[Core] Add multiple/external cog paths support (#853)

* WIP cog path manager

* Initial working state

* Get real reloading working

* Type error thingy

* And fix the tests

* Start UI shit

* path reordering

* Add install path getter/setter and fix config syntax

* Determine bot directory at runtime

* Add UI commands for install path

* Update downloader to use install path

* Add sane install path default

* Make evaluation of cog install path lazy

* Some typing fixes

* Add another line to codeowners

* Conditionally put install path in paths

* Always put install path first

* Dont allow people to add the installdir as an additional path, guarantee install dir isn't shown with paths command

* Make shit update loaded cogs

* Add tests

* Another one

core/bot.py

@@ -1,13 +1,22 @@
+import importlib.util
+from importlib.machinery import ModuleSpec
+
+import discord
 from discord.ext import commands
 from collections import Counter
 
+from discord.ext.commands import GroupMixin
+from pathlib import Path
+
 from core import Config
 from enum import Enum
 import os
 
+from core.cog_manager import CogManager
+
 
 class Red(commands.Bot):
-    def __init__(self, cli_flags, **kwargs):
+    def __init__(self, cli_flags, bot_dir: Path=Path.cwd(), **kwargs):
         self._shutdown_mode = ExitCodes.CRITICAL
         self.db = Config.get_core_conf(force_registration=True)
         self._co_owners = cli_flags.co_owner
@@ -51,6 +60,12 @@ class Red(commands.Bot):
 
         self.counter = Counter()
         self.uptime = None
+
+        self.main_dir = bot_dir
+
+        self.cog_mgr = CogManager(paths=(str(self.main_dir / 'cogs'),),
+                                  bot_dir=self.main_dir)
+
         super().__init__(**kwargs)
 
     async def is_owner(self, user):
@@ -84,12 +99,78 @@ class Red(commands.Bot):
         """Lists packages present in the cogs the folder"""
         return os.listdir("cogs")
 
-    async def save_packages_status(self):
-        loaded = []
-        for package in self.extensions:
-            if package.startswith("cogs."):
-                loaded.append(package)
-        await self.db.packages.set(loaded)
+    async def save_packages_status(self, packages):
+        await self.db.packages.set(packages)
+
+    async def add_loaded_package(self, pkg_name: str):
+        curr_pkgs = self.db.packages()
+        if pkg_name not in curr_pkgs:
+            curr_pkgs.append(pkg_name)
+            await self.save_packages_status(curr_pkgs)
+
+    async def remove_loaded_package(self, pkg_name: str):
+        curr_pkgs = self.db.packages()
+        if pkg_name in curr_pkgs:
+            await self.save_packages_status([p for p in curr_pkgs if p != pkg_name])
+
+    def load_extension(self, spec: ModuleSpec):
+        name = spec.name.split('.')[-1]
+        if name in self.extensions:
+            return
+
+        lib = spec.loader.load_module()
+        if not hasattr(lib, 'setup'):
+            del lib
+            raise discord.ClientException('extension does not have a setup function')
+
+        lib.setup(self)
+        self.extensions[name] = lib
+
+    def unload_extension(self, name):
+        lib = self.extensions.get(name)
+        if lib is None:
+            return
+
+        lib_name = lib.__name__  # Thank you
+
+        # find all references to the module
+
+        # remove the cogs registered from the module
+        for cogname, cog in self.cogs.copy().items():
+            if cog.__module__.startswith(lib_name):
+                self.remove_cog(cogname)
+
+        # first remove all the commands from the module
+        for cmd in self.all_commands.copy().values():
+            if cmd.module.startswith(lib_name):
+                if isinstance(cmd, GroupMixin):
+                    cmd.recursively_remove_all_commands()
+                self.remove_command(cmd.name)
+
+        # then remove all the listeners from the module
+        for event_list in self.extra_events.copy().values():
+            remove = []
+            for index, event in enumerate(event_list):
+                if event.__module__.startswith(lib_name):
+                    remove.append(index)
+
+            for index in reversed(remove):
+                del event_list[index]
+
+        try:
+            func = getattr(lib, 'teardown')
+        except AttributeError:
+            pass
+        else:
+            try:
+                func(self)
+            except:
+                pass
+        finally:
+            # finally remove the import..
+            del lib
+            del self.extensions[name]
+            # del sys.modules[name]
 
 
 class ExitCodes(Enum):