[Core] Add multiple/external cog paths support (#853)

* WIP cog path manager

* Initial working state

* Get real reloading working

* Type error thingy

* And fix the tests

* Start UI shit

* path reordering

* Add install path getter/setter and fix config syntax

* Determine bot directory at runtime

* Add UI commands for install path

* Update downloader to use install path

* Add sane install path default

* Make evaluation of cog install path lazy

* Some typing fixes

* Add another line to codeowners

* Conditionally put install path in paths

* Always put install path first

* Dont allow people to add the installdir as an additional path, guarantee install dir isn't shown with paths command

* Make shit update loaded cogs

* Add tests

* Another one

core/core_commands.py

@@ -1,3 +1,4 @@
+import itertools
 from discord.ext import commands
 from core import checks
 from string import ascii_letters, digits
@@ -5,11 +6,13 @@ from random import SystemRandom
 from collections import namedtuple
 import logging
 import importlib
-import os
+import sys
 import discord
 import aiohttp
 import asyncio
 
+from core.cog_manager import NoModuleFound
+
 log = logging.getLogger("red")
 
 OWNER_DISCLAIMER = ("⚠ **Only** the person who is hosting Red should be "
@@ -25,29 +28,30 @@ class Core:
     @checks.is_owner()
     async def load(self, ctx, *, cog_name: str):
         """Loads a package"""
-        if not cog_name.startswith("cogs."):
-            cog_name = "cogs." + cog_name
+        try:
+            spec = ctx.bot.cog_mgr.find_cog(cog_name)
+        except NoModuleFound:
+            await ctx.send("No module by that name was found in any"
+                           " cog path.")
+            return
 
         try:
-            ctx.bot.load_extension(cog_name)
+            ctx.bot.load_extension(spec)
         except Exception as e:
             log.exception("Package loading failed", exc_info=e)
             await ctx.send("Failed to load package. Check your console or "
                            "logs for details.")
         else:
-            await ctx.bot.save_packages_status()
+            await ctx.bot.add_loaded_package(cog_name)
             await ctx.send("Done.")
 
     @commands.group()
     @checks.is_owner()
     async def unload(self, ctx, *, cog_name: str):
         """Unloads a package"""
-        if not cog_name.startswith("cogs."):
-            cog_name = "cogs." + cog_name
-
         if cog_name in ctx.bot.extensions:
             ctx.bot.unload_extension(cog_name)
-            await ctx.bot.save_packages_status()
+            await ctx.bot.remove_loaded_package(cog_name)
             await ctx.send("Done.")
         else:
             await ctx.send("That extension is not loaded.")
@@ -56,17 +60,11 @@ class Core:
     @checks.is_owner()
     async def _reload(self, ctx, *, cog_name: str):
         """Reloads a package"""
-        if cog_name == "downloader":
-            await ctx.send("DONT RELOAD DOWNLOADER.")
-            return
-
-        if not cog_name.startswith("cogs."):
-            cog_name = "cogs." + cog_name
-
+        ctx.bot.unload_extension(cog_name)
+        self.cleanup_and_refresh_modules(cog_name)
         try:
-            self.refresh_modules(cog_name)
-            ctx.bot.unload_extension(cog_name)
-            ctx.bot.load_extension(cog_name)
+            spec = ctx.bot.cog_mgr.find_cog(cog_name)
+            ctx.bot.load_extension(spec)
         except Exception as e:
             log.exception("Package reloading failed", exc_info=e)
             await ctx.send("Failed to reload package. Check your console or "
@@ -75,18 +73,25 @@ class Core:
             await ctx.bot.save_packages_status()
             await ctx.send("Done.")
 
-    def refresh_modules(self, module):
+    def cleanup_and_refresh_modules(self, module_name: str):
         """Interally reloads modules so that changes are detected"""
-        module = module.replace(".", os.sep)
-        for root, dirs, files in os.walk(module):
-            for name in files:
-                if name.endswith(".py"):
-                    path = os.path.join(root, name)
-                    path, _ = os.path.splitext(path)
-                    path = ".".join(path.split(os.sep))
-                    print("Reloading " + path)
-                    m = importlib.import_module(path)
-                    importlib.reload(m)
+        splitted = module_name.split('.')
+
+        def maybe_reload(new_name):
+            try:
+                lib = sys.modules[new_name]
+            except KeyError:
+                pass
+            else:
+                importlib._bootstrap._exec(lib.__spec__, lib)
+
+        modules = itertools.accumulate(splitted, lambda old, next: "{}.{}".format(old, next))
+        for m in modules:
+            maybe_reload(m)
+
+        children = {name: lib for name, lib in sys.modules.items() if name.startswith(module_name)}
+        for child_name, lib in children.items():
+            importlib._bootstrap._exec(lib.__spec__, lib)
 
     @commands.group(name="set")
     async def _set(self, ctx):