From ee1db01a2f516b9b5549e11f71dd7a67f80f6106 Mon Sep 17 00:00:00 2001 From: Jakub Kuczys Date: Sun, 29 Mar 2026 22:25:04 +0200 Subject: [PATCH] Rip out Downloader's non-UI functionality into private core API (#6706) --- .github/labeler.yml | 11 +- redbot/__main__.py | 13 +- redbot/cogs/downloader/__init__.py | 1 - redbot/cogs/downloader/converters.py | 22 +- redbot/cogs/downloader/downloader.py | 1012 +++-------------- redbot/core/_downloader/__init__.py | 865 ++++++++++++++ .../downloader => core/_downloader}/errors.py | 0 .../_downloader}/info_schemas.py | 0 .../_downloader}/installable.py | 11 +- .../_downloader}/json_mixins.py | 0 redbot/core/_downloader/log.py | 3 + .../_downloader}/repo_manager.py | 19 +- redbot/core/bot.py | 20 +- redbot/core/core_commands.py | 7 +- redbot/core/utils/_internal_utils.py | 2 +- redbot/pytest/downloader.py | 4 +- .../_downloader}/__init__.py | 0 .../_downloader}/test_downloader.py | 18 +- .../_downloader}/test_git.py | 2 +- .../_downloader}/test_installable.py | 6 +- 20 files changed, 1120 insertions(+), 896 deletions(-) create mode 100644 redbot/core/_downloader/__init__.py rename redbot/{cogs/downloader => core/_downloader}/errors.py (100%) rename redbot/{cogs/downloader => core/_downloader}/info_schemas.py (100%) rename redbot/{cogs/downloader => core/_downloader}/installable.py (95%) rename redbot/{cogs/downloader => core/_downloader}/json_mixins.py (100%) create mode 100644 redbot/core/_downloader/log.py rename redbot/{cogs/downloader => core/_downloader}/repo_manager.py (98%) rename tests/{cogs/downloader => core/_downloader}/__init__.py (100%) rename tests/{cogs/downloader => core/_downloader}/test_downloader.py (94%) rename tests/{cogs/downloader => core/_downloader}/test_git.py (99%) rename tests/{cogs/downloader => core/_downloader}/test_installable.py (88%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 54467508b..9adf0b94d 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -50,10 +50,6 @@ - redbot/cogs/downloader/* # Docs - docs/cog_guides/downloader.rst - # Tests - - redbot/pytest/downloader.py - - redbot/pytest/downloader_testrepo.* - - tests/cogs/downloader/**/* "Category: Cogs - Economy": # Source - redbot/cogs/economy/* @@ -212,6 +208,13 @@ - redbot/core/_cli.py - redbot/core/_debuginfo.py - redbot/setup.py +"Category: Core - Downloader": + # Source + - redbot/core/_downloader/**/* + # Tests + - redbot/pytest/downloader.py + - redbot/pytest/downloader_testrepo.* + - tests/core/_downloader/**/* "Category: Core - Help": - redbot/core/commands/help.py "Category: Core - i18n": diff --git a/redbot/__main__.py b/redbot/__main__.py index e21880fe1..42f9a09b2 100644 --- a/redbot/__main__.py +++ b/redbot/__main__.py @@ -27,7 +27,7 @@ from redbot import __version__ from redbot.core.bot import Red, ExitCodes, _NoOwnerSet from redbot.core._cli import interactive_config, confirm, parse_cli_flags from redbot.setup import get_data_dir, get_name, save_config -from redbot.core import data_manager, _drivers +from redbot.core import data_manager, _drivers, _downloader from redbot.core._debuginfo import DebugInfo from redbot.core._sharedlibdeprecation import SharedLibImportWarner @@ -324,12 +324,13 @@ async def run_bot(red: Red, cli_flags: Namespace) -> None: log.debug("Data Path: %s", data_manager._base_data_path()) log.debug("Storage Type: %s", data_manager.storage_type()) + await _downloader._init(red) + # lib folder has to be in sys.path before trying to load any 3rd-party cog (GH-3061) # We might want to change handling of requirements in Downloader at later date - LIB_PATH = data_manager.cog_data_path(raw_name="Downloader") / "lib" - LIB_PATH.mkdir(parents=True, exist_ok=True) - if str(LIB_PATH) not in sys.path: - sys.path.append(str(LIB_PATH)) + lib_path = str(_downloader.LIB_PATH) + if lib_path not in sys.path: + sys.path.append(lib_path) # "It's important to note that the global `working_set` object is initialized from # `sys.path` when `pkg_resources` is first imported, but is only updated if you do @@ -339,7 +340,7 @@ async def run_bot(red: Red, cli_flags: Namespace) -> None: # Source: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#workingset-objects pkg_resources = sys.modules.get("pkg_resources") if pkg_resources is not None: - pkg_resources.working_set.add_entry(str(LIB_PATH)) + pkg_resources.working_set.add_entry(lib_path) sys.meta_path.insert(0, SharedLibImportWarner()) if cli_flags.token: diff --git a/redbot/cogs/downloader/__init__.py b/redbot/cogs/downloader/__init__.py index d535760c0..cd816d74d 100644 --- a/redbot/cogs/downloader/__init__.py +++ b/redbot/cogs/downloader/__init__.py @@ -6,4 +6,3 @@ from .downloader import Downloader async def setup(bot: Red) -> None: cog = Downloader(bot) await bot.add_cog(cog) - cog.create_init_task() diff --git a/redbot/cogs/downloader/converters.py b/redbot/cogs/downloader/converters.py index 483918d4a..7a9f6638b 100644 --- a/redbot/cogs/downloader/converters.py +++ b/redbot/cogs/downloader/converters.py @@ -1,7 +1,8 @@ import discord -from redbot.core import commands +from redbot.core import _downloader, commands from redbot.core.i18n import Translator -from .installable import InstalledModule +from redbot.core._downloader.installable import InstalledModule +from redbot.core._downloader.repo_manager import Repo as _Repo _ = Translator("Koala", __file__) @@ -9,14 +10,21 @@ _ = Translator("Koala", __file__) class InstalledCog(InstalledModule): @classmethod async def convert(cls, ctx: commands.Context, arg: str) -> InstalledModule: - downloader = ctx.bot.get_cog("Downloader") - if downloader is None: - raise commands.CommandError(_("No Downloader cog found.")) - - cog = discord.utils.get(await downloader.installed_cogs(), name=arg) + cog = discord.utils.get(await _downloader.installed_cogs(), name=arg) if cog is None: raise commands.BadArgument( _("Cog `{cog_name}` is not installed.").format(cog_name=arg) ) return cog + + +class Repo(_Repo): + @classmethod + async def convert(cls, ctx: commands.Context, argument: str) -> _Repo: + poss_repo = _downloader._repo_manager.get_repo(argument) + if poss_repo is None: + raise commands.BadArgument( + _('Repo by the name "{repo_name}" does not exist.').format(repo_name=argument) + ) + return poss_repo diff --git a/redbot/cogs/downloader/downloader.py b/redbot/cogs/downloader/downloader.py index 1cc353833..55f49f83e 100644 --- a/redbot/cogs/downloader/downloader.py +++ b/redbot/cogs/downloader/downloader.py @@ -1,29 +1,22 @@ import asyncio import contextlib -import os import re -import shutil -import sys -from pathlib import Path -from typing import Tuple, Union, Iterable, Collection, Optional, Dict, Set, List, cast -from collections import defaultdict +from typing import Tuple, Iterable, Collection, Optional, Set, List import discord -from redbot.core import commands, Config, version_info as red_version_info +from redbot.core import _downloader, commands, version_info as red_version_info +from redbot.core._downloader import errors +from redbot.core._downloader.installable import InstalledModule from redbot.core.bot import Red -from redbot.core.data_manager import cog_data_path from redbot.core.i18n import Translator, cog_i18n from redbot.core.utils import can_user_react_in from redbot.core.utils.chat_formatting import box, pagify, humanize_list, inline from redbot.core.utils.menus import start_adding_reactions from redbot.core.utils.predicates import MessagePredicate, ReactionPredicate -from . import errors from .checks import do_install_agreement -from .converters import InstalledCog -from .installable import InstallableType, Installable, InstalledModule +from .converters import InstalledCog, Repo from .log import log -from .repo_manager import RepoManager, Repo _ = Translator("Downloader", __file__) @@ -51,434 +44,12 @@ class Downloader(commands.Cog): def __init__(self, bot: Red): super().__init__() self.bot = bot - - self.config = Config.get_conf(self, identifier=998240343, force_registration=True) - - self.config.register_global(schema_version=0, installed_cogs={}, installed_libraries={}) - self.already_agreed = False - self.LIB_PATH = cog_data_path(self) / "lib" - self.SHAREDLIB_PATH = self.LIB_PATH / "cog_shared" - self.SHAREDLIB_INIT = self.SHAREDLIB_PATH / "__init__.py" - - self._create_lib_folder() - - self._repo_manager = RepoManager() - self._ready = asyncio.Event() - self._init_task = None - self._ready_raised = False - - def _create_lib_folder(self, *, remove_first: bool = False) -> None: - if remove_first: - shutil.rmtree(str(self.LIB_PATH)) - self.SHAREDLIB_PATH.mkdir(parents=True, exist_ok=True) - if not self.SHAREDLIB_INIT.exists(): - with self.SHAREDLIB_INIT.open(mode="w", encoding="utf-8") as _: - pass - - async def cog_before_invoke(self, ctx: commands.Context) -> None: - if not self._ready.is_set(): - async with ctx.typing(): - await self._ready.wait() - if self._ready_raised: - await ctx.send( - "There was an error during Downloader's initialization." - " Check logs for more information." - ) - raise commands.CheckFailure() - - def cog_unload(self): - if self._init_task is not None: - self._init_task.cancel() - async def red_delete_data_for_user(self, **kwargs): """Nothing to delete""" return - def create_init_task(self): - def _done_callback(task: asyncio.Task) -> None: - try: - exc = task.exception() - except asyncio.CancelledError: - pass - else: - if exc is None: - return - log.error( - "An unexpected error occurred during Downloader's initialization.", - exc_info=exc, - ) - self._ready_raised = True - self._ready.set() - - self._init_task = asyncio.create_task(self.initialize()) - self._init_task.add_done_callback(_done_callback) - - async def initialize(self) -> None: - await self._repo_manager.initialize() - await self._maybe_update_config() - self._ready.set() - - async def _maybe_update_config(self) -> None: - schema_version = await self.config.schema_version() - - if schema_version == 0: - await self._schema_0_to_1() - schema_version += 1 - await self.config.schema_version.set(schema_version) - - async def _schema_0_to_1(self): - """ - This contains migration to allow saving state - of both installed cogs and shared libraries. - """ - old_conf = await self.config.get_raw("installed", default=[]) - if not old_conf: - return - async with self.config.installed_cogs() as new_cog_conf: - for cog_json in old_conf: - repo_name = cog_json["repo_name"] - module_name = cog_json["cog_name"] - if repo_name not in new_cog_conf: - new_cog_conf[repo_name] = {} - new_cog_conf[repo_name][module_name] = { - "repo_name": repo_name, - "module_name": module_name, - "commit": "", - "pinned": False, - } - await self.config.clear_raw("installed") - # no reliable way to get installed libraries (i.a. missing repo name) - # but it only helps `[p]cog update` run faster so it's not an issue - - async def cog_install_path(self) -> Path: - """Get the current cog install path. - - Returns - ------- - pathlib.Path - The default cog install path. - - """ - return await self.bot._cog_mgr.install_path() - - async def installed_cogs(self) -> Tuple[InstalledModule, ...]: - """Get info on installed cogs. - - Returns - ------- - `tuple` of `InstalledModule` - All installed cogs. - - """ - installed = await self.config.installed_cogs() - # noinspection PyTypeChecker - return tuple( - InstalledModule.from_json(cog_json, self._repo_manager) - for repo_json in installed.values() - for cog_json in repo_json.values() - ) - - async def installed_libraries(self) -> Tuple[InstalledModule, ...]: - """Get info on installed shared libraries. - - Returns - ------- - `tuple` of `InstalledModule` - All installed shared libraries. - - """ - installed = await self.config.installed_libraries() - # noinspection PyTypeChecker - return tuple( - InstalledModule.from_json(lib_json, self._repo_manager) - for repo_json in installed.values() - for lib_json in repo_json.values() - ) - - async def installed_modules(self) -> Tuple[InstalledModule, ...]: - """Get info on installed cogs and shared libraries. - - Returns - ------- - `tuple` of `InstalledModule` - All installed cogs and shared libraries. - - """ - return await self.installed_cogs() + await self.installed_libraries() - - async def _save_to_installed(self, modules: Iterable[InstalledModule]) -> None: - """Mark modules as installed or updates their json in Config. - - Parameters - ---------- - modules : `list` of `InstalledModule` - The modules to check off. - - """ - async with self.config.all() as global_data: - installed_cogs = global_data["installed_cogs"] - installed_libraries = global_data["installed_libraries"] - for module in modules: - if module.type == InstallableType.COG: - installed = installed_cogs - elif module.type == InstallableType.SHARED_LIBRARY: - installed = installed_libraries - else: - continue - module_json = module.to_json() - repo_json = installed.setdefault(module.repo_name, {}) - repo_json[module.name] = module_json - - async def _remove_from_installed(self, modules: Iterable[InstalledModule]) -> None: - """Remove modules from the saved list - of installed modules (corresponding to type of module). - - Parameters - ---------- - modules : `list` of `InstalledModule` - The modules to remove. - - """ - async with self.config.all() as global_data: - installed_cogs = global_data["installed_cogs"] - installed_libraries = global_data["installed_libraries"] - for module in modules: - if module.type == InstallableType.COG: - installed = installed_cogs - elif module.type == InstallableType.SHARED_LIBRARY: - installed = installed_libraries - else: - continue - with contextlib.suppress(KeyError): - installed[module._json_repo_name].pop(module.name) - - async def _shared_lib_load_check(self, cog_name: str) -> Optional[Repo]: - is_installed, cog = await self.is_installed(cog_name) - # it's not gonna be None when `is_installed` is True - # if we'll use typing_extensions in future, `Literal` can solve this - cog = cast(InstalledModule, cog) - if is_installed and cog.repo is not None and cog.repo.available_libraries: - return cog.repo - return None - - async def _available_updates( - self, cogs: Iterable[InstalledModule] - ) -> Tuple[Tuple[Installable, ...], Tuple[Installable, ...]]: - """ - Get cogs and libraries which can be updated. - - Parameters - ---------- - cogs : `list` of `InstalledModule` - List of cogs, which should be checked against the updates. - - Returns - ------- - tuple - 2-tuple of cogs and libraries which can be updated. - - """ - repos = {cog.repo for cog in cogs if cog.repo is not None} - installed_libraries = await self.installed_libraries() - - modules: Set[InstalledModule] = set() - cogs_to_update: Set[Installable] = set() - libraries_to_update: Set[Installable] = set() - # split libraries and cogs into 2 categories: - # 1. `cogs_to_update`, `libraries_to_update` - module needs update, skip diffs - # 2. `modules` - module MAY need update, check diffs - for repo in repos: - for lib in repo.available_libraries: - try: - index = installed_libraries.index(lib) - except ValueError: - libraries_to_update.add(lib) - else: - modules.add(installed_libraries[index]) - for cog in cogs: - if cog.repo is None: - # cog had its repo removed, can't check for updates - continue - if cog.commit: - modules.add(cog) - continue - # marking cog for update if there's no commit data saved (back-compat, see GH-2571) - last_cog_occurrence = await cog.repo.get_last_module_occurrence(cog.name) - if last_cog_occurrence is not None and not last_cog_occurrence.disabled: - cogs_to_update.add(last_cog_occurrence) - - # Reduces diff requests to a single dict with no repeats - hashes: Dict[Tuple[Repo, str], Set[InstalledModule]] = defaultdict(set) - for module in modules: - module.repo = cast(Repo, module.repo) - if module.repo.commit != module.commit: - try: - should_add = await module.repo.is_ancestor(module.commit, module.repo.commit) - except errors.UnknownRevision: - # marking module for update if the saved commit data is invalid - last_module_occurrence = await module.repo.get_last_module_occurrence( - module.name - ) - if last_module_occurrence is not None and not last_module_occurrence.disabled: - if last_module_occurrence.type == InstallableType.COG: - cogs_to_update.add(last_module_occurrence) - elif last_module_occurrence.type == InstallableType.SHARED_LIBRARY: - libraries_to_update.add(last_module_occurrence) - else: - if should_add: - hashes[(module.repo, module.commit)].add(module) - - update_commits = [] - for (repo, old_hash), modules_to_check in hashes.items(): - modified = await repo.get_modified_modules(old_hash, repo.commit) - for module in modules_to_check: - try: - index = modified.index(module) - except ValueError: - # module wasn't modified - we just need to update its commit - module.commit = repo.commit - update_commits.append(module) - else: - modified_module = modified[index] - if modified_module.type == InstallableType.COG: - if not modified_module.disabled: - cogs_to_update.add(modified_module) - elif modified_module.type == InstallableType.SHARED_LIBRARY: - libraries_to_update.add(modified_module) - - await self._save_to_installed(update_commits) - - return (tuple(cogs_to_update), tuple(libraries_to_update)) - - async def _install_cogs( - self, cogs: Iterable[Installable] - ) -> Tuple[Tuple[InstalledModule, ...], Tuple[Installable, ...]]: - """Installs a list of cogs. - - Parameters - ---------- - cogs : `list` of `Installable` - Cogs to install. ``repo`` property of those objects can't be `None` - Returns - ------- - tuple - 2-tuple of installed and failed cogs. - """ - repos: Dict[str, Tuple[Repo, Dict[str, List[Installable]]]] = {} - for cog in cogs: - try: - repo_by_commit = repos[cog.repo_name] - except KeyError: - cog.repo = cast(Repo, cog.repo) # docstring specifies this already - repo_by_commit = repos[cog.repo_name] = (cog.repo, defaultdict(list)) - cogs_by_commit = repo_by_commit[1] - cogs_by_commit[cog.commit].append(cog) - installed = [] - failed = [] - for repo, cogs_by_commit in repos.values(): - exit_to_commit = repo.commit - for commit, cogs_to_install in cogs_by_commit.items(): - await repo.checkout(commit) - for cog in cogs_to_install: - if await cog.copy_to(await self.cog_install_path()): - installed.append(InstalledModule.from_installable(cog)) - else: - failed.append(cog) - await repo.checkout(exit_to_commit) - - # noinspection PyTypeChecker - return (tuple(installed), tuple(failed)) - - async def _reinstall_libraries( - self, libraries: Iterable[Installable] - ) -> Tuple[Tuple[InstalledModule, ...], Tuple[Installable, ...]]: - """Installs a list of shared libraries, used when updating. - - Parameters - ---------- - libraries : `list` of `Installable` - Libraries to reinstall. ``repo`` property of those objects can't be `None` - Returns - ------- - tuple - 2-tuple of installed and failed libraries. - """ - repos: Dict[str, Tuple[Repo, Dict[str, Set[Installable]]]] = {} - for lib in libraries: - try: - repo_by_commit = repos[lib.repo_name] - except KeyError: - lib.repo = cast(Repo, lib.repo) # docstring specifies this already - repo_by_commit = repos[lib.repo_name] = (lib.repo, defaultdict(set)) - libs_by_commit = repo_by_commit[1] - libs_by_commit[lib.commit].add(lib) - - all_installed: List[InstalledModule] = [] - all_failed: List[Installable] = [] - for repo, libs_by_commit in repos.values(): - exit_to_commit = repo.commit - for commit, libs in libs_by_commit.items(): - await repo.checkout(commit) - installed, failed = await repo.install_libraries( - target_dir=self.SHAREDLIB_PATH, req_target_dir=self.LIB_PATH, libraries=libs - ) - all_installed += installed - all_failed += failed - await repo.checkout(exit_to_commit) - - # noinspection PyTypeChecker - return (tuple(all_installed), tuple(all_failed)) - - async def _install_requirements(self, cogs: Iterable[Installable]) -> Tuple[str, ...]: - """ - Installs requirements for given cogs. - - Parameters - ---------- - cogs : `list` of `Installable` - Cogs whose requirements should be installed. - Returns - ------- - tuple - Tuple of failed requirements. - """ - - # Reduces requirements to a single list with no repeats - requirements = {requirement for cog in cogs for requirement in cog.requirements} - repos: List[Tuple[Repo, List[str]]] = [(repo, []) for repo in self._repo_manager.repos] - - # This for loop distributes the requirements across all repos - # which will allow us to concurrently install requirements - for i, req in enumerate(requirements): - repo_index = i % len(repos) - repos[repo_index][1].append(req) - - has_reqs = list(filter(lambda item: len(item[1]) > 0, repos)) - - failed_reqs = [] - for repo, reqs in has_reqs: - for req in reqs: - if not await repo.install_raw_requirements([req], self.LIB_PATH): - failed_reqs.append(req) - return tuple(failed_reqs) - - @staticmethod - async def _delete_cog(target: Path) -> None: - """ - Removes an (installed) cog. - :param target: Path pointing to an existing file or directory - :return: - """ - if not target.exists(): - return - - if target.is_dir(): - shutil.rmtree(str(target)) - elif target.is_file(): - os.remove(str(target)) - @staticmethod async def send_pagified(target: discord.abc.Messageable, content: str) -> None: for page in pagify(content): @@ -500,9 +71,8 @@ class Downloader(commands.Cog): - `` The package or packages you wish to install. """ - repo = Repo("", "", "", "", Path.cwd()) async with ctx.typing(): - success = await repo.install_raw_requirements(deps, self.LIB_PATH) + success = await _downloader.pip_install(*deps) if success: await ctx.send(_("Libraries installed.") if len(deps) > 1 else _("Library installed.")) @@ -547,6 +117,7 @@ class Downloader(commands.Cog): agreed = await do_install_agreement(ctx) if not agreed: return + # TODO: verify this in the Downloader APIs if name.startswith(".") or name.endswith("."): await ctx.send(_("Repo names cannot start or end with a dot.")) return @@ -561,7 +132,9 @@ class Downloader(commands.Cog): try: async with ctx.typing(): # noinspection PyTypeChecker - repo = await self._repo_manager.add_repo(name=name, url=repo_url, branch=branch) + repo = await _downloader._repo_manager.add_repo( + name=name, url=repo_url, branch=branch + ) except errors.ExistingGitRepo: await ctx.send( _("The repo name you provided is already in use. Please choose another name.") @@ -628,7 +201,7 @@ class Downloader(commands.Cog): - `` The repo or repos to remove. """ for repo in set(repos): - await self._repo_manager.delete_repo(repo.name) + await _downloader._repo_manager.delete_repo(repo.name) await ctx.send( ( @@ -642,7 +215,7 @@ class Downloader(commands.Cog): @repo.command(name="list") async def _repo_list(self, ctx: commands.Context) -> None: """List all installed repos.""" - repos = self._repo_manager.repos + repos = _downloader._repo_manager.repos sorted_repos = sorted(repos, key=lambda r: str.lower(r.name)) if len(repos) == 0: joined = _("There are no repos installed.") @@ -709,7 +282,7 @@ class Downloader(commands.Cog): async with ctx.typing(): updated: Set[str] - updated_repos, failed = await self._repo_manager.update_repos(repos) + updated_repos, failed = await _downloader._repo_manager.update_repos(repos) updated = {repo.name for repo in updated_repos} if updated: @@ -745,24 +318,8 @@ class Downloader(commands.Cog): because of change in minor version of Python. """ async with ctx.typing(): - self._create_lib_folder(remove_first=True) - installed_cogs = await self.installed_cogs() - cogs = [] - repos = set() - for cog in installed_cogs: - if cog.repo is None: - continue - repos.add(cog.repo) - cogs.append(cog) - failed_reqs = await self._install_requirements(cogs) - all_installed_libs: List[InstalledModule] = [] - all_failed_libs: List[Installable] = [] - for repo in repos: - installed_libs, failed_libs = await repo.install_libraries( - target_dir=self.SHAREDLIB_PATH, req_target_dir=self.LIB_PATH - ) - all_installed_libs += installed_libs - all_failed_libs += failed_libs + failed_reqs, failed_libs = await _downloader.reinstall_requirements() + message = "" if failed_reqs: message += ( @@ -770,11 +327,11 @@ class Downloader(commands.Cog): if len(failed_reqs) > 1 else _("Failed to install the requirement: ") ) + humanize_list(tuple(map(inline, failed_reqs))) - if all_failed_libs: + if failed_libs: libnames = [lib.name for lib in failed_libs] message += ( _("\nFailed to install shared libraries: ") - if len(all_failed_libs) > 1 + if len(failed_libs) > 1 else _("\nFailed to install shared library: ") ) + humanize_list(tuple(map(inline, libnames))) if message: @@ -836,60 +393,41 @@ class Downloader(commands.Cog): async def _cog_installrev( self, ctx: commands.Context, repo: Repo, rev: Optional[str], cog_names: Iterable[str] ) -> None: - commit = None async with ctx.typing(): - if rev is not None: - try: - commit = await repo.get_full_sha1(rev) - except errors.AmbiguousRevision as e: - msg = _( - "Error: short sha1 `{rev}` is ambiguous. Possible candidates:\n" - ).format(rev=rev) - for candidate in e.candidates: - msg += ( - f"**{candidate.object_type} {candidate.rev}**" - f" - {candidate.description}\n" - ) - await self.send_pagified(ctx, msg) - return - except errors.UnknownRevision: - await ctx.send( - _("Error: there is no revision `{rev}` in repo `{repo.name}`").format( - rev=rev, repo=repo - ) + try: + install_result = await _downloader.install_cogs(repo, rev, cog_names) + except errors.AmbiguousRevision as e: + msg = _("Error: short sha1 `{rev}` is ambiguous. Possible candidates:\n").format( + rev=rev + ) + for candidate in e.candidates: + msg += ( + f"**{candidate.object_type} {candidate.rev}**" + f" - {candidate.description}\n" ) - return - cog_names = set(cog_names) - - async with repo.checkout(commit, exit_to_rev=repo.branch): - cogs, message = await self._filter_incorrect_cogs_by_names(repo, cog_names) - if not cogs: - await self.send_pagified(ctx, message) - return - failed_reqs = await self._install_requirements(cogs) - if failed_reqs: - message += ( - _("\nFailed to install requirements: ") - if len(failed_reqs) > 1 - else _("\nFailed to install the requirement: ") - ) + humanize_list(tuple(map(inline, failed_reqs))) - await self.send_pagified(ctx, message) - return - - installed_cogs, failed_cogs = await self._install_cogs(cogs) + await self.send_pagified(ctx, msg) + return + except errors.UnknownRevision: + await ctx.send( + _("Error: there is no revision `{rev}` in repo `{repo.name}`").format( + rev=rev, repo=repo + ) + ) + return deprecation_notice = "" if repo.available_libraries: deprecation_notice = DEPRECATION_NOTICE.format(repo_list=inline(repo.name)) - installed_libs, failed_libs = await repo.install_libraries( - target_dir=self.SHAREDLIB_PATH, req_target_dir=self.LIB_PATH - ) - if rev is not None: - for cog in installed_cogs: - cog.pinned = True - await self._save_to_installed(installed_cogs + installed_libs) - if failed_libs: - libnames = [inline(lib.name) for lib in failed_libs] + + message = self._format_invalid_cogs(repo, install_result) + if install_result.failed_reqs: + message += ( + _("\nFailed to install requirements: ") + if len(install_result.failed_reqs) > 1 + else _("\nFailed to install the requirement: ") + ) + humanize_list(tuple(map(inline, install_result.failed_reqs))) + if install_result.failed_libs: + libnames = [inline(lib.name) for lib in install_result.failed_libs] message = ( ( _("\nFailed to install shared libraries for `{repo.name}` repo: ") @@ -899,23 +437,23 @@ class Downloader(commands.Cog): + humanize_list(libnames) + message ) - if failed_cogs: - cognames = [inline(cog.name) for cog in failed_cogs] + if install_result.failed_cogs: + cognames = [inline(cog.name) for cog in install_result.failed_cogs] message = ( ( _("\nFailed to install cogs: ") - if len(failed_cogs) > 1 + if len(install_result.failed_cogs) > 1 else _("\nFailed to install the cog: ") ) + humanize_list(cognames) + message ) - if installed_cogs: - cognames = [inline(cog.name) for cog in installed_cogs] + if install_result.installed_cogs: + cognames = [inline(cog.name) for cog in install_result.installed_cogs] message = ( ( _("Successfully installed cogs: ") - if len(installed_cogs) > 1 + if len(install_result.installed_cogs) > 1 else _("Successfully installed the cog: ") ) + humanize_list(cognames) @@ -938,7 +476,7 @@ class Downloader(commands.Cog): ) # "---" added to separate cog install messages from Downloader's message await self.send_pagified(ctx, f"{message}{deprecation_notice}\n---") - for cog in installed_cogs: + for cog in install_result.installed_cogs: if cog.install_msg: await ctx.send( cog.install_msg.replace("[p]", ctx.clean_prefix).replace( @@ -962,21 +500,7 @@ class Downloader(commands.Cog): - `` The cog or cogs to uninstall. """ async with ctx.typing(): - uninstalled_cogs = [] - failed_cogs = [] - for cog in set(cogs): - real_name = cog.name - - poss_installed_path = (await self.cog_install_path()) / real_name - if poss_installed_path.exists(): - with contextlib.suppress(commands.ExtensionNotLoaded): - await ctx.bot.unload_extension(real_name) - await ctx.bot.remove_loaded_package(real_name) - await self._delete_cog(poss_installed_path) - uninstalled_cogs.append(inline(real_name)) - else: - failed_cogs.append(real_name) - await self._remove_from_installed(cogs) + uninstalled_cogs, failed_cogs = await _downloader.uninstall_cogs(*cogs) message = "" if uninstalled_cogs: @@ -984,7 +508,7 @@ class Downloader(commands.Cog): _("Successfully uninstalled cogs: ") if len(uninstalled_cogs) > 1 else _("Successfully uninstalled the cog: ") - ) + humanize_list(uninstalled_cogs) + ) + humanize_list(tuple(map(inline, uninstalled_cogs))) if failed_cogs: if len(failed_cogs) > 1: message += ( @@ -1032,27 +556,20 @@ class Downloader(commands.Cog): - `` The cog or cogs to pin. Must already be installed. """ - already_pinned = [] - pinned = [] - for cog in set(cogs): - if cog.pinned: - already_pinned.append(inline(cog.name)) - continue - cog.pinned = True - pinned.append(cog) + pinned, already_pinned = await _downloader.pin_cogs(*cogs) message = "" if pinned: - await self._save_to_installed(pinned) cognames = [inline(cog.name) for cog in pinned] message += ( _("Pinned cogs: ") if len(pinned) > 1 else _("Pinned cog: ") ) + humanize_list(cognames) if already_pinned: + cognames = [inline(cog.name) for cog in already_pinned] message += ( _("\nThese cogs were already pinned: ") if len(already_pinned) > 1 else _("\nThis cog was already pinned: ") - ) + humanize_list(already_pinned) + ) + humanize_list(cognames) await self.send_pagified(ctx, message) @cog.command(name="unpin", require_var_positional=True) @@ -1066,33 +583,26 @@ class Downloader(commands.Cog): **Arguments** - `` The cog or cogs to unpin. Must already be installed and pinned.""" - not_pinned = [] - unpinned = [] - for cog in set(cogs): - if not cog.pinned: - not_pinned.append(inline(cog.name)) - continue - cog.pinned = False - unpinned.append(cog) + unpinned, not_pinned = await _downloader.unpin_cogs(*cogs) message = "" if unpinned: - await self._save_to_installed(unpinned) cognames = [inline(cog.name) for cog in unpinned] message += ( _("Unpinned cogs: ") if len(unpinned) > 1 else _("Unpinned cog: ") ) + humanize_list(cognames) if not_pinned: + cognames = [inline(cog.name) for cog in not_pinned] message += ( _("\nThese cogs weren't pinned: ") if len(not_pinned) > 1 else _("\nThis cog was already not pinned: ") - ) + humanize_list(not_pinned) + ) + humanize_list(cognames) await self.send_pagified(ctx, message) @cog.command(name="listpinned") async def _cog_listpinned(self, ctx: commands.Context): """List currently pinned cogs.""" - installed = await self.installed_cogs() + installed = await _downloader.installed_cogs() pinned_list = sorted( [cog for cog in installed if cog.pinned], key=lambda cog: cog.name.lower() ) @@ -1124,34 +634,33 @@ class Downloader(commands.Cog): """ async with ctx.typing(): - cogs_to_check, failed = await self._get_cogs_to_check() - cogs_to_update, libs_to_update = await self._available_updates(cogs_to_check) - cogs_to_update, filter_message = self._filter_incorrect_cogs(cogs_to_update) + update_check_result = await _downloader.check_cog_updates() + filter_message = self._format_incompatible_cogs(update_check_result) message = "" - if cogs_to_update: - cognames = [cog.name for cog in cogs_to_update] + if update_check_result.outdated_cogs: + cognames = [cog.name for cog in update_check_result.outdated_cogs] message += ( _("These cogs can be updated: ") if len(cognames) > 1 else _("This cog can be updated: ") ) + humanize_list(tuple(map(inline, cognames))) - if libs_to_update: - libnames = [cog.name for cog in libs_to_update] + if update_check_result.outdated_libs: + libnames = [cog.name for cog in update_check_result.outdated_libs] message += ( _("\nThese shared libraries can be updated: ") if len(libnames) > 1 else _("\nThis shared library can be updated: ") ) + humanize_list(tuple(map(inline, libnames))) - if not (cogs_to_update or libs_to_update) and filter_message: + if not update_check_result.updates_available and filter_message: message += _("No cogs can be updated.") message += filter_message if not message: message = _("All installed cogs are up to date.") - if failed: - message += "\n" + self.format_failed_repos(failed) + if update_check_result.failed_repos: + message += "\n" + self.format_failed_repos(update_check_result.failed_repos) await self.send_pagified(ctx, message) @@ -1237,23 +746,10 @@ class Downloader(commands.Cog): rev: Optional[str] = None, cogs: Optional[List[InstalledModule]] = None, ) -> None: - failed_repos = set() - updates_available = set() - async with ctx.typing(): - # this is enough to be sure that `rev` is not None (based on calls to this method) if repo is not None: - rev = cast(str, rev) - try: - await repo.update() - except errors.UpdateError: - message = self.format_failed_repos([repo.name]) - await self.send_pagified(ctx, message) - return - - try: - commit = await repo.get_full_sha1(rev) + update_result = await _downloader.update_repo_cogs(repo, cogs, rev=rev) except errors.AmbiguousRevision as e: msg = _( "Error: short sha1 `{rev}` is ambiguous. Possible candidates:\n" @@ -1271,74 +767,46 @@ class Downloader(commands.Cog): ).format(rev=rev, repo=repo) await ctx.send(message) return - - await repo.checkout(commit) - cogs_to_check, __ = await self._get_cogs_to_check( - repos=[repo], cogs=cogs, update_repos=False - ) - else: - cogs_to_check, check_failed = await self._get_cogs_to_check(repos=repos, cogs=cogs) - failed_repos.update(check_failed) - - pinned_cogs = {cog for cog in cogs_to_check if cog.pinned} - cogs_to_check -= pinned_cogs + update_result = await _downloader.update_cogs(cogs=cogs, repos=repos) message = "" - if not cogs_to_check: - cogs_to_update = libs_to_update = () + if not update_result.checked_cogs: message += _("There were no cogs to check.") - if pinned_cogs: - cognames = [cog.name for cog in pinned_cogs] - message += ( - _("\nThese cogs are pinned and therefore weren't checked: ") - if len(cognames) > 1 - else _("\nThis cog is pinned and therefore wasn't checked: ") - ) + humanize_list(tuple(map(inline, cognames))) + elif update_result.updates_available: + message = await self._format_cog_update_result(ctx, update_result) else: - cogs_to_update, libs_to_update = await self._available_updates(cogs_to_check) - - updates_available = cogs_to_update or libs_to_update - cogs_to_update, filter_message = self._filter_incorrect_cogs(cogs_to_update) - - if updates_available: - updated_cognames, message = await self._update_cogs_and_libs( - ctx, cogs_to_update, libs_to_update, current_cog_versions=cogs_to_check - ) - else: - if repos: - message += _("Cogs from provided repos are already up to date.") - elif repo: - if cogs: - message += _( - "Provided cogs are already up to date with this revision." - ) - else: - message += _( - "Cogs from provided repo are already up to date with this revision." - ) + if repos: + message += _("Cogs from provided repos are already up to date.") + elif repo: + if cogs: + message += _("Provided cogs are already up to date with this revision.") else: - if cogs: - message += _("Provided cogs are already up to date.") - else: - message += _("All installed cogs are already up to date.") - if repo is not None: - await repo.checkout(repo.branch) - if pinned_cogs: - cognames = [cog.name for cog in pinned_cogs] - message += ( - _("\nThese cogs are pinned and therefore weren't checked: ") - if len(cognames) > 1 - else _("\nThis cog is pinned and therefore wasn't checked: ") - ) + humanize_list(tuple(map(inline, cognames))) - message += filter_message + message += _( + "Cogs from provided repo are already up to date with this revision." + ) + else: + if cogs: + message += _("Provided cogs are already up to date.") + else: + message += _("All installed cogs are already up to date.") - if failed_repos: - message += "\n" + self.format_failed_repos(failed_repos) + if update_result.pinned_cogs: + cognames = [cog.name for cog in update_result.pinned_cogs] + message += ( + _("\nThese cogs are pinned and therefore weren't checked: ") + if len(cognames) > 1 + else _("\nThis cog is pinned and therefore wasn't checked: ") + ) + humanize_list(tuple(map(inline, cognames))) + + message += self._format_incompatible_cogs(update_result) + + if update_result.failed_repos: + message += "\n" + self.format_failed_repos(update_result.failed_repos) repos_with_libs = { inline(module.repo.name) - for module in cogs_to_update + libs_to_update + for module in update_result.updated_modules if module.repo.available_libraries } if repos_with_libs: @@ -1346,8 +814,8 @@ class Downloader(commands.Cog): await self.send_pagified(ctx, message) - if updates_available and updated_cognames: - await self._ask_for_cog_reload(ctx, updated_cognames) + if update_result.updated_cogs: + await self._ask_for_cog_reload(ctx, update_result.updated_cogs) @cog.command(name="list") async def _cog_list(self, ctx: commands.Context, repo: Repo) -> None: @@ -1361,7 +829,7 @@ class Downloader(commands.Cog): - `` The repo to list cogs from. """ sort_function = lambda x: x.name.lower() - all_installed_cogs = await self.installed_cogs() + all_installed_cogs = await _downloader.installed_cogs() installed_cogs_in_repo = [cog for cog in all_installed_cogs if cog.repo_name == repo.name] installed_str = "\n".join( "- {}{}".format(i.name, ": {}".format(i.short) if i.short else "") @@ -1432,218 +900,101 @@ class Downloader(commands.Cog): for page in pagify(msg): await ctx.send(box(page)) - async def is_installed( - self, cog_name: str - ) -> Union[Tuple[bool, InstalledModule], Tuple[bool, None]]: - """Check to see if a cog has been installed through Downloader. - - Parameters - ---------- - cog_name : str - The name of the cog to check for. - - Returns - ------- - `tuple` of (`bool`, `InstalledModule`) - :code:`(True, InstalledModule)` if the cog is installed, else - :code:`(False, None)`. - - """ - for installed_cog in await self.installed_cogs(): - if installed_cog.name == cog_name: - return True, installed_cog - return False, None - - async def _filter_incorrect_cogs_by_names( - self, repo: Repo, cog_names: Iterable[str] - ) -> Tuple[Tuple[Installable, ...], str]: - """Filter out incorrect cogs from list. - - Parameters - ---------- - repo : `Repo` - Repo which should be searched for `cog_names` - cog_names : `list` of `str` - Cog names to search for in repo. - Returns - ------- - tuple - 2-tuple of cogs to install and error message for incorrect cogs. - """ - installed_cogs = await self.installed_cogs() - cogs: List[Installable] = [] - unavailable_cogs: List[str] = [] - already_installed: List[str] = [] - name_already_used: List[str] = [] - - for cog_name in cog_names: - cog: Optional[Installable] = discord.utils.get(repo.available_cogs, name=cog_name) - if cog is None: - unavailable_cogs.append(inline(cog_name)) - continue - if cog in installed_cogs: - already_installed.append(inline(cog_name)) - continue - if discord.utils.get(installed_cogs, name=cog.name): - name_already_used.append(inline(cog_name)) - continue - cogs.append(cog) - + def _format_invalid_cogs( + self, repo: Repo, install_result: _downloader.CogInstallResult + ) -> str: message = "" - - if unavailable_cogs: + if install_result.unavailable_cogs: message = ( _("\nCouldn't find these cogs in {repo.name}: ") - if len(unavailable_cogs) > 1 + if len(install_result.unavailable_cogs) > 1 else _("\nCouldn't find this cog in {repo.name}: ") - ).format(repo=repo) + humanize_list(unavailable_cogs) - if already_installed: + ).format(repo=repo) + humanize_list(install_result.unavailable_cogs) + if install_result.already_installed: message += ( _("\nThese cogs were already installed: ") - if len(already_installed) > 1 + if len(install_result.already_installed) > 1 else _("\nThis cog was already installed: ") - ) + humanize_list(already_installed) - if name_already_used: + ) + humanize_list([cog.name for cog in install_result.already_installed]) + if install_result.name_already_used: message += ( _("\nSome cogs with these names are already installed from different repos: ") - if len(name_already_used) > 1 + if len(install_result.name_already_used) > 1 else _("\nCog with this name is already installed from a different repo: ") - ) + humanize_list(name_already_used) - correct_cogs, add_to_message = self._filter_incorrect_cogs(cogs) + ) + humanize_list([cog.name for cog in install_result.name_already_used]) + # TODO: resolve typing issue + add_to_message = self._format_incompatible_cogs(install_result) if add_to_message: - return correct_cogs, f"{message}{add_to_message}" - return correct_cogs, message + return f"{message}{add_to_message}" + return message - def _filter_incorrect_cogs( - self, cogs: Iterable[Installable] - ) -> Tuple[Tuple[Installable, ...], str]: - correct_cogs: List[Installable] = [] - outdated_python_version: List[str] = [] - outdated_bot_version: List[str] = [] - for cog in cogs: - if cog.min_python_version > sys.version_info: - outdated_python_version.append( + def _format_incompatible_cogs( + self, update_check_result: _downloader.CogUpdateCheckResult + ) -> str: + message = "" + if update_check_result.incompatible_python_version: + message += ( + _("\nThese cogs require higher python version than you have: ") + if len(update_check_result.incompatible_python_version) + else _("\nThis cog requires higher python version than you have: ") + ) + humanize_list( + [ inline(cog.name) + _(" (Minimum: {min_version})").format( min_version=".".join([str(n) for n in cog.min_python_version]) ) - ) - continue - ignore_max = cog.min_bot_version > cog.max_bot_version - if ( - cog.min_bot_version > red_version_info - or not ignore_max - and cog.max_bot_version < red_version_info - ): - outdated_bot_version.append( - inline(cog.name) - + _(" (Minimum: {min_version}").format(min_version=cog.min_bot_version) - + ( - "" - if ignore_max - else _(", at most: {max_version}").format(max_version=cog.max_bot_version) - ) - + ")" - ) - continue - correct_cogs.append(cog) - message = "" - if outdated_python_version: - message += ( - _("\nThese cogs require higher python version than you have: ") - if len(outdated_python_version) - else _("\nThis cog requires higher python version than you have: ") - ) + humanize_list(outdated_python_version) - if outdated_bot_version: + for cog in update_check_result.incompatible_python_version + ] + ) + if update_check_result.incompatible_bot_version: message += ( _( "\nThese cogs require different Red version" " than you currently have ({current_version}): " ) - if len(outdated_bot_version) > 1 + if len(update_check_result.incompatible_bot_version) > 1 else _( "\nThis cog requires different Red version than you currently " "have ({current_version}): " ) - ).format(current_version=red_version_info) + humanize_list(outdated_bot_version) - - return tuple(correct_cogs), message - - async def _get_cogs_to_check( - self, - *, - repos: Optional[Iterable[Repo]] = None, - cogs: Optional[Iterable[InstalledModule]] = None, - update_repos: bool = True, - ) -> Tuple[Set[InstalledModule], List[str]]: - failed = [] - if not (cogs or repos): - if update_repos: - __, failed = await self._repo_manager.update_repos() - - cogs_to_check = { - cog - for cog in await self.installed_cogs() - if cog.repo is not None and cog.repo.name not in failed - } - else: - # this is enough to be sure that `cogs` is not None (based on if above) - if not repos: - cogs = cast(Iterable[InstalledModule], cogs) - repos = {cog.repo for cog in cogs if cog.repo is not None} - - if update_repos: - __, failed = await self._repo_manager.update_repos(repos) - - if failed: - # remove failed repos - repos = {repo for repo in repos if repo.name not in failed} - - if cogs: - cogs_to_check = {cog for cog in cogs if cog.repo is not None and cog.repo in repos} - else: - cogs_to_check = { - cog - for cog in await self.installed_cogs() - if cog.repo is not None and cog.repo in repos - } - - return (cogs_to_check, failed) - - async def _update_cogs_and_libs( - self, - ctx: commands.Context, - cogs_to_update: Iterable[Installable], - libs_to_update: Iterable[Installable], - current_cog_versions: Iterable[InstalledModule], - ) -> Tuple[Set[str], str]: - current_cog_versions_map = {cog.name: cog for cog in current_cog_versions} - failed_reqs = await self._install_requirements(cogs_to_update) - if failed_reqs: - return ( - set(), - ( - _("Failed to install requirements: ") - if len(failed_reqs) > 1 - else _("Failed to install the requirement: ") - ) - + humanize_list(tuple(map(inline, failed_reqs))), + ).format(current_version=red_version_info) + humanize_list( + [ + inline(cog.name) + + _(" (Minimum: {min_version}").format(min_version=cog.min_bot_version) + + ( + "" + if cog.min_bot_version > cog.max_bot_version + else _(", at most: {max_version}").format(max_version=cog.max_bot_version) + ) + + ")" + for cog in update_check_result.incompatible_bot_version + ] ) - installed_cogs, failed_cogs = await self._install_cogs(cogs_to_update) - installed_libs, failed_libs = await self._reinstall_libraries(libs_to_update) - await self._save_to_installed(installed_cogs + installed_libs) + + return message + + async def _format_cog_update_result( + self, ctx: commands.Context, update_result: _downloader.CogUpdateResult + ) -> str: + current_cog_versions_map = {cog.name: cog for cog in update_result.checked_cogs} + if update_result.failed_reqs: + return ( + _("Failed to install requirements: ") + if len(update_result.failed_reqs) > 1 + else _("Failed to install the requirement: ") + ) + humanize_list(tuple(map(inline, update_result.failed_reqs))) + message = _("Cog update completed successfully.") - updated_cognames: Set[str] = set() - if installed_cogs: - updated_cognames = set() + if update_result.updated_cogs: cogs_with_changed_eud_statement = set() - for cog in installed_cogs: - updated_cognames.add(cog.name) + for cog in update_result.updated_cogs: current_eud_statement = current_cog_versions_map[cog.name].end_user_data_statement if current_eud_statement != cog.end_user_data_statement: cogs_with_changed_eud_statement.add(cog.name) - message += _("\nUpdated: ") + humanize_list(tuple(map(inline, updated_cognames))) + message += _("\nUpdated: ") + humanize_list( + [inline(cog.name) for cog in update_result.updated_cogs] + ) if cogs_with_changed_eud_statement: if len(cogs_with_changed_eud_statement) > 1: message += ( @@ -1667,37 +1018,40 @@ class Downloader(commands.Cog): message += _( "\nYou may need to resync your slash commands with `{prefix}slash sync`." ).format(prefix=ctx.prefix) - if failed_cogs: - cognames = [cog.name for cog in failed_cogs] + if update_result.failed_cogs: + cognames = [cog.name for cog in update_result.failed_cogs] message += ( _("\nFailed to update cogs: ") - if len(failed_cogs) > 1 + if len(update_result.failed_cogs) > 1 else _("\nFailed to update cog: ") ) + humanize_list(tuple(map(inline, cognames))) - if not cogs_to_update: + if not update_result.outdated_cogs: message = _("No cogs were updated.") - if installed_libs: + if update_result.updated_libs: message += ( _( "\nSome shared libraries were updated, you should restart the bot " "to bring the changes into effect." ) - if len(installed_libs) > 1 + if len(update_result.updated_libs) > 1 else _( "\nA shared library was updated, you should restart the " "bot to bring the changes into effect." ) ) - if failed_libs: - libnames = [lib.name for lib in failed_libs] + if update_result.failed_libs: + libnames = [lib.name for lib in update_result.failed_libs] message += ( _("\nFailed to install shared libraries: ") - if len(failed_cogs) > 1 + if len(update_result.failed_libs) > 1 else _("\nFailed to install shared library: ") ) + humanize_list(tuple(map(inline, libnames))) - return (updated_cognames, message) + return message - async def _ask_for_cog_reload(self, ctx: commands.Context, updated_cognames: Set[str]) -> None: + async def _ask_for_cog_reload( + self, ctx: commands.Context, updated_cogs: Tuple[InstalledModule, ...] + ) -> None: + updated_cognames = {cog.name for cog in updated_cogs} updated_cognames &= ctx.bot.extensions.keys() # only reload loaded cogs if not updated_cognames: await ctx.send(_("None of the updated cogs were previously loaded. Update complete.")) @@ -1784,7 +1138,7 @@ class Downloader(commands.Cog): cog = command.cog if cog: cog_pkg_name = self.cog_name_from_instance(cog) - installed, cog_installable = await self.is_installed(cog_pkg_name) + installed, cog_installable = await _downloader.is_installed(cog_pkg_name) if installed: made_by = ( humanize_list(cog_installable.author) diff --git a/redbot/core/_downloader/__init__.py b/redbot/core/_downloader/__init__.py new file mode 100644 index 000000000..abd8defd8 --- /dev/null +++ b/redbot/core/_downloader/__init__.py @@ -0,0 +1,865 @@ +# TODO list: +# - design ergonomic APIs instead of whatever you want to call what we have now +# - try to be consistent about requiring Installable vs cog name +# between cog install and other functionality +# - use immutable objects more +# - change Installable's equality to include its commit +# (note: we currently heavily rely on this *not* being the case) +# - add asyncio.Lock appropriately for things that Downloader does +# - avoid doing some of the work on RepoManager initialization to speedup bot startup + +from __future__ import annotations + +import contextlib +import dataclasses +import os +import shutil +import sys +import time +from collections import defaultdict +from pathlib import Path +from typing import ( + Dict, + Iterable, + List, + Literal, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, + TYPE_CHECKING, +) + +import discord +from redbot.core import commands, Config, version_info as red_version_info +from redbot.core._cog_manager import CogManager +from redbot.core.data_manager import cog_data_path + +from . import errors +from .log import log +from .installable import InstallableType, Installable, InstalledModule +from .repo_manager import RepoManager, Repo + +if TYPE_CHECKING: + from redbot.core.bot import Red + + +_SCHEMA_VERSION = 1 +_config: Config +_bot_ref: Optional[Red] +_cog_mgr: CogManager +_repo_manager: RepoManager + +LIB_PATH: Path +SHAREDLIB_PATH: Path +_SHAREDLIB_INIT: Path + + +async def _init(bot: Red) -> None: + global _bot_ref + _bot_ref = bot + + await _init_without_bot(_bot_ref._cog_mgr) + + +async def _init_without_bot(cog_manager: CogManager) -> None: + global _cog_mgr + _cog_mgr = cog_manager + + start = time.perf_counter() + + global _config + _config = Config.get_conf(None, 998240343, cog_name="Downloader", force_registration=True) + _config.register_global(schema_version=0, installed_cogs={}, installed_libraries={}) + await _migrate_config() + + global LIB_PATH, SHAREDLIB_PATH, _SHAREDLIB_INIT + LIB_PATH = cog_data_path(raw_name="Downloader") / "lib" + SHAREDLIB_PATH = LIB_PATH / "cog_shared" + _SHAREDLIB_INIT = SHAREDLIB_PATH / "__init__.py" + _create_lib_folder() + + global _repo_manager + _repo_manager = RepoManager() + await _repo_manager.initialize() + + stop = time.perf_counter() + + log.debug("Finished initialization in %.2fs", stop - start) + + +async def _migrate_config() -> None: + schema_version = await _config.schema_version() + + if schema_version == _SCHEMA_VERSION: + return + + if schema_version == 0: + await _schema_0_to_1() + schema_version += 1 + await _config.schema_version.set(schema_version) + + +async def _schema_0_to_1(): + """ + This contains migration to allow saving state + of both installed cogs and shared libraries. + """ + old_conf = await _config.get_raw("installed", default=[]) + if not old_conf: + return + async with _config.installed_cogs() as new_cog_conf: + for cog_json in old_conf: + repo_name = cog_json["repo_name"] + module_name = cog_json["cog_name"] + if repo_name not in new_cog_conf: + new_cog_conf[repo_name] = {} + new_cog_conf[repo_name][module_name] = { + "repo_name": repo_name, + "module_name": module_name, + "commit": "", + "pinned": False, + } + await _config.clear_raw("installed") + # no reliable way to get installed libraries (i.a. missing repo name) + # but it only helps `[p]cog update` run faster so it's not an issue + + +def _create_lib_folder(*, remove_first: bool = False) -> None: + if remove_first: + shutil.rmtree(str(LIB_PATH)) + SHAREDLIB_PATH.mkdir(parents=True, exist_ok=True) + if not _SHAREDLIB_INIT.exists(): + with _SHAREDLIB_INIT.open(mode="w", encoding="utf-8") as _: + pass + + +async def installed_cogs() -> Tuple[InstalledModule, ...]: + """Get info on installed cogs. + + Returns + ------- + `tuple` of `InstalledModule` + All installed cogs. + + """ + installed = await _config.installed_cogs() + # noinspection PyTypeChecker + return tuple( + InstalledModule.from_json(cog_json, _repo_manager) + for repo_json in installed.values() + for cog_json in repo_json.values() + ) + + +async def installed_libraries() -> Tuple[InstalledModule, ...]: + """Get info on installed shared libraries. + + Returns + ------- + `tuple` of `InstalledModule` + All installed shared libraries. + + """ + installed = await _config.installed_libraries() + # noinspection PyTypeChecker + return tuple( + InstalledModule.from_json(lib_json, _repo_manager) + for repo_json in installed.values() + for lib_json in repo_json.values() + ) + + +async def installed_modules() -> Tuple[InstalledModule, ...]: + """Get info on installed cogs and shared libraries. + + Returns + ------- + `tuple` of `InstalledModule` + All installed cogs and shared libraries. + + """ + return await installed_cogs() + await installed_libraries() + + +async def _save_to_installed(modules: Iterable[InstalledModule]) -> None: + """Mark modules as installed or updates their json in Config. + + Parameters + ---------- + modules : `list` of `InstalledModule` + The modules to check off. + + """ + async with _config.all() as global_data: + installed_cogs = global_data["installed_cogs"] + installed_libraries = global_data["installed_libraries"] + for module in modules: + if module.type is InstallableType.COG: + installed = installed_cogs + elif module.type is InstallableType.SHARED_LIBRARY: + installed = installed_libraries + else: + continue + module_json = module.to_json() + repo_json = installed.setdefault(module.repo_name, {}) + repo_json[module.name] = module_json + + +async def _remove_from_installed(modules: Iterable[InstalledModule]) -> None: + """Remove modules from the saved list + of installed modules (corresponding to type of module). + + Parameters + ---------- + modules : `list` of `InstalledModule` + The modules to remove. + + """ + async with _config.all() as global_data: + installed_cogs = global_data["installed_cogs"] + installed_libraries = global_data["installed_libraries"] + for module in modules: + if module.type is InstallableType.COG: + installed = installed_cogs + elif module.type is InstallableType.SHARED_LIBRARY: + installed = installed_libraries + else: + continue + with contextlib.suppress(KeyError): + installed[module._json_repo_name].pop(module.name) + + +async def _shared_lib_load_check(cog_name: str) -> Optional[Repo]: + _is_installed, cog = await is_installed(cog_name) + if _is_installed and cog.repo is not None and cog.repo.available_libraries: + return cog.repo + return None + + +async def is_installed( + cog_name: str, +) -> Union[Tuple[Literal[True], InstalledModule], Tuple[Literal[False], None]]: + """Check to see if a cog has been installed through Downloader. + + Parameters + ---------- + cog_name : str + The name of the cog to check for. + + Returns + ------- + `tuple` of (`bool`, `InstalledModule`) + :code:`(True, InstalledModule)` if the cog is installed, else + :code:`(False, None)`. + + """ + for installed_cog in await installed_cogs(): + if installed_cog.name == cog_name: + return True, installed_cog + return False, None + + +async def _available_updates( + cogs: Iterable[InstalledModule], +) -> Tuple[Tuple[Installable, ...], Tuple[Installable, ...]]: + """ + Get cogs and libraries which can be updated. + + Parameters + ---------- + cogs : `list` of `InstalledModule` + List of cogs, which should be checked against the updates. + + Returns + ------- + tuple + 2-tuple of cogs and libraries which can be updated. + + """ + repos = {cog.repo for cog in cogs if cog.repo is not None} + _installed_libraries = await installed_libraries() + + modules: Set[InstalledModule] = set() + cogs_to_update: Set[Installable] = set() + libraries_to_update: Set[Installable] = set() + # split libraries and cogs into 2 categories: + # 1. `cogs_to_update`, `libraries_to_update` - module needs update, skip diffs + # 2. `modules` - module MAY need update, check diffs + for repo in repos: + for lib in repo.available_libraries: + try: + index = _installed_libraries.index(lib) + except ValueError: + libraries_to_update.add(lib) + else: + modules.add(_installed_libraries[index]) + for cog in cogs: + if cog.repo is None: + # cog had its repo removed, can't check for updates + continue + if cog.commit: + modules.add(cog) + continue + # marking cog for update if there's no commit data saved (back-compat, see GH-2571) + last_cog_occurrence = await cog.repo.get_last_module_occurrence(cog.name) + if last_cog_occurrence is not None and not last_cog_occurrence.disabled: + cogs_to_update.add(last_cog_occurrence) + + # Reduces diff requests to a single dict with no repeats + hashes: Dict[Tuple[Repo, str], Set[InstalledModule]] = defaultdict(set) + for module in modules: + module.repo = cast(Repo, module.repo) + if module.repo.commit != module.commit: + try: + should_add = await module.repo.is_ancestor(module.commit, module.repo.commit) + except errors.UnknownRevision: + # marking module for update if the saved commit data is invalid + last_module_occurrence = await module.repo.get_last_module_occurrence(module.name) + if last_module_occurrence is not None and not last_module_occurrence.disabled: + if last_module_occurrence.type is InstallableType.COG: + cogs_to_update.add(last_module_occurrence) + elif last_module_occurrence.type is InstallableType.SHARED_LIBRARY: + libraries_to_update.add(last_module_occurrence) + else: + if should_add: + hashes[(module.repo, module.commit)].add(module) + + update_commits = [] + for (repo, old_hash), modules_to_check in hashes.items(): + modified = await repo.get_modified_modules(old_hash, repo.commit) + for module in modules_to_check: + try: + index = modified.index(module) + except ValueError: + # module wasn't modified - we just need to update its commit + module.commit = repo.commit + update_commits.append(module) + else: + modified_module = modified[index] + if modified_module.type is InstallableType.COG: + if not modified_module.disabled: + cogs_to_update.add(modified_module) + elif modified_module.type is InstallableType.SHARED_LIBRARY: + libraries_to_update.add(modified_module) + + await _save_to_installed(update_commits) + + return (tuple(cogs_to_update), tuple(libraries_to_update)) + + +async def _install_cogs( + cogs: Iterable[Installable], +) -> Tuple[Tuple[InstalledModule, ...], Tuple[Installable, ...]]: + """Installs a list of cogs. + + Parameters + ---------- + cogs : `list` of `Installable` + Cogs to install. ``repo`` property of those objects can't be `None` + Returns + ------- + tuple + 2-tuple of installed and failed cogs. + """ + repos: Dict[str, Tuple[Repo, Dict[str, List[Installable]]]] = {} + for cog in cogs: + try: + repo_by_commit = repos[cog.repo_name] + except KeyError: + cog.repo = cast(Repo, cog.repo) # docstring specifies this already + repo_by_commit = repos[cog.repo_name] = (cog.repo, defaultdict(list)) + cogs_by_commit = repo_by_commit[1] + cogs_by_commit[cog.commit].append(cog) + installed = [] + failed = [] + for repo, cogs_by_commit in repos.values(): + exit_to_commit = repo.commit + for commit, cogs_to_install in cogs_by_commit.items(): + await repo.checkout(commit) + for cog in cogs_to_install: + if await cog.copy_to(await _cog_mgr.install_path()): + installed.append(InstalledModule.from_installable(cog)) + else: + failed.append(cog) + await repo.checkout(exit_to_commit) + + # noinspection PyTypeChecker + return (tuple(installed), tuple(failed)) + + +async def _reinstall_libraries( + libraries: Iterable[Installable], +) -> Tuple[Tuple[InstalledModule, ...], Tuple[Installable, ...]]: + """Installs a list of shared libraries, used when updating. + + Parameters + ---------- + libraries : `list` of `Installable` + Libraries to reinstall. ``repo`` property of those objects can't be `None` + Returns + ------- + tuple + 2-tuple of installed and failed libraries. + """ + repos: Dict[str, Tuple[Repo, Dict[str, Set[Installable]]]] = {} + for lib in libraries: + try: + repo_by_commit = repos[lib.repo_name] + except KeyError: + lib.repo = cast(Repo, lib.repo) # docstring specifies this already + repo_by_commit = repos[lib.repo_name] = (lib.repo, defaultdict(set)) + libs_by_commit = repo_by_commit[1] + libs_by_commit[lib.commit].add(lib) + + all_installed: List[InstalledModule] = [] + all_failed: List[Installable] = [] + for repo, libs_by_commit in repos.values(): + exit_to_commit = repo.commit + for commit, libs in libs_by_commit.items(): + await repo.checkout(commit) + installed, failed = await repo.install_libraries( + target_dir=SHAREDLIB_PATH, req_target_dir=LIB_PATH, libraries=libs + ) + all_installed += installed + all_failed += failed + await repo.checkout(exit_to_commit) + + # noinspection PyTypeChecker + return (tuple(all_installed), tuple(all_failed)) + + +async def _install_requirements(cogs: Iterable[Installable]) -> Tuple[str, ...]: + """ + Installs requirements for given cogs. + + Parameters + ---------- + cogs : `list` of `Installable` + Cogs whose requirements should be installed. + Returns + ------- + tuple + Tuple of failed requirements. + """ + + # Reduces requirements to a single list with no repeats + requirements = {requirement for cog in cogs for requirement in cog.requirements} + repos: List[Tuple[Repo, List[str]]] = [(repo, []) for repo in _repo_manager.repos] + + # This for loop distributes the requirements across all repos + # which will allow us to concurrently install requirements + for i, req in enumerate(requirements): + repo_index = i % len(repos) + repos[repo_index][1].append(req) + + has_reqs = list(filter(lambda item: len(item[1]) > 0, repos)) + + failed_reqs = [] + for repo, reqs in has_reqs: + for req in reqs: + if not await repo.install_raw_requirements([req], LIB_PATH): + failed_reqs.append(req) + return tuple(failed_reqs) + + +async def _delete_cog(target: Path) -> None: + """ + Removes an (installed) cog. + :param target: Path pointing to an existing file or directory + :return: + """ + if not target.exists(): + return + + if target.is_dir(): + shutil.rmtree(str(target)) + elif target.is_file(): + os.remove(str(target)) + + +async def _get_cogs_to_check( + *, + repos: Optional[Iterable[Repo]] = None, + cogs: Optional[Iterable[InstalledModule]] = None, + update_repos: bool = True, +) -> Tuple[Set[InstalledModule], List[str]]: + failed = [] + if not (cogs or repos): + if update_repos: + __, failed = await _repo_manager.update_repos() + + cogs_to_check = { + cog + for cog in await installed_cogs() + if cog.repo is not None and cog.repo.name not in failed + } + else: + # this is enough to be sure that `cogs` is not None (based on if above) + if not repos: + cogs = cast(Iterable[InstalledModule], cogs) + repos = {cog.repo for cog in cogs if cog.repo is not None} + + if update_repos: + __, failed = await _repo_manager.update_repos(repos) + + if failed: + # remove failed repos + repos = {repo for repo in repos if repo.name not in failed} + + if cogs: + cogs_to_check = {cog for cog in cogs if cog.repo is not None and cog.repo in repos} + else: + cogs_to_check = { + cog for cog in await installed_cogs() if cog.repo is not None and cog.repo in repos + } + + return (cogs_to_check, failed) + + +# functionality extracted from command implementations +# TODO: make them into nice APIs instead of what they are now... + + +async def pip_install(*deps: str) -> bool: + repo = Repo("", "", "", "", Path.cwd()) + return await repo.install_raw_requirements(deps, LIB_PATH) + + +async def reinstall_requirements() -> tuple[List[str], List[str]]: + _create_lib_folder(remove_first=True) + _installed_cogs = await installed_cogs() + cogs = [] + repos = set() + for cog in _installed_cogs: + if cog.repo is None: + continue + repos.add(cog.repo) + cogs.append(cog) + failed_reqs = await _install_requirements(cogs) + all_installed_libs: List[InstalledModule] = [] + all_failed_libs: List[Installable] = [] + for repo in repos: + installed_libs, failed_libs = await repo.install_libraries( + target_dir=SHAREDLIB_PATH, req_target_dir=LIB_PATH + ) + all_installed_libs += installed_libs + all_failed_libs += failed_libs + + return failed_reqs, all_failed_libs + + +async def install_cogs( + repo: Repo, rev: Optional[str], cog_names: Iterable[str] +) -> CogInstallResult: + commit = None + + if rev is not None: + # raises errors.AmbiguousRevision and errors.UnknownRevision + commit = await repo.get_full_sha1(rev) + + cog_names = set(cog_names) + _installed_cogs = await installed_cogs() + + cogs: List[Installable] = [] + unavailable_cogs: List[str] = [] + already_installed: List[Installable] = [] + name_already_used: List[Installable] = [] + incompatible_python_version: List[Installable] = [] + incompatible_bot_version: List[Installable] = [] + + result_installed_cogs: Tuple[InstalledModule, ...] = () + result_failed_cogs: Tuple[Installable, ...] = () + result_failed_reqs: Tuple[str, ...] = () + result_installed_libs: Tuple[InstalledModule, ...] = () + result_failed_libs: Tuple[Installable, ...] = () + + async with repo.checkout(commit, exit_to_rev=repo.branch): + for cog_name in cog_names: + cog: Optional[Installable] = discord.utils.get(repo.available_cogs, name=cog_name) + if cog is None: + unavailable_cogs.append(cog_name) + elif cog in _installed_cogs: + already_installed.append(cog) + elif discord.utils.get(_installed_cogs, name=cog.name): + name_already_used.append(cog) + elif cog.min_python_version > sys.version_info: + incompatible_python_version.append(cog) + elif cog.min_bot_version > red_version_info or ( + # max version should be ignored when it's lower than min version + cog.min_bot_version <= cog.max_bot_version + and cog.max_bot_version < red_version_info + ): + incompatible_bot_version.append(cog) + else: + cogs.append(cog) + + if cogs: + result_failed_reqs = await _install_requirements(cogs) + if not result_failed_reqs: + result_installed_cogs, result_failed_cogs = await _install_cogs(cogs) + + if cogs and not result_failed_reqs: + result_installed_libs, result_failed_libs = await repo.install_libraries( + target_dir=SHAREDLIB_PATH, req_target_dir=LIB_PATH + ) + if rev is not None: + for cog in result_installed_cogs: + cog.pinned = True + await _save_to_installed(result_installed_cogs + result_installed_libs) + + return CogInstallResult( + installed_cogs=result_installed_cogs, + installed_libs=result_installed_libs, + failed_cogs=result_failed_cogs, + failed_libs=result_failed_libs, + failed_reqs=result_failed_reqs, + unavailable_cogs=tuple(unavailable_cogs), + already_installed=tuple(already_installed), + name_already_used=tuple(name_already_used), + incompatible_python_version=tuple(incompatible_python_version), + incompatible_bot_version=tuple(incompatible_bot_version), + ) + + +async def uninstall_cogs(*cogs: InstalledModule) -> tuple[list[str], list[str]]: + uninstalled_cogs = [] + failed_cogs = [] + for cog in set(cogs): + real_name = cog.name + + poss_installed_path = (await _cog_mgr.install_path()) / real_name + if poss_installed_path.exists(): + if _bot_ref is not None: + with contextlib.suppress(commands.ExtensionNotLoaded): + await _bot_ref.unload_extension(real_name) + await _bot_ref.remove_loaded_package(real_name) + await _delete_cog(poss_installed_path) + uninstalled_cogs.append(real_name) + else: + failed_cogs.append(real_name) + await _remove_from_installed(cogs) + + return uninstalled_cogs, failed_cogs + + +async def check_cog_updates( + *, + repos: Optional[Iterable[Repo]] = None, + cogs: Optional[Iterable[InstalledModule]] = None, + update_repos: bool = True, +) -> CogUpdateCheckResult: + cogs_to_check, failed_repos = await _get_cogs_to_check( + repos=repos, cogs=cogs, update_repos=update_repos + ) + outdated_cogs, outdated_libs = await _available_updates(cogs_to_check) + + updatable_cogs: List[Installable] = [] + incompatible_python_version: List[Installable] = [] + incompatible_bot_version: List[Installable] = [] + for cog in outdated_cogs: + if cog.min_python_version > sys.version_info: + incompatible_python_version.append(cog) + elif cog.min_bot_version > red_version_info or ( + # max version should be ignored when it's lower than min version + cog.min_bot_version <= cog.max_bot_version + and cog.max_bot_version < red_version_info + ): + incompatible_bot_version.append(cog) + else: + updatable_cogs.append(cog) + + return CogUpdateCheckResult( + outdated_cogs=outdated_cogs, + outdated_libs=outdated_libs, + updatable_cogs=tuple(updatable_cogs), + failed_repos=tuple(failed_repos), + incompatible_python_version=tuple(incompatible_python_version), + incompatible_bot_version=tuple(incompatible_bot_version), + ) + + +# update given cogs or all cogs +async def update_cogs( + *, cogs: Optional[List[InstalledModule]] = None, repos: Optional[List[Repo]] = None +) -> CogUpdateResult: + if cogs is not None and repos is not None: + raise ValueError("You can specify cogs or repos argument, not both") + + cogs_to_check, failed_repos = await _get_cogs_to_check(repos=repos, cogs=cogs) + return await _update_cogs(cogs_to_check, failed_repos=failed_repos) + + +# update given cogs or all cogs from the specified repo +# using the specified revision (or latest if not specified) +async def update_repo_cogs( + repo: Repo, cogs: Optional[List[InstalledModule]] = None, *, rev: Optional[str] = None +) -> CogUpdateResult: + try: + await repo.update() + except errors.UpdateError: + return await _update_cogs(set(), failed_repos=[repo]) + + # TODO: should this be set to `repo.branch` when `rev` is None? + commit = None + if rev is not None: + # raises errors.AmbiguousRevision and errors.UnknownRevision + commit = await repo.get_full_sha1(rev) + async with repo.checkout(commit, exit_to_rev=repo.branch): + cogs_to_check, __ = await _get_cogs_to_check(repos=[repo], cogs=cogs, update_repos=False) + return await _update_cogs(cogs_to_check, failed_repos=()) + + +async def _update_cogs( + cogs_to_check: Set[InstalledModule], *, failed_repos: Sequence[Repo] +) -> CogUpdateResult: + pinned_cogs = {cog for cog in cogs_to_check if cog.pinned} + cogs_to_check -= pinned_cogs + + outdated_cogs: Tuple[Installable, ...] = () + outdated_libs: Tuple[Installable, ...] = () + updatable_cogs: List[Installable] = [] + incompatible_python_version: List[Installable] = [] + incompatible_bot_version: List[Installable] = [] + + updated_cogs: Tuple[InstalledModule, ...] = () + failed_cogs: Tuple[Installable, ...] = () + failed_reqs: Tuple[str, ...] = () + updated_libs: Tuple[InstalledModule, ...] = () + failed_libs: Tuple[Installable, ...] = () + + if cogs_to_check: + outdated_cogs, outdated_libs = await _available_updates(cogs_to_check) + + for cog in outdated_cogs: + if cog.min_python_version > sys.version_info: + incompatible_python_version.append(cog) + elif cog.min_bot_version > red_version_info or ( + # max version should be ignored when it's lower than min version + cog.min_bot_version <= cog.max_bot_version + and cog.max_bot_version < red_version_info + ): + incompatible_bot_version.append(cog) + else: + updatable_cogs.append(cog) + + if updatable_cogs or outdated_libs: + failed_reqs = await _install_requirements(updatable_cogs) + if not failed_reqs: + updated_cogs, failed_cogs = await _install_cogs(updatable_cogs) + updated_libs, failed_libs = await _reinstall_libraries(outdated_libs) + await _save_to_installed(updated_cogs + updated_libs) + + return CogUpdateResult( + checked_cogs=frozenset(cogs_to_check), + pinned_cogs=frozenset(pinned_cogs), + updated_cogs=updated_cogs, + updated_libs=updated_libs, + failed_cogs=failed_cogs, + failed_libs=failed_libs, + failed_reqs=failed_reqs, + outdated_cogs=outdated_cogs, + outdated_libs=outdated_libs, + updatable_cogs=tuple(updatable_cogs), + failed_repos=tuple(failed_repos), + incompatible_python_version=tuple(incompatible_python_version), + incompatible_bot_version=tuple(incompatible_bot_version), + ) + + +async def pin_cogs( + *cogs: InstalledModule, +) -> tuple[tuple[InstalledModule, ...], tuple[InstalledModule, ...]]: + already_pinned = [] + pinned = [] + for cog in set(cogs): + if cog.pinned: + already_pinned.append(cog) + continue + cog.pinned = True + pinned.append(cog) + if pinned: + await _save_to_installed(pinned) + + return tuple(pinned), tuple(already_pinned) + + +async def unpin_cogs( + *cogs: InstalledModule, +) -> tuple[tuple[InstalledModule, ...], tuple[InstalledModule, ...]]: + not_pinned = [] + unpinned = [] + for cog in set(cogs): + if not cog.pinned: + not_pinned.append(cog) + continue + cog.pinned = False + unpinned.append(cog) + if unpinned: + await _save_to_installed(unpinned) + + return tuple(unpinned), tuple(not_pinned) + + +# TODO: make kw_only +@dataclasses.dataclass +class CogInstallResult: + installed_cogs: Tuple[InstalledModule, ...] + installed_libs: Tuple[InstalledModule, ...] + failed_cogs: Tuple[Installable, ...] + failed_libs: Tuple[Installable, ...] + failed_reqs: Tuple[str, ...] + unavailable_cogs: Tuple[str, ...] + already_installed: Tuple[Installable, ...] + name_already_used: Tuple[Installable, ...] + incompatible_python_version: Tuple[Installable, ...] + incompatible_bot_version: Tuple[Installable, ...] + + +# TODO: make kw_only +@dataclasses.dataclass +class CogUpdateCheckResult: + outdated_cogs: Tuple[Installable, ...] + outdated_libs: Tuple[Installable, ...] + updatable_cogs: Tuple[Installable, ...] + failed_repos: Tuple[Repo, ...] + incompatible_python_version: Tuple[Installable, ...] + incompatible_bot_version: Tuple[Installable, ...] + + @property + def updates_available(self) -> bool: + return bool(self.outdated_cogs or self.outdated_libs) + + @property + def updates_installable(self) -> bool: + return bool(self.updatable_cogs or self.outdated_libs) + + @property + def incompatible_cogs(self) -> Tuple[Installable, ...]: + return self.incompatible_python_version + self.incompatible_bot_version + + +# TODO: make kw_only +@dataclasses.dataclass +class CogUpdateResult(CogUpdateCheckResult): + # checked_cogs contains old modules, before update + checked_cogs: Set[InstalledModule] + pinned_cogs: Set[InstalledModule] + updated_cogs: Tuple[InstalledModule, ...] + updated_libs: Tuple[InstalledModule, ...] + failed_cogs: Tuple[Installable, ...] + failed_libs: Tuple[Installable, ...] + failed_reqs: Tuple[str, ...] + + @property + def updated_modules(self) -> Tuple[InstalledModule, ...]: + return self.updated_cogs + self.updated_libs + + +class CogUnavailableError(Exception): + def __init__(self, repo_name: str, cog_name: str) -> None: + self.repo_name = repo_name + self.cog_name = cog_name + super().__init__(f"Couldn't find cog {cog_name!r} in {repo_name!r}") diff --git a/redbot/cogs/downloader/errors.py b/redbot/core/_downloader/errors.py similarity index 100% rename from redbot/cogs/downloader/errors.py rename to redbot/core/_downloader/errors.py diff --git a/redbot/cogs/downloader/info_schemas.py b/redbot/core/_downloader/info_schemas.py similarity index 100% rename from redbot/cogs/downloader/info_schemas.py rename to redbot/core/_downloader/info_schemas.py diff --git a/redbot/cogs/downloader/installable.py b/redbot/core/_downloader/installable.py similarity index 95% rename from redbot/cogs/downloader/installable.py rename to redbot/core/_downloader/installable.py index abda7d924..7fdd8c0bf 100644 --- a/redbot/cogs/downloader/installable.py +++ b/redbot/core/_downloader/installable.py @@ -2,7 +2,7 @@ from __future__ import annotations import functools import shutil -from enum import IntEnum +from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union, cast @@ -16,8 +16,7 @@ if TYPE_CHECKING: from .repo_manager import RepoManager, Repo -class InstallableType(IntEnum): - # using IntEnum, because hot-reload breaks its identity +class InstallableType(Enum): UNKNOWN = 0 COG = 1 SHARED_LIBRARY = 2 @@ -139,7 +138,7 @@ class Installable(RepoJSONMixin): super()._read_info_file() update_mixin(self, INSTALLABLE_SCHEMA) - if self.type == InstallableType.SHARED_LIBRARY: + if self.type is InstallableType.SHARED_LIBRARY: self.hidden = True @@ -163,7 +162,7 @@ class InstalledModule(Installable): json_repo_name: str = "", ): super().__init__(location=location, repo=repo, commit=commit) - self.pinned: bool = pinned if self.type == InstallableType.COG else False + self.pinned: bool = pinned if self.type is InstallableType.COG else False # this is here so that Downloader could use real repo name instead of "MISSING_REPO" self._json_repo_name = json_repo_name @@ -173,7 +172,7 @@ class InstalledModule(Installable): "module_name": self.name, "commit": self.commit, } - if self.type == InstallableType.COG: + if self.type is InstallableType.COG: module_json["pinned"] = self.pinned return module_json diff --git a/redbot/cogs/downloader/json_mixins.py b/redbot/core/_downloader/json_mixins.py similarity index 100% rename from redbot/cogs/downloader/json_mixins.py rename to redbot/core/_downloader/json_mixins.py diff --git a/redbot/core/_downloader/log.py b/redbot/core/_downloader/log.py new file mode 100644 index 000000000..ce0f8df06 --- /dev/null +++ b/redbot/core/_downloader/log.py @@ -0,0 +1,3 @@ +import logging + +log = logging.getLogger("red.core.downloader") diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/core/_downloader/repo_manager.py similarity index 98% rename from redbot/cogs/downloader/repo_manager.py rename to redbot/core/_downloader/repo_manager.py index 8d9b3fa06..401a6b530 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/core/_downloader/repo_manager.py @@ -169,21 +169,6 @@ class Repo(RepoJSONMixin): except ValueError: return self.url - @classmethod - async def convert(cls, ctx: commands.Context, argument: str) -> Repo: - downloader_cog = ctx.bot.get_cog("Downloader") - if downloader_cog is None: - raise commands.CommandError(_("No Downloader cog found.")) - - # noinspection PyProtectedMember - repo_manager = downloader_cog._repo_manager - poss_repo = repo_manager.get_repo(argument) - if poss_repo is None: - raise commands.BadArgument( - _('Repo by the name "{repo_name}" does not exist.').format(repo_name=argument) - ) - return poss_repo - def _existing_git_repo(self) -> Tuple[bool, Path]: git_path = self.folder_path / ".git" return git_path.exists(), git_path @@ -1001,7 +986,7 @@ class Repo(RepoJSONMixin): """ # noinspection PyTypeChecker return tuple( - [m for m in self.available_modules if m.type == InstallableType.COG and not m.disabled] + [m for m in self.available_modules if m.type is InstallableType.COG and not m.disabled] ) @property @@ -1011,7 +996,7 @@ class Repo(RepoJSONMixin): """ # noinspection PyTypeChecker return tuple( - [m for m in self.available_modules if m.type == InstallableType.SHARED_LIBRARY] + [m for m in self.available_modules if m.type is InstallableType.SHARED_LIBRARY] ) @classmethod diff --git a/redbot/core/bot.py b/redbot/core/bot.py index d274526fd..84f1c2c62 100644 --- a/redbot/core/bot.py +++ b/redbot/core/bot.py @@ -37,7 +37,18 @@ import discord from discord.ext import commands as dpy_commands from discord.ext.commands import when_mentioned_or -from . import Config, _i18n, i18n, app_commands, commands, errors, _drivers, modlog, bank +from . import ( + Config, + _i18n, + i18n, + app_commands, + commands, + errors, + _drivers, + modlog, + bank, + _downloader, +) from ._cli import ExitCodes from ._cog_manager import CogManager, CogManagerUI from .core_commands import Core @@ -1200,12 +1211,11 @@ class Red( ver_info = list(sys.version_info[:2]) python_version_changed = False - LIB_PATH = cog_data_path(raw_name="Downloader") / "lib" if ver_info != last_system_info["python_version"]: await self._config.last_system_info.python_version.set(ver_info) - if any(LIB_PATH.iterdir()): - shutil.rmtree(str(LIB_PATH)) - LIB_PATH.mkdir() + if any(_downloader.LIB_PATH.iterdir()): + shutil.rmtree(str(_downloader.LIB_PATH)) + _downloader.LIB_PATH.mkdir() asyncio.create_task( send_to_owners_with_prefix_replaced( self, diff --git a/redbot/core/core_commands.py b/redbot/core/core_commands.py index f8525e0a4..007ca853c 100644 --- a/redbot/core/core_commands.py +++ b/redbot/core/core_commands.py @@ -49,6 +49,7 @@ from . import ( i18n, bank, modlog, + _downloader, ) from ._diagnoser import IssueDiagnoser from .utils import AsyncIter, can_user_send_messages_in @@ -215,12 +216,8 @@ class CoreLogic: else: await bot.add_loaded_package(name) loaded_packages.append(name) - # remove in Red 3.4 - downloader = bot.get_cog("Downloader") - if downloader is None: - continue try: - maybe_repo = await downloader._shared_lib_load_check(name) + maybe_repo = await _downloader._shared_lib_load_check(name) except Exception: log.exception( "Shared library check failed," diff --git a/redbot/core/utils/_internal_utils.py b/redbot/core/utils/_internal_utils.py index a8a49ec72..cd70a760a 100644 --- a/redbot/core/utils/_internal_utils.py +++ b/redbot/core/utils/_internal_utils.py @@ -236,7 +236,7 @@ async def create_backup(dest: Path = Path.home()) -> Optional[Path]: ] # Avoiding circular imports - from ...cogs.downloader.repo_manager import RepoManager + from redbot.core._downloader.repo_manager import RepoManager repo_mgr = RepoManager() await repo_mgr.initialize() diff --git a/redbot/pytest/downloader.py b/redbot/pytest/downloader.py index 2f961fbac..4457cc3e9 100644 --- a/redbot/pytest/downloader.py +++ b/redbot/pytest/downloader.py @@ -6,8 +6,8 @@ import shutil import pytest -from redbot.cogs.downloader.repo_manager import RepoManager, Repo, ProcessFormatter -from redbot.cogs.downloader.installable import Installable, InstalledModule +from redbot.core._downloader.repo_manager import RepoManager, Repo, ProcessFormatter +from redbot.core._downloader.installable import Installable, InstalledModule __all__ = [ "GIT_VERSION", diff --git a/tests/cogs/downloader/__init__.py b/tests/core/_downloader/__init__.py similarity index 100% rename from tests/cogs/downloader/__init__.py rename to tests/core/_downloader/__init__.py diff --git a/tests/cogs/downloader/test_downloader.py b/tests/core/_downloader/test_downloader.py similarity index 94% rename from tests/cogs/downloader/test_downloader.py rename to tests/core/_downloader/test_downloader.py index e60d401f4..f8ce2b2cd 100644 --- a/tests/cogs/downloader/test_downloader.py +++ b/tests/core/_downloader/test_downloader.py @@ -9,9 +9,9 @@ from pytest_mock import MockFixture from redbot.pytest.downloader import * -from redbot.cogs.downloader.repo_manager import Installable -from redbot.cogs.downloader.repo_manager import Candidate, ProcessFormatter, RepoManager, Repo -from redbot.cogs.downloader.errors import ( +from redbot.core._downloader.repo_manager import Installable +from redbot.core._downloader.repo_manager import Candidate, ProcessFormatter, RepoManager, Repo +from redbot.core._downloader.errors import ( AmbiguousRevision, ExistingGitRepo, GitException, @@ -322,9 +322,9 @@ async def test_update(mocker, repo): async def test_add_repo(monkeypatch, repo_manager): - monkeypatch.setattr("redbot.cogs.downloader.repo_manager.Repo._run", fake_run_noprint) + monkeypatch.setattr("redbot.core._downloader.repo_manager.Repo._run", fake_run_noprint) monkeypatch.setattr( - "redbot.cogs.downloader.repo_manager.Repo.current_commit", fake_current_commit + "redbot.core._downloader.repo_manager.Repo.current_commit", fake_current_commit ) squid = await repo_manager.add_repo( @@ -335,9 +335,9 @@ async def test_add_repo(monkeypatch, repo_manager): async def test_lib_install_requirements(monkeypatch, library_installable, repo, tmpdir): - monkeypatch.setattr("redbot.cogs.downloader.repo_manager.Repo._run", fake_run_noprint) + monkeypatch.setattr("redbot.core._downloader.repo_manager.Repo._run", fake_run_noprint) monkeypatch.setattr( - "redbot.cogs.downloader.repo_manager.Repo.available_libraries", (library_installable,) + "redbot.core._downloader.repo_manager.Repo.available_libraries", (library_installable,) ) lib_path = Path(str(tmpdir)) / "cog_data_path" / "lib" @@ -353,9 +353,9 @@ async def test_lib_install_requirements(monkeypatch, library_installable, repo, async def test_remove_repo(monkeypatch, repo_manager): - monkeypatch.setattr("redbot.cogs.downloader.repo_manager.Repo._run", fake_run_noprint) + monkeypatch.setattr("redbot.core._downloader.repo_manager.Repo._run", fake_run_noprint) monkeypatch.setattr( - "redbot.cogs.downloader.repo_manager.Repo.current_commit", fake_current_commit + "redbot.core._downloader.repo_manager.Repo.current_commit", fake_current_commit ) await repo_manager.add_repo( diff --git a/tests/cogs/downloader/test_git.py b/tests/core/_downloader/test_git.py similarity index 99% rename from tests/cogs/downloader/test_git.py rename to tests/core/_downloader/test_git.py index a2e99734f..d00c5b69f 100644 --- a/tests/cogs/downloader/test_git.py +++ b/tests/core/_downloader/test_git.py @@ -3,7 +3,7 @@ import subprocess as sp import pytest -from redbot.cogs.downloader.repo_manager import ProcessFormatter, Repo +from redbot.core._downloader.repo_manager import ProcessFormatter, Repo from redbot.pytest.downloader import ( GIT_VERSION, cloned_git_repo, diff --git a/tests/cogs/downloader/test_installable.py b/tests/core/_downloader/test_installable.py similarity index 88% rename from tests/cogs/downloader/test_installable.py rename to tests/core/_downloader/test_installable.py index 825945baf..e9106ffa2 100644 --- a/tests/cogs/downloader/test_installable.py +++ b/tests/core/_downloader/test_installable.py @@ -4,14 +4,14 @@ from pathlib import Path import pytest from redbot.pytest.downloader import * -from redbot.cogs.downloader.installable import Installable, InstallableType +from redbot.core._downloader.installable import Installable, InstallableType from redbot.core import VersionInfo def test_process_info_file(installable): for k, v in INFO_JSON.items(): if k == "type": - assert installable.type == InstallableType.COG + assert installable.type is InstallableType.COG elif k in ("min_bot_version", "max_bot_version"): assert getattr(installable, k) == VersionInfo.from_str(v) else: @@ -21,7 +21,7 @@ def test_process_info_file(installable): def test_process_lib_info_file(library_installable): for k, v in LIBRARY_INFO_JSON.items(): if k == "type": - assert library_installable.type == InstallableType.SHARED_LIBRARY + assert library_installable.type is InstallableType.SHARED_LIBRARY elif k in ("min_bot_version", "max_bot_version"): assert getattr(library_installable, k) == VersionInfo.from_str(v) elif k == "hidden":