Merge pull request #2019 from Salamandar/rework_autoupdater

Rework autoupdater - part 2

autoupdate_app_sources/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

autoupdate_app_sources/autoupdate_app_sources.py

@@ -1,25 +1,28 @@
 #!/usr/bin/env python3
 
 import argparse
-import glob
 import hashlib
-import os
+import multiprocessing
+import logging
+from typing import Any
 import re
 import sys
-import time
+import textwrap
 from pathlib import Path
+from functools import cache
 from datetime import datetime
 
 import requests
 import toml
 import tqdm
-from tqdm.contrib.logging import logging_redirect_tqdm
+import github
 
 # add apps/tools to sys.path
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
-from rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType
+from rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType  # noqa: E402,E501 pylint: disable=import-error,wrong-import-position
-from appslib.utils import REPO_APPS_ROOT, get_catalog  # pylint: disable=import-error
+from appslib.utils import REPO_APPS_ROOT, get_catalog  # noqa: E402 pylint: disable=import-error,wrong-import-position
+from app_caches import app_cache_folder  # noqa: E402 pylint: disable=import-error,wrong-import-position
 
 
 STRATEGIES = [
@@ -34,15 +37,24 @@ STRATEGIES = [
     "latest_gitea_commit",
     "latest_forgejo_release",
     "latest_forgejo_tag",
-    "latest_forgejo_commit"
+    "latest_forgejo_commit",
-    ]
+]
 
-dry_run = True
 
-# For github authentication
+@cache
-auth = None
+def get_github() -> tuple[tuple[str, str] | None, github.Github | None, github.InputGitAuthor | None]:
-github = None
+    try:
-author = None
+        github_login = (REPO_APPS_ROOT / ".github_login").open("r", encoding="utf-8").read().strip()
+        github_token = (REPO_APPS_ROOT / ".github_token").open("r", encoding="utf-8").read().strip()
+        github_email = (REPO_APPS_ROOT / ".github_email").open("r", encoding="utf-8").read().strip()
+
+        auth = (github_login, github_token)
+        github_api = github.Github(github_token)
+        author = github.InputGitAuthor(github_login, github_email)
+        return auth, github_api, author
+    except Exception as e:
+        logging.warning(f"Could not get github: {e}")
+        return None, None, None
 
 
 def apps_to_run_auto_update_for():
@@ -53,61 +65,202 @@ def apps_to_run_auto_update_for():
         and "/github.com/yunohost-apps" in infos["url"].lower()
     ]
 
-    manifest_tomls = glob.glob(
+    relevant_apps = []
-        os.path.dirname(__file__) + "/../../.apps_cache/*/manifest.toml"
+    for app in apps_flagged_as_working_and_on_yunohost_apps_org:
-    )
+        manifest_toml = app_cache_folder(app) / "manifest.toml"
-
+        if manifest_toml.exists():
-    apps_with_manifest_toml = [path.split("/")[-2] for path in manifest_tomls]
+            manifest = toml.load(manifest_toml.open("r", encoding="utf-8"))
-
-    relevant_apps = list(
-        sorted(
-            set(apps_flagged_as_working_and_on_yunohost_apps_org)
-            & set(apps_with_manifest_toml)
-        )
-    )
-
-    out = []
-    for app in relevant_apps:
-        manifest = toml.load(
-            os.path.dirname(__file__) + f"/../../.apps_cache/{app}/manifest.toml"
-        )
             sources = manifest.get("resources", {}).get("sources", {})
             if any("autoupdate" in source for source in sources.values()):
-            out.append(app)
+                relevant_apps.append(app)
-    return out
+    return relevant_apps
 
 
-def filter_and_get_latest_tag(tags, app_id):
+class LocalOrRemoteRepo:
-    filter_keywords = ["start", "rc", "beta", "alpha"]
+    def __init__(self, app: str | Path) -> None:
-    tags = [t for t in tags if not any(keyword in t for keyword in filter_keywords)]
+        self.local = False
+        self.remote = False
 
-    tag_dict = {}
+        self.app = app
-    for t in tags:
+        if isinstance(app, Path):
-        t_to_check = t
+            # It's local
-        if t.startswith(app_id + "-"):
+            self.local = True
-            t_to_check = t.split("-", 1)[-1]
+            self.manifest_path = app / "manifest.toml"
-        # Boring special case for dokuwiki...
+
-        elif t.startswith("release-"):
+            if not self.manifest_path.exists():
-            t_to_check = t.split("-", 1)[-1].replace("-", ".")
+                raise RuntimeError(f"{app.name}: manifest.toml doesnt exists?")
+            # app is in fact a path
+            self.manifest_raw = (app / "manifest.toml").open("r", encoding="utf-8").read()
+
+        elif isinstance(app, str):
+            # It's remote
+            self.remote = True
+            github = get_github()[1]
+            assert github, "Could not get github authentication!"
+            self.repo = github.get_repo(f"Yunohost-Apps/{app}_ynh")
+            self.pr_branch = None
+            # Determine base branch, either `testing` or default branch
+            try:
+                self.base_branch = self.repo.get_branch("testing").name
+            except Exception:
+                self.base_branch = self.repo.default_branch
+            contents = self.repo.get_contents("manifest.toml", ref=self.base_branch)
+            assert not isinstance(contents, list)
+            self.manifest_raw = contents.decoded_content.decode()
+            self.manifest_raw_sha = contents.sha
 
-        if not re.match(r"^v?[\d\.]*\-?\d$", t_to_check):
-            print(f"Ignoring tag {t_to_check}, doesn't look like a version number")
         else:
-            tag_dict[t] = tag_to_int_tuple(t_to_check)
+            raise TypeError(f"Invalid argument type for app: {type(app)}")
 
-    tags = sorted(list(tag_dict.keys()), key=tag_dict.get)
+    def edit_manifest(self, content: str):
+        self.manifest_raw = content
+        if self.local:
+            self.manifest_path.open("w", encoding="utf-8").write(content)
 
-    return tags[-1], ".".join([str(i) for i in tag_dict[tags[-1]]])
+    def commit(self, message: str):
+        if self.remote:
+            author = get_github()[2]
+            assert author, "Could not get Github author!"
+            assert self.pr_branch is not None, "Did you forget to create a branch?"
+            self.repo.update_file(
+                "manifest.toml",
+                message=message,
+                content=self.manifest_raw,
+                sha=self.manifest_raw_sha,
+                branch=self.pr_branch,
+                author=author,
+            )
+
+    def new_branch(self, name: str) -> bool:
+        if self.local:
+            logging.warning("Can't create branches for local repositories")
+            return False
+        if self.remote:
+            self.pr_branch = name
+            commit_sha = self.repo.get_branch(self.base_branch).commit.sha
+            if self.pr_branch in [branch.name for branch in self.repo.get_branches()]:
+                return False
+            self.repo.create_git_ref(ref=f"refs/heads/{name}", sha=commit_sha)
+            return True
+        return False
+
+    def create_pr(self, branch: str, title: str, message: str) -> str | None:
+        if self.local:
+            logging.warning("Can't create pull requests for local repositories")
+            return
+        if self.remote:
+            # Open the PR
+            pr = self.repo.create_pull(
+                title=title, body=message, head=branch, base=self.base_branch
+            )
+            return pr.url
+
+    def get_pr(self, branch: str) -> str:
+        return next(pull.html_url for pull in self.repo.get_pulls(head=branch))
 
 
-def tag_to_int_tuple(tag):
+class AppAutoUpdater:
+    def __init__(self, app_id: str | Path) -> None:
+        self.repo = LocalOrRemoteRepo(app_id)
+        self.manifest = toml.loads(self.repo.manifest_raw)
+
+        self.app_id = self.manifest["id"]
+        self.current_version = self.manifest["version"].split("~")[0]
+        self.sources = self.manifest.get("resources", {}).get("sources")
+        self.main_upstream = self.manifest.get("upstream", {}).get("code")
+
+        if not self.sources:
+            raise RuntimeError("There's no resources.sources in manifest.toml ?")
+
+        self.main_upstream = self.manifest.get("upstream", {}).get("code")
+
+    def run(self, edit: bool = False, commit: bool = False, pr: bool = False) -> bool | tuple[str | None, str | None, str | None]:
+        has_updates = False
+        main_version = None
+        pr_url = None
+
+        # Default message
+        pr_title = commit_msg = "Upgrade sources"
+        branch_name = "ci-auto-update-sources"
+
+        for source, infos in self.sources.items():
+            update = self.get_source_update(source, infos)
+            if update is None:
+                continue
+            has_updates = True
+            version, assets, msg = update
+
+            if source == "main":
+                main_version = version
+                branch_name = f"ci-auto-update-{version}"
+                pr_title = commit_msg = f"Upgrade to v{version}"
+                if msg:
+                    commit_msg += f"\n{msg}"
+
+            self.repo.manifest_raw = self.replace_version_and_asset_in_manifest(
+                self.repo.manifest_raw, version, assets, infos, is_main=source == "main",
+            )
+
+        if not has_updates:
+            return False
+
+        if edit:
+            self.repo.edit_manifest(self.repo.manifest_raw)
+
+        try:
+            if pr:
+                self.repo.new_branch(branch_name)
+            if commit:
+                self.repo.commit(commit_msg)
+            if pr:
+                pr_url = self.repo.create_pr(branch_name, pr_title, commit_msg)
+        except github.GithubException as e:
+            if e.status == 422 or e.status == 409:
+                pr_url = f"already existing pr: {self.repo.get_pr(branch_name)}"
+            else:
+                raise
+        return self.current_version, main_version, pr_url
+
+    @staticmethod
+    def filter_and_get_latest_tag(tags: list[str], app_id: str) -> tuple[str, str]:
+        def version_numbers(tag: str) -> tuple[int, ...] | None:
+            filter_keywords = ["start", "rc", "beta", "alpha"]
+            if any(keyword in tag for keyword in filter_keywords):
+                logging.debug(f"Tag {tag} contains filtered keyword from {filter_keywords}.")
+                return None
+
+            t_to_check = tag
+            if tag.startswith(app_id + "-"):
+                t_to_check = tag.split("-", 1)[-1]
+            # Boring special case for dokuwiki...
+            elif tag.startswith("release-"):
+                t_to_check = tag.split("-", 1)[-1].replace("-", ".")
+
+            if re.match(r"^v?[\d\.]*\-?\d$", t_to_check):
+                return AppAutoUpdater.tag_to_int_tuple(t_to_check)
+            print(f"Ignoring tag {t_to_check}, doesn't look like a version number")
+            return None
+
+        # sorted will sort by keys
+        tags_dict = {version_numbers(tag): tag for tag in tags}
+        tags_dict.pop(None, None)
+        # reverse=True will set the last release as first element
+        tags_dict = dict(sorted(tags_dict.items(), reverse=True))
+        if not tags_dict:
+            raise RuntimeError("No tags were found after sanity filtering!")
+        the_tag_list, the_tag = next(iter(tags_dict.items()))
+        assert the_tag_list is not None
+        return the_tag, ".".join(str(i) for i in the_tag_list)
+
+    @staticmethod
+    def tag_to_int_tuple(tag: str) -> tuple[int, ...]:
         tag = tag.strip("v").replace("-", ".").strip(".")
         int_tuple = tag.split(".")
         assert all(i.isdigit() for i in int_tuple), f"Cant convert {tag} to int tuple :/"
         return tuple(int(i) for i in int_tuple)
 
-
+    @staticmethod
-def sha256_of_remote_file(url):
+    def sha256_of_remote_file(url: str) -> str:
         print(f"Computing sha256sum for {url} ...")
         try:
             r = requests.get(url, stream=True)
@@ -116,70 +269,24 @@ def sha256_of_remote_file(url):
                 m.update(data)
             return m.hexdigest()
         except Exception as e:
-        print(f"Failed to compute sha256 for {url} : {e}")
+            raise RuntimeError(f"Failed to compute sha256 for {url} : {e}") from e
+
+    def get_source_update(self, name: str, infos: dict[str, Any]) -> tuple[str, str | dict[str, str], str] | None:
+        if "autoupdate" not in infos:
             return None
 
-
+        print(f"\n  Checking {name} ...")
-class AppAutoUpdater:
+        asset = infos.get("autoupdate", {}).get("asset", "tarball")
-    def __init__(self, app_id, app_id_is_local_app_dir=False):
-        if app_id_is_local_app_dir:
-            if not os.path.exists(app_id + "/manifest.toml"):
-                raise Exception("manifest.toml doesnt exists?")
-            # app_id is in fact a path
-            manifest = toml.load(open(app_id + "/manifest.toml"))
-
-        else:
-            # We actually want to look at the manifest on the "testing" (or default) branch
-            self.repo = github.get_repo(f"Yunohost-Apps/{app_id}_ynh")
-            # Determine base branch, either `testing` or default branch
-            try:
-                self.base_branch = self.repo.get_branch("testing").name
-            except:
-                self.base_branch = self.repo.default_branch
-
-            contents = self.repo.get_contents("manifest.toml", ref=self.base_branch)
-            self.manifest_raw = contents.decoded_content.decode()
-            self.manifest_raw_sha = contents.sha
-            manifest = toml.loads(self.manifest_raw)
-
-        self.app_id = manifest["id"]
-        self.current_version = manifest["version"].split("~")[0]
-        self.sources = manifest.get("resources", {}).get("sources")
-
-        if not self.sources:
-            raise Exception("There's no resources.sources in manifest.toml ?")
-
-        self.main_upstream = manifest.get("upstream", {}).get("code")
-
-    def run(self):
-        todos = {}
-
-        for source, infos in self.sources.items():
-            if "autoupdate" not in infos:
-                continue
-
         strategy = infos.get("autoupdate", {}).get("strategy")
         if strategy not in STRATEGIES:
-                raise Exception(
+            raise ValueError(f"Unknown update strategy '{strategy}' for '{name}', expected one of {STRATEGIES}")
-                    f"Unknown strategy to autoupdate {source}, expected one of {STRATEGIES}, got {strategy}"
-                )
 
-            asset = infos.get("autoupdate", {}).get("asset", "tarball")
+        result = self.get_latest_version_and_asset(strategy, asset, infos)
+        if result is None:
+            return None
+        new_version, assets, more_info = result
 
-            print(f"\n  Checking {source} ...")
+        if name == "main":
-
-            if strategy.endswith("_release"):
-                (
-                    new_version,
-                    new_asset_urls,
-                    changelog_url,
-                ) = self.get_latest_version_and_asset(strategy, asset, infos, source)
-            else:
-                (new_version, new_asset_urls) = self.get_latest_version_and_asset(
-                    strategy, asset, infos, source
-                )
-
-            if source == "main":
             print(f"Current version in manifest: {self.current_version}")
             print(f"Newest  version on upstream: {new_version}")
 
@@ -188,248 +295,144 @@ class AppAutoUpdater:
             # which is ignored by this script
             # Though we wrap this in a try/except pass, because don't want to miserably crash
             # if the tag can't properly be converted to int tuple ...
-                try:
-                    if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(
-                        new_version
-                    ):
-                        print(
-                            "Up to date (current version appears more recent than newest version found)"
-                        )
-                        continue
-                except:
-                    pass
-
             if self.current_version == new_version:
                 print("Up to date")
-                    continue
+                return None
-
-            if (
-                isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)
-            ) or (
-                isinstance(new_asset_urls, str)
-                and not isinstance(infos.get("url"), str)
-            ):
-                raise Exception(
-                    f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}"
-                )
-
-            if isinstance(new_asset_urls, str) and infos["url"] == new_asset_urls:
-                print(f"URL for asset {source} is up to date")
-                continue
-            elif isinstance(new_asset_urls, dict) and new_asset_urls == {
-                k: infos[k]["url"] for k in new_asset_urls.keys()
-            }:
-                print(f"URLs for asset {source} are up to date")
-                continue
-            else:
-                print(f"Update needed for {source}")
-                todos[source] = {
-                    "new_asset_urls": new_asset_urls,
-                    "old_assets": infos,
-                }
-
-            if source == "main":
-                todos[source]["new_version"] = new_version
-
-        if dry_run or not todos:
-            return bool(todos)
-
-        if "main" in todos:
-            if strategy.endswith("_release"):
-                title = f"Upgrade to v{new_version}"
-                message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}"
-            else:
-                title = message = f"Upgrade to v{new_version}"
-            new_version = todos["main"]["new_version"]
-            new_branch = f"ci-auto-update-{new_version}"
-        else:
-            title = message = "Upgrade sources"
-            new_branch = "ci-auto-update-sources"
-
             try:
-            # Get the commit base for the new branch, and create it
+                if self.tag_to_int_tuple(self.current_version) > self.tag_to_int_tuple(new_version):
-            commit_sha = self.repo.get_branch(self.base_branch).commit.sha
+                    print("Up to date (current version appears more recent than newest version found)")
-            self.repo.create_git_ref(ref=f"refs/heads/{new_branch}", sha=commit_sha)
+                    return None
-        except:
+            except (AssertionError, ValueError):
-            print("... Branch already exists, skipping")
+                pass
-            return False
 
-        manifest_new = self.manifest_raw
+        if isinstance(assets, dict) and isinstance(infos.get("url"), str) or \
-        for source, infos in todos.items():
+           isinstance(assets, str) and not isinstance(infos.get("url"), str):
-            manifest_new = self.replace_version_and_asset_in_manifest(
+            raise RuntimeError(
-                manifest_new,
+                "It looks like there's an inconsistency between the old asset list and the new ones... "
-                infos.get("new_version"),
+                "One is arch-specific, the other is not... Did you forget to define arch-specific regexes? "
-                infos["new_asset_urls"],
+                f"New asset url is/are : {assets}"
-                infos["old_assets"],
-                is_main=source == "main",
             )
 
-        self.repo.update_file(
+        if isinstance(assets, str) and infos["url"] == assets:
-            "manifest.toml",
+            print(f"URL for asset {name} is up to date")
-            message=message,
+            return
-            content=manifest_new,
+        if isinstance(assets, dict) and assets == {k: infos[k]["url"] for k in assets.keys()}:
-            sha=self.manifest_raw_sha,
+            print(f"URLs for asset {name} are up to date")
-            branch=new_branch,
+            return
-            author=author,
+        print(f"Update needed for {name}")
-        )
+        return new_version, assets, more_info
 
-        # Wait a bit to preserve the API rate limit
+    @staticmethod
-        time.sleep(1.5)
+    def find_matching_asset(assets: dict[str, str], regex: str) -> tuple[str, str]:
+        matching_assets = {
+            name: url for name, url in assets.items() if re.match(regex, name)
+        }
+        if not matching_assets:
+            raise RuntimeError(f"No assets matching regex '{regex}' in {list(assets.keys())}")
+        if len(matching_assets) > 1:
+            raise RuntimeError(f"Too many assets matching regex '{regex}': {matching_assets}")
+        return next(iter(matching_assets.items()))
 
-        # Open the PR
+    def get_latest_version_and_asset(self, strategy: str, asset: str | dict, infos
-        pr = self.repo.create_pull(
+                                     ) -> tuple[str, str | dict[str, str], str] | None:
-            title=title, body=message, head=new_branch, base=self.base_branch
+        upstream = (infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/"))
-        )
+        _, remote_type, revision_type = strategy.split("_")
 
-        print("Created PR " + self.repo.full_name + " updated with PR #" + str(pr.id))
+        if remote_type == "github":
-
-        return bool(todos)
-
-    def get_latest_version_and_asset(self, strategy, asset, infos, source):
-        upstream = (
-            infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")
-        )
-
-        if "github" in strategy:
             assert (
                 upstream and upstream.startswith("https://github.com/")
             ), f"When using strategy {strategy}, having a defined upstream code repo on github.com is required"
-            api = GithubAPI(upstream, auth=auth)
+            api = GithubAPI(upstream, auth=get_github()[0])
-        elif "gitlab" in strategy:
+        if remote_type == "gitlab":
             api = GitlabAPI(upstream)
-        elif "gitea" in strategy or "forgejo" in strategy:
+        if remote_type in ["gitea", "forgejo"]:
             api = GiteaForgejoAPI(upstream)
 
-        if strategy.endswith("_release"):
+        if revision_type == "release":
-            releases = api.releases()
+            releases: dict[str, dict[str, Any]] = {
-            tags = [
+                release["tag_name"]: release
-                release["tag_name"]
+                for release in api.releases()
-                for release in releases
                 if not release["draft"] and not release["prerelease"]
-            ]
+            }
-            latest_version_orig, latest_version = filter_and_get_latest_tag(
+            latest_version_orig, latest_version = self.filter_and_get_latest_tag(list(releases.keys()), self.app_id)
-                tags, self.app_id
+            latest_release = releases[latest_version_orig]
-            )
-            latest_release = [
-                release
-                for release in releases
-                if release["tag_name"] == latest_version_orig
-            ][0]
             latest_assets = {
                 a["name"]: a["browser_download_url"]
                 for a in latest_release["assets"]
                 if not a["name"].endswith(".md5")
             }
-            if ("gitea" in strategy or "forgejo" in strategy) and latest_assets == "":
+            if remote_type in ["gitea", "forgejo"] and latest_assets == "":
                 # if empty (so only the base asset), take the tarball_url
                 latest_assets = latest_release["tarball_url"]
             # get the release changelog link
             latest_release_html_url = latest_release["html_url"]
             if asset == "tarball":
-                latest_tarball = (
+                latest_tarball = api.url_for_ref(latest_version_orig, RefType.tags)
-                    api.url_for_ref(latest_version_orig, RefType.tags)
-                )
                 return latest_version, latest_tarball, latest_release_html_url
             # FIXME
-            else:
             if isinstance(asset, str):
-                    matching_assets_urls = [
+                try:
-                        url
+                    _, url = self.find_matching_asset(latest_assets, asset)
-                        for name, url in latest_assets.items()
+                    return latest_version, url, latest_release_html_url
-                        if re.match(asset, name)
+                except RuntimeError as e:
-                    ]
+                    raise RuntimeError(f"{e}.\nFull release details on {latest_release_html_url}.") from e
-                    if not matching_assets_urls:
+
-                        raise Exception(
+            if isinstance(asset, dict):
-                            f"No assets matching regex '{asset}' for release {latest_version} among {list(latest_assets.keys())}. Full release details on {latest_release_html_url}"
+                new_assets = {}
-                        )
-                    elif len(matching_assets_urls) > 1:
-                        raise Exception(
-                            f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
-                        )
-                    return (
-                        latest_version,
-                        matching_assets_urls[0],
-                        latest_release_html_url,
-                    )
-                elif isinstance(asset, dict):
-                    matching_assets_dicts = {}
                 for asset_name, asset_regex in asset.items():
-                        matching_assets_urls = [
+                    try:
-                            url
+                        _, url = self.find_matching_asset(latest_assets, asset_regex)
-                            for name, url in latest_assets.items()
+                        new_assets[asset_name] = url
-                            if re.match(asset_regex, name)
+                    except RuntimeError as e:
-                        ]
+                        raise RuntimeError(f"{e}.\nFull release details on {latest_release_html_url}.") from e
-                        if not matching_assets_urls:
+                return latest_version, new_assets, latest_release_html_url
-                            raise Exception(
-                                f"No assets matching regex '{asset_regex}' for release {latest_version} among {list(latest_assets.keys())}. Full release details on {latest_release_html_url}"
-                            )
-                        elif len(matching_assets_urls) > 1:
-                            raise Exception(
-                                f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
-                            )
-                        matching_assets_dicts[asset_name] = matching_assets_urls[0]
-                    return (
-                        latest_version.strip("v"),
-                        matching_assets_dicts,
-                        latest_release_html_url,
-                    )
 
-        elif strategy.endswith("_tag"):
+            return None
+
+        if revision_type == "tag":
             if asset != "tarball":
-                raise Exception(
+                raise ValueError("For the latest tag strategies, only asset = 'tarball' is supported")
-                    "For the latest tag strategy, only asset = 'tarball' is supported"
+            tags = [t["name"] for t in api.tags()]
-                )
+            latest_version_orig, latest_version = self.filter_and_get_latest_tag(tags, self.app_id)
-            tags = api.tags()
-            latest_version_orig, latest_version = filter_and_get_latest_tag(
-                [t["name"] for t in tags], self.app_id
-            )
             latest_tarball = api.url_for_ref(latest_version_orig, RefType.tags)
-            return latest_version, latest_tarball
+            return latest_version, latest_tarball, ""
 
-        elif strategy.endswith("_commit"):
+        if revision_type == "commit":
             if asset != "tarball":
-                raise Exception(
+                raise ValueError("For the latest commit strategies, only asset = 'tarball' is supported")
-                    "For the latest release strategy, only asset = 'tarball' is supported"
-                )
             commits = api.commits()
             latest_commit = commits[0]
             latest_tarball = api.url_for_ref(latest_commit["sha"], RefType.commits)
             # Let's have the version as something like "2023.01.23"
-            latest_commit_date = datetime.strptime(
+            latest_commit_date = datetime.strptime(latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d")
-                latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d"
+            version_format = infos.get("autoupdate", {}).get("force_version", "%Y.%m.%d")
-            )
-            version_format = infos.get("autoupdate", {}).get(
-                "force_version", "%Y.%m.%d"
-            )
             latest_version = latest_commit_date.strftime(version_format)
+            return latest_version, latest_tarball, ""
 
-            return latest_version, latest_tarball
+    def replace_version_and_asset_in_manifest(self, content: str, new_version: str, new_assets_urls: str | dict,
-
+                                              current_assets: dict, is_main: bool):
-    def replace_version_and_asset_in_manifest(
+        replacements = []
-        self, content, new_version, new_assets_urls, current_assets, is_main
-    ):
         if isinstance(new_assets_urls, str):
-            sha256 = sha256_of_remote_file(new_assets_urls)
+            replacements = [
-        elif isinstance(new_assets_urls, dict):
+                (current_assets["url"], new_assets_urls),
-            sha256 = {
+                (current_assets["sha256"], self.sha256_of_remote_file(new_assets_urls)),
-                url: sha256_of_remote_file(url) for url in new_assets_urls.values()
+            ]
-            }
+        if isinstance(new_assets_urls, dict):
+            replacements = [
+                repl
+                for key, url in new_assets_urls.items() for repl in (
+                    (current_assets[key]["url"], url),
+                    (current_assets[key]["sha256"], self.sha256_of_remote_file(url))
+                )
+            ]
 
         if is_main:
-
+            def repl(m: re.Match) -> str:
-            def repl(m):
                 return m.group(1) + new_version + '~ynh1"'
+            content = re.sub(r"(\s*version\s*=\s*[\"\'])([\d\.]+)(\~ynh\d+[\"\'])", repl, content)
 
-            content = re.sub(
+        for old, new in replacements:
-                r"(\s*version\s*=\s*[\"\'])([\d\.]+)(\~ynh\d+[\"\'])", repl, content
+            content = content.replace(old, new)
-            )
-        if isinstance(new_assets_urls, str):
-            content = content.replace(current_assets["url"], new_assets_urls)
-            content = content.replace(current_assets["sha256"], sha256)
-        elif isinstance(new_assets_urls, dict):
-            for key, url in new_assets_urls.items():
-                content = content.replace(current_assets[key]["url"], url)
-                content = content.replace(current_assets[key]["sha256"], sha256[url])
 
         return content
 
@@ -438,75 +441,116 @@ def paste_on_haste(data):
     # NB: we hardcode this here and can't use the yunopaste command
     # because this script runs on the same machine than haste is hosted on...
     # and doesn't have the proper front-end LE cert in this context
-    SERVER_URL = "http://paste.yunohost.org"
+    SERVER_HOST = "http://paste.yunohost.org"
     TIMEOUT = 3
     try:
-        url = SERVER_URL + "/documents"
+        url = f"{SERVER_HOST}/documents"
         response = requests.post(url, data=data.encode("utf-8"), timeout=TIMEOUT)
         response.raise_for_status()
         dockey = response.json()["key"]
-        return SERVER_URL + "/raw/" + dockey
+        return f"{SERVER_HOST}/raw/{dockey}"
     except requests.exceptions.RequestException as e:
-        print("\033[31mError: {}\033[0m".format(e))
+        logging.error("\033[31mError: {}\033[0m".format(e))
-        sys.exit(1)
+        raise
+
+
+class StdoutSwitch:
+
+    class DummyFile:
+        def __init__(self):
+            self.result = ""
+
+        def write(self, x):
+            self.result += x
+
+    def __init__(self) -> None:
+        self.save_stdout = sys.stdout
+        sys.stdout = self.DummyFile()
+
+    def reset(self) -> str:
+        result = ""
+        if isinstance(sys.stdout, self.DummyFile):
+            result = sys.stdout.result
+            sys.stdout = self.save_stdout
+        return result
+
+    def __exit__(self) -> None:
+        sys.stdout = self.save_stdout
+
+
+def run_autoupdate_for_multiprocessing(data) -> tuple[bool, str, Any] | None:
+    app, edit, commit, pr = data
+    stdoutswitch = StdoutSwitch()
+    try:
+        result = AppAutoUpdater(app).run(edit=edit, commit=commit, pr=pr)
+        if result is not False:
+            return True, app, result
+    except Exception:
+        result = stdoutswitch.reset()
+        import traceback
+        t = traceback.format_exc()
+        return False, app, f"{result}\n{t}"
 
 
 def main() -> None:
     parser = argparse.ArgumentParser()
-    parser.add_argument("app_dir", nargs="?", type=Path)
+    parser.add_argument("apps", nargs="*", type=Path,
-    parser.add_argument("--commit-and-create-PR", action="store_true")
+                        help="If not passed, the script will run on the catalog. Github keys required.")
+    parser.add_argument("--edit", action=argparse.BooleanOptionalAction, help="Edit the local files", default=True)
+    parser.add_argument("--commit", action=argparse.BooleanOptionalAction, help="Create a commit with the changes")
+    parser.add_argument("--pr", action=argparse.BooleanOptionalAction, help="Create a pull request with the changes")
+    parser.add_argument("--paste", action="store_true")
+    parser.add_argument("-j", "--processes", type=int, default=multiprocessing.cpu_count())
     args = parser.parse_args()
 
-    global dry_run, auth, github, author
+    if args.commit and not args.edit:
-    dry_run = args.commit_and_create_PR
+        parser.error("--commit requires --edit")
+    if args.pr and not args.commit:
+        parser.error("--pr requires --commit")
 
-    if args.app_dir:
+    # Handle apps or no apps
-        AppAutoUpdater(str(args.app_dir), app_id_is_local_app_dir=True).run()
+    apps = list(args.apps) if args.apps else apps_to_run_auto_update_for()
+    apps_failed = {}
+    apps_updated = {}
+
+    with multiprocessing.Pool(processes=args.processes) as pool:
+        tasks = pool.imap(run_autoupdate_for_multiprocessing,
+                          ((app, args.edit, args.commit, args.pr) for app in apps))
+        for result in tqdm.tqdm(tasks, total=len(apps), ascii=" ·#"):
+            if result is None:
+                continue
+            is_ok, app, info = result
+            if is_ok:
+                apps_updated[app] = info
             else:
-        GITHUB_LOGIN = (REPO_APPS_ROOT / ".github_login").open("r", encoding="utf-8").read().strip()
+                apps_failed[app] = info
-        GITHUB_TOKEN = (REPO_APPS_ROOT / ".github_token").open("r", encoding="utf-8").read().strip()
+            pass
-        GITHUB_EMAIL = (REPO_APPS_ROOT / ".github_email").open("r", encoding="utf-8").read().strip()
 
-        from github import Github, InputGitAuthor
+    result_message = ""
-
+    if apps_updated:
-        auth = (GITHUB_LOGIN, GITHUB_TOKEN)
+        result_message += f"\n{'=' * 80}\nApps updated:"
-        github = Github(GITHUB_TOKEN)
+    for app, info in apps_updated.items():
-        author = InputGitAuthor(GITHUB_LOGIN, GITHUB_EMAIL)
+        result_message += f"\n- {app}"
-
+        if isinstance(info, tuple):
-        apps_failed = []
+            print(info)
-        apps_failed_details = {}
+            result_message += f" ({info[0]} -> {info[1]})"
-        apps_updated = []
+            if info[2] is not None:
-
+                result_message += f" see {info[2]}"
-        with logging_redirect_tqdm():
-            for app in tqdm.tqdm(apps_to_run_auto_update_for(), ascii=" ·#"):
-                try:
-                    updated = AppAutoUpdater(app).run()
-                except Exception as e:
-                    apps_failed.append(app)
-                    import traceback
-
-                    t = traceback.format_exc()
-                    apps_failed_details[app] = t
-                    print(t)
-                else:
-                    if updated:
-                        apps_updated.append(app)
 
     if apps_failed:
-            print(f"Apps failed: {', '.join(apps_failed)}")
+        result_message += f"\n{'=' * 80}\nApps failed:"
-            if os.path.exists("/usr/bin/sendxmpppy"):
+    for app, info in apps_failed.items():
-                paste = "\n=========\n".join(
+        result_message += f"\n{'='*40}\n{app}\n{'-'*40}\n{info}\n\n"
-                    [
+
-                        app + "\n-------\n" + trace + "\n\n"
+    if apps_failed and args.paste:
-                        for app, trace in apps_failed_details.items()
+        paste_url = paste_on_haste(result_message)
-                    ]
+        logging.error(textwrap.dedent(f"""
-                )
+        Failed to run the source auto-update for: {', '.join(apps_failed.keys())}
-                paste_url = paste_on_haste(paste)
+        Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening!
-                os.system(
+        See the debug log here: {paste_url}"
-                    f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'"
+        """))
-                )
+
-        if apps_updated:
+    print(result_message)
-            print(f"Apps updated: {', '.join(apps_updated)}")
 
 
 if __name__ == "__main__":

autoupdate_app_sources/rest_api.py

@@ -2,7 +2,7 @@
 
 import re
 from enum import Enum
-from typing import List
+from typing import Any
 
 import requests
 
@@ -13,7 +13,7 @@ class RefType(Enum):
 
 
 class GithubAPI:
-    def __init__(self, upstream: str, auth: tuple[str, str] = None):
+    def __init__(self, upstream: str, auth: tuple[str, str] | None = None):
         self.upstream = upstream
         self.upstream_repo = upstream.replace("https://github.com/", "")\
             .strip("/")
@@ -22,13 +22,13 @@ class GithubAPI:
             ), f"'{upstream}' doesn't seem to be a github repository ?"
         self.auth = auth
 
-    def internal_api(self, uri: str):
+    def internal_api(self, uri: str) -> Any:
         url = f"https://api.github.com/{uri}"
         r = requests.get(url, auth=self.auth)
         assert r.status_code == 200, r
         return r.json()
 
-    def tags(self) -> List[str]:
+    def tags(self) -> list[dict[str, str]]:
         """Get a list of tags for project."""
         return self.internal_api(f"repos/{self.upstream_repo}/tags")
 
@@ -53,25 +53,28 @@ class GithubAPI:
 class GitlabAPI:
     def __init__(self, upstream: str):
         split = re.search("(?P<host>https?://.+)/(?P<group>[^/]+)/(?P<project>[^/]+)/?$", upstream)
+        assert split is not None
         self.upstream = split.group("host")
         self.upstream_repo = f"{split.group('group')}/{split.group('project')}"
         self.project_id = self.find_project_id(self.upstream_repo)
 
     def find_project_id(self, project: str) -> int:
         project = self.internal_api(f"projects/{project.replace('/', '%2F')}")
-        return project["id"]
+        assert isinstance(project, dict)
+        project_id = project.get("id", None)
+        return project_id
 
-    def internal_api(self, uri: str):
+    def internal_api(self, uri: str) -> Any:
         url = f"{self.upstream}/api/v4/{uri}"
         r = requests.get(url)
         assert r.status_code == 200, r
         return r.json()
 
-    def tags(self) -> List[str]:
+    def tags(self) -> list[dict[str, str]]:
         """Get a list of tags for project."""
         return self.internal_api(f"projects/{self.project_id}/repository/tags")
 
-    def commits(self) -> List[str]:
+    def commits(self) -> list[dict[str, Any]]:
         """Get a list of commits for project."""
         return [
             {
@@ -85,7 +88,7 @@ class GitlabAPI:
             for commit in self.internal_api(f"projects/{self.project_id}/repository/commits")
         ]
 
-    def releases(self) -> List[str]:
+    def releases(self) -> list[dict[str, Any]]:
         """Get a list of releases for project."""
         releases = self.internal_api(f"projects/{self.project_id}/releases")
         retval = []