Add apps_caches.py script that handles the download of the apps cache for the other scripts

app_caches.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+import argparse
+import shutil
+import logging
+from multiprocessing import Pool
+from pathlib import Path
+from typing import Any
+
+import tqdm
+
+from appslib.utils import (REPO_APPS_ROOT,  # pylint: disable=import-error
+                           get_catalog, git_repo_age, progressbar)
+from git import Repo
+
+
+def app_cache_folder(app: str) -> Path:
+    return REPO_APPS_ROOT / ".apps_cache" / app
+
+
+def app_cache_clone(app: str, infos: dict[str, str]) -> None:
+    logging.info("Cloning %s...", app)
+    git_depths = {
+        "notworking": 5,
+        "inprogress": 20,
+        "default": 40,
+    }
+    if app_cache_folder(app).exists():
+        shutil.rmtree(app_cache_folder(app))
+    Repo.clone_from(
+        infos["url"],
+        to_path=app_cache_folder(app),
+        depth=git_depths.get(infos["state"], git_depths["default"]),
+        single_branch=True, branch=infos.get("branch", "master"),
+    )
+
+
+def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None:
+    app_path = app_cache_folder(app)
+
+    # Don't refresh if already refreshed during last hour
+    age = git_repo_age(app_path)
+    if age is False:
+        app_cache_clone(app, infos)
+        return
+
+    # if age < 3600:
+    #     logging.info(f"Skipping {app}, it's been updated recently.")
+    #     return
+
+    logging.info("Updating %s...", app)
+    repo = Repo(app_path)
+    repo.remote("origin").set_url(infos["url"])
+
+    branch = infos.get("branch", "master")
+    if repo.active_branch != branch:
+        all_branches = [str(b) for b in repo.branches]
+        if branch in all_branches:
+            repo.git.checkout(branch, "--force")
+        else:
+            repo.git.remote("set-branches", "--add", "origin", branch)
+            repo.remote("origin").fetch(f"{branch}:{branch}")
+
+    repo.remote("origin").fetch(refspec=branch, force=True)
+    repo.git.reset("--hard", f"origin/{branch}")
+
+
+def __app_cache_clone_or_update_mapped(data):
+    name, info = data
+    try:
+        app_cache_clone_or_update(name, info)
+    except Exception as err:
+        logging.error("Error while updating %s: %s", name, err)
+
+
+def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None:
+    with Pool(processes=parallel) as pool:
+        tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items())
+        for _ in tqdm.tqdm(tasks, total=len(apps.keys())):
+            pass
+
+
+def __run_for_catalog():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--verbose", action="store_true")
+    parser.add_argument("-j", "--processes", type=int, default=8)
+    args = parser.parse_args()
+    if args.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+
+    apps_cache_update_all(get_catalog(), parallel=args.processes)
+
+
+if __name__ == "__main__":
+    __run_for_catalog()