#!/usr/bin/env python3 import argparse import shutil import logging from multiprocessing import Pool from pathlib import Path from typing import Any import tqdm from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error get_catalog, git_repo_age) from git import Repo def app_cache_folder(app: str) -> Path: return REPO_APPS_ROOT / ".apps_cache" / app def app_cache_clone(app: str, infos: dict[str, str]) -> None: logging.info("Cloning %s...", app) git_depths = { "notworking": 5, "inprogress": 20, "default": 40, } if app_cache_folder(app).exists(): shutil.rmtree(app_cache_folder(app)) Repo.clone_from( infos["url"], to_path=app_cache_folder(app), depth=git_depths.get(infos["state"], git_depths["default"]), single_branch=True, branch=infos.get("branch", "master"), ) def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None: app_path = app_cache_folder(app) # Don't refresh if already refreshed during last hour age = git_repo_age(app_path) if age is False: app_cache_clone(app, infos) return # if age < 3600: # logging.info(f"Skipping {app}, it's been updated recently.") # return logging.info("Updating %s...", app) repo = Repo(app_path) repo.remote("origin").set_url(infos["url"]) branch = infos.get("branch", "master") if repo.active_branch != branch: all_branches = [str(b) for b in repo.branches] if branch in all_branches: repo.git.checkout(branch, "--force") else: repo.git.remote("set-branches", "--add", "origin", branch) repo.remote("origin").fetch(f"{branch}:{branch}") repo.remote("origin").fetch(refspec=branch, force=True) repo.git.reset("--hard", f"origin/{branch}") def __app_cache_clone_or_update_mapped(data): name, info = data try: app_cache_clone_or_update(name, info) except Exception as err: logging.error("Error while updating %s: %s", name, err) def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None: with Pool(processes=parallel) as pool: tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items()) for _ in tqdm.tqdm(tasks, total=len(apps.keys()), ascii=" ยท#"): pass def __run_for_catalog(): parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("-j", "--processes", type=int, default=8) args = parser.parse_args() if args.verbose: logging.getLogger().setLevel(logging.INFO) apps_cache_update_all(get_catalog(), parallel=args.processes) if __name__ == "__main__": __run_for_catalog()