From 01330c5d7c128a2b5e2cb867160059f0365c47fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Tue, 21 May 2024 10:59:49 +0200 Subject: [PATCH] app_caches.py: Add --ssh and --all-branches Those arguments allow to customize how repositories are cloned and what is fetched. --- app_caches.py | 57 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/app_caches.py b/app_caches.py index 069f5f2..6cb79dd 100755 --- a/app_caches.py +++ b/app_caches.py @@ -3,6 +3,7 @@ import argparse import shutil import logging +from itertools import repeat from multiprocessing import Pool from pathlib import Path from typing import Any @@ -24,7 +25,7 @@ def app_cache_folder(app: str) -> Path: return APPS_CACHE_DIR / app -def app_cache_clone(app: str, infos: dict[str, str]) -> None: +def app_cache_clone(app: str, infos: dict[str, str], all_branches: bool = False) -> None: logging.info("Cloning %s...", app) git_depths = { "notworking": 5, @@ -37,18 +38,26 @@ def app_cache_clone(app: str, infos: dict[str, str]) -> None: infos["url"], to_path=app_cache_folder(app), depth=git_depths.get(infos["state"], git_depths["default"]), - single_branch=True, + single_branch=not all_branches, branch=infos.get("branch", "master"), ) -def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None: +def app_cache_clone_or_update( + app: str, + infos: dict[str, str], + ssh_clone: bool = False, + fetch_all_branches: bool = False) -> None: app_path = app_cache_folder(app) + # Patch url for ssh clone + if ssh_clone: + infos["url"] = infos["url"].replace("https://github.com/", "git@github.com:") + # Don't refresh if already refreshed during last hour age = git_repo_age(app_path) if age is False: - app_cache_clone(app, infos) + app_cache_clone(app, infos, fetch_all_branches) return # if age < 3600: @@ -60,29 +69,38 @@ def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None: repo.remote("origin").set_url(infos["url"]) branch = infos.get("branch", "master") - if repo.active_branch != branch: - all_branches = [str(b) for b in repo.branches] - if branch in all_branches: - repo.git.checkout(branch, "--force") - else: - repo.git.remote("set-branches", "--add", "origin", branch) - repo.remote("origin").fetch(f"{branch}:{branch}") + if fetch_all_branches: + repo.git.remote("set-branches", "origin", "*") + repo.remote("origin").fetch() + else: + if repo.active_branch != branch: + all_branches = [str(b) for b in repo.branches] + if branch in all_branches: + repo.git.checkout(branch, "--force") + else: + repo.git.remote("set-branches", "--add", "origin", branch) + repo.remote("origin").fetch(f"{branch}:{branch}") - repo.remote("origin").fetch(refspec=branch, force=True) - repo.git.reset("--hard", f"origin/{branch}") + repo.remote("origin").fetch(refspec=branch, force=True) + repo.git.reset("--hard", f"origin/{branch}") def __app_cache_clone_or_update_mapped(data): - name, info = data + name, info, ssh_clone, all_branches = data try: - app_cache_clone_or_update(name, info) + app_cache_clone_or_update(name, info, ssh_clone, all_branches) except Exception as err: logging.error("[App caches] Error while updating %s: %s", name, err) -def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None: +def apps_cache_update_all( + apps: dict[str, dict[str, Any]], + parallel: int = 8, + ssh_clone: bool = False, + all_branches: bool = False) -> None: with Pool(processes=parallel) as pool: - tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items()) + tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, + zip(apps.keys(), apps.values(), repeat(ssh_clone), repeat(all_branches))) for _ in tqdm.tqdm(tasks, total=len(apps.keys()), ascii=" ยท#"): pass @@ -101,6 +119,8 @@ def __run_for_catalog(): parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("-j", "--processes", type=int, default=8) + parser.add_argument("-s", "--ssh", action=argparse.BooleanOptionalAction, default=False, help="Use ssh clones instead of https") + parser.add_argument("-b", "--all-branches", action=argparse.BooleanOptionalAction, default=False, help="Download all branches from repo") parser.add_argument( "-c", "--cleanup", @@ -114,7 +134,8 @@ def __run_for_catalog(): if args.cleanup: apps_cache_cleanup(get_catalog()) - apps_cache_update_all(get_catalog(), parallel=args.processes) + apps_cache_update_all(get_catalog(), parallel=args.processes, + ssh_clone=args.ssh, all_branches=args.all_branches) if __name__ == "__main__":