app_caches.py: Add --ssh and --all-branches

Those arguments allow to customize how repositories are cloned and what is fetched.

app_caches.py

@@ -3,6 +3,7 @@
 import argparse
 import shutil
 import logging
+from itertools import repeat
 from multiprocessing import Pool
 from pathlib import Path
 from typing import Any
@@ -24,7 +25,7 @@ def app_cache_folder(app: str) -> Path:
     return APPS_CACHE_DIR / app
 
 
-def app_cache_clone(app: str, infos: dict[str, str]) -> None:
+def app_cache_clone(app: str, infos: dict[str, str], all_branches: bool = False) -> None:
     logging.info("Cloning %s...", app)
     git_depths = {
         "notworking": 5,
@@ -37,18 +38,26 @@ def app_cache_clone(app: str, infos: dict[str, str]) -> None:
         infos["url"],
         to_path=app_cache_folder(app),
         depth=git_depths.get(infos["state"], git_depths["default"]),
-        single_branch=True,
+        single_branch=not all_branches,
         branch=infos.get("branch", "master"),
     )
 
 
-def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None:
+def app_cache_clone_or_update(
+        app: str,
+        infos: dict[str, str],
+        ssh_clone: bool = False,
+        fetch_all_branches: bool = False) -> None:
     app_path = app_cache_folder(app)
 
+    # Patch url for ssh clone
+    if ssh_clone:
+        infos["url"] = infos["url"].replace("https://github.com/", "git@github.com:")
+
     # Don't refresh if already refreshed during last hour
     age = git_repo_age(app_path)
     if age is False:
-        app_cache_clone(app, infos)
+        app_cache_clone(app, infos, fetch_all_branches)
         return
 
     # if age < 3600:
@@ -60,29 +69,38 @@ def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None:
     repo.remote("origin").set_url(infos["url"])
 
     branch = infos.get("branch", "master")
-    if repo.active_branch != branch:
+    if fetch_all_branches:
-        all_branches = [str(b) for b in repo.branches]
+        repo.git.remote("set-branches", "origin", "*")
-        if branch in all_branches:
+        repo.remote("origin").fetch()
-            repo.git.checkout(branch, "--force")
+    else:
-        else:
+        if repo.active_branch != branch:
-            repo.git.remote("set-branches", "--add", "origin", branch)
+            all_branches = [str(b) for b in repo.branches]
-            repo.remote("origin").fetch(f"{branch}:{branch}")
+            if branch in all_branches:
+                repo.git.checkout(branch, "--force")
+            else:
+                repo.git.remote("set-branches", "--add", "origin", branch)
+                repo.remote("origin").fetch(f"{branch}:{branch}")
 
-    repo.remote("origin").fetch(refspec=branch, force=True)
+        repo.remote("origin").fetch(refspec=branch, force=True)
-    repo.git.reset("--hard", f"origin/{branch}")
+        repo.git.reset("--hard", f"origin/{branch}")
 
 
 def __app_cache_clone_or_update_mapped(data):
-    name, info = data
+    name, info, ssh_clone, all_branches = data
     try:
-        app_cache_clone_or_update(name, info)
+        app_cache_clone_or_update(name, info, ssh_clone, all_branches)
     except Exception as err:
         logging.error("[App caches] Error while updating %s: %s", name, err)
 
 
-def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None:
+def apps_cache_update_all(
+        apps: dict[str, dict[str, Any]],
+        parallel: int = 8,
+        ssh_clone: bool = False,
+        all_branches: bool = False) -> None:
     with Pool(processes=parallel) as pool:
-        tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items())
+        tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped,
+                                    zip(apps.keys(), apps.values(), repeat(ssh_clone), repeat(all_branches)))
         for _ in tqdm.tqdm(tasks, total=len(apps.keys()), ascii=" ·#"):
             pass
 
@@ -101,6 +119,8 @@ def __run_for_catalog():
     parser = argparse.ArgumentParser()
     parser.add_argument("-v", "--verbose", action="store_true")
     parser.add_argument("-j", "--processes", type=int, default=8)
+    parser.add_argument("-s", "--ssh", action=argparse.BooleanOptionalAction, default=False, help="Use ssh clones instead of https")
+    parser.add_argument("-b", "--all-branches", action=argparse.BooleanOptionalAction, default=False, help="Download all branches from repo")
     parser.add_argument(
         "-c",
         "--cleanup",
@@ -114,7 +134,8 @@ def __run_for_catalog():
 
     if args.cleanup:
         apps_cache_cleanup(get_catalog())
-    apps_cache_update_all(get_catalog(), parallel=args.processes)
+    apps_cache_update_all(get_catalog(), parallel=args.processes,
+                          ssh_clone=args.ssh, all_branches=args.all_branches)
 
 
 if __name__ == "__main__":