hosting/ynh-apps_tools
1
0
Fork 0
Code Activity

app_caches.py: Add --ssh and --all-branches

Browse source 
Those arguments allow to customize how repositories are cloned and what is fetched.
This commit is contained in:
Félix Piédallu 2024-05-21 10:59:49 +02:00 committed by Salamandar
parent 2df6241485
commit 01330c5d7c
1 changed files with 39 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
app_caches.py
View file

@ -3,6 +3,7 @@
import argparse
import shutil
import logging
from itertools import repeat
from multiprocessing import Pool
from pathlib import Path
from typing import Any
@ -24,7 +25,7 @@ def app_cache_folder(app: str) -> Path:
return APPS_CACHE_DIR / app
def app_cache_clone(app: str, infos: dict[str, str]) -> None:
def app_cache_clone(app: str, infos: dict[str, str], all_branches: bool = False) -> None:
logging.info("Cloning %s...", app)
git_depths = {
"notworking": 5,
@ -37,18 +38,26 @@ def app_cache_clone(app: str, infos: dict[str, str]) -> None:
infos["url"],
to_path=app_cache_folder(app),
depth=git_depths.get(infos["state"], git_depths["default"]),
single_branch=True,
single_branch=not all_branches,
branch=infos.get("branch", "master"),
)
def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None:
def app_cache_clone_or_update(
app: str,
infos: dict[str, str],
ssh_clone: bool = False,
fetch_all_branches: bool = False) -> None:
app_path = app_cache_folder(app)
# Patch url for ssh clone
if ssh_clone:
infos["url"] = infos["url"].replace("https://github.com/", "git@github.com:")
# Don't refresh if already refreshed during last hour
age = git_repo_age(app_path)
if age is False:
app_cache_clone(app, infos)
app_cache_clone(app, infos, fetch_all_branches)
return
# if age < 3600:
@ -60,6 +69,10 @@ def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None:
repo.remote("origin").set_url(infos["url"])
branch = infos.get("branch", "master")
if fetch_all_branches:
repo.git.remote("set-branches", "origin", "*")
repo.remote("origin").fetch()
else:
if repo.active_branch != branch:
all_branches = [str(b) for b in repo.branches]
if branch in all_branches:
@ -73,16 +86,21 @@ def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None:
def __app_cache_clone_or_update_mapped(data):
name, info = data
name, info, ssh_clone, all_branches = data
try:
app_cache_clone_or_update(name, info)
app_cache_clone_or_update(name, info, ssh_clone, all_branches)
except Exception as err:
logging.error("[App caches] Error while updating %s: %s", name, err)
def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None:
def apps_cache_update_all(
apps: dict[str, dict[str, Any]],
parallel: int = 8,
ssh_clone: bool = False,
all_branches: bool = False) -> None:
with Pool(processes=parallel) as pool:
tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items())
tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped,
zip(apps.keys(), apps.values(), repeat(ssh_clone), repeat(all_branches)))
for _ in tqdm.tqdm(tasks, total=len(apps.keys()), ascii=" ·#"):
pass
@ -101,6 +119,8 @@ def __run_for_catalog():
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", action="store_true")
parser.add_argument("-j", "--processes", type=int, default=8)
parser.add_argument("-s", "--ssh", action=argparse.BooleanOptionalAction, default=False, help="Use ssh clones instead of https")
parser.add_argument("-b", "--all-branches", action=argparse.BooleanOptionalAction, default=False, help="Download all branches from repo")
parser.add_argument(
"-c",
"--cleanup",
@ -114,7 +134,8 @@ def __run_for_catalog():
if args.cleanup:
apps_cache_cleanup(get_catalog())
apps_cache_update_all(get_catalog(), parallel=args.processes)
apps_cache_update_all(get_catalog(), parallel=args.processes,
ssh_clone=args.ssh, all_branches=args.all_branches)
if __name__ == "__main__":