From 0af4cc21481c0442fbf175469dcd15c67d7a1e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 10 Feb 2024 13:46:11 +0100 Subject: [PATCH 1/5] Use main() function --- autoupdate_app_sources/autoupdate_app_sources.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index e10db4d..40ebc13 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -484,7 +484,7 @@ def paste_on_haste(data): sys.exit(1) -if __name__ == "__main__": +def main() -> None: args = [arg for arg in sys.argv[1:] if arg != "--commit-and-create-PR"] if len(args): @@ -522,3 +522,7 @@ if __name__ == "__main__": ) if apps_updated: print(f"Apps updated: {', '.join(apps_updated)}") + + +if __name__ == "__main__": + main() From b5af26a872e98e03b9d843ec7d172f550592b654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 10 Feb 2024 13:54:36 +0100 Subject: [PATCH 2/5] Use argparse, move github auth logic temporarily in main() --- .../autoupdate_app_sources.py | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 40ebc13..79ff60d 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import glob import hashlib import os @@ -27,31 +28,12 @@ STRATEGIES = [ "latest_forgejo_commit" ] -if "--commit-and-create-PR" not in sys.argv: - dry_run = True -else: - dry_run = False +dry_run = True -args = [arg for arg in sys.argv[1:] if arg != "--commit-and-create-PR"] - -if len(args): - auth = None -else: - GITHUB_LOGIN = ( - open(os.path.dirname(__file__) + "/../../.github_login").read().strip() - ) - GITHUB_TOKEN = ( - open(os.path.dirname(__file__) + "/../../.github_token").read().strip() - ) - GITHUB_EMAIL = ( - open(os.path.dirname(__file__) + "/../../.github_email").read().strip() - ) - - from github import Github, InputGitAuthor - - auth = (GITHUB_LOGIN, GITHUB_TOKEN) - github = Github(GITHUB_TOKEN) - author = InputGitAuthor(GITHUB_LOGIN, GITHUB_EMAIL) +# For github authentication +auth = None +github = None +author = None def apps_to_run_auto_update_for(): @@ -485,11 +467,33 @@ def paste_on_haste(data): def main() -> None: - args = [arg for arg in sys.argv[1:] if arg != "--commit-and-create-PR"] + parser = argparse.ArgumentParser() + parser.add_argument("app_dir", nargs="?", type=str) + parser.add_argument("--commit-and-create-PR", action="store_true") + args = parser.parse_args() - if len(args): - AppAutoUpdater(args[0], app_id_is_local_app_dir=True).run() + global dry_run, auth, github, author + dry_run = args.commit_and_create_PR + + if args.app_dir: + AppAutoUpdater(args.app_dir, app_id_is_local_app_dir=True).run() else: + GITHUB_LOGIN = ( + open(os.path.dirname(__file__) + "/../../.github_login").read().strip() + ) + GITHUB_TOKEN = ( + open(os.path.dirname(__file__) + "/../../.github_token").read().strip() + ) + GITHUB_EMAIL = ( + open(os.path.dirname(__file__) + "/../../.github_email").read().strip() + ) + + from github import Github, InputGitAuthor + + auth = (GITHUB_LOGIN, GITHUB_TOKEN) + github = Github(GITHUB_TOKEN) + author = InputGitAuthor(GITHUB_LOGIN, GITHUB_EMAIL) + apps_failed = [] apps_failed_details = {} apps_updated = [] From 9ed2f8d017a880e0f6916713b86a7c36a0b31759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 10 Feb 2024 15:02:05 +0100 Subject: [PATCH 3/5] Use pathlib.Path --- .../autoupdate_app_sources.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 79ff60d..0d41404 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -4,15 +4,21 @@ import argparse import glob import hashlib import os +import subprocess import re import sys import time +from pathlib import Path from datetime import datetime import requests import toml from rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType +REPO_APPS_ROOT = Path(subprocess.check_output([ + "git", "-C", Path(__file__).parent, "rev-parse", "--show-toplevel"]).decode("utf-8").strip()) + + STRATEGIES = [ "latest_github_release", "latest_github_tag", @@ -468,7 +474,7 @@ def paste_on_haste(data): def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("app_dir", nargs="?", type=str) + parser.add_argument("app_dir", nargs="?", type=Path) parser.add_argument("--commit-and-create-PR", action="store_true") args = parser.parse_args() @@ -476,17 +482,11 @@ def main() -> None: dry_run = args.commit_and_create_PR if args.app_dir: - AppAutoUpdater(args.app_dir, app_id_is_local_app_dir=True).run() + AppAutoUpdater(str(args.app_dir), app_id_is_local_app_dir=True).run() else: - GITHUB_LOGIN = ( - open(os.path.dirname(__file__) + "/../../.github_login").read().strip() - ) - GITHUB_TOKEN = ( - open(os.path.dirname(__file__) + "/../../.github_token").read().strip() - ) - GITHUB_EMAIL = ( - open(os.path.dirname(__file__) + "/../../.github_email").read().strip() - ) + GITHUB_LOGIN = (REPO_APPS_ROOT / ".github_login").open("r", encoding="utf-8").read().strip() + GITHUB_TOKEN = (REPO_APPS_ROOT / ".github_token").open("r", encoding="utf-8").read().strip() + GITHUB_EMAIL = (REPO_APPS_ROOT / ".github_email").open("r", encoding="utf-8").read().strip() from github import Github, InputGitAuthor From 350d08312267770e4f13dbf732bcb019214ba3bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 10 Feb 2024 15:13:45 +0100 Subject: [PATCH 4/5] Add sys.path.insert to import appslib --- autoupdate_app_sources/autoupdate_app_sources.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 0d41404..ad7f2d1 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -4,7 +4,6 @@ import argparse import glob import hashlib import os -import subprocess import re import sys import time @@ -13,10 +12,12 @@ from datetime import datetime import requests import toml -from rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType -REPO_APPS_ROOT = Path(subprocess.check_output([ - "git", "-C", Path(__file__).parent, "rev-parse", "--show-toplevel"]).decode("utf-8").strip()) +# add apps/tools to sys.path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType +from appslib.utils import REPO_APPS_ROOT, get_catalog # pylint: disable=import-error STRATEGIES = [ @@ -43,11 +44,9 @@ author = None def apps_to_run_auto_update_for(): - catalog = toml.load(open(os.path.dirname(__file__) + "/../../apps.toml")) - apps_flagged_as_working_and_on_yunohost_apps_org = [ app - for app, infos in catalog.items() + for app, infos in get_catalog().items() if infos["state"] == "working" and "/github.com/yunohost-apps" in infos["url"].lower() ] From 20081a9620677e34d23a852df1d205eadb923d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sun, 11 Feb 2024 20:04:34 +0100 Subject: [PATCH 5/5] Use tqdm instead of home-made progressbar --- .../autoupdate_app_sources.py | 50 ++++++------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index ad7f2d1..f541eb9 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -12,6 +12,8 @@ from datetime import datetime import requests import toml +import tqdm +from tqdm.contrib.logging import logging_redirect_tqdm # add apps/tools to sys.path sys.path.insert(0, str(Path(__file__).parent.parent)) @@ -432,28 +434,6 @@ class AppAutoUpdater: return content -# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 -def progressbar(it, prefix="", size=60, file=sys.stdout): - it = list(it) - count = len(it) - - def show(j, name=""): - name += " " - x = int(size * j / count) - file.write( - "\n%s[%s%s] %i/%i %s\n" - % (prefix, "#" * x, "." * (size - x), j, count, name) - ) - file.flush() - - show(0) - for i, item in enumerate(it): - show(i + 1, item) - yield item - file.write("\n") - file.flush() - - def paste_on_haste(data): # NB: we hardcode this here and can't use the yunopaste command # because this script runs on the same machine than haste is hosted on... @@ -496,19 +476,21 @@ def main() -> None: apps_failed = [] apps_failed_details = {} apps_updated = [] - for app in progressbar(apps_to_run_auto_update_for(), "Checking: ", 40): - try: - updated = AppAutoUpdater(app).run() - except Exception as e: - apps_failed.append(app) - import traceback - t = traceback.format_exc() - apps_failed_details[app] = t - print(t) - else: - if updated: - apps_updated.append(app) + with logging_redirect_tqdm(): + for app in tqdm.tqdm(apps_to_run_auto_update_for(), ascii=" ยท#"): + try: + updated = AppAutoUpdater(app).run() + except Exception as e: + apps_failed.append(app) + import traceback + + t = traceback.format_exc() + apps_failed_details[app] = t + print(t) + else: + if updated: + apps_updated.append(app) if apps_failed: print(f"Apps failed: {', '.join(apps_failed)}")