From e02e5fb9cbb6a67de7f5006774614c52c566860a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 18 Dec 2023 19:00:53 +0100 Subject: [PATCH 01/11] Improve linter to also check notworking apps and graveyard entries --- catalog_linter.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/catalog_linter.py b/catalog_linter.py index 3c5453e..3657180 100755 --- a/catalog_linter.py +++ b/catalog_linter.py @@ -37,6 +37,12 @@ def get_wishlist() -> Dict[str, Dict[str, str]]: return toml.load(wishlist_path) +@cache +def get_graveyard() -> Dict[str, Dict[str, str]]: + wishlist_path = APPS_ROOT / "graveyard.toml" + return toml.load(wishlist_path) + + def validate_schema() -> Generator[str, None, None]: with open(APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file: apps_catalog_schema = json.load(file) @@ -50,9 +56,6 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], No yield "state is missing", True return - if infos["state"] != "working": - return - # validate that the app is not (anymore?) in the wishlist # we use fuzzy matching because the id in catalog may not be the same exact id as in the wishlist # some entries are ignore-hard-coded, because e.g. radarr an readarr are really different apps... @@ -66,6 +69,14 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], No if wishlist_matches: yield f"app seems to be listed in wishlist: {wishlist_matches}", True + graveyard_matches = [ + grave + for grave in get_graveyard() + if SequenceMatcher(None, app, grave).ratio() > 0.9 + ] + if graveyard_matches: + yield f"app seems to be listed in graveyard: {graveyard_matches}", True + repo_name = infos.get("url", "").split("/")[-1] if repo_name != f"{app}_ynh": yield f"repo name should be {app}_ynh, not in {repo_name}", True From f73a2c183cd0c8ed5f8c58de9d7f3b8e8f35cf16 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 18 Dec 2023 19:03:23 +0100 Subject: [PATCH 02/11] Fix false positive mailman<->mailman3 --- catalog_linter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catalog_linter.py b/catalog_linter.py index 3657180..4c383a9 100755 --- a/catalog_linter.py +++ b/catalog_linter.py @@ -69,10 +69,12 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], No if wishlist_matches: yield f"app seems to be listed in wishlist: {wishlist_matches}", True + ignored_graveyard_entries = ["mailman"] graveyard_matches = [ grave for grave in get_graveyard() - if SequenceMatcher(None, app, grave).ratio() > 0.9 + if grave not in ignored_graveyard_entries + and SequenceMatcher(None, app, grave).ratio() > 0.9 ] if graveyard_matches: yield f"app seems to be listed in graveyard: {graveyard_matches}", True From 19c956970006705ad32f0d01bdab98b887a03de9 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 4 Jan 2024 22:09:00 +0100 Subject: [PATCH 03/11] add changelog link for latest_github_release --- .../autoupdate_app_sources.py | 102 ++++++++++++------ 1 file changed, 70 insertions(+), 32 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index ca70698..72a418d 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -38,7 +38,6 @@ else: def apps_to_run_auto_update_for(): - catalog = toml.load(open(os.path.dirname(__file__) + "/../../apps.toml")) apps_flagged_as_working_and_on_yunohost_apps_org = [ @@ -96,7 +95,6 @@ def filter_and_get_latest_tag(tags, app_id): def tag_to_int_tuple(tag): - tag = tag.strip("v").strip(".") int_tuple = tag.split(".") assert all(i.isdigit() for i in int_tuple), f"Cant convert {tag} to int tuple :/" @@ -118,7 +116,6 @@ def sha256_of_remote_file(url): class AppAutoUpdater: def __init__(self, app_id, app_id_is_local_app_dir=False): - if app_id_is_local_app_dir: if not os.path.exists(app_id + "/manifest.toml"): raise Exception("manifest.toml doesnt exists?") @@ -149,11 +146,9 @@ class AppAutoUpdater: self.main_upstream = manifest.get("upstream", {}).get("code") def run(self): - todos = {} for source, infos in self.sources.items(): - if "autoupdate" not in infos: continue @@ -167,9 +162,16 @@ class AppAutoUpdater: print(f"\n Checking {source} ...") - new_version, new_asset_urls = self.get_latest_version_and_asset( - strategy, asset, infos, source - ) + if strategy == "latest_github_release": + ( + new_version, + new_asset_urls, + changelog_url, + ) = self.get_latest_version_and_asset(strategy, asset, infos, source) + else: + (new_version, new_asset_urls) = self.get_latest_version_and_asset( + strategy, asset, infos, source + ) if source == "main": print(f"Current version in manifest: {self.current_version}") @@ -181,8 +183,12 @@ class AppAutoUpdater: # Though we wrap this in a try/except pass, because don't want to miserably crash # if the tag can't properly be converted to int tuple ... try: - if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(new_version): - print("Up to date (current version appears more recent than newest version found)") + if tag_to_int_tuple(self.current_version) > tag_to_int_tuple( + new_version + ): + print( + "Up to date (current version appears more recent than newest version found)" + ) continue except: pass @@ -191,9 +197,15 @@ class AppAutoUpdater: print("Up to date") continue - if (isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)) \ - or (isinstance(new_asset_urls, str) and not isinstance(infos.get("url"), str)): - raise Exception(f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}") + if ( + isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str) + ) or ( + isinstance(new_asset_urls, str) + and not isinstance(infos.get("url"), str) + ): + raise Exception( + f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}" + ) if isinstance(new_asset_urls, str) and infos["url"] == new_asset_urls: print(f"URL for asset {source} is up to date") @@ -217,8 +229,11 @@ class AppAutoUpdater: return bool(todos) if "main" in todos: + if strategy == "latest_github_release": + message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}" + else: + message = f"Upgrade to v{new_version}" new_version = todos["main"]["new_version"] - message = f"Upgrade to v{new_version}" new_branch = f"ci-auto-update-{new_version}" else: message = "Upgrade sources" @@ -264,12 +279,13 @@ class AppAutoUpdater: return bool(todos) def get_latest_version_and_asset(self, strategy, asset, infos, source): - - upstream = infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/") + upstream = ( + infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/") + ) if "github" in strategy: - assert upstream and upstream.startswith( - "https://github.com/" + assert ( + upstream and upstream.startswith("https://github.com/") ), f"When using strategy {strategy}, having a defined upstream code repo on github.com is required" upstream_repo = upstream.replace("https://github.com/", "").strip("/") assert ( @@ -318,7 +334,11 @@ class AppAutoUpdater: raise Exception( f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}" ) - return latest_version, matching_assets_urls[0] + return ( + latest_version, + matching_assets_urls[0], + latest_release_html_url, + ) elif isinstance(asset, dict): matching_assets_dicts = {} for asset_name, asset_regex in asset.items(): @@ -336,7 +356,11 @@ class AppAutoUpdater: f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}" ) matching_assets_dicts[asset_name] = matching_assets_urls[0] - return latest_version.strip("v"), matching_assets_dicts + return ( + latest_version.strip("v"), + matching_assets_dicts, + latest_release_html_url, + ) elif strategy == "latest_github_tag": if asset != "tarball": @@ -347,7 +371,9 @@ class AppAutoUpdater: latest_version_orig, latest_version = filter_and_get_latest_tag( [t["name"] for t in tags], self.app_id ) - latest_tarball = f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz" + latest_tarball = ( + f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz" + ) return latest_version, latest_tarball elif strategy == "latest_github_commit": @@ -359,14 +385,17 @@ class AppAutoUpdater: latest_commit = commits[0] latest_tarball = f"https://github.com/{upstream_repo}/archive/{latest_commit['sha']}.tar.gz" # Let's have the version as something like "2023.01.23" - latest_commit_date = datetime.strptime(latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d") - version_format = infos.get("autoupdate", {}).get("force_version", "%Y.%m.%d") + latest_commit_date = datetime.strptime( + latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d" + ) + version_format = infos.get("autoupdate", {}).get( + "force_version", "%Y.%m.%d" + ) latest_version = latest_commit_date.strftime(version_format) return latest_version, latest_tarball def github_api(self, uri): - r = requests.get(f"https://api.github.com/{uri}", auth=auth) assert r.status_code == 200, r return r.json() @@ -374,7 +403,6 @@ class AppAutoUpdater: def replace_version_and_asset_in_manifest( self, content, new_version, new_assets_urls, current_assets, is_main ): - if isinstance(new_assets_urls, str): sha256 = sha256_of_remote_file(new_assets_urls) elif isinstance(new_assets_urls, dict): @@ -385,7 +413,7 @@ class AppAutoUpdater: if is_main: def repl(m): - return m.group(1) + new_version + "~ynh1\"" + return m.group(1) + new_version + '~ynh1"' content = re.sub( r"(\s*version\s*=\s*[\"\'])([\d\.]+)(\~ynh\d+[\"\'])", repl, content @@ -410,7 +438,8 @@ def progressbar(it, prefix="", size=60, file=sys.stdout): name += " " x = int(size * j / count) file.write( - "\n%s[%s%s] %i/%i %s\n" % (prefix, "#" * x, "." * (size - x), j, count, name) + "\n%s[%s%s] %i/%i %s\n" + % (prefix, "#" * x, "." * (size - x), j, count, name) ) file.flush() @@ -430,14 +459,15 @@ def paste_on_haste(data): TIMEOUT = 3 try: url = SERVER_URL + "/documents" - response = requests.post(url, data=data.encode('utf-8'), timeout=TIMEOUT) + response = requests.post(url, data=data.encode("utf-8"), timeout=TIMEOUT) response.raise_for_status() - dockey = response.json()['key'] + dockey = response.json()["key"] return SERVER_URL + "/raw/" + dockey except requests.exceptions.RequestException as e: print("\033[31mError: {}\033[0m".format(e)) sys.exit(1) + if __name__ == "__main__": args = [arg for arg in sys.argv[1:] if arg != "--commit-and-create-PR"] @@ -453,6 +483,7 @@ if __name__ == "__main__": except Exception as e: apps_failed.append(app) import traceback + t = traceback.format_exc() apps_failed_details[app] = t print(t) @@ -463,8 +494,15 @@ if __name__ == "__main__": if apps_failed: print(f"Apps failed: {', '.join(apps_failed)}") if os.path.exists("/usr/bin/sendxmpppy"): - paste = '\n=========\n'.join([app + "\n-------\n" + trace + "\n\n" for app, trace in apps_failed_details.items()]) - paste_url = paste_on_haste(paste) - os.system(f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'") + paste = "\n=========\n".join( + [ + app + "\n-------\n" + trace + "\n\n" + for app, trace in apps_failed_details.items() + ] + ) + paste_url = paste_on_haste(paste) + os.system( + f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'" + ) if apps_updated: print(f"Apps updated: {', '.join(apps_updated)}") From b9dd575b91db3e71309d85c839c31fa6dc4eab5a Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 4 Jan 2024 22:34:04 +0100 Subject: [PATCH 04/11] fix if --- autoupdate_app_sources/autoupdate_app_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 72a418d..72dde3c 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -162,7 +162,7 @@ class AppAutoUpdater: print(f"\n Checking {source} ...") - if strategy == "latest_github_release": + if strategy == "latest_github_release" & asset != "tarball": ( new_version, new_asset_urls, @@ -229,7 +229,7 @@ class AppAutoUpdater: return bool(todos) if "main" in todos: - if strategy == "latest_github_release": + if strategy == "latest_github_release" & asset != "tarball": message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}" else: message = f"Upgrade to v{new_version}" From 2a18e99b6b4525d37299858bb432396983fddbf8 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 4 Jan 2024 22:47:54 +0100 Subject: [PATCH 05/11] Update tools/autoupdate_app_sources/autoupdate_app_sources.py Co-authored-by: Alexandre Aubin <4533074+alexAubin@users.noreply.github.com> --- autoupdate_app_sources/autoupdate_app_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 72dde3c..2b6d68a 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -162,7 +162,7 @@ class AppAutoUpdater: print(f"\n Checking {source} ...") - if strategy == "latest_github_release" & asset != "tarball": + if strategy == "latest_github_release" and asset != "tarball": ( new_version, new_asset_urls, From 6b32b5f25736ace2baa52d8d3b6cc3cf223eeab3 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 4 Jan 2024 22:47:58 +0100 Subject: [PATCH 06/11] Update tools/autoupdate_app_sources/autoupdate_app_sources.py Co-authored-by: Alexandre Aubin <4533074+alexAubin@users.noreply.github.com> --- autoupdate_app_sources/autoupdate_app_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 2b6d68a..2660775 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -229,7 +229,7 @@ class AppAutoUpdater: return bool(todos) if "main" in todos: - if strategy == "latest_github_release" & asset != "tarball": + if strategy == "latest_github_release" and asset != "tarball": message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}" else: message = f"Upgrade to v{new_version}" From 06927ae7247f8cadeaf8834d319e4adbad81aee9 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 4 Jan 2024 23:21:07 +0100 Subject: [PATCH 07/11] get changelog link for release tarball too --- .../autoupdate_app_sources.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 2660775..9464807 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -162,7 +162,7 @@ class AppAutoUpdater: print(f"\n Checking {source} ...") - if strategy == "latest_github_release" and asset != "tarball": + if strategy == "latest_github_release": ( new_version, new_asset_urls, @@ -229,7 +229,7 @@ class AppAutoUpdater: return bool(todos) if "main" in todos: - if strategy == "latest_github_release" and asset != "tarball": + if strategy == "latest_github_release": message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}" else: message = f"Upgrade to v{new_version}" @@ -302,6 +302,17 @@ class AppAutoUpdater: latest_version_orig, latest_version = filter_and_get_latest_tag( tags, self.app_id ) + latest_release = [ + release + for release in releases + if release["tag_name"] == latest_version_orig + ][0] + latest_assets = { + a["name"]: a["browser_download_url"] + for a in latest_release["assets"] + if not a["name"].endswith(".md5") + } + latest_release_html_url = latest_release["html_url"] if asset == "tarball": latest_tarball = ( f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz" @@ -309,17 +320,6 @@ class AppAutoUpdater: return latest_version, latest_tarball # FIXME else: - latest_release = [ - release - for release in releases - if release["tag_name"] == latest_version_orig - ][0] - latest_assets = { - a["name"]: a["browser_download_url"] - for a in latest_release["assets"] - if not a["name"].endswith(".md5") - } - latest_release_html_url = latest_release["html_url"] if isinstance(asset, str): matching_assets_urls = [ url From 14d79aa8af974061913a2f79709ca2a0b0280e69 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 4 Jan 2024 23:22:20 +0100 Subject: [PATCH 08/11] fix return --- autoupdate_app_sources/autoupdate_app_sources.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 9464807..73971b3 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -239,6 +239,8 @@ class AppAutoUpdater: message = "Upgrade sources" new_branch = "ci-auto-update-sources" + print(message) + try: # Get the commit base for the new branch, and create it commit_sha = self.repo.get_branch(self.base_branch).commit.sha @@ -317,7 +319,7 @@ class AppAutoUpdater: latest_tarball = ( f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz" ) - return latest_version, latest_tarball + return latest_version, latest_tarball, latest_release_html_url # FIXME else: if isinstance(asset, str): From f9cad0880e938445f20f38ee8ce6f6e6508b5810 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 4 Jan 2024 23:26:37 +0100 Subject: [PATCH 09/11] remove debug print --- autoupdate_app_sources/autoupdate_app_sources.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 73971b3..b7bd956 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -239,8 +239,6 @@ class AppAutoUpdater: message = "Upgrade sources" new_branch = "ci-auto-update-sources" - print(message) - try: # Get the commit base for the new branch, and create it commit_sha = self.repo.get_branch(self.base_branch).commit.sha From eafc9606e41fbd185df746217d59e040aaf7c680 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 5 Jan 2024 20:54:10 +0100 Subject: [PATCH 10/11] fix title --- autoupdate_app_sources/autoupdate_app_sources.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index b7bd956..9788d6a 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -230,15 +230,19 @@ class AppAutoUpdater: if "main" in todos: if strategy == "latest_github_release": + title = f"Upgrade to v{new_version}" message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}" else: - message = f"Upgrade to v{new_version}" + title = message = f"Upgrade to v{new_version}" new_version = todos["main"]["new_version"] new_branch = f"ci-auto-update-{new_version}" else: - message = "Upgrade sources" + title = message = "Upgrade sources" new_branch = "ci-auto-update-sources" + print(f"Title: {title}") + print(f"Message: {message}") + try: # Get the commit base for the new branch, and create it commit_sha = self.repo.get_branch(self.base_branch).commit.sha @@ -271,7 +275,7 @@ class AppAutoUpdater: # Open the PR pr = self.repo.create_pull( - title=message, body=message, head=new_branch, base=self.base_branch + title=title, body=message, head=new_branch, base=self.base_branch ) print("Created PR " + self.repo.full_name + " updated with PR #" + str(pr.id)) From 50ce88fe1758260d02720799f6ae97d1b991cdef Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 5 Jan 2024 20:56:36 +0100 Subject: [PATCH 11/11] remove debug print --- autoupdate_app_sources/autoupdate_app_sources.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 9788d6a..bb2743b 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -240,9 +240,6 @@ class AppAutoUpdater: title = message = "Upgrade sources" new_branch = "ci-auto-update-sources" - print(f"Title: {title}") - print(f"Message: {message}") - try: # Get the commit base for the new branch, and create it commit_sha = self.repo.get_branch(self.base_branch).commit.sha