From 5f1a16d36c7034a729bd07723a28520051e27f02 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 03:40:26 +0100 Subject: [PATCH 01/11] Initial support for Gitea & Forgejo --- .../autoupdate_app_sources.py | 29 +++++++++++------- autoupdate_app_sources/rest_api.py | 30 +++++++++++++++++++ 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index b42dc78..92b758c 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -10,7 +10,7 @@ from datetime import datetime import requests import toml -from rest_api import GithubAPI, GitlabAPI, RefType +from rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType STRATEGIES = [ "latest_github_release", @@ -18,7 +18,10 @@ STRATEGIES = [ "latest_github_commit", "latest_gitlab_release", "latest_gitlab_tag", - "latest_gitlab_commit" + "latest_gitlab_commit", + "latest_giteaforgejo_release", + "latest_giteaforgejo_tag", + "latest_giteaforgejo_commit" ] if "--commit-and-create-PR" not in sys.argv: @@ -173,7 +176,7 @@ class AppAutoUpdater: print(f"\n Checking {source} ...") - if strategy == "latest_github_release" or strategy == "latest_gitlab_release": + if "_release" in strategy: ( new_version, new_asset_urls, @@ -240,7 +243,7 @@ class AppAutoUpdater: return bool(todos) if "main" in todos: - if strategy == "latest_github_release": + if "_release" in strategy: title = f"Upgrade to v{new_version}" message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}" else: @@ -302,8 +305,10 @@ class AppAutoUpdater: api = GithubAPI(upstream, auth=auth) elif "gitlab" in strategy: api = GitlabAPI(upstream) + elif "gitea" in strategy or "forgejo" in strategy: + api = GiteaForgejoAPI(upstream) - if strategy == "latest_github_release" or strategy == "latest_gitlab_release": + if "_release" in strategy: releases = api.releases() tags = [ release["tag_name"] @@ -323,7 +328,11 @@ class AppAutoUpdater: for a in latest_release["assets"] if not a["name"].endswith(".md5") } - latest_release_html_url = latest_release["html_url"] + if strategy == "latest_gitlab_release": + # gitlab's API is different for that + latest_release_html_url = latest_release["_links"]["self"] + else: + latest_release_html_url = latest_release["html_url"] if asset == "tarball": latest_tarball = ( api.url_for_ref(latest_version_orig, RefType.tags) @@ -373,10 +382,10 @@ class AppAutoUpdater: latest_release_html_url, ) - elif strategy == "latest_github_tag" or strategy == "latest_gitlab_tag": + elif "_tag" in strategy: if asset != "tarball": raise Exception( - "For the latest_github_tag strategy, only asset = 'tarball' is supported" + "For the latest tag strategy, only asset = 'tarball' is supported" ) tags = api.tags() latest_version_orig, latest_version = filter_and_get_latest_tag( @@ -385,10 +394,10 @@ class AppAutoUpdater: latest_tarball = api.url_for_ref(latest_version_orig, RefType.tags) return latest_version, latest_tarball - elif strategy == "latest_github_commit" or strategy == "latest_gitlab_commit": + elif "_commit" in strategy: if asset != "tarball": raise Exception( - "For the latest_github_release strategy, only asset = 'tarball' is supported" + "For the latest release strategy, only asset = 'tarball' is supported" ) commits = api.commits() latest_commit = commits[0] diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index ccc6b4d..22374bb 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -111,3 +111,33 @@ class GitlabAPI: def url_for_ref(self, ref: str, ref_type: RefType) -> str: return f"{self.upstream}/api/v4/projects/{self.project_id}/repository/archive.tar.gz/?sha={ref}" + + +class GiteaForgejoAPI: + def __init__(self, upstream: str): + split = re.search("(?Phttps?://.+)/(?P[^/]+)/(?P[^/]+)/?$", upstream) + self.upstream = split.group("host") + self.upstream_repo = f"{split.group('group')}/{split.group('project')}" + self.project_id = self.find_project_id(self.upstream_repo) + + def internal_api(self, uri: str): + url = f"{self.upstream}/api/v1/{uri}" + r = requests.get(url) + assert r.status_code == 200, r + return r.json() + + def tags(self) -> List[str]: + """Get a list of tags for project.""" + return self.internal_api(f"repos/{self.upstream_repo}/tags") + + def commits(self) -> List[str]: + """Get a list of commits for project.""" + return self.internal_api(f"repos/{self.upstream_repo}/commits") + + def releases(self) -> List[str]: + """Get a list of releases for project.""" + return self.internal_api(f"repos/{self.upstream_repo}/releases") + + def url_for_ref(self, ref: str, ref_type: RefType) -> str: + """Get a URL for a ref.""" + return f"{self.upstream}/{self.upstream_repo}/archive/{ref}.tar.gz" From d65a43a89dc1a82493e0ffb4d96d4b86b48edca0 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 03:43:16 +0100 Subject: [PATCH 02/11] fix strategies (split gitea & forgejo) --- autoupdate_app_sources/autoupdate_app_sources.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 92b758c..1029b96 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -19,9 +19,12 @@ STRATEGIES = [ "latest_gitlab_release", "latest_gitlab_tag", "latest_gitlab_commit", - "latest_giteaforgejo_release", - "latest_giteaforgejo_tag", - "latest_giteaforgejo_commit" + "latest_gitea_release", + "latest_gitea_tag", + "latest_gitea_commit", + "latest_forgejo_release", + "latest_forgejo_tag", + "latest_forgejo_commit" ] if "--commit-and-create-PR" not in sys.argv: From 0ef037d7def2092e8b68f870d319de21a63b7427 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 03:58:37 +0100 Subject: [PATCH 03/11] small fixes --- .../autoupdate_app_sources.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 1029b96..0facec4 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -326,12 +326,22 @@ class AppAutoUpdater: for release in releases if release["tag_name"] == latest_version_orig ][0] - latest_assets = { - a["name"]: a["browser_download_url"] - for a in latest_release["assets"] - if not a["name"].endswith(".md5") - } - if strategy == "latest_gitlab_release": + if "github" in strategy or "gitlab" in strategy: + latest_assets = { + a["name"]: a["browser_download_url"] + for a in latest_release["assets"] + if not a["name"].endswith(".md5") + } + elif "gitea" in strategy or "forgejo" in strategy: + latest_assets = { + a["name"]: a["browser_download_url"] + for a in latest_release["assets"] + if not a["name"].endswith(".md5") + } + if latest_assets == "": + # if empty (so only the base asset), take the tarball_url + latest_assets = latest_release["tarball_url"] + if strategy == "_release": # gitlab's API is different for that latest_release_html_url = latest_release["_links"]["self"] else: From 2189b3e6026a103ba804442f96eabc97a6eeca07 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 04:08:31 +0100 Subject: [PATCH 04/11] removing irrelevant project_id --- autoupdate_app_sources/rest_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index 22374bb..9da6751 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -118,7 +118,6 @@ class GiteaForgejoAPI: split = re.search("(?Phttps?://.+)/(?P[^/]+)/(?P[^/]+)/?$", upstream) self.upstream = split.group("host") self.upstream_repo = f"{split.group('group')}/{split.group('project')}" - self.project_id = self.find_project_id(self.upstream_repo) def internal_api(self, uri: str): url = f"{self.upstream}/api/v1/{uri}" From f58c3aeb0ad53fed36d5b108fb40f9b41f07f0af Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 04:21:10 +0100 Subject: [PATCH 05/11] supports 1.13.1-2 version numbers --- autoupdate_app_sources/autoupdate_app_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 0facec4..3c0c58c 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -101,7 +101,7 @@ def filter_and_get_latest_tag(tags, app_id): elif t.startswith("release-"): t_to_check = t.split("-", 1)[-1].replace("-", ".") - if not re.match(r"^v?[\d\.]*\d$", t_to_check): + if not re.match(r"^v?[\d\.]*\-?\d$", t_to_check): print(f"Ignoring tag {t_to_check}, doesn't look like a version number") else: tag_dict[t] = tag_to_int_tuple(t_to_check) @@ -112,7 +112,7 @@ def filter_and_get_latest_tag(tags, app_id): def tag_to_int_tuple(tag): - tag = tag.strip("v").strip(".") + tag = tag.strip("v").replace("-", ".").strip(".") int_tuple = tag.split(".") assert all(i.isdigit() for i in int_tuple), f"Cant convert {tag} to int tuple :/" return tuple(int(i) for i in int_tuple) From a83dd3fdf0d3b527cf9d0b36e6ccfd04395e284b Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 04:23:03 +0100 Subject: [PATCH 06/11] typo --- autoupdate_app_sources/autoupdate_app_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 3c0c58c..4b588cc 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -101,7 +101,7 @@ def filter_and_get_latest_tag(tags, app_id): elif t.startswith("release-"): t_to_check = t.split("-", 1)[-1].replace("-", ".") - if not re.match(r"^v?[\d\.]*\-?\d$", t_to_check): + if not re.match(r"^v?[\d\.]*\-\d$", t_to_check): print(f"Ignoring tag {t_to_check}, doesn't look like a version number") else: tag_dict[t] = tag_to_int_tuple(t_to_check) From 5570b003ab748e452716d9d46dea16b58a82e7b1 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 04:24:02 +0100 Subject: [PATCH 07/11] wasn't a typo lmao i'm too tired --- autoupdate_app_sources/autoupdate_app_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 4b588cc..3c0c58c 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -101,7 +101,7 @@ def filter_and_get_latest_tag(tags, app_id): elif t.startswith("release-"): t_to_check = t.split("-", 1)[-1].replace("-", ".") - if not re.match(r"^v?[\d\.]*\-\d$", t_to_check): + if not re.match(r"^v?[\d\.]*\-?\d$", t_to_check): print(f"Ignoring tag {t_to_check}, doesn't look like a version number") else: tag_dict[t] = tag_to_int_tuple(t_to_check) From fb0e0e12d1c27e8fa1a7c13b5a5f6f62f316d090 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 05:00:07 +0100 Subject: [PATCH 08/11] small refactor --- .../autoupdate_app_sources.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 3c0c58c..e8ea9e4 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -326,21 +326,14 @@ class AppAutoUpdater: for release in releases if release["tag_name"] == latest_version_orig ][0] - if "github" in strategy or "gitlab" in strategy: - latest_assets = { - a["name"]: a["browser_download_url"] - for a in latest_release["assets"] - if not a["name"].endswith(".md5") - } - elif "gitea" in strategy or "forgejo" in strategy: - latest_assets = { - a["name"]: a["browser_download_url"] - for a in latest_release["assets"] - if not a["name"].endswith(".md5") - } - if latest_assets == "": - # if empty (so only the base asset), take the tarball_url - latest_assets = latest_release["tarball_url"] + latest_assets = { + a["name"]: a["browser_download_url"] + for a in latest_release["assets"] + if not a["name"].endswith(".md5") + } + if ("gitea" in strategy or "forgejo" in strategy) and latest_assets == "": + # if empty (so only the base asset), take the tarball_url + latest_assets = latest_release["tarball_url"] if strategy == "_release": # gitlab's API is different for that latest_release_html_url = latest_release["_links"]["self"] From 9f6036d471d25a68df87adb5ba1834d2acfc6c56 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 05:27:25 +0100 Subject: [PATCH 09/11] small enhancement --- autoupdate_app_sources/autoupdate_app_sources.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index e8ea9e4..0cce06d 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -179,7 +179,7 @@ class AppAutoUpdater: print(f"\n Checking {source} ...") - if "_release" in strategy: + if strategy.endswith("_release"): ( new_version, new_asset_urls, @@ -246,7 +246,7 @@ class AppAutoUpdater: return bool(todos) if "main" in todos: - if "_release" in strategy: + if strategy.endswith("_release"): title = f"Upgrade to v{new_version}" message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}" else: @@ -257,6 +257,9 @@ class AppAutoUpdater: title = message = "Upgrade sources" new_branch = "ci-auto-update-sources" + print(f"Title: {title}") + print(f"Message: {message}") + try: # Get the commit base for the new branch, and create it commit_sha = self.repo.get_branch(self.base_branch).commit.sha @@ -311,7 +314,7 @@ class AppAutoUpdater: elif "gitea" in strategy or "forgejo" in strategy: api = GiteaForgejoAPI(upstream) - if "_release" in strategy: + if strategy.endswith("_release"): releases = api.releases() tags = [ release["tag_name"] @@ -388,7 +391,7 @@ class AppAutoUpdater: latest_release_html_url, ) - elif "_tag" in strategy: + elif strategy.endswith("_tag"): if asset != "tarball": raise Exception( "For the latest tag strategy, only asset = 'tarball' is supported" @@ -400,7 +403,7 @@ class AppAutoUpdater: latest_tarball = api.url_for_ref(latest_version_orig, RefType.tags) return latest_version, latest_tarball - elif "_commit" in strategy: + elif strategy.endswith("_commit"): if asset != "tarball": raise Exception( "For the latest release strategy, only asset = 'tarball' is supported" From 2c960ebc3bf960da9538f0adb9540fa860dc8785 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 05:28:37 +0100 Subject: [PATCH 10/11] lol forgot debug prints --- autoupdate_app_sources/autoupdate_app_sources.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 0cce06d..9a265e4 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -257,9 +257,6 @@ class AppAutoUpdater: title = message = "Upgrade sources" new_branch = "ci-auto-update-sources" - print(f"Title: {title}") - print(f"Message: {message}") - try: # Get the commit base for the new branch, and create it commit_sha = self.repo.get_branch(self.base_branch).commit.sha From 480b337914d0105a643d5138dd081b72ae8f1b37 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Wed, 14 Feb 2024 05:42:04 +0100 Subject: [PATCH 11/11] we don't need that, it's handled by GitlabAPI --- autoupdate_app_sources/autoupdate_app_sources.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 9a265e4..7a086e7 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -335,9 +335,6 @@ class AppAutoUpdater: # if empty (so only the base asset), take the tarball_url latest_assets = latest_release["tarball_url"] if strategy == "_release": - # gitlab's API is different for that - latest_release_html_url = latest_release["_links"]["self"] - else: latest_release_html_url = latest_release["html_url"] if asset == "tarball": latest_tarball = (