From 09533313551e630bcd12b173380779563b2242db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 15 Feb 2024 23:13:16 +0100 Subject: [PATCH 1/9] Fix duplicate code --- autoupdate_app_sources/autoupdate_app_sources.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 1dc4e43..2cf0612 100755 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -570,15 +570,6 @@ def main() -> None: if info[2] is not None: result_message += f" see {info[2]}" - if apps_updated: - result_message += f"\n{'=' * 80}\nApps updated:" - for app, info in apps_updated.items(): - result_message += f"\n- {app}" - if isinstance(info, tuple): - result_message += f" ({info[0]} -> {info[1]})" - if info[2] is not None: - result_message += f" see {info[2]}" - if apps_failed: result_message += f"\n{'=' * 80}\nApps failed:" for app, logs in apps_failed.items(): From 94b05b8fd8bd8a8e4a6b315a8f64c327164f4a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 15 Feb 2024 23:46:12 +0100 Subject: [PATCH 2/9] Handle cases with already existing branch / commit --- autoupdate_app_sources/autoupdate_app_sources.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 2cf0612..aa65352 100755 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -145,6 +145,7 @@ class LocalOrRemoteRepo: self.pr_branch = name commit_sha = self.repo.get_branch(self.base_branch).commit.sha if self.pr_branch in [branch.name for branch in self.repo.get_branches()]: + print("already existing") return False self.repo.create_git_ref(ref=f"refs/heads/{name}", sha=commit_sha) return True @@ -156,7 +157,7 @@ class LocalOrRemoteRepo: pr = self.repo.create_pull( title=title, body=message, head=branch, base=self.base_branch ) - return pr.url + return pr.html_url logging.warning("Can't create pull requests for local repositories") return None @@ -223,8 +224,17 @@ class AppAutoUpdater: try: if pr: self.repo.new_branch(branch_name) + except github.GithubException as e: + if e.status == 409: + print("Branch already exists!") + + try: if commit: self.repo.commit(commit_msg) + except github.GithubException as e: + if e.status == 409: + print("Commits were already commited on branch!") + try: if pr: pr_url = self.repo.create_pr(branch_name, pr_title, commit_msg) or "" except github.GithubException as e: From 1f301b6a628fb62c5282b5abcc84e31d92b5c560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 15 Feb 2024 23:54:36 +0100 Subject: [PATCH 3/9] remove roge double quote --- autoupdate_app_sources/autoupdate_app_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index aa65352..b836748 100755 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -590,7 +590,7 @@ def main() -> None: logging.error(textwrap.dedent(f""" Failed to run the source auto-update for: {', '.join(apps_failed.keys())} Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! - See the debug log here: {paste_url}" + See the debug log here: {paste_url} """)) print(result_message) From 657119f338c43b9c3c2eb7422e6148c3e78beca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 16 Feb 2024 00:20:52 +0100 Subject: [PATCH 4/9] Clean message issues --- autoupdate_app_sources/autoupdate_app_sources.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index b836748..2e90c48 100755 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -566,19 +566,17 @@ def main() -> None: result_message += f"\n{'=' * 80}\nApps already with an update PR:" for app, info in apps_already.items(): result_message += f"\n- {app}" - if isinstance(info, tuple): - result_message += f" ({info[0]} -> {info[1]})" - if info[2] is not None: - result_message += f" see {info[2]}" + result_message += f" ({info[0]} -> {info[1]})" if info[1] else " (app version did not change)" + if info[2]: + result_message += f" see {info[2]}" if apps_updated: result_message += f"\n{'=' * 80}\nApps updated:" for app, info in apps_updated.items(): result_message += f"\n- {app}" - if isinstance(info, tuple): - result_message += f" ({info[0]} -> {info[1]})" - if info[2] is not None: - result_message += f" see {info[2]}" + result_message += f" ({info[0]} -> {info[1]})" if info[1] else " (app version did not change)" + if info[2]: + result_message += f" see {info[2]}" if apps_failed: result_message += f"\n{'=' * 80}\nApps failed:" From 408827f902b8d2ee5136add005250b9b6fd0710f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 16 Feb 2024 19:07:20 +0100 Subject: [PATCH 5/9] rest_api: rename properties for readability --- autoupdate_app_sources/rest_api.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index 09c44c6..e780302 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -54,9 +54,9 @@ class GitlabAPI: def __init__(self, upstream: str): split = re.search("(?Phttps?://.+)/(?P[^/]+)/(?P[^/]+)/?$", upstream) assert split is not None - self.upstream = split.group("host") - self.upstream_repo = f"{split.group('group')}/{split.group('project')}" - self.project_id = self.find_project_id(self.upstream_repo) + self.forge_root = split.group("host") + self.project_path = f"{split.group('group')}/{split.group('project')}" + self.project_id = self.find_project_id(self.project_path) def find_project_id(self, project: str) -> int: project = self.internal_api(f"projects/{project.replace('/', '%2F')}") @@ -65,7 +65,7 @@ class GitlabAPI: return project_id def internal_api(self, uri: str) -> Any: - url = f"{self.upstream}/api/v4/{uri}" + url = f"{self.forge_root}/api/v4/{uri}" r = requests.get(url) assert r.status_code == 200, r return r.json() @@ -113,35 +113,35 @@ class GitlabAPI: return retval def url_for_ref(self, ref: str, ref_type: RefType) -> str: - name = self.upstream_repo.split("/")[-1] - return f"{self.upstream}/{self.upstream_repo}/-/archive/{ref}/{name}-{ref}.tar.bz2" + name = self.project_path.split("/")[-1] + return f"{self.forge_root}/{self.project_path}/-/archive/{ref}/{name}-{ref}.tar.bz2" class GiteaForgejoAPI: def __init__(self, upstream: str): split = re.search("(?Phttps?://.+)/(?P[^/]+)/(?P[^/]+)/?$", upstream) assert split is not None - self.upstream = split.group("host") - self.upstream_repo = f"{split.group('group')}/{split.group('project')}" + self.forge_root = split.group("host") + self.project_path = f"{split.group('group')}/{split.group('project')}" def internal_api(self, uri: str): - url = f"{self.upstream}/api/v1/{uri}" + url = f"{self.forge_root}/api/v1/{uri}" r = requests.get(url) assert r.status_code == 200, r return r.json() def tags(self) -> list[dict[str, Any]]: """Get a list of tags for project.""" - return self.internal_api(f"repos/{self.upstream_repo}/tags") + return self.internal_api(f"repos/{self.project_path}/tags") def commits(self) -> list[dict[str, Any]]: """Get a list of commits for project.""" - return self.internal_api(f"repos/{self.upstream_repo}/commits") + return self.internal_api(f"repos/{self.project_path}/commits") def releases(self) -> list[dict[str, Any]]: """Get a list of releases for project.""" - return self.internal_api(f"repos/{self.upstream_repo}/releases") + return self.internal_api(f"repos/{self.project_path}/releases") def url_for_ref(self, ref: str, ref_type: RefType) -> str: """Get a URL for a ref.""" - return f"{self.upstream}/{self.upstream_repo}/archive/{ref}.tar.gz" + return f"{self.forge_root}/{self.project_path}/archive/{ref}.tar.gz" From 189462259cc18e2764364b4d55ecdf153c8356db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 16 Feb 2024 19:36:52 +0100 Subject: [PATCH 6/9] Fix autoupdater for forges allowing for non-domain-root install and multiple levels of projects --- autoupdate_app_sources/rest_api.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index e780302..611544f 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -52,12 +52,19 @@ class GithubAPI: class GitlabAPI: def __init__(self, upstream: str): - split = re.search("(?Phttps?://.+)/(?P[^/]+)/(?P[^/]+)/?$", upstream) - assert split is not None - self.forge_root = split.group("host") - self.project_path = f"{split.group('group')}/{split.group('project')}" + # Find gitlab api root... + self.forge_root = self.get_forge_root(upstream) + self.project_path = upstream.replace(self.forge_root, "").lstrip("/") self.project_id = self.find_project_id(self.project_path) + def get_forge_root(self, project_url: str) -> str: + """A small heuristic based on the content of the html page...""" + r = requests.get(project_url) + r.raise_for_status() + match = re.search(r"const url = `(.*)/api/graphql`", r.text) + assert match is not None + return match.group(1) + def find_project_id(self, project: str) -> int: project = self.internal_api(f"projects/{project.replace('/', '%2F')}") assert isinstance(project, dict) @@ -119,10 +126,17 @@ class GitlabAPI: class GiteaForgejoAPI: def __init__(self, upstream: str): - split = re.search("(?Phttps?://.+)/(?P[^/]+)/(?P[^/]+)/?$", upstream) - assert split is not None - self.forge_root = split.group("host") - self.project_path = f"{split.group('group')}/{split.group('project')}" + # Find gitea/forgejo api root... + self.forge_root = self.get_forge_root(upstream) + self.project_path = upstream.replace(self.forge_root, "").lstrip("/") + + def get_forge_root(self, project_url: str) -> str: + """A small heuristic based on the content of the html page...""" + r = requests.get(project_url) + r.raise_for_status() + match = re.search(r"appUrl: '([^']*)',", r.text) + assert match is not None + return match.group(1).replace("\\", "") def internal_api(self, uri: str): url = f"{self.forge_root}/api/v1/{uri}" From 7d4402e9c24d135cacdd00922067dc481ffc4fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 16 Feb 2024 22:10:00 +0100 Subject: [PATCH 7/9] autoupdater: Call infos.get("autoupdate") way sooner for cleaner code --- autoupdate_app_sources/autoupdate_app_sources.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 2e90c48..8ee8fad 100755 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -297,16 +297,17 @@ class AppAutoUpdater: def get_source_update(self, name: str, infos: dict[str, Any] ) -> Optional[tuple[str, Union[str, dict[str, str]], str]]: - if "autoupdate" not in infos: + autoupdate = infos.get("autoupdate") + if autoupdate is None: return None print(f"\n Checking {name} ...") - asset = infos.get("autoupdate", {}).get("asset", "tarball") - strategy = infos.get("autoupdate", {}).get("strategy") + asset = autoupdate.get("asset", "tarball") + strategy = autoupdate.get("strategy") if strategy not in STRATEGIES: raise ValueError(f"Unknown update strategy '{strategy}' for '{name}', expected one of {STRATEGIES}") - result = self.get_latest_version_and_asset(strategy, asset, infos) + result = self.get_latest_version_and_asset(strategy, asset, autoupdate) if result is None: return None new_version, assets, more_info = result @@ -358,9 +359,9 @@ class AppAutoUpdater: raise RuntimeError(f"Too many assets matching regex '{regex}': {matching_assets}") return next(iter(matching_assets.items())) - def get_latest_version_and_asset(self, strategy: str, asset: Union[str, dict], infos + def get_latest_version_and_asset(self, strategy: str, asset: Union[str, dict], autoupdate ) -> Optional[tuple[str, Union[str, dict[str, str]], str]]: - upstream = (infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")) + upstream = autoupdate.get("upstream", self.main_upstream).strip("/") _, remote_type, revision_type = strategy.split("_") api: Union[GithubAPI, GitlabAPI, GiteaForgejoAPI] @@ -431,7 +432,7 @@ class AppAutoUpdater: latest_tarball = api.url_for_ref(latest_commit["sha"], RefType.commits) # Let's have the version as something like "2023.01.23" latest_commit_date = datetime.strptime(latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d") - version_format = infos.get("autoupdate", {}).get("force_version", "%Y.%m.%d") + version_format = autoupdate.get("force_version", "%Y.%m.%d") latest_version = latest_commit_date.strftime(version_format) return latest_version, latest_tarball, "" return None From 60ee37c24074e95a121e1296f1665e6afa507e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 16 Feb 2024 23:53:55 +0100 Subject: [PATCH 8/9] Gitlab replaces / with - in tag tarballs --- autoupdate_app_sources/rest_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autoupdate_app_sources/rest_api.py b/autoupdate_app_sources/rest_api.py index 611544f..6cf6d43 100644 --- a/autoupdate_app_sources/rest_api.py +++ b/autoupdate_app_sources/rest_api.py @@ -121,7 +121,8 @@ class GitlabAPI: def url_for_ref(self, ref: str, ref_type: RefType) -> str: name = self.project_path.split("/")[-1] - return f"{self.forge_root}/{self.project_path}/-/archive/{ref}/{name}-{ref}.tar.bz2" + clean_ref = ref.replace("/", "-") + return f"{self.forge_root}/{self.project_path}/-/archive/{ref}/{name}-{clean_ref}.tar.bz2" class GiteaForgejoAPI: From 37adbb13c0fbff4d31d884ced6c7fdcf1e9b6d32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 16 Feb 2024 23:59:54 +0100 Subject: [PATCH 9/9] Apps can now provide a "version_regex" that aims to extract the version from tag an release names. For example, version_regex = "^release_(.*)$" will extract "1.0" from "release_1.0". --- .../autoupdate_app_sources.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 8ee8fad..3b7fbf8 100755 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -246,7 +246,17 @@ class AppAutoUpdater: return (state, self.current_version, main_version, pr_url) @staticmethod - def filter_and_get_latest_tag(tags: list[str], app_id: str) -> tuple[str, str]: + def relevant_versions(tags: list[str], app_id: str, version_regex: Optional[str]) -> tuple[str, str]: + + def apply_version_regex(tag: str) -> Optional[str]: + # First preprocessing according to the manifest version_regex… + if not version_regex: + return tag + match = re.match(version_regex, tag) + if match is None: + return None + return match.group(1) + def version_numbers(tag: str) -> Optional[tuple[int, ...]]: filter_keywords = ["start", "rc", "beta", "alpha"] if any(keyword in tag for keyword in filter_keywords): @@ -265,16 +275,24 @@ class AppAutoUpdater: print(f"Ignoring tag {t_to_check}, doesn't look like a version number") return None - # sorted will sort by keys - tags_dict = {version_numbers(tag): tag for tag in tags} - tags_dict.pop(None, None) + tags_dict: dict[tuple[int, ...], tuple[str, str]] = {} + for tag in tags: + tag_clean = apply_version_regex(tag) + if tag_clean is None: + continue + tag_as_ints = version_numbers(tag_clean) + if tag_as_ints is None: + continue + tags_dict[tag_as_ints] = (tag, tag_clean) + + # sorted will sort by keys, tag_as_ints # reverse=True will set the last release as first element tags_dict = dict(sorted(tags_dict.items(), reverse=True)) if not tags_dict: raise RuntimeError("No tags were found after sanity filtering!") - the_tag_list, the_tag = next(iter(tags_dict.items())) + the_tag_list, (the_tag_orig, the_tag_clean) = next(iter(tags_dict.items())) assert the_tag_list is not None - return the_tag, ".".join(str(i) for i in the_tag_list) + return the_tag_orig, the_tag_clean @staticmethod def tag_to_int_tuple(tag: str) -> tuple[int, ...]: @@ -362,6 +380,7 @@ class AppAutoUpdater: def get_latest_version_and_asset(self, strategy: str, asset: Union[str, dict], autoupdate ) -> Optional[tuple[str, Union[str, dict[str, str]], str]]: upstream = autoupdate.get("upstream", self.main_upstream).strip("/") + version_re = autoupdate.get("version_regex", None) _, remote_type, revision_type = strategy.split("_") api: Union[GithubAPI, GitlabAPI, GiteaForgejoAPI] @@ -381,7 +400,7 @@ class AppAutoUpdater: for release in api.releases() if not release["draft"] and not release["prerelease"] } - latest_version_orig, latest_version = self.filter_and_get_latest_tag(list(releases.keys()), self.app_id) + latest_version_orig, latest_version = self.relevant_versions(list(releases.keys()), self.app_id, version_re) latest_release = releases[latest_version_orig] latest_assets = { a["name"]: a["browser_download_url"] @@ -420,7 +439,7 @@ class AppAutoUpdater: if asset != "tarball": raise ValueError("For the latest tag strategies, only asset = 'tarball' is supported") tags = [t["name"] for t in api.tags()] - latest_version_orig, latest_version = self.filter_and_get_latest_tag(tags, self.app_id) + latest_version_orig, latest_version = self.relevant_versions(tags, self.app_id, version_re) latest_tarball = api.url_for_ref(latest_version_orig, RefType.tags) return latest_version, latest_tarball, ""