Merge pull request #1972 from orhtej2/support_gitlab_autoupgrade

Support for GitLab autoupgrade.

autoupdate_app_sources/.gitignore

@@ -0,0 +1,188 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python,venv
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,venv
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### venv ###
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+pip-selfcheck.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python,venv

autoupdate_app_sources/autoupdate_app_sources.py

@@ -8,7 +8,16 @@ import os
 import glob
 from datetime import datetime
 
-STRATEGIES = ["latest_github_release", "latest_github_tag", "latest_github_commit"]
+from rest_api import GithubAPI, GitlabAPI, RefType
+
+STRATEGIES = [
+    "latest_github_release",
+    "latest_github_tag",
+    "latest_github_commit",
+    "latest_gitlab_release",
+    "latest_gitlab_tag",
+    "latest_gitlab_commit"
+    ]
 
 if "--commit-and-create-PR" not in sys.argv:
     dry_run = True
@@ -288,13 +297,12 @@ class AppAutoUpdater:
             assert (
                 upstream and upstream.startswith("https://github.com/")
             ), f"When using strategy {strategy}, having a defined upstream code repo on github.com is required"
-            upstream_repo = upstream.replace("https://github.com/", "").strip("/")
+            api = GithubAPI(upstream, auth=auth)
-            assert (
+        elif "gitlab" in strategy:
-                len(upstream_repo.split("/")) == 2
+            api = GitlabAPI(upstream)
-            ), f"'{upstream}' doesn't seem to be a github repository ?"
 
-        if strategy == "latest_github_release":
+        if strategy == "latest_github_release" or strategy == "latest_gitlab_release":
-            releases = self.github_api(f"repos/{upstream_repo}/releases")
+            releases = api.releases()
             tags = [
                 release["tag_name"]
                 for release in releases
@@ -316,7 +324,7 @@ class AppAutoUpdater:
             latest_release_html_url = latest_release["html_url"]
             if asset == "tarball":
                 latest_tarball = (
-                    f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz"
+                    api.url_for_ref(latest_version_orig, RefType.tags)
                 )
                 return latest_version, latest_tarball, latest_release_html_url
             # FIXME
@@ -363,28 +371,26 @@ class AppAutoUpdater:
                         latest_release_html_url,
                     )
 
-        elif strategy == "latest_github_tag":
+        elif strategy == "latest_github_tag" or strategy == "latest_gitlab_tag":
             if asset != "tarball":
                 raise Exception(
                     "For the latest_github_tag strategy, only asset = 'tarball' is supported"
                 )
-            tags = self.github_api(f"repos/{upstream_repo}/tags")
+            tags = api.tags()
             latest_version_orig, latest_version = filter_and_get_latest_tag(
                 [t["name"] for t in tags], self.app_id
             )
-            latest_tarball = (
+            latest_tarball = api.url_for_ref(latest_version_orig, RefType.tags)
-                f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz"
-            )
             return latest_version, latest_tarball
 
-        elif strategy == "latest_github_commit":
+        elif strategy == "latest_github_commit" or strategy == "latest_gitlab_commit":
             if asset != "tarball":
                 raise Exception(
                     "For the latest_github_release strategy, only asset = 'tarball' is supported"
                 )
-            commits = self.github_api(f"repos/{upstream_repo}/commits")
+            commits = api.commits()
             latest_commit = commits[0]
-            latest_tarball = f"https://github.com/{upstream_repo}/archive/{latest_commit['sha']}.tar.gz"
+            latest_tarball = api.url_for_ref(latest_commit["sha"], RefType.commits)
             # Let's have the version as something like "2023.01.23"
             latest_commit_date = datetime.strptime(
                 latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d"
@@ -396,11 +402,6 @@ class AppAutoUpdater:
 
             return latest_version, latest_tarball
 
-    def github_api(self, uri):
-        r = requests.get(f"https://api.github.com/{uri}", auth=auth)
-        assert r.status_code == 200, r
-        return r.json()
-
     def replace_version_and_asset_in_manifest(
         self, content, new_version, new_assets_urls, current_assets, is_main
     ):

autoupdate_app_sources/requirements.txt

@@ -0,0 +1,3 @@
+requests
+github
+toml

autoupdate_app_sources/rest_api.py

@@ -0,0 +1,103 @@
+from enum import Enum
+import re
+import requests
+from typing import List
+
+
+class RefType(Enum):
+    tags = 1
+    commits = 2
+
+
+class GithubAPI:
+    def __init__(self, upstream: str, auth: tuple[str, str] = None):
+        self.upstream = upstream
+        self.upstream_repo = upstream.replace("https://github.com/", "")\
+            .strip("/")
+        assert (
+                len(self.upstream_repo.split("/")) == 2
+            ), f"'{upstream}' doesn't seem to be a github repository ?"
+        self.auth = auth
+
+    def internal_api(self, uri: str):
+        url = f"https://api.github.com/{uri}"
+        r = requests.get(url, auth=self.auth)
+        assert r.status_code == 200, r
+        return r.json()
+
+    def tags(self) -> List[str]:
+        """Get a list of tags for project."""
+        return self.internal_api(f"repos/{self.upstream_repo}/tags")
+
+    def commits(self) -> List[str]:
+        """Get a list of commits for project."""
+        return self.internal_api(f"repos/{self.upstream_repo}/commits")
+
+    def releases(self) -> List[str]:
+        """Get a list of releases for project."""
+        return self.internal_api(f"repos/{self.upstream_repo}/releases")
+
+    def url_for_ref(self, ref: str, ref_type: RefType) -> str:
+        """Get a URL for a ref."""
+        if ref_type == RefType.tags:
+            return f"{self.upstream}/archive/refs/tags/{ref}.tar.gz"
+        elif ref_type == RefType.commits:
+            return f"{self.upstream}/archive/{ref}.tar.gz"
+        else:
+            raise NotImplementedError
+
+
+class GitlabAPI:
+    def __init__(self, upstream: str):
+        split = re.search("(?P<host>https?://.+)/(?P<group>[^/]+)/(?P<project>[^/]+)/?$", upstream)
+        self.upstream = split.group("host")
+        self.upstream_repo = f"{split.group('group')}/{split.group('project')}"
+        self.project_id = self.find_project_id(split.group('project'))
+
+    def find_project_id(self, project: str) -> int:
+        projects = self.internal_api(f"projects?search={project}")
+        for project in projects:
+            if project["path_with_namespace"] == self.upstream_repo:
+                return project["id"]
+        raise ValueError(f"Project {project} not found")
+
+    def internal_api(self, uri: str):
+        url = f"{self.upstream}/api/v4/{uri}"
+        r = requests.get(url)
+        assert r.status_code == 200, r
+        return r.json()
+
+    def tags(self) -> List[str]:
+        """Get a list of tags for project."""
+        return self.internal_api(f"projects/{self.project_id}/repository/tags")
+
+    def commits(self) -> List[str]:
+        """Get a list of commits for project."""
+        return [
+            {
+                "sha": commit["id"],
+                "commit": {
+                    "author": { 
+                        "date": commit["committed_date"]
+                        }
+                    }
+            }
+            for commit in self.internal_api(f"projects/{self.project_id}/repository/commits")
+            ]
+
+    def releases(self) -> List[str]:
+        """Get a list of releases for project."""
+        releases = self.internal_api(f"projects/{self.project_id}/releases")
+        return [{
+                "tag_name": release["tag_name"],
+                "prerelease": False,
+                "draft": False,
+                "html_url": release["_links"]["self"],
+                "assets": [{
+                    "name": asset["name"],
+                    "browser_download_url": asset["direct_asset_url"]
+                    } for asset in release["assets"]["links"]],
+                } for release in releases]
+
+    def url_for_ref(self, ref: str, ref_type: RefType) -> str:
+        return f"{self.upstream}/api/v4/projects/{self.project_id}/repository/archive.tar.gz/?sha={ref}"