hosting/ynh-apps_tools
1
0
Fork 0
Code Activity

add changelog link for latest_github_release

Browse source
This commit is contained in:
OniriCorpe 2024-01-04 22:09:00 +01:00
parent 4825fa7c22
commit 19c9569700
1 changed files with 70 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
autoupdate_app_sources/autoupdate_app_sources.py
View file

@ -38,7 +38,6 @@ else:
def apps_to_run_auto_update_for():
catalog = toml.load(open(os.path.dirname(__file__) + "/../../apps.toml"))
apps_flagged_as_working_and_on_yunohost_apps_org = [
@ -96,7 +95,6 @@ def filter_and_get_latest_tag(tags, app_id):
def tag_to_int_tuple(tag):
tag = tag.strip("v").strip(".")
int_tuple = tag.split(".")
assert all(i.isdigit() for i in int_tuple), f"Cant convert {tag} to int tuple :/"
@ -118,7 +116,6 @@ def sha256_of_remote_file(url):
class AppAutoUpdater:
def __init__(self, app_id, app_id_is_local_app_dir=False):
if app_id_is_local_app_dir:
if not os.path.exists(app_id + "/manifest.toml"):
raise Exception("manifest.toml doesnt exists?")
@ -149,11 +146,9 @@ class AppAutoUpdater:
self.main_upstream = manifest.get("upstream", {}).get("code")
def run(self):
todos = {}
for source, infos in self.sources.items():
if "autoupdate" not in infos:
continue
@ -167,9 +162,16 @@ class AppAutoUpdater:
print(f"\n Checking {source} ...")
new_version, new_asset_urls = self.get_latest_version_and_asset(
strategy, asset, infos, source
)
if strategy == "latest_github_release":
(
new_version,
new_asset_urls,
changelog_url,
) = self.get_latest_version_and_asset(strategy, asset, infos, source)
else:
(new_version, new_asset_urls) = self.get_latest_version_and_asset(
strategy, asset, infos, source
)
if source == "main":
print(f"Current version in manifest: {self.current_version}")
@ -181,8 +183,12 @@ class AppAutoUpdater:
# Though we wrap this in a try/except pass, because don't want to miserably crash
# if the tag can't properly be converted to int tuple ...
try:
if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(new_version):
print("Up to date (current version appears more recent than newest version found)")
if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(
new_version
):
print(
"Up to date (current version appears more recent than newest version found)"
)
continue
except:
pass
@ -191,9 +197,15 @@ class AppAutoUpdater:
print("Up to date")
continue
if (isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)) \
or (isinstance(new_asset_urls, str) and not isinstance(infos.get("url"), str)):
raise Exception(f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}")
if (
isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)
) or (
isinstance(new_asset_urls, str)
and not isinstance(infos.get("url"), str)
):
raise Exception(
f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}"
)
if isinstance(new_asset_urls, str) and infos["url"] == new_asset_urls:
print(f"URL for asset {source} is up to date")
@ -217,8 +229,11 @@ class AppAutoUpdater:
return bool(todos)
if "main" in todos:
if strategy == "latest_github_release":
message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}"
else:
message = f"Upgrade to v{new_version}"
new_version = todos["main"]["new_version"]
message = f"Upgrade to v{new_version}"
new_branch = f"ci-auto-update-{new_version}"
else:
message = "Upgrade sources"
@ -264,12 +279,13 @@ class AppAutoUpdater:
return bool(todos)
def get_latest_version_and_asset(self, strategy, asset, infos, source):
upstream = infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")
upstream = (
infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")
)
if "github" in strategy:
assert upstream and upstream.startswith(
"https://github.com/"
assert (
upstream and upstream.startswith("https://github.com/")
), f"When using strategy {strategy}, having a defined upstream code repo on github.com is required"
upstream_repo = upstream.replace("https://github.com/", "").strip("/")
assert (
@ -318,7 +334,11 @@ class AppAutoUpdater:
raise Exception(
f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
)
return latest_version, matching_assets_urls[0]
return (
latest_version,
matching_assets_urls[0],
latest_release_html_url,
)
elif isinstance(asset, dict):
matching_assets_dicts = {}
for asset_name, asset_regex in asset.items():
@ -336,7 +356,11 @@ class AppAutoUpdater:
f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
)
matching_assets_dicts[asset_name] = matching_assets_urls[0]
return latest_version.strip("v"), matching_assets_dicts
return (
latest_version.strip("v"),
matching_assets_dicts,
latest_release_html_url,
)
elif strategy == "latest_github_tag":
if asset != "tarball":
@ -347,7 +371,9 @@ class AppAutoUpdater:
latest_version_orig, latest_version = filter_and_get_latest_tag(
[t["name"] for t in tags], self.app_id
)
latest_tarball = f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz"
latest_tarball = (
f"{upstream}/archive/refs/tags/{latest_version_orig}.tar.gz"
)
return latest_version, latest_tarball
elif strategy == "latest_github_commit":
@ -359,14 +385,17 @@ class AppAutoUpdater:
latest_commit = commits[0]
latest_tarball = f"https://github.com/{upstream_repo}/archive/{latest_commit['sha']}.tar.gz"
# Let's have the version as something like "2023.01.23"
latest_commit_date = datetime.strptime(latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d")
version_format = infos.get("autoupdate", {}).get("force_version", "%Y.%m.%d")
latest_commit_date = datetime.strptime(
latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d"
)
version_format = infos.get("autoupdate", {}).get(
"force_version", "%Y.%m.%d"
)
latest_version = latest_commit_date.strftime(version_format)
return latest_version, latest_tarball
def github_api(self, uri):
r = requests.get(f"https://api.github.com/{uri}", auth=auth)
assert r.status_code == 200, r
return r.json()
@ -374,7 +403,6 @@ class AppAutoUpdater:
def replace_version_and_asset_in_manifest(
self, content, new_version, new_assets_urls, current_assets, is_main
):
if isinstance(new_assets_urls, str):
sha256 = sha256_of_remote_file(new_assets_urls)
elif isinstance(new_assets_urls, dict):
@ -385,7 +413,7 @@ class AppAutoUpdater:
if is_main:
def repl(m):
return m.group(1) + new_version + "~ynh1\""
return m.group(1) + new_version + '~ynh1"'
content = re.sub(
r"(\s*version\s*=\s*[\"\'])([\d\.]+)(\~ynh\d+[\"\'])", repl, content
@ -410,7 +438,8 @@ def progressbar(it, prefix="", size=60, file=sys.stdout):
name += " "
x = int(size * j / count)
file.write(
"\n%s[%s%s] %i/%i %s\n" % (prefix, "#" * x, "." * (size - x), j, count, name)
"\n%s[%s%s] %i/%i %s\n"
% (prefix, "#" * x, "." * (size - x), j, count, name)
)
file.flush()
@ -430,14 +459,15 @@ def paste_on_haste(data):
TIMEOUT = 3
try:
url = SERVER_URL + "/documents"
response = requests.post(url, data=data.encode('utf-8'), timeout=TIMEOUT)
response = requests.post(url, data=data.encode("utf-8"), timeout=TIMEOUT)
response.raise_for_status()
dockey = response.json()['key']
dockey = response.json()["key"]
return SERVER_URL + "/raw/" + dockey
except requests.exceptions.RequestException as e:
print("\033[31mError: {}\033[0m".format(e))
sys.exit(1)
if __name__ == "__main__":
args = [arg for arg in sys.argv[1:] if arg != "--commit-and-create-PR"]
@ -453,6 +483,7 @@ if __name__ == "__main__":
except Exception as e:
apps_failed.append(app)
import traceback
t = traceback.format_exc()
apps_failed_details[app] = t
print(t)
@ -463,8 +494,15 @@ if __name__ == "__main__":
if apps_failed:
print(f"Apps failed: {', '.join(apps_failed)}")
if os.path.exists("/usr/bin/sendxmpppy"):
paste = '\n=========\n'.join([app + "\n-------\n" + trace + "\n\n" for app, trace in apps_failed_details.items()])
paste_url = paste_on_haste(paste)
os.system(f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'")
paste = "\n=========\n".join(
[
app + "\n-------\n" + trace + "\n\n"
for app, trace in apps_failed_details.items()
]
)
paste_url = paste_on_haste(paste)
os.system(
f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'"
)
if apps_updated:
print(f"Apps updated: {', '.join(apps_updated)}")