From 7b2c85c2ff5d328dc03954c1f652aac0c17d5dc2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 20 Jan 2023 00:02:59 +0100 Subject: [PATCH 1/4] Propagate the json/yml -> toml change to README generator --- README-generator/make_readme.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index 9788d89..5f004ea 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -4,7 +4,6 @@ import argparse import json import toml import os -import yaml from pathlib import Path from jinja2 import Environment, FileSystemLoader @@ -33,11 +32,10 @@ def generate_READMEs(app_path: str): upstream = manifest.get("upstream", {}) - catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) + catalog = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.toml")) from_catalog = catalog.get(manifest['id'], {}) - antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader) - antifeatures_list = { e['id']: e for e in antifeatures_list } + antifeatures_list = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.toml")) if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( From d5299825e6ed08ce1f20a36b2b995f95ce5eb899 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 20 Jan 2023 16:44:00 +0100 Subject: [PATCH 2/4] Add a new script to update levels according to CI results and create the corresponding PR --- .gitignore | 2 + update_app_levels/update_app_levels.py | 99 ++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 update_app_levels/update_app_levels.py diff --git a/.gitignore b/.gitignore index ce6c63a..ce60bf3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ tools/bot-repo-cleanup/.github_token tools/autopatches/login tools/autopatches/token + +.github_token diff --git a/update_app_levels/update_app_levels.py b/update_app_levels/update_app_levels.py new file mode 100644 index 0000000..0770440 --- /dev/null +++ b/update_app_levels/update_app_levels.py @@ -0,0 +1,99 @@ +import time +import toml +import requests +import tempfile +import os +import sys +import json + +token = open(".github_token").read().strip() + +tmpdir = tempfile.mkdtemp(prefix="update_app_levels_") +os.system(f"git clone 'https://oauth2:{token}@github.com/yunohost/apps' {tmpdir}") +os.system(f"git -C {tmpdir} checkout -b update_app_levels") + +# Load the app catalog and filter out the non-working ones +catalog = toml.load(open(f"{tmpdir}/apps.toml")) + +# Fetch results from the CI +ci_results = requests.get("https://ci-apps.yunohost.org/ci/logs/list_level_stable_amd64.json").json() + +comment = { + "major_regressions": [], + "minor_regressions": [], + "improvements": [], + "outdated": [], + "missing": [], +} + +for app, infos in catalog.items(): + + if infos.get("state") != "working": + continue + + if app not in ci_results: + comment["missing"].append(app) + continue + + # 3600 * 24 * 60 = ~2 months + if (int(time.time()) - ci_results[app].get("timestamp", 0)) > 3600 * 24 * 60: + comment["outdated"].append(app) + continue + + ci_level = ci_results[app]["level"] + current_level = infos.get("level") + + if ci_level == current_level: + continue + elif current_level is None or ci_level > current_level: + comment["improvements"].append((app, current_level, ci_level)) + elif ci_level < current_level: + if ci_level < 4 and current_level >= 4: + comment["major_regressions"].append((app, current_level, ci_level)) + else: + comment["minor_regressions"].append((app, current_level, ci_level)) + + infos["level"] = ci_level + +updated_catalog = toml.dumps(catalog) +updated_catalog = updated_catalog.replace(",]", " ]") +open(f"{tmpdir}/apps.toml", "w").write(updated_catalog) + +os.system(f"git -C {tmpdir} commit apps.toml -m 'Update app levels according to CI results'") +os.system(f"git -C {tmpdir} push origin update_app_levels --force") +os.system(f"rm -rf {tmpdir}") + +PR_body = "" +if comment["major_regressions"]: + PR_body += "\n### Major regressions\n\n" + for app, current_level, new_level in comment['major_regressions']: + PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["minor_regressions"]: + PR_body += "\n### Minor regressions\n\n" + for app, current_level, new_level in comment['minor_regressions']: + PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["improvements"]: + PR_body += "\n### Improvements\n\n" + for app, current_level, new_level in comment['improvements']: + PR_body += f"- {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["missing"]: + PR_body += "\n### Missing results\n\n" + for app in comment['missing']: + PR_body += f"- {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["outdated"]: + PR_body += "\n### Outdated results\n\n" + for app in comment['outdated']: + PR_body += f"- [ ] {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" + +PR = {"title": "Update app levels accoring to CI results", + "body": PR_body, + "head": "update_app_levels", + "base": "master"} + +with requests.Session() as s: + s.headers.update({"Authorization": f"token {token}"}) +r = s.post("https://api.github.com/repos/yunohost/apps/pulls", json.dumps(PR)) + +if r.status_code != 200: + print(r.text) + sys.exit(1) From 98b87d4f72113224ba56aec7fff72103155c17e8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 20 Jan 2023 17:14:30 +0100 Subject: [PATCH 3/4] update_app_levels: autosort keys --- update_app_levels/update_app_levels.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/update_app_levels/update_app_levels.py b/update_app_levels/update_app_levels.py index 0770440..12713de 100644 --- a/update_app_levels/update_app_levels.py +++ b/update_app_levels/update_app_levels.py @@ -5,6 +5,7 @@ import tempfile import os import sys import json +from collections import OrderedDict token = open(".github_token").read().strip() @@ -16,7 +17,8 @@ os.system(f"git -C {tmpdir} checkout -b update_app_levels") catalog = toml.load(open(f"{tmpdir}/apps.toml")) # Fetch results from the CI -ci_results = requests.get("https://ci-apps.yunohost.org/ci/logs/list_level_stable_amd64.json").json() +CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/logs/list_level_stable_amd64.json" +ci_results = requests.get(CI_RESULTS_URL).json() comment = { "major_regressions": [], @@ -55,6 +57,11 @@ for app, infos in catalog.items(): infos["level"] = ci_level +# Also re-sort the catalog keys / subkeys +for app, infos in catalog.items(): + catalog[app] = OrderedDict(sorted(infos.items())) +catalog = OrderedDict(sorted(catalog.items())) + updated_catalog = toml.dumps(catalog) updated_catalog = updated_catalog.replace(",]", " ]") open(f"{tmpdir}/apps.toml", "w").write(updated_catalog) From e22aab15c17e6b17721f343af809afc944df2fe0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 20 Jan 2023 17:33:33 +0100 Subject: [PATCH 4/4] ci: Add a proper catalog linter instead of bash mess --- catalog_linter.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 catalog_linter.py diff --git a/catalog_linter.py b/catalog_linter.py new file mode 100644 index 0000000..6e17117 --- /dev/null +++ b/catalog_linter.py @@ -0,0 +1,28 @@ +import toml +import sys + +catalog = toml.load(open('apps.toml')) +catalog = {app: infos for app, infos in catalog.items() if infos.get('state') == "working"} +categories = toml.load(open('categories.toml')).keys() + +def check_apps(): + + for app, infos in catalog.items(): + + repo_name = infos.get("url", "").split("/")[-1] + if repo_name != app + "_ynh": + yield f"{app}: repo name should be {app}_ynh, not in {repo_name}" + + category = infos.get("category") + if not category: + yield f"{app}: missing category" + if category not in categories: + yield f"{app}: category {category} is not defined in categories.toml" + +errors = list(check_apps()) + +for error in errors: + print(error) + +if errors: + sys.exit(1)