From 97a6013868eaae37f1bb945d869030dc84a2e62b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sun, 24 Sep 2023 18:16:15 +0200 Subject: [PATCH 1/4] Rework the catalog_linter.py: * chmod +x * split into functions * add python typing * Now run jsonschema on apps.toml * Now check category subtags * Now check antifeatures --- catalog_linter.py | 114 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 25 deletions(-) mode change 100644 => 100755 catalog_linter.py diff --git a/catalog_linter.py b/catalog_linter.py old mode 100644 new mode 100755 index 3265989..acd345a --- a/catalog_linter.py +++ b/catalog_linter.py @@ -1,37 +1,101 @@ -import toml +#!/usr/bin/env python3 + +import json import sys +from functools import cache +from pathlib import Path +from typing import Any, Dict, Generator, List, Tuple -errors = [] +import jsonschema +import toml -catalog = toml.load(open('apps.toml')) +APPS_ROOT = Path(__file__).parent.parent -for app, infos in catalog.items(): + +@cache +def get_catalog() -> Dict[str, Dict[str, Any]]: + catalog_path = APPS_ROOT / "apps.toml" + return toml.load(catalog_path) + + +@cache +def get_categories() -> Dict[str, Any]: + categories_path = APPS_ROOT / "categories.toml" + return toml.load(categories_path) + + +@cache +def get_antifeatures() -> Dict[str, Any]: + antifeatures_path = APPS_ROOT / "antifeatures.toml" + return toml.load(antifeatures_path) + + +def validate_schema() -> Generator[str, None, None]: + with open(APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file: + apps_catalog_schema = json.load(file) + validator = jsonschema.Draft202012Validator(apps_catalog_schema) + for error in validator.iter_errors(get_catalog()): + yield f"at .{'.'.join(error.path)}: {error.message}" + + +def check_app(app: str, infos: Dict[str, Any]) -> Generator[str, None, None]: if "state" not in infos: - errors.append(f"{app}: missing state info") + yield "state is missing" + return -catalog = {app: infos for app, infos in catalog.items() if infos.get('state') == "working"} -categories = toml.load(open('categories.toml')).keys() + if infos["state"] != "working": + return + + repo_name = infos.get("url", "").split("/")[-1] + if repo_name != f"{app}_ynh": + yield f"repo name should be {app}_ynh, not in {repo_name}" + + antifeatures = infos.get("antifeatures", []) + for antifeature in antifeatures: + if antifeature not in get_antifeatures(): + yield f"unknown antifeature {antifeature}" + + category = infos.get("category") + if not category: + yield "category is missing" + else: + if category not in get_categories(): + yield f"unknown category {category}" + + subtags = infos.get("subtags", []) + for subtag in subtags: + if subtag not in get_categories()[category].get("subtags", []): + yield f"unknown subtag {category} / {subtag}" -def check_apps(): - - for app, infos in catalog.items(): - - repo_name = infos.get("url", "").split("/")[-1] - if repo_name != app + "_ynh": - yield f"{app}: repo name should be {app}_ynh, not in {repo_name}" - - category = infos.get("category") - if not category: - yield f"{app}: missing category" - if category not in categories: - yield f"{app}: category {category} is not defined in categories.toml" +def check_all_apps() -> Generator[Tuple[str, List[str]], None, None]: + for app, info in get_catalog().items(): + errors = list(check_app(app, info)) + if errors: + yield app, errors -errors = errors + list(check_apps()) +def main() -> None: + has_errors = False -for error in errors: - print(error) + schema_errors = list(validate_schema()) + if schema_errors: + has_errors = True + print("Error while validating catalog against schema:") + for error in schema_errors: + print(f" - {error}") + if schema_errors: + print() -if errors: - sys.exit(1) + for app, errors in check_all_apps(): + has_errors = True + print(f"{app}:") + for error in errors: + print(f" - {error}") + + if has_errors: + sys.exit(1) + + +if __name__ == "__main__": + main() From d26266e1f399fbd5aa3d1cbdea7014655bc4a169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sun, 24 Sep 2023 20:30:04 +0200 Subject: [PATCH 2/4] Pass subtags errors as warnings only --- catalog_linter.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/catalog_linter.py b/catalog_linter.py index acd345a..9ba6689 100755 --- a/catalog_linter.py +++ b/catalog_linter.py @@ -38,7 +38,7 @@ def validate_schema() -> Generator[str, None, None]: yield f"at .{'.'.join(error.path)}: {error.message}" -def check_app(app: str, infos: Dict[str, Any]) -> Generator[str, None, None]: +def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], None, None]: if "state" not in infos: yield "state is missing" return @@ -48,27 +48,27 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[str, None, None]: repo_name = infos.get("url", "").split("/")[-1] if repo_name != f"{app}_ynh": - yield f"repo name should be {app}_ynh, not in {repo_name}" + yield f"repo name should be {app}_ynh, not in {repo_name}", True antifeatures = infos.get("antifeatures", []) for antifeature in antifeatures: if antifeature not in get_antifeatures(): - yield f"unknown antifeature {antifeature}" + yield f"unknown antifeature {antifeature}", True category = infos.get("category") if not category: - yield "category is missing" + yield "category is missing", True else: if category not in get_categories(): - yield f"unknown category {category}" + yield f"unknown category {category}", True subtags = infos.get("subtags", []) for subtag in subtags: - if subtag not in get_categories()[category].get("subtags", []): - yield f"unknown subtag {category} / {subtag}" + if subtag not in get_categories().get(category, {}).get("subtags", []): + yield f"unknown subtag {category} / {subtag}", False -def check_all_apps() -> Generator[Tuple[str, List[str]], None, None]: +def check_all_apps() -> Generator[Tuple[str, List[Tuple[str, bool]]], None, None]: for app, info in get_catalog().items(): errors = list(check_app(app, info)) if errors: @@ -88,10 +88,12 @@ def main() -> None: print() for app, errors in check_all_apps(): - has_errors = True print(f"{app}:") - for error in errors: - print(f" - {error}") + for error, is_fatal in errors: + if is_fatal: + has_errors = True + level = "error" if is_fatal else "warning" + print(f" - {level}: {error}") if has_errors: sys.exit(1) From 4d3b63f2f863869af430de9a20bac26d2ed4d8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sun, 24 Sep 2023 22:12:11 +0200 Subject: [PATCH 3/4] catalog_linter: Fix missing is_fatal on yield --- catalog_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog_linter.py b/catalog_linter.py index 9ba6689..6925cf8 100755 --- a/catalog_linter.py +++ b/catalog_linter.py @@ -40,7 +40,7 @@ def validate_schema() -> Generator[str, None, None]: def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], None, None]: if "state" not in infos: - yield "state is missing" + yield "state is missing", True return if infos["state"] != "working": From cef9fc6a160fe8683bfdf50644f716117ac0c4c7 Mon Sep 17 00:00:00 2001 From: orhtej2 <2871798+orhtej2@users.noreply.github.com> Date: Sun, 24 Sep 2023 23:01:45 +0200 Subject: [PATCH 4/4] Reset YNH integration version to 1 on upstream version bumps --- autoupdate_app_sources/autoupdate_app_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate_app_sources/autoupdate_app_sources.py b/autoupdate_app_sources/autoupdate_app_sources.py index 3e4b4d7..48ff84f 100644 --- a/autoupdate_app_sources/autoupdate_app_sources.py +++ b/autoupdate_app_sources/autoupdate_app_sources.py @@ -385,7 +385,7 @@ class AppAutoUpdater: if is_main: def repl(m): - return m.group(1) + new_version + m.group(3) + return m.group(1) + new_version + "~ynh1" content = re.sub( r"(\s*version\s*=\s*[\"\'])([\d\.]+)(\~ynh\d+[\"\'])", repl, content