From a286aa3c6137750b3cfe98609a8c8fe689063a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:13:11 +0100 Subject: [PATCH] Commonize code to read catalog, categories etc --- appslib/utils.py | 27 +++++++++++++++++++++- catalog_linter.py | 40 ++++---------------------------- list_builder.py | 58 +++++++++++++++++++---------------------------- 3 files changed, 53 insertions(+), 72 deletions(-) diff --git a/appslib/utils.py b/appslib/utils.py index f80650e..46600af 100644 --- a/appslib/utils.py +++ b/appslib/utils.py @@ -61,7 +61,7 @@ def progressbar( @cache -def get_catalog(working_only=False): +def get_catalog(working_only: bool = False) -> dict[str, dict[str, Any]]: """Load the app catalog and filter out the non-working ones""" catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8")) if working_only: @@ -70,3 +70,28 @@ def get_catalog(working_only=False): if infos.get("state") != "notworking" } return catalog + + +@cache +def get_categories() -> dict[str, Any]: + categories_path = REPO_APPS_ROOT / "categories.toml" + return toml.load(categories_path) + + +@cache +def get_antifeatures() -> dict[str, Any]: + antifeatures_path = REPO_APPS_ROOT / "antifeatures.toml" + return toml.load(antifeatures_path) + + +@cache +def get_wishlist() -> dict[str, dict[str, str]]: + wishlist_path = REPO_APPS_ROOT / "wishlist.toml" + return toml.load(wishlist_path) + + +@cache +def get_graveyard() -> dict[str, dict[str, str]]: + wishlist_path = REPO_APPS_ROOT / "graveyard.toml" + return toml.load(wishlist_path) + diff --git a/catalog_linter.py b/catalog_linter.py index 6ce7ef9..53a05aa 100755 --- a/catalog_linter.py +++ b/catalog_linter.py @@ -3,48 +3,16 @@ import json import sys from difflib import SequenceMatcher -from functools import cache -from pathlib import Path from typing import Any, Dict, Generator, List, Tuple import jsonschema -import toml - -APPS_ROOT = Path(__file__).parent.parent - - -@cache -def get_catalog() -> Dict[str, Dict[str, Any]]: - catalog_path = APPS_ROOT / "apps.toml" - return toml.load(catalog_path) - - -@cache -def get_categories() -> Dict[str, Any]: - categories_path = APPS_ROOT / "categories.toml" - return toml.load(categories_path) - - -@cache -def get_antifeatures() -> Dict[str, Any]: - antifeatures_path = APPS_ROOT / "antifeatures.toml" - return toml.load(antifeatures_path) - - -@cache -def get_wishlist() -> Dict[str, Dict[str, str]]: - wishlist_path = APPS_ROOT / "wishlist.toml" - return toml.load(wishlist_path) - - -@cache -def get_graveyard() -> Dict[str, Dict[str, str]]: - wishlist_path = APPS_ROOT / "graveyard.toml" - return toml.load(wishlist_path) +from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error + get_antifeatures, get_catalog, get_categories, + get_graveyard, get_wishlist) def validate_schema() -> Generator[str, None, None]: - with open(APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file: + with open(REPO_APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file: apps_catalog_schema = json.load(file) validator = jsonschema.Draft202012Validator(apps_catalog_schema) for error in validator.iter_errors(get_catalog()): diff --git a/list_builder.py b/list_builder.py index 1bb947a..d069bda 100755 --- a/list_builder.py +++ b/list_builder.py @@ -20,41 +20,29 @@ from packaging_v2.convert_v1_manifest_to_v2_for_catalog import \ convert_v1_manifest_to_v2_for_catalog # pylint: disable=import-error from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error - get_catalog, git_repo_age) - -# Automatically enables error-to-xmpp -import appslib.xmpplogger # pylint: disable=import-error - - + get_antifeatures, get_catalog, get_categories) now = time.time() -# Load categories and reformat the structure to have a list with an "id" key -categories = toml.load((REPO_APPS_ROOT / "categories.toml").open("r", encoding="utf-8")) -for category_id, infos in categories.items(): - infos["id"] = category_id - for subtag_id, subtag_infos in infos.get("subtags", {}).items(): - subtag_infos["id"] = subtag_id - infos["subtags"] = list(infos.get('subtags', {}).values()) +@cache +def categories_list(): + # Load categories and reformat the structure to have a list with an "id" key + new_categories = get_categories() + for category_id, infos in new_categories.items(): + infos["id"] = category_id + for subtag_id, subtag_infos in infos.get("subtags", {}).items(): + subtag_infos["id"] = subtag_id + infos["subtags"] = list(infos.get('subtags', {}).values()) + return list(new_categories.values()) -categories = list(categories.values()) -# (Same for antifeatures) -antifeatures = toml.load((REPO_APPS_ROOT / "antifeatures.toml").open("r", encoding="utf-8")) -for antifeature_id, infos in antifeatures.items(): - infos["id"] = antifeature_id -antifeatures = list(antifeatures.values()) - -# Load the app catalog and filter out the non-working ones -catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8")) -catalog = { - app: infos for app, infos in catalog.items() if infos.get("state") != "notworking" -} - -my_env = os.environ.copy() -my_env["GIT_TERMINAL_PROMPT"] = "0" - -(REPO_APPS_ROOT / "builds").mkdir(exist_ok=True) +@cache +def antifeatures_list(): + # (Same for antifeatures) + new_antifeatures = get_antifeatures() + for antifeature_id, infos in new_antifeatures.items(): + infos["id"] = antifeature_id + return list(new_antifeatures.values()) ################################ @@ -92,8 +80,8 @@ def write_catalog_v2(base_catalog, target_dir: Path) -> None: } full_catalog = { "apps": result_dict_with_manifest_v1, - "categories": categories, - "antifeatures": antifeatures, + "categories": categories_list(), + "antifeatures": antifeatures_list(), } target_file = target_dir / "apps.json" @@ -130,8 +118,8 @@ def write_catalog_v3(base_catalog, target_dir: Path) -> None: full_catalog = { "apps": result_dict_with_manifest_v2, - "categories": categories, - "antifeatures": antifeatures, + "categories": categories_list(), + "antifeatures": antifeatures_list(), } target_file = target_dir / "apps.json" @@ -166,7 +154,7 @@ def write_catalog_doc(base_catalog, target_dir: Path) -> None: } full_catalog = { "apps": result_dict_doc, - "categories": categories + "categories": categories_list() } target_file = target_dir / "apps.json"