From 6ea4acaab2a0ee86764dec3fa3e8a8be79330409 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 13 Jun 2021 04:50:21 +0200 Subject: [PATCH 01/34] [mod] use with tempfile.TemporaryDirectory --- README-generator/webhook.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README-generator/webhook.py b/README-generator/webhook.py index e33f994..5f52c7b 100755 --- a/README-generator/webhook.py +++ b/README-generator/webhook.py @@ -1,6 +1,6 @@ import subprocess import os -import shutil +import tempfile from github_webhook import Webhook from flask import Flask @@ -43,8 +43,7 @@ def on_push(data): repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] - folder = subprocess.check_output(["mktemp", "-d"]).decode('utf-8').strip() - try: + with tempfile.TemporaryDirectory() as folder: git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) @@ -56,9 +55,7 @@ def on_push(data): git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) git(["push", "origin", branch, "--quiet"], in_folder=folder) - finally: - if os.path.exists(folder): - shutil.rmtree(folder) + if __name__ == "__main__": app.run(host="0.0.0.0", port=8123) From b68fb5988e77572b7b7fef5e1533c930efc750b9 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 13 Jun 2021 05:12:34 +0200 Subject: [PATCH 02/34] [mod] port to sanic for perfs --- README-generator/requirements.txt | 4 +- README-generator/webhook.py | 64 ++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/README-generator/requirements.txt b/README-generator/requirements.txt index 3e3ec9d..88cd6e2 100644 --- a/README-generator/requirements.txt +++ b/README-generator/requirements.txt @@ -1,4 +1,2 @@ -argparse jinja2 -github-webhook==1.0.4 -gunicorn==20.1.0 +sanic diff --git a/README-generator/webhook.py b/README-generator/webhook.py index 5f52c7b..b78b36d 100755 --- a/README-generator/webhook.py +++ b/README-generator/webhook.py @@ -1,15 +1,19 @@ -import subprocess import os +import hmac +import shlex +import hashlib +import asyncio import tempfile -from github_webhook import Webhook -from flask import Flask +from sanic import Sanic +from sanic.response import text +from sanic.exceptions import abort + from make_readme import generate_READMEs -app = Flask(__name__) +app = Sanic(__name__) github_webhook_secret = open("github_webhook_secret", "r").read().strip() -webhook = Webhook(app, endpoint="/github", secret=github_webhook_secret) login = open("login").read().strip() token = open("token").read().strip() @@ -22,39 +26,63 @@ my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot" my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" -def git(cmd, in_folder=None): +async def git(cmd, in_folder=None): if not isinstance(cmd, list): cmd = cmd.split() if in_folder: cmd = ["-C", in_folder] + cmd cmd = ["git"] + cmd - return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + cmd = " ".join(map(shlex.quote, cmd)) + command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) + data = await command.stdout.read() + return data.decode().strip() -@app.route("/github") -def main_route(): - return "You aren't supposed to go on this page using a browser, it's for webhooks push instead." +@app.route("/github", methods=["GET"]) +def main_route(request): + return text("You aren't supposed to go on this page using a browser, it's for webhooks push instead.") -@webhook.hook() -def on_push(data): +@app.route("/github", methods=["POST"]) +async def on_push(request): + header_signature = request.headers.get("X-Hub-Signature") + if header_signature is None: + print("no header X-Hub-Signature") + abort(403) + + sha_name, signature = header_signature.split("=") + if sha_name != "sha1": + print("signing algo isn't sha1, it's '%s'" % sha_name) + abort(501) + + # HMAC requires the key to be bytes, but data is string + mac = hmac.new(github_webhook_secret.encode(), msg=request.body, digestmod=hashlib.sha1) + + if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): + abort(403) + + data = request.json repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] with tempfile.TemporaryDirectory() as folder: - git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) + await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) - git(["add", "README*.md"], in_folder=folder) + await git(["add", "README*.md"], in_folder=folder) - diff_not_empty = bool(subprocess.check_output(["git", "diff", "HEAD", "--compact-summary"], cwd=folder).strip().decode("utf-8")) + diff_not_empty = await asyncio.create_subprocess_shell(" ".join(["git", "diff", "HEAD", "--compact-summary"]), cwd=folder, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) + diff_not_empty = await diff_not_empty.stdout.read() + diff_not_empty = diff_not_empty.decode().strip() if not diff_not_empty: - return + return text("nothing to do") - git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) - git(["push", "origin", branch, "--quiet"], in_folder=folder) + await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) + await git(["push", "origin", branch, "--quiet"], in_folder=folder) + + return text("ok") if __name__ == "__main__": From 559f5ee0b38a773a2931adc799a26092f80d0336 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 14 Jun 2021 16:34:26 +0200 Subject: [PATCH 03/34] [mod] add some debug logging --- README-generator/webhook.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README-generator/webhook.py b/README-generator/webhook.py index b78b36d..a9165e6 100755 --- a/README-generator/webhook.py +++ b/README-generator/webhook.py @@ -34,6 +34,7 @@ async def git(cmd, in_folder=None): cmd = ["-C", in_folder] + cmd cmd = ["git"] + cmd cmd = " ".join(map(shlex.quote, cmd)) + print(cmd) command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) data = await command.stdout.read() return data.decode().strip() @@ -67,6 +68,8 @@ async def on_push(request): repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] + print(f"{repository} -> branch '{branch}'") + with tempfile.TemporaryDirectory() as folder: await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) @@ -77,6 +80,7 @@ async def on_push(request): diff_not_empty = await diff_not_empty.stdout.read() diff_not_empty = diff_not_empty.decode().strip() if not diff_not_empty: + print("nothing to do") return text("nothing to do") await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) From f2e5574c45cc3be0107006d1c1e60cdef732bcf1 Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Sun, 22 Aug 2021 16:57:07 +0200 Subject: [PATCH 04/34] Add description --- README-generator/make_readme.py | 9 +++++++++ README-generator/templates/README.md.j2 | 2 +- README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index 5491860..ddcc3d2 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -30,6 +30,14 @@ def generate_READMEs(app_path: str): template = env.get_template(f"README{lang_suffix}.md.j2") + if (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").exists(): + description = (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").read_text() + # Fallback to english if maintainer too lazy to translate the description + elif (app_path / "doc" / "DESCRIPTION.md").exists(): + description = (app_path / "doc" / "DESCRIPTION.md").read_text() + else: + description = None + if (app_path / "doc" / "screenshots").exists(): screenshots = os.listdir(os.path.join(app_path, "doc", "screenshots")) if ".gitkeep" in screenshots: @@ -48,6 +56,7 @@ def generate_READMEs(app_path: str): out = template.render( lang=lang, upstream=upstream, + description=description, screenshots=screenshots, disclaimer=disclaimer, manifest=manifest, diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 9ceda16..6e58002 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -29,7 +29,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Overview -{{manifest.description[lang]}} +{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} **Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 2fa3850..9169cbb 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -11,7 +11,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Vue d'ensemble -{{manifest.description[lang]}} +{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} **Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} From d13cda4951cc7f0278d88b694b20f788ded05706 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Sat, 4 Sep 2021 00:42:51 +0200 Subject: [PATCH 05/34] Fix author and committer name for README auto-update --- README-generator/webhook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README-generator/webhook.py b/README-generator/webhook.py index a9165e6..a2183df 100755 --- a/README-generator/webhook.py +++ b/README-generator/webhook.py @@ -20,9 +20,9 @@ token = open("token").read().strip() my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" -my_env["GIT_AUTHOR_NAME"] = "Yunohost-Bot" +my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" -my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot" +my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" From 11301cd9e2223048f09ea569a461a93c1ff7d021 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Tue, 26 Oct 2021 23:00:46 +0200 Subject: [PATCH 06/34] Homogenize bot's name/email b/w author/committer --- README-generator/webhook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-generator/webhook.py b/README-generator/webhook.py index a2183df..5f295de 100755 --- a/README-generator/webhook.py +++ b/README-generator/webhook.py @@ -83,7 +83,7 @@ async def on_push(request): print("nothing to do") return text("nothing to do") - await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) + await git(["commit", "-a", "-m", "Auto-update README", "--author='yunohost-bot '"], in_folder=folder) await git(["push", "origin", branch, "--quiet"], in_folder=folder) return text("ok") From 84ccf50f604b15badd2c27e76b2e1a3e5739f242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Mennetrier?= Date: Fri, 29 Oct 2021 09:17:02 +0200 Subject: [PATCH 07/34] Add Readme generator venv in gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 12dfc78..55c973f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .apps_cache builds +tools/README-generator/venv/ From 306d447e972ba359873eff253b9ad40b7d492b87 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Tue, 22 Feb 2022 23:44:44 +0000 Subject: [PATCH 08/34] Add a warning when not on a master branch --- README-generator/make_readme.py | 4 ++++ README-generator/templates/README.md.j2 | 2 +- README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index ddcc3d2..d00e1ed 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -53,6 +53,9 @@ def generate_READMEs(app_path: str): else: disclaimer = None + # Get the current branch using git inside the app path + branch = os.popen("git --git-dir=%s/.git --work-tree=%s rev-parse --abbrev-ref HEAD" % (app_path, app_path)).read().strip() + out = template.render( lang=lang, upstream=upstream, @@ -60,6 +63,7 @@ def generate_READMEs(app_path: str): screenshots=screenshots, disclaimer=disclaimer, manifest=manifest, + branch=branch, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 6e58002..b0eb5c2 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -31,7 +31,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(This is the `{{ branch }}` branch. See the [`master` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) for the version in the catalog.)*{% endif %} {% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 9169cbb..63ca8c4 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -13,7 +13,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `master`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) pour la version du catalogue.)*{% endif %} {% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %} From cf0343f6a178047274da2003bc96d838d0215b56 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 23 Feb 2022 06:36:19 +0000 Subject: [PATCH 09/34] Get the default branch from the apps.json --- README-generator/make_readme.py | 4 ++++ README-generator/templates/README.md.j2 | 2 +- README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index d00e1ed..4b34ef5 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -18,6 +18,9 @@ def generate_READMEs(app_path: str): manifest = json.load(open(app_path / "manifest.json")) upstream = manifest.get("upstream", {}) + catalog = json.load(open(Path(__file__).parent.parent.parent / "apps.json")) + from_catalog = catalog.get(manifest['id'], {}) + if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( "There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet." @@ -64,6 +67,7 @@ def generate_READMEs(app_path: str): disclaimer=disclaimer, manifest=manifest, branch=branch, + default_branch=from_catalog.get('branch', 'master'), ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index b0eb5c2..a0fdf07 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -31,7 +31,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(This is the `{{ branch }}` branch. See the [`master` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) for the version in the catalog.)*{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(This is the `{{ branch }}` branch. See the [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) for the version in the catalog.)*{% endif %} {% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 63ca8c4..14bd727 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -13,7 +13,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `master`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) pour la version du catalogue.)*{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) pour la version du catalogue.)*{% endif %} {% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %} From 388337d8bf3686bb5696ecd20808aff3259bcf43 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 23 Feb 2022 07:07:45 +0000 Subject: [PATCH 10/34] fix --- README-generator/make_readme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index 4b34ef5..ac47a04 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -18,7 +18,7 @@ def generate_READMEs(app_path: str): manifest = json.load(open(app_path / "manifest.json")) upstream = manifest.get("upstream", {}) - catalog = json.load(open(Path(__file__).parent.parent.parent / "apps.json")) + catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) from_catalog = catalog.get(manifest['id'], {}) if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): From 448dd80c5f1ec2af2487bc8a75ee07efbb0c8399 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 23 Feb 2022 07:18:05 +0000 Subject: [PATCH 11/34] Add Anti-Features in READMEs --- README-generator/make_readme.py | 12 ++++++++++++ README-generator/requirements.txt | 1 + README-generator/templates/README.md.j2 | 8 ++++++++ README-generator/templates/README_fr.md.j2 | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index ddcc3d2..3e8be5a 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -3,6 +3,7 @@ import argparse import json import os +import yaml from pathlib import Path from jinja2 import Environment, FileSystemLoader @@ -18,6 +19,11 @@ def generate_READMEs(app_path: str): manifest = json.load(open(app_path / "manifest.json")) upstream = manifest.get("upstream", {}) + catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) + from_catalog = catalog.get(manifest['id'], {}) + + antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yaml")) + if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( "There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet." @@ -53,6 +59,12 @@ def generate_READMEs(app_path: str): else: disclaimer = None + # TODO: Add url to the documentation... and actually create that documentation :D + antifeatures = [antifeatures_list[a] for a in from_catalog.get('antifeatures', [])] + for antifeature in antifeatures: + antifeature['title'] = antifeature['title'].get(lang_suffix, 'en') + antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') + out = template.render( lang=lang, upstream=upstream, diff --git a/README-generator/requirements.txt b/README-generator/requirements.txt index 88cd6e2..33fe25a 100644 --- a/README-generator/requirements.txt +++ b/README-generator/requirements.txt @@ -1,2 +1,3 @@ jinja2 sanic +pyyaml diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 6e58002..8713a9a 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -49,6 +49,14 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {{ disclaimer }} {% endif -%} +{% if antifeatures -%} +## Antifeatures + +{% for antifeature in antifeatures -%} + - **{{ antifeature.title }}**: {{ antifeature.description }} +{% endfor -%} +{% endif -%} + ## Documentation and resources {% if upstream.website -%}* Official app website: {{ upstream.website }} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 9169cbb..9e89c10 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -31,6 +31,14 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {{ disclaimer }} {% endif -%} +{% if antifeatures -%} +## Fonctions indésirables + +{% for antifeature in antifeatures -%} + - **{{ antifeature.title }}**: {{ antifeature.description }} +{% endfor -%} +{% endif -%} + ## Documentations et ressources {% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }} From b14cb5e82a3b39fc04c6a1ece5d6d9c0fa5a2074 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 00:13:27 +0200 Subject: [PATCH 12/34] Readmes improvements --- README-generator/templates/README.md.j2 | 8 +++++--- README-generator/templates/README_fr.md.j2 | 14 ++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 6e58002..19f87ef 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -19,7 +19,7 @@ It shall NOT be edited by hand. # {{manifest.name}} for YunoHost -[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Working status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Maintenance status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Install {{manifest.name}} with YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Lire ce readme en français.](./README_fr.md)* @@ -39,7 +39,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Screenshots {% for screenshot in screenshots -%} - ![](./doc/screenshots/{{screenshot}}) + ![Screenshot of {{manifest.name}}](./doc/screenshots/{{screenshot}}) {% endfor %} {% endif -%} @@ -67,10 +67,12 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in Please send your pull request to the [testing branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing). To try the testing branch, please proceed like that. -``` + +``` bash sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug or sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` **More info regarding app packaging:** https://yunohost.org/packaging_apps + diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 9169cbb..f531abe 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -1,10 +1,14 @@ + + # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Status du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* -*[Lire ce readme en français.](./README_fr.md)* > *Ce package vous permet d'installer {{manifest.name}} rapidement et simplement sur un serveur YunoHost. Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour savoir comment l'installer et en profiter.* @@ -21,7 +25,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Captures d'écran {% for screenshot in screenshots -%} - ![](./doc/screenshots/{{screenshot}}) + ![Capture d'écran de {{manifest.name}}](./doc/screenshots/{{screenshot}}) {% endfor %} {% endif -%} @@ -49,10 +53,12 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour Merci de faire vos pull request sur la [branche testing](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing). Pour essayer la branche testing, procédez comme suit. -``` + +``` bash sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ou sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` **Plus d'infos sur le packaging d'applications :** https://yunohost.org/packaging_apps + From 9b890e36b93f2b654be324dcb1e801449a50f08c Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 01:19:44 +0200 Subject: [PATCH 13/34] layout corrections (line breaks) --- README-generator/templates/README.md.j2 | 9 ++++++--- README-generator/templates/README_fr.md.j2 | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 19f87ef..e0f09d0 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -31,11 +31,14 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} -{% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %} +{% if upstream.demo %} +**Demo:** {{upstream.demo}} +{% endif -%} -{% if screenshots -%} +{% if screenshots %} ## Screenshots {% for screenshot in screenshots -%} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index f531abe..a15ba8c 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -17,11 +17,13 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} -{% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %} +{% if upstream.demo %}**Démo :** {{upstream.demo}} +{% endif -%} -{% if screenshots -%} +{% if screenshots %} ## Captures d'écran {% for screenshot in screenshots -%} From 6c0a2a50fb772692d91fe5ca658eef4fb3e56cfa Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 01:25:17 +0200 Subject: [PATCH 14/34] forgotten layout corrections (line breaks) --- README-generator/templates/README_fr.md.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index a15ba8c..de77988 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -20,7 +20,8 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour **Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} {% endif -%} -{% if upstream.demo %}**Démo :** {{upstream.demo}} +{% if upstream.demo %} +**Démo :** {{upstream.demo}} {% endif -%} {% if screenshots %} From d4ff6d83a11b2a5f8dcf496613cf7f40d22d3207 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 18:15:34 +0200 Subject: [PATCH 15/34] add angle brackets around bare URLs (fix MD034) --- README-generator/templates/README.md.j2 | 14 +++++++------- README-generator/templates/README_fr.md.j2 | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index e0f09d0..4931754 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -54,16 +54,16 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Documentation and resources -{% if upstream.website -%}* Official app website: {{ upstream.website }} +{% if upstream.website -%}* Official app website: <{{ upstream.website }}> {% endif -%} -{% if upstream.userdoc -%}* Official user documentation: {{ upstream.userdoc }} +{% if upstream.userdoc -%}* Official user documentation: <{{ upstream.userdoc }}> {% endif -%} -{% if upstream.admindoc -%}* Official admin documentation: {{ upstream.admindoc }} +{% if upstream.admindoc -%}* Official admin documentation: <{{ upstream.admindoc }}> {% endif -%} -{% if upstream.code -%}* Upstream app code repository: {{ upstream.code }} +{% if upstream.code -%}* Upstream app code repository: <{{ upstream.code }}> {% endif -%} -* YunoHost documentation for this app: https://yunohost.org/app_{{manifest.id}} -* Report a bug: https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues +* YunoHost documentation for this app: +* Report a bug: ## Developer info @@ -77,5 +77,5 @@ or sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` -**More info regarding app packaging:** https://yunohost.org/packaging_apps +**More info regarding app packaging:** diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index de77988..6f383ae 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -40,16 +40,16 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Documentations et ressources -{% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }} +{% if upstream.website -%}* Site officiel de l'app : <{{ upstream.website }}> {% endif -%} -{% if upstream.userdoc -%}* Documentation officielle utilisateur : {{ upstream.userdoc }} +{% if upstream.userdoc -%}* Documentation officielle utilisateur : <{{ upstream.userdoc }}> {% endif -%} -{% if upstream.admindoc -%}* Documentation officielle de l'admin : {{ upstream.admindoc }} +{% if upstream.admindoc -%}* Documentation officielle de l'admin : <{{ upstream.admindoc }}> {% endif -%} -{% if upstream.code -%}* Dépôt de code officiel de l'app : {{ upstream.code }} +{% if upstream.code -%}* Dépôt de code officiel de l'app : <{{ upstream.code }}> {% endif -%} -* Documentation YunoHost pour cette app : https://yunohost.org/app_{{manifest.id}} -* Signaler un bug : https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues +* Documentation YunoHost pour cette app : +* Signaler un bug : ## Informations pour les développeurs @@ -63,5 +63,5 @@ ou sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` -**Plus d'infos sur le packaging d'applications :** https://yunohost.org/packaging_apps +**Plus d'infos sur le packaging d'applications :** From d9227835800dff0b1d885b40dfd3fcb0e5864688 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sat, 21 May 2022 16:09:09 +0200 Subject: [PATCH 16/34] harmonising the style of lists (MD004) --- README-generator/templates/README.md.j2 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 4931754..78a1139 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -1,13 +1,13 @@ {% if manifest.id == "example" -%} # Packaging your an app, starting from this example -- Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. -- Edit the `manifest.json` with app specific info. -- Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`. - - Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers) -- Add a `LICENSE` file for the package. -- Edit `doc/DISCLAIMER*.md` -- The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator +* Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. +* Edit the `manifest.json` with app specific info. +* Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`. + * Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers) +* Add a `LICENSE` file for the package. +* Edit `doc/DISCLAIMER*.md` +* The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator --- {% endif -%} From 8dd2a460a494e9790822cb124f9f53a6dfa8c6c5 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 25 May 2022 09:26:35 +0200 Subject: [PATCH 17/34] =?UTF-8?q?Coquille=20dans=20le=20g=C3=A9n=C3=A9rate?= =?UTF-8?q?ur=20de=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: nicofrand --- README-generator/templates/README_fr.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 6f383ae..7c2bf63 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -5,7 +5,7 @@ It shall NOT be edited by hand. # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Status du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* From 04fa9897d01319c7c898a2707cef296b2d3f789f Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 25 May 2022 10:04:17 +0200 Subject: [PATCH 18/34] =?UTF-8?q?Coquille=20dans=20le=20g=C3=A9n=C3=A9rate?= =?UTF-8?q?ur=20de=20README,=20la=20suite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: nicofrand --- README-generator/templates/README_fr.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 7c2bf63..ae51374 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -5,7 +5,7 @@ It shall NOT be edited by hand. # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Statut de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* From 1d19f28c9a8f51ae0bb27b517fbd13d778cc2644 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Mon, 30 May 2022 11:59:54 +0000 Subject: [PATCH 19/34] fix --- README-generator/make_readme.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index 3e8be5a..788b8a7 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -22,7 +22,8 @@ def generate_READMEs(app_path: str): catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) from_catalog = catalog.get(manifest['id'], {}) - antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yaml")) + antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml")) + antifeatures_list = { e['id']: e for e in antifeatures_list } if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( @@ -60,10 +61,13 @@ def generate_READMEs(app_path: str): disclaimer = None # TODO: Add url to the documentation... and actually create that documentation :D - antifeatures = [antifeatures_list[a] for a in from_catalog.get('antifeatures', [])] - for antifeature in antifeatures: - antifeature['title'] = antifeature['title'].get(lang_suffix, 'en') - antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') + antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} + for k, v in antifeatures: + v['title'] = v['title'].get(lang_suffix, 'en') + if manifest.get("antifeatures", {}).get(k, 'en'): + v['description'] = manifest.get("antifeatures", {}).get(k, 'en') + else: + antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') out = template.render( lang=lang, From 7507671f7a854f8ae430345209583b0652aa128c Mon Sep 17 00:00:00 2001 From: yalh76 Date: Tue, 7 Jun 2022 22:48:50 +0200 Subject: [PATCH 20/34] Update README.md.j2 --- README-generator/templates/README.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 78a1139..d04393b 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -1,5 +1,5 @@ {% if manifest.id == "example" -%} -# Packaging your an app, starting from this example +# Packaging an app, starting from this example * Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. * Edit the `manifest.json` with app specific info. From a88e5eef3e66cba09b3265241580f20cf3c42085 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 21:02:49 +0200 Subject: [PATCH 21/34] Yoloadd an app catalog v3 + tools to convert apps from v1 to v2 --- __init__.py | 0 packaging_v2/__init__.py | 0 packaging_v2/convert_app_to_packaging_v2.py | 321 ++++++++++++++++++ .../convert_v1_manifest_to_v2_for_catalog.py | 55 +++ 4 files changed, 376 insertions(+) create mode 100644 __init__.py create mode 100644 packaging_v2/__init__.py create mode 100644 packaging_v2/convert_app_to_packaging_v2.py create mode 100644 packaging_v2/convert_v1_manifest_to_v2_for_catalog.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packaging_v2/__init__.py b/packaging_v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packaging_v2/convert_app_to_packaging_v2.py b/packaging_v2/convert_app_to_packaging_v2.py new file mode 100644 index 0000000..fd66325 --- /dev/null +++ b/packaging_v2/convert_app_to_packaging_v2.py @@ -0,0 +1,321 @@ +import argparse +import os +import re +import json +import subprocess + + +def check_output(cmd): + return ( + subprocess.check_output(cmd, shell=True) + .decode("utf-8") + .strip() + ) + + +def _convert_v1_manifest_to_v2(app_path): + + manifest = json.load(open(app_path + "/manifest.json")) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost"), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + "disk": "50M", + "ram.build": "50M", + "ram.runtime": "50M" + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + # Rename is_public to init_main_permission + manifest["install"] = {(k if k != "is_public" else "init_main_permission"): v for k, v in manifest["install"].items()} + + if "init_main_permission" in manifest["install"]: + manifest["install"]["init_main_permission"]["type"] = "group" + if manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "visitors" + elif manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "all_users" + + if "domain" in manifest["install"] and "path" not in manifest["install"]: + manifest["install"]["domain"]["full_domain"] = True + + manifest["resources"] = {} + manifest["resources"]["system_user"] = {} + manifest["resources"]["install_dir"] = {} + + if os.system(f"grep -q 'datadir=' {app_path}/scripts/install") == 0: + manifest["resources"]["data_dir"] = {} + + manifest["resources"]["permissions"] = {} + + if os.system(f"grep -q 'ynh_webpath_register' '{app_path}/scripts/install'") == 0: + manifest["resources"]["permissions"]["main.url"] = "/" + + # FIXME: Parse ynh_permission_create --permission="admin" --url="/wp-login.php" --additional_urls="/wp-admin.php" --allowed=$admin_wordpress + + ports = check_output(f"sed -nr 's/(\\w+)=.*ynh_find_port[^0-9]*([0-9]+)\\)/\\1,\\2/p' '{app_path}/scripts/install'") + if ports: + manifest["resources"]["ports"] = {} + for port in ports.split("\n"): + name, default = port.split(",") + exposed = check_output(f"sed -nr 's/.*yunohost firewall allow .*(TCP|UDP|Both).*${name}/\\1/p' '{app_path}/scripts/install'") + if exposed == "Both": + exposed = True + + name = name.replace("_port", "").replace("port_", "") + if name == "port": + name = "main" + + manifest["resources"]["ports"][f"{name}.default"] = int(default) + if exposed: + manifest["resources"]["ports"][f"{name}.exposed"] = exposed + + maybequote = "[\"'\"'\"']?" + apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d '\"' | sed 's@ @\\n@g'") + php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d \"\\\"'\"") + if apt_dependencies.strip(): + if php_version: + apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version) + apt_dependencies = ', '.join([d for d in apt_dependencies.split("\n") if d]) + manifest["resources"]["apt"] = {"packages": apt_dependencies} + + extra_apt_repos = check_output(r"sed -nr 's/.*_extra_app_dependencies.*repo=\"(.*)\".*package=\"(.*)\".*key=\"(.*)\"/\1,\2,\3/p' %s/scripts/install" % app_path) + if extra_apt_repos: + for i, extra_apt_repo in enumerate(extra_apt_repos.split("\n")): + repo, packages, key = extra_apt_repo.split(",") + packages = packages.replace('$', '#FIXME#$') + if "apt" not in manifest["resources"]: + manifest["resources"]["apt"] = {} + if "extras" not in manifest["resources"]["apt"]: + manifest["resources"]["apt"]["extras"] = [] + manifest["resources"]["apt"]["extras"].append({ + "repo": repo, + "key": key, + "packages": packages, + }) + + if os.system(f"grep -q 'ynh_mysql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "mysql"} + elif os.system(f"grep -q 'ynh_psql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "postgresql"} + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + + return manifest + + +def _dump_v2_manifest_as_toml(manifest): + + import re + from tomlkit import document, nl, table, dumps, comment + + toml_manifest = document() + toml_manifest.add("packaging_format", 2) + toml_manifest.add(nl()) + toml_manifest.add("id", manifest["id"]) + toml_manifest.add("name", manifest["name"]) + for lang, value in manifest["description"].items(): + toml_manifest.add(f"description.{lang}", value) + toml_manifest.add(nl()) + toml_manifest.add("version", manifest["version"]) + toml_manifest.add(nl()) + toml_manifest.add("maintainers", manifest["maintainers"]) + + upstream = table() + for key, value in manifest["upstream"].items(): + upstream[key] = value + toml_manifest["upstream"] = upstream + + integration = table() + for key, value in manifest["integration"].items(): + integration.add(key, value) + integration["architectures"].comment('FIXME: can be replaced by a list of supported archs using the dpkg --print-architecture nomenclature (amd64/i386/armhf/arm64/armel), for example: ["amd64", "i386"]') + integration["ldap"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["sso"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["disk"].comment('FIXME: replace with an **estimate** minimum disk requirement. e.g. 20M, 400M, 1G, ...') + integration["ram.build"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + integration["ram.runtime"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + toml_manifest["integration"] = integration + + install = table() + for key, value in manifest["install"].items(): + install[key] = table() + install[key].indent(4) + + if key in ["domain", "path", "admin", "is_public", "password"]: + install[key].add(comment("this is a generic question - ask strings are automatically handled by Yunohost's core")) + + for lang, value2 in value.get("ask", {}).items(): + install[key].add(f"ask.{lang}", value2) + + for lang, value2 in value.get("help", {}).items(): + install[key].add(f"help.{lang}", value2) + + for key2, value2 in value.items(): + if key2 in ["ask", "help"]: + continue + install[key].add(key2, value2) + + toml_manifest["install"] = install + + resources = table() + for key, value in manifest["resources"].items(): + resources[key] = table() + resources[key].indent(4) + for key2, value2 in value.items(): + resources[key].add(key2, value2) + if key == "apt" and key2 == "extras": + for extra in resources[key][key2]: + extra.indent(8) + + toml_manifest["resources"] = resources + + toml_manifest_dump = dumps(toml_manifest) + + regex = re.compile(r'\"((description|ask|help)\.[a-z]{2})\"') + toml_manifest_dump = regex.sub(r'\1', toml_manifest_dump) + toml_manifest_dump = toml_manifest_dump.replace('"ram.build"', "ram.build") + toml_manifest_dump = toml_manifest_dump.replace('"ram.runtime"', "ram.runtime") + toml_manifest_dump = toml_manifest_dump.replace('"main.url"', "main.url") + toml_manifest_dump = toml_manifest_dump.replace('"main.default"', "main.default") + return toml_manifest_dump + + +def cleanup_scripts_and_conf(folder): + + patterns_to_remove_in_scripts = [ + "^.*ynh_abort_if_errors.*$", + "^.*YNH_APP_ARG.*$", + "^.*YNH_APP_INSTANCE_NAME.*$", + r"^ *final_path=", + r"^\s*final_path=", + "^.*test .*-(e|d) .*final_path.*$", + "^.*ynh_webpath_register.*$", + "^.*ynh_webpath_available.*$", + "^.*ynh_system_user_create.*$", + "^.*ynh_system_user_delete.*$", + "^.*ynh_permission_update.*$", + "^.*ynh_permission_create.*$", + "^.*if .*ynh_permission_exists.*$", + "^.*if .*ynh_legacy_permissions_exists.*$", + "^.*ynh_legacy_permissions_delete_all.*$", + "^.*ynh_app_setting_set .*(domain|path|final_path|admin|password|port|datadir|db_name|db_user|db_pwd).*$", + "^.*ynh_app_setting_.* is_public.*$", + r"^.*if.*\$is_public.*$", + "^.*_dependencies=.*$", + "^.*ynh_install_app_dependencies.*$", + "^.*ynh_install_extra_app_dependencies.*$", + "^.*ynh_remove_app_dependencies.*$", + r"^.*\$\(ynh_app_setting_get.*$", + r"^.*ynh_secure_remove .*\$final_path.*$", + r"^.*ynh_secure_remove .*\$datadir.*$", + "^.*ynh_backup_before_upgrade.*$", + "^.*ynh_clean_setup.*$", + "^.*ynh_restore_upgradebackup.*$", + "^db_name=.*$", + "^db_user=.*$", + "^db_pwd=.*$", + "^datadir=.*$", + "^.*ynh_psql_test_if_first_run.*$", + "^.*ynh_mysql_setup_db.*$", + "^.*ynh_psql_setup_db.*$", + "^.*ynh_mysql_remove_db.*$", + "^.*ynh_psql_remove_db.*$", + "^.*ynh_find_port.*$", + "^.*ynh_script_progression.*Finding an available port", + "^.*ynh_script_progression.*Backing up the app before upgrading", + "^.*ynh_script_progression.*Creating data directory", + "^.*ynh_script_progression.*system user", + "^.*ynh_script_progression.*installation settings", + "^.*ynh_print_info.*installation settings", + r"^.*ynh_script_progression.*\w+ dependencies", + "^.*ynh_script_progression.*Removing app main dir", + "^.*ynh_script_progression.*Validating.*parameters", + "^.*ynh_script_progression.*SQL database", + "^.*ynh_script_progression.*Configuring permissions", + ] + patterns_to_remove_in_scripts = [re.compile(f"({p})", re.MULTILINE) for p in patterns_to_remove_in_scripts] + + replaces = [ + ("path_url", "path"), + ("PATH_URL", "PATH"), + ("final_path", "install_dir"), + ("FINALPATH", "INSTALL_DIR"), + ("datadir", "data_dir"), + ("DATADIR", "DATA_DIR"), + ] + + for s in ["_common.sh", "install", "remove", "upgrade", "backup", "restore"]: + + script = f"{folder}/scripts/{s}" + + content = open(script).read() + + for pattern in patterns_to_remove_in_scripts: + content = pattern.sub(r"#REMOVEME? \1", content) + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + open(script, "w").write(content) + + for conf in os.listdir(f"{folder}/conf"): + + conf = f"{folder}/conf/{conf}" + + if not os.path.isfile(conf): + continue + + content = open(conf).read() + content_init = content + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + if content_init != content: + open(conf, "w").write(content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Attempt to automatically convert a v1 YunoHost app to v2 (at least as much as possible) : parse the app scripts to auto-generate the manifest.toml, and remove now-useless lines from the app scripts" + ) + parser.add_argument( + "app_path", help="Path to the app to convert" + ) + + args = parser.parse_args() + + manifest = _convert_v1_manifest_to_v2(args.app_path) + open(args.app_path + "/manifest.toml", "w").write(_dump_v2_manifest_as_toml(manifest)) + + cleanup_scripts_and_conf(args.app_path) diff --git a/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py b/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py new file mode 100644 index 0000000..29f24ee --- /dev/null +++ b/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py @@ -0,0 +1,55 @@ +import copy + + +def convert_v1_manifest_to_v2_for_catalog(manifest): + + manifest = copy.deepcopy(manifest) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + "disk": "50M", + "ram": {"build": "50M", "runtime": "10M"} + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + manifest["resources"] = { + "system_user": {}, + "install_dir": { + "alias": "final_path" + } + } + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + + return manifest From 549724942d6e85b2aafc32e5062662921ffce1d1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 21:31:17 +0200 Subject: [PATCH 22/34] Fix my previous commit /o\ --- packaging_v2/convert_app_to_packaging_v2.py | 9 +++++++-- packaging_v2/convert_v1_manifest_to_v2_for_catalog.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packaging_v2/convert_app_to_packaging_v2.py b/packaging_v2/convert_app_to_packaging_v2.py index fd66325..21fe774 100644 --- a/packaging_v2/convert_app_to_packaging_v2.py +++ b/packaging_v2/convert_app_to_packaging_v2.py @@ -37,8 +37,13 @@ def _convert_v1_manifest_to_v2(app_path): "ram.runtime": "50M" } - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] + maintainers = manifest.get("maintainer", {}) + if isinstance(maintainers, list): + maintainers = [m['name'] for m in maintainers] + else: + maintainers = [maintainers["name"]] if maintainers.get("name") else [] + + manifest["maintainers"] = maintainers install_questions = manifest["arguments"]["install"] manifest["install"] = {} diff --git a/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py b/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py index 29f24ee..0130c29 100644 --- a/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py +++ b/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py @@ -24,8 +24,13 @@ def convert_v1_manifest_to_v2_for_catalog(manifest): "ram": {"build": "50M", "runtime": "10M"} } - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] + maintainers = manifest.get("maintainer", {}) + if isinstance(maintainers, list): + maintainers = [m['name'] for m in maintainers] + else: + maintainers = [maintainers["name"]] if maintainers.get("name") else [] + + manifest["maintainers"] = maintainers install_questions = manifest["arguments"]["install"] From 55e2b5d0dd33470ab4ab09ca157318afe0b2fb68 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 11:17:46 +0200 Subject: [PATCH 23/34] move autopatches/ inside tools/ --- autopatches/autopatch.py | 188 ++++++++++++++++++ .../patches/issue-and-pr-template/patch.sh | 14 ++ .../patches/issue-and-pr-template/pr_body.md | 2 + .../patches/issue-and-pr-template/pr_title.md | 1 + .../missing-seteu-in-change_url/patch.sh | 10 + .../missing-seteu-in-change_url/pr_body.md | 2 + .../missing-seteu-in-change_url/pr_title.md | 1 + .../patches/new-permission-system/patch.sh | 66 ++++++ .../patches/new-permission-system/pr_body.md | 11 + .../patches/new-permission-system/pr_title.md | 1 + 10 files changed, 296 insertions(+) create mode 100755 autopatches/autopatch.py create mode 100644 autopatches/patches/issue-and-pr-template/patch.sh create mode 100644 autopatches/patches/issue-and-pr-template/pr_body.md create mode 100644 autopatches/patches/issue-and-pr-template/pr_title.md create mode 100644 autopatches/patches/missing-seteu-in-change_url/patch.sh create mode 100644 autopatches/patches/missing-seteu-in-change_url/pr_body.md create mode 100644 autopatches/patches/missing-seteu-in-change_url/pr_title.md create mode 100644 autopatches/patches/new-permission-system/patch.sh create mode 100644 autopatches/patches/new-permission-system/pr_body.md create mode 100644 autopatches/patches/new-permission-system/pr_title.md diff --git a/autopatches/autopatch.py b/autopatches/autopatch.py new file mode 100755 index 0000000..746ee51 --- /dev/null +++ b/autopatches/autopatch.py @@ -0,0 +1,188 @@ +#!/usr/bin/python3 +import json +import sys +import requests +import os +import subprocess + +catalog = requests.get("https://raw.githubusercontent.com/YunoHost/apps/master/apps.json").json() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +os.makedirs(".apps_cache", exist_ok=True) + +login = open("login").read().strip() +token = open("token").read().strip() +github_api = "https://api.github.com" + + +def apps(min_level=4): + + for app, infos in catalog.items(): + if infos.get("state") == "working" and infos.get("level", -1) > min_level: + infos["id"] = app + yield infos + + +def app_cache_folder(app): + return os.path.join(".apps_cache", app) + + +def git(cmd, in_folder=None): + + if not isinstance(cmd, list): + cmd = cmd.split() + if in_folder: + cmd = ["-C", in_folder] + cmd + cmd = ["git"] + cmd + return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + + +# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 +def progressbar(it, prefix="", size=60, file=sys.stdout): + it = list(it) + count = len(it) + def show(j, name=""): + name += " " + x = int(size*j/count) + file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name)) + file.flush() + show(0) + for i, item in enumerate(it): + yield item + show(i+1, item["id"]) + file.write("\n") + file.flush() + + +def build_cache(): + + for app in progressbar(apps(), "Git cloning: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + reponame = app["url"].rsplit("/", 1)[-1] + git(f"clone --quiet --depth 1 --single-branch {app['url']} {folder}") + git(f"remote add fork https://{login}:{token}@github.com/{login}/{reponame}", in_folder=folder) + + +def apply(patch): + + patch_path = os.path.abspath(os.path.join("patches", patch, "patch.sh")) + + for app in progressbar(apps(), "Apply to: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset --hard origin/{current_branch}", in_folder=folder) + os.system(f"cd {folder} && bash {patch_path}") + + +def diff(): + + for app in apps(): + folder = os.path.join(".apps_cache", app["id"]) + if bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")): + print("\n\n\n") + print("=================================") + print("Changes in : " + app["id"]) + print("=================================") + print("\n") + os.system(f"cd {folder} && git --no-pager diff") + + +def push(patch): + + title = "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip() + + def diff_not_empty(app): + folder = os.path.join(".apps_cache", app["id"]) + return bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")) + + def app_is_on_github(app): + return "github.com" in app["url"] + + apps_to_push = [app for app in apps() if diff_not_empty(app) and app_is_on_github(app)] + + with requests.Session() as s: + s.headers.update({"Authorization": f"token {token}"}) + for app in progressbar(apps_to_push, "Forking: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + fork_if_needed(app["repo"], s) + + for app in progressbar(apps_to_push, "Pushing: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + app_repo_name = app["url"].rsplit("/", 1)[-1] + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset origin/{current_branch}", in_folder=folder) + git(["commit", "-a", "-m", title, "--author='Yunohost-Bot <>'"], in_folder=folder) + try: + git(f"remote remove fork", in_folder=folder) + except Exception: + pass + git(f"remote add fork https://{login}:{token}@github.com/{login}/{app_repo_name}", in_folder=folder) + git(f"push fork {current_branch}:{patch} --quiet --force", in_folder=folder) + create_pull_request(app["repo"], patch, current_branch, s) + + +def fork_if_needed(repo, s): + + repo_name = repo.split("/")[-1] + r = s.get(github_api + f"/repos/{login}/{repo_name}") + + if r.status_code == 200: + return + + r = s.post(github_api + f"/repos/{repo}/forks") + + if r.status_code != 200: + print(r.text) + + +def create_pull_request(repo, patch, base_branch, s): + + PR = {"title": "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip(), + "body": "This is an automatic PR\n\n" + open(os.path.join("patches", patch, "pr_body.md")).read().strip(), + "head": login + ":" + patch, + "base": base_branch, + "maintainer_can_modify": True} + + r = s.post(github_api + f"/repos/{repo}/pulls", json.dumps(PR)) + + if r.status_code != 200: + print(r.text) + else: + json.loads(r.text)["html_url"] + + +def main(): + + action = sys.argv[1] + if action == "--help": + print(""" + Example usage: + +# Init local git clone for all apps +./autopatch.py --build-cache + +# Apply patch in all local clones +./autopatch.py --apply explicit-php-version-in-deps + +# Inspect diff for all apps +./autopatch.py --diff + +# Push and create pull requests on all apps with non-empty diff +./autopatch.py --push explicit-php-version-in-deps +""") + + elif action == "--build-cache": + build_cache() + elif action == "--apply": + apply(sys.argv[2]) + elif action == "--diff": + diff() + elif action == "--push": + push(sys.argv[2]) + else: + print("Unknown action %s" % action) + + +main() diff --git a/autopatches/patches/issue-and-pr-template/patch.sh b/autopatches/patches/issue-and-pr-template/patch.sh new file mode 100644 index 0000000..d52d3b0 --- /dev/null +++ b/autopatches/patches/issue-and-pr-template/patch.sh @@ -0,0 +1,14 @@ + +[ ! -e issue_template.md ] || git rm issue_template.md +[ ! -e pull_request_template.md ] || git rm pull_request_template.md + +[ ! -e .github ] || git rm -rf .github +mkdir -p .github + +# Sleep 1 to avoid too many requests on github (there's a rate limit anyway) +sleep 1 + +wget -O .github/ISSUE_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/ISSUE_TEMPLATE.md +wget -O .github/PULL_REQUEST_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/PULL_REQUEST_TEMPLATE.md + +git add .github diff --git a/autopatches/patches/issue-and-pr-template/pr_body.md b/autopatches/patches/issue-and-pr-template/pr_body.md new file mode 100644 index 0000000..471273e --- /dev/null +++ b/autopatches/patches/issue-and-pr-template/pr_body.md @@ -0,0 +1,2 @@ + +This is an ***automated*** patch to sync the issue and PR template with the official templates in https://github.com/YunoHost/example_ynh/tree/master/.github diff --git a/autopatches/patches/issue-and-pr-template/pr_title.md b/autopatches/patches/issue-and-pr-template/pr_title.md new file mode 100644 index 0000000..f0bed1c --- /dev/null +++ b/autopatches/patches/issue-and-pr-template/pr_title.md @@ -0,0 +1 @@ +Update issue and PR templates diff --git a/autopatches/patches/missing-seteu-in-change_url/patch.sh b/autopatches/patches/missing-seteu-in-change_url/patch.sh new file mode 100644 index 0000000..f930134 --- /dev/null +++ b/autopatches/patches/missing-seteu-in-change_url/patch.sh @@ -0,0 +1,10 @@ + +cd scripts/ + +if [ ! -e change_url ] || grep -q 'ynh_abort_if_errors' change_url +then + # The app doesn't has any change url script or already has ynh_abort_if_error + exit 0 +fi + +sed 's@\(source /usr/share/yunohost/helpers\)@\1\nynh_abort_if_errors@g' -i change_url diff --git a/autopatches/patches/missing-seteu-in-change_url/pr_body.md b/autopatches/patches/missing-seteu-in-change_url/pr_body.md new file mode 100644 index 0000000..bb67bd0 --- /dev/null +++ b/autopatches/patches/missing-seteu-in-change_url/pr_body.md @@ -0,0 +1,2 @@ + +This is an ***automated*** patch to fix the lack of `ynh_abort_if_errors` in change_url script diff --git a/autopatches/patches/missing-seteu-in-change_url/pr_title.md b/autopatches/patches/missing-seteu-in-change_url/pr_title.md new file mode 100644 index 0000000..efd0e73 --- /dev/null +++ b/autopatches/patches/missing-seteu-in-change_url/pr_title.md @@ -0,0 +1 @@ +Missing ynh_abort_if_errors in change_url scripts diff --git a/autopatches/patches/new-permission-system/patch.sh b/autopatches/patches/new-permission-system/patch.sh new file mode 100644 index 0000000..f372a44 --- /dev/null +++ b/autopatches/patches/new-permission-system/patch.sh @@ -0,0 +1,66 @@ + +cd scripts/ + +if grep -q 'ynh_legacy_permissions' upgrade || grep -q 'ynh_permission_' install +then + # App already using the new permission system - not patching anything + exit 0 +fi + +if ! grep -q "protected_\|skipped_" install +then + # App doesn't has any (un)protected / skipped setting ? + # Probably not a webapp or permission ain't relevant for it ? + exit 0 +fi + +CONFIGURE_PERMISSION_DURING_INSTALL=' +# Make app public if necessary +if [ \"\$is_public\" -eq 1 ] +then + ynh_permission_update --permission=\"main\" --add=\"visitors\" +fi +' + +MIGRATE_LEGACY_PERMISSIONS=' +#================================================= +# Migrate legacy permissions to new system +#================================================= +if ynh_legacy_permissions_exists +then + ynh_legacy_permissions_delete_all + + ynh_app_setting_delete --app=\$app --key=is_public +fi' + +for SCRIPT in "remove upgrade backup restore change_url" +do + [[ -e $SCRIPT ]] || continue + + perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i $SCRIPT + perl -p0e 's@\s*if.*-z.*is_public.*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@\s*if.*is_public.*(-eq|=).*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@is_public=.*\n@@g' -i $SCRIPT + perl -p0e 's@ynh_app_setting_.*is_public.*@@g' -i $SCRIPT + perl -p0e 's@.*# Make app .*@@g' -i $SCRIPT + perl -p0e 's@.*# Fix is_public as a boolean.*@@g' -i $SCRIPT + perl -p0e 's@.*# If app is public.*@@g' -i $SCRIPT + perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_script_progression.*SSOwat.*@@g' -i $SCRIPT + perl -p0e 's@#=*\s#.*SETUP SSOWAT.*\s#=*\s@@g' -i $SCRIPT +done + + +perl -p0e 's@.*domain_regex.*@@g' -i install +perl -p0e 's@.*# If app is public.*@@g' -i install +perl -p0e 's@.*# Make app .*@@g' -i install +perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i install +perl -p0e "s@if.*is_public.*(-eq|=)(.|\n){0,100}setting(.|\n)*?fi\n@$CONFIGURE_PERMISSION_DURING_INSTALL@g" -i install +perl -p0e 's@.*ynh_app_setting_.*is_public.*\s@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i install + +grep -q 'is_public=' install || perl -p0e 's@(.*Configuring SSOwat.*)@\1\nynh_permission_update --permission=\"main\" --add=\"visitors\"@g' -i install + +perl -p0e "s@ynh_abort_if_errors@ynh_abort_if_errors\n$MIGRATE_LEGACY_PERMISSIONS@g" -i upgrade diff --git a/autopatches/patches/new-permission-system/pr_body.md b/autopatches/patches/new-permission-system/pr_body.md new file mode 100644 index 0000000..07f4e85 --- /dev/null +++ b/autopatches/patches/new-permission-system/pr_body.md @@ -0,0 +1,11 @@ + +NB. : this is an ***automated*** attempt to migrate the app to the new permission system + +You should ***not*** blindly trust the proposed changes. In particular, the auto-patch will not handle: +- situations which are more complex than "if is_public is true, allow visitors" +- situations where the app needs to be temporarily public (then possible private) during initial configuration +- apps that need to define extra permission for specific section of the app (such as admin interface) +- apps using non-standard syntax +- other specific use cases + +***PLEASE*** carefully review, test and amend the proposed changes if you find that the autopatch did not do a proper job. diff --git a/autopatches/patches/new-permission-system/pr_title.md b/autopatches/patches/new-permission-system/pr_title.md new file mode 100644 index 0000000..0f9554c --- /dev/null +++ b/autopatches/patches/new-permission-system/pr_title.md @@ -0,0 +1 @@ +Autopatch to migrate to new permission system From 0749b9fd6ffea39b30d54893a4d46669e5cd6bcc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 17:05:46 +0200 Subject: [PATCH 24/34] Fix v2 packaging convert script (_common.sh may not exist) --- packaging_v2/convert_app_to_packaging_v2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packaging_v2/convert_app_to_packaging_v2.py b/packaging_v2/convert_app_to_packaging_v2.py index 21fe774..b2a11c3 100644 --- a/packaging_v2/convert_app_to_packaging_v2.py +++ b/packaging_v2/convert_app_to_packaging_v2.py @@ -101,8 +101,8 @@ def _convert_v1_manifest_to_v2(app_path): manifest["resources"]["ports"][f"{name}.exposed"] = exposed maybequote = "[\"'\"'\"']?" - apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d '\"' | sed 's@ @\\n@g'") - php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d \"\\\"'\"") + apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d '\"' | sed 's@ @\\n@g'") + php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d \"\\\"'\"") if apt_dependencies.strip(): if php_version: apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version) @@ -283,6 +283,9 @@ def cleanup_scripts_and_conf(folder): script = f"{folder}/scripts/{s}" + if not os.path.exists(script): + continue + content = open(script).read() for pattern in patterns_to_remove_in_scripts: From 6d8e8df390da924bfa2245ebf8053a3110f73161 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Aug 2022 14:56:52 +0200 Subject: [PATCH 25/34] packaging v2: add a 'CPE' field in upstream section --- packaging_v2/convert_app_to_packaging_v2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging_v2/convert_app_to_packaging_v2.py b/packaging_v2/convert_app_to_packaging_v2.py index b2a11c3..ba00153 100644 --- a/packaging_v2/convert_app_to_packaging_v2.py +++ b/packaging_v2/convert_app_to_packaging_v2.py @@ -26,6 +26,8 @@ def _convert_v1_manifest_to_v2(app_path): if "url" in manifest and "website" not in manifest["upstream"]: manifest["upstream"]["website"] = manifest["url"] + manifest["upstream"]["cpe"] = "???" + manifest["integration"] = { "yunohost": manifest.get("requirements", {}).get("yunohost"), "architectures": "all", @@ -158,6 +160,7 @@ def _dump_v2_manifest_as_toml(manifest): upstream = table() for key, value in manifest["upstream"].items(): upstream[key] = value + upstream["cpe"].comment("FIXME: optional but recommended if relevant, this is meant to contain the Common Platform Enumeration, which is sort of a standard id for applications defined by the NIST. In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)") toml_manifest["upstream"] = upstream integration = table() From 3b8576149ab930296f1f30f5f90839f49bcc0f41 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 5 Aug 2022 16:39:38 +0200 Subject: [PATCH 26/34] Fix antifeatures list generation --- README-generator/make_readme.py | 22 ++++++++++++++++------ README-generator/templates/README.md.j2 | 3 ++- README-generator/templates/README_fr.md.j2 | 4 +++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index 788b8a7..d9c03db 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -8,6 +8,15 @@ from pathlib import Path from jinja2 import Environment, FileSystemLoader +def value_for_lang(values, lang): + if not isinstance(values, dict): + return values + if lang in values: + return values[lang] + elif "en" in values: + return values["en"] + else: + return list(values.values())[0] def generate_READMEs(app_path: str): @@ -22,7 +31,7 @@ def generate_READMEs(app_path: str): catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) from_catalog = catalog.get(manifest['id'], {}) - antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml")) + antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader) antifeatures_list = { e['id']: e for e in antifeatures_list } if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): @@ -62,12 +71,12 @@ def generate_READMEs(app_path: str): # TODO: Add url to the documentation... and actually create that documentation :D antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} - for k, v in antifeatures: - v['title'] = v['title'].get(lang_suffix, 'en') - if manifest.get("antifeatures", {}).get(k, 'en'): - v['description'] = manifest.get("antifeatures", {}).get(k, 'en') + for k, v in antifeatures.items(): + antifeatures[k]['title'] = value_for_lang(v['title'], lang_suffix) + if manifest.get("antifeatures", {}).get(k, None): + antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang_suffix) else: - antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') + antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang_suffix) out = template.render( lang=lang, @@ -75,6 +84,7 @@ def generate_READMEs(app_path: str): description=description, screenshots=screenshots, disclaimer=disclaimer, + antifeatures=antifeatures, manifest=manifest, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 8713a9a..990b5e0 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -52,8 +52,9 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if antifeatures -%} ## Antifeatures -{% for antifeature in antifeatures -%} +{% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} + {% endfor -%} {% endif -%} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 9e89c10..864dc60 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -34,11 +34,13 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if antifeatures -%} ## Fonctions indésirables -{% for antifeature in antifeatures -%} +{% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} + {% endfor -%} {% endif -%} + ## Documentations et ressources {% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }} From 55cd460a4ad2cd835da4b4ef5292d4f0617f3879 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 5 Aug 2022 16:41:27 +0200 Subject: [PATCH 27/34] Add red circle to antifeatures title Co-authored-by: Alexandre Aubin --- README-generator/templates/README.md.j2 | 2 +- README-generator/templates/README_fr.md.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 990b5e0..91d6510 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -50,7 +50,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% endif -%} {% if antifeatures -%} -## Antifeatures +## :red_circle: Antifeatures {% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 864dc60..22f4ebb 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -32,7 +32,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% endif -%} {% if antifeatures -%} -## Fonctions indésirables +## :red_circle: Fonctions indésirables {% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} From 0a9495483fd4ff16742ac0c3f1130c58a403f9a1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Aug 2022 23:06:38 +0200 Subject: [PATCH 28/34] README-generator: improve mechanism to only add the 'specific branch' note when version is different from master + display master's version --- README-generator/make_readme.py | 14 +++++++++++--- README-generator/templates/README.md.j2 | 2 +- README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index ac47a04..049aa21 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -57,7 +57,14 @@ def generate_READMEs(app_path: str): disclaimer = None # Get the current branch using git inside the app path - branch = os.popen("git --git-dir=%s/.git --work-tree=%s rev-parse --abbrev-ref HEAD" % (app_path, app_path)).read().strip() + default_branch = from_catalog.get('branch', 'master') + current_branch = os.popen(f"git -C {app_path} rev-parse --abbrev-ref HEAD").read().strip() + + if default_branch != current_branch: + os.system(f"git -C {app_path} fetch origin {default_branch} 2>/dev/null") + default_branch_version = os.popen(f"git -C {app_path} show FETCH_HEAD:manifest.json | jq -r .version").read().strip() + else: + default_branch_version = None # we don't care in that case out = template.render( lang=lang, @@ -66,8 +73,9 @@ def generate_READMEs(app_path: str): screenshots=screenshots, disclaimer=disclaimer, manifest=manifest, - branch=branch, - default_branch=from_catalog.get('branch', 'master'), + current_branch=current_branch, + default_branch=default_branch, + default_branch_version=default_branch_version, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 125833c..ec99391 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -31,7 +31,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(This is the `{{ branch }}` branch. See the [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) for the version in the catalog.)*{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: This is the `{{ current_branch }}` branch. The [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) used in the catalog is currently on version {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} {% if upstream.demo %} **Demo:** {{upstream.demo}} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 7350947..d616280 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -17,7 +17,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) pour la version du catalogue.)*{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: Il s'agit de la branche `{{ current_branch }}`. La [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) utilisée dans le catalogue est actuellement en {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} {% if upstream.demo %} **Démo :** {{upstream.demo}} From dcc3725f3057aa63fb2617aed11a79fc9e69c629 Mon Sep 17 00:00:00 2001 From: Hadrien Date: Sun, 7 Aug 2022 15:16:34 +0200 Subject: [PATCH 29/34] Add to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 55c973f..97f8654 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ *.swp *~ +.DS_Store .apps_cache builds tools/README-generator/venv/ + +tools/autopatches/login +tools/autopatches/token \ No newline at end of file From 3a5dc749d1ccfc5ac1a2e8e207d5fc3cd402cc9e Mon Sep 17 00:00:00 2001 From: Hadrien Date: Sun, 7 Aug 2022 15:15:40 +0200 Subject: [PATCH 30/34] Add a script to add cpe id to manifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit indent + utf8 --- autopatches/patches/add-cpe/cpe.csv | 507 +++++++++++++++++++++++++++ autopatches/patches/add-cpe/patch.sh | 19 + 2 files changed, 526 insertions(+) create mode 100644 autopatches/patches/add-cpe/cpe.csv create mode 100644 autopatches/patches/add-cpe/patch.sh diff --git a/autopatches/patches/add-cpe/cpe.csv b/autopatches/patches/add-cpe/cpe.csv new file mode 100644 index 0000000..f44e455 --- /dev/null +++ b/autopatches/patches/add-cpe/cpe.csv @@ -0,0 +1,507 @@ +App YnH,Common Platform Enumeration +20euros, +243, +2fauth, +abantecart,cpe:2.3:a:abantecart:abantecart +acropolis, +adguardhome,cpe:2.3:a:adguard:adguardhome +adhocserver, +adminer,cpe:2.3:a:adminer:adminer +aeneria, +agendav, +agora, +airsonic,cpe:2.3:a:airsonic_project:airsonic +alltube, +ampache,cpe:2.3:a:ampache:ampache +anarchism, +anfora, +archivebox, +archivist,cpe:2.3:a:archivista:archivistabox +armadietto, +askbot,cpe:2.3:a:askbot:askbot +audiobookshelf, +automad,cpe:2.3:a:automad:automad +backdrop,cpe:2.3:a:backdropcms:backdrop +baikal, +bazarr, +beehive,cpe:2.3:a:beehive_forum:beehive_forum +bibliogram, +biboumi, +bicbucstriim, +blogotext,cpe:2.3:a:blogotext_project:blogotext +bludit,cpe:2.3:a:bludit:bludit +bolt, +bookstack,cpe:2.3:a:bookstackapp:bookstack +borg,cpe:2.3:a:borgbackup:borg +borgserver, +bozon, +cachet, +calibreweb, +castopod, +cesium, +cheky, +chtickynotes, +chuwiki, +cinny,cpe:2.3:a:cinny_project:cinny +civicrm_drupal7, +cockpit,cpe:2.3:a:agentejo:cockpit +code-server,cpe:2.3:a:coder:code-server +codimd,cpe:2.3:a:hackmd:codimd +coin, +collabora, +commento, +compteur_du_gase, +concrete5,cpe:2.3:a:concrete5:concrete5 +converse,cpe:2.3:a:conversejs:converse.js +cops, +coquelicot, +coturn,cpe:2.3:a:coturn_project:coturn +couchdb,cpe:2.3:a:apache:couchdb +couchpotato, +covoiturage, +cowyo, +cryptpad,cpe:2.3:a:xwiki:cryptpad +cubiks-2048, +cypht, +dato, +decidim, +democracyos, +dendrite, +dex, +diagramsnet, +diaspora, +digisteps, +digitools, +digiwords, +discourse,cpe:2.3:a:discourse:discourse +dispatch, +distbin, +django-fmd, +django-for-runners, +django-fritzconnection, +django_app, +docker-registry, +dockercontainer, +dockerrstudio, +dockerui, +documize,cpe:2.3:a:documize:documize +dokuwiki,cpe:2.3:a:dokuwiki:dokuwiki +dolibarr,cpe:2.3:a:dolibarr:dolibarr +domoticz,cpe:2.3:a:domoticz:domoticz +dotclear2, +droppy,cpe:2.3:a:droppy_project:droppy +drupal,cpe:2.3:a:drupal:drupal +drupal7, +duniter, +dynette, +easyappointments,cpe:2.3:a:easyappointments:easyappointments +ecko, +elabftw,cpe:2.3:a:elabftw:elabftw +element, +emailpoubelle, +emoncms,cpe:2.3:a:openenergymonitor:emoncms +encryptic, +encryptor-decryptor, +epicyon, +ergo, +ethercalc, +etherpad_mypads, +excalidraw, +fab-manager, +facette, +facilmap, +fallback, +ffsync, +filebrowser, +filepizza, +firefly-iii,cpe:2.3:a:firefly-iii:firefly_iii +flarum,cpe:2.3:a:flarum:flarum +flask,cpe:2.3:a:palletsprojects:flask +flood, +fluxbb,cpe:2.3:a:fluxbb:fluxbb +focalboard,cpe:2.3:a:mattermost:focalboard +foodsoft, +framaestro, +framaestro_hub, +framaforms, +framagames, +freeboard, +freepbx,cpe:2.3:a:freepbx:freepbx +freshrss,cpe:2.3:a:freshrss:freshrss +friendica,cpe:2.3:a:friendica:friendica +ftp_webapp, +ftssolr, +funkwhale, +galene, +galette,cpe:2.3:a:galette:galette +gamja, +garradin, +gateone,cpe:2.3:a:liftoffsoftware:gate_one +gekko, +gemserv, +getsimple,cpe:2.3:a:get-simple:getsimple_cms +ghost,cpe:2.3:a:ghost:ghost +gitea,cpe:2.3:a:gitea:gitea +gitlab,cpe:2.3:a:gitlab:gitlab +gitlab-runner,cpe:2.3:a:gitlab:runner +gitolite,cpe:2.3:a:gitolite:gitolite +gitrepositories, +gitweb, +glitchsoc, +glowingbear, +glpi,cpe:2.3:a:glpi-project:glpi +gnusocial, +gogs,cpe:2.3:a:gogs:gogs +gogswebhost, +gollum,cpe:2.3:a:gollum_project:gollum +gossa, +gotify, +gotosocial, +grafana,cpe:2.3:a:grafana:grafana +grammalecte, +grav, +grocy,cpe:2.3:a:grocy_project:grocy +grr,cpe:2.3:a:devome:grr +guacamole,cpe:2.3:a:apache:guacamole +h5ai,cpe:2.3:a:h5ai_project:h5ai +halcyon, +haste, +hat, +headphones, +hedgedoc,cpe:2.3:a:hedgedoc:hedgedoc +helloworld, +hextris, +homeassistant, +horde,cpe:2.3:a:horde:horde_application_framework +hotspot, +htmltool, +htpc-manager, +hubzilla,cpe:2.3:a:hubzilla:hubzilla +huginn, +humhub,cpe:2.3:a:humhub:humhub +hydrogen, +icecoder,cpe:2.3:a:icecoder:icecoder +ifconfig-io, +ifm, +ihatemoney,cpe:2.3:a:ihatemoney:i_hate_money +indexhibit,cpe:2.3:a:indexhibit:indexhibit +internetarchive, +invidious, +invoiceninja,cpe:2.3:a:invoiceninja:invoice_ninja +invoiceninja5, +jackett, +jappix,cpe:2.3:a:jappix_project:jappix +jappix_mini, +jeedom,cpe:2.3:a:jeedom:jeedom +jellyfin,cpe:2.3:a:jellyfin:jellyfin +jenkins,cpe:2.3:a:jenkins:jenkins +jirafeau,cpe:2.3:a:jirafeau:jirafeau +jitsi,cpe:2.3:a:jitsi:jitsi +joomla,cpe:2.3:a:joomla:joomla\! +jupyterlab,cpe:2.3:a:jupyter:nbdime-jupyterlab +kanboard,cpe:2.3:a:kanboard:kanboard +keeweb, +kimai2,cpe:2.3:a:kimai:kimai_2 +kiwiirc, +kiwix,cpe:2.3:a:kiwix:kiwix +kodi,cpe:2.3:a:kodi:kodi +komga, +kresus, +languagetool, +laverna, +lbcalerte, +leed, +lektor, +lemmy, +librarian, +libreddit, +libreerp, +librephotos, +librespeed, +libreto, +librex, +lidarr, +limesurvey,cpe:2.3:a:limesurvey:limesurvey +linuxdash, +lionwiki-t2t, +listmonk, +lstu, +luckysheet, +lufi, +lutim,cpe:2.3:a:lutim_project:lutim +lxd, +lxd-dashboard, +lychee,cpe:2.3:a:lycheeorganisation:lychee-v3 +mailman,cpe:2.3:a:gnu:mailman +mailman3, +mantis,cpe:2.3:a:mantisbt:mantisbt +mastodon,cpe:2.3:a:joinmastodon:mastodon +matomo,cpe:2.3:a:matomo:matomo +matrix-puppet-discord, +matterbridge, +mattermost,cpe:2.3:a:mattermost:mattermost +mautic,cpe:2.3:a:acquia:mautic +mautrix_facebook, +mautrix_signal, +mautrix_telegram, +mautrix_whatsapp, +mediadrop, +mediawiki,cpe:2.3:a:mediawiki:mediawiki +medusa, +meilisearch, +menu, +metabase,cpe:2.3:a:metabase:metabase +minchat, +mindmaps, +minetest,cpe:2.3:a:minetest:minetest +mineweb, +minidlna,cpe:2.3:a:readymedia_project:readymedia +miniflux, +minio,cpe:2.3:a:minio:minio +misskey,cpe:2.3:a:misskey:misskey +mobilizon, +modernpaste, +monica,cpe:2.3:a:monicahq:monica +monit, +monitorix,cpe:2.3:a:fibranet:monitorix +moodle,cpe:2.3:a:moodle:moodle +mopidy, +mosquitto,cpe:2.3:a:eclipse:mosquitto +movim,cpe:2.3:a:movim:movim +multi_webapp, +mumble-web, +mumble_admin_plugin, +mumbleserver, +munin,cpe:2.3:a:munin-monitoring:munin +my-mind, +my_capsule, +my_webapp, +mybb,cpe:2.3:a:mybb:mybb +mycryptochat, +mygpo, +mytinytodo,cpe:2.3:a:mytinytodo:mytinytodo +navidrome,cpe:2.3:a:navidrome:navidrome +netdata,cpe:2.3:a:netdata:netdata +neutrinet, +nextcloud,cpe:2.3:a:nextcloud:nextcloud +nexusoss, +nitter, +noalyss, +nocodb,cpe:2.3:a:xgenecloud:nocodb +nodebb,cpe:2.3:a:nodebb:nodebb +nodered,cpe:2.3:a:nodered:node-red-dashboard +nomad,cpe:2.3:a:jenkins:nomad +ntopng,cpe:2.3:a:ntop:ntopng +nullboard, +ofbiz,cpe:2.3:a:apache:ofbiz +omeka-s, +onlyoffice,cpe:2.3:a:onlyoffice:document_server +openidsimplesamlphp, +opennote, +openproject,cpe:2.3:a:openproject:openproject +opensondage, +opentracker, +osada, +osjs, +osmw, +osticket,cpe:2.3:a:osticket:osticket +outline, +overleaf, +owncast,cpe:2.3:a:owncast_project:owncast +owncloud,cpe:2.3:a:owncloud:owncloud +owntracks, +pagure,cpe:2.3:a:fedoraproject:389_directory_server +paperless-ngx, +peachpub, +peertube,cpe:2.3:a:framasoft:peertube +peertube-search-index, +pelican, +pepettes, +petitesannonces, +petrolette, +pgadmin,cpe:2.3:a:phppgadmin_project:phppgadmin +photonix, +photoprism, +photoview, +phpback, +phpbb,cpe:2.3:a:phpbb:phpbb +phpboost, +phpinfo, +phpipam,cpe:2.3:a:phpipam:phpipam +phpldapadmin,cpe:2.3:a:phpldapadmin_project:phpldapadmin +phplicensewatcher, +phpmyadmin,cpe:2.3:a:phpmyadmin:phpmyadmin +phpservermon,cpe:2.3:a:phpservermonitor:php_server_monitor +phpsysinfo, +pia, +pico, +pihole,cpe:2.3:a:pi-hole:pi-hole +piratebox, +piwigo,cpe:2.3:a:piwigo:piwigo +pixelfed, +plainpad, +pleroma, +plonecms, +plume,cpe:2.3:a:plume-cms:plume_cms +pluxml,cpe:2.3:a:pluxml:pluxml +pmwiki,cpe:2.3:a:pmwiki:pmwiki +portainer,cpe:2.3:a:portainer:portainer +prestashop,cpe:2.3:a:prestashop:prestashop +prettynoemiecms, +privatebin,cpe:2.3:a:privatebin:privatebin +proftpd,cpe:2.3:a:proftpd:proftpd +prometheus,cpe:2.3:a:prometheus:prometheus +prosody,cpe:2.3:a:prosody:prosody +prowlarr, +proxitok, +psitransfer, +pterodactyl,cpe:2.3:a:pterodactyl:panel +pufferpanel, +pydio,cpe:2.3:a:pydio:pydio +pyinventory, +pyload, +pytition, +qr, +question2answer,cpe:2.3:a:question2answer:question2answer +quizzes, +radarr, +radicale,cpe:2.3:a:radicale:radicale +rainloop,cpe:2.3:a:rainloop:webmail +redirect, +redmine,cpe:2.3:a:redmine:redmine +reel2bits, +remotestorage, +restic, +retroarch,cpe:2.3:a:libretro:retroarch +riot, +roadiz, +rocketchat,cpe:2.3:a:rocket.chat:rocket.chat +roundcube,cpe:2.3:a:roundcube:webmail +rportd, +rspamdui, +rss-bridge, +rutorrent, +samba,cpe:2.3:a:samba:samba +sat, +satdress, +scm, +scratch, +scrumblr, +seafile,cpe:2.3:a:seafile:seafile +searx, +seenthis, +selfoss, +send, +shaarli,cpe:2.3:a:shaarli_project:shaarli +shadowsocks,cpe:2.3:a:shadowsocks:shadowsocks-libev +shellinabox,cpe:2.3:a:shellinabox_project:shellinabox +shinken, +shiori, +shsd, +shuri, +sickbeard, +sickrage,cpe:2.3:a:sickrage:sickrage +signaturepdf, +simple-hash-generator, +simple-torrent, +sitemagiccms,cpe:2.3:a:sitemagic:sitemagic +slingcode, +snappymail, +snipeit,cpe:2.3:a:snipeitapp:snipe-it +snserver, +snweb, +soapbox,cpe:2.3:a:soapbox_project:soapbox +sogo,cpe:2.3:a:inverse:sogo +sonarr, +sonerezh, +spacedeck, +spftoolbox, +sphinx,cpe:2.3:a:sphinxsearch:sphinx +spip,cpe:2.3:a:spip:spip +squid3, +ssbroom, +ssh_chroot_dir, +staticwebapp, +streama, +strut, +subscribe, +subsonic,cpe:2.3:a:subsonic:subsonic +sutom, +svgedit, +sympa,cpe:2.3:a:sympa:sympa +synapse, +synapse-admin, +syncthing,cpe:2.3:a:syncthing:syncthing +tagspaces, +tailoredflow, +teampass,cpe:2.3:a:teampass:teampass +technitium-dns, +teddit, +telegram_chatbot, +tes3mp, +thelounge, +tiddlywiki, +tiki,cpe:2.3:a:tiki:tiki +timemachine, +timeoff, +tooljet, +torrelay, +tracim, +traggo, +transfersh, +transmission, +transpay, +transwhat, +trilium,cpe:2.3:a:trilium_project:trilium +trustyhash, +ttrss, +turtl,cpe:2.3:a:lyonbros:turtl +tutao, +tvheadend, +tyto, +ulogger, +umami, +umap, +ums, +unattended_upgrades, +unbound,cpe:2.3:a:nlnetlabs:unbound +uptime-kuma, +vaultwarden, +veloren, +vikunja, +vpnclient, +vpnserver, +wallabag2, +weblate,cpe:2.3:a:weblate:weblate +webmin,cpe:2.3:a:webmin:webmin +webogram, +webtrees,cpe:2.3:a:webtrees:webtrees +wekan,cpe:2.3:a:wekan_project:wekan +wemawema, +wetty, +whitebophir, +wifiwithme, +wikijs, +wildfly,cpe:2.3:a:redhat:jboss_wildfly_application_server +wireguard, +wisemapping, +wondercms,cpe:2.3:a:wondercms:wondercms +wordpress,cpe:2.3:a:wordpress:wordpress +writefreely, +yacy, +yellow, +yeswiki,cpe:2.3:a:yeswiki:yeswiki +yourls,cpe:2.3:a:yourls:yourls +youtube-dl-webui, +yunofav, +yunohost,cpe:2.3:o:yunohost:yunohost +yunomonitor, +yunorunner, +z-push, +zabbix,cpe:2.3:a:zabbix:zabbix +zap, +zerobin, +zeronet, +zomburl, +ztncui, +zusam, diff --git a/autopatches/patches/add-cpe/patch.sh b/autopatches/patches/add-cpe/patch.sh new file mode 100644 index 0000000..02a697c --- /dev/null +++ b/autopatches/patches/add-cpe/patch.sh @@ -0,0 +1,19 @@ +#!/usr/bin/python3 + +import json +import csv + +def find_cpe(app_id): + with open("../../patches/add-cpe/cpe.csv", newline='') as f: + cpe_list = csv.reader(f) + for row in cpe_list: + if row[0] == app_id: + return row[1] + return False + +manifest = json.load(open("manifest.json")) +app_id = manifest['id'] +cpe = find_cpe(app_id) +if cpe: + manifest['upstream']['cpe'] = cpe + json.dump(manifest, open("manifest.json", "w"), indent=4, ensure_ascii=False) From 3f4a57aeefbd06fa509a7492ebf3f87db8f93f05 Mon Sep 17 00:00:00 2001 From: Hadrien Date: Sun, 7 Aug 2022 16:32:25 +0200 Subject: [PATCH 31/34] patch message --- autopatches/patches/add-cpe/pr_body.md | 6 ++++++ autopatches/patches/add-cpe/pr_title.md | 1 + 2 files changed, 7 insertions(+) create mode 100644 autopatches/patches/add-cpe/pr_body.md create mode 100644 autopatches/patches/add-cpe/pr_title.md diff --git a/autopatches/patches/add-cpe/pr_body.md b/autopatches/patches/add-cpe/pr_body.md new file mode 100644 index 0000000..5f08ae8 --- /dev/null +++ b/autopatches/patches/add-cpe/pr_body.md @@ -0,0 +1,6 @@ + +This is an ***automated*** patch to add the (optional but recommended if relevant) Common Platform Enumeration (CPE) id, which is sort of a standard id for applications, defined by the NIST. + +In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. + +The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)"). \ No newline at end of file diff --git a/autopatches/patches/add-cpe/pr_title.md b/autopatches/patches/add-cpe/pr_title.md new file mode 100644 index 0000000..5944c62 --- /dev/null +++ b/autopatches/patches/add-cpe/pr_title.md @@ -0,0 +1 @@ +Add Common Platform Enumeration id to `manifest.json` From 5aaa60e92e3b47a3615d23f90238784227f09f91 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Aug 2022 02:15:40 +0200 Subject: [PATCH 32/34] Packaging v2: add a 'fund' key in the upstream section --- packaging_v2/convert_app_to_packaging_v2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging_v2/convert_app_to_packaging_v2.py b/packaging_v2/convert_app_to_packaging_v2.py index ba00153..4d6d97a 100644 --- a/packaging_v2/convert_app_to_packaging_v2.py +++ b/packaging_v2/convert_app_to_packaging_v2.py @@ -27,6 +27,7 @@ def _convert_v1_manifest_to_v2(app_path): manifest["upstream"]["website"] = manifest["url"] manifest["upstream"]["cpe"] = "???" + manifest["upstream"]["fund"] = "???" manifest["integration"] = { "yunohost": manifest.get("requirements", {}).get("yunohost"), @@ -161,6 +162,7 @@ def _dump_v2_manifest_as_toml(manifest): for key, value in manifest["upstream"].items(): upstream[key] = value upstream["cpe"].comment("FIXME: optional but recommended if relevant, this is meant to contain the Common Platform Enumeration, which is sort of a standard id for applications defined by the NIST. In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)") + upstream["fund"].comment("FIXME: optional but recommended (or remove if irrelevant / not applicable). This is meant to be an URL where people can financially support this app, especially when its development is based on volunteers and/or financed by its community. YunoHost may later advertise it in the webadmin.") toml_manifest["upstream"] = upstream integration = table() From 0e1da9afb7526b29b503f92e0bf02bc086523a1b Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Thu, 6 Oct 2022 23:12:37 +0200 Subject: [PATCH 33/34] Revert "Add a warning on the README when not on the master branch" --- README-generator/make_readme.py | 17 ++--------------- README-generator/templates/README.md.j2 | 3 ++- README-generator/templates/README_fr.md.j2 | 3 ++- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index 592f009..d9c03db 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -32,7 +32,7 @@ def generate_READMEs(app_path: str): from_catalog = catalog.get(manifest['id'], {}) antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader) - antifeatures_list = {e['id']: e for e in antifeatures_list} + antifeatures_list = { e['id']: e for e in antifeatures_list } if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( @@ -69,18 +69,8 @@ def generate_READMEs(app_path: str): else: disclaimer = None - # Get the current branch using git inside the app path - default_branch = from_catalog.get('branch', 'master') - current_branch = os.popen(f"git -C {app_path} rev-parse --abbrev-ref HEAD").read().strip() - - if default_branch != current_branch: - os.system(f"git -C {app_path} fetch origin {default_branch} 2>/dev/null") - default_branch_version = os.popen(f"git -C {app_path} show FETCH_HEAD:manifest.json | jq -r .version").read().strip() - else: - default_branch_version = None # we don't care in that case - # TODO: Add url to the documentation... and actually create that documentation :D - antifeatures = {a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} + antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} for k, v in antifeatures.items(): antifeatures[k]['title'] = value_for_lang(v['title'], lang_suffix) if manifest.get("antifeatures", {}).get(k, None): @@ -96,9 +86,6 @@ def generate_READMEs(app_path: str): disclaimer=disclaimer, antifeatures=antifeatures, manifest=manifest, - current_branch=current_branch, - default_branch=default_branch, - default_branch_version=default_branch_version, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/README-generator/templates/README.md.j2 b/README-generator/templates/README.md.j2 index 0c5f9b0..e2d428f 100644 --- a/README-generator/templates/README.md.j2 +++ b/README-generator/templates/README.md.j2 @@ -31,7 +31,8 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: This is the `{{ current_branch }}` branch. The [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) used in the catalog is currently on version {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} {% if upstream.demo %} **Demo:** {{upstream.demo}} diff --git a/README-generator/templates/README_fr.md.j2 b/README-generator/templates/README_fr.md.j2 index 6171d44..95fb8f4 100644 --- a/README-generator/templates/README_fr.md.j2 +++ b/README-generator/templates/README_fr.md.j2 @@ -17,7 +17,8 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: Il s'agit de la branche `{{ current_branch }}`. La [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) utilisée dans le catalogue est actuellement en {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} {% if upstream.demo %} **Démo :** {{upstream.demo}} From 0361a0d604c4ec8f25b77cc00c5f41e6f22d52bb Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Thu, 6 Oct 2022 23:20:19 +0200 Subject: [PATCH 34/34] typo --- README-generator/make_readme.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README-generator/make_readme.py b/README-generator/make_readme.py index d9c03db..cb65525 100755 --- a/README-generator/make_readme.py +++ b/README-generator/make_readme.py @@ -72,11 +72,11 @@ def generate_READMEs(app_path: str): # TODO: Add url to the documentation... and actually create that documentation :D antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} for k, v in antifeatures.items(): - antifeatures[k]['title'] = value_for_lang(v['title'], lang_suffix) + antifeatures[k]['title'] = value_for_lang(v['title'], lang) if manifest.get("antifeatures", {}).get(k, None): - antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang_suffix) + antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang) else: - antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang_suffix) + antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang) out = template.render( lang=lang,