Merge pull request #2 from YunoHost/master

update

.gitignore

@@ -1,5 +1,10 @@
 *.swp
 *~
+.DS_Store
 
 .apps_cache
 builds
+tools/README-generator/venv/
+
+tools/autopatches/login
+tools/autopatches/token

README-generator/make_readme.py

@@ -3,10 +3,20 @@
 import argparse
 import json
 import os
+import yaml
 from pathlib import Path
 
 from jinja2 import Environment, FileSystemLoader
 
+def value_for_lang(values, lang):
+     if not isinstance(values, dict):
+         return values
+     if lang in values:
+         return values[lang]
+     elif "en" in values:
+         return values["en"]
+     else:
+         return list(values.values())[0]
 
 def generate_READMEs(app_path: str):
 
@@ -18,6 +28,12 @@ def generate_READMEs(app_path: str):
     manifest = json.load(open(app_path / "manifest.json"))
     upstream = manifest.get("upstream", {})
 
+    catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json"))
+    from_catalog = catalog.get(manifest['id'], {})
+
+    antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader)
+    antifeatures_list = { e['id']: e for e in antifeatures_list }
+
     if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists():
         print(
             "There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet."
@@ -30,6 +46,14 @@ def generate_READMEs(app_path: str):
 
         template = env.get_template(f"README{lang_suffix}.md.j2")
 
+        if (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").exists():
+            description = (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").read_text()
+        # Fallback to english if maintainer too lazy to translate the description
+        elif (app_path / "doc" / "DESCRIPTION.md").exists():
+            description = (app_path / "doc" / "DESCRIPTION.md").read_text()
+        else:
+            description = None
+
         if (app_path / "doc" / "screenshots").exists():
             screenshots = os.listdir(os.path.join(app_path, "doc", "screenshots"))
             if ".gitkeep" in screenshots:
@@ -45,11 +69,22 @@ def generate_READMEs(app_path: str):
         else:
             disclaimer = None
 
+        # TODO: Add url to the documentation... and actually create that documentation :D
+        antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])}
+        for k, v in antifeatures.items():
+            antifeatures[k]['title'] = value_for_lang(v['title'], lang)
+            if manifest.get("antifeatures", {}).get(k, None):
+                antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang)
+            else:
+                antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang)
+
         out = template.render(
             lang=lang,
             upstream=upstream,
+            description=description,
             screenshots=screenshots,
             disclaimer=disclaimer,
+            antifeatures=antifeatures,
             manifest=manifest,
         )
         (app_path / f"README{lang_suffix}.md").write_text(out)

README-generator/requirements.txt

@@ -1,4 +1,3 @@
-argparse
 jinja2
-github-webhook==1.0.4
+sanic
-gunicorn==20.1.0
+pyyaml

README-generator/templates/README.md.j2

@@ -1,13 +1,13 @@
 {% if manifest.id == "example" -%}
-# Packaging your an app, starting from this example
+# Packaging an app, starting from this example
 
-- Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo.
+* Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo.
-- Edit the `manifest.json` with app specific info.
+* Edit the `manifest.json` with app specific info.
-- Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`.
+* Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`.
-  - Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers)
+  * Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers)
-- Add a `LICENSE` file for the package.
+* Add a `LICENSE` file for the package.
-- Edit `doc/DISCLAIMER*.md`
+* Edit `doc/DISCLAIMER*.md`
-- The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator
+* The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator
 
 ---
 {% endif -%}
@@ -19,7 +19,7 @@ It shall NOT be edited by hand.
 
 # {{manifest.name}} for YunoHost
 
-[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg)  
+[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Working status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Maintenance status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg)  
 [![Install {{manifest.name}} with YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}})
 
 *[Lire ce readme en français.](./README_fr.md)*
@@ -29,17 +29,20 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in
 
 ## Overview
 
-{{manifest.description[lang]}}
+{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %}
 
-**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}
+**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}
+{% endif -%}
 
-{% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %}
+{% if upstream.demo %}
+**Demo:** {{upstream.demo}}
+{% endif -%}
 
-{% if screenshots -%}
+{% if screenshots %}
 ## Screenshots
 
 {% for screenshot in screenshots -%}
-    ![](./doc/screenshots/{{screenshot}})
+    ![Screenshot of {{manifest.name}}](./doc/screenshots/{{screenshot}})
 {% endfor %}
 {% endif -%}
 
@@ -49,28 +52,39 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in
 {{ disclaimer }}
 {% endif -%}
 
+{% if antifeatures -%}
+## :red_circle: Antifeatures
+
+{% for antifeature in antifeatures.values() -%}
+  - **{{ antifeature.title }}**: {{ antifeature.description }}
+
+{% endfor -%}
+{% endif -%}
+
 ## Documentation and resources
 
-{% if upstream.website -%}* Official app website: {{ upstream.website }}
+{% if upstream.website -%}* Official app website: <{{ upstream.website }}>
 {% endif -%}
-{% if upstream.userdoc -%}* Official user documentation: {{ upstream.userdoc }}
+{% if upstream.userdoc -%}* Official user documentation: <{{ upstream.userdoc }}>
 {% endif -%}
-{% if upstream.admindoc -%}* Official admin documentation: {{ upstream.admindoc }}
+{% if upstream.admindoc -%}* Official admin documentation: <{{ upstream.admindoc }}>
 {% endif -%}
-{% if upstream.code -%}* Upstream app code repository: {{ upstream.code }}
+{% if upstream.code -%}* Upstream app code repository: <{{ upstream.code }}>
 {% endif -%}
-* YunoHost documentation for this app: https://yunohost.org/app_{{manifest.id}}
+* YunoHost documentation for this app: <https://yunohost.org/app_{{manifest.id}}>
-* Report a bug: https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues
+* Report a bug: <https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues>
 
 ## Developer info
 
 Please send your pull request to the [testing branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing).
 
 To try the testing branch, please proceed like that.
-```
+
+``` bash
 sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
 or
 sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
 ```
 
-**More info regarding app packaging:** https://yunohost.org/packaging_apps
+**More info regarding app packaging:** <https://yunohost.org/packaging_apps>
+

README-generator/templates/README_fr.md.j2

@@ -1,27 +1,34 @@
+<!--
+N.B.: This README was automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator
+It shall NOT be edited by hand.
+-->
+
 # {{manifest.name}} pour YunoHost
 
-[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg)  
+[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Statut de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg)  
 [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}})
 
 *[Read this readme in english.](./README.md)*
-*[Lire ce readme en français.](./README_fr.md)*
 
 > *Ce package vous permet d'installer {{manifest.name}} rapidement et simplement sur un serveur YunoHost.
 Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour savoir comment l'installer et en profiter.*
 
 ## Vue d'ensemble
 
-{{manifest.description[lang]}}
+{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %}
 
-**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}
+**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}
+{% endif -%}
 
-{% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %}
+{% if upstream.demo %}
+**Démo :** {{upstream.demo}}
+{% endif -%}
 
-{% if screenshots -%}
+{% if screenshots %}
 ## Captures d'écran
 
 {% for screenshot in screenshots -%}
-    ![](./doc/screenshots/{{screenshot}})
+    ![Capture d'écran de {{manifest.name}}](./doc/screenshots/{{screenshot}})
 {% endfor %}
 {% endif -%}
 
@@ -31,28 +38,40 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour
 {{ disclaimer }}
 {% endif -%}
 
+{% if antifeatures -%}
+## :red_circle: Fonctions indésirables
+
+{% for antifeature in antifeatures.values() -%}
+  - **{{ antifeature.title }}**: {{ antifeature.description }}
+
+{% endfor -%}
+{% endif -%}
+
+
 ## Documentations et ressources
 
-{% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }}
+{% if upstream.website -%}* Site officiel de l'app : <{{ upstream.website }}>
 {% endif -%}
-{% if upstream.userdoc -%}* Documentation officielle utilisateur : {{ upstream.userdoc }}
+{% if upstream.userdoc -%}* Documentation officielle utilisateur : <{{ upstream.userdoc }}>
 {% endif -%}
-{% if upstream.admindoc -%}* Documentation officielle de l'admin : {{ upstream.admindoc }}
+{% if upstream.admindoc -%}* Documentation officielle de l'admin : <{{ upstream.admindoc }}>
 {% endif -%}
-{% if upstream.code -%}* Dépôt de code officiel de l'app : {{ upstream.code }}
+{% if upstream.code -%}* Dépôt de code officiel de l'app : <{{ upstream.code }}>
 {% endif -%}
-* Documentation YunoHost pour cette app : https://yunohost.org/app_{{manifest.id}}
+* Documentation YunoHost pour cette app : <https://yunohost.org/app_{{manifest.id}}>
-* Signaler un bug : https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues
+* Signaler un bug : <https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues>
 
 ## Informations pour les développeurs
 
 Merci de faire vos pull request sur la [branche testing](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing).
 
 Pour essayer la branche testing, procédez comme suit.
-```
+
+``` bash
 sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
 ou
 sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
 ```
 
-**Plus d'infos sur le packaging d'applications :** https://yunohost.org/packaging_apps
+**Plus d'infos sur le packaging d'applications :** <https://yunohost.org/packaging_apps>
+

README-generator/webhook.py

@@ -1,64 +1,93 @@
-import subprocess
 import os
-import shutil
+import hmac
+import shlex
+import hashlib
+import asyncio
+import tempfile
+
+from sanic import Sanic
+from sanic.response import text
+from sanic.exceptions import abort
 
-from github_webhook import Webhook
-from flask import Flask
 from make_readme import generate_READMEs
 
-app = Flask(__name__)
+app = Sanic(__name__)
 
 github_webhook_secret = open("github_webhook_secret", "r").read().strip()
-webhook = Webhook(app, endpoint="/github", secret=github_webhook_secret)
 
 login = open("login").read().strip()
 token = open("token").read().strip()
 
 my_env = os.environ.copy()
 my_env["GIT_TERMINAL_PROMPT"] = "0"
-my_env["GIT_AUTHOR_NAME"] = "Yunohost-Bot"
+my_env["GIT_AUTHOR_NAME"] = "yunohost-bot"
 my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org"
-my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot"
+my_env["GIT_COMMITTER_NAME"] = "yunohost-bot"
 my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org"
 
 
-def git(cmd, in_folder=None):
+async def git(cmd, in_folder=None):
 
     if not isinstance(cmd, list):
         cmd = cmd.split()
     if in_folder:
         cmd = ["-C", in_folder] + cmd
     cmd = ["git"] + cmd
-    return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8")
+    cmd = " ".join(map(shlex.quote, cmd))
+    print(cmd)
+    command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+    data = await command.stdout.read()
+    return data.decode().strip()
 
 
-@app.route("/github")
+@app.route("/github", methods=["GET"])
-def main_route():
+def main_route(request):
-    return "You aren't supposed to go on this page using a browser, it's for webhooks push instead."
+    return text("You aren't supposed to go on this page using a browser, it's for webhooks push instead.")
 
 
-@webhook.hook()
+@app.route("/github", methods=["POST"])
-def on_push(data):
+async def on_push(request):
+    header_signature = request.headers.get("X-Hub-Signature")
+    if header_signature is None:
+        print("no header X-Hub-Signature")
+        abort(403)
+
+    sha_name, signature = header_signature.split("=")
+    if sha_name != "sha1":
+        print("signing algo isn't sha1, it's '%s'" % sha_name)
+        abort(501)
+
+    # HMAC requires the key to be bytes, but data is string
+    mac = hmac.new(github_webhook_secret.encode(), msg=request.body, digestmod=hashlib.sha1)
+
+    if not hmac.compare_digest(str(mac.hexdigest()), str(signature)):
+        abort(403)
+
+    data = request.json
 
     repository = data["repository"]["full_name"]
     branch = data["ref"].split("/", 2)[2]
 
-    folder = subprocess.check_output(["mktemp", "-d"]).decode('utf-8').strip()
+    print(f"{repository} -> branch '{branch}'")
-    try:
+
-        git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder])
+    with tempfile.TemporaryDirectory() as folder:
+        await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder])
         generate_READMEs(folder)
 
-        git(["add", "README*.md"], in_folder=folder)
+        await git(["add", "README*.md"], in_folder=folder)
 
-        diff_not_empty = bool(subprocess.check_output(["git", "diff", "HEAD", "--compact-summary"], cwd=folder).strip().decode("utf-8"))
+        diff_not_empty = await asyncio.create_subprocess_shell(" ".join(["git", "diff", "HEAD", "--compact-summary"]), cwd=folder, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+        diff_not_empty = await diff_not_empty.stdout.read()
+        diff_not_empty = diff_not_empty.decode().strip()
         if not diff_not_empty:
-            return
+            print("nothing to do")
+            return text("nothing to do")
+
+        await git(["commit", "-a", "-m", "Auto-update README", "--author='yunohost-bot <yunohost@yunohost.org>'"], in_folder=folder)
+        await git(["push", "origin", branch, "--quiet"], in_folder=folder)
+
+    return text("ok")
 
-        git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder)
-        git(["push", "origin", branch, "--quiet"], in_folder=folder)
-    finally:
-        if os.path.exists(folder):
-            shutil.rmtree(folder)
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=8123)

autopatches/autopatch.py

@@ -0,0 +1,188 @@
+#!/usr/bin/python3
+import json
+import sys
+import requests
+import os
+import subprocess
+
+catalog = requests.get("https://raw.githubusercontent.com/YunoHost/apps/master/apps.json").json()
+
+my_env = os.environ.copy()
+my_env["GIT_TERMINAL_PROMPT"] = "0"
+os.makedirs(".apps_cache", exist_ok=True)
+
+login = open("login").read().strip()
+token = open("token").read().strip()
+github_api = "https://api.github.com"
+
+
+def apps(min_level=4):
+
+    for app, infos in catalog.items():
+        if infos.get("state") == "working" and infos.get("level", -1) > min_level:
+            infos["id"] = app
+            yield infos
+
+
+def app_cache_folder(app):
+    return os.path.join(".apps_cache", app)
+
+
+def git(cmd, in_folder=None):
+
+    if not isinstance(cmd, list):
+        cmd = cmd.split()
+    if in_folder:
+        cmd = ["-C", in_folder] + cmd
+    cmd = ["git"] + cmd
+    return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8")
+
+
+# Progress bar helper, stolen from https://stackoverflow.com/a/34482761
+def progressbar(it, prefix="", size=60, file=sys.stdout):
+    it = list(it)
+    count = len(it)
+    def show(j, name=""):
+        name += "          "
+        x = int(size*j/count)
+        file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j,  count, name))
+        file.flush()
+    show(0)
+    for i, item in enumerate(it):
+        yield item
+        show(i+1, item["id"])
+    file.write("\n")
+    file.flush()
+
+
+def build_cache():
+
+    for app in progressbar(apps(), "Git cloning: ", 40):
+        folder = os.path.join(".apps_cache", app["id"])
+        reponame = app["url"].rsplit("/", 1)[-1]
+        git(f"clone --quiet --depth 1 --single-branch {app['url']} {folder}")
+        git(f"remote add fork https://{login}:{token}@github.com/{login}/{reponame}", in_folder=folder)
+
+
+def apply(patch):
+
+    patch_path = os.path.abspath(os.path.join("patches", patch, "patch.sh"))
+
+    for app in progressbar(apps(), "Apply to: ", 40):
+        folder = os.path.join(".apps_cache", app["id"])
+        current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder)
+        git(f"reset --hard origin/{current_branch}", in_folder=folder)
+        os.system(f"cd {folder} && bash {patch_path}")
+
+
+def diff():
+
+    for app in apps():
+        folder = os.path.join(".apps_cache", app["id"])
+        if bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")):
+            print("\n\n\n")
+            print("=================================")
+            print("Changes in : " + app["id"])
+            print("=================================")
+            print("\n")
+            os.system(f"cd {folder} && git --no-pager diff")
+
+
+def push(patch):
+
+    title = "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip()
+
+    def diff_not_empty(app):
+        folder = os.path.join(".apps_cache", app["id"])
+        return bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8"))
+
+    def app_is_on_github(app):
+        return "github.com" in app["url"]
+
+    apps_to_push = [app for app in apps() if diff_not_empty(app) and app_is_on_github(app)]
+
+    with requests.Session() as s:
+        s.headers.update({"Authorization": f"token {token}"})
+        for app in progressbar(apps_to_push, "Forking: ", 40):
+            app["repo"] = app["url"][len("https://github.com/"):].strip("/")
+            fork_if_needed(app["repo"], s)
+
+        for app in progressbar(apps_to_push, "Pushing: ", 40):
+            app["repo"] = app["url"][len("https://github.com/"):].strip("/")
+            app_repo_name = app["url"].rsplit("/", 1)[-1]
+            folder = os.path.join(".apps_cache", app["id"])
+            current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder)
+            git(f"reset origin/{current_branch}", in_folder=folder)
+            git(["commit", "-a", "-m", title, "--author='Yunohost-Bot <>'"], in_folder=folder)
+            try:
+                git(f"remote remove fork", in_folder=folder)
+            except Exception:
+                pass
+            git(f"remote add fork https://{login}:{token}@github.com/{login}/{app_repo_name}", in_folder=folder)
+            git(f"push fork {current_branch}:{patch} --quiet --force", in_folder=folder)
+            create_pull_request(app["repo"], patch, current_branch, s)
+
+
+def fork_if_needed(repo, s):
+
+    repo_name = repo.split("/")[-1]
+    r = s.get(github_api + f"/repos/{login}/{repo_name}")
+
+    if r.status_code == 200:
+        return
+
+    r = s.post(github_api + f"/repos/{repo}/forks")
+
+    if r.status_code != 200:
+        print(r.text)
+
+
+def create_pull_request(repo, patch, base_branch, s):
+
+    PR = {"title": "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip(),
+          "body": "This is an automatic PR\n\n" + open(os.path.join("patches", patch, "pr_body.md")).read().strip(),
+          "head": login + ":" + patch,
+          "base": base_branch,
+          "maintainer_can_modify": True}
+
+    r = s.post(github_api + f"/repos/{repo}/pulls", json.dumps(PR))
+
+    if r.status_code != 200:
+        print(r.text)
+    else:
+        json.loads(r.text)["html_url"]
+
+
+def main():
+
+    action = sys.argv[1]
+    if action == "--help":
+        print("""
+    Example usage:
+
+# Init local git clone for all apps
+./autopatch.py --build-cache
+
+# Apply patch in all local clones
+./autopatch.py --apply explicit-php-version-in-deps
+
+# Inspect diff for all apps
+./autopatch.py --diff
+
+# Push and create pull requests on all apps with non-empty diff
+./autopatch.py --push explicit-php-version-in-deps
+""")
+
+    elif action == "--build-cache":
+        build_cache()
+    elif action == "--apply":
+        apply(sys.argv[2])
+    elif action == "--diff":
+        diff()
+    elif action == "--push":
+        push(sys.argv[2])
+    else:
+        print("Unknown action %s" % action)
+
+
+main()

autopatches/patches/add-cpe/cpe.csv

@@ -0,0 +1,507 @@
+App YnH,Common Platform Enumeration
+20euros,
+243,
+2fauth,
+abantecart,cpe:2.3:a:abantecart:abantecart
+acropolis,
+adguardhome,cpe:2.3:a:adguard:adguardhome
+adhocserver,
+adminer,cpe:2.3:a:adminer:adminer
+aeneria,
+agendav,
+agora,
+airsonic,cpe:2.3:a:airsonic_project:airsonic
+alltube,
+ampache,cpe:2.3:a:ampache:ampache
+anarchism,
+anfora,
+archivebox,
+archivist,cpe:2.3:a:archivista:archivistabox
+armadietto,
+askbot,cpe:2.3:a:askbot:askbot
+audiobookshelf,
+automad,cpe:2.3:a:automad:automad
+backdrop,cpe:2.3:a:backdropcms:backdrop
+baikal,
+bazarr,
+beehive,cpe:2.3:a:beehive_forum:beehive_forum
+bibliogram,
+biboumi,
+bicbucstriim,
+blogotext,cpe:2.3:a:blogotext_project:blogotext
+bludit,cpe:2.3:a:bludit:bludit
+bolt,
+bookstack,cpe:2.3:a:bookstackapp:bookstack
+borg,cpe:2.3:a:borgbackup:borg
+borgserver,
+bozon,
+cachet,
+calibreweb,
+castopod,
+cesium,
+cheky,
+chtickynotes,
+chuwiki,
+cinny,cpe:2.3:a:cinny_project:cinny
+civicrm_drupal7,
+cockpit,cpe:2.3:a:agentejo:cockpit
+code-server,cpe:2.3:a:coder:code-server
+codimd,cpe:2.3:a:hackmd:codimd
+coin,
+collabora,
+commento,
+compteur_du_gase,
+concrete5,cpe:2.3:a:concrete5:concrete5
+converse,cpe:2.3:a:conversejs:converse.js
+cops,
+coquelicot,
+coturn,cpe:2.3:a:coturn_project:coturn
+couchdb,cpe:2.3:a:apache:couchdb
+couchpotato,
+covoiturage,
+cowyo,
+cryptpad,cpe:2.3:a:xwiki:cryptpad
+cubiks-2048,
+cypht,
+dato,
+decidim,
+democracyos,
+dendrite,
+dex,
+diagramsnet,
+diaspora,
+digisteps,
+digitools,
+digiwords,
+discourse,cpe:2.3:a:discourse:discourse
+dispatch,
+distbin,
+django-fmd,
+django-for-runners,
+django-fritzconnection,
+django_app,
+docker-registry,
+dockercontainer,
+dockerrstudio,
+dockerui,
+documize,cpe:2.3:a:documize:documize
+dokuwiki,cpe:2.3:a:dokuwiki:dokuwiki
+dolibarr,cpe:2.3:a:dolibarr:dolibarr
+domoticz,cpe:2.3:a:domoticz:domoticz
+dotclear2,
+droppy,cpe:2.3:a:droppy_project:droppy
+drupal,cpe:2.3:a:drupal:drupal
+drupal7,
+duniter,
+dynette,
+easyappointments,cpe:2.3:a:easyappointments:easyappointments
+ecko,
+elabftw,cpe:2.3:a:elabftw:elabftw
+element,
+emailpoubelle,
+emoncms,cpe:2.3:a:openenergymonitor:emoncms
+encryptic,
+encryptor-decryptor,
+epicyon,
+ergo,
+ethercalc,
+etherpad_mypads,
+excalidraw,
+fab-manager,
+facette,
+facilmap,
+fallback,
+ffsync,
+filebrowser,
+filepizza,
+firefly-iii,cpe:2.3:a:firefly-iii:firefly_iii
+flarum,cpe:2.3:a:flarum:flarum
+flask,cpe:2.3:a:palletsprojects:flask
+flood,
+fluxbb,cpe:2.3:a:fluxbb:fluxbb
+focalboard,cpe:2.3:a:mattermost:focalboard
+foodsoft,
+framaestro,
+framaestro_hub,
+framaforms,
+framagames,
+freeboard,
+freepbx,cpe:2.3:a:freepbx:freepbx
+freshrss,cpe:2.3:a:freshrss:freshrss
+friendica,cpe:2.3:a:friendica:friendica
+ftp_webapp,
+ftssolr,
+funkwhale,
+galene,
+galette,cpe:2.3:a:galette:galette
+gamja,
+garradin,
+gateone,cpe:2.3:a:liftoffsoftware:gate_one
+gekko,
+gemserv,
+getsimple,cpe:2.3:a:get-simple:getsimple_cms
+ghost,cpe:2.3:a:ghost:ghost
+gitea,cpe:2.3:a:gitea:gitea
+gitlab,cpe:2.3:a:gitlab:gitlab
+gitlab-runner,cpe:2.3:a:gitlab:runner
+gitolite,cpe:2.3:a:gitolite:gitolite
+gitrepositories,
+gitweb,
+glitchsoc,
+glowingbear,
+glpi,cpe:2.3:a:glpi-project:glpi
+gnusocial,
+gogs,cpe:2.3:a:gogs:gogs
+gogswebhost,
+gollum,cpe:2.3:a:gollum_project:gollum
+gossa,
+gotify,
+gotosocial,
+grafana,cpe:2.3:a:grafana:grafana
+grammalecte,
+grav,
+grocy,cpe:2.3:a:grocy_project:grocy
+grr,cpe:2.3:a:devome:grr
+guacamole,cpe:2.3:a:apache:guacamole
+h5ai,cpe:2.3:a:h5ai_project:h5ai
+halcyon,
+haste,
+hat,
+headphones,
+hedgedoc,cpe:2.3:a:hedgedoc:hedgedoc
+helloworld,
+hextris,
+homeassistant,
+horde,cpe:2.3:a:horde:horde_application_framework
+hotspot,
+htmltool,
+htpc-manager,
+hubzilla,cpe:2.3:a:hubzilla:hubzilla
+huginn,
+humhub,cpe:2.3:a:humhub:humhub
+hydrogen,
+icecoder,cpe:2.3:a:icecoder:icecoder
+ifconfig-io,
+ifm,
+ihatemoney,cpe:2.3:a:ihatemoney:i_hate_money
+indexhibit,cpe:2.3:a:indexhibit:indexhibit
+internetarchive,
+invidious,
+invoiceninja,cpe:2.3:a:invoiceninja:invoice_ninja
+invoiceninja5,
+jackett,
+jappix,cpe:2.3:a:jappix_project:jappix
+jappix_mini,
+jeedom,cpe:2.3:a:jeedom:jeedom
+jellyfin,cpe:2.3:a:jellyfin:jellyfin
+jenkins,cpe:2.3:a:jenkins:jenkins
+jirafeau,cpe:2.3:a:jirafeau:jirafeau
+jitsi,cpe:2.3:a:jitsi:jitsi
+joomla,cpe:2.3:a:joomla:joomla\!
+jupyterlab,cpe:2.3:a:jupyter:nbdime-jupyterlab
+kanboard,cpe:2.3:a:kanboard:kanboard
+keeweb,
+kimai2,cpe:2.3:a:kimai:kimai_2
+kiwiirc,
+kiwix,cpe:2.3:a:kiwix:kiwix
+kodi,cpe:2.3:a:kodi:kodi
+komga,
+kresus,
+languagetool,
+laverna,
+lbcalerte,
+leed,
+lektor,
+lemmy,
+librarian,
+libreddit,
+libreerp,
+librephotos,
+librespeed,
+libreto,
+librex,
+lidarr,
+limesurvey,cpe:2.3:a:limesurvey:limesurvey
+linuxdash,
+lionwiki-t2t,
+listmonk,
+lstu,
+luckysheet,
+lufi,
+lutim,cpe:2.3:a:lutim_project:lutim
+lxd,
+lxd-dashboard,
+lychee,cpe:2.3:a:lycheeorganisation:lychee-v3
+mailman,cpe:2.3:a:gnu:mailman
+mailman3,
+mantis,cpe:2.3:a:mantisbt:mantisbt
+mastodon,cpe:2.3:a:joinmastodon:mastodon
+matomo,cpe:2.3:a:matomo:matomo
+matrix-puppet-discord,
+matterbridge,
+mattermost,cpe:2.3:a:mattermost:mattermost
+mautic,cpe:2.3:a:acquia:mautic
+mautrix_facebook,
+mautrix_signal,
+mautrix_telegram,
+mautrix_whatsapp,
+mediadrop,
+mediawiki,cpe:2.3:a:mediawiki:mediawiki
+medusa,
+meilisearch,
+menu,
+metabase,cpe:2.3:a:metabase:metabase
+minchat,
+mindmaps,
+minetest,cpe:2.3:a:minetest:minetest
+mineweb,
+minidlna,cpe:2.3:a:readymedia_project:readymedia
+miniflux,
+minio,cpe:2.3:a:minio:minio
+misskey,cpe:2.3:a:misskey:misskey
+mobilizon,
+modernpaste,
+monica,cpe:2.3:a:monicahq:monica
+monit,
+monitorix,cpe:2.3:a:fibranet:monitorix
+moodle,cpe:2.3:a:moodle:moodle
+mopidy,
+mosquitto,cpe:2.3:a:eclipse:mosquitto
+movim,cpe:2.3:a:movim:movim
+multi_webapp,
+mumble-web,
+mumble_admin_plugin,
+mumbleserver,
+munin,cpe:2.3:a:munin-monitoring:munin
+my-mind,
+my_capsule,
+my_webapp,
+mybb,cpe:2.3:a:mybb:mybb
+mycryptochat,
+mygpo,
+mytinytodo,cpe:2.3:a:mytinytodo:mytinytodo
+navidrome,cpe:2.3:a:navidrome:navidrome
+netdata,cpe:2.3:a:netdata:netdata
+neutrinet,
+nextcloud,cpe:2.3:a:nextcloud:nextcloud
+nexusoss,
+nitter,
+noalyss,
+nocodb,cpe:2.3:a:xgenecloud:nocodb
+nodebb,cpe:2.3:a:nodebb:nodebb
+nodered,cpe:2.3:a:nodered:node-red-dashboard
+nomad,cpe:2.3:a:jenkins:nomad
+ntopng,cpe:2.3:a:ntop:ntopng
+nullboard,
+ofbiz,cpe:2.3:a:apache:ofbiz
+omeka-s,
+onlyoffice,cpe:2.3:a:onlyoffice:document_server
+openidsimplesamlphp,
+opennote,
+openproject,cpe:2.3:a:openproject:openproject
+opensondage,
+opentracker,
+osada,
+osjs,
+osmw,
+osticket,cpe:2.3:a:osticket:osticket
+outline,
+overleaf,
+owncast,cpe:2.3:a:owncast_project:owncast
+owncloud,cpe:2.3:a:owncloud:owncloud
+owntracks,
+pagure,cpe:2.3:a:fedoraproject:389_directory_server
+paperless-ngx,
+peachpub,
+peertube,cpe:2.3:a:framasoft:peertube
+peertube-search-index,
+pelican,
+pepettes,
+petitesannonces,
+petrolette,
+pgadmin,cpe:2.3:a:phppgadmin_project:phppgadmin
+photonix,
+photoprism,
+photoview,
+phpback,
+phpbb,cpe:2.3:a:phpbb:phpbb
+phpboost,
+phpinfo,
+phpipam,cpe:2.3:a:phpipam:phpipam
+phpldapadmin,cpe:2.3:a:phpldapadmin_project:phpldapadmin
+phplicensewatcher,
+phpmyadmin,cpe:2.3:a:phpmyadmin:phpmyadmin
+phpservermon,cpe:2.3:a:phpservermonitor:php_server_monitor
+phpsysinfo,
+pia,
+pico,
+pihole,cpe:2.3:a:pi-hole:pi-hole
+piratebox,
+piwigo,cpe:2.3:a:piwigo:piwigo
+pixelfed,
+plainpad,
+pleroma,
+plonecms,
+plume,cpe:2.3:a:plume-cms:plume_cms
+pluxml,cpe:2.3:a:pluxml:pluxml
+pmwiki,cpe:2.3:a:pmwiki:pmwiki
+portainer,cpe:2.3:a:portainer:portainer
+prestashop,cpe:2.3:a:prestashop:prestashop
+prettynoemiecms,
+privatebin,cpe:2.3:a:privatebin:privatebin
+proftpd,cpe:2.3:a:proftpd:proftpd
+prometheus,cpe:2.3:a:prometheus:prometheus
+prosody,cpe:2.3:a:prosody:prosody
+prowlarr,
+proxitok,
+psitransfer,
+pterodactyl,cpe:2.3:a:pterodactyl:panel
+pufferpanel,
+pydio,cpe:2.3:a:pydio:pydio
+pyinventory,
+pyload,
+pytition,
+qr,
+question2answer,cpe:2.3:a:question2answer:question2answer
+quizzes,
+radarr,
+radicale,cpe:2.3:a:radicale:radicale
+rainloop,cpe:2.3:a:rainloop:webmail
+redirect,
+redmine,cpe:2.3:a:redmine:redmine
+reel2bits,
+remotestorage,
+restic,
+retroarch,cpe:2.3:a:libretro:retroarch
+riot,
+roadiz,
+rocketchat,cpe:2.3:a:rocket.chat:rocket.chat
+roundcube,cpe:2.3:a:roundcube:webmail
+rportd,
+rspamdui,
+rss-bridge,
+rutorrent,
+samba,cpe:2.3:a:samba:samba
+sat,
+satdress,
+scm,
+scratch,
+scrumblr,
+seafile,cpe:2.3:a:seafile:seafile
+searx,
+seenthis,
+selfoss,
+send,
+shaarli,cpe:2.3:a:shaarli_project:shaarli
+shadowsocks,cpe:2.3:a:shadowsocks:shadowsocks-libev
+shellinabox,cpe:2.3:a:shellinabox_project:shellinabox
+shinken,
+shiori,
+shsd,
+shuri,
+sickbeard,
+sickrage,cpe:2.3:a:sickrage:sickrage
+signaturepdf,
+simple-hash-generator,
+simple-torrent,
+sitemagiccms,cpe:2.3:a:sitemagic:sitemagic
+slingcode,
+snappymail,
+snipeit,cpe:2.3:a:snipeitapp:snipe-it
+snserver,
+snweb,
+soapbox,cpe:2.3:a:soapbox_project:soapbox
+sogo,cpe:2.3:a:inverse:sogo
+sonarr,
+sonerezh,
+spacedeck,
+spftoolbox,
+sphinx,cpe:2.3:a:sphinxsearch:sphinx
+spip,cpe:2.3:a:spip:spip
+squid3,
+ssbroom,
+ssh_chroot_dir,
+staticwebapp,
+streama,
+strut,
+subscribe,
+subsonic,cpe:2.3:a:subsonic:subsonic
+sutom,
+svgedit,
+sympa,cpe:2.3:a:sympa:sympa
+synapse,
+synapse-admin,
+syncthing,cpe:2.3:a:syncthing:syncthing
+tagspaces,
+tailoredflow,
+teampass,cpe:2.3:a:teampass:teampass
+technitium-dns,
+teddit,
+telegram_chatbot,
+tes3mp,
+thelounge,
+tiddlywiki,
+tiki,cpe:2.3:a:tiki:tiki
+timemachine,
+timeoff,
+tooljet,
+torrelay,
+tracim,
+traggo,
+transfersh,
+transmission,
+transpay,
+transwhat,
+trilium,cpe:2.3:a:trilium_project:trilium
+trustyhash,
+ttrss,
+turtl,cpe:2.3:a:lyonbros:turtl
+tutao,
+tvheadend,
+tyto,
+ulogger,
+umami,
+umap,
+ums,
+unattended_upgrades,
+unbound,cpe:2.3:a:nlnetlabs:unbound
+uptime-kuma,
+vaultwarden,
+veloren,
+vikunja,
+vpnclient,
+vpnserver,
+wallabag2,
+weblate,cpe:2.3:a:weblate:weblate
+webmin,cpe:2.3:a:webmin:webmin
+webogram,
+webtrees,cpe:2.3:a:webtrees:webtrees
+wekan,cpe:2.3:a:wekan_project:wekan
+wemawema,
+wetty,
+whitebophir,
+wifiwithme,
+wikijs,
+wildfly,cpe:2.3:a:redhat:jboss_wildfly_application_server
+wireguard,
+wisemapping,
+wondercms,cpe:2.3:a:wondercms:wondercms
+wordpress,cpe:2.3:a:wordpress:wordpress
+writefreely,
+yacy,
+yellow,
+yeswiki,cpe:2.3:a:yeswiki:yeswiki
+yourls,cpe:2.3:a:yourls:yourls
+youtube-dl-webui,
+yunofav,
+yunohost,cpe:2.3:o:yunohost:yunohost
+yunomonitor,
+yunorunner,
+z-push,
+zabbix,cpe:2.3:a:zabbix:zabbix
+zap,
+zerobin,
+zeronet,
+zomburl,
+ztncui,
+zusam,

autopatches/patches/add-cpe/patch.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/python3
+
+import json
+import csv
+
+def find_cpe(app_id):
+    with open("../../patches/add-cpe/cpe.csv", newline='') as f:
+        cpe_list = csv.reader(f)
+        for row in cpe_list:
+            if row[0] == app_id:
+                return row[1]
+        return False
+
+manifest = json.load(open("manifest.json"))
+app_id = manifest['id']
+cpe = find_cpe(app_id)
+if cpe:
+    manifest['upstream']['cpe'] = cpe
+    json.dump(manifest, open("manifest.json", "w"), indent=4, ensure_ascii=False)

autopatches/patches/add-cpe/pr_body.md

@@ -0,0 +1,6 @@
+
+This is an ***automated*** patch to add the (optional but recommended if relevant) Common Platform Enumeration (CPE) id, which is sort of a standard id for applications, defined by the NIST.
+
+In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps.
+
+The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)").

autopatches/patches/add-cpe/pr_title.md

@@ -0,0 +1 @@
+Add Common Platform Enumeration id to `manifest.json`

autopatches/patches/issue-and-pr-template/patch.sh

@@ -0,0 +1,14 @@
+
+[ ! -e issue_template.md ] || git rm issue_template.md
+[ ! -e pull_request_template.md ] || git rm pull_request_template.md
+
+[ ! -e .github ] || git rm -rf .github
+mkdir -p .github
+
+# Sleep 1 to avoid too many requests on github (there's a rate limit anyway)
+sleep 1
+
+wget -O .github/ISSUE_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/ISSUE_TEMPLATE.md
+wget -O .github/PULL_REQUEST_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/PULL_REQUEST_TEMPLATE.md
+
+git add .github

autopatches/patches/issue-and-pr-template/pr_body.md

@@ -0,0 +1,2 @@
+
+This is an ***automated*** patch to sync the issue and PR template with the official templates in https://github.com/YunoHost/example_ynh/tree/master/.github

autopatches/patches/issue-and-pr-template/pr_title.md

@@ -0,0 +1 @@
+Update issue and PR templates

autopatches/patches/missing-seteu-in-change_url/patch.sh

@@ -0,0 +1,10 @@
+
+cd scripts/
+
+if [ ! -e change_url ] || grep -q 'ynh_abort_if_errors' change_url
+then
+    # The app doesn't has any change url script or already has ynh_abort_if_error
+    exit 0
+fi
+
+sed 's@\(source /usr/share/yunohost/helpers\)@\1\nynh_abort_if_errors@g' -i change_url

autopatches/patches/missing-seteu-in-change_url/pr_body.md

@@ -0,0 +1,2 @@
+
+This is an ***automated*** patch to fix the lack of `ynh_abort_if_errors` in change_url script

autopatches/patches/missing-seteu-in-change_url/pr_title.md

@@ -0,0 +1 @@
+Missing ynh_abort_if_errors in change_url scripts

autopatches/patches/new-permission-system/patch.sh

@@ -0,0 +1,66 @@
+
+cd scripts/
+
+if grep -q 'ynh_legacy_permissions' upgrade || grep -q 'ynh_permission_' install
+then
+    # App already using the new permission system - not patching anything
+    exit 0
+fi
+
+if ! grep -q "protected_\|skipped_" install
+then
+    # App doesn't has any (un)protected / skipped setting ?
+    # Probably not a webapp or permission ain't relevant for it ?
+    exit 0
+fi
+
+CONFIGURE_PERMISSION_DURING_INSTALL='
+# Make app public if necessary
+if [ \"\$is_public\" -eq 1 ]
+then
+	ynh_permission_update --permission=\"main\" --add=\"visitors\"
+fi
+'
+
+MIGRATE_LEGACY_PERMISSIONS='
+#=================================================
+# Migrate legacy permissions to new system
+#=================================================
+if ynh_legacy_permissions_exists
+then
+	ynh_legacy_permissions_delete_all
+
+	ynh_app_setting_delete --app=\$app --key=is_public
+fi'
+
+for SCRIPT in "remove upgrade backup restore change_url"
+do
+    [[ -e $SCRIPT ]] || continue
+
+    perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i $SCRIPT
+    perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i $SCRIPT
+    perl -p0e 's@\s*if.*-z.*is_public.*(.|\n)*?fi\s@\n@g' -i $SCRIPT
+    perl -p0e 's@\s*if.*is_public.*(-eq|=).*(.|\n)*?fi\s@\n@g' -i $SCRIPT
+    perl -p0e 's@is_public=.*\n@@g' -i $SCRIPT
+    perl -p0e 's@ynh_app_setting_.*is_public.*@@g' -i $SCRIPT
+    perl -p0e 's@.*# Make app .*@@g' -i $SCRIPT
+    perl -p0e 's@.*# Fix is_public as a boolean.*@@g' -i $SCRIPT
+    perl -p0e 's@.*# If app is public.*@@g' -i $SCRIPT
+    perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i $SCRIPT
+    perl -p0e 's@.*ynh_script_progression.*SSOwat.*@@g' -i $SCRIPT
+    perl -p0e 's@#=*\s#.*SETUP SSOWAT.*\s#=*\s@@g' -i $SCRIPT
+done
+
+
+perl -p0e 's@.*domain_regex.*@@g' -i install
+perl -p0e 's@.*# If app is public.*@@g' -i install
+perl -p0e 's@.*# Make app .*@@g' -i install
+perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i install
+perl -p0e "s@if.*is_public.*(-eq|=)(.|\n){0,100}setting(.|\n)*?fi\n@$CONFIGURE_PERMISSION_DURING_INSTALL@g" -i install
+perl -p0e 's@.*ynh_app_setting_.*is_public.*\s@@g' -i install
+perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i install
+perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i install
+
+grep -q 'is_public=' install || perl -p0e 's@(.*Configuring SSOwat.*)@\1\nynh_permission_update --permission=\"main\" --add=\"visitors\"@g' -i install
+
+perl -p0e "s@ynh_abort_if_errors@ynh_abort_if_errors\n$MIGRATE_LEGACY_PERMISSIONS@g" -i upgrade

autopatches/patches/new-permission-system/pr_body.md

@@ -0,0 +1,11 @@
+
+NB. : this is an ***automated*** attempt to migrate the app to the new permission system
+
+You should ***not*** blindly trust the proposed changes. In particular, the auto-patch will not handle:
+- situations which are more complex than "if is_public is true, allow visitors"
+- situations where the app needs to be temporarily public (then possible private) during initial configuration
+- apps that need to define extra permission for specific section of the app (such as admin interface)
+- apps using non-standard syntax
+- other specific use cases
+
+***PLEASE*** carefully review, test and amend the proposed changes if you find that the autopatch did not do a proper job.

autopatches/patches/new-permission-system/pr_title.md

@@ -0,0 +1 @@
+Autopatch to migrate to new permission system

packaging_v2/convert_app_to_packaging_v2.py

@@ -0,0 +1,334 @@
+import argparse
+import os
+import re
+import json
+import subprocess
+
+
+def check_output(cmd):
+    return (
+        subprocess.check_output(cmd, shell=True)
+        .decode("utf-8")
+        .strip()
+    )
+
+
+def _convert_v1_manifest_to_v2(app_path):
+
+    manifest = json.load(open(app_path + "/manifest.json"))
+
+    if "upstream" not in manifest:
+        manifest["upstream"] = {}
+
+    if "license" in manifest and "license" not in manifest["upstream"]:
+        manifest["upstream"]["license"] = manifest["license"]
+
+    if "url" in manifest and "website" not in manifest["upstream"]:
+        manifest["upstream"]["website"] = manifest["url"]
+
+    manifest["upstream"]["cpe"] = "???"
+    manifest["upstream"]["fund"] = "???"
+
+    manifest["integration"] = {
+        "yunohost": manifest.get("requirements", {}).get("yunohost"),
+        "architectures": "all",
+        "multi_instance": manifest.get("multi_instance", False),
+        "ldap": "?",
+        "sso": "?",
+        "disk": "50M",
+        "ram.build": "50M",
+        "ram.runtime": "50M"
+    }
+
+    maintainers = manifest.get("maintainer", {})
+    if isinstance(maintainers, list):
+        maintainers = [m['name'] for m in maintainers]
+    else:
+        maintainers = [maintainers["name"]] if maintainers.get("name") else []
+
+    manifest["maintainers"] = maintainers
+
+    install_questions = manifest["arguments"]["install"]
+    manifest["install"] = {}
+    for question in install_questions:
+        name = question.pop("name")
+        if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]:
+            question.pop("ask")
+        if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]:
+            question.pop("example")
+
+        manifest["install"][name] = question
+
+    # Rename is_public to init_main_permission
+    manifest["install"] = {(k if k != "is_public" else "init_main_permission"): v for k, v in manifest["install"].items()}
+
+    if "init_main_permission" in manifest["install"]:
+        manifest["install"]["init_main_permission"]["type"] = "group"
+        if manifest["install"]["init_main_permission"].get("default") is True:
+            manifest["install"]["init_main_permission"]["default"] = "visitors"
+        elif manifest["install"]["init_main_permission"].get("default") is True:
+            manifest["install"]["init_main_permission"]["default"] = "all_users"
+
+    if "domain" in manifest["install"] and "path" not in manifest["install"]:
+        manifest["install"]["domain"]["full_domain"] = True
+
+    manifest["resources"] = {}
+    manifest["resources"]["system_user"] = {}
+    manifest["resources"]["install_dir"] = {}
+
+    if os.system(f"grep -q 'datadir=' {app_path}/scripts/install") == 0:
+        manifest["resources"]["data_dir"] = {}
+
+    manifest["resources"]["permissions"] = {}
+
+    if os.system(f"grep -q 'ynh_webpath_register' '{app_path}/scripts/install'") == 0:
+        manifest["resources"]["permissions"]["main.url"] = "/"
+
+    # FIXME: Parse ynh_permission_create --permission="admin" --url="/wp-login.php" --additional_urls="/wp-admin.php" --allowed=$admin_wordpress
+
+    ports = check_output(f"sed -nr 's/(\\w+)=.*ynh_find_port[^0-9]*([0-9]+)\\)/\\1,\\2/p' '{app_path}/scripts/install'")
+    if ports:
+        manifest["resources"]["ports"] = {}
+        for port in ports.split("\n"):
+            name, default = port.split(",")
+            exposed = check_output(f"sed -nr 's/.*yunohost firewall allow .*(TCP|UDP|Both).*${name}/\\1/p' '{app_path}/scripts/install'")
+            if exposed == "Both":
+                exposed = True
+
+            name = name.replace("_port", "").replace("port_", "")
+            if name == "port":
+                name = "main"
+
+            manifest["resources"]["ports"][f"{name}.default"] = int(default)
+            if exposed:
+                manifest["resources"]["ports"][f"{name}.exposed"] = exposed
+
+    maybequote = "[\"'\"'\"']?"
+    apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d '\"' | sed 's@ @\\n@g'")
+    php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d \"\\\"'\"")
+    if apt_dependencies.strip():
+        if php_version:
+            apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version)
+        apt_dependencies = ', '.join([d for d in apt_dependencies.split("\n") if d])
+        manifest["resources"]["apt"] = {"packages": apt_dependencies}
+
+    extra_apt_repos = check_output(r"sed -nr 's/.*_extra_app_dependencies.*repo=\"(.*)\".*package=\"(.*)\".*key=\"(.*)\"/\1,\2,\3/p' %s/scripts/install" % app_path)
+    if extra_apt_repos:
+        for i, extra_apt_repo in enumerate(extra_apt_repos.split("\n")):
+            repo, packages, key = extra_apt_repo.split(",")
+            packages = packages.replace('$', '#FIXME#$')
+            if "apt" not in manifest["resources"]:
+                manifest["resources"]["apt"] = {}
+            if "extras" not in manifest["resources"]["apt"]:
+                manifest["resources"]["apt"]["extras"] = []
+            manifest["resources"]["apt"]["extras"].append({
+                "repo": repo,
+                "key": key,
+                "packages": packages,
+            })
+
+    if os.system(f"grep -q 'ynh_mysql_setup_db' {app_path}/scripts/install") == 0:
+        manifest["resources"]["database"] = {"type": "mysql"}
+    elif os.system(f"grep -q 'ynh_psql_setup_db' {app_path}/scripts/install") == 0:
+        manifest["resources"]["database"] = {"type": "postgresql"}
+
+    keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"]
+
+    keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep]
+    for key in keys_to_del:
+        del manifest[key]
+
+    return manifest
+
+
+def _dump_v2_manifest_as_toml(manifest):
+
+    import re
+    from tomlkit import document, nl, table, dumps, comment
+
+    toml_manifest = document()
+    toml_manifest.add("packaging_format", 2)
+    toml_manifest.add(nl())
+    toml_manifest.add("id", manifest["id"])
+    toml_manifest.add("name", manifest["name"])
+    for lang, value in manifest["description"].items():
+        toml_manifest.add(f"description.{lang}", value)
+    toml_manifest.add(nl())
+    toml_manifest.add("version", manifest["version"])
+    toml_manifest.add(nl())
+    toml_manifest.add("maintainers", manifest["maintainers"])
+
+    upstream = table()
+    for key, value in manifest["upstream"].items():
+        upstream[key] = value
+    upstream["cpe"].comment("FIXME: optional but recommended if relevant, this is meant to contain the Common Platform Enumeration, which is sort of a standard id for applications defined by the NIST. In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)")
+    upstream["fund"].comment("FIXME: optional but recommended (or remove if irrelevant / not applicable). This is meant to be an URL where people can financially support this app, especially when its development is based on volunteers and/or financed by its community. YunoHost may later advertise it in the webadmin.")
+    toml_manifest["upstream"] = upstream
+
+    integration = table()
+    for key, value in manifest["integration"].items():
+        integration.add(key, value)
+    integration["architectures"].comment('FIXME: can be replaced by a list of supported archs using the dpkg --print-architecture nomenclature (amd64/i386/armhf/arm64/armel), for example: ["amd64", "i386"]')
+    integration["ldap"].comment('FIXME: replace with true, false, or "not_relevant"')
+    integration["sso"].comment('FIXME: replace with true, false, or "not_relevant"')
+    integration["disk"].comment('FIXME: replace with an **estimate** minimum disk requirement. e.g. 20M, 400M, 1G, ...')
+    integration["ram.build"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...')
+    integration["ram.runtime"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...')
+    toml_manifest["integration"] = integration
+
+    install = table()
+    for key, value in manifest["install"].items():
+        install[key] = table()
+        install[key].indent(4)
+
+        if key in ["domain", "path", "admin", "is_public", "password"]:
+            install[key].add(comment("this is a generic question - ask strings are automatically handled by Yunohost's core"))
+
+        for lang, value2 in value.get("ask", {}).items():
+            install[key].add(f"ask.{lang}", value2)
+
+        for lang, value2 in value.get("help", {}).items():
+            install[key].add(f"help.{lang}", value2)
+
+        for key2, value2 in value.items():
+            if key2 in ["ask", "help"]:
+                continue
+            install[key].add(key2, value2)
+
+    toml_manifest["install"] = install
+
+    resources = table()
+    for key, value in manifest["resources"].items():
+        resources[key] = table()
+        resources[key].indent(4)
+        for key2, value2 in value.items():
+            resources[key].add(key2, value2)
+            if key == "apt" and key2 == "extras":
+                for extra in resources[key][key2]:
+                    extra.indent(8)
+
+    toml_manifest["resources"] = resources
+
+    toml_manifest_dump = dumps(toml_manifest)
+
+    regex = re.compile(r'\"((description|ask|help)\.[a-z]{2})\"')
+    toml_manifest_dump = regex.sub(r'\1', toml_manifest_dump)
+    toml_manifest_dump = toml_manifest_dump.replace('"ram.build"', "ram.build")
+    toml_manifest_dump = toml_manifest_dump.replace('"ram.runtime"', "ram.runtime")
+    toml_manifest_dump = toml_manifest_dump.replace('"main.url"', "main.url")
+    toml_manifest_dump = toml_manifest_dump.replace('"main.default"', "main.default")
+    return toml_manifest_dump
+
+
+def cleanup_scripts_and_conf(folder):
+
+    patterns_to_remove_in_scripts = [
+        "^.*ynh_abort_if_errors.*$",
+        "^.*YNH_APP_ARG.*$",
+        "^.*YNH_APP_INSTANCE_NAME.*$",
+        r"^ *final_path=",
+        r"^\s*final_path=",
+        "^.*test .*-(e|d) .*final_path.*$",
+        "^.*ynh_webpath_register.*$",
+        "^.*ynh_webpath_available.*$",
+        "^.*ynh_system_user_create.*$",
+        "^.*ynh_system_user_delete.*$",
+        "^.*ynh_permission_update.*$",
+        "^.*ynh_permission_create.*$",
+        "^.*if .*ynh_permission_exists.*$",
+        "^.*if .*ynh_legacy_permissions_exists.*$",
+        "^.*ynh_legacy_permissions_delete_all.*$",
+        "^.*ynh_app_setting_set .*(domain|path|final_path|admin|password|port|datadir|db_name|db_user|db_pwd).*$",
+        "^.*ynh_app_setting_.* is_public.*$",
+        r"^.*if.*\$is_public.*$",
+        "^.*_dependencies=.*$",
+        "^.*ynh_install_app_dependencies.*$",
+        "^.*ynh_install_extra_app_dependencies.*$",
+        "^.*ynh_remove_app_dependencies.*$",
+        r"^.*\$\(ynh_app_setting_get.*$",
+        r"^.*ynh_secure_remove .*\$final_path.*$",
+        r"^.*ynh_secure_remove .*\$datadir.*$",
+        "^.*ynh_backup_before_upgrade.*$",
+        "^.*ynh_clean_setup.*$",
+        "^.*ynh_restore_upgradebackup.*$",
+        "^db_name=.*$",
+        "^db_user=.*$",
+        "^db_pwd=.*$",
+        "^datadir=.*$",
+        "^.*ynh_psql_test_if_first_run.*$",
+        "^.*ynh_mysql_setup_db.*$",
+        "^.*ynh_psql_setup_db.*$",
+        "^.*ynh_mysql_remove_db.*$",
+        "^.*ynh_psql_remove_db.*$",
+        "^.*ynh_find_port.*$",
+        "^.*ynh_script_progression.*Finding an available port",
+        "^.*ynh_script_progression.*Backing up the app before upgrading",
+        "^.*ynh_script_progression.*Creating data directory",
+        "^.*ynh_script_progression.*system user",
+        "^.*ynh_script_progression.*installation settings",
+        "^.*ynh_print_info.*installation settings",
+        r"^.*ynh_script_progression.*\w+ dependencies",
+        "^.*ynh_script_progression.*Removing app main dir",
+        "^.*ynh_script_progression.*Validating.*parameters",
+        "^.*ynh_script_progression.*SQL database",
+        "^.*ynh_script_progression.*Configuring permissions",
+    ]
+    patterns_to_remove_in_scripts = [re.compile(f"({p})", re.MULTILINE) for p in patterns_to_remove_in_scripts]
+
+    replaces = [
+        ("path_url", "path"),
+        ("PATH_URL", "PATH"),
+        ("final_path", "install_dir"),
+        ("FINALPATH", "INSTALL_DIR"),
+        ("datadir", "data_dir"),
+        ("DATADIR", "DATA_DIR"),
+    ]
+
+    for s in ["_common.sh", "install", "remove", "upgrade", "backup", "restore"]:
+
+        script = f"{folder}/scripts/{s}"
+
+        if not os.path.exists(script):
+            continue
+
+        content = open(script).read()
+
+        for pattern in patterns_to_remove_in_scripts:
+            content = pattern.sub(r"#REMOVEME? \1", content)
+
+        for pattern, replace in replaces:
+            content = content.replace(pattern, replace)
+
+        open(script, "w").write(content)
+
+    for conf in os.listdir(f"{folder}/conf"):
+
+        conf = f"{folder}/conf/{conf}"
+
+        if not os.path.isfile(conf):
+            continue
+
+        content = open(conf).read()
+        content_init = content
+
+        for pattern, replace in replaces:
+            content = content.replace(pattern, replace)
+
+        if content_init != content:
+            open(conf, "w").write(content)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Attempt to automatically convert a v1 YunoHost app to v2 (at least as much as possible) : parse the app scripts to auto-generate the manifest.toml, and remove now-useless lines from the app scripts"
+    )
+    parser.add_argument(
+        "app_path", help="Path to the app to convert"
+    )
+
+    args = parser.parse_args()
+
+    manifest = _convert_v1_manifest_to_v2(args.app_path)
+    open(args.app_path + "/manifest.toml", "w").write(_dump_v2_manifest_as_toml(manifest))
+
+    cleanup_scripts_and_conf(args.app_path)

packaging_v2/convert_v1_manifest_to_v2_for_catalog.py

@@ -0,0 +1,60 @@
+import copy
+
+
+def convert_v1_manifest_to_v2_for_catalog(manifest):
+
+    manifest = copy.deepcopy(manifest)
+
+    if "upstream" not in manifest:
+        manifest["upstream"] = {}
+
+    if "license" in manifest and "license" not in manifest["upstream"]:
+        manifest["upstream"]["license"] = manifest["license"]
+
+    if "url" in manifest and "website" not in manifest["upstream"]:
+        manifest["upstream"]["website"] = manifest["url"]
+
+    manifest["integration"] = {
+        "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""),
+        "architectures": "all",
+        "multi_instance": manifest.get("multi_instance", False),
+        "ldap": "?",
+        "sso": "?",
+        "disk": "50M",
+        "ram": {"build": "50M", "runtime": "10M"}
+    }
+
+    maintainers = manifest.get("maintainer", {})
+    if isinstance(maintainers, list):
+        maintainers = [m['name'] for m in maintainers]
+    else:
+        maintainers = [maintainers["name"]] if maintainers.get("name") else []
+
+    manifest["maintainers"] = maintainers
+
+    install_questions = manifest["arguments"]["install"]
+
+    manifest["install"] = {}
+    for question in install_questions:
+        name = question.pop("name")
+        if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]:
+            question.pop("ask")
+        if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]:
+            question.pop("example")
+
+        manifest["install"][name] = question
+
+    manifest["resources"] = {
+        "system_user": {},
+        "install_dir": {
+            "alias": "final_path"
+        }
+    }
+
+    keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"]
+
+    keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep]
+    for key in keys_to_del:
+        del manifest[key]
+
+    return manifest