8a0cf61637
This properties is not used by the core. It's deduced from the fact that there's no path properties.
479 lines
20 KiB
Python
479 lines
20 KiB
Python
import argparse
|
|
import os
|
|
import re
|
|
import json
|
|
import subprocess
|
|
from glob import glob
|
|
|
|
|
|
def check_output(cmd):
|
|
return (
|
|
subprocess.check_output(cmd, shell=True)
|
|
.decode("utf-8")
|
|
.strip()
|
|
)
|
|
|
|
|
|
def convert_app_sources(folder):
|
|
|
|
def parse_and_convert_src(filename):
|
|
|
|
D = {}
|
|
raw = open(filename).read()
|
|
for line in raw.split("\n"):
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
key = key.replace("SOURCE_", "").lower()
|
|
D[key] = value
|
|
|
|
new_D = {
|
|
"url": D["url"],
|
|
"sha256": D["sum"],
|
|
}
|
|
|
|
if D.get("format", "tar.gz") not in ["zip", "tar.gz", "tar.xz", "tgz", "tar.bz2"]:
|
|
new_D["format"] = D["format"]
|
|
if "filename" in D:
|
|
new_D["rename"] = D["filename"]
|
|
elif "in_subdir" in D and D["in_subdir"] != "true":
|
|
new_D["in_subdir"] = False
|
|
|
|
return new_D
|
|
|
|
sources = {}
|
|
|
|
remap_id = {
|
|
"app": "main",
|
|
"amd64": "main.amd64",
|
|
"i386": "main.i386",
|
|
"arm64": "main.arm64",
|
|
"armhf": "main.armhf",
|
|
"arm7": "main.armhf",
|
|
"app.arm64": "main.arm64",
|
|
"app.x86_64": "main.amd64",
|
|
"app.x64": "main.amd64",
|
|
"app.arm": "main.armhf",
|
|
"app.armhf": "main.armhf",
|
|
"app.armv7": "main.armhf",
|
|
"app.386": "main.i386",
|
|
"app.x86": "main.i386",
|
|
"app.armel": "main.armel",
|
|
"armel": "main.armel",
|
|
"aarch64": "main.arm64",
|
|
"x86-64": "main.amd64",
|
|
"armv6": "main.armel",
|
|
"armv7": "main.armhf",
|
|
}
|
|
|
|
for filename in glob(folder + "/conf/*.src"):
|
|
id_ = os.path.basename(filename).rsplit(".", 1)[0]
|
|
if id_ in remap_id:
|
|
id_ = remap_id[id_]
|
|
|
|
sources[id_] = parse_and_convert_src(filename)
|
|
|
|
if id_.startswith("main."):
|
|
if "main" not in sources:
|
|
sources["main"] = sources[id_]
|
|
arch = id_.split(".")[1]
|
|
sources["main"][arch + ".url"] = sources[id_]["url"]
|
|
sources["main"][arch + ".sha256"] = sources[id_]["sha256"]
|
|
del sources[id_]["url"]
|
|
del sources[id_]["sha256"]
|
|
del sources[id_]
|
|
|
|
os.system(f"rm '{filename}'")
|
|
|
|
return sources
|
|
|
|
|
|
def _convert_v1_manifest_to_v2(app_path):
|
|
|
|
manifest = json.load(open(app_path + "/manifest.json"))
|
|
|
|
if "upstream" not in manifest:
|
|
manifest["upstream"] = {}
|
|
|
|
if "license" in manifest and "license" not in manifest["upstream"]:
|
|
manifest["upstream"]["license"] = manifest["license"]
|
|
|
|
if "url" in manifest and "website" not in manifest["upstream"]:
|
|
manifest["upstream"]["website"] = manifest["url"]
|
|
|
|
manifest["upstream"]["cpe"] = "???"
|
|
manifest["upstream"]["fund"] = "???"
|
|
|
|
manifest["integration"] = {
|
|
"yunohost": manifest.get("requirements", {}).get("yunohost"),
|
|
"architectures": "all",
|
|
"multi_instance": manifest.get("multi_instance", False),
|
|
"ldap": "?",
|
|
"sso": "?",
|
|
"disk": "50M",
|
|
"ram.build": "50M",
|
|
"ram.runtime": "50M"
|
|
}
|
|
|
|
maintainers = manifest.get("maintainer", {})
|
|
if isinstance(maintainers, list):
|
|
maintainers = [m['name'] for m in maintainers]
|
|
else:
|
|
maintainers = [maintainers["name"]] if maintainers.get("name") else []
|
|
|
|
manifest["maintainers"] = maintainers
|
|
|
|
install_questions = manifest["arguments"]["install"]
|
|
manifest["install"] = {}
|
|
for question in install_questions:
|
|
name = question.pop("name")
|
|
if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]:
|
|
question.pop("ask")
|
|
if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]:
|
|
question.pop("example")
|
|
|
|
manifest["install"][name] = question
|
|
|
|
# Rename is_public to init_main_permission
|
|
manifest["install"] = {(k if k != "is_public" else "init_main_permission"): v for k, v in manifest["install"].items()}
|
|
|
|
if "init_main_permission" in manifest["install"]:
|
|
manifest["install"]["init_main_permission"]["type"] = "group"
|
|
if manifest["install"]["init_main_permission"].get("default") is True:
|
|
manifest["install"]["init_main_permission"]["default"] = "visitors"
|
|
elif manifest["install"]["init_main_permission"].get("default") is True:
|
|
manifest["install"]["init_main_permission"]["default"] = "all_users"
|
|
|
|
manifest["resources"] = {}
|
|
|
|
sources = convert_app_sources(app_path)
|
|
if sources:
|
|
manifest["resources"]["sources"] = sources
|
|
|
|
manifest["resources"]["system_user"] = {}
|
|
manifest["resources"]["install_dir"] = {}
|
|
|
|
if os.system(f"grep -q 'datadir=' {app_path}/scripts/install") == 0:
|
|
manifest["resources"]["data_dir"] = {}
|
|
|
|
manifest["resources"]["permissions"] = {}
|
|
|
|
if os.system(f"grep -q 'ynh_webpath_register' '{app_path}/scripts/install'") == 0:
|
|
manifest["resources"]["permissions"]["main.url"] = "/"
|
|
|
|
# FIXME: Parse ynh_permission_create --permission="admin" --url="/wp-login.php" --additional_urls="/wp-admin.php" --allowed=$admin_wordpress
|
|
|
|
ports = check_output(f"sed -nr 's/(\\w+)=.*ynh_find_port[^0-9]*([0-9]+)\\)/\\1,\\2/p' '{app_path}/scripts/install'")
|
|
if ports:
|
|
manifest["resources"]["ports"] = {}
|
|
for port in ports.split("\n"):
|
|
name, default = port.split(",")
|
|
exposed = check_output(f"sed -nr 's/.*yunohost firewall allow .*(TCP|UDP|Both).*${name}/\\1/p' '{app_path}/scripts/install'")
|
|
if exposed == "Both":
|
|
exposed = True
|
|
|
|
name = name.replace("_port", "").replace("port_", "")
|
|
if name == "port":
|
|
name = "main"
|
|
|
|
if not default.isdigit():
|
|
print(f"Failed to parse '{default}' as a port number ... Will use 12345 instead")
|
|
default = 12345
|
|
|
|
manifest["resources"]["ports"][f"{name}.default"] = int(default)
|
|
if exposed:
|
|
manifest["resources"]["ports"][f"{name}.exposed"] = exposed
|
|
|
|
maybequote = "[\"'\"'\"']?"
|
|
apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d '\"' | sed 's@ @\\n@g'")
|
|
php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d \"\\\"'\"")
|
|
if apt_dependencies.strip():
|
|
if php_version:
|
|
apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version)
|
|
apt_dependencies = ', '.join([d for d in apt_dependencies.split("\n") if d])
|
|
manifest["resources"]["apt"] = {"packages": apt_dependencies}
|
|
|
|
extra_apt_repos = check_output(r"sed -nr 's/.*_extra_app_dependencies.*repo=\"(.*)\".*package=\"(.*)\".*key=\"(.*)\"/\1,\2,\3/p' %s/scripts/install" % app_path)
|
|
if extra_apt_repos:
|
|
for i, extra_apt_repo in enumerate(extra_apt_repos.split("\n")):
|
|
repo, packages, key = extra_apt_repo.split(",")
|
|
packages = packages.replace('$', '#FIXME#$')
|
|
if "apt" not in manifest["resources"]:
|
|
manifest["resources"]["apt"] = {}
|
|
if "extras" not in manifest["resources"]["apt"]:
|
|
manifest["resources"]["apt"]["extras"] = []
|
|
manifest["resources"]["apt"]["extras"].append({
|
|
"repo": repo,
|
|
"key": key,
|
|
"packages": packages,
|
|
})
|
|
|
|
if os.system(f"grep -q 'ynh_mysql_setup_db' {app_path}/scripts/install") == 0:
|
|
manifest["resources"]["database"] = {"type": "mysql"}
|
|
elif os.system(f"grep -q 'ynh_psql_setup_db' {app_path}/scripts/install") == 0:
|
|
manifest["resources"]["database"] = {"type": "postgresql"}
|
|
|
|
keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"]
|
|
|
|
keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep]
|
|
for key in keys_to_del:
|
|
del manifest[key]
|
|
|
|
return manifest
|
|
|
|
|
|
def _dump_v2_manifest_as_toml(manifest):
|
|
|
|
import re
|
|
from tomlkit import document, nl, table, dumps, comment
|
|
|
|
toml_manifest = document()
|
|
toml_manifest.add("packaging_format", 2)
|
|
toml_manifest.add(nl())
|
|
toml_manifest.add("id", manifest["id"])
|
|
toml_manifest.add("name", manifest["name"])
|
|
for lang, value in manifest["description"].items():
|
|
toml_manifest.add(f"description.{lang}", value)
|
|
toml_manifest.add(nl())
|
|
toml_manifest.add("version", manifest["version"])
|
|
toml_manifest.add(nl())
|
|
toml_manifest.add("maintainers", manifest["maintainers"])
|
|
|
|
upstream = table()
|
|
for key, value in manifest["upstream"].items():
|
|
upstream[key] = value
|
|
upstream["cpe"].comment("FIXME: optional but recommended if relevant, this is meant to contain the Common Platform Enumeration, which is sort of a standard id for applications defined by the NIST. In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)")
|
|
upstream["fund"].comment("FIXME: optional but recommended (or remove if irrelevant / not applicable). This is meant to be an URL where people can financially support this app, especially when its development is based on volunteers and/or financed by its community. YunoHost may later advertise it in the webadmin.")
|
|
toml_manifest["upstream"] = upstream
|
|
|
|
integration = table()
|
|
for key, value in manifest["integration"].items():
|
|
integration.add(key, value)
|
|
integration["architectures"].comment('FIXME: can be replaced by a list of supported archs using the dpkg --print-architecture nomenclature (amd64/i386/armhf/arm64), for example: ["amd64", "i386"]')
|
|
integration["ldap"].comment('FIXME: replace with true, false, or "not_relevant". Not to confuse with the "sso" key : the "ldap" key corresponds to wether or not a user *can* login on the app using its YunoHost credentials.')
|
|
integration["sso"].comment('FIXME: replace with true, false, or "not_relevant". Not to confuse with the "ldap" key : the "sso" key corresponds to wether or not a user is *automatically logged-in* on the app when logged-in on the YunoHost portal.')
|
|
integration["disk"].comment('FIXME: replace with an **estimate** minimum disk requirement. e.g. 20M, 400M, 1G, ...')
|
|
integration["ram.build"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...')
|
|
integration["ram.runtime"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...')
|
|
toml_manifest["integration"] = integration
|
|
|
|
install = table()
|
|
for key, value in manifest["install"].items():
|
|
install[key] = table()
|
|
install[key].indent(4)
|
|
|
|
if key in ["domain", "path", "admin", "is_public", "password"]:
|
|
install[key].add(comment("this is a generic question - ask strings are automatically handled by Yunohost's core"))
|
|
|
|
for lang, value2 in value.get("ask", {}).items():
|
|
install[key].add(f"ask.{lang}", value2)
|
|
|
|
for lang, value2 in value.get("help", {}).items():
|
|
install[key].add(f"help.{lang}", value2)
|
|
|
|
for key2, value2 in value.items():
|
|
if key2 in ["ask", "help"]:
|
|
continue
|
|
install[key].add(key2, value2)
|
|
|
|
toml_manifest["install"] = install
|
|
|
|
resources = table()
|
|
for key, value in manifest["resources"].items():
|
|
resources[key] = table()
|
|
resources[key].indent(4)
|
|
for key2, value2 in value.items():
|
|
if not isinstance(value2, dict):
|
|
resources[key].add(key2, value2)
|
|
else:
|
|
t = table()
|
|
t.indent(4)
|
|
resources[key].add(key2, t)
|
|
for key3, value3 in value2.items():
|
|
t.add(key3, value3)
|
|
t.add(nl())
|
|
|
|
if key == "apt" and key2 == "extras":
|
|
for extra in resources[key][key2]:
|
|
extra.indent(8)
|
|
|
|
toml_manifest["resources"] = resources
|
|
|
|
toml_manifest_dump = dumps(toml_manifest)
|
|
|
|
regex = re.compile(r'\"((description|ask|help)\.[a-z]{2})\"')
|
|
toml_manifest_dump = regex.sub(r'\1', toml_manifest_dump)
|
|
toml_manifest_dump = toml_manifest_dump.replace('"ram.build"', "ram.build")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"ram.runtime"', "ram.runtime")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"main.url"', "main.url")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"main.default"', "main.default")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"armhf.url"', "armhf.url")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"armhf.sha256"', "armhf.sha256")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"arm64.url"', "arm64.url")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"arm64.sha256"', "arm64.sha256")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"amd64.url"', "amd64.url")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"amd64.sha256"', "amd64.sha256")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"i386.url"', "i386.url")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"i386.sha256"', "i386.sha256")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"armel.url"', "armel.url")
|
|
toml_manifest_dump = toml_manifest_dump.replace('"armel.sha256"', "armel.sha256")
|
|
|
|
if "ports" in manifest["resources"]:
|
|
for port_thing in manifest["resources"]["ports"].keys():
|
|
toml_manifest_dump = toml_manifest_dump.replace(f'"{port_thing}"', f"{port_thing}")
|
|
|
|
return toml_manifest_dump
|
|
|
|
|
|
def cleanup_scripts_and_conf(folder):
|
|
|
|
patterns_to_remove_in_scripts = [
|
|
"^.*ynh_abort_if_errors.*$",
|
|
"^.*YNH_APP_ARG.*$",
|
|
"^.*YNH_APP_INSTANCE_NAME.*$",
|
|
r"^ *final_path=",
|
|
r"^\s*final_path=",
|
|
"^.*test .*-(e|d) .*final_path.*$",
|
|
"^.*ynh_webpath_register.*$",
|
|
"^.*ynh_webpath_available.*$",
|
|
"^.*ynh_system_user_create.*$",
|
|
"^.*ynh_system_user_delete.*$",
|
|
"^.*ynh_permission_update.*$",
|
|
"^.*ynh_permission_create.*$",
|
|
"^.*if .*ynh_permission_exists.*$",
|
|
"^.*if .*ynh_legacy_permissions_exists.*$",
|
|
"^.*ynh_legacy_permissions_delete_all.*$",
|
|
"^.*ynh_app_setting_set .*(domain|path|final_path|admin|password|port|datadir|db_name|db_user|db_pwd).*$",
|
|
"^.*ynh_app_setting_.* is_public.*$",
|
|
r"^.*if.*\$is_public.*$",
|
|
"^.*_dependencies=.*$",
|
|
"^.*ynh_install_app_dependencies.*$",
|
|
"^.*ynh_install_extra_app_dependencies.*$",
|
|
"^.*ynh_remove_app_dependencies.*$",
|
|
r"^.*\$\(ynh_app_setting_get.*$",
|
|
r"^.*ynh_secure_remove .*\$final_path.*$",
|
|
r"^.*ynh_secure_remove .*\$datadir.*$",
|
|
"^.*ynh_backup_before_upgrade.*$",
|
|
"^.*ynh_clean_setup.*$",
|
|
"^.*ynh_restore_upgradebackup.*$",
|
|
"^db_name=.*$",
|
|
"^db_user=.*$",
|
|
"^db_pwd=.*$",
|
|
"^old_domain=.*$",
|
|
"^new_domain=.*$",
|
|
"^old_path=.*$",
|
|
"^new_path=.*$",
|
|
r"change_path=(0|1)\s*$",
|
|
r"change_domain=(0|1)\s*$",
|
|
r"^\s*if.*old_domain.*new_domain.*$",
|
|
r"^\s*if.*old_path.*new_path.*$",
|
|
"^datadir=.*$",
|
|
"^phpversion=.*$",
|
|
"^YNH_PHP_VERSION=.*$",
|
|
"^.*ynh_psql_test_if_first_run.*$",
|
|
"^.*ynh_mysql_setup_db.*$",
|
|
"^.*ynh_psql_setup_db.*$",
|
|
"^.*ynh_mysql_remove_db.*$",
|
|
"^.*ynh_psql_remove_db.*$",
|
|
"^.*ynh_find_port.*$",
|
|
"^.*ynh_script_progression.*Finding an available port",
|
|
"^.*ynh_script_progression.*Backing up the app before upgrading",
|
|
"^.*ynh_script_progression.*Backing up the app before changing its URL",
|
|
"^.*ynh_script_progression.*Creating data directory",
|
|
"^.*ynh_script_progression.*system user",
|
|
"^.*ynh_script_progression.*installation settings",
|
|
"^.*ynh_print_info.*installation settings",
|
|
r"^.*ynh_script_progression.*\w+ dependencies",
|
|
"^.*ynh_script_progression.*Removing app main dir",
|
|
"^.*ynh_script_progression.*Validating.*parameters",
|
|
"^.*ynh_script_progression.*SQL database",
|
|
"^.*ynh_script_progression.*Configuring permissions",
|
|
"^.*ynh_script_progression.*Reloading NGINX web server",
|
|
"^.*ynh_systemd_action --service_name=nginx --action=reload",
|
|
]
|
|
patterns_to_remove_in_scripts = [re.compile(f"({p})", re.MULTILINE) for p in patterns_to_remove_in_scripts]
|
|
|
|
replaces = [
|
|
("path_url", "path"),
|
|
("PATH_URL", "PATH"),
|
|
("final_path", "install_dir"),
|
|
("FINALPATH", "INSTALL_DIR"),
|
|
("datadir", "data_dir"),
|
|
("DATADIR", "DATA_DIR"),
|
|
('--source_id="$architecture"', ''),
|
|
('--source_id="$YNH_ARCH"', ''),
|
|
('--source_id=app', ''),
|
|
('--source_id="app.$architecture"', ''),
|
|
]
|
|
|
|
for s in ["_common.sh", "install", "remove", "upgrade", "backup", "restore", "change_url"]:
|
|
|
|
script = f"{folder}/scripts/{s}"
|
|
|
|
if not os.path.exists(script):
|
|
continue
|
|
|
|
content = open(script).read()
|
|
|
|
for pattern in patterns_to_remove_in_scripts:
|
|
if "^.*ynh_script_progression.*Reloading NGINX web server" in pattern.pattern and s == "restore":
|
|
# This case is legit
|
|
continue
|
|
if "^.*ynh_systemd_action --service_name=nginx --action=reload" in pattern.pattern and s == "restore":
|
|
# This case is legit
|
|
continue
|
|
content = pattern.sub(r"#REMOVEME? \1", content)
|
|
|
|
for pattern, replace in replaces:
|
|
content = content.replace(pattern, replace)
|
|
|
|
if s == "change_url":
|
|
|
|
pattern = re.compile("(^.*nginx.*$)", re.MULTILINE)
|
|
content = pattern.sub(r"#REMOVEME? \1", content)
|
|
|
|
pattern = re.compile("(^.*ynh_script_progress.*Updat.* NGINX.*conf.*$)", re.MULTILINE)
|
|
content = pattern.sub(r"\1\n\nynh_change_url_nginx_config", content)
|
|
|
|
pattern = re.compile(r"(ynh_clean_check_starting)", re.MULTILINE)
|
|
content = pattern.sub(r"#REMOVEME? \1", content)
|
|
pattern = re.compile(r"(^\s+domain=.*$)", re.MULTILINE)
|
|
content = pattern.sub(r"#REMOVEME? \1", content)
|
|
pattern = re.compile(r"(^\s+path=.*$)", re.MULTILINE)
|
|
content = pattern.sub(r"#REMOVEME? \1", content)
|
|
|
|
|
|
open(script, "w").write(content)
|
|
|
|
for conf in os.listdir(f"{folder}/conf"):
|
|
|
|
conf = f"{folder}/conf/{conf}"
|
|
|
|
if not os.path.isfile(conf):
|
|
continue
|
|
|
|
content = open(conf).read()
|
|
content_init = content
|
|
|
|
for pattern, replace in replaces:
|
|
content = content.replace(pattern, replace)
|
|
|
|
if content_init != content:
|
|
open(conf, "w").write(content)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(
|
|
description="Attempt to automatically convert a v1 YunoHost app to v2 (at least as much as possible) : parse the app scripts to auto-generate the manifest.toml, and remove now-useless lines from the app scripts"
|
|
)
|
|
parser.add_argument(
|
|
"app_path", help="Path to the app to convert"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
manifest = _convert_v1_manifest_to_v2(args.app_path)
|
|
open(args.app_path + "/manifest.toml", "w").write(_dump_v2_manifest_as_toml(manifest))
|
|
|
|
cleanup_scripts_and_conf(args.app_path)
|