From 0a0cf347bd4948f85529f97143ffd4d81d3a427d Mon Sep 17 00:00:00 2001 From: tituspijean Date: Sun, 29 Jan 2023 18:32:39 +0100 Subject: [PATCH 1/2] Add auto-updater upgrader --- autoupdater-upgrader/README.md | 14 ++++ autoupdater-upgrader/autoupdater-upgrader.py | 87 ++++++++++++++++++++ autoupdater-upgrader/requirements.txt | 3 + 3 files changed, 104 insertions(+) create mode 100644 autoupdater-upgrader/README.md create mode 100755 autoupdater-upgrader/autoupdater-upgrader.py create mode 100644 autoupdater-upgrader/requirements.txt diff --git a/autoupdater-upgrader/README.md b/autoupdater-upgrader/README.md new file mode 100644 index 0000000..854899d --- /dev/null +++ b/autoupdater-upgrader/README.md @@ -0,0 +1,14 @@ +# Auto-README generation + +Browses all repositories in YunoHost-Apps organization, and updates `updater.yml` with latest actions versions. + +### Initial install + +``` +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +This script requires a `.github_token` file with a token with public.repo permission. + diff --git a/autoupdater-upgrader/autoupdater-upgrader.py b/autoupdater-upgrader/autoupdater-upgrader.py new file mode 100755 index 0000000..85aed84 --- /dev/null +++ b/autoupdater-upgrader/autoupdater-upgrader.py @@ -0,0 +1,87 @@ +#!venv/bin/python3 + +import sys, os, time +import urllib.request, json +import re + +# Debug +from rich.traceback import install +install(show_locals=True) + +from github import Github + +##### +# +# CONFIG +# +##### + +# API token for yunohost-bot, need public.repo permission +g = Github(open(".github_token").read().strip()) + +# Path to the file to be updated +path=".github/workflows/updater.yml" + +# Body of the PR message +body=""" +Auto-updater actions need upgrading to continue working: +- actions/checkout@v3 +- peter-evans/create-pull-request@v4 +""" + +# Name of the branch created for the PR +new_branch="upgrade-auto-updater" + +##### +# +# CRAWL REPOSITORIES +# +##### + +u = g.get_user("yunohost-bot") +org = g.get_organization("yunohost-apps") + +# For each repositories belonging to the bot (user `u`) +for repo in org.get_repos(): + # Determine base branch, either `testing` or default branch + try: + base_branch = repo.get_branch("testing").name + except: + base_branch = repo.default_branch + # Make sure the repository has an auto-updater + try: + repo.get_contents(path, ref="refs/heads/"+base_branch) + except: + print("No updater in "+repo.full_name) + continue + # Process the repo + try: + print("Processing "+repo.full_name) + + # Get the commit base for the new branch, and create it + commit_sha = repo.get_branch(base_branch).commit.sha + new_branch_ref = repo.create_git_ref(ref="refs/heads/"+new_branch, sha=commit_sha) + + # Get current file contents + contents = repo.get_contents(path, ref=new_branch_ref.ref) + + # Update the file + updater_yml = contents.decoded_content.decode("unicode_escape") + updater_yml = re.sub(r'(?m)uses: actions/checkout@v[\d]+', "uses: actions/checkout@v3", updater_yml) + updater_yml = re.sub(r'(?m)uses: peter-evans/create-pull-request@v[\d]+', "uses: peter-evans/create-pull-request@v4", updater_yml) + updated = repo.update_file(path=contents.path, + message="Upgrade auto-updater", + content=updater_yml, + sha=contents.sha, + branch=new_branch) + + # Open the PR + pr = repo.create_pull(title="Upgrade auto-updater", body=body, head=new_branch, base=base_branch) + + print(repo.full_name+" updated with PR \#"+ pr.id) + break + except Exception as e: + print(e) + print("...failed. Deleting new branch.") + repo.get_git_ref("heads/"+new_branch).delete() + break diff --git a/autoupdater-upgrader/requirements.txt b/autoupdater-upgrader/requirements.txt new file mode 100644 index 0000000..981d980 --- /dev/null +++ b/autoupdater-upgrader/requirements.txt @@ -0,0 +1,3 @@ +rich +PyGithub + From 9248a5958582680e247aa5d8a3f513f03b9afac6 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 24 Feb 2023 00:50:32 +0100 Subject: [PATCH 2/2] Fix auto-updater upgrader --- .gitignore | 1 + autoupdater-upgrader/README.md | 6 +- autoupdater-upgrader/autoupdater-upgrader.py | 94 +++++++++++++------- 3 files changed, 68 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index ce6c63a..b219748 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ builds tools/README-generator/venv/ tools/bot-repo-cleanup/.github_token +tools/autoupdater-upgrader/.* tools/autopatches/login tools/autopatches/token diff --git a/autoupdater-upgrader/README.md b/autoupdater-upgrader/README.md index 854899d..e63903b 100644 --- a/autoupdater-upgrader/README.md +++ b/autoupdater-upgrader/README.md @@ -10,5 +10,7 @@ source venv/bin/activate pip install -r requirements.txt ``` -This script requires a `.github_token` file with a token with public.repo permission. - +This script requires the following files: +- `.github_token` containing a token with `public.repo` and `workflow` permission +- `.github_login` containing the author's username +- `.github_email` containing the author's email address diff --git a/autoupdater-upgrader/autoupdater-upgrader.py b/autoupdater-upgrader/autoupdater-upgrader.py index 85aed84..7252042 100755 --- a/autoupdater-upgrader/autoupdater-upgrader.py +++ b/autoupdater-upgrader/autoupdater-upgrader.py @@ -4,11 +4,12 @@ import sys, os, time import urllib.request, json import re +from github import Github +import github + # Debug from rich.traceback import install -install(show_locals=True) - -from github import Github +install(width=150, show_locals=True, locals_max_length=None, locals_max_string=None) ##### # @@ -22,6 +23,9 @@ g = Github(open(".github_token").read().strip()) # Path to the file to be updated path=".github/workflows/updater.yml" +# Title of the PR +title="[autopatch] Upgrade auto-updater" + # Body of the PR message body=""" Auto-updater actions need upgrading to continue working: @@ -29,9 +33,21 @@ Auto-updater actions need upgrading to continue working: - peter-evans/create-pull-request@v4 """ +# Author of the commit +author=github.InputGitAuthor(open(".github_login").read().strip(), open(".github_email").read().strip()) + # Name of the branch created for the PR new_branch="upgrade-auto-updater" +##### +# +# CACHE +# +##### + +with open('processed.txt') as f: + processed = f.read().splitlines() + ##### # # CRAWL REPOSITORIES @@ -42,25 +58,34 @@ u = g.get_user("yunohost-bot") org = g.get_organization("yunohost-apps") # For each repositories belonging to the bot (user `u`) +i=0 for repo in org.get_repos(): - # Determine base branch, either `testing` or default branch - try: - base_branch = repo.get_branch("testing").name - except: - base_branch = repo.default_branch - # Make sure the repository has an auto-updater - try: - repo.get_contents(path, ref="refs/heads/"+base_branch) - except: - print("No updater in "+repo.full_name) - continue - # Process the repo - try: + if repo.full_name not in processed: + + # Determine base branch, either `testing` or default branch + try: + base_branch = repo.get_branch("testing").name + except: + base_branch = repo.default_branch + + # Make sure the repository has an auto-updater + try: + repo.get_contents(path, ref="refs/heads/"+base_branch) + except: + with open('processed.txt', 'a') as pfile: + pfile.write(repo.full_name+'\n') + time.sleep(1.5) + continue + + # Process the repo print("Processing "+repo.full_name) - # Get the commit base for the new branch, and create it - commit_sha = repo.get_branch(base_branch).commit.sha - new_branch_ref = repo.create_git_ref(ref="refs/heads/"+new_branch, sha=commit_sha) + try: + # Get the commit base for the new branch, and create it + commit_sha = repo.get_branch(base_branch).commit.sha + new_branch_ref = repo.create_git_ref(ref="refs/heads/"+new_branch, sha=commit_sha) + except: + new_branch_ref = repo.get_git_ref(ref="heads/"+new_branch) # Get current file contents contents = repo.get_contents(path, ref=new_branch_ref.ref) @@ -69,19 +94,26 @@ for repo in org.get_repos(): updater_yml = contents.decoded_content.decode("unicode_escape") updater_yml = re.sub(r'(?m)uses: actions/checkout@v[\d]+', "uses: actions/checkout@v3", updater_yml) updater_yml = re.sub(r'(?m)uses: peter-evans/create-pull-request@v[\d]+', "uses: peter-evans/create-pull-request@v4", updater_yml) - updated = repo.update_file(path=contents.path, - message="Upgrade auto-updater", - content=updater_yml, - sha=contents.sha, - branch=new_branch) + updated = repo.update_file(contents.path, + message=title, + content=updater_yml, + sha=contents.sha, + branch=new_branch, + author=author) + + # Wait a bit to preserve the API rate limit + time.sleep(1.5) # Open the PR pr = repo.create_pull(title="Upgrade auto-updater", body=body, head=new_branch, base=base_branch) - print(repo.full_name+" updated with PR \#"+ pr.id) - break - except Exception as e: - print(e) - print("...failed. Deleting new branch.") - repo.get_git_ref("heads/"+new_branch).delete() - break + print(repo.full_name+" updated with PR #"+ str(pr.id)) + i=i+1 + + # Wait a bit to preserve the API rate limit + time.sleep(1.5) + + with open('processed.txt', 'a') as pfile: + pfile.write(repo.full_name+'\n') + +print("Done. "+str(i)+" repos processed")