1
0
Fork 0

Merge pull request #2067 from Salamandar/revert-2058-only_local_repos

Revert "Use local app cache to reduce github api calls"
This commit is contained in:
Alexandre Aubin 2024-02-27 19:28:42 +01:00 committed by GitHub
commit 3a3ecfa573

View file

@ -17,7 +17,6 @@ import requests
import toml import toml
import tqdm import tqdm
import github import github
from git import Repo, GitCommandError
# add apps/tools to sys.path # add apps/tools to sys.path
sys.path.insert(0, str(Path(__file__).parent.parent)) sys.path.insert(0, str(Path(__file__).parent.parent))
@ -83,76 +82,87 @@ def apps_to_run_auto_update_for() -> list[str]:
return relevant_apps return relevant_apps
class LocalAndRemoteRepo: class LocalOrRemoteRepo:
def __init__(self, app: Union[str, Path]) -> None: def __init__(self, app: Union[str, Path]) -> None:
self.local = False
self.remote = False
self.app = app
if isinstance(app, Path): if isinstance(app, Path):
self.app_path = app # It's local
self.app = self.app_path.name self.local = True
self.manifest_path = app / "manifest.toml"
if not self.manifest_path.exists():
raise RuntimeError(f"{app.name}: manifest.toml doesnt exists?")
# app is in fact a path
self.manifest_raw = (app / "manifest.toml").open("r", encoding="utf-8").read()
elif isinstance(app, str):
# It's remote
self.remote = True
github = get_github()[1]
assert github, "Could not get github authentication!"
self.repo = github.get_repo(f"Yunohost-Apps/{app}_ynh")
self.pr_branch: Optional[str] = None
# Determine base branch, either `testing` or default branch
try:
self.base_branch = self.repo.get_branch("testing").name
except Exception:
self.base_branch = self.repo.default_branch
contents = self.repo.get_contents("manifest.toml", ref=self.base_branch)
assert not isinstance(contents, list)
self.manifest_raw = contents.decoded_content.decode()
self.manifest_raw_sha = contents.sha
else: else:
self.app = app raise TypeError(f"Invalid argument type for app: {type(app)}")
self.app_path = app_cache_folder(app)
self.manifest_path = self.app_path / "manifest.toml"
if not self.manifest_path.exists():
raise RuntimeError(f"{self.app_path.name}: manifest.toml doesnt exists?")
self.manifest_raw = self.manifest_path.open("r", encoding="utf-8").read()
# Mock the Git blob sha, to prevent a request to Github API
manifest_blob = f"blob {len(self.manifest_raw)}\0{self.manifest_raw}"
self.manifest_sha = hashlib.sha1(manifest_blob.encode("utf-8")).hexdigest()
def edit_manifest(self, content: str): def edit_manifest(self, content: str):
self.manifest_raw = content self.manifest_raw = content
self.manifest_path.open("w", encoding="utf-8").write(content) if self.local:
self.manifest_path.open("w", encoding="utf-8").write(content)
def connect_api(self) -> None: def commit(self, message: str):
github = get_github()[1] if self.remote:
assert github, "Could not get github authentication!" author = get_github()[2]
path = Repo(self.app_path).remote("origin").url.split("github.com")[1].strip(":/") assert author, "Could not get Github author!"
self.github = github.get_repo(path) assert self.pr_branch is not None, "Did you forget to create a branch?"
self.repo.update_file(
"manifest.toml",
message=message,
content=self.manifest_raw,
sha=self.manifest_raw_sha,
branch=self.pr_branch,
author=author,
)
def new_branch(self, name: str) -> None: def new_branch(self, name: str) -> bool:
self.pr_branch = name if self.local:
repo = Repo(self.app_path) logging.warning("Can't create branches for local repositories")
try:
repo.git.checkout(name)
except GitCommandError:
branch = repo.create_head(self.pr_branch)
branch.checkout()
def commit(self, message: str) -> bool:
author = get_github()[2]
author_str = f"{author._identity['name']} <{author._identity['email']}>" \
if author else None
repo = Repo(self.app_path)
if not repo.index.diff(None):
logging.info("No local changes to be commited, maybe update branch already contains changes!")
return False return False
repo.git.add("manifest.toml") if self.remote:
repo.git.commit(m=message, author=author_str) self.pr_branch = name
return True commit_sha = self.repo.get_branch(self.base_branch).commit.sha
if self.pr_branch in [branch.name for branch in self.repo.get_branches()]:
def push(self) -> None: print("already existing")
repo = Repo(self.app_path) return False
origin = repo.remote("origin") self.repo.create_git_ref(ref=f"refs/heads/{name}", sha=commit_sha)
origin.push(repo.active_branch.name) return True
return False
def create_pr(self, branch: str, title: str, message: str) -> Optional[str]: def create_pr(self, branch: str, title: str, message: str) -> Optional[str]:
# Determine base branch, either `testing` or default branch if self.remote:
try: # Open the PR
self.base_branch = self.github.get_branch("testing").name pr = self.repo.create_pull(
except Exception: title=title, body=message, head=branch, base=self.base_branch
self.base_branch = self.github.default_branch )
return pr.html_url
# Open the PR logging.warning("Can't create pull requests for local repositories")
pr = self.github.create_pull( return None
title=title, body=message, head=branch, base=self.base_branch
)
return pr.html_url
def get_pr(self, branch: str) -> str: def get_pr(self, branch: str) -> str:
return next(pull.html_url for pull in self.github.get_pulls(head=branch)) return next(pull.html_url for pull in self.repo.get_pulls(head=branch))
class State(Enum): class State(Enum):
@ -164,7 +174,7 @@ class State(Enum):
class AppAutoUpdater: class AppAutoUpdater:
def __init__(self, app_id: Union[str, Path]) -> None: def __init__(self, app_id: Union[str, Path]) -> None:
self.repo = LocalAndRemoteRepo(app_id) self.repo = LocalOrRemoteRepo(app_id)
self.manifest = toml.loads(self.repo.manifest_raw) self.manifest = toml.loads(self.repo.manifest_raw)
self.app_id = self.manifest["id"] self.app_id = self.manifest["id"]
@ -208,19 +218,24 @@ class AppAutoUpdater:
if state == State.up_to_date: if state == State.up_to_date:
return (State.up_to_date, "", "", "") return (State.up_to_date, "", "", "")
# if pr:
self.repo.new_branch(branch_name)
if edit: if edit:
self.repo.edit_manifest(self.repo.manifest_raw) self.repo.edit_manifest(self.repo.manifest_raw)
commited = False try:
if commit: if pr:
commited = self.repo.commit(commit_msg) self.repo.new_branch(branch_name)
except github.GithubException as e:
if e.status == 409:
print("Branch already exists!")
try: try:
if pr and commited: if commit:
self.repo.push() self.repo.commit(commit_msg)
except github.GithubException as e:
if e.status == 409:
print("Commits were already commited on branch!")
try:
if pr:
pr_url = self.repo.create_pr(branch_name, pr_title, commit_msg) or "" pr_url = self.repo.create_pr(branch_name, pr_title, commit_msg) or ""
except github.GithubException as e: except github.GithubException as e:
if e.status == 422 or e.status == 409: if e.status == 422 or e.status == 409:
@ -559,7 +574,7 @@ def main() -> None:
with multiprocessing.Pool(processes=args.processes) as pool: with multiprocessing.Pool(processes=args.processes) as pool:
tasks = pool.imap(run_autoupdate_for_multiprocessing, tasks = pool.imap(run_autoupdate_for_multiprocessing,
((app, args.edit, args.commit, args.pr) for app in apps)) ((app, args.edit, args.commit, args.pr) for app in apps))
for app, result in tqdm.tqdm(tasks, total=len(apps), mininterval=0, ascii=" ·#"): for app, result in tqdm.tqdm(tasks, total=len(apps), ascii=" ·#"):
state, current_version, main_version, pr_url = result state, current_version, main_version, pr_url = result
if state == State.up_to_date: if state == State.up_to_date:
continue continue