maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity
llvm-project/llvm/utils/git/github-automation.py

454 lines
18 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
#
# ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ==-------------------------------------------------------------------------==#
import argparse
from git import Repo # type: ignore
import github
import os
import re
import requests
import sys
import time
from typing import *
class IssueSubscriber:
@property
def team_name(self) -> str:
return self._team_name
def __init__(self, token:str, repo:str, issue_number:int, label_name:str):
self.repo = github.Github(token).get_repo(repo)
self.org = github.Github(token).get_organization(self.repo.organization.login)
self.issue = self.repo.get_issue(issue_number)
self._team_name = 'issue-subscribers-{}'.format(label_name).lower()
def run(self) -> bool:
for team in self.org.get_teams():
if self.team_name != team.name.lower():
continue
comment = '@llvm/{}'.format(team.slug)
self.issue.create_comment(comment)
return True
return False
def setup_llvmbot_git(git_dir = '.'):
"""
Configure the git repo in `git_dir` with the llvmbot account so
commits are attributed to llvmbot.
"""
repo = Repo(git_dir)
with repo.config_writer() as config:
config.set_value('user', 'name', 'llvmbot')
config.set_value('user', 'email', 'llvmbot@llvm.org')
def phab_api_call(phab_token:str, url:str, args:dict) -> dict:
"""
Make an API call to the Phabricator web service and return a dictionary
containing the json response.
"""
data = { "api.token" : phab_token }
data.update(args)
response = requests.post(url, data = data)
return response.json()
def phab_login_to_github_login(phab_token:str, repo:github.Repository.Repository, phab_login:str) -> str:
"""
Tries to translate a Phabricator login to a github login by
finding a commit made in Phabricator's Differential.
The commit's SHA1 is then looked up in the github repo and
the committer's login associated with that commit is returned.
:param str phab_token: The Conduit API token to use for communication with Pabricator
:param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
:param str phab_login: The Phabricator login to be translated.
"""
args = {
"constraints[authors][0]" : phab_login,
# PHID for "LLVM Github Monorepo" repository
"constraints[repositories][0]" : "PHID-REPO-f4scjekhnkmh7qilxlcy",
"limit" : 1
}
# API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
r = phab_api_call(phab_token, "https://reviews.llvm.org/api/diffusion.commit.search", args)
data = r['result']['data']
if len(data) == 0:
# Can't find any commits associated with this user
return None
commit_sha = data[0]['fields']['identifier']
committer = repo.get_commit(commit_sha).committer
if not committer:
# This committer had an email address GitHub could not recognize, so
# it can't link the user to a GitHub account.
print(f"Warning: Can't find github account for {phab_login}")
return None
return committer.login
def phab_get_commit_approvers(phab_token:str, repo:github.Repository.Repository, commit:github.Commit.Commit) -> list:
args = { "corpus" : commit.commit.message }
# API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/
r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.parsecommitmessage", args)
review_id = r['result']['revisionIDFieldInfo']['value']
if not review_id:
# No Phabricator revision for this commit
return []
args = {
'constraints[ids][0]' : review_id,
'attachments[reviewers]' : True
}
# API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/
r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.revision.search", args)
reviewers = r['result']['data'][0]['attachments']['reviewers']['reviewers']
accepted = []
for reviewer in reviewers:
if reviewer['status'] != 'accepted':
continue
phid = reviewer['reviewerPHID']
args = { 'constraints[phids][0]' : phid }
# API documentation: https://reviews.llvm.org/conduit/method/user.search/
r = phab_api_call(phab_token, "https://reviews.llvm.org/api/user.search", args)
accepted.append(r['result']['data'][0]['fields']['username'])
return accepted
class ReleaseWorkflow:
CHERRY_PICK_FAILED_LABEL = 'release:cherry-pick-failed'
"""
This class implements the sub-commands for the release-workflow command.
The current sub-commands are:
* create-branch
* create-pull-request
The execute_command method will automatically choose the correct sub-command
based on the text in stdin.
"""
def __init__(self, token:str, repo:str, issue_number:int,
branch_repo_name:str, branch_repo_token:str,
llvm_project_dir:str, phab_token:str) -> None:
self._token = token
self._repo_name = repo
self._issue_number = issue_number
self._branch_repo_name = branch_repo_name
if branch_repo_token:
self._branch_repo_token = branch_repo_token
else:
self._branch_repo_token = self.token
self._llvm_project_dir = llvm_project_dir
self._phab_token = phab_token
@property
def token(self) -> str:
return self._token
@property
def repo_name(self) -> str:
return self._repo_name
@property
def issue_number(self) -> int:
return self._issue_number
@property
def branch_repo_name(self) -> str:
return self._branch_repo_name
@property
def branch_repo_token(self) -> str:
return self._branch_repo_token
@property
def llvm_project_dir(self) -> str:
return self._llvm_project_dir
@property
def phab_token(self) -> str:
return self._phab_token
@property
def repo(self) -> github.Repository.Repository:
return github.Github(self.token).get_repo(self.repo_name)
@property
def issue(self) -> github.Issue.Issue:
return self.repo.get_issue(self.issue_number)
@property
def push_url(self) -> str:
return 'https://{}@github.com/{}'.format(self.branch_repo_token, self.branch_repo_name)
@property
def branch_name(self) -> str:
return 'issue{}'.format(self.issue_number)
@property
def release_branch_for_issue(self) -> Optional[str]:
issue = self.issue
milestone = issue.milestone
if milestone is None:
return None
m = re.search('branch: (.+)',milestone.description)
if m:
return m.group(1)
return None
def print_release_branch(self) -> None:
print(self.release_branch_for_issue)
def issue_notify_branch(self) -> None:
self.issue.create_comment('/branch {}/{}'.format(self.branch_repo_name, self.branch_name))
def issue_notify_pull_request(self, pull:github.PullRequest.PullRequest) -> None:
self.issue.create_comment('/pull-request {}#{}'.format(self.branch_repo_name, pull.number))
def make_ignore_comment(self, comment: str) -> str:
"""
Returns the comment string with a prefix that will cause
a Github workflow to skip parsing this comment.
:param str comment: The comment to ignore
"""
return "<!--IGNORE-->\n"+comment
def issue_notify_no_milestone(self, comment:List[str]) -> None:
message = "{}\n\nError: Command failed due to missing milestone.".format(''.join(['>' + line for line in comment]))
self.issue.create_comment(self.make_ignore_comment(message))
@property
def action_url(self) -> str:
if os.getenv('CI'):
return 'https://github.com/{}/actions/runs/{}'.format(os.getenv('GITHUB_REPOSITORY'), os.getenv('GITHUB_RUN_ID'))
return ""
def issue_notify_cherry_pick_failure(self, commit:str) -> github.IssueComment.IssueComment:
message = self.make_ignore_comment("Failed to cherry-pick: {}\n\n".format(commit))
action_url = self.action_url
if action_url:
message += action_url + "\n\n"
message += "Please manually backport the fix and push it to your github fork. Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`"
issue = self.issue
comment = issue.create_comment(message)
issue.add_to_labels(self.CHERRY_PICK_FAILED_LABEL)
return comment
def issue_notify_pull_request_failure(self, branch:str) -> github.IssueComment.IssueComment:
message = "Failed to create pull request for {} ".format(branch)
message += self.action_url
return self.issue.create_comment(message)
def issue_remove_cherry_pick_failed_label(self):
if self.CHERRY_PICK_FAILED_LABEL in [l.name for l in self.issue.labels]:
self.issue.remove_from_labels(self.CHERRY_PICK_FAILED_LABEL)
def pr_request_review(self, pr:github.PullRequest.PullRequest):
"""
This function will try to find the best reviewers for `commits` and
then add a comment requesting review of the backport and assign the
pull request to the selected reviewers.
The reviewers selected are those users who approved the patch in
Phabricator.
"""
reviewers = []
for commit in pr.get_commits():
approvers = phab_get_commit_approvers(self.phab_token, self.repo, commit)
for a in approvers:
login = phab_login_to_github_login(self.phab_token, self.repo, a)
if not login:
continue
reviewers.append(login)
if len(reviewers):
message = "{} What do you think about merging this PR to the release branch?".format(
" ".join(["@" + r for r in reviewers]))
pr.create_issue_comment(message)
pr.add_to_assignees(*reviewers)
def create_branch(self, commits:List[str]) -> bool:
"""
This function attempts to backport `commits` into the branch associated
with `self.issue_number`.
If this is successful, then the branch is pushed to `self.branch_repo_name`, if not,
a comment is added to the issue saying that the cherry-pick failed.
:param list commits: List of commits to cherry-pick.
"""
print('cherry-picking', commits)
branch_name = self.branch_name
local_repo = Repo(self.llvm_project_dir)
local_repo.git.checkout(self.release_branch_for_issue)
for c in commits:
try:
local_repo.git.cherry_pick('-x', c)
except Exception as e:
self.issue_notify_cherry_pick_failure(c)
raise e
push_url = self.push_url
print('Pushing to {} {}'.format(push_url, branch_name))
local_repo.git.push(push_url, 'HEAD:{}'.format(branch_name), force=True)
self.issue_notify_branch()
self.issue_remove_cherry_pick_failed_label()
return True
def check_if_pull_request_exists(self, repo:github.Repository.Repository, head:str) -> bool:
pulls = repo.get_pulls(head=head)
return pulls.totalCount != 0
def create_pull_request(self, owner:str, repo_name:str, branch:str) -> bool:
"""
reate a pull request in `self.branch_repo_name`. The base branch of the
pull request will be chosen based on the the milestone attached to
the issue represented by `self.issue_number` For example if the milestone
is Release 13.0.1, then the base branch will be release/13.x. `branch`
will be used as the compare branch.
https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch
https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch
"""
repo = github.Github(self.token).get_repo(self.branch_repo_name)
issue_ref = '{}#{}'.format(self.repo_name, self.issue_number)
pull = None
release_branch_for_issue = self.release_branch_for_issue
if release_branch_for_issue is None:
return False
head_branch = branch
if not repo.fork:
# If the target repo is not a fork of llvm-project, we need to copy
# the branch into the target repo. GitHub only supports cross-repo pull
# requests on forked repos.
head_branch = f'{owner}-{branch}'
local_repo = Repo(self.llvm_project_dir)
push_done = False
for i in range(0,5):
try:
local_repo.git.fetch(f'https://github.com/{owner}/{repo_name}', f'{branch}:{branch}')
local_repo.git.push(self.push_url, f'{branch}:{head_branch}', force=True)
push_done = True
break
except Exception as e:
print(e)
time.sleep(30)
continue
if not push_done:
raise Exception("Failed to mirror branch into {}".format(self.push_url))
owner = repo.owner.login
head = f"{owner}:{head_branch}"
if self.check_if_pull_request_exists(repo, head):
print("PR already exists...")
return True
try:
pull = repo.create_pull(title=f"PR for {issue_ref}",
body='resolves {}'.format(issue_ref),
base=release_branch_for_issue,
head=head,
maintainer_can_modify=False)
try:
if self.phab_token:
self.pr_request_review(pull)
except Exception as e:
print("error: Failed while searching for reviewers", e)
except Exception as e:
self.issue_notify_pull_request_failure(branch)
raise e
if pull is None:
return False
self.issue_notify_pull_request(pull)
self.issue_remove_cherry_pick_failed_label()
# TODO(tstellar): Do you really want to always return True?
return True
def execute_command(self) -> bool:
"""
This function reads lines from STDIN and executes the first command
that it finds. The 2 supported commands are:
/cherry-pick commit0 <commit1> <commit2> <...>
/branch <owner>/<repo>/<branch>
"""
for line in sys.stdin:
line.rstrip()
m = re.search("/([a-z-]+)\s(.+)", line)
if not m:
continue
command = m.group(1)
args = m.group(2)
if command == 'cherry-pick':
return self.create_branch(args.split())
if command == 'branch':
m = re.match('([^/]+)/([^/]+)/(.+)', args)
if m:
owner = m.group(1)
repo = m.group(2)
branch = m.group(3)
return self.create_pull_request(owner, repo, branch)
print("Do not understand input:")
print(sys.stdin.readlines())
return False
parser = argparse.ArgumentParser()
parser.add_argument('--token', type=str, required=True, help='GitHub authentiation token')
parser.add_argument('--repo', type=str, default=os.getenv('GITHUB_REPOSITORY', 'llvm/llvm-project'),
help='The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)')
subparsers = parser.add_subparsers(dest='command')
issue_subscriber_parser = subparsers.add_parser('issue-subscriber')
issue_subscriber_parser.add_argument('--label-name', type=str, required=True)
issue_subscriber_parser.add_argument('--issue-number', type=int, required=True)
release_workflow_parser = subparsers.add_parser('release-workflow')
release_workflow_parser.add_argument('--llvm-project-dir', type=str, default='.', help='directory containing the llvm-project checout')
release_workflow_parser.add_argument('--issue-number', type=int, required=True, help='The issue number to update')
release_workflow_parser.add_argument('--phab-token', type=str, help='Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/')
release_workflow_parser.add_argument('--branch-repo-token', type=str,
help='GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.')
release_workflow_parser.add_argument('--branch-repo', type=str, default='llvm/llvm-project-release-prs',
help='The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)')
release_workflow_parser.add_argument('sub_command', type=str, choices=['print-release-branch', 'auto'],
help='Print to stdout the name of the release branch ISSUE_NUMBER should be backported to')
llvmbot_git_config_parser = subparsers.add_parser('setup-llvmbot-git', help='Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot')
args = parser.parse_args()
if args.command == 'issue-subscriber':
issue_subscriber = IssueSubscriber(args.token, args.repo, args.issue_number, args.label_name)
issue_subscriber.run()
elif args.command == 'release-workflow':
release_workflow = ReleaseWorkflow(args.token, args.repo, args.issue_number,
args.branch_repo, args.branch_repo_token,
args.llvm_project_dir, args.phab_token)
if not release_workflow.release_branch_for_issue:
release_workflow.issue_notify_no_milestone(sys.stdin.readlines())
sys.exit(1)
if args.sub_command == 'print-release-branch':
release_workflow.print_release_branch()
else:
if not release_workflow.execute_command():
sys.exit(1)
elif args.command == 'setup-llvmbot-git':
setup_llvmbot_git()
Reference in New Issue View Git Blame Copy Permalink