From 14f16e75ad91f2ede196fad2349027efb6148ef2 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Wed, 26 May 2021 14:57:39 -0700 Subject: [PATCH] Large rework to enable sending pull requests as user. This is done by using two GitHub Apps; (a) [OpenROAD Pull Request Sender ](https://github.com/organizations/The-OpenROAD-Project-staging/settings/apps/openroad-pull-request-sender) -- The GitHub App which is given permission by a user to actually send pull requests as them and then does the request. It is installed on the `upstream` and `staging` repositories. It needs the ability to modify pull requests. (b) [OpenROAD Pull Request Sender Auth Check ](https://github.com/organizations/The-OpenROAD-Project-private/settings/apps/openroad-pr-sender-auth-check) -- The GitHub App which checks that a user has given the OpenROAD Pull Request Sender the permission to send the pull requests. It is installed on the `private` repositories. It only needs the ability to update check status. The GitHub Apps run using [Google Cloud Run](https://cloud.google.com/run) (on Google Cloud Platform) under the `openroad-robot` project. Signed-off-by: Tim 'mithro' Ansell --- apps/.gitignore | 2 + apps/Makefile | 35 +++ apps/common/.dockerignore | 7 + apps/common/Dockerfile | 23 ++ apps/common/config.mk | 18 ++ apps/common/db.py | 121 ++++++++++ apps/common/inc.mk | 72 ++++++ apps/common/index.yaml | 6 + apps/common/utils.py | 197 ++++++++++++++++ apps/pull-request-sender-auth-check/Makefile | 23 ++ apps/pull-request-sender-auth-check/app.py | 177 ++++++++++++++ apps/pull-request-sender-auth-check/main.py | 28 +++ .../requirements.txt | 5 + apps/pull-request-sender/Makefile | 22 ++ apps/pull-request-sender/app.py | 223 ++++++++++++++++++ apps/pull-request-sender/main.py | 28 +++ apps/pull-request-sender/requirements.txt | 5 + apps/upstream-sync/Makefile | 25 ++ apps/upstream-sync/app.py | 148 ++++++++++++ apps/upstream-sync/main.py | 28 +++ apps/upstream-sync/requirements.txt | 4 + github_api/__init__.py | 99 +++++--- github_api/app_token.py | 54 ++++- github_api/checks.py | 214 +++++++++++++++++ github_api/env.py | 20 +- send_pr/action.py | 60 ++--- 26 files changed, 1563 insertions(+), 81 deletions(-) create mode 100644 apps/.gitignore create mode 100644 apps/Makefile create mode 100644 apps/common/.dockerignore create mode 100644 apps/common/Dockerfile create mode 100644 apps/common/config.mk create mode 100755 apps/common/db.py create mode 100644 apps/common/inc.mk create mode 100644 apps/common/index.yaml create mode 100755 apps/common/utils.py create mode 100644 apps/pull-request-sender-auth-check/Makefile create mode 100755 apps/pull-request-sender-auth-check/app.py create mode 100755 apps/pull-request-sender-auth-check/main.py create mode 100644 apps/pull-request-sender-auth-check/requirements.txt create mode 100644 apps/pull-request-sender/Makefile create mode 100644 apps/pull-request-sender/app.py create mode 100755 apps/pull-request-sender/main.py create mode 100644 apps/pull-request-sender/requirements.txt create mode 100644 apps/upstream-sync/Makefile create mode 100644 apps/upstream-sync/app.py create mode 100755 apps/upstream-sync/main.py create mode 100644 apps/upstream-sync/requirements.txt create mode 100755 github_api/checks.py diff --git a/apps/.gitignore b/apps/.gitignore new file mode 100644 index 0000000..6a57d58 --- /dev/null +++ b/apps/.gitignore @@ -0,0 +1,2 @@ +*-build +*.pem diff --git a/apps/Makefile b/apps/Makefile new file mode 100644 index 0000000..7f426f2 --- /dev/null +++ b/apps/Makefile @@ -0,0 +1,35 @@ +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +include common/config.mk + +APPS=pull-request-sender pull-request-sender-auth-check + +env: + virtualenv env + source env/bin/activate; pip install -r pull-request-sender/requirements.txt + source env/bin/activate; pip install -r pull-request-sender-auth-check/requirements.txt + +enter: + source env/bin/activate; bash + +build: + for A in $(APPS); do (cd $$A; make build); done + +deploy: + make build + (cd common; yes | gcloud --project=$(PROJECT_ID) datastore indexes create index.yaml) + for A in $(APPS); do (cd $$A; make deploy); done diff --git a/apps/common/.dockerignore b/apps/common/.dockerignore new file mode 100644 index 0000000..3e4bdd9 --- /dev/null +++ b/apps/common/.dockerignore @@ -0,0 +1,7 @@ +Dockerfile +README.md +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/apps/common/Dockerfile b/apps/common/Dockerfile new file mode 100644 index 0000000..027c537 --- /dev/null +++ b/apps/common/Dockerfile @@ -0,0 +1,23 @@ +# Use the official lightweight Python image. +# https://hub.docker.com/_/python +FROM python:3.9-slim + +# Allow statements and log messages to immediately appear in the Knative logs +ENV PYTHONUNBUFFERED True + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ +RUN find | sort +RUN mkdir /keys + +# Install production dependencies. +RUN pip install -r requirements.txt + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. +CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 app:app diff --git a/apps/common/config.mk b/apps/common/config.mk new file mode 100644 index 0000000..87d82bd --- /dev/null +++ b/apps/common/config.mk @@ -0,0 +1,18 @@ +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +PROJECT_ID=openroad-robot +REGION=us-central1 diff --git a/apps/common/db.py b/apps/common/db.py new file mode 100755 index 0000000..293e0da --- /dev/null +++ b/apps/common/db.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +import datetime +import pprint +import requests +import urllib + + +from google.cloud import ndb + + +class Token(ndb.Model): + login = ndb.StringProperty(indexed=True, required=True) + access_token = ndb.StringProperty(required=True) + expires_in = ndb.IntegerProperty(required=True) + refresh_token = ndb.StringProperty(required=True) + refresh_token_expires_in = ndb.IntegerProperty(required=True) + token_type = ndb.StringProperty(required=True) + updated = ndb.DateTimeProperty(auto_now=True) + created = ndb.DateTimeProperty(auto_now_add=True) + + def to_table(self): + c = (datetime.datetime.utcnow() - self.created).seconds + u = (datetime.datetime.utcnow() - self.updated).seconds + ein = self.created + datetime.timedelta(seconds=self.expires_in) + rin = self.created + datetime.timedelta(seconds=self.refresh_token_expires_in) + return f"""\ + + + + + + + +
Login {self.login}
Created {c}s ago {self.created}
Last updated {u}s ago {self.updated}
Auth Expires In t + {self.expires_in}s {ein}
Refresh Expires Int + {self.refresh_token_expires_in}s{rin}
Type {self.token_type}
+""" + + def to_html(self): + return f"""\ + + +{self.to_table()} + + +""" + + @classmethod + def latest(cls, login): + with client.context(): + q = cls.query().filter(cls.login==login).order(-cls.created) + for t in q: + return t + + @classmethod + def delete(cls, login): + with client.context(): + q = cls.query().filter(cls.login==login).order(-cls.created) + for t in q: + t.key.delete() + + @classmethod + def from_response(cls, login, d): + od = dict(urllib.parse.parse_qsl(d.text)) + + if 'error' in od: + ods = pprint.pformat(od) + return f"""\ + + + {od['error_description']} +
+ 
{ods}
+ + +""" + # Use the token to figure out the username of the user associated with the + # token. + headers = { + "Accept": "application/vnd.github.v3+json", + "Authorization": f"{od['token_type']} {od['access_token']}", + } + ud = requests.get( + url="https://api.github.com/user", + headers=headers, + ).json() + assert ud['login'] == login, (ud['login'], login) + + od['login'] = login + return cls.from_json(od) + + @classmethod + def from_json(cls, od): + assert 'access_token' in od, od + assert 'token_type' in od, od + assert 'expires_in' in od, od + od['expires_in'] = int(od['expires_in']) + assert 'refresh_token' in od, od + assert 'refresh_token_expires_in' in od, od + od['refresh_token_expires_in'] = int(od['refresh_token_expires_in']) + return cls(**od) + + +client = ndb.Client() diff --git a/apps/common/inc.mk b/apps/common/inc.mk new file mode 100644 index 0000000..8d424b4 --- /dev/null +++ b/apps/common/inc.mk @@ -0,0 +1,72 @@ +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +MAKE_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) + +include $(MAKE_DIR)/config.mk + +SRCS=\ + ../common/Dockerfile \ + ../common/.dockerignore \ + ../common/db.py \ + ../common/utils.py \ + *.py \ + *.txt \ + ../../github_api + +PYTHONPATH=$(abspath $(PWD)/../common):$(abspath $(PWD)/../../) +export PYTHONPATH + +BUILD_DIR=$(PROJECT_NAME)-build + +build: + rm -rf $(BUILD_DIR) + mkdir $(BUILD_DIR) + cp -a $(SRCS) ./$(BUILD_DIR)/ + find $(BUILD_DIR) -name '*.pem' -delete + find $(BUILD_DIR) -name '*.pyc' -delete + find $(BUILD_DIR) -name '.*.sw*' -delete + find $(BUILD_DIR) -type d -empty -delete + find $(BUILD_DIR) | sort + cd $(BUILD_DIR); gcloud --project=$(PROJECT_ID) \ + builds submit \ + --tag gcr.io/$(PROJECT_ID)/$(PROJECT_NAME) + +.PHONY: build + +SECRETS_ENV=CLIENT_ID CLIENT_SECRET APP_ID +SECRETS_CENV=CLIENT_TOKEN + +SECRETS_CMDLINE_ENV=$(shell echo $(SECRETS_ENV) | sed -e's/\([^ ]\+\) \?/\1=$(SECRETS_PREFIX)_\1:latest,/g' -e's/,$$//') +SECRETS_CMDLINE_CENV=$(shell echo $(SECRETS_CENV) | sed -e's/\([^ ]\+\) \?/,\1=COMMON_\1:latest,/g' -e's/,$$//') +SECRETS_CMDLINE_PEM=,/keys/app.private-key.pem=$(SECRETS_PREFIX)_CLIENT_KEY:latest + +deploy: + gcloud --project=$(PROJECT_ID) beta \ + run deploy \ + $(PROJECT_NAME) \ + --image gcr.io/$(PROJECT_ID)/$(PROJECT_NAME) \ + --platform managed --region=$(REGION) \ + --allow-unauthenticated \ + --set-env-vars PROJECT_ID=$(PROJECT_ID),CLIENT_KEY=/keys/app.private-key.pem \ + --service-account=$(SERVICE_ACCOUNT)@$(PROJECT_ID).iam.gserviceaccount.com \ + --set-secrets=$(SECRETS_CMDLINE_ENV)$(SECRETS_CMDLINE_CENV)$(SECRETS_CMDLINE_PEM) + +.PHONY: deploy + +DATASTORE_EMULATOR_HOST=localhost:8081 +db: + gcloud beta emulators datastore start diff --git a/apps/common/index.yaml b/apps/common/index.yaml new file mode 100644 index 0000000..fd9abce --- /dev/null +++ b/apps/common/index.yaml @@ -0,0 +1,6 @@ +indexes: +- kind: Token + properties: + - name: login + - name: created + direction: desc diff --git a/apps/common/utils.py b/apps/common/utils.py new file mode 100755 index 0000000..7a16779 --- /dev/null +++ b/apps/common/utils.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +import os +from collections import namedtuple + +import tokenlib + + +_PRInfo = namedtuple("PRInfo", "user org repo pr rev") + + +class PRInfo(_PRInfo): + + @classmethod + def _tm(cls, tm_store=[]): + if not tm_store: + tm_store.append(SimpleTokenManager( + secret=os.environ['CLIENT_TOKEN'])) + return tm_store[0] + + @property + def pr_url(self): + """ + >>> i = PRInfo.parse("mithro:The-OpenROAD-Project/OpenROAD/2@main") + >>> i.pr_url + 'https://github.com/The-OpenROAD-Project/OpenROAD/pull/2' + """ + return f"https://github.com/{self.org}/{self.repo}/pull/{self.pr}" + + @property + def auth_url(self): + return f'https://pr.gha.openroad.tools/auth?state='+self.to_token() + + @property + def hook_url(self): + return f'https://a.gha.openroad.tools/hook?state='+self.to_token() + + @property + def revoke_url(self): + return f'https://a.gha.openroad.tools/revoke?state='+self.to_token() + + @property + def slug(self): + return f"{self.org}/{self.repo}" + + @classmethod + def check(cls, s): + """ + >>> PRInfo.check("mithro:The-OpenROAD-Project/OpenROAD/2@main") + True + >>> PRInfo.check("mithro:The-OpenROAD-Project/OpenROAD/2@") + False + """ + try: + user, org, repo, pr, rev = cls.parse(s) + assert user, s + assert org, s + assert repo, s + assert pr, s + assert rev , s + return True + except Exception as e: + return False + + @classmethod + def from_token(cls, estate): + assert estate is not None, estate + state = cls._tm().parse_token(estate) + return cls.parse(state) + + def to_token(self, salt=None): + """ + >>> i = PRInfo.parse("mithro:The-OpenROAD-Project/OpenROAD/2@main") + >>> t = i.to_token(b'AAA') + >>> t + 'QUFBbWl0aHJvOlRoZS1PcGVuUk9BRC1Qcm9qZWN0L09wZW5ST0FELzJAbWFpbtXQMDVIcDMABpYwq0bR16ASrpgL6HVjgv8ibtahoZzg' + >>> PRInfo.from_token(t) + PRInfo(user='mithro', org='The-OpenROAD-Project', repo='OpenROAD', pr=2, rev='main') + >>> t = i.to_token(b'AAB') + >>> t + 'QUFCbWl0aHJvOlRoZS1PcGVuUk9BRC1Qcm9qZWN0L09wZW5ST0FELzJAbWFpbgwv3YQAVjJL-ewHxQqxEA8aBzOSopyLwO5nW9IUnlma' + >>> PRInfo.from_token(t) + PRInfo(user='mithro', org='The-OpenROAD-Project', repo='OpenROAD', pr=2, rev='main') + """ + return self._tm().make_token(str(self), salt=salt) + + @classmethod + def from_request(cls, r): + estate = r.args.get('state') + assert estate is not None, r.args + return cls.from_token(estate) + + @classmethod + def from_json(cls, j): + return cls( + j['user']['login'], + *j['base']['repo']['full_name'].split('/'), + j['number'], + j['head']['sha']) + + @classmethod + def parse(cls, s): + """ + >>> PRInfo.parse("mithro:The-OpenROAD-Project/OpenROAD/2@main") + PRInfo(user='mithro', org='The-OpenROAD-Project', repo='OpenROAD', pr=2, rev='main') + + """ + assert ':' in s, s + assert '/' in s, s + assert '@' in s, s + rest = s + + user, rest = rest.split(':', maxsplit=1) + org, repo, rest = rest.split('/', maxsplit=2) + pr, rev = rest.split('@', maxsplit=1) + pr = int(pr) + return cls(user, org, repo, pr, rev) + + def __str__(self): + """ + + >>> str(PRInfo('mithro', 'The-OpenROAD-Project', 'OpenROAD', 2, 'main')) + 'mithro:The-OpenROAD-Project/OpenROAD/2@main' + + >>> str(PRInfo('mithro', 'The-OpenROAD-Project', 'OpenROAD', '2', 'main')) + 'mithro:The-OpenROAD-Project/OpenROAD/2@main' + + """ + return f"{self.user}:{self.org}/{self.repo}/{self.pr}@{self.rev}" + + +class SimpleTokenManager(tokenlib.TokenManager): + salt_length = 3 + + def make_token(self, data, salt=None): + """Generate a new token embedding the given dict of data. + + The token is a JSON dump of the given data along with an expiry + time and salt. It has a HMAC signature appended and is b64-encoded + for transmission. + """ + if salt is None: + salt = os.urandom(self.salt_length) + else: + assert len(salt) == self.salt_length, self.salt_length + payload = salt+data.encode("utf8") + sig = self._get_signature(payload) + assert len(sig) == self.hashmod_digest_size + return tokenlib.utils.encode_token_bytes(payload + sig) + + def parse_token(self, token, now=None): + """Extract the data embedded in the given token, if valid. + + The token is valid if it has a valid signature and if the embedded + expiry time has not passed. If the token is not valid then this + method raises ValueError. + """ + # Parse the payload and signature from the token. + try: + decoded_token = tokenlib.utils.decode_token_bytes(token) + except (TypeError, ValueError) as e: + raise tokenlib.errors.MalformedTokenError(str(e)) + payload = decoded_token[:-self.hashmod_digest_size] + sig = decoded_token[-self.hashmod_digest_size:] + # Carefully check the signature. + # This is a deliberately slow string-compare to avoid timing attacks. + # Read the docstring of strings_differ for more details. + expected_sig = self._get_signature(payload) + if tokenlib.utils.strings_differ(sig, expected_sig): + raise tokenlib.errors.InvalidSignatureError() + # Only decode *after* we've confirmed the signature. + # This should never fail, but well, you can't be too careful. + return payload[self.salt_length:].decode('utf-8') + + +if __name__ == "__main__": + os.environ['CLIENT_TOKEN'] = 'doctest' + import doctest + doctest.testmod() diff --git a/apps/pull-request-sender-auth-check/Makefile b/apps/pull-request-sender-auth-check/Makefile new file mode 100644 index 0000000..9eb0552 --- /dev/null +++ b/apps/pull-request-sender-auth-check/Makefile @@ -0,0 +1,23 @@ +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +PROJECT_NAME=pull-request-sender-auth-check +SECRETS_PREFIX=AUTH +SERVICE_ACCOUNT=pull-request-sender + + +include ../common/inc.mk diff --git a/apps/pull-request-sender-auth-check/app.py b/apps/pull-request-sender-auth-check/app.py new file mode 100755 index 0000000..d0d2f70 --- /dev/null +++ b/apps/pull-request-sender-auth-check/app.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +import datetime +import json +import os +import pprint + +import flask + +import db +import utils + +import github_api as gapi +from github_api import checks + + +app = flask.Flask(__name__) + + +BASE_URL = "a.gha.openroad.tools" + +PROJECT_ID = os.environ['PROJECT_ID'] + +CLIENT_ID = os.environ['CLIENT_ID'] +CLIENT_SECRET = os.environ['CLIENT_SECRET'] + +# ---------------------------- +NAME = 'PR Sender Authentication' +EXTERNAL_ID = 'pr-auth' + + +def auth_check_on_private_pr(info): + assert isinstance(info, utils.PRInfo), info + + if info.slug not in ('mithro/OpenROAD-1',): + return + + url = f"https://api.github.com/repos/{info.slug}/commits/{info.rev}/check-runs" + os.environ['GITHUB_REPOSITORY'] = info.slug + check_runs_json = gapi.get_github_json(url) + assert 'check_runs' in check_runs_json, pprint.pformat(check_runs_json) + + existing_checks = [] + for c in check_runs_json['check_runs']: + c.pop('app') + if c['external_id'] != EXTERNAL_ID: + continue + if c['id'] in (2688818725,): + continue + existing_checks.append(c) + + t = db.Token.latest(info.user) + + now = datetime.datetime.utcnow() + if t is None: + new_check = checks.CheckRunCreate( + name = NAME, + head_sha = info.rev, + external_id = EXTERNAL_ID, + + status = checks.CheckStatus.completed, + started_at = now, + + conclusion = checks.CheckConclusion.action_required, + details_url = info.auth_url, + completed_at = now, + + output = checks.CheckRunCreateOutput( + title = "Missing credentials for " +info.user, + summary = f"""\ +The pull request sending robot has not be authorized to send pull requests for {info.user}. + +[Resolve by having {info.user} click this link.]({info.auth_url}) +""", + text = None, + ), + actions = [], + ) + else: + # Create the login okay status check. + new_check = checks.CheckRunCreate( + name = NAME, + head_sha = info.rev, + external_id = EXTERNAL_ID, + + status = checks.CheckStatus.completed, + started_at = now, + + conclusion = checks.CheckConclusion.success, + details_url = None, + completed_at = now, + + output = checks.CheckRunCreateOutput( + title = "Credentials found for "+info.user, + summary = f"""\ +The pull request sending robot found authorization for {info.user}. + +[Double check by having {info.user} click this link.]({info.auth_url}) + +([Revoke authorization]({info.revoke_url})) +""", + text = None, + ), + actions = [], + ) + + uurl = f"https://api.github.com/repos/{info.slug}/check-runs" + if not existing_checks: + r = gapi.send_github_json(uurl, 'POST', new_check) + return 'Need to *create* this check.'+'\n'+pprint.pformat(r)+'\n' + else: + existing_id = existing_checks[0]['id'] + purl = uurl+'/'+str(existing_id) + r = gapi.send_github_json(purl, 'PATCH', new_check) + return f'Needed to *update* this check with {existing_id}.'+'\n'+pprint.pformat(r)+'\n' + +# ---------------------------- + +@app.route("/revoke", methods=['GET', 'POST']) +def revoke(): + info = utils.PRInfo.from_request(flask.request) + db.Token.delete(info.user) + r = auth_check_on_private_pr(info) + return flask.redirect(info.pr_url) + +# ---------------------------- + +@app.route("/hook", methods=['GET', 'POST']) +def hook(): + if flask.request.method == 'POST': + payload = json.loads(flask.request.get_data()) + if 'pull_request' in payload: + info = utils.PRInfo.from_json(payload['pull_request']) + r = auth_check_on_private_pr(info) + if r is not None: + return r + return 'Unknown hook action?\n'+json.dumps(payload, sort_keys=True, indent=2) + else: + info = utils.PRInfo.from_request(flask.request) + r = auth_check_on_private_pr(info) + return flask.redirect(info.pr_url) + +# ---------------------------- + +# Redirect {BASE_URL}/localhost/ to enable OAuth authentication when +# developing locally via redirected through the production URL. +@app.route("/localhost/") +def localhost(rurl): + qs = flask.request.query_string.decode('utf-8') + lurl = f"http://localhost:8080/{rurl}?{qs}" + return flask.redirect(lurl) + + +if __name__ == "__main__": + import sys + d = json.loads(open(sys.argv[1]).read()) + assert 'pull_request' in d, pprint.pformat(d) + info = utils.PRInfo.from_json(d['pull_request']) + print(auth_check_on_private_pr(info)) diff --git a/apps/pull-request-sender-auth-check/main.py b/apps/pull-request-sender-auth-check/main.py new file mode 100755 index 0000000..39b0df7 --- /dev/null +++ b/apps/pull-request-sender-auth-check/main.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +sys.path.insert(0, '../') +sys.path.insert(0, '../../') + +import app + +app.BASE_URL = f"{app.BASE_URL}/localhost" +app.app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080))) diff --git a/apps/pull-request-sender-auth-check/requirements.txt b/apps/pull-request-sender-auth-check/requirements.txt new file mode 100644 index 0000000..36b5d85 --- /dev/null +++ b/apps/pull-request-sender-auth-check/requirements.txt @@ -0,0 +1,5 @@ +Flask==1.0.2 +google-cloud-ndb +tokenlib +gunicorn +jwt diff --git a/apps/pull-request-sender/Makefile b/apps/pull-request-sender/Makefile new file mode 100644 index 0000000..92c3b21 --- /dev/null +++ b/apps/pull-request-sender/Makefile @@ -0,0 +1,22 @@ +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +PROJECT_NAME=pull-request-sender +SECRETS_PREFIX=PR +SERVICE_ACCOUNT=pull-request-sender + +include ../common/inc.mk diff --git a/apps/pull-request-sender/app.py b/apps/pull-request-sender/app.py new file mode 100644 index 0000000..15453b1 --- /dev/null +++ b/apps/pull-request-sender/app.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +import json +import os +import pprint +import requests +import urllib + +import flask + +import db +import utils + +import github_api as gapi +from github_api import env as genv + +app = flask.Flask(__name__) + + +BASE_URL = "pr.gha.openroad.tools" + +PROJECT_ID = os.environ['PROJECT_ID'] + +CLIENT_ID = os.environ['CLIENT_ID'] +CLIENT_SECRET = os.environ['CLIENT_SECRET'] + +# ---------------------------- + +@app.route("/send", methods=['GET', 'POST']) +def send(): + data = json.loads(flask.request.get_data()) + event_json = data['event_json'] + private, staging, upstream, pr_sha = genv.details_from_json(data['env']) + + assert private.sender is not None, (private, staging, upstream) + + # Refresh the auth token + t = db.Token.latest(private.sender) + with db.client.context(): + t = refresh_token(t) + if not isinstance(t, db.Token): + return t+'\n' + assert t.access_token is not None, t + + pr_api_url = f'https://api.github.com/repos/{staging.slug}/commits/{pr_sha}/pulls' + prs_json = gapi.get_github_json( + pr_api_url, + preview="groot-preview", + access_token=t.access_token, + ) + + # Only include open pull requests + prs_json = list(filter(lambda pr: pr['state'] == 'open', prs_json)) + + if not prs_json: + # Need to create a new pull request. + pr_api_url = f'https://api.github.com/repos/{upstream.slug}/pulls' + + create_pr_json = { + "base": upstream.branch, + "head": f"{staging.owner}:{staging.branch}", + "title": event_json["pull_request"]["title"], + "body": event_json["pull_request"]["body"], + "maintainer_can_modify": True, + "draft": True, + } + r = gapi.send_github_json( + pr_api_url, + "POST", + create_pr_json, + access_token=t.access_token, + ) + prs_json.append(r) + + if 'number' not in prs_json[-1]: + return json.dumps(prs_json)+'\n' + return json.dumps(prs_json[-1])+'\n' + + +@app.route("/revoke", methods=['GET', 'POST']) +def revoke(): + user = flask.request.args.get('user') + db.Token.delete(user) + return f'Removed tokens for {user}' + + +@app.route("/auth") +def auth(): + info = utils.PRInfo.from_request(flask.request) + + # Redirect user to GitHub login + rjson = { + 'client_id': CLIENT_ID, + 'redirect_uri': f'https://{BASE_URL}/token', + 'state': info.to_token(), + 'login': info.user, + 'allow_signup': False, + } + + d = requests.get( + url="https://github.com/login/oauth/authorize", + json=rjson, + allow_redirects=False, + ) + return flask.redirect(d.headers['Location']) + + +@app.route("/token", methods=['GET']) +def token(): + info = utils.PRInfo.from_request(flask.request) + + code = flask.request.args.get('code') + + d = requests.get( + url="https://github.com/login/oauth/access_token", + json = { + 'client_id': CLIENT_ID, + 'client_secret': CLIENT_SECRET, + 'code': code, + }, + allow_redirects=False, + ) + + with db.client.context(): + t = db.Token.from_response(info.user, d) + if not isinstance(t, db.Token): + return t + + t.put() + + # Ask the check to refresh + return flask.redirect(info.hook_url) + + +def refresh_token(t): + d = requests.get( + url="https://github.com/login/oauth/access_token", + json = { + 'refresh_token': t.refresh_token, + 'grant_type': 'refresh_token', + 'client_id': CLIENT_ID, + 'client_secret': CLIENT_SECRET, + }, + allow_redirects=False, + ) + + t = db.Token.from_response(t.login, d) + if not isinstance(t, db.Token): + return t + + t.put() + return t + + +@app.route("/refresh") +def refresh(): + user = flask.request.args.get('user') + + # Get existing token + t = db.Token.latest(user) + before = t.to_table() + + # Refresh the token + with db.client.context(): + t = refresh_token(t) + after = t.to_table() + + return f"""\ + + +

Before Refresh

+{before} + +
+ +

After Refresh

+{after} + + + +""" + + +@app.route("/") +def index(): + info = utils.PRInfo.from_request(flask.request) + + t = db.Token.latest(info.user) + return t.to_html() + + +@app.route("/hook", methods=['GET', 'POST']) +def hook(): + data = json.loads(flask.request.get_data()) + return json.dumps(data, sort_keys=True, indent=2) + +# ---------------------------- + +# Redirect {BASE_URL}/localhost/ to enable OAuth authentication when +# developing locally via redirected through the production URL. +@app.route("/localhost/") +def localhost(rurl): + qs = flask.request.query_string.decode('utf-8') + lurl = f"http://localhost:8080/{rurl}?{qs}" + return flask.redirect(lurl) diff --git a/apps/pull-request-sender/main.py b/apps/pull-request-sender/main.py new file mode 100755 index 0000000..39b0df7 --- /dev/null +++ b/apps/pull-request-sender/main.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +sys.path.insert(0, '../') +sys.path.insert(0, '../../') + +import app + +app.BASE_URL = f"{app.BASE_URL}/localhost" +app.app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080))) diff --git a/apps/pull-request-sender/requirements.txt b/apps/pull-request-sender/requirements.txt new file mode 100644 index 0000000..36b5d85 --- /dev/null +++ b/apps/pull-request-sender/requirements.txt @@ -0,0 +1,5 @@ +Flask==1.0.2 +google-cloud-ndb +tokenlib +gunicorn +jwt diff --git a/apps/upstream-sync/Makefile b/apps/upstream-sync/Makefile new file mode 100644 index 0000000..6c2c749 --- /dev/null +++ b/apps/upstream-sync/Makefile @@ -0,0 +1,25 @@ +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +PROJECT_NAME=upstream-sync +SECRETS_PREFIX=UP +SERVICE_ACCOUNT=upstream-sync + +include ../common/inc.mk + +# Doesn't use the common secrets. +SECRETS_CENV = diff --git a/apps/upstream-sync/app.py b/apps/upstream-sync/app.py new file mode 100644 index 0000000..336cee9 --- /dev/null +++ b/apps/upstream-sync/app.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import pprint +import threading +import traceback +import time + +import flask + +import github_api as gapi + +app = flask.Flask(__name__) + + +BASE_URL = "up.gha.openroad.tools" + +PROJECT_ID = os.environ['PROJECT_ID'] + +CLIENT_ID = os.environ['CLIENT_ID'] +CLIENT_SECRET = os.environ['CLIENT_SECRET'] + +# ---------------------------- + +WORKFLOW_ID = 'github-actions-cron-sync-fork-from-upstream.yml' +HEAD_REF = 'refs/heads/' + + +class DispatchTargets: + _cache = None + _cache_updated = 0 + _cache_lock = threading.Lock() + + @classmethod + def get(cls): + now = time.time() + + targets = cls._cache + if now - cls._cache_updated > 60*5: + if not cls._cache_lock.acquire(blocking=False): + # Is another thread is updating the cache, first try and + # return the old values, otherwise wait for them to be + # populated. + while True: + targets = cls._cache + if targets: + return targets + time.sleep(0.1) + + try: + installs = gapi.get_github_json( + 'https://api.github.com/app/installations', jwt=True) + + targets = [] + for i in installs: + targets.append(i["account"]["login"]) + targets = tuple(targets) + + cls._cache = targets + cls._cache_updated = now + + finally: + cls._cache_lock.release() + + return targets + + +def trigger_workflow_dispatch(owner, repo, branch): + # https://docs.github.com/en/rest/reference/actions#create-a-workflow-dispatch-event + # /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches + # workflow_id: The ID of the workflow. You can also pass the workflow file name as a string. + url = f'https://api.github.com/repos/{owner}/{repo}/actions/workflows/{WORKFLOW_ID}/dispatches' + # Payload: + # ref: The git reference for the workflow. The reference can be a branch or tag name. + # inputs: Input keys and values configured in the workflow file. + payload = {'ref': branch} + return gapi.send_github_json(url, 'POST', payload, slug=f'{owner}/{repo}') + + +def trigger_workflow_dispatches(repo, branch): + results = {} + for owner in DispatchTargets.get(): + try: + results[owner] = trigger_workflow_dispatch(owner, repo, branch) + except Exception as e: + results[owner] = { + 'message': str(e), + 'owner': owner, + 'repo': repo, + 'breanch': branch, + 'traceback': traceback.format_exc(), + } + return results + + +@app.route("/hook", methods=['GET', 'POST']) +def hook(): + data = json.loads(flask.request.get_data()) + try: + repo = data['repository']['name'] + head = data['ref'] + assert head.startswith(HEAD_REF), (head, HEAD_REF) + branch = head[len(HEAD_REF):] + + results = [] + if branch in ('master', 'main'): + results = trigger_workflow_dispatches(repo, branch) + + return json.dumps(results, sort_keys=True, indent=2) + + except Exception as e: + return str(e)+'\n'+json.dumps(data, sort_keys=True, indent=2) + + +@app.route("/manual", methods=['GET', 'POST']) +def manual(): + repo = flask.request.args.get('repo') + if not repo: + return 'Missing required "repo" argument.' + branch = flask.request.args.get('branch') + if not branch: + return 'Missing required "branch" argument.' + + results = trigger_workflow_dispatches(repo, branch) + return json.dumps(results, sort_keys=True, indent=2) + + +@app.route("/") +def installs(): + return json.dumps(DispatchTargets.get()) diff --git a/apps/upstream-sync/main.py b/apps/upstream-sync/main.py new file mode 100755 index 0000000..39b0df7 --- /dev/null +++ b/apps/upstream-sync/main.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +sys.path.insert(0, '../') +sys.path.insert(0, '../../') + +import app + +app.BASE_URL = f"{app.BASE_URL}/localhost" +app.app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080))) diff --git a/apps/upstream-sync/requirements.txt b/apps/upstream-sync/requirements.txt new file mode 100644 index 0000000..1d453e5 --- /dev/null +++ b/apps/upstream-sync/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +requests +gunicorn +jwt diff --git a/github_api/__init__.py b/github_api/__init__.py index 0258764..975cf53 100644 --- a/github_api/__init__.py +++ b/github_api/__init__.py @@ -24,8 +24,7 @@ import os import requests - -from datetime import datetime +import datetime as dt def fromisoformat(s): @@ -41,46 +40,70 @@ def fromisoformat(s): return None if s.endswith('Z'): s = s[:-1]+'+00:00' - return datetime.fromisoformat(s) + return dt.datetime.fromisoformat(s) def toisoformat(s): """ - >>> toisoformat(datetime(2021, 5, 3, 1, 48, 37, tzinfo=timezone.utc)) + >>> toisoformat(dt.datetime(2021, 5, 3, 1, 48, 37, tzinfo=dt.timezone.utc)) '2021-05-03T01:48:37Z' - >>> toisoformat(datetime(2021, 5, 3, 1, 48, 37, tzinfo=None)) + >>> toisoformat(dt.datetime(2021, 5, 3, 1, 48, 37, tzinfo=None)) '2021-05-03T01:48:37Z' """ if s is None: return None else: - assert s.tzinfo in (timezone.utc, None), (s.tzinfo, repr(s), str(s)) - if s.tzinfo == timezone.utc: - return datetime.isoformat(s).replace('+00:00', 'Z') + assert s.tzinfo in (dt.timezone.utc, None), (s.tzinfo, repr(s), str(s)) + if s.tzinfo == dt.timezone.utc: + return dt.datetime.isoformat(s).replace('+00:00', 'Z') elif s.tzinfo is None: - return datetime.isoformat(s) + 'Z' + return dt.datetime.isoformat(s) + 'Z' TOKEN_ENV_NAME = 'GITHUB_TOKEN' -def github_headers(preview=None, _headers={}): - if not _headers: - # Figure out the GitHub access token. - access_token = os.environ.get(TOKEN_ENV_NAME, None) +_ACCESS_TOKEN_CACHE = {} + + +def get_access_token(slug=None): + if slug not in _ACCESS_TOKEN_CACHE: + env_token = os.environ.get(TOKEN_ENV_NAME, None) + + if slug is None: + if env_token is not None: + return env_token + else: + slug = os.environ.get('GITHUB_REPOSITORY') + + assert slug is not None, slug + + from . import app_token + access_token = app_token.get_token(slug=slug) if not access_token: - from . import app_token - access_token = app_token.get_token() - if not access_token: - raise SystemError( - f'Did not find an access token of `{TOKEN_ENV_NAME}`') - _headers['Authorization'] = 'token ' + access_token + raise SystemError( + f'Did not find an access token of `{TOKEN_ENV_NAME}`') + + _ACCESS_TOKEN_CACHE[slug] = access_token + + assert _ACCESS_TOKEN_CACHE[slug] is not None, (slug, _ACCESS_TOKEN_CACHE) + return _ACCESS_TOKEN_CACHE[slug] + + +def github_headers(preview=None, slug=None, access_token=None): + if access_token is None: + access_token = get_access_token(slug) + + h = {} + if access_token: + h['Authorization'] = 'token ' + access_token if preview is None: - _headers['Accept'] = 'application/vnd.github.v3+json' + h['Accept'] = 'application/vnd.github.v3+json' else: - _headers['Accept'] = f'application/vnd.github.{preview}+json' - return _headers + h['Accept'] = f'application/vnd.github.{preview}+json' + + return h def cleanup_json_dict(d): @@ -98,11 +121,11 @@ def cleanup_json_dict(d): del d[k] elif isinstance(v, enum.Enum): d[k] = v.value - elif isinstance(v, datetime): - d[k] = fromisoformat(v) + elif isinstance(v, dt.datetime): + d[k] = toisoformat(v) -def send_github_json(url, mode, json_data=None, preview=None): +def send_github_json(url, mode, json_data=None, preview=None, slug=None, access_token=None): assert mode in ('GET', 'POST', 'PATCH', 'DELETE'), f"Unknown mode {mode}" if dataclasses.is_dataclass(json_data): @@ -111,7 +134,7 @@ def send_github_json(url, mode, json_data=None, preview=None): kw = { 'url': url, - 'headers': github_headers(preview=preview), + 'headers': github_headers(preview=preview, slug=slug, access_token=access_token), } if mode == 'POST': f = requests.post @@ -129,14 +152,32 @@ def send_github_json(url, mode, json_data=None, preview=None): f = requests.delete return f(**kw).json() - json_data = f(**kw).json() - return json_data + print(url, mode, json_data, slug) + r = f(**kw) + if r.status_code in (204,): + return (r.status_code, r.reason) + try: + json_data = r.json() + return json_data + except json.decoder.JSONDecodeError: + return (r.status_code, r.text) def get_github_json(url, *args, **kw): + jwt = kw.pop('jwt', False) preview = kw.pop('preview', None) + slug = kw.get('slug', None) + access_token = kw.get('access_token', None) + full_url = url.format(*args, **kw) - return send_github_json(full_url, 'GET', preview=preview) + if jwt: + from . import app_token + return app_token.get_github_json_jwt(full_url) + return send_github_json( + full_url, 'GET', + preview=preview, + slug=slug, + access_token=access_token) if __name__ == "__main__": diff --git a/github_api/app_token.py b/github_api/app_token.py index f192949..df7c364 100755 --- a/github_api/app_token.py +++ b/github_api/app_token.py @@ -1,4 +1,22 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + import json import os @@ -11,14 +29,20 @@ import jwt -GH_APP_PRIVATE_KEY = pathlib.Path(__file__).parent / pathlib.Path("app.private-key.pem") +if 'CLIENT_KEY' in os.environ: + GH_APP_PRIVATE_KEY = pathlib.Path(os.environ['CLIENT_KEY']).resolve() + assert GH_APP_PRIVATE_KEY.exists(), GH_APP_PRIVATE_KEY +else: + GH_APP_PRIVATE_KEY = pathlib.Path(__file__).parent / pathlib.Path("app.private-key.pem") def get_bearer_token(): if not GH_APP_PRIVATE_KEY.exists(): + raise SystemError(f'Needed a bearer token but missing {GH_APP_PRIVATE_KEY}') return None - app_id = os.environ['GITHUB_APP_ID'] + app_id = int(os.environ.get('APP_ID', os.environ.get('GITHUB_APP_ID', 0))) + assert app_id != 0 with open(GH_APP_PRIVATE_KEY, 'rb') as fh: private_key = jwt.jwk_from_pem(fh.read()) @@ -40,23 +64,27 @@ def get_bearer_token(): return i.encode(payload, private_key, alg="RS256") -def get_token(slug=os.environ.get('GITHUB_REPOSITORY', None)): - assert slug is not None +def get_github_json_jwt(url, method='GET'): + btoken = get_bearer_token() + assert btoken is not None, (url, method) headers = { "Accept": "application/vnd.github.v3+json", - "Authorization": "Bearer "+get_bearer_token(), + "Authorization": "Bearer "+btoken, } - install_data = requests.get( - url=f"https://api.github.com/repos/{slug}/installation", - headers=headers, - ).json() + f = getattr(requests, method.lower()) + return f(url=url, headers=headers).json() + + +def get_token(slug=os.environ.get('GITHUB_REPOSITORY', None)): + assert slug is not None + install_data = get_github_json_jwt( + f"https://api.github.com/repos/{slug}/installation") + assert 'id' in install_data, pprint.pformat(install_data) install_id = install_data['id'] - access_data = requests.post( - url=install_data['access_tokens_url'], - headers=headers, - ).json() + access_data = get_github_json_jwt( + install_data['access_tokens_url'], method='POST') return access_data['token'] diff --git a/github_api/checks.py b/github_api/checks.py new file mode 100755 index 0000000..3707276 --- /dev/null +++ b/github_api/checks.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2021 OpenROAD Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +import enum +import json +import pprint +import dataclasses + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Optional + + +""" +Small library for working with GitHub Check Runs / Suites. +""" + + +def datetime_field(): + return field( +# metadata=config( +# encoder=toisoformat, +# decoder=fromisoformat, +# ), + default = None, + ) + + +# ------------------------------------------------------------------- +# Info for an existing check run. +# ------------------------------------------------------------------- + + +# Annotations +# https://api.github.com/repos/octocat/hello-world/check-runs/42/annotations +# ------------------------------------------------------------------- + + +@enum.unique +class CheckRunAnnotationLevel(enum.Enum): + notice = 'notice' + warning = 'warning' + failure = 'failure' + + +@dataclass +class CheckRunAnnotation: + path: str + start_line: int + end_line: int + + annotation_level: CheckRunAnnotationLevel + message: str + title: str + raw_details: str + + start_column: Optional[int] = None + end_column: Optional[int] = None + + +@enum.unique +class CheckStatus(enum.Enum): + queued = 'queued' + in_progress = 'in_progress' + completed = 'completed' + + +# Check Runs +# https://api.github.com/repos/octocat/hello-world/check-runs/42 +# ------------------------------------------------------------------- + + +@enum.unique +class CheckConclusion(enum.Enum): + action_required = 'action_required' + cancelled = 'cancelled' + failure = 'failure' + neutral = 'neutral' + success = 'success' + skipped = 'skipped' + stale = 'stale' + timed_out = 'timed_out' + + +@dataclass +class CheckRunOutput: + title: Optional[str] = None + summary: Optional[str] = None + text: Optional[str] = None + + annotations_count: int = 0 + annotations_url: Optional[str] = None + + +@dataclass +class CheckSuite: + id: int + + +@dataclass +class CheckRun: + + id: int + node_id: str + + name: str + head_sha: str + + details_url: str + external_id: str + + output: CheckRunOutput + + url: str + html_url: str + + #app: dict + check_suite: Optional[CheckSuite] = None + + status: Optional[CheckStatus] = None + started_at: Optional[datetime] = datetime_field() + completed_at: Optional[datetime] = datetime_field() + conclusion: Optional[CheckConclusion] = None + + pull_requests: Optional[list[dict]] = None + + @staticmethod + def _pprint(p, object, stream, indent, allowance, context, level): + p._pprint_dict(dataclasses.asdict(object), stream, indent, allowance, context, level) + + +pprint.PrettyPrinter._dispatch[CheckRun.__repr__] = CheckRun._pprint + + +# ------------------------------------------------------------------- +# Creating a check run. +# ------------------------------------------------------------------- + + +@dataclass +class CheckRunCreateAction: + label: str + description: str + identifier: str + + +@dataclass +class CheckRunCreateOutputImage: + alt: str + image_url: str + caption: str + + +@dataclass +class CheckRunCreateOutput: + title: str # Required + summary: str # Required + text: str + + annotations: list[CheckRunAnnotation] = field(default_factory=list) + images: list[CheckRunCreateOutputImage] = field(default_factory=list) + + +@dataclass +class CheckRunCreate: + # Header: accept = "application/vnd.github.v3+json" + # URL: owner: str = "" + # URL: repo: str = "" + # URL: check_run_id: str = "" + + name: str + head_sha: str + + details_url: Optional[str] = None + external_id: Optional[str] = None + + status: Optional[CheckStatus] = CheckStatus.queued + started_at: Optional[datetime] = datetime_field() + + conclusion: Optional[CheckConclusion] = None + completed_at: Optional[datetime] = datetime_field() + + output: Optional[CheckRunCreateOutput] = None + + actions: list[CheckRunCreateAction] = field(default_factory=list) + + @staticmethod + def _pprint(p, object, stream, indent, allowance, context, level): + p._pprint_dict(dataclasses.asdict(object), stream, indent, allowance, context, level) + + +pprint.PrettyPrinter._dispatch[CheckRunCreate.__repr__] = CheckRunCreate._pprint + + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/github_api/env.py b/github_api/env.py index 8c64475..0239f58 100755 --- a/github_api/env.py +++ b/github_api/env.py @@ -38,6 +38,7 @@ class Repo: repo: str branch: str pr: Optional[int] = None + sender: Optional[str] = None @property def slug(self): @@ -52,6 +53,9 @@ def pr_url(self): assert self.pr is not None, self return f"https://github.com/{self.slug}/pull/{self.pr}" + def as_dict(self): + return dataclasses.asdict(self) + def get_event_json(debug=(os.environ.get('ACTIONS_STEP_DEBUG', None)=='true')): event_json_path = os.environ.get('GITHUB_EVENT_PATH', None) @@ -91,6 +95,14 @@ def get_repo_default_name(key, private, _cache={}): return _cache[key] +def details_from_json(data): + private = Repo(**data['private']) + staging = Repo(**data['staging']) + upstream = Repo(**data['upstream']) + pr_sha = data['pr_sha'] + return (private, staging, upstream, pr_sha) + + def details(event_json=None): # As there are three repositories involved here, things can get a bit # confusing. @@ -124,17 +136,20 @@ def details(event_json=None): branch = event_json['pull_request']['head']['ref'] pr = event_json['pull_request']['number'] pr_sha = event_json['pull_request']['head']['sha'] + sender = event_json['pull_request']['user']['login'] else: repo_json = event_json['repository'] branch = event_json['ref'] pr = None pr_sha = None + sender = event_json['sender']['login'] private_defaults = Repo( owner = repo_json['owner']['login'], repo = repo_json['name'], branch = branch, pr = pr, + sender = sender, ) else: private_defaults = Repo( @@ -155,6 +170,7 @@ def details(event_json=None): 'PRIVATE_BRANCH', private_defaults.branch), pr = private_defaults.pr, + sender = private_defaults.sender, ) staging = Repo( @@ -189,9 +205,7 @@ def details(event_json=None): print(" Staging:", staging.slug, "@", staging.branch, "(", staging.branch_url, ")") print("Upstream:", upstream.slug, "@", upstream.branch, "(", upstream.branch_url, ")") print() - pr_sha_url = f"https://github.com/{private.slug}/commits/{pr_sha}" - print() - print(" Pull request @", pr_sha, "(", pr_sha_url, ")") + print(" Private Pull request @", pr_sha, "(", private.pr_url, ")", "created by", private.sender) print() return (private, staging, upstream, pr_sha) diff --git a/send_pr/action.py b/send_pr/action.py index 64b5522..fc31f93 100755 --- a/send_pr/action.py +++ b/send_pr/action.py @@ -23,7 +23,9 @@ import os import pathlib import pprint +import requests import sys +import urllib sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) @@ -35,51 +37,37 @@ def send_pr(): - github_api.TOKEN_ENV_NAME = 'STAGING_GITHUB_TOKEN' event_json = genv.get_event_json() private, staging, upstream, pr_sha = genv.details(event_json) - - # Figure out if there are any pull requests associated with the sha at the - # moment. - pr_api_url = f'https://api.github.com/repos/{staging.slug}/commits/{pr_sha}/pulls' - prs_json = get_github_json(pr_api_url, preview="groot-preview") + assert private.sender is not None, (private, staging, upstream) + + data = { + 'event_json': event_json, + 'env': { + 'private': private.as_dict(), + 'staging': staging.as_dict(), + 'upstream': upstream.as_dict(), + 'pr_sha': pr_sha, + }, + } print() - print(f"::group::Current pull requests from {staging.slug} for {pr_sha}") - pprint.pprint(prs_json) + print("::group::Sending JSON Data") + print(json.dumps(data)) print("::endgroup::") print() - if not prs_json: - # Need to create a new pull request. - pr_api_url = f'https://api.github.com/repos/{upstream.slug}/pulls' - - create_pr_json = { - "base": upstream.branch, - "head": f"{staging.owner}:{staging.branch}", - "title": event_json["pull_request"]["title"], - "body": event_json["pull_request"]["body"], - "maintainer_can_modify": True, - "draft": True, - } - r = send_github_json(pr_api_url, "POST", create_pr_json) - print(f"::group::Created pull request from {staging.slug} {staging.branch} to {upstream.slug}") - pprint.pprint(r) - print("::endgroup::") - print() - prs_json.append(r) - else: - print() - print("Pull request already existed!") - print() - - upstream.pr = prs_json[-1]["number"] + pr_json = requests.post( + url='https://pr.gha.openroad.tools/send', + json=data, + ).json() print() - print(" Private PR:", private.pr, private.pr_url) - print("Upstream PR:", upstream.pr, upstream.pr_url) - - print("::set-output name=pr::"+str(upstream.pr)) + print("::group::Response JSON Data") + print(json.dumps(pr_json)) + print("::endgroup::") + print() + print("::set-output name=pr::"+str(pr_json['number'])) return