From 93a093953edd9e593f58c881c71b90d997651dd9 Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Thu, 26 Jan 2023 14:22:16 -0800 Subject: [PATCH 01/12] first commit --- main.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ 2 files changed, 52 insertions(+) create mode 100644 main.py create mode 100644 requirements.txt diff --git a/main.py b/main.py new file mode 100644 index 0000000..c0f4c2e --- /dev/null +++ b/main.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass + +import flask +import functions_framework +import requests + + +@dataclass +class GitHubLabel: + id: int + node_id: str + url: str + name: str + color: str + default: bool + description: str + + +@functions_framework.http +def forwarder(request: flask.Request): + """ + + """ + + event = request.headers.get("X-GitHub-Event") + print(f"{event = }") + if event == "pull_request": + labels = [ + GitHubLabel(**label) + for label in request.json["pull_request"]["labels"] + ] + if not labels: + return "No labels", 400 + + forward_to = [ + l.name.split("fwd:")[-1] + for l in labels + if l.name.startswith("fwd:") + ] + if not forward_to: + return "No labels starting with 'fwd:'", 400 + + for recipient in forward_to: + print(f"Forwarding GitHub webhook to {recipient = }") + requests.post( + f"https://{recipient}", + headers=request.headers, + json=request.json, + ) + return f"Forwarded to {forward_to}", 200 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b8435b0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +functions-framework==3.3.0 +requests==2.28.2 From 17b68b4f84421a5442a155634859930b6988c300 Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Thu, 26 Jan 2023 17:58:17 -0800 Subject: [PATCH 02/12] make this a heroku app --- Procfile | 1 + app.json | 17 +++++++++++++++ dev-requirements.txt | 1 + main.py | 51 ++++++++++++++++++++++++-------------------- requirements.txt | 6 ++++-- runtime.txt | 1 + test.py | 2 ++ 7 files changed, 54 insertions(+), 25 deletions(-) create mode 100644 Procfile create mode 100644 app.json create mode 100644 dev-requirements.txt create mode 100644 runtime.txt create mode 100644 test.py diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..11e8596 --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: gunicorn -w 2 -k uvicorn.workers.UvicornWorker main:app \ No newline at end of file diff --git a/app.json b/app.json new file mode 100644 index 0000000..928c83b --- /dev/null +++ b/app.json @@ -0,0 +1,17 @@ +{ + "name": "GitHub Webhook Forwarder", + "description": "Forwards GitHub webhooks to dev instances of Pangeo Forge API.", + "image": "heroku/python", + "repository": "https://github.com/pangeo-forge/github-webhook-forwarder", + "keywords": [], + "addons": [], + "env": {}, + "environments": { + "test": { + "scripts": { + "test-setup": "python -m pip install -r dev-requirements.txt", + "test": "pytest test.py -v" + } + } + } + } diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..55b033e --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1 @@ +pytest \ No newline at end of file diff --git a/main.py b/main.py index c0f4c2e..300ae48 100644 --- a/main.py +++ b/main.py @@ -1,36 +1,34 @@ -from dataclasses import dataclass +from fastapi import FastAPI, HTTPException, Request, status +from pydantic import BaseModel -import flask -import functions_framework -import requests +app = FastAPI() -@dataclass -class GitHubLabel: - id: int - node_id: str - url: str +class GitHubLabel(BaseModel): name: str - color: str - default: bool - description: str -@functions_framework.http -def forwarder(request: flask.Request): +@app.post("/", status_code=status.HTTP_200_OK) +async def forwarder(request: Request): """ """ event = request.headers.get("X-GitHub-Event") print(f"{event = }") + + request_json = await request.json() + if event == "pull_request": labels = [ GitHubLabel(**label) - for label in request.json["pull_request"]["labels"] + for label in request_json["pull_request"]["labels"] ] if not labels: - return "No labels", 400 + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No labels." + ) forward_to = [ l.name.split("fwd:")[-1] @@ -38,13 +36,20 @@ def forwarder(request: flask.Request): if l.name.startswith("fwd:") ] if not forward_to: - return "No labels starting with 'fwd:'", 400 + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No labels starting with 'fwd:'", + ) for recipient in forward_to: print(f"Forwarding GitHub webhook to {recipient = }") - requests.post( - f"https://{recipient}", - headers=request.headers, - json=request.json, - ) - return f"Forwarded to {forward_to}", 200 + # FIXME: use httpx to make this request (aiohttp doesnt work w python 3.11) + # async with aiohttp.ClientSession() as session: + # async with session.post( + # f"https://{recipient}", + # headers=request.headers, + # json=request_json, + # ) as response: + # ... + + return f"Forwarded to {forward_to}" diff --git a/requirements.txt b/requirements.txt index b8435b0..5205612 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ -functions-framework==3.3.0 -requests==2.28.2 +fastapi==0.87.0 +gunicorn==20.1.0 +httpx==0.23.3 +uvicorn==0.20.0 diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 0000000..cd6f130 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.11.1 \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..d587138 --- /dev/null +++ b/test.py @@ -0,0 +1,2 @@ +def test_main(): + ... From 7d64fcc5ed40428fb191476355c1cd7e3e9c8dde Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 27 Jan 2023 11:33:52 -0800 Subject: [PATCH 03/12] add httpx forwarding block --- main.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index 300ae48..b4e3eae 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +import httpx from fastapi import FastAPI, HTTPException, Request, status from pydantic import BaseModel @@ -41,15 +42,16 @@ async def forwarder(request: Request): detail="No labels starting with 'fwd:'", ) + responses = {} for recipient in forward_to: print(f"Forwarding GitHub webhook to {recipient = }") - # FIXME: use httpx to make this request (aiohttp doesnt work w python 3.11) - # async with aiohttp.ClientSession() as session: - # async with session.post( - # f"https://{recipient}", - # headers=request.headers, - # json=request_json, - # ) as response: - # ... - - return f"Forwarded to {forward_to}" + async with httpx.AsyncClient() as client: + r = await client.post( + f"https://{recipient}", + headers=request.headers, + json=request_json, + ) + print(f"{recipient = } responded with {r.status_code = }") + responses |= {recipient: r.status_code} + + return f"Forwarded to {responses}" From b95d0e5772ee6c74d22ad3ee4c24dccd8c78305e Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 27 Jan 2023 12:40:17 -0800 Subject: [PATCH 04/12] use bytes instead of json --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index b4e3eae..191cfa8 100644 --- a/main.py +++ b/main.py @@ -19,6 +19,7 @@ async def forwarder(request: Request): print(f"{event = }") request_json = await request.json() + request_bytes = await request.body() if event == "pull_request": labels = [ @@ -49,7 +50,7 @@ async def forwarder(request: Request): r = await client.post( f"https://{recipient}", headers=request.headers, - json=request_json, + json=request_bytes, ) print(f"{recipient = } responded with {r.status_code = }") responses |= {recipient: r.status_code} From 3e0fa63a7ea9d8f0e40baa0232f6359e91f3fc89 Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 27 Jan 2023 12:52:58 -0800 Subject: [PATCH 05/12] maybe decode payload bytes --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 191cfa8..f05e2be 100644 --- a/main.py +++ b/main.py @@ -50,7 +50,7 @@ async def forwarder(request: Request): r = await client.post( f"https://{recipient}", headers=request.headers, - json=request_bytes, + json=request_bytes.decode("utf-8"), ) print(f"{recipient = } responded with {r.status_code = }") responses |= {recipient: r.status_code} From 07191b5045c3343976eb247c562ea049f57d936a Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 27 Jan 2023 13:15:01 -0800 Subject: [PATCH 06/12] pass request_bytes to data, not json --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index f05e2be..c531eb5 100644 --- a/main.py +++ b/main.py @@ -50,7 +50,7 @@ async def forwarder(request: Request): r = await client.post( f"https://{recipient}", headers=request.headers, - json=request_bytes.decode("utf-8"), + data=request_bytes, ) print(f"{recipient = } responded with {r.status_code = }") responses |= {recipient: r.status_code} From 1377151dae968917e52e0f74a584d15989b06632 Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 27 Jan 2023 15:36:59 -0800 Subject: [PATCH 07/12] might need to cast headers to dict --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index c531eb5..0cc77d8 100644 --- a/main.py +++ b/main.py @@ -49,7 +49,7 @@ async def forwarder(request: Request): async with httpx.AsyncClient() as client: r = await client.post( f"https://{recipient}", - headers=request.headers, + headers=dict(request.headers), data=request_bytes, ) print(f"{recipient = } responded with {r.status_code = }") From 94a5837c2bf147b86396d6d6ddc52daa49ea2749 Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 27 Jan 2023 16:32:43 -0800 Subject: [PATCH 08/12] subset headers for forwarding; getting 404 if not --- main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 0cc77d8..eecd923 100644 --- a/main.py +++ b/main.py @@ -49,7 +49,10 @@ async def forwarder(request: Request): async with httpx.AsyncClient() as client: r = await client.post( f"https://{recipient}", - headers=dict(request.headers), + headers={ + "X-GitHub-Event": request.headers.get("X-GitHub-Event"), + "X-Hub-Signature-256": request.headers.get("X-Hub-Signature-256"), + }, data=request_bytes, ) print(f"{recipient = } responded with {r.status_code = }") From 0e69e069a8410b43b55111cbaab20b7c496bc056 Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 27 Jan 2023 17:28:49 -0800 Subject: [PATCH 09/12] support issue_comment events --- main.py | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/main.py b/main.py index eecd923..5458436 100644 --- a/main.py +++ b/main.py @@ -22,26 +22,32 @@ async def forwarder(request: Request): request_bytes = await request.body() if event == "pull_request": - labels = [ - GitHubLabel(**label) - for label in request_json["pull_request"]["labels"] - ] - if not labels: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="No labels." - ) + label_containing_obj = "pull_request" + elif event == "issue_comment": + label_containing_obj = "issue" + else: + raise NotImplementedError - forward_to = [ - l.name.split("fwd:")[-1] - for l in labels - if l.name.startswith("fwd:") - ] - if not forward_to: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="No labels starting with 'fwd:'", - ) + labels = [ + GitHubLabel(**label) + for label in request_json[label_containing_obj]["labels"] + ] + if not labels: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No labels." + ) + + forward_to = [ + l.name.split("fwd:")[-1] + for l in labels + if l.name.startswith("fwd:") + ] + if not forward_to: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No labels starting with 'fwd:'", + ) responses = {} for recipient in forward_to: From c2ba72a21429792aa2bf3bc9da8c528d5e0c569f Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:21:43 -0800 Subject: [PATCH 10/12] forward json as well --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 5458436..c86b17b 100644 --- a/main.py +++ b/main.py @@ -60,6 +60,7 @@ async def forwarder(request: Request): "X-Hub-Signature-256": request.headers.get("X-Hub-Signature-256"), }, data=request_bytes, + json=request_json, ) print(f"{recipient = } responded with {r.status_code = }") responses |= {recipient: r.status_code} From 4e0b7d4c55059097210fe2b845d0ca03967f5875 Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:26:57 -0800 Subject: [PATCH 11/12] add content type --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index c86b17b..f3f4938 100644 --- a/main.py +++ b/main.py @@ -58,6 +58,7 @@ async def forwarder(request: Request): headers={ "X-GitHub-Event": request.headers.get("X-GitHub-Event"), "X-Hub-Signature-256": request.headers.get("X-Hub-Signature-256"), + "content-type": "application/json", }, data=request_bytes, json=request_json, From 6581e52431e54882b2c6ea1aad1057d89b31bddb Mon Sep 17 00:00:00 2001 From: Charles Stern <62192187+cisaacstern@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:29:30 -0800 Subject: [PATCH 12/12] drop json from fwded request --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index f3f4938..88ae44f 100644 --- a/main.py +++ b/main.py @@ -61,7 +61,6 @@ async def forwarder(request: Request): "content-type": "application/json", }, data=request_bytes, - json=request_json, ) print(f"{recipient = } responded with {r.status_code = }") responses |= {recipient: r.status_code}