Merge remote-tracking branch 'origin/main' into test-dataflow

.pre-commit-config.yaml

@@ -22,20 +22,17 @@ repos:
       - id: debug-statements
       - id: mixed-line-ending
 
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-      - id: black
-
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.241'
     hooks:
-      - id: flake8
+      - id: ruff
+        args: ['--fix']
 
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.2.0
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
     hooks:
-      - id: seed-isort-config
+      - id: black
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: 'v0.991'
@@ -44,11 +41,6 @@ repos:
         additional_dependencies:
           - 'pydantic'
 
-  - repo: https://github.com/pycqa/isort
-    rev: 5.11.4
-    hooks:
-      - id: isort
-
   - repo: https://github.com/rstcheck/rstcheck
     rev: v6.1.1
     hooks:

pangeo_forge_orchestrator/config.py

@@ -1,7 +1,7 @@
 import json
 import os
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Optional
 
 import yaml  # type: ignore
 from pydantic import BaseModel, Extra, Field, SecretStr
@@ -19,7 +19,7 @@ class GitHubAppConfig(BaseModel):
     app_name: str
     webhook_secret: str
     private_key: str
-    run_only_on: Optional[List[str]] = None
+    run_only_on: Optional[list[str]] = None
 
 
 def get_gcr_container_image_url():
@@ -102,7 +102,7 @@ def export_with_secrets(self):
 class Config(BaseModel):
     fastapi: FastAPIConfig
     github_app: GitHubAppConfig
-    bakeries: Dict[str, Bakery]
+    bakeries: dict[str, Bakery]
 
 
 def get_app_config_path() -> str:
@@ -140,7 +140,7 @@ def get_secrets_dir():
     return f"{root}/secrets"
 
 
-def get_secret_bakery_args_paths() -> List[str]:
+def get_secret_bakery_args_paths() -> list[str]:
     return [p for p in os.listdir(get_secrets_dir()) if p.startswith("bakery-args")]
 
 

pangeo_forge_orchestrator/model_builders.py

@@ -1,6 +1,6 @@
 import types
 from dataclasses import dataclass
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 from sqlmodel import Field, Relationship, SQLModel
 
@@ -21,7 +21,7 @@ class RelationBuilder:
     """
 
     field: str
-    annotation: Union[SQLModel, List[str]]
+    annotation: Union[SQLModel, list[str]]
     back_populates: str
 
 
@@ -65,7 +65,7 @@ class MultipleModels:
     response: SQLModel
     descriptive_name: str
     extended_response: Optional[SQLModel] = None
-    relations: Optional[List[RelationBuilder]] = None
+    relations: Optional[list[RelationBuilder]] = None
 
     def __post_init__(self):
         self.creation: SQLModel = self.make_creator_cls()

pangeo_forge_orchestrator/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from enum import Enum
-from typing import List, Optional
+from typing import Optional
 
 from sqlmodel import Field, SQLModel
 
@@ -155,11 +155,11 @@ class RecipeRunRead(RecipeRunBase):
 
 
 class BakeryReadWithRecipeRuns(BakeryRead):
-    recipe_runs: List[RecipeRunRead]
+    recipe_runs: list[RecipeRunRead]
 
 
 class FeedstockReadWithRecipeRuns(FeedstockRead):
-    recipe_runs: List[RecipeRunRead]
+    recipe_runs: list[RecipeRunRead]
 
 
 class RecipeRunReadWithBakeryAndFeedstock(RecipeRunRead):
@@ -179,7 +179,7 @@ class RecipeRunReadWithBakeryAndFeedstock(RecipeRunRead):
     relations=[
         RelationBuilder(
             field="recipe_runs",
-            annotation=List["RecipeRun"],  # type: ignore # noqa: F821
+            annotation=list["RecipeRun"],  # type: ignore # noqa: F821
             back_populates="bakery",
         ),
     ],
@@ -193,7 +193,7 @@ class RecipeRunReadWithBakeryAndFeedstock(RecipeRunRead):
     relations=[
         RelationBuilder(
             field="recipe_runs",
-            annotation=List["RecipeRun"],  # type: ignore # noqa: F821
+            annotation=list["RecipeRun"],  # type: ignore # noqa: F821
             back_populates="feedstock",
         ),
     ],

pangeo_forge_orchestrator/routers/github_app.py

@@ -7,7 +7,7 @@
 import time
 from datetime import datetime
 from textwrap import dedent
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 from urllib.parse import parse_qs, urlparse
 
 import aiohttp
@@ -320,13 +320,11 @@ async def handle_dataflow_event(
     gh: GitHubAPI,
     gh_kws: dict,
 ):
-
     logger.info(f"Received dataflow webhook with {payload = }")
 
     action = payload["action"]
 
     if action == "completed":
-
         if payload["conclusion"] not in ("success", "failure"):
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -363,7 +361,7 @@ async def handle_dataflow_event(
 
         # Wow not every day you google a error and see a comment on it by Guido van Rossum
         # https://github.com/python/mypy/issues/1174#issuecomment-175854832
-        args: List[SQLModel] = [recipe_run]  # type: ignore
+        args: list[SQLModel] = [recipe_run]  # type: ignore
         if recipe_run.is_test:
             args.append(feedstock.spec)  # type: ignore
             logger.info(f"Calling `triage_test_run_complete` with {args=}")
@@ -407,7 +405,6 @@ async def handle_pr_comment_event(
     pr = await gh.getitem(payload["issue"]["pull_request"]["url"], **gh_kws)
 
     if action == "created":
-
         comment_body = comment["body"]
         if not comment_body.startswith("/"):
             # Exit early if this isn't a slash command, so we don't end up spamming *every* issue
@@ -458,10 +455,10 @@ async def handle_pr_comment_event(
 
 async def handle_pr_event(
     *,
-    payload: Dict,
-    gh_kws: Dict[str, Any],
+    payload: dict,
+    gh_kws: dict[str, Any],
     gh: GitHubAPI,
-    session_kws: Dict[str, Any],
+    session_kws: dict[str, Any],
     background_tasks: BackgroundTasks,
 ):
     """Process a PR event."""
@@ -472,7 +469,6 @@ async def handle_pr_event(
     pr = payload["pull_request"]
 
     if action in ("synchronize", "opened"):
-
         base_repo_name = pr["base"]["repo"]["full_name"]
         if ignore_repo(base_repo_name):
             return {"status": "ok", "message": f"Skipping synchronize for repo {base_repo_name}"}
@@ -559,7 +555,6 @@ async def parse_payload(request, payload_bytes, event):
 
 async def verify_hash_signature(request: Request, payload_bytes: bytes) -> None:
     if hash_signature := request.headers.get("X-Hub-Signature-256", None):
-
         github_app = get_config().github_app
         webhook_secret = bytes(github_app.webhook_secret, encoding="utf-8")  # type: ignore
         h = hmac.new(webhook_secret, payload_bytes, hashlib.sha256)
@@ -625,7 +620,7 @@ async def make_dataflow_job_name(recipe_run: SQLModel, gh: GitHubAPI):
     # server, and so we do need to preseve that.
     to_encode = p.netloc if p.path == "/github/hooks/" else p.netloc + p.path
     to_encode += "%"  # control character to separate webhook url encoding from recipe run id
-    as_hex = "".join(["{:02x}".format(ord(x)) for x in to_encode])
+    as_hex = "".join([f"{ord(x):02x}" for x in to_encode])
     # decoding is easier if recipe_run id encoded with an even length, so zero-pad if odd.
     # note we are using ``f"0{recipe_run.id}"``, *not* ``f"{recipe_run_id:02}"`` to pad if odd,
     # because we want the *number of digits* to be even regardless of the length of the odd input.

pangeo_forge_orchestrator/routers/model_router.py

@@ -1,4 +1,4 @@
-from typing import List, Literal
+from typing import Literal
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import and_
@@ -116,7 +116,7 @@ def delete(
         model.path,
         make_read_range_endpoint(model),
         methods=["GET"],
-        response_model=List[model.response],  # type: ignore
+        response_model=list[model.response],  # type: ignore
         summary=f"Read a range of {model.descriptive_name} objects",
         tags=[model.descriptive_name, "public"],
     )
@@ -158,7 +158,6 @@ def get_feedstock_datasets(
         "all", description="Filter by whether the dataset is a production or test dataset"
     ),
 ):
-
     model = MODELS["recipe_run"]
 
     statement = and_(

pangeo_forge_orchestrator/routers/repr.py

@@ -13,14 +13,12 @@ def xarray(
         example="https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/HadISST-feedstock/hadisst.zarr",
     )
 ):
-
     import xarray as xr
     import zarr
 
     error_message = f"An error occurred while fetching the data from URL: {url}"
 
     try:
-
         with xr.open_dataset(url, engine="zarr", chunks={}) as ds:
             html = ds._repr_html_().strip().encode("utf-8", "replace").decode("utf-8")
 

pyproject.toml

@@ -10,3 +10,57 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools_scm]
 write_to = "pangeo_forge_orchestrator/_version.py"
 write_to_template = "__version__ = '{version}'"
+
+[tool.ruff]
+line-length = 100
+target-version = "py39"
+builtins = ["ellipsis"]
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+per-file-ignores = {}
+# E402: module level import not at top of file
+# E501: line too long - let black worry about that
+# E731: do not assign a lambda expression, use a def
+ignore = [
+    "E402",
+    "E501",
+    "E731",
+]
+select = [
+    # Pyflakes
+    "F",
+    # Pycodestyle
+    "E",
+    "W",
+    # isort
+    "I",
+    # Pyupgrade
+    "UP",
+]
+
+
+[tool.ruff.mccabe]
+max-complexity = 18
+
+[tool.ruff.isort]
+known-first-party = ["pangeo_forge_orchestrator"]

scripts.develop/new_github_app.py

@@ -50,7 +50,6 @@ def get_app_name_and_desc(github_username, deployment, pr_number=None):
 
 
 def main(github_username, deployment, pr_number=None):
-
     app_name, description = get_app_name_and_desc(github_username, deployment, pr_number=pr_number)
 
     # We're going to serve from the `CACHEDIR` as base,
@@ -109,7 +108,6 @@ def main(github_username, deployment, pr_number=None):
 
 
 if __name__ == "__main__":
-
     user_token = os.environ.get("GITHUB_PAT", None)
     if not user_token:
         raise ValueError("Env variable 'GITHUB_PAT' required, but unset.")

setup.cfg

@@ -54,21 +54,6 @@ dev =
 console_scripts =
     pangeo-forge = pangeo_forge_orchestrator.cli:cli
 
-[flake8]
-ignore =
-max-line-length = 100
-max-complexity = 18
-select = B,C,E,F,W,T4,B9
-extend-ignore = E203,E501,E402,W605,W503
-
-[isort]
-known_first_party=pangeo_forge_orchestrator
-known_third_party=aiohttp,alembic,asgi_lifespan,cryptography,fastapi,gidgethub,httpx,jwt,pydantic,pytest,pytest_asyncio,pytest_lazyfixture,requests,setuptools,sqlalchemy,sqlmodel,starlette,yaml
-multi_line_output=3
-include_trailing_comma=True
-force_grid_wrap=0
-combine_as_imports=True
-line_length=100
 
 [tool:pytest]
 log_cli = False

tests/github_app/mock_gidgethub.py

@@ -1,7 +1,7 @@
 import hashlib
 import random
 from dataclasses import dataclass
-from typing import Dict, List, Optional, Union
+from typing import Optional, Union
 
 from pangeo_forge_orchestrator.http import HttpSession
 from pangeo_forge_orchestrator.routers.github_app import get_jwt
@@ -22,13 +22,13 @@ def mock_access_token_from_jwt(jwt: str):
 @dataclass
 class _MockGitHubBackend:
     _app_hook_config_url: Optional[str] = None
-    _accessible_repos: Optional[List[dict]] = None
-    _repositories: Optional[Dict[str, dict]] = None
-    _app_hook_deliveries: Optional[List[dict]] = None
-    _app_installations: Optional[List[dict]] = None
-    _check_runs: Optional[List[dict]] = None
-    _pulls: Optional[Dict[str, Dict[int, dict]]] = None
-    _pulls_files: Optional[Dict[str, Dict[int, dict]]] = None
+    _accessible_repos: Optional[list[dict]] = None
+    _repositories: Optional[dict[str, dict]] = None
+    _app_hook_deliveries: Optional[list[dict]] = None
+    _app_installations: Optional[list[dict]] = None
+    _check_runs: Optional[list[dict]] = None
+    _pulls: Optional[dict[str, dict[int, dict]]] = None
+    _pulls_files: Optional[dict[str, dict[int, dict]]] = None
 
 
 @dataclass
@@ -46,7 +46,7 @@ async def getitem(
         accept: Optional[str] = None,
         jwt: Optional[str] = None,
         oauth_token: Optional[str] = None,
-    ) -> Union[dict, List[dict]]:
+    ) -> Union[dict, list[dict]]:
         if path == "/app/hook/config":
             return {"url": self._backend._app_hook_config_url}
         # gidgethub allows us to call either the relative or absolute path, and we do both