From f754c31df1638ab359416cdf3446f56320a12fd9 Mon Sep 17 00:00:00 2001 From: Hraban Luyat Date: Sun, 8 Dec 2024 01:20:16 -0500 Subject: [PATCH] feat: POC: brrr lib & demo Functional POC. Co-authored-by: Jesse Zwaan --- .editorconfig | 11 + .github/workflows/ci.yaml | 12 + .gitignore | 8 +- README.md | 9 + brrr-demo.nix | 42 +++ brrr.py | 269 ---------------- brrr_demo.py | 68 ++++ flake.lock | 212 +++++++++++++ flake.nix | 236 ++++++++++++++ localstack.nix | 41 +++ poetry.lock | 564 --------------------------------- pyproject.toml | 27 +- src/brrr/__init__.py | 10 + src/brrr/asyncrrr.py | 9 + src/brrr/backends/__init__.py | 0 src/brrr/backends/dynamo.py | 111 +++++++ src/brrr/backends/in_memory.py | 26 ++ src/brrr/backends/redis.py | 205 ++++++++++++ src/brrr/backends/sqs.py | 40 +++ src/brrr/brrr.py | 408 ++++++++++++++++++++++++ src/brrr/compat.py | 48 +++ src/brrr/queue.py | 34 ++ src/brrr/store.py | 132 ++++++++ uv.lock | 288 +++++++++++++++++ 24 files changed, 1965 insertions(+), 845 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/workflows/ci.yaml create mode 100644 brrr-demo.nix delete mode 100644 brrr.py create mode 100755 brrr_demo.py create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 localstack.nix delete mode 100644 poetry.lock create mode 100644 src/brrr/__init__.py create mode 100644 src/brrr/asyncrrr.py create mode 100644 src/brrr/backends/__init__.py create mode 100644 src/brrr/backends/dynamo.py create mode 100644 src/brrr/backends/in_memory.py create mode 100644 src/brrr/backends/redis.py create mode 100644 src/brrr/backends/sqs.py create mode 100644 src/brrr/brrr.py create mode 100644 src/brrr/compat.py create mode 100644 src/brrr/queue.py create mode 100644 src/brrr/store.py create mode 100644 uv.lock diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4ce789b --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 + +[*.py] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..c88f412 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,12 @@ +on: + pull_request: + push: + +name: Syntax +jobs: + nocommit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: "nocommit checker" + uses: nobssoftware/nocommit@v2 diff --git a/.gitignore b/.gitignore index 1d17dae..126b2e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ -.venv +.venv +.envrc +.terraform* +data +__pycache__ +result +.direnv diff --git a/README.md b/README.md index e69de29..0369d6c 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,9 @@ +# Brrr: high performance workflow scheduling + +Brrr is a POC for ultra high performance workflow scheduling. + +## Copyright & License + +Brrr was written by Hraban Luyat and Jesse Zwaan. It is available under the AGPLv3 license (not later). + +See the [LICENSE](LICENSE) file. diff --git a/brrr-demo.nix b/brrr-demo.nix new file mode 100644 index 0000000..898fdab --- /dev/null +++ b/brrr-demo.nix @@ -0,0 +1,42 @@ +# Copyright © 2024 Brrr Authors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Brrr-demo module for process-compose-flake + +{ config, pkgs, name, lib, ... }: { + options = with lib.types; { + # You’ll want to override this unless you use an overlay + package = lib.mkPackageOption pkgs "brrr-demo" { }; + args = lib.mkOption { + default = []; + type = listOf str; + }; + environment = lib.mkOption { + type = types.attrsOf types.str; + default = { }; + example = { + AWS_ENDPOINT_URL = "http://localhost:12345"; + }; + description = '' + Extra environment variables passed to the `brrr-demo` process. + ''; + }; + }; + config = { + outputs.settings.processes.${name} = { + environment = config.environment; + command = "${lib.getExe config.package} ${lib.escapeShellArgs config.args}"; + }; + }; +} diff --git a/brrr.py b/brrr.py deleted file mode 100644 index 634fe5f..0000000 --- a/brrr.py +++ /dev/null @@ -1,269 +0,0 @@ -from typing import Any -import pickle -import sys -import json -from dataclasses import dataclass -from hashlib import sha256 -import asyncio -from queue import Queue - -import boto3 - -# Takes a number of task lambdas and calls each of them. -# If they've all been computed, return their values, -# Otherwise raise jobs for those that haven't been computed -def gather(*task_lambdas): - defer = Call.Defer([]) - values = [] - for task in task_lambdas: - try: - values.append(task()) - except Call.Defer as d: - print("DEFER", d) - defer.calls.extend(d.calls) - if defer.calls: - raise defer - return values - - -def input_hash(*args): - # return ":".join(map(repr, args)) - return sha256(":".join(map(str, args)).encode()).hexdigest() - -class PickleJar: - def __init__(self): - self.pickles = {} - - def __contains__(self, key): - return key in self.pickles - - def __getitem__(self, key): - return pickle.loads(self.pickles[key]) - - def __setitem__(self, key, value): - self.pickles[key] = pickle.dumps(value) - -# A queue of frame keys to be processed -queue = Queue() - -# A store of task frame contexts. A frame looks something like: -# -# memo_key: A hash of the task and its arguments -# children: A dictionary of child tasks keys, with a bool indicating whether they have been computed (This could be a set) -# parent: The caller's frame_key -# -# In the real world, this would be some sort of distributed store, -# optimised for specific access patterns -frames = {} - -# A memoization cache for tasks that have already been computed, based on their task name and input arguments -memos = PickleJar() - -# Using the same memo key, we store the task and its argv here so we can retrieve them in workers -calls = PickleJar() - - -class Task: - """ - A decorator to turn a function into a task. - When it is called, it checks whether it has already been computed. - If so, it returns the value, otherwise it raises a Call job - - A task can not write to the store, only read from it - """ - _tasks: dict[str, 'Task'] = {} - - fn: Any - def __init__(self, fn): - self.fn = fn - self._tasks[fn.__name__] = self - - def __str__(self): - return f"{self.fn.__name__}()" - - def __repr__(self): - return f"Task({self.fn.__name__})" - - @classmethod - def from_name(cls, name): - return cls._tasks[name] - - def to_name(self): - return self.fn.__name__ - - # @property - # def context(self): - # return context[threading.get_ident()] - - # Calling a function returns the value if it has already been computed. - # Otherwise, it raises a Call exception to schedule the computation - def __call__(self, *args, **kwargs): - print("Calling", self.fn.__name__, kwargs) - key = self.memo_key((args, kwargs)) - if key not in memos: - raise Call.Defer([Call(self, (args, kwargs))]) - return memos[key] - - def to_lambda(self, *args, **kwargs): - """ - Is a separate function to capture a closure - """ - return lambda: self(*args, **kwargs) - - # Fanning out, a map function returns the values if they have already been computed. - # Otherwise, it raises a list of Call exceptions to schedule the computation, - # for the ones that aren't already computed - def map(self, args): - print("MAP", args) - return gather(*(self.to_lambda(**arg) for arg in args)) - - def resolve(self, argv): - return self.fn(*argv[0], **argv[1]) - - # Some deterministic hash of a task and its arguments - def memo_key(self, argv): - """A deterministic hash of a task and its arguments""" - return input_hash(self.to_name(), argv) - -task = Task - -class Frame: - """ - A frame represents a function call with a parent and a number of child frames - """ - memo_key: str - children: dict - parent_key: str - - def __repr__(self): - return f"Frame({self.memo_key}, {self.children}, {self.parent_key})" - - def __init__(self, memo_key, parent_key): - self.memo_key = memo_key - self.children = {} - self.parent_key = parent_key - - # A key is a unique identifier for a task and its arguments - def resolve(self): - print("RESOLVE", self) - name, argv = calls[self.memo_key] - return Task.from_name(name).resolve(argv) - - @property - def is_waiting(frame_key): - return not all(frames[frame_key].children.values()) - - -class Call: - class Defer(Exception): - calls: list['Call'] - def __init__(self, calls: list['Call']): - print("Deferring", calls) - self.calls = calls - - task: Task - argv: tuple - - def __repr__(self): - return f"Call({self.task.fn.__name__}, {json.dumps(self.argv)})" - - def __init__(self, task: Task, argv: tuple[tuple, dict]): - self.task = task - self.argv = argv - - @property - def memo_key(self): - return self.task.memo_key(self.argv) - - def frame_key(self, parent_key): - return input_hash(parent_key, self.memo_key) - - def schedule(self, parent_key=None): - memo_key = self.memo_key - if memo_key not in calls: - calls[memo_key] = (self.task.to_name(), self.argv) - - child_key = self.frame_key(parent_key) - if child_key not in frames: - frames[child_key] = Frame(memo_key, parent_key) - - if memo_key in memos: - if parent_key is not None: - frames[parent_key].children[child_key] = True - print("💝", child_key) - queue.put(parent_key) - else: - print("🚀", child_key) - if child_key is not None: - queue.put(child_key) - - -def handle_frame(frame_key): - print("Handle frame", frame_key) - frame = frames[frame_key] - try: - memos[frame.memo_key] = frame.resolve() - # This is a redundant step, we could just check the store whether the children have memoized values - if frame.parent_key is not None: - frames[frame.parent_key].children[frame_key] = True - queue.put(frame.parent_key) - except Call.Defer as defer: - for call in defer.calls: - call.schedule(frame_key) - - -class Worker: - def __init__(self, queue: boto3, kv): - self.sqs = sqs - self.kv = kv - -async def worker(): - """ - Workers take jobs from the queue, one at a time, and handles them. - They have read and write access to the store, and are responsible for - Managing the output of tasks and scheduling new ones - """ - print("WORK WORK") - while not queue.empty(): - # print() - print() - print("QUEUE", len(queue.queue)) - # for item in queue.queue: - # print(" ", item) - # print() - # print() - print("MEMOS", len(memos.pickles)) - # for key, val in memos.items(): - # print(" ", key, ":", val, ":", calls[key]) - # print() - print() - - - handle_frame(queue.get()) - await asyncio.sleep(.1) - -@task -def fib(n): - match n: - case 0: return 0 - case 1: return 1 - case _: return sum(fib.map([{"n": n - 2}, {"n": n - 1}])) - -@task -def output(n): - n = fib(n=n) - print("output", n) - -async def main(): - sqs = boto3.client("sqs") - kv = boto3.client("dynamo") - call = Call(output, {"n": 9}) - call.schedule(None) - - await asyncio.gather( - *(worker() for _ in range(int(sys.argv[1]) if len(sys.argv) > 1 else 3)) - ) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/brrr_demo.py b/brrr_demo.py new file mode 100755 index 0000000..d74c086 --- /dev/null +++ b/brrr_demo.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import os +import sys + +import redis +import boto3 + +from brrr.backends import redis as redis_, dynamo +from brrr import task, wrrrk, srrrv, setup + +def init_brrr(reset_backends): + # Check credentials + boto3.client('sts').get_caller_identity() + + redis_client = redis.Redis() + queue = redis_.RedisQueue(redis_client, os.environ.get("REDIS_QUEUE_KEY", "r1")) + + dynamo_client = boto3.client("dynamodb") + store = dynamo.DynamoDbMemStore(dynamo_client, os.environ.get("DYNAMODB_TABLE_NAME", "brrr")) + if reset_backends: + store.create_table() + + setup(queue, store) + +@task +def fib(n: int, salt=None): + match n: + case 0: return 0 + case 1: return 1 + case _: return sum(fib.map([[n - 2, salt], [n - 1, salt]])) + +@task +def fib_and_print(n: str, salt = None): + f = fib(int(n), salt) + print(f"fib({n}) = {f}", flush=True) + return f + +@task +def hello(greetee: str): + greeting = f"Hello, {greetee}!" + print(greeting, flush=True) + return greeting + +cmds = {} +def cmd(f): + cmds[f.__name__] = f + +@cmd +def worker(): + init_brrr(False) + wrrrk(1) + +@cmd +def server(): + init_brrr(True) + srrrv([hello, fib, fib_and_print]) + +def main(): + f = cmds.get(sys.argv[1]) if len(sys.argv) > 1 else None + if f: + f() + else: + print(f"Usage: brrr_demo.py <{" | ".join(cmds.keys())}>") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..8e2e2ed --- /dev/null +++ b/flake.lock @@ -0,0 +1,212 @@ +{ + "nodes": { + "devshell": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1728330715, + "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=", + "owner": "numtide", + "repo": "devshell", + "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1733312601, + "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1722073938, + "narHash": "sha256-OpX0StkL8vpXyWOGUD6G+MA26wAXK6SpT94kLJXo6B4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e36e9f57337d0ff0cf77aceb58af4c805472bfae", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1733096140, + "narHash": "sha256-1qRH7uAUsyQI7R1Uwl4T+XvdNv778H0Nb5njNrqvylY=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/5487e69da40cbd611ab2cadee0b4637225f7cfae.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/5487e69da40cbd611ab2cadee0b4637225f7cfae.tar.gz" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1733376361, + "narHash": "sha256-aLJxoTDDSqB+/3orsulE6/qdlX6MzDLIITLZqdgMpqo=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "929116e316068c7318c54eb4d827f7d9756d5e9c", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "process-compose-flake": { + "locked": { + "lastModified": 1733325752, + "narHash": "sha256-79tzPuXNRo1NUllafYW6SjeLtjqfnLGq7tHCM7cAXNg=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "1012530b582f1bd3b102295c799358d95abf42d7", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, + "pyproject-build-systems": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ], + "uv2nix": [ + "uv2nix" + ] + }, + "locked": { + "lastModified": 1733628997, + "narHash": "sha256-brzCRGaUUnye+i1Xua2TxGYog1LRwDn9fNS/s/pym4Q=", + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "rev": "d7f1157ecc005714e2e853bf06ef9daec79fa43d", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1733540292, + "narHash": "sha256-UAW/Rs8hx6ai+lAv2c0fP21NXi3SWuJO9p0DNiEYS90=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "e2393999b79ea26c1d399f8fcc34db4a01665e9b", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devshell": "devshell", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs_2", + "process-compose-flake": "process-compose-flake", + "pyproject-build-systems": "pyproject-build-systems", + "pyproject-nix": "pyproject-nix", + "services-flake": "services-flake", + "systems": "systems", + "uv2nix": "uv2nix" + } + }, + "services-flake": { + "locked": { + "lastModified": 1733618124, + "narHash": "sha256-7tQoSMlZLN4a9m3C5RPHQfSqtMwIzp0X6usqWhsqm6g=", + "owner": "juspay", + "repo": "services-flake", + "rev": "eadd9b43e060054cb01ac3af18a862273acc01b8", + "type": "github" + }, + "original": { + "owner": "juspay", + "repo": "services-flake", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "uv2nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ] + }, + "locked": { + "lastModified": 1733530735, + "narHash": "sha256-btOk4ryl5g+RScvJeyGaxC0TRAbmVGDccait5aZT8mo=", + "owner": "pyproject-nix", + "repo": "uv2nix", + "rev": "b38118307bbd489aa21193e472a84018a5d9fe32", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "uv2nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..0c7ebd9 --- /dev/null +++ b/flake.nix @@ -0,0 +1,236 @@ +# Copyright © 2024 Brrr Authors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + systems.url = "systems"; + flake-parts.url = "github:hercules-ci/flake-parts"; + devshell.url = "github:numtide/devshell"; + services-flake.url = "github:juspay/services-flake"; + process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; + # Heavily inspired by + # https://pyproject-nix.github.io/uv2nix/usage/hello-world.html + pyproject-nix = { + url = "github:pyproject-nix/pyproject.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + uv2nix = { + url = "github:pyproject-nix/uv2nix"; + inputs.pyproject-nix.follows = "pyproject-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + pyproject-build-systems = { + url = "github:pyproject-nix/build-system-pkgs"; + inputs.pyproject-nix.follows = "pyproject-nix"; + inputs.uv2nix.follows = "uv2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { self, flake-parts, ... }@inputs: flake-parts.lib.mkFlake { inherit inputs; } { + systems = import inputs.systems; + imports = [ + inputs.process-compose-flake.flakeModule + inputs.devshell.flakeModule + ]; + # A reusable process-compose module (for flake-parts) with either a full + # demo environment, or just the dependencies if you want to run a server + # manually. + flake = { + processComposeModules.default = { pkgs, ... }: { + imports = [ + ./localstack.nix + (inputs.services-flake.lib.multiService ./brrr-demo.nix) + ]; + services = let + demoEnv = { + AWS_ENDPOINT_URL = "http://localhost:4566"; + AWS_ACCESS_KEY_ID = "000000000000"; + AWS_SECRET_ACCESS_KEY = "localstack-foo"; + }; + in { + redis.r1.enable = true; + localstack.enable = true; + ollama.test1 = { + enable = true; + models = [ "llama3.2" ]; + }; + brrr-demo.worker = { + package = self.packages.${pkgs.system}.brrr-demo; + args = [ "worker" ]; + environment = demoEnv; + }; + brrr-demo.server = { + package = self.packages.${pkgs.system}.brrr-demo; + args = [ "server" ]; + environment = demoEnv; + }; + }; + }; + }; + perSystem = { config, self', inputs', pkgs, lib, system, ... }: let + uvWorkspace = inputs.uv2nix.lib.workspace.loadWorkspace { + workspaceRoot = ./.; + }; + uvOverlay = uvWorkspace.mkPyprojectOverlay { + sourcePreference = "wheel"; + }; + python = pkgs.python312; + pythonSet = (pkgs.callPackage inputs.pyproject-nix.build.packages { + inherit python; + }).overrideScope ( + lib.composeManyExtensions [ + inputs.pyproject-build-systems.overlays.default + uvOverlay + ] + ); + in { + config = { + process-compose.demo = { + imports = [ + inputs.services-flake.processComposeModules.default + self.processComposeModules.default + ]; + services.brrr-demo.server.enable = true; + services.brrr-demo.worker.enable = true; + }; + process-compose.deps = { + imports = [ + inputs.services-flake.processComposeModules.default + self.processComposeModules.default + ]; + services.brrr-demo.server.enable = false; + services.brrr-demo.worker.enable = false; + }; + packages = { + inherit python; + inherit (pkgs) uv; + # As far as I understand pyprojectnix and uv2nix, you want to use + # virtual envs even for prod-level final derivations because a + # virtualenv includes all the dependencies and a python which knows + # how to find them. + default = pythonSet.mkVirtualEnv "brrr-env" uvWorkspace.deps.default; + # Bare package without any env setup for other packages to include as + # a lib (again: I think?) + brrr = pythonSet.brrr; + # Stand-alone brrr_demo.py script + brrr-demo = pkgs.stdenvNoCC.mkDerivation { + name = "brrr-demo.py"; + dontUnpack = true; + installPhase = '' + mkdir -p $out/bin + cp ${./brrr_demo.py} $out/bin/brrr_demo.py + ''; + buildInputs = [ + # Dependencies for the demo are marked as ‘dev’ + (pythonSet.mkVirtualEnv "brrr-dev-env" uvWorkspace.deps.all) + ]; + # The patch phase will automatically use the python from the venv as + # the interpreter for the demo script. + meta.mainProgram = "brrr_demo.py"; + }; + docker = let + pkg = self'.packages.brrr-demo; + in pkgs.dockerTools.buildLayeredImage { + name = "brrr-demo"; + tag = "latest"; + config.Entrypoint = [ "${lib.getExe pkg}" ]; + }; + }; + devshells = { + impure = { + packages = with self'.packages; [ + python + uv + ]; + env = [ + { + name = "PYTHONPATH"; + unset = true; + } + { + name = "UV_PYTHON_DOWNLOADS"; + value = "never"; + } + ]; + }; + default = let + editableOverlay = uvWorkspace.mkEditablePyprojectOverlay { + # Set by devshell + root = "$PRJ_ROOT"; + }; + editablePythonSet = pythonSet.overrideScope editableOverlay; + virtualenv = editablePythonSet.mkVirtualEnv "brrr-dev-env" uvWorkspace.deps.all; + in { + env = [ + { + name = "PYTHONPATH"; + unset = true; + } + { + name = "UV_PYTHON_DOWNLOADS"; + value = "never"; + } + { + name = "UV_NO_SYNC"; + value = "1"; + } + ]; + packages = [ + pkgs.process-compose + self'.packages.uv + self'.packages.brrr-demo + virtualenv + ]; + commands = [ + # Always build aarch64-linux + { + name = "brrr-build-docker"; + category = "build"; + help = "Build and load a Docker image (requires a Nix Linux builder)"; + command = let + drv = self'.packages.docker; + in '' + ( + set -o pipefail + if nix build --no-link --print-out-paths .#packages.aarch64-linux.docker | xargs cat | docker load; then + echo 'Start a new worker with `docker run ${drv.imageName}:${drv.imageTag}`' + fi + ) + ''; + } + { + name = "brrr-demo-full"; + category = "demo"; + help = "Launch a full demo locally"; + command = '' + nix run .#demo + ''; + } + { + name = "brrr-demo-deps"; + category = "demo"; + help = "Start all dependent services without any brrr workers / server"; + command = '' + nix run .#deps + ''; + } + ]; + }; + }; + }; + }; + }; +} diff --git a/localstack.nix b/localstack.nix new file mode 100644 index 0000000..ab1bc08 --- /dev/null +++ b/localstack.nix @@ -0,0 +1,41 @@ +# Copyright © 2024 Brrr Authors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Localstack module for process-compose-flake + +{ config, pkgs, lib, ... }: { + options.services.localstack = with lib.types; { + enable = lib.mkEnableOption "Enable localstack service"; + package = lib.mkPackageOption pkgs "localstack" { }; + args = lib.mkOption { + default = []; + type = listOf str; + }; + }; + config = let + cfg = config.services.localstack; + in + lib.mkIf cfg.enable { + settings.processes.localstack = let + localstack = lib.getExe cfg.package; + in { + command = '' + ( + trap "${localstack} stop" EXIT + ${localstack} start ${lib.escapeShellArgs cfg.args} + ) + ''; + }; + }; +} diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index ca2e6fa..0000000 --- a/poetry.lock +++ /dev/null @@ -1,564 +0,0 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. - -[[package]] -name = "boto3" -version = "1.35.54" -description = "The AWS SDK for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "boto3-1.35.54-py3-none-any.whl", hash = "sha256:2d5e160b614db55fbee7981001c54476cb827c441cef65b2fcb2c52a62019909"}, - {file = "boto3-1.35.54.tar.gz", hash = "sha256:7d9c359bbbc858a60b51c86328db813353c8bd1940212cdbd0a7da835291c2e1"}, -] - -[package.dependencies] -botocore = ">=1.35.54,<1.36.0" -jmespath = ">=0.7.1,<2.0.0" -s3transfer = ">=0.10.0,<0.11.0" - -[package.extras] -crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] - -[[package]] -name = "boto3-stubs" -version = "1.35.54" -description = "Type annotations for boto3 1.35.54 generated with mypy-boto3-builder 8.1.4" -optional = false -python-versions = ">=3.8" -files = [ - {file = "boto3_stubs-1.35.54-py3-none-any.whl", hash = "sha256:17519739590614d50a4bfb82d10e02a9bf324ab87e90d54ecddf68044ba219b7"}, - {file = "boto3_stubs-1.35.54.tar.gz", hash = "sha256:ecf6cce737abc9bc212d48238021ff4aa3fb20ed887ecdee72a750bd329c902e"}, -] - -[package.dependencies] -botocore-stubs = "*" -types-s3transfer = "*" - -[package.extras] -accessanalyzer = ["mypy-boto3-accessanalyzer (>=1.35.0,<1.36.0)"] -account = ["mypy-boto3-account (>=1.35.0,<1.36.0)"] -acm = ["mypy-boto3-acm (>=1.35.0,<1.36.0)"] -acm-pca = ["mypy-boto3-acm-pca (>=1.35.0,<1.36.0)"] -all = ["mypy-boto3-accessanalyzer (>=1.35.0,<1.36.0)", "mypy-boto3-account (>=1.35.0,<1.36.0)", "mypy-boto3-acm (>=1.35.0,<1.36.0)", "mypy-boto3-acm-pca (>=1.35.0,<1.36.0)", "mypy-boto3-amp (>=1.35.0,<1.36.0)", "mypy-boto3-amplify (>=1.35.0,<1.36.0)", "mypy-boto3-amplifybackend (>=1.35.0,<1.36.0)", "mypy-boto3-amplifyuibuilder (>=1.35.0,<1.36.0)", "mypy-boto3-apigateway (>=1.35.0,<1.36.0)", "mypy-boto3-apigatewaymanagementapi (>=1.35.0,<1.36.0)", "mypy-boto3-apigatewayv2 (>=1.35.0,<1.36.0)", "mypy-boto3-appconfig (>=1.35.0,<1.36.0)", "mypy-boto3-appconfigdata (>=1.35.0,<1.36.0)", "mypy-boto3-appfabric (>=1.35.0,<1.36.0)", "mypy-boto3-appflow (>=1.35.0,<1.36.0)", "mypy-boto3-appintegrations (>=1.35.0,<1.36.0)", "mypy-boto3-application-autoscaling (>=1.35.0,<1.36.0)", "mypy-boto3-application-insights (>=1.35.0,<1.36.0)", "mypy-boto3-application-signals (>=1.35.0,<1.36.0)", "mypy-boto3-applicationcostprofiler (>=1.35.0,<1.36.0)", "mypy-boto3-appmesh (>=1.35.0,<1.36.0)", "mypy-boto3-apprunner (>=1.35.0,<1.36.0)", "mypy-boto3-appstream (>=1.35.0,<1.36.0)", "mypy-boto3-appsync (>=1.35.0,<1.36.0)", "mypy-boto3-apptest (>=1.35.0,<1.36.0)", "mypy-boto3-arc-zonal-shift (>=1.35.0,<1.36.0)", "mypy-boto3-artifact (>=1.35.0,<1.36.0)", "mypy-boto3-athena (>=1.35.0,<1.36.0)", "mypy-boto3-auditmanager (>=1.35.0,<1.36.0)", "mypy-boto3-autoscaling (>=1.35.0,<1.36.0)", "mypy-boto3-autoscaling-plans (>=1.35.0,<1.36.0)", "mypy-boto3-b2bi (>=1.35.0,<1.36.0)", "mypy-boto3-backup (>=1.35.0,<1.36.0)", "mypy-boto3-backup-gateway (>=1.35.0,<1.36.0)", "mypy-boto3-batch (>=1.35.0,<1.36.0)", "mypy-boto3-bcm-data-exports (>=1.35.0,<1.36.0)", "mypy-boto3-bedrock (>=1.35.0,<1.36.0)", "mypy-boto3-bedrock-agent (>=1.35.0,<1.36.0)", "mypy-boto3-bedrock-agent-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-bedrock-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-billingconductor (>=1.35.0,<1.36.0)", "mypy-boto3-braket (>=1.35.0,<1.36.0)", "mypy-boto3-budgets (>=1.35.0,<1.36.0)", "mypy-boto3-ce (>=1.35.0,<1.36.0)", "mypy-boto3-chatbot (>=1.35.0,<1.36.0)", "mypy-boto3-chime (>=1.35.0,<1.36.0)", "mypy-boto3-chime-sdk-identity (>=1.35.0,<1.36.0)", "mypy-boto3-chime-sdk-media-pipelines (>=1.35.0,<1.36.0)", "mypy-boto3-chime-sdk-meetings (>=1.35.0,<1.36.0)", "mypy-boto3-chime-sdk-messaging (>=1.35.0,<1.36.0)", "mypy-boto3-chime-sdk-voice (>=1.35.0,<1.36.0)", "mypy-boto3-cleanrooms (>=1.35.0,<1.36.0)", "mypy-boto3-cleanroomsml (>=1.35.0,<1.36.0)", "mypy-boto3-cloud9 (>=1.35.0,<1.36.0)", "mypy-boto3-cloudcontrol (>=1.35.0,<1.36.0)", "mypy-boto3-clouddirectory (>=1.35.0,<1.36.0)", "mypy-boto3-cloudformation (>=1.35.0,<1.36.0)", "mypy-boto3-cloudfront (>=1.35.0,<1.36.0)", "mypy-boto3-cloudfront-keyvaluestore (>=1.35.0,<1.36.0)", "mypy-boto3-cloudhsm (>=1.35.0,<1.36.0)", "mypy-boto3-cloudhsmv2 (>=1.35.0,<1.36.0)", "mypy-boto3-cloudsearch (>=1.35.0,<1.36.0)", "mypy-boto3-cloudsearchdomain (>=1.35.0,<1.36.0)", "mypy-boto3-cloudtrail (>=1.35.0,<1.36.0)", "mypy-boto3-cloudtrail-data (>=1.35.0,<1.36.0)", "mypy-boto3-cloudwatch (>=1.35.0,<1.36.0)", "mypy-boto3-codeartifact (>=1.35.0,<1.36.0)", "mypy-boto3-codebuild (>=1.35.0,<1.36.0)", "mypy-boto3-codecatalyst (>=1.35.0,<1.36.0)", "mypy-boto3-codecommit (>=1.35.0,<1.36.0)", "mypy-boto3-codeconnections (>=1.35.0,<1.36.0)", "mypy-boto3-codedeploy (>=1.35.0,<1.36.0)", "mypy-boto3-codeguru-reviewer (>=1.35.0,<1.36.0)", "mypy-boto3-codeguru-security (>=1.35.0,<1.36.0)", "mypy-boto3-codeguruprofiler (>=1.35.0,<1.36.0)", "mypy-boto3-codepipeline (>=1.35.0,<1.36.0)", "mypy-boto3-codestar-connections (>=1.35.0,<1.36.0)", "mypy-boto3-codestar-notifications (>=1.35.0,<1.36.0)", "mypy-boto3-cognito-identity (>=1.35.0,<1.36.0)", "mypy-boto3-cognito-idp (>=1.35.0,<1.36.0)", "mypy-boto3-cognito-sync (>=1.35.0,<1.36.0)", "mypy-boto3-comprehend (>=1.35.0,<1.36.0)", "mypy-boto3-comprehendmedical (>=1.35.0,<1.36.0)", "mypy-boto3-compute-optimizer (>=1.35.0,<1.36.0)", "mypy-boto3-config (>=1.35.0,<1.36.0)", "mypy-boto3-connect (>=1.35.0,<1.36.0)", "mypy-boto3-connect-contact-lens (>=1.35.0,<1.36.0)", "mypy-boto3-connectcampaigns (>=1.35.0,<1.36.0)", "mypy-boto3-connectcases (>=1.35.0,<1.36.0)", "mypy-boto3-connectparticipant (>=1.35.0,<1.36.0)", "mypy-boto3-controlcatalog (>=1.35.0,<1.36.0)", "mypy-boto3-controltower (>=1.35.0,<1.36.0)", "mypy-boto3-cost-optimization-hub (>=1.35.0,<1.36.0)", "mypy-boto3-cur (>=1.35.0,<1.36.0)", "mypy-boto3-customer-profiles (>=1.35.0,<1.36.0)", "mypy-boto3-databrew (>=1.35.0,<1.36.0)", "mypy-boto3-dataexchange (>=1.35.0,<1.36.0)", "mypy-boto3-datapipeline (>=1.35.0,<1.36.0)", "mypy-boto3-datasync (>=1.35.0,<1.36.0)", "mypy-boto3-datazone (>=1.35.0,<1.36.0)", "mypy-boto3-dax (>=1.35.0,<1.36.0)", "mypy-boto3-deadline (>=1.35.0,<1.36.0)", "mypy-boto3-detective (>=1.35.0,<1.36.0)", "mypy-boto3-devicefarm (>=1.35.0,<1.36.0)", "mypy-boto3-devops-guru (>=1.35.0,<1.36.0)", "mypy-boto3-directconnect (>=1.35.0,<1.36.0)", "mypy-boto3-discovery (>=1.35.0,<1.36.0)", "mypy-boto3-dlm (>=1.35.0,<1.36.0)", "mypy-boto3-dms (>=1.35.0,<1.36.0)", "mypy-boto3-docdb (>=1.35.0,<1.36.0)", "mypy-boto3-docdb-elastic (>=1.35.0,<1.36.0)", "mypy-boto3-drs (>=1.35.0,<1.36.0)", "mypy-boto3-ds (>=1.35.0,<1.36.0)", "mypy-boto3-ds-data (>=1.35.0,<1.36.0)", "mypy-boto3-dynamodb (>=1.35.0,<1.36.0)", "mypy-boto3-dynamodbstreams (>=1.35.0,<1.36.0)", "mypy-boto3-ebs (>=1.35.0,<1.36.0)", "mypy-boto3-ec2 (>=1.35.0,<1.36.0)", "mypy-boto3-ec2-instance-connect (>=1.35.0,<1.36.0)", "mypy-boto3-ecr (>=1.35.0,<1.36.0)", "mypy-boto3-ecr-public (>=1.35.0,<1.36.0)", "mypy-boto3-ecs (>=1.35.0,<1.36.0)", "mypy-boto3-efs (>=1.35.0,<1.36.0)", "mypy-boto3-eks (>=1.35.0,<1.36.0)", "mypy-boto3-eks-auth (>=1.35.0,<1.36.0)", "mypy-boto3-elastic-inference (>=1.35.0,<1.36.0)", "mypy-boto3-elasticache (>=1.35.0,<1.36.0)", "mypy-boto3-elasticbeanstalk (>=1.35.0,<1.36.0)", "mypy-boto3-elastictranscoder (>=1.35.0,<1.36.0)", "mypy-boto3-elb (>=1.35.0,<1.36.0)", "mypy-boto3-elbv2 (>=1.35.0,<1.36.0)", "mypy-boto3-emr (>=1.35.0,<1.36.0)", "mypy-boto3-emr-containers (>=1.35.0,<1.36.0)", "mypy-boto3-emr-serverless (>=1.35.0,<1.36.0)", "mypy-boto3-entityresolution (>=1.35.0,<1.36.0)", "mypy-boto3-es (>=1.35.0,<1.36.0)", "mypy-boto3-events (>=1.35.0,<1.36.0)", "mypy-boto3-evidently (>=1.35.0,<1.36.0)", "mypy-boto3-finspace (>=1.35.0,<1.36.0)", "mypy-boto3-finspace-data (>=1.35.0,<1.36.0)", "mypy-boto3-firehose (>=1.35.0,<1.36.0)", "mypy-boto3-fis (>=1.35.0,<1.36.0)", "mypy-boto3-fms (>=1.35.0,<1.36.0)", "mypy-boto3-forecast (>=1.35.0,<1.36.0)", "mypy-boto3-forecastquery (>=1.35.0,<1.36.0)", "mypy-boto3-frauddetector (>=1.35.0,<1.36.0)", "mypy-boto3-freetier (>=1.35.0,<1.36.0)", "mypy-boto3-fsx (>=1.35.0,<1.36.0)", "mypy-boto3-gamelift (>=1.35.0,<1.36.0)", "mypy-boto3-geo-maps (>=1.35.0,<1.36.0)", "mypy-boto3-geo-places (>=1.35.0,<1.36.0)", "mypy-boto3-geo-routes (>=1.35.0,<1.36.0)", "mypy-boto3-glacier (>=1.35.0,<1.36.0)", "mypy-boto3-globalaccelerator (>=1.35.0,<1.36.0)", "mypy-boto3-glue (>=1.35.0,<1.36.0)", "mypy-boto3-grafana (>=1.35.0,<1.36.0)", "mypy-boto3-greengrass (>=1.35.0,<1.36.0)", "mypy-boto3-greengrassv2 (>=1.35.0,<1.36.0)", "mypy-boto3-groundstation (>=1.35.0,<1.36.0)", "mypy-boto3-guardduty (>=1.35.0,<1.36.0)", "mypy-boto3-health (>=1.35.0,<1.36.0)", "mypy-boto3-healthlake (>=1.35.0,<1.36.0)", "mypy-boto3-iam (>=1.35.0,<1.36.0)", "mypy-boto3-identitystore (>=1.35.0,<1.36.0)", "mypy-boto3-imagebuilder (>=1.35.0,<1.36.0)", "mypy-boto3-importexport (>=1.35.0,<1.36.0)", "mypy-boto3-inspector (>=1.35.0,<1.36.0)", "mypy-boto3-inspector-scan (>=1.35.0,<1.36.0)", "mypy-boto3-inspector2 (>=1.35.0,<1.36.0)", "mypy-boto3-internetmonitor (>=1.35.0,<1.36.0)", "mypy-boto3-iot (>=1.35.0,<1.36.0)", "mypy-boto3-iot-data (>=1.35.0,<1.36.0)", "mypy-boto3-iot-jobs-data (>=1.35.0,<1.36.0)", "mypy-boto3-iot1click-devices (>=1.35.0,<1.36.0)", "mypy-boto3-iot1click-projects (>=1.35.0,<1.36.0)", "mypy-boto3-iotanalytics (>=1.35.0,<1.36.0)", "mypy-boto3-iotdeviceadvisor (>=1.35.0,<1.36.0)", "mypy-boto3-iotevents (>=1.35.0,<1.36.0)", "mypy-boto3-iotevents-data (>=1.35.0,<1.36.0)", "mypy-boto3-iotfleethub (>=1.35.0,<1.36.0)", "mypy-boto3-iotfleetwise (>=1.35.0,<1.36.0)", "mypy-boto3-iotsecuretunneling (>=1.35.0,<1.36.0)", "mypy-boto3-iotsitewise (>=1.35.0,<1.36.0)", "mypy-boto3-iotthingsgraph (>=1.35.0,<1.36.0)", "mypy-boto3-iottwinmaker (>=1.35.0,<1.36.0)", "mypy-boto3-iotwireless (>=1.35.0,<1.36.0)", "mypy-boto3-ivs (>=1.35.0,<1.36.0)", "mypy-boto3-ivs-realtime (>=1.35.0,<1.36.0)", "mypy-boto3-ivschat (>=1.35.0,<1.36.0)", "mypy-boto3-kafka (>=1.35.0,<1.36.0)", "mypy-boto3-kafkaconnect (>=1.35.0,<1.36.0)", "mypy-boto3-kendra (>=1.35.0,<1.36.0)", "mypy-boto3-kendra-ranking (>=1.35.0,<1.36.0)", "mypy-boto3-keyspaces (>=1.35.0,<1.36.0)", "mypy-boto3-kinesis (>=1.35.0,<1.36.0)", "mypy-boto3-kinesis-video-archived-media (>=1.35.0,<1.36.0)", "mypy-boto3-kinesis-video-media (>=1.35.0,<1.36.0)", "mypy-boto3-kinesis-video-signaling (>=1.35.0,<1.36.0)", "mypy-boto3-kinesis-video-webrtc-storage (>=1.35.0,<1.36.0)", "mypy-boto3-kinesisanalytics (>=1.35.0,<1.36.0)", "mypy-boto3-kinesisanalyticsv2 (>=1.35.0,<1.36.0)", "mypy-boto3-kinesisvideo (>=1.35.0,<1.36.0)", "mypy-boto3-kms (>=1.35.0,<1.36.0)", "mypy-boto3-lakeformation (>=1.35.0,<1.36.0)", "mypy-boto3-lambda (>=1.35.0,<1.36.0)", "mypy-boto3-launch-wizard (>=1.35.0,<1.36.0)", "mypy-boto3-lex-models (>=1.35.0,<1.36.0)", "mypy-boto3-lex-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-lexv2-models (>=1.35.0,<1.36.0)", "mypy-boto3-lexv2-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-license-manager (>=1.35.0,<1.36.0)", "mypy-boto3-license-manager-linux-subscriptions (>=1.35.0,<1.36.0)", "mypy-boto3-license-manager-user-subscriptions (>=1.35.0,<1.36.0)", "mypy-boto3-lightsail (>=1.35.0,<1.36.0)", "mypy-boto3-location (>=1.35.0,<1.36.0)", "mypy-boto3-logs (>=1.35.0,<1.36.0)", "mypy-boto3-lookoutequipment (>=1.35.0,<1.36.0)", "mypy-boto3-lookoutmetrics (>=1.35.0,<1.36.0)", "mypy-boto3-lookoutvision (>=1.35.0,<1.36.0)", "mypy-boto3-m2 (>=1.35.0,<1.36.0)", "mypy-boto3-machinelearning (>=1.35.0,<1.36.0)", "mypy-boto3-macie2 (>=1.35.0,<1.36.0)", "mypy-boto3-mailmanager (>=1.35.0,<1.36.0)", "mypy-boto3-managedblockchain (>=1.35.0,<1.36.0)", "mypy-boto3-managedblockchain-query (>=1.35.0,<1.36.0)", "mypy-boto3-marketplace-agreement (>=1.35.0,<1.36.0)", "mypy-boto3-marketplace-catalog (>=1.35.0,<1.36.0)", "mypy-boto3-marketplace-deployment (>=1.35.0,<1.36.0)", "mypy-boto3-marketplace-entitlement (>=1.35.0,<1.36.0)", "mypy-boto3-marketplace-reporting (>=1.35.0,<1.36.0)", "mypy-boto3-marketplacecommerceanalytics (>=1.35.0,<1.36.0)", "mypy-boto3-mediaconnect (>=1.35.0,<1.36.0)", "mypy-boto3-mediaconvert (>=1.35.0,<1.36.0)", "mypy-boto3-medialive (>=1.35.0,<1.36.0)", "mypy-boto3-mediapackage (>=1.35.0,<1.36.0)", "mypy-boto3-mediapackage-vod (>=1.35.0,<1.36.0)", "mypy-boto3-mediapackagev2 (>=1.35.0,<1.36.0)", "mypy-boto3-mediastore (>=1.35.0,<1.36.0)", "mypy-boto3-mediastore-data (>=1.35.0,<1.36.0)", "mypy-boto3-mediatailor (>=1.35.0,<1.36.0)", "mypy-boto3-medical-imaging (>=1.35.0,<1.36.0)", "mypy-boto3-memorydb (>=1.35.0,<1.36.0)", "mypy-boto3-meteringmarketplace (>=1.35.0,<1.36.0)", "mypy-boto3-mgh (>=1.35.0,<1.36.0)", "mypy-boto3-mgn (>=1.35.0,<1.36.0)", "mypy-boto3-migration-hub-refactor-spaces (>=1.35.0,<1.36.0)", "mypy-boto3-migrationhub-config (>=1.35.0,<1.36.0)", "mypy-boto3-migrationhuborchestrator (>=1.35.0,<1.36.0)", "mypy-boto3-migrationhubstrategy (>=1.35.0,<1.36.0)", "mypy-boto3-mq (>=1.35.0,<1.36.0)", "mypy-boto3-mturk (>=1.35.0,<1.36.0)", "mypy-boto3-mwaa (>=1.35.0,<1.36.0)", "mypy-boto3-neptune (>=1.35.0,<1.36.0)", "mypy-boto3-neptune-graph (>=1.35.0,<1.36.0)", "mypy-boto3-neptunedata (>=1.35.0,<1.36.0)", "mypy-boto3-network-firewall (>=1.35.0,<1.36.0)", "mypy-boto3-networkmanager (>=1.35.0,<1.36.0)", "mypy-boto3-networkmonitor (>=1.35.0,<1.36.0)", "mypy-boto3-oam (>=1.35.0,<1.36.0)", "mypy-boto3-omics (>=1.35.0,<1.36.0)", "mypy-boto3-opensearch (>=1.35.0,<1.36.0)", "mypy-boto3-opensearchserverless (>=1.35.0,<1.36.0)", "mypy-boto3-opsworks (>=1.35.0,<1.36.0)", "mypy-boto3-opsworkscm (>=1.35.0,<1.36.0)", "mypy-boto3-organizations (>=1.35.0,<1.36.0)", "mypy-boto3-osis (>=1.35.0,<1.36.0)", "mypy-boto3-outposts (>=1.35.0,<1.36.0)", "mypy-boto3-panorama (>=1.35.0,<1.36.0)", "mypy-boto3-payment-cryptography (>=1.35.0,<1.36.0)", "mypy-boto3-payment-cryptography-data (>=1.35.0,<1.36.0)", "mypy-boto3-pca-connector-ad (>=1.35.0,<1.36.0)", "mypy-boto3-pca-connector-scep (>=1.35.0,<1.36.0)", "mypy-boto3-pcs (>=1.35.0,<1.36.0)", "mypy-boto3-personalize (>=1.35.0,<1.36.0)", "mypy-boto3-personalize-events (>=1.35.0,<1.36.0)", "mypy-boto3-personalize-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-pi (>=1.35.0,<1.36.0)", "mypy-boto3-pinpoint (>=1.35.0,<1.36.0)", "mypy-boto3-pinpoint-email (>=1.35.0,<1.36.0)", "mypy-boto3-pinpoint-sms-voice (>=1.35.0,<1.36.0)", "mypy-boto3-pinpoint-sms-voice-v2 (>=1.35.0,<1.36.0)", "mypy-boto3-pipes (>=1.35.0,<1.36.0)", "mypy-boto3-polly (>=1.35.0,<1.36.0)", "mypy-boto3-pricing (>=1.35.0,<1.36.0)", "mypy-boto3-privatenetworks (>=1.35.0,<1.36.0)", "mypy-boto3-proton (>=1.35.0,<1.36.0)", "mypy-boto3-qapps (>=1.35.0,<1.36.0)", "mypy-boto3-qbusiness (>=1.35.0,<1.36.0)", "mypy-boto3-qconnect (>=1.35.0,<1.36.0)", "mypy-boto3-qldb (>=1.35.0,<1.36.0)", "mypy-boto3-qldb-session (>=1.35.0,<1.36.0)", "mypy-boto3-quicksight (>=1.35.0,<1.36.0)", "mypy-boto3-ram (>=1.35.0,<1.36.0)", "mypy-boto3-rbin (>=1.35.0,<1.36.0)", "mypy-boto3-rds (>=1.35.0,<1.36.0)", "mypy-boto3-rds-data (>=1.35.0,<1.36.0)", "mypy-boto3-redshift (>=1.35.0,<1.36.0)", "mypy-boto3-redshift-data (>=1.35.0,<1.36.0)", "mypy-boto3-redshift-serverless (>=1.35.0,<1.36.0)", "mypy-boto3-rekognition (>=1.35.0,<1.36.0)", "mypy-boto3-repostspace (>=1.35.0,<1.36.0)", "mypy-boto3-resiliencehub (>=1.35.0,<1.36.0)", "mypy-boto3-resource-explorer-2 (>=1.35.0,<1.36.0)", "mypy-boto3-resource-groups (>=1.35.0,<1.36.0)", "mypy-boto3-resourcegroupstaggingapi (>=1.35.0,<1.36.0)", "mypy-boto3-robomaker (>=1.35.0,<1.36.0)", "mypy-boto3-rolesanywhere (>=1.35.0,<1.36.0)", "mypy-boto3-route53 (>=1.35.0,<1.36.0)", "mypy-boto3-route53-recovery-cluster (>=1.35.0,<1.36.0)", "mypy-boto3-route53-recovery-control-config (>=1.35.0,<1.36.0)", "mypy-boto3-route53-recovery-readiness (>=1.35.0,<1.36.0)", "mypy-boto3-route53domains (>=1.35.0,<1.36.0)", "mypy-boto3-route53profiles (>=1.35.0,<1.36.0)", "mypy-boto3-route53resolver (>=1.35.0,<1.36.0)", "mypy-boto3-rum (>=1.35.0,<1.36.0)", "mypy-boto3-s3 (>=1.35.0,<1.36.0)", "mypy-boto3-s3control (>=1.35.0,<1.36.0)", "mypy-boto3-s3outposts (>=1.35.0,<1.36.0)", "mypy-boto3-sagemaker (>=1.35.0,<1.36.0)", "mypy-boto3-sagemaker-a2i-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-sagemaker-edge (>=1.35.0,<1.36.0)", "mypy-boto3-sagemaker-featurestore-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-sagemaker-geospatial (>=1.35.0,<1.36.0)", "mypy-boto3-sagemaker-metrics (>=1.35.0,<1.36.0)", "mypy-boto3-sagemaker-runtime (>=1.35.0,<1.36.0)", "mypy-boto3-savingsplans (>=1.35.0,<1.36.0)", "mypy-boto3-scheduler (>=1.35.0,<1.36.0)", "mypy-boto3-schemas (>=1.35.0,<1.36.0)", "mypy-boto3-sdb (>=1.35.0,<1.36.0)", "mypy-boto3-secretsmanager (>=1.35.0,<1.36.0)", "mypy-boto3-securityhub (>=1.35.0,<1.36.0)", "mypy-boto3-securitylake (>=1.35.0,<1.36.0)", "mypy-boto3-serverlessrepo (>=1.35.0,<1.36.0)", "mypy-boto3-service-quotas (>=1.35.0,<1.36.0)", "mypy-boto3-servicecatalog (>=1.35.0,<1.36.0)", "mypy-boto3-servicecatalog-appregistry (>=1.35.0,<1.36.0)", "mypy-boto3-servicediscovery (>=1.35.0,<1.36.0)", "mypy-boto3-ses (>=1.35.0,<1.36.0)", "mypy-boto3-sesv2 (>=1.35.0,<1.36.0)", "mypy-boto3-shield (>=1.35.0,<1.36.0)", "mypy-boto3-signer (>=1.35.0,<1.36.0)", "mypy-boto3-simspaceweaver (>=1.35.0,<1.36.0)", "mypy-boto3-sms (>=1.35.0,<1.36.0)", "mypy-boto3-sms-voice (>=1.35.0,<1.36.0)", "mypy-boto3-snow-device-management (>=1.35.0,<1.36.0)", "mypy-boto3-snowball (>=1.35.0,<1.36.0)", "mypy-boto3-sns (>=1.35.0,<1.36.0)", "mypy-boto3-socialmessaging (>=1.35.0,<1.36.0)", "mypy-boto3-sqs (>=1.35.0,<1.36.0)", "mypy-boto3-ssm (>=1.35.0,<1.36.0)", "mypy-boto3-ssm-contacts (>=1.35.0,<1.36.0)", "mypy-boto3-ssm-incidents (>=1.35.0,<1.36.0)", "mypy-boto3-ssm-quicksetup (>=1.35.0,<1.36.0)", "mypy-boto3-ssm-sap (>=1.35.0,<1.36.0)", "mypy-boto3-sso (>=1.35.0,<1.36.0)", "mypy-boto3-sso-admin (>=1.35.0,<1.36.0)", "mypy-boto3-sso-oidc (>=1.35.0,<1.36.0)", "mypy-boto3-stepfunctions (>=1.35.0,<1.36.0)", "mypy-boto3-storagegateway (>=1.35.0,<1.36.0)", "mypy-boto3-sts (>=1.35.0,<1.36.0)", "mypy-boto3-supplychain (>=1.35.0,<1.36.0)", "mypy-boto3-support (>=1.35.0,<1.36.0)", "mypy-boto3-support-app (>=1.35.0,<1.36.0)", "mypy-boto3-swf (>=1.35.0,<1.36.0)", "mypy-boto3-synthetics (>=1.35.0,<1.36.0)", "mypy-boto3-taxsettings (>=1.35.0,<1.36.0)", "mypy-boto3-textract (>=1.35.0,<1.36.0)", "mypy-boto3-timestream-influxdb (>=1.35.0,<1.36.0)", "mypy-boto3-timestream-query (>=1.35.0,<1.36.0)", "mypy-boto3-timestream-write (>=1.35.0,<1.36.0)", "mypy-boto3-tnb (>=1.35.0,<1.36.0)", "mypy-boto3-transcribe (>=1.35.0,<1.36.0)", "mypy-boto3-transfer (>=1.35.0,<1.36.0)", "mypy-boto3-translate (>=1.35.0,<1.36.0)", "mypy-boto3-trustedadvisor (>=1.35.0,<1.36.0)", "mypy-boto3-verifiedpermissions (>=1.35.0,<1.36.0)", "mypy-boto3-voice-id (>=1.35.0,<1.36.0)", "mypy-boto3-vpc-lattice (>=1.35.0,<1.36.0)", "mypy-boto3-waf (>=1.35.0,<1.36.0)", "mypy-boto3-waf-regional (>=1.35.0,<1.36.0)", "mypy-boto3-wafv2 (>=1.35.0,<1.36.0)", "mypy-boto3-wellarchitected (>=1.35.0,<1.36.0)", "mypy-boto3-wisdom (>=1.35.0,<1.36.0)", "mypy-boto3-workdocs (>=1.35.0,<1.36.0)", "mypy-boto3-workmail (>=1.35.0,<1.36.0)", "mypy-boto3-workmailmessageflow (>=1.35.0,<1.36.0)", "mypy-boto3-workspaces (>=1.35.0,<1.36.0)", "mypy-boto3-workspaces-thin-client (>=1.35.0,<1.36.0)", "mypy-boto3-workspaces-web (>=1.35.0,<1.36.0)", "mypy-boto3-xray (>=1.35.0,<1.36.0)"] -amp = ["mypy-boto3-amp (>=1.35.0,<1.36.0)"] -amplify = ["mypy-boto3-amplify (>=1.35.0,<1.36.0)"] -amplifybackend = ["mypy-boto3-amplifybackend (>=1.35.0,<1.36.0)"] -amplifyuibuilder = ["mypy-boto3-amplifyuibuilder (>=1.35.0,<1.36.0)"] -apigateway = ["mypy-boto3-apigateway (>=1.35.0,<1.36.0)"] -apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (>=1.35.0,<1.36.0)"] -apigatewayv2 = ["mypy-boto3-apigatewayv2 (>=1.35.0,<1.36.0)"] -appconfig = ["mypy-boto3-appconfig (>=1.35.0,<1.36.0)"] -appconfigdata = ["mypy-boto3-appconfigdata (>=1.35.0,<1.36.0)"] -appfabric = ["mypy-boto3-appfabric (>=1.35.0,<1.36.0)"] -appflow = ["mypy-boto3-appflow (>=1.35.0,<1.36.0)"] -appintegrations = ["mypy-boto3-appintegrations (>=1.35.0,<1.36.0)"] -application-autoscaling = ["mypy-boto3-application-autoscaling (>=1.35.0,<1.36.0)"] -application-insights = ["mypy-boto3-application-insights (>=1.35.0,<1.36.0)"] -application-signals = ["mypy-boto3-application-signals (>=1.35.0,<1.36.0)"] -applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (>=1.35.0,<1.36.0)"] -appmesh = ["mypy-boto3-appmesh (>=1.35.0,<1.36.0)"] -apprunner = ["mypy-boto3-apprunner (>=1.35.0,<1.36.0)"] -appstream = ["mypy-boto3-appstream (>=1.35.0,<1.36.0)"] -appsync = ["mypy-boto3-appsync (>=1.35.0,<1.36.0)"] -apptest = ["mypy-boto3-apptest (>=1.35.0,<1.36.0)"] -arc-zonal-shift = ["mypy-boto3-arc-zonal-shift (>=1.35.0,<1.36.0)"] -artifact = ["mypy-boto3-artifact (>=1.35.0,<1.36.0)"] -athena = ["mypy-boto3-athena (>=1.35.0,<1.36.0)"] -auditmanager = ["mypy-boto3-auditmanager (>=1.35.0,<1.36.0)"] -autoscaling = ["mypy-boto3-autoscaling (>=1.35.0,<1.36.0)"] -autoscaling-plans = ["mypy-boto3-autoscaling-plans (>=1.35.0,<1.36.0)"] -b2bi = ["mypy-boto3-b2bi (>=1.35.0,<1.36.0)"] -backup = ["mypy-boto3-backup (>=1.35.0,<1.36.0)"] -backup-gateway = ["mypy-boto3-backup-gateway (>=1.35.0,<1.36.0)"] -batch = ["mypy-boto3-batch (>=1.35.0,<1.36.0)"] -bcm-data-exports = ["mypy-boto3-bcm-data-exports (>=1.35.0,<1.36.0)"] -bedrock = ["mypy-boto3-bedrock (>=1.35.0,<1.36.0)"] -bedrock-agent = ["mypy-boto3-bedrock-agent (>=1.35.0,<1.36.0)"] -bedrock-agent-runtime = ["mypy-boto3-bedrock-agent-runtime (>=1.35.0,<1.36.0)"] -bedrock-runtime = ["mypy-boto3-bedrock-runtime (>=1.35.0,<1.36.0)"] -billingconductor = ["mypy-boto3-billingconductor (>=1.35.0,<1.36.0)"] -boto3 = ["boto3 (==1.35.54)", "botocore (==1.35.54)"] -braket = ["mypy-boto3-braket (>=1.35.0,<1.36.0)"] -budgets = ["mypy-boto3-budgets (>=1.35.0,<1.36.0)"] -ce = ["mypy-boto3-ce (>=1.35.0,<1.36.0)"] -chatbot = ["mypy-boto3-chatbot (>=1.35.0,<1.36.0)"] -chime = ["mypy-boto3-chime (>=1.35.0,<1.36.0)"] -chime-sdk-identity = ["mypy-boto3-chime-sdk-identity (>=1.35.0,<1.36.0)"] -chime-sdk-media-pipelines = ["mypy-boto3-chime-sdk-media-pipelines (>=1.35.0,<1.36.0)"] -chime-sdk-meetings = ["mypy-boto3-chime-sdk-meetings (>=1.35.0,<1.36.0)"] -chime-sdk-messaging = ["mypy-boto3-chime-sdk-messaging (>=1.35.0,<1.36.0)"] -chime-sdk-voice = ["mypy-boto3-chime-sdk-voice (>=1.35.0,<1.36.0)"] -cleanrooms = ["mypy-boto3-cleanrooms (>=1.35.0,<1.36.0)"] -cleanroomsml = ["mypy-boto3-cleanroomsml (>=1.35.0,<1.36.0)"] -cloud9 = ["mypy-boto3-cloud9 (>=1.35.0,<1.36.0)"] -cloudcontrol = ["mypy-boto3-cloudcontrol (>=1.35.0,<1.36.0)"] -clouddirectory = ["mypy-boto3-clouddirectory (>=1.35.0,<1.36.0)"] -cloudformation = ["mypy-boto3-cloudformation (>=1.35.0,<1.36.0)"] -cloudfront = ["mypy-boto3-cloudfront (>=1.35.0,<1.36.0)"] -cloudfront-keyvaluestore = ["mypy-boto3-cloudfront-keyvaluestore (>=1.35.0,<1.36.0)"] -cloudhsm = ["mypy-boto3-cloudhsm (>=1.35.0,<1.36.0)"] -cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (>=1.35.0,<1.36.0)"] -cloudsearch = ["mypy-boto3-cloudsearch (>=1.35.0,<1.36.0)"] -cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (>=1.35.0,<1.36.0)"] -cloudtrail = ["mypy-boto3-cloudtrail (>=1.35.0,<1.36.0)"] -cloudtrail-data = ["mypy-boto3-cloudtrail-data (>=1.35.0,<1.36.0)"] -cloudwatch = ["mypy-boto3-cloudwatch (>=1.35.0,<1.36.0)"] -codeartifact = ["mypy-boto3-codeartifact (>=1.35.0,<1.36.0)"] -codebuild = ["mypy-boto3-codebuild (>=1.35.0,<1.36.0)"] -codecatalyst = ["mypy-boto3-codecatalyst (>=1.35.0,<1.36.0)"] -codecommit = ["mypy-boto3-codecommit (>=1.35.0,<1.36.0)"] -codeconnections = ["mypy-boto3-codeconnections (>=1.35.0,<1.36.0)"] -codedeploy = ["mypy-boto3-codedeploy (>=1.35.0,<1.36.0)"] -codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (>=1.35.0,<1.36.0)"] -codeguru-security = ["mypy-boto3-codeguru-security (>=1.35.0,<1.36.0)"] -codeguruprofiler = ["mypy-boto3-codeguruprofiler (>=1.35.0,<1.36.0)"] -codepipeline = ["mypy-boto3-codepipeline (>=1.35.0,<1.36.0)"] -codestar-connections = ["mypy-boto3-codestar-connections (>=1.35.0,<1.36.0)"] -codestar-notifications = ["mypy-boto3-codestar-notifications (>=1.35.0,<1.36.0)"] -cognito-identity = ["mypy-boto3-cognito-identity (>=1.35.0,<1.36.0)"] -cognito-idp = ["mypy-boto3-cognito-idp (>=1.35.0,<1.36.0)"] -cognito-sync = ["mypy-boto3-cognito-sync (>=1.35.0,<1.36.0)"] -comprehend = ["mypy-boto3-comprehend (>=1.35.0,<1.36.0)"] -comprehendmedical = ["mypy-boto3-comprehendmedical (>=1.35.0,<1.36.0)"] -compute-optimizer = ["mypy-boto3-compute-optimizer (>=1.35.0,<1.36.0)"] -config = ["mypy-boto3-config (>=1.35.0,<1.36.0)"] -connect = ["mypy-boto3-connect (>=1.35.0,<1.36.0)"] -connect-contact-lens = ["mypy-boto3-connect-contact-lens (>=1.35.0,<1.36.0)"] -connectcampaigns = ["mypy-boto3-connectcampaigns (>=1.35.0,<1.36.0)"] -connectcases = ["mypy-boto3-connectcases (>=1.35.0,<1.36.0)"] -connectparticipant = ["mypy-boto3-connectparticipant (>=1.35.0,<1.36.0)"] -controlcatalog = ["mypy-boto3-controlcatalog (>=1.35.0,<1.36.0)"] -controltower = ["mypy-boto3-controltower (>=1.35.0,<1.36.0)"] -cost-optimization-hub = ["mypy-boto3-cost-optimization-hub (>=1.35.0,<1.36.0)"] -cur = ["mypy-boto3-cur (>=1.35.0,<1.36.0)"] -customer-profiles = ["mypy-boto3-customer-profiles (>=1.35.0,<1.36.0)"] -databrew = ["mypy-boto3-databrew (>=1.35.0,<1.36.0)"] -dataexchange = ["mypy-boto3-dataexchange (>=1.35.0,<1.36.0)"] -datapipeline = ["mypy-boto3-datapipeline (>=1.35.0,<1.36.0)"] -datasync = ["mypy-boto3-datasync (>=1.35.0,<1.36.0)"] -datazone = ["mypy-boto3-datazone (>=1.35.0,<1.36.0)"] -dax = ["mypy-boto3-dax (>=1.35.0,<1.36.0)"] -deadline = ["mypy-boto3-deadline (>=1.35.0,<1.36.0)"] -detective = ["mypy-boto3-detective (>=1.35.0,<1.36.0)"] -devicefarm = ["mypy-boto3-devicefarm (>=1.35.0,<1.36.0)"] -devops-guru = ["mypy-boto3-devops-guru (>=1.35.0,<1.36.0)"] -directconnect = ["mypy-boto3-directconnect (>=1.35.0,<1.36.0)"] -discovery = ["mypy-boto3-discovery (>=1.35.0,<1.36.0)"] -dlm = ["mypy-boto3-dlm (>=1.35.0,<1.36.0)"] -dms = ["mypy-boto3-dms (>=1.35.0,<1.36.0)"] -docdb = ["mypy-boto3-docdb (>=1.35.0,<1.36.0)"] -docdb-elastic = ["mypy-boto3-docdb-elastic (>=1.35.0,<1.36.0)"] -drs = ["mypy-boto3-drs (>=1.35.0,<1.36.0)"] -ds = ["mypy-boto3-ds (>=1.35.0,<1.36.0)"] -ds-data = ["mypy-boto3-ds-data (>=1.35.0,<1.36.0)"] -dynamodb = ["mypy-boto3-dynamodb (>=1.35.0,<1.36.0)"] -dynamodbstreams = ["mypy-boto3-dynamodbstreams (>=1.35.0,<1.36.0)"] -ebs = ["mypy-boto3-ebs (>=1.35.0,<1.36.0)"] -ec2 = ["mypy-boto3-ec2 (>=1.35.0,<1.36.0)"] -ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (>=1.35.0,<1.36.0)"] -ecr = ["mypy-boto3-ecr (>=1.35.0,<1.36.0)"] -ecr-public = ["mypy-boto3-ecr-public (>=1.35.0,<1.36.0)"] -ecs = ["mypy-boto3-ecs (>=1.35.0,<1.36.0)"] -efs = ["mypy-boto3-efs (>=1.35.0,<1.36.0)"] -eks = ["mypy-boto3-eks (>=1.35.0,<1.36.0)"] -eks-auth = ["mypy-boto3-eks-auth (>=1.35.0,<1.36.0)"] -elastic-inference = ["mypy-boto3-elastic-inference (>=1.35.0,<1.36.0)"] -elasticache = ["mypy-boto3-elasticache (>=1.35.0,<1.36.0)"] -elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (>=1.35.0,<1.36.0)"] -elastictranscoder = ["mypy-boto3-elastictranscoder (>=1.35.0,<1.36.0)"] -elb = ["mypy-boto3-elb (>=1.35.0,<1.36.0)"] -elbv2 = ["mypy-boto3-elbv2 (>=1.35.0,<1.36.0)"] -emr = ["mypy-boto3-emr (>=1.35.0,<1.36.0)"] -emr-containers = ["mypy-boto3-emr-containers (>=1.35.0,<1.36.0)"] -emr-serverless = ["mypy-boto3-emr-serverless (>=1.35.0,<1.36.0)"] -entityresolution = ["mypy-boto3-entityresolution (>=1.35.0,<1.36.0)"] -es = ["mypy-boto3-es (>=1.35.0,<1.36.0)"] -essential = ["mypy-boto3-cloudformation (>=1.35.0,<1.36.0)", "mypy-boto3-dynamodb (>=1.35.0,<1.36.0)", "mypy-boto3-ec2 (>=1.35.0,<1.36.0)", "mypy-boto3-lambda (>=1.35.0,<1.36.0)", "mypy-boto3-rds (>=1.35.0,<1.36.0)", "mypy-boto3-s3 (>=1.35.0,<1.36.0)", "mypy-boto3-sqs (>=1.35.0,<1.36.0)"] -events = ["mypy-boto3-events (>=1.35.0,<1.36.0)"] -evidently = ["mypy-boto3-evidently (>=1.35.0,<1.36.0)"] -finspace = ["mypy-boto3-finspace (>=1.35.0,<1.36.0)"] -finspace-data = ["mypy-boto3-finspace-data (>=1.35.0,<1.36.0)"] -firehose = ["mypy-boto3-firehose (>=1.35.0,<1.36.0)"] -fis = ["mypy-boto3-fis (>=1.35.0,<1.36.0)"] -fms = ["mypy-boto3-fms (>=1.35.0,<1.36.0)"] -forecast = ["mypy-boto3-forecast (>=1.35.0,<1.36.0)"] -forecastquery = ["mypy-boto3-forecastquery (>=1.35.0,<1.36.0)"] -frauddetector = ["mypy-boto3-frauddetector (>=1.35.0,<1.36.0)"] -freetier = ["mypy-boto3-freetier (>=1.35.0,<1.36.0)"] -fsx = ["mypy-boto3-fsx (>=1.35.0,<1.36.0)"] -full = ["boto3-stubs-full"] -gamelift = ["mypy-boto3-gamelift (>=1.35.0,<1.36.0)"] -geo-maps = ["mypy-boto3-geo-maps (>=1.35.0,<1.36.0)"] -geo-places = ["mypy-boto3-geo-places (>=1.35.0,<1.36.0)"] -geo-routes = ["mypy-boto3-geo-routes (>=1.35.0,<1.36.0)"] -glacier = ["mypy-boto3-glacier (>=1.35.0,<1.36.0)"] -globalaccelerator = ["mypy-boto3-globalaccelerator (>=1.35.0,<1.36.0)"] -glue = ["mypy-boto3-glue (>=1.35.0,<1.36.0)"] -grafana = ["mypy-boto3-grafana (>=1.35.0,<1.36.0)"] -greengrass = ["mypy-boto3-greengrass (>=1.35.0,<1.36.0)"] -greengrassv2 = ["mypy-boto3-greengrassv2 (>=1.35.0,<1.36.0)"] -groundstation = ["mypy-boto3-groundstation (>=1.35.0,<1.36.0)"] -guardduty = ["mypy-boto3-guardduty (>=1.35.0,<1.36.0)"] -health = ["mypy-boto3-health (>=1.35.0,<1.36.0)"] -healthlake = ["mypy-boto3-healthlake (>=1.35.0,<1.36.0)"] -iam = ["mypy-boto3-iam (>=1.35.0,<1.36.0)"] -identitystore = ["mypy-boto3-identitystore (>=1.35.0,<1.36.0)"] -imagebuilder = ["mypy-boto3-imagebuilder (>=1.35.0,<1.36.0)"] -importexport = ["mypy-boto3-importexport (>=1.35.0,<1.36.0)"] -inspector = ["mypy-boto3-inspector (>=1.35.0,<1.36.0)"] -inspector-scan = ["mypy-boto3-inspector-scan (>=1.35.0,<1.36.0)"] -inspector2 = ["mypy-boto3-inspector2 (>=1.35.0,<1.36.0)"] -internetmonitor = ["mypy-boto3-internetmonitor (>=1.35.0,<1.36.0)"] -iot = ["mypy-boto3-iot (>=1.35.0,<1.36.0)"] -iot-data = ["mypy-boto3-iot-data (>=1.35.0,<1.36.0)"] -iot-jobs-data = ["mypy-boto3-iot-jobs-data (>=1.35.0,<1.36.0)"] -iot1click-devices = ["mypy-boto3-iot1click-devices (>=1.35.0,<1.36.0)"] -iot1click-projects = ["mypy-boto3-iot1click-projects (>=1.35.0,<1.36.0)"] -iotanalytics = ["mypy-boto3-iotanalytics (>=1.35.0,<1.36.0)"] -iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (>=1.35.0,<1.36.0)"] -iotevents = ["mypy-boto3-iotevents (>=1.35.0,<1.36.0)"] -iotevents-data = ["mypy-boto3-iotevents-data (>=1.35.0,<1.36.0)"] -iotfleethub = ["mypy-boto3-iotfleethub (>=1.35.0,<1.36.0)"] -iotfleetwise = ["mypy-boto3-iotfleetwise (>=1.35.0,<1.36.0)"] -iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (>=1.35.0,<1.36.0)"] -iotsitewise = ["mypy-boto3-iotsitewise (>=1.35.0,<1.36.0)"] -iotthingsgraph = ["mypy-boto3-iotthingsgraph (>=1.35.0,<1.36.0)"] -iottwinmaker = ["mypy-boto3-iottwinmaker (>=1.35.0,<1.36.0)"] -iotwireless = ["mypy-boto3-iotwireless (>=1.35.0,<1.36.0)"] -ivs = ["mypy-boto3-ivs (>=1.35.0,<1.36.0)"] -ivs-realtime = ["mypy-boto3-ivs-realtime (>=1.35.0,<1.36.0)"] -ivschat = ["mypy-boto3-ivschat (>=1.35.0,<1.36.0)"] -kafka = ["mypy-boto3-kafka (>=1.35.0,<1.36.0)"] -kafkaconnect = ["mypy-boto3-kafkaconnect (>=1.35.0,<1.36.0)"] -kendra = ["mypy-boto3-kendra (>=1.35.0,<1.36.0)"] -kendra-ranking = ["mypy-boto3-kendra-ranking (>=1.35.0,<1.36.0)"] -keyspaces = ["mypy-boto3-keyspaces (>=1.35.0,<1.36.0)"] -kinesis = ["mypy-boto3-kinesis (>=1.35.0,<1.36.0)"] -kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (>=1.35.0,<1.36.0)"] -kinesis-video-media = ["mypy-boto3-kinesis-video-media (>=1.35.0,<1.36.0)"] -kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (>=1.35.0,<1.36.0)"] -kinesis-video-webrtc-storage = ["mypy-boto3-kinesis-video-webrtc-storage (>=1.35.0,<1.36.0)"] -kinesisanalytics = ["mypy-boto3-kinesisanalytics (>=1.35.0,<1.36.0)"] -kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (>=1.35.0,<1.36.0)"] -kinesisvideo = ["mypy-boto3-kinesisvideo (>=1.35.0,<1.36.0)"] -kms = ["mypy-boto3-kms (>=1.35.0,<1.36.0)"] -lakeformation = ["mypy-boto3-lakeformation (>=1.35.0,<1.36.0)"] -lambda = ["mypy-boto3-lambda (>=1.35.0,<1.36.0)"] -launch-wizard = ["mypy-boto3-launch-wizard (>=1.35.0,<1.36.0)"] -lex-models = ["mypy-boto3-lex-models (>=1.35.0,<1.36.0)"] -lex-runtime = ["mypy-boto3-lex-runtime (>=1.35.0,<1.36.0)"] -lexv2-models = ["mypy-boto3-lexv2-models (>=1.35.0,<1.36.0)"] -lexv2-runtime = ["mypy-boto3-lexv2-runtime (>=1.35.0,<1.36.0)"] -license-manager = ["mypy-boto3-license-manager (>=1.35.0,<1.36.0)"] -license-manager-linux-subscriptions = ["mypy-boto3-license-manager-linux-subscriptions (>=1.35.0,<1.36.0)"] -license-manager-user-subscriptions = ["mypy-boto3-license-manager-user-subscriptions (>=1.35.0,<1.36.0)"] -lightsail = ["mypy-boto3-lightsail (>=1.35.0,<1.36.0)"] -location = ["mypy-boto3-location (>=1.35.0,<1.36.0)"] -logs = ["mypy-boto3-logs (>=1.35.0,<1.36.0)"] -lookoutequipment = ["mypy-boto3-lookoutequipment (>=1.35.0,<1.36.0)"] -lookoutmetrics = ["mypy-boto3-lookoutmetrics (>=1.35.0,<1.36.0)"] -lookoutvision = ["mypy-boto3-lookoutvision (>=1.35.0,<1.36.0)"] -m2 = ["mypy-boto3-m2 (>=1.35.0,<1.36.0)"] -machinelearning = ["mypy-boto3-machinelearning (>=1.35.0,<1.36.0)"] -macie2 = ["mypy-boto3-macie2 (>=1.35.0,<1.36.0)"] -mailmanager = ["mypy-boto3-mailmanager (>=1.35.0,<1.36.0)"] -managedblockchain = ["mypy-boto3-managedblockchain (>=1.35.0,<1.36.0)"] -managedblockchain-query = ["mypy-boto3-managedblockchain-query (>=1.35.0,<1.36.0)"] -marketplace-agreement = ["mypy-boto3-marketplace-agreement (>=1.35.0,<1.36.0)"] -marketplace-catalog = ["mypy-boto3-marketplace-catalog (>=1.35.0,<1.36.0)"] -marketplace-deployment = ["mypy-boto3-marketplace-deployment (>=1.35.0,<1.36.0)"] -marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (>=1.35.0,<1.36.0)"] -marketplace-reporting = ["mypy-boto3-marketplace-reporting (>=1.35.0,<1.36.0)"] -marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (>=1.35.0,<1.36.0)"] -mediaconnect = ["mypy-boto3-mediaconnect (>=1.35.0,<1.36.0)"] -mediaconvert = ["mypy-boto3-mediaconvert (>=1.35.0,<1.36.0)"] -medialive = ["mypy-boto3-medialive (>=1.35.0,<1.36.0)"] -mediapackage = ["mypy-boto3-mediapackage (>=1.35.0,<1.36.0)"] -mediapackage-vod = ["mypy-boto3-mediapackage-vod (>=1.35.0,<1.36.0)"] -mediapackagev2 = ["mypy-boto3-mediapackagev2 (>=1.35.0,<1.36.0)"] -mediastore = ["mypy-boto3-mediastore (>=1.35.0,<1.36.0)"] -mediastore-data = ["mypy-boto3-mediastore-data (>=1.35.0,<1.36.0)"] -mediatailor = ["mypy-boto3-mediatailor (>=1.35.0,<1.36.0)"] -medical-imaging = ["mypy-boto3-medical-imaging (>=1.35.0,<1.36.0)"] -memorydb = ["mypy-boto3-memorydb (>=1.35.0,<1.36.0)"] -meteringmarketplace = ["mypy-boto3-meteringmarketplace (>=1.35.0,<1.36.0)"] -mgh = ["mypy-boto3-mgh (>=1.35.0,<1.36.0)"] -mgn = ["mypy-boto3-mgn (>=1.35.0,<1.36.0)"] -migration-hub-refactor-spaces = ["mypy-boto3-migration-hub-refactor-spaces (>=1.35.0,<1.36.0)"] -migrationhub-config = ["mypy-boto3-migrationhub-config (>=1.35.0,<1.36.0)"] -migrationhuborchestrator = ["mypy-boto3-migrationhuborchestrator (>=1.35.0,<1.36.0)"] -migrationhubstrategy = ["mypy-boto3-migrationhubstrategy (>=1.35.0,<1.36.0)"] -mq = ["mypy-boto3-mq (>=1.35.0,<1.36.0)"] -mturk = ["mypy-boto3-mturk (>=1.35.0,<1.36.0)"] -mwaa = ["mypy-boto3-mwaa (>=1.35.0,<1.36.0)"] -neptune = ["mypy-boto3-neptune (>=1.35.0,<1.36.0)"] -neptune-graph = ["mypy-boto3-neptune-graph (>=1.35.0,<1.36.0)"] -neptunedata = ["mypy-boto3-neptunedata (>=1.35.0,<1.36.0)"] -network-firewall = ["mypy-boto3-network-firewall (>=1.35.0,<1.36.0)"] -networkmanager = ["mypy-boto3-networkmanager (>=1.35.0,<1.36.0)"] -networkmonitor = ["mypy-boto3-networkmonitor (>=1.35.0,<1.36.0)"] -oam = ["mypy-boto3-oam (>=1.35.0,<1.36.0)"] -omics = ["mypy-boto3-omics (>=1.35.0,<1.36.0)"] -opensearch = ["mypy-boto3-opensearch (>=1.35.0,<1.36.0)"] -opensearchserverless = ["mypy-boto3-opensearchserverless (>=1.35.0,<1.36.0)"] -opsworks = ["mypy-boto3-opsworks (>=1.35.0,<1.36.0)"] -opsworkscm = ["mypy-boto3-opsworkscm (>=1.35.0,<1.36.0)"] -organizations = ["mypy-boto3-organizations (>=1.35.0,<1.36.0)"] -osis = ["mypy-boto3-osis (>=1.35.0,<1.36.0)"] -outposts = ["mypy-boto3-outposts (>=1.35.0,<1.36.0)"] -panorama = ["mypy-boto3-panorama (>=1.35.0,<1.36.0)"] -payment-cryptography = ["mypy-boto3-payment-cryptography (>=1.35.0,<1.36.0)"] -payment-cryptography-data = ["mypy-boto3-payment-cryptography-data (>=1.35.0,<1.36.0)"] -pca-connector-ad = ["mypy-boto3-pca-connector-ad (>=1.35.0,<1.36.0)"] -pca-connector-scep = ["mypy-boto3-pca-connector-scep (>=1.35.0,<1.36.0)"] -pcs = ["mypy-boto3-pcs (>=1.35.0,<1.36.0)"] -personalize = ["mypy-boto3-personalize (>=1.35.0,<1.36.0)"] -personalize-events = ["mypy-boto3-personalize-events (>=1.35.0,<1.36.0)"] -personalize-runtime = ["mypy-boto3-personalize-runtime (>=1.35.0,<1.36.0)"] -pi = ["mypy-boto3-pi (>=1.35.0,<1.36.0)"] -pinpoint = ["mypy-boto3-pinpoint (>=1.35.0,<1.36.0)"] -pinpoint-email = ["mypy-boto3-pinpoint-email (>=1.35.0,<1.36.0)"] -pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (>=1.35.0,<1.36.0)"] -pinpoint-sms-voice-v2 = ["mypy-boto3-pinpoint-sms-voice-v2 (>=1.35.0,<1.36.0)"] -pipes = ["mypy-boto3-pipes (>=1.35.0,<1.36.0)"] -polly = ["mypy-boto3-polly (>=1.35.0,<1.36.0)"] -pricing = ["mypy-boto3-pricing (>=1.35.0,<1.36.0)"] -privatenetworks = ["mypy-boto3-privatenetworks (>=1.35.0,<1.36.0)"] -proton = ["mypy-boto3-proton (>=1.35.0,<1.36.0)"] -qapps = ["mypy-boto3-qapps (>=1.35.0,<1.36.0)"] -qbusiness = ["mypy-boto3-qbusiness (>=1.35.0,<1.36.0)"] -qconnect = ["mypy-boto3-qconnect (>=1.35.0,<1.36.0)"] -qldb = ["mypy-boto3-qldb (>=1.35.0,<1.36.0)"] -qldb-session = ["mypy-boto3-qldb-session (>=1.35.0,<1.36.0)"] -quicksight = ["mypy-boto3-quicksight (>=1.35.0,<1.36.0)"] -ram = ["mypy-boto3-ram (>=1.35.0,<1.36.0)"] -rbin = ["mypy-boto3-rbin (>=1.35.0,<1.36.0)"] -rds = ["mypy-boto3-rds (>=1.35.0,<1.36.0)"] -rds-data = ["mypy-boto3-rds-data (>=1.35.0,<1.36.0)"] -redshift = ["mypy-boto3-redshift (>=1.35.0,<1.36.0)"] -redshift-data = ["mypy-boto3-redshift-data (>=1.35.0,<1.36.0)"] -redshift-serverless = ["mypy-boto3-redshift-serverless (>=1.35.0,<1.36.0)"] -rekognition = ["mypy-boto3-rekognition (>=1.35.0,<1.36.0)"] -repostspace = ["mypy-boto3-repostspace (>=1.35.0,<1.36.0)"] -resiliencehub = ["mypy-boto3-resiliencehub (>=1.35.0,<1.36.0)"] -resource-explorer-2 = ["mypy-boto3-resource-explorer-2 (>=1.35.0,<1.36.0)"] -resource-groups = ["mypy-boto3-resource-groups (>=1.35.0,<1.36.0)"] -resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (>=1.35.0,<1.36.0)"] -robomaker = ["mypy-boto3-robomaker (>=1.35.0,<1.36.0)"] -rolesanywhere = ["mypy-boto3-rolesanywhere (>=1.35.0,<1.36.0)"] -route53 = ["mypy-boto3-route53 (>=1.35.0,<1.36.0)"] -route53-recovery-cluster = ["mypy-boto3-route53-recovery-cluster (>=1.35.0,<1.36.0)"] -route53-recovery-control-config = ["mypy-boto3-route53-recovery-control-config (>=1.35.0,<1.36.0)"] -route53-recovery-readiness = ["mypy-boto3-route53-recovery-readiness (>=1.35.0,<1.36.0)"] -route53domains = ["mypy-boto3-route53domains (>=1.35.0,<1.36.0)"] -route53profiles = ["mypy-boto3-route53profiles (>=1.35.0,<1.36.0)"] -route53resolver = ["mypy-boto3-route53resolver (>=1.35.0,<1.36.0)"] -rum = ["mypy-boto3-rum (>=1.35.0,<1.36.0)"] -s3 = ["mypy-boto3-s3 (>=1.35.0,<1.36.0)"] -s3control = ["mypy-boto3-s3control (>=1.35.0,<1.36.0)"] -s3outposts = ["mypy-boto3-s3outposts (>=1.35.0,<1.36.0)"] -sagemaker = ["mypy-boto3-sagemaker (>=1.35.0,<1.36.0)"] -sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (>=1.35.0,<1.36.0)"] -sagemaker-edge = ["mypy-boto3-sagemaker-edge (>=1.35.0,<1.36.0)"] -sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (>=1.35.0,<1.36.0)"] -sagemaker-geospatial = ["mypy-boto3-sagemaker-geospatial (>=1.35.0,<1.36.0)"] -sagemaker-metrics = ["mypy-boto3-sagemaker-metrics (>=1.35.0,<1.36.0)"] -sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (>=1.35.0,<1.36.0)"] -savingsplans = ["mypy-boto3-savingsplans (>=1.35.0,<1.36.0)"] -scheduler = ["mypy-boto3-scheduler (>=1.35.0,<1.36.0)"] -schemas = ["mypy-boto3-schemas (>=1.35.0,<1.36.0)"] -sdb = ["mypy-boto3-sdb (>=1.35.0,<1.36.0)"] -secretsmanager = ["mypy-boto3-secretsmanager (>=1.35.0,<1.36.0)"] -securityhub = ["mypy-boto3-securityhub (>=1.35.0,<1.36.0)"] -securitylake = ["mypy-boto3-securitylake (>=1.35.0,<1.36.0)"] -serverlessrepo = ["mypy-boto3-serverlessrepo (>=1.35.0,<1.36.0)"] -service-quotas = ["mypy-boto3-service-quotas (>=1.35.0,<1.36.0)"] -servicecatalog = ["mypy-boto3-servicecatalog (>=1.35.0,<1.36.0)"] -servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (>=1.35.0,<1.36.0)"] -servicediscovery = ["mypy-boto3-servicediscovery (>=1.35.0,<1.36.0)"] -ses = ["mypy-boto3-ses (>=1.35.0,<1.36.0)"] -sesv2 = ["mypy-boto3-sesv2 (>=1.35.0,<1.36.0)"] -shield = ["mypy-boto3-shield (>=1.35.0,<1.36.0)"] -signer = ["mypy-boto3-signer (>=1.35.0,<1.36.0)"] -simspaceweaver = ["mypy-boto3-simspaceweaver (>=1.35.0,<1.36.0)"] -sms = ["mypy-boto3-sms (>=1.35.0,<1.36.0)"] -sms-voice = ["mypy-boto3-sms-voice (>=1.35.0,<1.36.0)"] -snow-device-management = ["mypy-boto3-snow-device-management (>=1.35.0,<1.36.0)"] -snowball = ["mypy-boto3-snowball (>=1.35.0,<1.36.0)"] -sns = ["mypy-boto3-sns (>=1.35.0,<1.36.0)"] -socialmessaging = ["mypy-boto3-socialmessaging (>=1.35.0,<1.36.0)"] -sqs = ["mypy-boto3-sqs (>=1.35.0,<1.36.0)"] -ssm = ["mypy-boto3-ssm (>=1.35.0,<1.36.0)"] -ssm-contacts = ["mypy-boto3-ssm-contacts (>=1.35.0,<1.36.0)"] -ssm-incidents = ["mypy-boto3-ssm-incidents (>=1.35.0,<1.36.0)"] -ssm-quicksetup = ["mypy-boto3-ssm-quicksetup (>=1.35.0,<1.36.0)"] -ssm-sap = ["mypy-boto3-ssm-sap (>=1.35.0,<1.36.0)"] -sso = ["mypy-boto3-sso (>=1.35.0,<1.36.0)"] -sso-admin = ["mypy-boto3-sso-admin (>=1.35.0,<1.36.0)"] -sso-oidc = ["mypy-boto3-sso-oidc (>=1.35.0,<1.36.0)"] -stepfunctions = ["mypy-boto3-stepfunctions (>=1.35.0,<1.36.0)"] -storagegateway = ["mypy-boto3-storagegateway (>=1.35.0,<1.36.0)"] -sts = ["mypy-boto3-sts (>=1.35.0,<1.36.0)"] -supplychain = ["mypy-boto3-supplychain (>=1.35.0,<1.36.0)"] -support = ["mypy-boto3-support (>=1.35.0,<1.36.0)"] -support-app = ["mypy-boto3-support-app (>=1.35.0,<1.36.0)"] -swf = ["mypy-boto3-swf (>=1.35.0,<1.36.0)"] -synthetics = ["mypy-boto3-synthetics (>=1.35.0,<1.36.0)"] -taxsettings = ["mypy-boto3-taxsettings (>=1.35.0,<1.36.0)"] -textract = ["mypy-boto3-textract (>=1.35.0,<1.36.0)"] -timestream-influxdb = ["mypy-boto3-timestream-influxdb (>=1.35.0,<1.36.0)"] -timestream-query = ["mypy-boto3-timestream-query (>=1.35.0,<1.36.0)"] -timestream-write = ["mypy-boto3-timestream-write (>=1.35.0,<1.36.0)"] -tnb = ["mypy-boto3-tnb (>=1.35.0,<1.36.0)"] -transcribe = ["mypy-boto3-transcribe (>=1.35.0,<1.36.0)"] -transfer = ["mypy-boto3-transfer (>=1.35.0,<1.36.0)"] -translate = ["mypy-boto3-translate (>=1.35.0,<1.36.0)"] -trustedadvisor = ["mypy-boto3-trustedadvisor (>=1.35.0,<1.36.0)"] -verifiedpermissions = ["mypy-boto3-verifiedpermissions (>=1.35.0,<1.36.0)"] -voice-id = ["mypy-boto3-voice-id (>=1.35.0,<1.36.0)"] -vpc-lattice = ["mypy-boto3-vpc-lattice (>=1.35.0,<1.36.0)"] -waf = ["mypy-boto3-waf (>=1.35.0,<1.36.0)"] -waf-regional = ["mypy-boto3-waf-regional (>=1.35.0,<1.36.0)"] -wafv2 = ["mypy-boto3-wafv2 (>=1.35.0,<1.36.0)"] -wellarchitected = ["mypy-boto3-wellarchitected (>=1.35.0,<1.36.0)"] -wisdom = ["mypy-boto3-wisdom (>=1.35.0,<1.36.0)"] -workdocs = ["mypy-boto3-workdocs (>=1.35.0,<1.36.0)"] -workmail = ["mypy-boto3-workmail (>=1.35.0,<1.36.0)"] -workmailmessageflow = ["mypy-boto3-workmailmessageflow (>=1.35.0,<1.36.0)"] -workspaces = ["mypy-boto3-workspaces (>=1.35.0,<1.36.0)"] -workspaces-thin-client = ["mypy-boto3-workspaces-thin-client (>=1.35.0,<1.36.0)"] -workspaces-web = ["mypy-boto3-workspaces-web (>=1.35.0,<1.36.0)"] -xray = ["mypy-boto3-xray (>=1.35.0,<1.36.0)"] - -[[package]] -name = "botocore" -version = "1.35.54" -description = "Low-level, data-driven core of boto 3." -optional = false -python-versions = ">=3.8" -files = [ - {file = "botocore-1.35.54-py3-none-any.whl", hash = "sha256:9cca1811094b6cdc144c2c063a3ec2db6d7c88194b04d4277cd34fc8e3473aff"}, - {file = "botocore-1.35.54.tar.gz", hash = "sha256:131bb59ce59c8a939b31e8e647242d70cf11d32d4529fa4dca01feea1e891a76"}, -] - -[package.dependencies] -jmespath = ">=0.7.1,<2.0.0" -python-dateutil = ">=2.1,<3.0.0" -urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} - -[package.extras] -crt = ["awscrt (==0.22.0)"] - -[[package]] -name = "botocore-stubs" -version = "1.35.54" -description = "Type annotations and code completion for botocore" -optional = false -python-versions = ">=3.8" -files = [ - {file = "botocore_stubs-1.35.54-py3-none-any.whl", hash = "sha256:26ba65907eed959dddc644ab1cd72e3a2cc9761dad79e0b45ff3b8676c47e5ec"}, - {file = "botocore_stubs-1.35.54.tar.gz", hash = "sha256:49e28813324308bfc5a92bde118df5c9c41a01237eef1e1628891770f3f68a94"}, -] - -[package.dependencies] -types-awscrt = "*" - -[package.extras] -botocore = ["botocore"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -optional = false -python-versions = ">=3.7" -files = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "s3transfer" -version = "0.10.3" -description = "An Amazon S3 Transfer Manager" -optional = false -python-versions = ">=3.8" -files = [ - {file = "s3transfer-0.10.3-py3-none-any.whl", hash = "sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d"}, - {file = "s3transfer-0.10.3.tar.gz", hash = "sha256:4f50ed74ab84d474ce614475e0b8d5047ff080810aac5d01ea25231cfc944b0c"}, -] - -[package.dependencies] -botocore = ">=1.33.2,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "types-awscrt" -version = "0.23.0" -description = "Type annotations and code completion for awscrt" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types_awscrt-0.23.0-py3-none-any.whl", hash = "sha256:517d9d06f19cf58d778ca90ad01e52e0489466bf70dcf78c7f47f74fdf151a60"}, - {file = "types_awscrt-0.23.0.tar.gz", hash = "sha256:3fd1edeac923d1956c0e907c973fb83bda465beae7f054716b371b293f9b5fdc"}, -] - -[[package]] -name = "types-s3transfer" -version = "0.10.3" -description = "Type annotations and code completion for s3transfer" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types_s3transfer-0.10.3-py3-none-any.whl", hash = "sha256:d34c5a82f531af95bb550927136ff5b737a1ed3087f90a59d545591dfde5b4cc"}, - {file = "types_s3transfer-0.10.3.tar.gz", hash = "sha256:f761b2876ac4c208e6c6b75cdf5f6939009768be9950c545b11b0225e7703ee7"}, -] - -[[package]] -name = "urllib3" -version = "2.2.3" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.8" -files = [ - {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, - {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.12" -content-hash = "cdf40eccb9f0ab07241e1e17b9536a5f53eccf34f87ec43af1ddaced21c0ec98" diff --git a/pyproject.toml b/pyproject.toml index 129b3a0..c7cea3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,24 @@ -[tool.poetry] +[project] name = "brrr" version = "0.1.0" -description = "" +description = "Horizontally scalable workflow scheduling with pluggable backends" authors = [ - "Hraban Luyat ", - "Jesse Zwaan " + {name = "Hraban Luyat", email = "hraban@0brg.net"}, + {name = "Jesse Zwaan", email = "j.k.zwaan@gmail.com"}, ] readme = "README.md" +requires-python = "<3.13,>=3.12" +dependencies = [] -[tool.poetry.dependencies] -python = "^3.12" -boto3 = "^1.35.54" -boto3-stubs = "^1.35.54" - +[dependency-groups] +dev = [ + "boto3>=1.35.71", + "boto3-stubs[essential]>=1.35.71", + "pyright>=1.1.389", + "redis>=5.2.0", + "ruff>=0.8.1", +] [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/src/brrr/__init__.py b/src/brrr/__init__.py new file mode 100644 index 0000000..5a9a4ca --- /dev/null +++ b/src/brrr/__init__.py @@ -0,0 +1,10 @@ +from .brrr import Brrr + +# For ergonomics, we provide a singleton and a bunch of proxies as the module interface +brrr = Brrr() + +setup = brrr.setup +gather = brrr.gather +wrrrk = brrr.wrrrk +srrrv = brrr.srrrv +task = brrr.register_task diff --git a/src/brrr/asyncrrr.py b/src/brrr/asyncrrr.py new file mode 100644 index 0000000..082a6ea --- /dev/null +++ b/src/brrr/asyncrrr.py @@ -0,0 +1,9 @@ +import asyncio + +async def async_wrrrk(workers: int = 1): + """ + Spin up a number of worker threads + """ + return asyncio.gather( + *[asyncio.create_task(asyncio_worker()) for _ in range(workers)] + ) diff --git a/src/brrr/backends/__init__.py b/src/brrr/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/brrr/backends/dynamo.py b/src/brrr/backends/dynamo.py new file mode 100644 index 0000000..878672c --- /dev/null +++ b/src/brrr/backends/dynamo.py @@ -0,0 +1,111 @@ +import os + +from mypy_boto3_dynamodb import DynamoDBClient + +from ..store import MemKey, Store + +# The frame table layout is: +# +# pk: FRAME_KEY +# sk: FRAME_KEY +# parent: The parent frame key +# memo: The memo key +# +# OR +# +# pk: FRAME_KEY +# sk: A child's frame key +# +# OR +# +# pk: MEMO_KEY +# sk: "call" +# task: The task name +# argv: bytes (pickled) +# +# OR +# +# pk: MEMO_KEY +# sk: "value" +# value: bytes (pickled) +# +# TODO It is possible we'll add versioning in there as pk or somethin +class DynamoDbMemStore(Store): + client: DynamoDBClient + table_name: str + + def key(self, mem_key: MemKey) -> dict: + return { + "pk": {"S": mem_key.id}, + "sk": {"S": mem_key.type} + } + + def __init__(self, client: DynamoDBClient, table_name: str): + self.client = client + self.table_name = table_name + + def __contains__(self, key: MemKey): + return "Item" in self.client.get_item( + TableName=self.table_name, + Key=self.key(key), + ) + + def __getitem__(self, key: MemKey) -> bytes: + return self.client.get_item( + TableName=self.table_name, + Key=self.key(key), + )["Item"]["value"]["B"] + + def __setitem__(self, key: MemKey, value: bytes): + self.client.put_item( + TableName=self.table_name, + Item={ + **self.key(key), + "value": {"B": value} + } + ) + + def __delitem__(self, key: MemKey): + self.client.delete_item( + TableName=self.table_name, + Key=self.key(key), + ) + + def __iter__(self): + raise NotImplementedError + + def __len__(self): + raise NotImplementedError + + def create_table(self): + try: + self.client.create_table( + TableName=self.table_name, + KeySchema=[ + { + "AttributeName": "pk", + "KeyType": "HASH" + }, + { + "AttributeName": "sk", + "KeyType": "RANGE" + } + ], + AttributeDefinitions=[ + { + "AttributeName": "pk", + "AttributeType": "S" + }, + { + "AttributeName": "sk", + "AttributeType": "S" + } + ], + # TODO make this configurable? Should this method even exist? + ProvisionedThroughput={ + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5 + } + ) + except self.client.exceptions.ResourceInUseException: + pass diff --git a/src/brrr/backends/in_memory.py b/src/brrr/backends/in_memory.py new file mode 100644 index 0000000..25ba14d --- /dev/null +++ b/src/brrr/backends/in_memory.py @@ -0,0 +1,26 @@ +import collections +from ..queue import Queue, Message, QueueIsEmpty + +class InMemoryQueue(Queue): + """ + This queue does not do receipts + """ + messages = collections.deque() + + def put(self, message: str): + self.messages.append(message) + + def get_message(self) -> Message: + if not self.messages: + raise QueueIsEmpty + return Message(self.messages.popleft(), '') + + async def get_message_async(self): + return self.get_message() + + def delete_message(self, receipt_handle: str): + pass + + +# Just to drive the point home +InMemoryMemStore = dict diff --git a/src/brrr/backends/redis.py b/src/brrr/backends/redis.py new file mode 100644 index 0000000..9315b1a --- /dev/null +++ b/src/brrr/backends/redis.py @@ -0,0 +1,205 @@ +import time +import redis + +from ..queue import Message, Queue, RichQueue, QueueIsEmpty +from ..store import MemKey, Store + +# This script takes care of all of the queue semantics beyond at least once delivery +# - Rate limiting +# - Max concurrency +# - Job timeout +# - Requeueing stuck jobs +# +# Features to consider: +# - Dead letter queue +# - Cleaning up from failed scripts. +DEQUEUE_FUNCTION = """ +-- This script is used to dequeue a message from a Redis stream +-- It respects max_concurrency as well as rate limits using a simple token bucket implementation +-- Usage: DEQUEUE_SCRIPT 1 +-- TODO make rate_limiting and max_concurrency optional + +local stream = KEYS[1] +local rate_limiters = stream .. ":rate_limiters" +local group = ARGV[1] +local consumer = ARGV[2] +local rate_limit_pool_capacity = tonumber(ARGV[3]) +local replenish_rate_per_second = tonumber(ARGV[4]) +local max_concurrency = tonumber(ARGV[5]) +local job_timeout_ms = tonumber(ARGV[6]) + +-- Before dequeueing, restore all stuck jobs to the stream +-- TODO do we want to set timeout per job? +-- Count of 100 is the default, presumably we want this to be 'inf' +local stuck_jobs = redis.call('XAUTOCLAIM', stream, group, 'watchdog', job_timeout_ms, '0-0', 'COUNT', 100) +for _, job in ipairs(stuck_jobs[2] or {}) do + -- Add before removing to avoid race conditions + redis.call('XADD', stream, '*', 'body', job[2][2]) + redis.call('SREM', rate_limiters, job[1]) +end + +-- The PEL holds all currently active jobs, grabbed by a consumer that haven't been acked +-- If the PEL is full, we can't take any more jobs, we do need to make sure that +-- jobs don't get stuck in the PEL forever +if (tonumber(redis.call('XPENDING', stream, group)[1]) or 0) >= max_concurrency then + return nil +end + +-- Rate limits are implemented by adding each grabbed task to a set of rate limiters, with a TTL +-- Calculated by the rate limit pool capacity and the replenish rate +-- If the rate limiter set is full, we can't take any more jobs +if (tonumber(redis.call('SCARD', rate_limiters)) or 0) >= rate_limit_pool_capacity then + return nil +end + +-- Grab one message from the stream +local message = redis.call('XREADGROUP', 'GROUP', group, consumer, 'COUNT', 1, 'STREAMS', stream, '>') + +if message and #message > 0 then + local stream_messages = message[1][2] + if #stream_messages > 0 then + local msg_id = stream_messages[1][1] + -- If we have more key, value pairs, we'd need to find the body key here + local msg_body = stream_messages[1][2][2] + + -- Before returning the message, add expiry to the key, then add it to the rate limiters (in this order to avoid immortal rate limiters) + local max_rate_limit_lookback_seconds = rate_limit_pool_capacity / replenish_rate_per_second + redis.call("EXPIRE", msg_id, max_rate_limit_lookback_seconds) + redis.call("SADD", rate_limiters, msg_id) + + return {msg_body, msg_id} + end +end + +return nil +--end) +""".strip() + +ACK_FUNCTION = """ +-- This script is used to delete a message from a Redis stream +-- Usage: DEQUEUE_SCRIPT 1 +-- TODO make rate_limiting and max_concurrency optional + +local stream = KEYS[1] +local group = ARGV[1] +local msg_id = ARGV[2] + +redis.call('XACK', stream, group, msg_id) +redis.call('XDEL', stream, msg_id) +""" + +class RedisQueue(Queue): + client: redis.Redis + key: str + + def __init__(self, client: redis.Redis, key: str): + self.client = client + self.key = key + + def put(self, message: str): + self.client.rpush(self.key, message) + + def get_message(self) -> Message: + message_bytes = self.client.lpop(self.key) + if not message_bytes: + raise QueueIsEmpty + message = message_bytes.decode('utf-8') + return Message(message, '') + + def delete_message(self, receipt_handle: str): + pass + + +class RedisStream(RichQueue): + client: redis.Redis + + queue: str + group = "workers" + + rate_limit_pool_capacity = 1000 + replenish_rate_per_second = 100 + max_concurrency = 4 + job_timeout_ms = 1000 + + consumer = "TODO_WORKER_ID" + lib_name = "brrr" + func_name = "dequeue" + + def __init__(self, client: redis.Redis, queue: str): + self.client = client + self.queue = queue + + def clear(self): + self.client.delete(self.queue) + self.client.delete(self.queue + ":rate_limiters") + + def setup(self): + try: + self.client.xgroup_create(self.queue, self.group, id='0', mkstream=True) + except redis.exceptions.ResponseError as e: + if "BUSYGROUP Consumer Group name already exists" not in str(e): + raise + + # Don't register functions but run eval instead + # assert DEQUEUE_FUNCTION.startswith(f"#!lua name={self.lib_name}") + # assert f"redis.register_function('{self.func_name}'" in DEQUEUE_FUNCTION + # self.client.function_load(DEQUEUE_FUNCTION, replace=True) + + def put(self, body: str): + # Messages can not be added to specific groups, so we just create a stream per topic + self.client.xadd(self.queue, {'body': body}) + + def get_message(self) -> Message: + keys = self.queue, + argv = self.group, self.consumer, self.rate_limit_pool_capacity, self.replenish_rate_per_second, self.max_concurrency, self.job_timeout_ms + response = self.client.eval(DEQUEUE_FUNCTION, len(keys), *keys, *argv) + if not response: + time.sleep(1) + raise QueueIsEmpty + body = response[0].decode('utf-8') + receipt_handle = response[1].decode('utf-8') + print(self.client.xpending(self.queue, self.group)) + return Message(body, receipt_handle) + + # TODO: This has a bug; xack does not remove the message from the stream + def delete_message(self, receipt_handle: str): + # The receipt handle here must match the message ID + # self.client.xack(self.queue, self.group, receipt_handle) + keys = self.queue, + argv = self.group, receipt_handle, + self.client.eval(ACK_FUNCTION, len(keys), *keys, *argv) + + def set_message_timeout(self, receipt_handle: str, seconds: int): + # The seconds don't do anything for now; I wasn't sure how to translate a fixed timeout to the Redis model + # At least this resets the idle time @jkz + self.client.xclaim(self.queue, self.group, self.consumer, min_idle_time=0, message_ids=[receipt_handle]) + +class RedisMemStore(Store): + client: redis.Redis + + def __init__(self, client: redis.Redis): + self.client = client + + def key(self, key: MemKey) -> str: + return f"{key.type}:{key.id}" + + def __getitem__(self, key: MemKey) -> bytes: + value = self.client.get(self.key(key)) + if value is None: + raise KeyError(key) + return value + + def __setitem__(self, key: str, value: bytes): + self.client.set(self.key(key), value) + + def __delitem__(self, key: str): + self.client.delete(self.key(key)) + + def __iter__(self): + raise NotImplementedError + + def __contains__(self, key: str) -> bool: + return self.client.exists(self.key(key)) == 1 + + def __len__(self) -> int: + raise NotImplementedError diff --git a/src/brrr/backends/sqs.py b/src/brrr/backends/sqs.py new file mode 100644 index 0000000..258139d --- /dev/null +++ b/src/brrr/backends/sqs.py @@ -0,0 +1,40 @@ +from mypy_boto3_sqs import SQSClient + +from ..queue import Queue, Message, QueueIsEmpty + +class SqsQueue(Queue): + def __init__(self, client: SQSClient, url: str): + self.client = client + self.url = url + + def put(self, message: str): + self.client.send_message( + QueueUrl=self.url, + MessageBody=message + ) + + def get_message(self) -> Message: + response = self.client.receive_message( + QueueUrl=self.url, + MaxNumberOfMessages=1, + WaitTimeSeconds=3, + ) + + if "Messages" not in response: + raise QueueIsEmpty + + return Message(response["Messages"][0]["Body"], response["Messages"][0]["ReceiptHandle"]) + + def delete_message(self, receipt_handle: str): + self.client.delete_message( + QueueUrl=self.url, + ReceiptHandle=receipt_handle + ) + + def set_message_timeout(self, receipt_handle, seconds): + self.client.change_message_visibility( + QueueUrl=self.url, + ReceiptHandle=receipt_handle, + VisibilityTimeout=seconds + ) + diff --git a/src/brrr/brrr.py b/src/brrr/brrr.py new file mode 100644 index 0000000..32266c4 --- /dev/null +++ b/src/brrr/brrr.py @@ -0,0 +1,408 @@ +from typing import Any, Callable, Union + +import asyncio +import threading +import os +import http.server +import socketserver +import json +from urllib.parse import parse_qsl + +from .store import Call, Frame, Memory, Store, input_hash +from .queue import Queue, QueueIsEmpty + +class Defer(Exception): + """ + When a task is called and hasn't been computed yet, a Defer exception is raised + Workers catch this exception and schedule the task to be computed + """ + calls: list[Call] + def __init__(self, calls: list[Call]): + self.calls = calls + +# Quick n dirty hack to achieve lazy initialization without fully rewriting this +# entire global using file. The real solution is to use a singleton and make +# the top level functions proxies to the singleton. +class Brrr: + """ + All state for brrr to function wrapped in a container. + """ + # The worker loop (as of writing) is synchronous so it can safely set a + # local global variable to indicate that it is a worker thread, which, for + # tasks, means that their Defer raises will be caught and handled by the + # worker + worker_singleton: Union['Wrrrker', None] + + # For threaded workers, each worker registers itself in this dict by thread id + worker_threads: dict[int, 'Wrrrker'] + + # For async workers, + worker_loops: dict[int, 'Wrrrker'] + + # A storage backend for frames, calls and values + memory: Memory | None + # A queue of frame keys to be processed + queue: Queue | None + + # Dictionary of task_name to task instance + tasks = dict[str, 'Task'] + + def __init__(self): + self.worker_singleton = None + self.worker_threads = {} + self.worker_loops = {} + self.tasks = {} + self.queue = None + self.memory = None + + # TODO do we like the idea of brrr as a context manager? + def __enter__(self): + pass + def __exit__(self, exc_type, exc_value, traceback): + if hasattr(self.queue, "__exit__"): + self.queue.__exit__(exc_type, exc_value, traceback) + if hasattr(self.memory, "__exit__"): + self.memory.__exit__(exc_type, exc_value, traceback) + + # TODO Do we want to pass in a memstore/kv instead? + def setup(self, queue: Queue, store: Store): + # TODO throw if already instantiated? + self.queue = queue + self.memory = Memory(store) + + # TODO would we like this to be a decorator? + def require_setup(self): + if self.queue is None or self.memory is None: + raise Exception("Brrr not set up") + + def are_we_inside_worker_context(self): + if self.worker_singleton: + return True + elif self.worker_threads: + # For synchronous workers, we can use a thread-local global variable + return threading.current_thread() in self.worker_threads + elif self.worker_loops: + try: + # For async workers, we can check the asyncio loop + return asyncio.get_running_loop() in self.worker_loops + except RuntimeError: + return False + else: + return False + + + def gather(self, *task_lambdas) -> list[Any]: + """ + Takes a number of task lambdas and calls each of them. + If they've all been computed, return their values, + Otherwise raise jobs for those that haven't been computed + """ + self.require_setup() + + if not self.are_we_inside_worker_context(): + return [task_lambda() for task_lambda in task_lambdas] + + defers = [] + values = [] + + for task_lambda in task_lambdas: + try: + values.append(task_lambda()) + except Defer as d: + defers.extend(d.calls) + + if defers: + raise Defer(defers) + + return values + + def schedule(self, call: Call, parent_key=None) -> Frame: + self.require_setup() + + # Value has been computed already, return straight to the parent (if there is one) + if self.memory.has_value(call.memo_key): + if parent_key is not None: + self.queue.put(parent_key) + return + + # If not, schedule the child. We don't care if it has been scheduled already for now + # but we could check, as an optimisation. The set calls are idempotent. + self.memory.set_call(call) + child = Frame(call.memo_key, parent_key) + self.memory.set_frame(child) + self.queue.put(child.frame_key) + + return child + + + def evaluate(self, frame: Frame) -> Any: + """ + Evaluate a frame, which means calling the tasks function with its arguments + """ + self.require_setup() + + call = self.memory.get_call(frame.memo_key) + task = self.tasks[call.task_name] + return task.evaluate(call.argv) + + def register_task(self, fn: Callable, name: str = None) -> 'Task': + task = Task(self, fn, name) + if task.name in self.tasks: + raise Exception(f"Task {task.name} already exists") + self.tasks[task.name] = task + return task + + def task(self, fn: Callable, name: str = None) -> 'Task': + return Task(self, fn, name) + + def srrrv(self, tasks: list['Task'], port: int = int(os.getenv("SRRRVER_PORT", "8333"))): + """ + Spin up a webserver that that translates HTTP requests to tasks + """ + # Srrrver.tasks.update({task.name: task for task in tasks}) + with socketserver.TCPServer(("", port), Srrrver.subclass_with_brrr(self)) as httpd: + print(f"Srrrver running on port {port}") + print("Available tasks:") + for task in tasks: + print(" ", task.name) + httpd.serve_forever() + + async def wrrrk_async(self, workers: int = 1): + """ + Start a number of async worker loops + """ + await asyncio.gather( + *(Wrrrker(self).loop_async() for _ in range(workers)) + ) + + def wrrrk(self, threads: int = 1): + """ + Spin up a number of worker threads + """ + if threads == 1: + Wrrrker(self).loop() + else: + for _ in range(threads): + threading.Thread(target=Wrrrker(self).loop).start() + + +class Task: + """ + A decorator to turn a function into a task. + When it is called, within the context of a worker, it checks whether it has already been computed. + If so, it returns the value, otherwise it raises a Call job, which causes the worker to schedule the computation. + + A task can not write to the store, only read from it + """ + + fn: Any + name: str + brrr: Brrr + + def __init__(self, brrr: Brrr, fn, name: str = None): + self.brrr = brrr + self.fn = fn + self.name = name or fn.__name__ + + # Calling a function returns the value if it has already been computed. + # Otherwise, it raises a Call exception to schedule the computation + def __call__(self, *args, **kwargs): + argv = (args, kwargs) + if not self.brrr.are_we_inside_worker_context(): + return self.evaluate(argv) + memo_key = input_hash(self.name, argv) + try: + return self.brrr.memory.get_value(memo_key) + except KeyError: + raise Defer([Call(self.name, argv)]) + + def to_lambda(self, *args, **kwargs): + """ + Separate function to capture a closure + """ + return lambda: self(*args, **kwargs) + + def map(self, args: list[Union[dict, list, tuple[tuple, dict]]]): + """ + Fanning out, a map function returns the values if they have already been computed. + Otherwise, it raises a list of Call exceptions to schedule the computation, + for the ones that aren't already computed + + Offers a few syntaxes, TBD whether that is useful + #TODO we _could_ support a list of elements to get passed as a single arg each + """ + argvs = [ + (arg, {}) if isinstance(arg, list) else ((), arg) if isinstance(arg, dict) else arg + for arg in args + ] + return self.brrr.gather(*(self.to_lambda(*argv[0], **argv[1]) for argv in argvs)) + + def evaluate(self, argv): + return self.fn(*argv[0], **argv[1]) + + def schedule(self, *args, **kwargs): + """ + This puts the task call on the queue, but doesn't return the result! + """ + return self.brrr.schedule(Call(self.name, (args, kwargs))) + +class Wrrrker: + def __init__(self, brrr: Brrr): + self.brrr = brrr + + # The context manager maintains a thread-local global variable to indicate that the thread is a worker + # and that any invoked tasks can raise Defer exceptions + def __enter__(self): + if self.brrr.worker_singleton is not None: + raise Exception("Worker already running") + self.brrr.worker_singleton = self + + def __exit__(self, exc_type, exc_value, traceback): + self.brrr.worker_singleton = None + + def resolve_frame(self, frame_key: str): + """ + A queue message is a frame key and a receipt handle + The frame key is used to look up the job to be done, + the receipt handle is used to tell the queue that the job is done + """ + + frame = self.brrr.memory.get_frame(frame_key) + try: + self.brrr.memory.set_value(frame.memo_key, self.brrr.evaluate(frame)) + if frame.parent_key is not None: + # This is a redundant step, we could just check the store whether the children have memoized values + # frames[frame.parent_key].children[frame_key] = True + self.brrr.queue.put(frame.parent_key) + except Defer as defer: + for call in defer.calls: + self.brrr.schedule(call, frame_key) + + # TODO exit when queue empty? + def loop(self): + """ + Workers take jobs from the queue, one at a time, and handle them. + They have read and write access to the store, and are responsible for + Managing the output of tasks and scheduling new ones + """ + with self: + print("Worker Started") + while True: + try: + # This is presumed to be a long poll + message = self.brrr.queue.get_message() + except QueueIsEmpty: + continue + + frame_key = message.body + self.resolve_frame(frame_key) + + self.brrr.queue.delete_message(message.receipt_handle) + + async def loop_async(self): + with self: + while True: + try: + message = await self.brrr.queue.get_message_async() + except QueueIsEmpty: + continue + + frame_key = message.body + self.resolve_frame(frame_key) + + self.brrr.queue.delete_message(message.receipt_handle) + + +class ThreadWrrrker(Wrrrker): + # The context manager maintains a thread-local global variable to indicate that the thread is a worker + # and that any invoked tasks can raise Defer exceptions + def __enter__(self): + tid = threading.current_thread() + if tid in self.brrr.worker_threads: + raise Exception("Worker already running in this thread") + + self.brrr.worker_threads[tid] = self + return self + + def __exit__(self, exc_type, exc_value, traceback): + del self.brrr.worker_threads[threading.current_thread()] + + +# A "Front Office" worker, that is publically exposed and takes requests +# to schedule tasks and return their results via webhooks +class Srrrver(http.server.SimpleHTTPRequestHandler): + brrr: Brrr + + @classmethod + def subclass_with_brrr(cls, brrr: Brrr) -> 'Srrrver': + """ + For some reason the python server class needs to be instantiated without args + """ + return type("SrrrverWithBrrr", (cls,), {"brrr": brrr}) + + """ + A simple HTTP server that takes a JSON payload and schedules a task + """ + def do_GET(self): + """ + GET /task_name?argv={"..."} + """ + task_name = self.path.split("?")[0].strip("/") + # TODO parse the argv properly + kwargs = dict(parse_qsl(self.path.split("?")[-1])) + argv = ((), kwargs) + + try: + task = self.brrr.tasks[task_name] + call = Call(task.name, argv) + except KeyError: + self.send_response(404) + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({"error": "Task not found"}).encode()) + return + + try: + memo_key = call.memo_key + result = self.brrr.memory.get_value(memo_key) + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({"status": "ok", "result": result}).encode()) + except KeyError: + self.brrr.schedule(call) + self.send_response(202) + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({"status": "accepted"}).encode()) + return + + def do_POST(self): + """ + POST /{task_name} with a JSON payload { + "args": [], + "kwargs": {}, + } + + """ + # Custom behavior for POST requests + content_length = int(self.headers['Content-Length']) + post_data = self.rfile.read(content_length) + self.wfile.write(post_data) + + # Schedule a task and submit PUT request to the webhook with the result if one is provided + # once it's done + try: + data = json.loads(post_data) + call = Call(data["task_name"], (data["args"], data["kwargs"])) + frame = self.brrr.schedule(call) + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.end_headers() + error_response = {"status": "OK", "frame_key": frame.frame_key} + except json.JSONDecodeError: + self.send_response(400) + self.send_header('Content-type', 'application/json') + self.end_headers() + error_response = {"error": "Invalid JSON"} + self.wfile.write(json.dumps(error_response)) diff --git a/src/brrr/compat.py b/src/brrr/compat.py new file mode 100644 index 0000000..9ec8c8d --- /dev/null +++ b/src/brrr/compat.py @@ -0,0 +1,48 @@ +from typing import Callable + +from .brrr import Task + +class CompatTask(Task): + def to_deployment(self): + return self + +def serve(*tasks): + pass + +def deploy(*tasks): + pass + +def task( + fn=None, + *, + name: str = None, + description: str = None, + timeout_seconds: int = None, + cache_key_fn: Callable[[tuple[tuple, dict]], str] = None, + cache_expiration: int = None, + retries: int = None, + retry_delay_seconds: int = None, + log_prints: bool, + **kwargs +): + def decorator(_fn): + return CompatTask( + _fn, + name=name or _fn.__name__, + ) + return decorator if fn is None else decorator(fn) + + +def flow( + fn=None, + *, + name: str = None, + description: str = None, + timeout_seconds: int = None, + retries: int = None, + retry_delay_seconds: int = None, + flow_run_name: str = None, + validate_parameters: bool = None, + version: str = None, +): + return task(fn, name=name, description=description, timeout_seconds=timeout_seconds, retries=retries, retry_delay_seconds=retry_delay_seconds) diff --git a/src/brrr/queue.py b/src/brrr/queue.py new file mode 100644 index 0000000..8d14a35 --- /dev/null +++ b/src/brrr/queue.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass + + +class QueueIsEmpty(Exception): + pass + +@dataclass +class Message: + body: str + receipt_handle: str + + +# Infra abstractions + +class Queue: + async def get_message_async(self) -> Message: + return self.get_message() + def get_message(self) -> Message: + raise NotImplementedError + def delete_message(self, receipt_handle: str): + raise NotImplementedError + def set_message_timeout(self, receipt_handle: str, seconds: int): + raise NotImplementedError + + +class RichQueue(Queue): + # Max number of jobs that can be processed concurrently + max_concurrency: int + + # Every job requires a token from the pool to be dequeued + rate_limit_pool_capacity: int + + # The number of tokens restored to the pool per second + replenish_rate_per_second: float diff --git a/src/brrr/store.py b/src/brrr/store.py new file mode 100644 index 0000000..b391637 --- /dev/null +++ b/src/brrr/store.py @@ -0,0 +1,132 @@ +from dataclasses import dataclass +from typing import Any +from collections import namedtuple +from collections.abc import MutableMapping + +import pickle + +from hashlib import sha256 + +def input_hash(*args): + return sha256(":".join(map(str, args)).encode()).hexdigest() + + +# Objects to be stored + +# A memoization cache for tasks that have already been computed, based on their task name and input arguments + +# Using the same memo key, we store the task and its argv here so we can retrieve them in workers +@dataclass +class Call: + task_name: str + argv: tuple[tuple, dict] + + @property + def memo_key(self): + return input_hash(self.task_name, self.argv) + + +# A store of task frame contexts. A frame looks something like: +# +# memo_key: A hash of the task and its arguments +# children: A dictionary of child tasks keys, with a bool indicating whether they have been computed (This could be a set) +# parent: The caller's frame_key +# +# In the real world, this would be some sort of distributed store, +# optimised for specific access patterns +@dataclass +class Frame: + """ + A frame represents a function call with a parent and a number of child frames + """ + memo_key: str + # The empty string means no parent + parent_key: str + # This one is redundant + # children: dict + + @property + def frame_key(self): + return input_hash(self.parent_key, self.memo_key) + + + +@dataclass +class Info: + """ + Optional information about a task. + Does not affect the computation, but may instruct orchestration + """ + description: str | None + timeout_seconds: int | None + retries: int | None + retry_delay_seconds: int | None + log_prints: bool | None + + +MemKey = namedtuple("MemKey", ["type", "id"]) + +# All operations MUST be idempotent +# All getters MUST throw a KeyError for missing keys +Store = MutableMapping[MemKey, bytes] + +class PickleJar: + """ + A dict-like object that pickles on set and unpickles on get + """ + pickles: MutableMapping[MemKey, bytes] + + def __init__(self, store: Store): + self.pickles = store + + def __contains__(self, key: MemKey): + return key in self.pickles + + def __getitem__(self, key: MemKey): + return pickle.loads(self.pickles[key]) + + def __setitem__(self, key: MemKey, value): + self.pickles[key] = pickle.dumps(value) + + +class Memory: + """ + A memstore that uses a PickleJar as its backend + """ + def __init__(self, store: Store): + self.pickles = PickleJar(store) + + def get_frame(self, frame_key: str) -> Frame: + return self.pickles[MemKey("frame", frame_key)] + + def set_frame(self, frame: Frame): + self.pickles[MemKey("frame", frame.frame_key)] = frame + + def get_call(self, memo_key: str) -> Call: + return self.pickles[MemKey("call", memo_key)] + + def set_call(self, call: Call): + self.pickles[MemKey("call", call.memo_key)] = call + + def has_value(self, memo_key: str) -> bool: + return MemKey("value", memo_key) in self.pickles + + def get_value(self, memo_key: str) -> Any: + return self.pickles[MemKey("value", memo_key)] + + def set_value(self, memo_key: str, value: Any): + self.pickles[MemKey("value", memo_key)] = value + + def get_info(self, task_name: str) -> Info: + return self.pickles[MemKey("info", task_name)] + + def set_info(self, task_name: str, value: Info): + self.pickles[MemKey("info", task_name)] = value + + def get_stack_trace(self, frame_key: str) -> list[Frame]: + frames = [] + while frame_key: + frame = self.get_frame(frame_key) + frames.append(frame) + frame_key = frame.parent_key + return frames diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..5b69c80 --- /dev/null +++ b/uv.lock @@ -0,0 +1,288 @@ +version = 1 +requires-python = "==3.12.*" + +[[package]] +name = "boto3" +version = "1.35.71" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/6d/8e89a60e756c5da4ef56afa738aabed4aa16945676e98b23ede17dffb007/boto3-1.35.71.tar.gz", hash = "sha256:3ed7172b3d4fceb6218bb0ec3668c4d40c03690939c2fca4f22bb875d741a07f", size = 111006 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/db/c544d414b6c903011489fc33e2c171d497437b9914a7b587576ee31694b3/boto3-1.35.71-py3-none-any.whl", hash = "sha256:e2969a246bb3208122b3c349c49cc6604c6fc3fc2b2f65d99d3e8ccd745b0c16", size = 139177 }, +] + +[[package]] +name = "boto3-stubs" +version = "1.35.71" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore-stubs" }, + { name = "types-s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/85/86243ad2792f8506b567c645d97ece548258203c55bcc165fd5801f4372f/boto3_stubs-1.35.71.tar.gz", hash = "sha256:50e20fa74248c96b3e3498b2d81388585583e38b9f0609d2fa58257e49c986a5", size = 93776 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/d1/aedf5f4a92e1e74ee29a4d43084780f2d77aeef3d734e550aa2ab304e1fb/boto3_stubs-1.35.71-py3-none-any.whl", hash = "sha256:4abf357250bdb16d1a56489a59bfc385d132a43677956bd984f6578638d599c0", size = 62964 }, +] + +[package.optional-dependencies] +essential = [ + { name = "mypy-boto3-cloudformation" }, + { name = "mypy-boto3-dynamodb" }, + { name = "mypy-boto3-ec2" }, + { name = "mypy-boto3-lambda" }, + { name = "mypy-boto3-rds" }, + { name = "mypy-boto3-s3" }, + { name = "mypy-boto3-sqs" }, +] + +[[package]] +name = "botocore" +version = "1.35.71" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/7b/c3f9babe738d5efeb96bd5b250bafcd733c2fd5d8650d8986daa86ee45a1/botocore-1.35.71.tar.gz", hash = "sha256:f9fa058e0393660c3fe53c1e044751beb64b586def0bd2212448a7c328b0cbba", size = 13238393 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/49/b821048ae671518c00064a936ca08ab4ef7a715d866c3f0331688febeedd/botocore-1.35.71-py3-none-any.whl", hash = "sha256:fc46e7ab1df3cef66dfba1633f4da77c75e07365b36f03bd64a3793634be8fc1", size = 13034516 }, +] + +[[package]] +name = "botocore-stubs" +version = "1.35.71" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-awscrt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/0d/a76cdd9457268ad6efedd8abdc9f81511e297d20331fd6dc7de52ffc0701/botocore_stubs-1.35.71.tar.gz", hash = "sha256:c5f7208b20ae19400fa73eb569017f1e372990f7a5505a72116ed6420904f666", size = 40166 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/ce/aaccd20b8c4c90dc688e128ab9d4446403e692866961c979e29ed3fb28a1/botocore_stubs-1.35.71-py3-none-any.whl", hash = "sha256:7e938bb169c28faf05ce14e67bb0b5e5583092ab6ccc9d3d68d698530edb6584", size = 61049 }, +] + +[[package]] +name = "brrr" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "boto3" }, + { name = "boto3-stubs", extra = ["essential"] }, + { name = "pyright" }, + { name = "redis" }, + { name = "ruff" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "boto3", specifier = ">=1.35.71" }, + { name = "boto3-stubs", extras = ["essential"], specifier = ">=1.35.71" }, + { name = "pyright", specifier = ">=1.1.389" }, + { name = "redis", specifier = ">=5.2.0" }, + { name = "ruff", specifier = ">=0.8.1" }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256 }, +] + +[[package]] +name = "mypy-boto3-cloudformation" +version = "1.35.64" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/97/04c21049fb94ea9b5dc52cb5350042b822a5e3f8495e06403e442abc510a/mypy_boto3_cloudformation-1.35.64.tar.gz", hash = "sha256:d1a1500df811ac8ebd459640f5b31c14daac784d8a00fc4f67bc6eb391e7b5a8", size = 53575 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/c3/c7de8091b98747e134c68f94442ea906b1ee91e7b3e1831088b708e2da85/mypy_boto3_cloudformation-1.35.64-py3-none-any.whl", hash = "sha256:aba213f3411a65096a8d95633c36e0c57a775ac6ac9ccf1e6fd9bea4002073bc", size = 65168 }, +] + +[[package]] +name = "mypy-boto3-dynamodb" +version = "1.35.60" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/e7/09a49462ea33bd31d8d552ffe84f9fffb12ae16b350a54c9b0f9074d218a/mypy_boto3_dynamodb-1.35.60.tar.gz", hash = "sha256:92eac35c49e9f3ff23a4ad6dee5dc54e410e0c49a98b4d93493c7000ebe74568", size = 45142 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a7/8ec2958a20bef1e6956c7420a4eb23059d982149ed14a7f0c9e7a73de404/mypy_boto3_dynamodb-1.35.60-py3-none-any.whl", hash = "sha256:187915c781f352bc79d35b08a094605515ecc54f30107f629972c3358b864a5c", size = 54183 }, +] + +[[package]] +name = "mypy-boto3-ec2" +version = "1.35.70" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/a4/a92feffd3ccba3415e1f2cb5e0b482b9bfba885d0a01a4aadf97133243c3/mypy_boto3_ec2-1.35.70.tar.gz", hash = "sha256:93f9ddadac303d63f34cd4c0a60a682c008d655d3b2cfa74d1234fbde9a0b401", size = 379694 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/2f/8d75aa08d7dbdb29ad7374a80ee544fbe1801f350fc4b588da147baa4de7/mypy_boto3_ec2-1.35.70-py3-none-any.whl", hash = "sha256:d5b27b79b1749fb10a4eb9508069995a8e4bf2614f4e171224637596403e42c8", size = 371241 }, +] + +[[package]] +name = "mypy-boto3-lambda" +version = "1.35.68" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/2e/b80f84512a1d62a6d8c6d076ad21449bff06c5c4ec69f21617f66ae1167e/mypy_boto3_lambda-1.35.68.tar.gz", hash = "sha256:577a9465ac63ac564efc2755a7e72c28a9d2f496747c1faf242cb13d5017b262", size = 40260 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/75/d6c0381da41cddc89fa8afa7e4d7fd0007cc0fb9de0596d97712cc6894ef/mypy_boto3_lambda-1.35.68-py3-none-any.whl", hash = "sha256:00499898236fe423c9292f77644102d4bd6699b3c16b8c4062eb759c022447f5", size = 47192 }, +] + +[[package]] +name = "mypy-boto3-rds" +version = "1.35.66" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/e2/0351901e2b48f39c5fc43e9b0d6544f7ac2ff4d09dafc468903fdd5352fa/mypy_boto3_rds-1.35.66.tar.gz", hash = "sha256:0850cb5bddda1853c6ba44bb8dc1bf0d303ea4729f8cdf982d0e4d91f08ab2d9", size = 82939 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/be/d3d5bebcb505fa2437c33a42fa8875ad7f8e76bf67d3e9b2e925643826fc/mypy_boto3_rds-1.35.66-py3-none-any.whl", hash = "sha256:7bfeadfbd361aaf53a5f161c571886d3cadbdf05c15591761280fe6f079ab273", size = 89590 }, +] + +[[package]] +name = "mypy-boto3-s3" +version = "1.35.69" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/13/4603c44a16d3ed5de032a1dbdb3f6fcaea2bf82ba46cad85df5b3d6d5498/mypy_boto3_s3-1.35.69.tar.gz", hash = "sha256:97f7944a84a4a49282825bef1483a25680dcdce75da6017745d709d2cf2aa1c0", size = 70301 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/e9/b2548bfd7358057e102fcf9b0705a40d2b4dce86cfeaf1b604959b44f15e/mypy_boto3_s3-1.35.69-py3-none-any.whl", hash = "sha256:11a34259983e09d67e4d3a322fd47904a006bbfff19984e4e36a77e30f2014bb", size = 77391 }, +] + +[[package]] +name = "mypy-boto3-sqs" +version = "1.35.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/e5/e8228deac8720ddd152c8ff8f55fa5ef27f1c256835dd98063cd754570c7/mypy_boto3_sqs-1.35.0.tar.gz", hash = "sha256:61752f1c2bf2efa3815f64d43c25b4a39dbdbd9e472ae48aa18d7c6d2a7a6eb8", size = 22253 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/46/5ad44cf9d4725a4530b9ea9d0fdb01893d2f5fa7b23e51b0d483635350ff/mypy_boto3_sqs-1.35.0-py3-none-any.whl", hash = "sha256:9fd6e622ed231c06f7542ba6f8f0eea92046cace24defa95d0d0ce04e7caee0c", size = 33032 }, +] + +[[package]] +name = "nodeenv" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314 }, +] + +[[package]] +name = "pyright" +version = "1.1.389" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/4e/9a5ab8745e7606b88c2c7ca223449ac9d82a71fd5e31df47b453f2cb39a1/pyright-1.1.389.tar.gz", hash = "sha256:716bf8cc174ab8b4dcf6828c3298cac05c5ed775dda9910106a5dcfe4c7fe220", size = 21940 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/26/c288cabf8cfc5a27e1aa9e5029b7682c0f920b8074f45d22bf844314d66a/pyright-1.1.389-py3-none-any.whl", hash = "sha256:41e9620bba9254406dc1f621a88ceab5a88af4c826feb4f614d95691ed243a60", size = 18581 }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, +] + +[[package]] +name = "redis" +version = "5.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/17/2f4a87ffa4cd93714cf52edfa3ea94589e9de65f71e9f99cbcfa84347a53/redis-5.2.0.tar.gz", hash = "sha256:0b1087665a771b1ff2e003aa5bdd354f15a70c9e25d5a7dbf9c722c16528a7b0", size = 4607878 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/f5/ffa560ecc4bafbf25f7961c3d6f50d627a90186352e27e7d0ba5b1f6d87d/redis-5.2.0-py3-none-any.whl", hash = "sha256:ae174f2bb3b1bf2b09d54bf3e51fbc1469cf6c10aa03e21141f51969801a7897", size = 261428 }, +] + +[[package]] +name = "ruff" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/d0/8ff5b189d125f4260f2255d143bf2fa413b69c2610c405ace7a0a8ec81ec/ruff-0.8.1.tar.gz", hash = "sha256:3583db9a6450364ed5ca3f3b4225958b24f78178908d5c4bc0f46251ccca898f", size = 3313222 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/d6/1a6314e568db88acdbb5121ed53e2c52cebf3720d3437a76f82f923bf171/ruff-0.8.1-py3-none-linux_armv6l.whl", hash = "sha256:fae0805bd514066f20309f6742f6ee7904a773eb9e6c17c45d6b1600ca65c9b5", size = 10532605 }, + { url = "https://files.pythonhosted.org/packages/89/a8/a957a8812e31facffb6a26a30be0b5b4af000a6e30c7d43a22a5232a3398/ruff-0.8.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b8a4f7385c2285c30f34b200ca5511fcc865f17578383db154e098150ce0a087", size = 10278243 }, + { url = "https://files.pythonhosted.org/packages/a8/23/9db40fa19c453fabf94f7a35c61c58f20e8200b4734a20839515a19da790/ruff-0.8.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cd054486da0c53e41e0086e1730eb77d1f698154f910e0cd9e0d64274979a209", size = 9917739 }, + { url = "https://files.pythonhosted.org/packages/e2/a0/6ee2d949835d5701d832fc5acd05c0bfdad5e89cfdd074a171411f5ccad5/ruff-0.8.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2029b8c22da147c50ae577e621a5bfbc5d1fed75d86af53643d7a7aee1d23871", size = 10779153 }, + { url = "https://files.pythonhosted.org/packages/7a/25/9c11dca9404ef1eb24833f780146236131a3c7941de394bc356912ef1041/ruff-0.8.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2666520828dee7dfc7e47ee4ea0d928f40de72056d929a7c5292d95071d881d1", size = 10304387 }, + { url = "https://files.pythonhosted.org/packages/c8/b9/84c323780db1b06feae603a707d82dbbd85955c8c917738571c65d7d5aff/ruff-0.8.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:333c57013ef8c97a53892aa56042831c372e0bb1785ab7026187b7abd0135ad5", size = 11360351 }, + { url = "https://files.pythonhosted.org/packages/6b/e1/9d4bbb2ace7aad14ded20e4674a48cda5b902aed7a1b14e6b028067060c4/ruff-0.8.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:288326162804f34088ac007139488dcb43de590a5ccfec3166396530b58fb89d", size = 12022879 }, + { url = "https://files.pythonhosted.org/packages/75/28/752ff6120c0e7f9981bc4bc275d540c7f36db1379ba9db9142f69c88db21/ruff-0.8.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b12c39b9448632284561cbf4191aa1b005882acbc81900ffa9f9f471c8ff7e26", size = 11610354 }, + { url = "https://files.pythonhosted.org/packages/ba/8c/967b61c2cc8ebd1df877607fbe462bc1e1220b4a30ae3352648aec8c24bd/ruff-0.8.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:364e6674450cbac8e998f7b30639040c99d81dfb5bbc6dfad69bc7a8f916b3d1", size = 12813976 }, + { url = "https://files.pythonhosted.org/packages/7f/29/e059f945d6bd2d90213387b8c360187f2fefc989ddcee6bbf3c241329b92/ruff-0.8.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b22346f845fec132aa39cd29acb94451d030c10874408dbf776af3aaeb53284c", size = 11154564 }, + { url = "https://files.pythonhosted.org/packages/55/47/cbd05e5a62f3fb4c072bc65c1e8fd709924cad1c7ec60a1000d1e4ee8307/ruff-0.8.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b2f2f7a7e7648a2bfe6ead4e0a16745db956da0e3a231ad443d2a66a105c04fa", size = 10760604 }, + { url = "https://files.pythonhosted.org/packages/bb/ee/4c3981c47147c72647a198a94202633130cfda0fc95cd863a553b6f65c6a/ruff-0.8.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:adf314fc458374c25c5c4a4a9270c3e8a6a807b1bec018cfa2813d6546215540", size = 10391071 }, + { url = "https://files.pythonhosted.org/packages/6b/e6/083eb61300214590b188616a8ac6ae1ef5730a0974240fb4bec9c17de78b/ruff-0.8.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a885d68342a231b5ba4d30b8c6e1b1ee3a65cf37e3d29b3c74069cdf1ee1e3c9", size = 10896657 }, + { url = "https://files.pythonhosted.org/packages/77/bd/aacdb8285d10f1b943dbeb818968efca35459afc29f66ae3bd4596fbf954/ruff-0.8.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d2c16e3508c8cc73e96aa5127d0df8913d2290098f776416a4b157657bee44c5", size = 11228362 }, + { url = "https://files.pythonhosted.org/packages/39/72/fcb7ad41947f38b4eaa702aca0a361af0e9c2bf671d7fd964480670c297e/ruff-0.8.1-py3-none-win32.whl", hash = "sha256:93335cd7c0eaedb44882d75a7acb7df4b77cd7cd0d2255c93b28791716e81790", size = 8803476 }, + { url = "https://files.pythonhosted.org/packages/e4/ea/cae9aeb0f4822c44651c8407baacdb2e5b4dcd7b31a84e1c5df33aa2cc20/ruff-0.8.1-py3-none-win_amd64.whl", hash = "sha256:2954cdbe8dfd8ab359d4a30cd971b589d335a44d444b6ca2cb3d1da21b75e4b6", size = 9614463 }, + { url = "https://files.pythonhosted.org/packages/eb/76/fbb4bd23dfb48fa7758d35b744413b650a9fd2ddd93bca77e30376864414/ruff-0.8.1-py3-none-win_arm64.whl", hash = "sha256:55873cc1a473e5ac129d15eccb3c008c096b94809d693fc7053f588b67822737", size = 8959621 }, +] + +[[package]] +name = "s3transfer" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/0a/1cdbabf9edd0ea7747efdf6c9ab4e7061b085aa7f9bfc36bb1601563b069/s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7", size = 145287 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/05/7957af15543b8c9799209506df4660cba7afc4cf94bfb60513827e96bed6/s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e", size = 83175 }, +] + +[[package]] +name = "six" +version = "1.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", size = 34041 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", size = 11053 }, +] + +[[package]] +name = "types-awscrt" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/9a/534fdab439cc6c6d3a9756dd5883b71cb3cb0f7359c6119be79770db1210/types_awscrt-0.23.1.tar.gz", hash = "sha256:a20b425dabb258bc3d07a5e7de503fd9558dd1542d72de796e74e402c6d493b2", size = 14942 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/34/ab41dec9a871665330daf279d3fb5eb0eb0b2bd9e6a8ad8faac10496fed3/types_awscrt-0.23.1-py3-none-any.whl", hash = "sha256:0d362a5d62d68ca4216f458172f41c1123ec04791d68364de8ee8b61b528b262", size = 18452 }, +] + +[[package]] +name = "types-s3transfer" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/8f/5cf8bea1470f9d0af8a8a8e232bc9d94eb2b8c040f1c19e673fcd3ba488c/types_s3transfer-0.10.4.tar.gz", hash = "sha256:03123477e3064c81efe712bf9d372c7c72f2790711431f9baa59cf96ea607267", size = 13791 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/de/38872bc9414018e223a4c7193bc2f7ed5ef8ab7a01ab3bb8d7de4f3c2720/types_s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:22ac1aabc98f9d7f2928eb3fb4d5c02bf7435687f0913345a97dd3b84d0c217d", size = 18744 }, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, +] + +[[package]] +name = "urllib3" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/63/22ba4ebfe7430b76388e7cd448d5478814d3032121827c12a2cc287e2260/urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9", size = 300677 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", size = 126338 }, +]