From 0387b652fb2c0079844b6d339aaa9a6c88128147 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Fri, 5 Jul 2024 16:38:09 +0200 Subject: [PATCH] nixish v1 --- .envrc.example | 6 + .github/workflows/nix-auto.yml | 33 +++ .gitignore | 7 + ddsketch.nix | 32 +++ ddtrace.nix | 71 +++++++ flake.lock | 201 ++++++++++++++++++ flake.nix | 142 +++++++++++++ shell.nix | 1 + .../test_trace_distributed_propagated.json | 2 + tests/test_agent.py | 1 + tests/test_snapshot_integration.py | 5 +- treefmt.nix | 11 + use-example.Dockerfile | 3 + 13 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 .envrc.example create mode 100644 .github/workflows/nix-auto.yml create mode 100644 ddsketch.nix create mode 100644 ddtrace.nix create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 shell.nix create mode 100644 treefmt.nix create mode 100644 use-example.Dockerfile diff --git a/.envrc.example b/.envrc.example new file mode 100644 index 00000000..ce72b8fa --- /dev/null +++ b/.envrc.example @@ -0,0 +1,6 @@ +# use https://github.com/nix-community/nix-direnv to load all tools needed to develop in the repository +# $ cp .envrc.example .envrc +# $ direnv allow . +use flake --impure . + +# alternatively just use `nix develop` or `nix-shell` diff --git a/.github/workflows/nix-auto.yml b/.github/workflows/nix-auto.yml new file mode 100644 index 00000000..59b5e21b --- /dev/null +++ b/.github/workflows/nix-auto.yml @@ -0,0 +1,33 @@ +name: Nix Flake actions + +on: + pull_request: + push: + branches: + - master + - main + +jobs: + nix-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v27 + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + matrix="$(nix eval --json '.#githubActions.matrix')" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + + nix: + needs: nix-matrix + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v27 + - run: nix build -L '.#githubActions.checks.${{ matrix.attr }}' \ No newline at end of file diff --git a/.gitignore b/.gitignore index f3efc880..423803da 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,10 @@ htmlcov/ .coverage .coverage.* .cache + +# Automated dev env setup - see .envrc.example +/.envrc +/.direnv/ + +# nix +/result \ No newline at end of file diff --git a/ddsketch.nix b/ddsketch.nix new file mode 100644 index 00000000..069698d3 --- /dev/null +++ b/ddsketch.nix @@ -0,0 +1,32 @@ +{ python, pkgs, ... }: +python.pkgs.buildPythonPackage rec { + name = "ddsketch"; + version = "3.0.1"; + + src = pkgs.fetchFromGitHub { + owner = "datadog"; + repo = "sketches-py"; + rev = "refs/tags/v${version}"; + hash = "sha256-SmdKq5aXi5B3FNBxPQDNKNBujGGEPXF132YGadGFPpo="; + }; + + propagatedBuildInputs = with python.pkgs; [ + six + protobuf + setuptools + ]; + nativeBuildInputs = with python.pkgs; [ setuptools_scm ]; + checkInputs = with python.pkgs; [ + pytest + numpy + ]; + env.SETUPTOOLS_SCM_PRETEND_VERSION = version; + + pythonImportsCheck = [ "ddsketch" ]; + + postPatch = '' + patchShebangs setup.py + ls -lah + echo version=\"${version}\" > ddsketch/__version.py + ''; +} diff --git a/ddtrace.nix b/ddtrace.nix new file mode 100644 index 00000000..c6acab2a --- /dev/null +++ b/ddtrace.nix @@ -0,0 +1,71 @@ +{ + python, + pkgs, + ddsketch, + ... +}: + +let + envier = python.pkgs.buildPythonPackage rec { + pname = "envier"; + version = "0.5.2"; + + pyproject = true; + propagatedBuildInputs = with python.pkgs; [ + hatchling + hatch-vcs + ]; + + src = pkgs.fetchPypi { + inherit pname version; + sha256 = "sha256-Tn45jLCajdNgUI734SURoVI1VCbSVEuEh6NNrSfMIK0="; + }; + }; + + ddtrace = python.pkgs.buildPythonPackage rec { + pname = "ddtrace"; + version = "2.9.2"; + pyproject = true; + + nativeBuildInputs = + [ pkgs.cmake ] + ++ (with python.pkgs; [ + cmake + setuptools + setuptools_scm + cython + ]) + ++ pkgs.lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.IOKit ]; + + propagatedBuildInputs = with python.pkgs; [ + attrs + cattrs + ddsketch + envier + opentelemetry-api + protobuf + six + xmltodict + bytecode + ]; + + postPatch = '' + substituteInPlace setup.py --replace "cmake>=3.24.2,<3.28" "cmake" + + # downloading artifacts is impossible in sandboxed build + substituteInPlace setup.py --replace "cls.download_artifacts()" "pass" + + substituteInPlace pyproject.toml --replace "cmake>=3.24.2,<3.28" "cmake" + ''; + + dontUseCmakeConfigure = true; + + src = pkgs.fetchFromGitHub { + owner = "datadog"; + repo = "dd-trace-py"; + rev = "refs/tags/v${version}"; + hash = "sha256-Ax220/uBNwSZNBFYxbxAe0rmLrqYYf3a8K/PIuSE150="; + }; + }; +in +(ddtrace) diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..9157a9e0 --- /dev/null +++ b/flake.lock @@ -0,0 +1,201 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-filter": { + "locked": { + "lastModified": 1710156097, + "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=", + "owner": "numtide", + "repo": "nix-filter", + "rev": "3342559a24e85fc164b295c3444e8a139924675b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "nix-filter", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1720066371, + "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nix2containerPkg": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1712990762, + "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1712920918, + "narHash": "sha256-1yxFvUcJfUphK9V91KufIQom7gCsztza0H4Rz2VCWUU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "92323443a56f4e9fc4e4b712e3119f66d0969297", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1717179513, + "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1719690277, + "narHash": "sha256-0xSej1g7eP2kaUF+JQp8jdyNmpmCJKRpO12mKl/36Kc=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "2741b4b489b55df32afac57bc4bfd220e8bf617e", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nix-filter": "nix-filter", + "nix-github-actions": "nix-github-actions", + "nix2containerPkg": "nix2containerPkg", + "nixpkgs": "nixpkgs_2", + "treefmt-nix": "treefmt-nix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1720507012, + "narHash": "sha256-QIeZ43t9IVB4dLsFaWh2f4C7JSRfK7p+Y1U9dULsLXU=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "8b63fe8cf7892c59b3df27cbcab4d5644035d72f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..5affce91 --- /dev/null +++ b/flake.nix @@ -0,0 +1,142 @@ +{ + description = "ddapm-test-agent"; + nixConfig.bash-prompt-prefix = "\[ddapm-test-agent\] "; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/24.05"; + + flake-utils.url = "github:numtide/flake-utils"; + nix-filter.url = "github:numtide/nix-filter"; + + nix2containerPkg.url = "github:nlewo/nix2container"; + treefmt-nix.url = "github:numtide/treefmt-nix"; + + nix-github-actions.url = "github:nix-community/nix-github-actions/"; + nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = + { + self, + nixpkgs, + flake-utils, + nix-filter, + treefmt-nix, + nix2containerPkg, + nix-github-actions, + }: + (flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + python = pkgs.python312; + + treefmt = treefmt-nix.lib.evalModule pkgs ./treefmt.nix; + + nix2container = nix2containerPkg.packages.${system}.nix2container; + + ddsketch = pkgs.callPackage ./ddsketch.nix { inherit python pkgs; }; + ddtrace = pkgs.callPackage ./ddtrace.nix { inherit python pkgs ddsketch; }; + + ddapm-test-agent = python.pkgs.buildPythonApplication rec { + name = "ddapm-test-agent"; + version = "0.0.0"; + src = ./.; + + postPatch = '' + # remove riot since its not available from nixpkgs + substituteInPlace test_deps.txt --replace "riot==0.13.0" "" + ''; + + dontUseCmakeConfigure = true; + + propagatedBuildInputs = with python.pkgs; [ + aiohttp + msgpack + ddsketch + requests + yarl + ]; + nativeBuildInputs = with python.pkgs; [ + setuptools + setuptools_scm + ]; + + checkInputs = [ + python.pkgs.pytest + ddtrace + pkgs.cmake + ]; + + doCheck = false; + + installCheckPhase = '' + runHook preCheck + export TEST_AGENT="$out/bin/ddapm-test-agent" + $TEST_AGENT --version + + # use nix provided agent for testing + substituteInPlace \ + tests/test_snapshot_integration.py \ + tests/test_agent.py \ + tests/conftest.py \ + --replace "ddapm-test-agent" "$TEST_AGENT" + + ${python.pkgs.pytest}/bin/pytest -vv + + runHook postCheck + ''; + + env.SETUPTOOLS_SCM_PRETEND_VERSION = version; + }; + toolContainer = nix2container.buildImage { + name = "ghcr.io/pawelchcki/ddapm-test-agent"; + tag = "latest"; + config = { + entrypoint = [ "/bin/ddapm-test-agent" ]; + }; + + copyToRoot = pkgs.buildEnv { + name = "root"; + paths = [ ddapm-test-agent ]; + pathsToLink = [ "/bin" ]; + }; + }; + in + { + packages = { + inherit + python + ddapm-test-agent + ddtrace + ddsketch + toolContainer + ; + default = ddapm-test-agent; + reno = pkgs.reno; + }; + + formatter = treefmt.config.build.wrapper; + + checks = { + ddapm-test-agent = ddapm-test-agent.override { doCheck = true; }; + formatting = treefmt.config.build.check self; + }; + + devShells.default = pkgs.mkShell { + venvDir = "./.venv"; + nativeBuildInputs = ddapm-test-agent.nativeBuildInputs ++ [ ddapm-test-agent ]; + }; + } + )) + // { + githubActions = nix-github-actions.lib.mkGithubMatrix { + attrPrefix = ""; + checks = { + inherit (self.checks) x86_64-linux; + + aarch64-darwin = builtins.removeAttrs (self.checks.aarch64-darwin) [ "formatting" ]; + }; + }; + }; +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000..8745f502 --- /dev/null +++ b/shell.nix @@ -0,0 +1 @@ +(builtins.getFlake ("git+file://" + toString ./.)).devShells.${builtins.currentSystem}.default diff --git a/tests/integration_snapshots/test_trace_distributed_propagated.json b/tests/integration_snapshots/test_trace_distributed_propagated.json index 6f94dd85..6c350c4a 100644 --- a/tests/integration_snapshots/test_trace_distributed_propagated.json +++ b/tests/integration_snapshots/test_trace_distributed_propagated.json @@ -9,12 +9,14 @@ "type": "", "error": 0, "meta": { + "_dd.p.dm": "-3", "language": "python", "runtime-id": "ce77bb0dd15e4f36aa106905fdddbc19" }, "metrics": { "_dd.top_level": 1, "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2, "process_id": 94638 }, "duration": 162000, diff --git a/tests/test_agent.py b/tests/test_agent.py index ae4e7126..c58d4e29 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -86,6 +86,7 @@ async def test_info(agent): "endpoints": [ "/v0.4/traces", "/v0.5/traces", + "/v0.7/traces", "/v0.6/stats", "/telemetry/proxy/", "/v0.7/config", diff --git a/tests/test_snapshot_integration.py b/tests/test_snapshot_integration.py index 77e2aa5b..91f8ba1b 100644 --- a/tests/test_snapshot_integration.py +++ b/tests/test_snapshot_integration.py @@ -169,7 +169,8 @@ async def test_trace_distributed_propagated(testagent_url, testagent, tracer): resp = await testagent.get( f"{testagent_url}/test/session/snapshot?test_session_token=test_trace_distributed_propagated" ) - assert resp.status == 200 + + assert resp.status == 200, await resp.text() async def test_trace_missing_received(testagent_url, testagent, tracer): @@ -245,7 +246,7 @@ async def test_tracestats( if fail: assert resp.status == 400 else: - assert resp.status == 200 + assert resp.status == 200, await resp.text() async def test_cmd(testagent_url: str, testagent: aiohttp.ClientSession, tracer: Tracer) -> None: diff --git a/treefmt.nix b/treefmt.nix new file mode 100644 index 00000000..5285b18b --- /dev/null +++ b/treefmt.nix @@ -0,0 +1,11 @@ +# treefmt.nix +{ pkgs, ... }: +{ + # Used to find the project root + projectRootFile = "flake.nix"; + # Enable the Nix formatter + programs.nixfmt.enable = true; + # Enable the Python formatter + programs.black.enable = true; + settings.formatter.black.excludes = [ "ddsketch/pb/*" ]; +} diff --git a/use-example.Dockerfile b/use-example.Dockerfile new file mode 100644 index 00000000..8a2862e3 --- /dev/null +++ b/use-example.Dockerfile @@ -0,0 +1,3 @@ +FROM bash +COPY --from=ghcr.io/pawelchcki/ddapm-test-agent / / +