Merge branch 'main' into clamoriniere/langdetection-patcher-config-cmp

.circleci/README.md

@@ -11,12 +11,12 @@ This image is now built alongside other images in [agent-buildimages](https://gi
 Once you have created a new image by building a new version of agent-buildimages, you can test your modification with the associated invoke task:
 
 ```bash
-invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5
+invoke -e buildimages.update --tag v12345678-c0mm1t5
 ```
-This will update the configuration of circleci and gitlab to use the __test version__ of these images.
-Once your test is successful, you can either move the `_test_version` from files or invoke
+This will update the configuration of circleci and gitlab to use the __test__ version of these images.
+Once your test is successful, you can either move the `_test_only` from files or invoke
 ```bash
-invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5 --no-test-version
+invoke -e buildimages.update --tag v12345678-c0mm1t5 --no-test
 ```
 
 If everything is green, get a review and merge the PR.

.copyright-overrides.yml

@@ -365,6 +365,7 @@ github.com/signalfx/sapm-proto/*: Copyright 2019 Splunk, Inc.
 go.mongodb.org/atlas/mongodbatlas: Copyright 2022 MongoDB Inc
 github.com/go-jose/go-jose/v4: Copyright 2014 Square Inc.
 github.com/gocolly/colly: Copyright 2018 Adam Tauber
+github.com/puzpuzpuz/xsync/v3: Copyright (c) 2021 Andrey Pechkurov
 
 github.com/vibrantbyte/go-antpath/antpath: vibrantbyte <[email protected]>|suchao <[email protected]>
 github.com/vibrantbyte/go-antpath/extend: vibrantbyte <[email protected]>|suchao <[email protected]>

.ddqa/config.toml

@@ -57,12 +57,20 @@ jira_statuses = ["To Do", "In Progress", "Done"]
 github_team = "universal-service-monitoring"
 github_labels = ["team/usm"]
 
-[teams."Network Device Monitoring"]
+[teams."Network Device Monitoring - Core"]
 jira_project = "NDMII"
 jira_issue_type = "Task"
 jira_statuses = ["To Do", "In Progress", "Done"]
-github_team = "network-device-monitoring"
-github_labels = ["team/network-device-monitoring"]
+github_team = "ndm-core"
+github_labels = ["team/ndm-core", "team/network-device-monitoring"]
+exclude_members = ["leeavital", "heyronhay"]
+
+[teams."Network Device Monitoring - Integrations"]
+jira_project = "NDINT"
+jira_issue_type = "Task"
+jira_statuses = ["To Do", "In Progress", "Done"]
+github_team = "ndm-integrations"
+github_labels = ["team/ndm-integrations"]
 exclude_members = ["leeavital", "heyronhay"]
 
 [teams."Network Performance Monitoring"]
@@ -203,9 +211,9 @@ jira_statuses = ["💼 To Do", "💡 In Progress", "✅ Done"]
 github_team = "container-app"
 github_labels = ["team/container-app"]
 
-[teams."APM Onboarding"]
-jira_project = "APMON"
-jira_issue_type = "Story"
+[teams."APM Injection Platform"]
+jira_project = "INPLAT"
+jira_issue_type = "Task"
 jira_statuses = ["Selected For Development", "In Progress", "Done"]
 github_team = "apm-onboarding"
 github_labels = ["team/apm-onboarding"]
@@ -222,4 +230,4 @@ jira_project = "DEBUG"
 jira_issue_type = "Task"
 jira_statuses = ["To Do", "In Progress", "Done"]
 github_team = "debugger-go"
-github_labels = ["team/dynamic-instrumentation"]
+github_labels = ["team/dynamic-instrumentation"]

.github/CODEOWNERS

@@ -139,10 +139,8 @@
 /.gitlab/container_build/fakeintake.yml              @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 
-/.gitlab/functional_test/security_agent.yml          @DataDog/agent-security @DataDog/agent-devx-infra
 /.gitlab/functional_test/serverless.yml              @DataDog/serverless @Datadog/serverless-aws @DataDog/agent-devx-infra
 /.gitlab/functional_test_cleanup                     @DataDog/agent-security @DataDog/windows-kernel-integrations @DataDog/agent-devx-infra
-/.gitlab/functional_test/common.yml                  @DataDog/agent-devx-infra @DataDog/windows-kernel-integrations
 /.gitlab/functional_test/oracle.yml                  @DataDog/agent-devx-infra @DataDog/database-monitoring
 
 /.gitlab/powershell_script_deploy                    @DataDog/agent-delivery @DataDog/windows-agent
@@ -192,7 +190,7 @@
 /cmd/agent/subcommands/dogstatsd*       @DataDog/agent-metrics-logs
 /cmd/agent/subcommands/integrations     @DataDog/agent-integrations @DataDog/agent-shared-components
 /cmd/agent/subcommands/remoteconfig     @Datadog/remote-config
-/cmd/agent/subcommands/snmp             @DataDog/network-device-monitoring
+/cmd/agent/subcommands/snmp             @DataDog/ndm-core
 /cmd/agent/subcommands/streamlogs       @DataDog/agent-metrics-logs
 /cmd/agent/subcommands/streamep         @DataDog/container-integrations
 /cmd/agent/subcommands/taggerlist       @DataDog/container-platform
@@ -209,7 +207,7 @@
 /cmd/agent/dist/conf.d/oracle-dbm.d/    @DataDog/database-monitoring
 /cmd/agent/dist/conf.d/network_path.d/  @DataDog/Networks @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/sbom.d/          @DataDog/container-integrations
-/cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
+/cmd/agent/dist/conf.d/snmp.d/          @DataDog/ndm-core
 /cmd/agent/dist/conf.d/win32_event_log.d/ @DataDog/windows-agent
 /cmd/agent/install*.sh                  @DataDog/container-ecosystems @DataDog/agent-delivery
 /cmd/cluster-agent/                     @DataDog/container-platform
@@ -232,7 +230,7 @@
 /cmd/system-probe/modules/eventmonitor* @DataDog/agent-security
 /cmd/system-probe/modules/tcp_queue_tracer* @DataDog/container-integrations
 /cmd/system-probe/modules/traceroute*   @DataDog/network-device-monitoring @Datadog/Networks
-/cmd/system-probe/modules/ping*         @DataDog/network-device-monitoring
+/cmd/system-probe/modules/ping*         @DataDog/ndm-core
 /cmd/system-probe/modules/language_detection* @DataDog/processes @DataDog/universal-service-monitoring
 /cmd/system-probe/modules/dynamic_instrumentation* @DataDog/debugger
 /cmd/system-probe/windows_resources/    @DataDog/windows-kernel-integrations
@@ -276,7 +274,7 @@
 /omnibus/config/software/datadog-security-agent*.rb       @Datadog/agent-security @DataDog/agent-delivery
 /omnibus/config/software/openscap.rb                      @DataDog/agent-cspm
 /omnibus/config/software/sds.rb                           @DataDog/agent-processing-and-routing
-/omnibus/config/software/snmp-traps.rb                    @DataDog/network-device-monitoring
+/omnibus/config/software/snmp-traps.rb                    @DataDog/ndm-core
 /omnibus/resources/*/msi/                                 @DataDog/windows-agent
 
 # The following is managed by `inv lint-components` -- DO NOT EDIT
@@ -292,13 +290,13 @@
 /comp/forwarder @DataDog/agent-processing-and-routing
 /comp/logs @DataDog/agent-metrics-logs
 /comp/metadata @DataDog/agent-shared-components
-/comp/ndmtmp @DataDog/network-device-monitoring
-/comp/netflow @DataDog/network-device-monitoring
+/comp/ndmtmp @DataDog/ndm-core
+/comp/netflow @DataDog/ndm-integrations
 /comp/networkpath @DataDog/Networks @DataDog/network-device-monitoring
 /comp/otelcol @DataDog/opentelemetry
 /comp/process @DataDog/processes
 /comp/remote-config @DataDog/remote-config
-/comp/snmptraps @DataDog/network-device-monitoring
+/comp/snmptraps @DataDog/ndm-core
 /comp/systray @DataDog/windows-agent
 /comp/trace @DataDog/agent-apm
 /comp/updater @DataDog/fleet @DataDog/windows-agent
@@ -318,9 +316,9 @@
 /comp/autoscaling/datadogclient @DataDog/container-integrations
 /comp/etw @DataDog/windows-agent
 /comp/languagedetection/client @DataDog/container-platform
-/comp/rdnsquerier @DataDog/network-device-monitoring
+/comp/rdnsquerier @DataDog/ndm-integrations
 /comp/serializer/compression @DataDog/agent-metrics-logs
-/comp/snmpscan @DataDog/network-device-monitoring
+/comp/snmpscan @DataDog/ndm-core
 # END COMPONENTS
 
 # pkg
@@ -356,7 +354,7 @@
 /pkg/trace/transform/                   @DataDog/opentelemetry
 /comp/core/autodiscovery/listeners/           @DataDog/container-platform
 /comp/core/autodiscovery/listeners/cloudfoundry*.go  @DataDog/platform-integrations
-/comp/core/autodiscovery/listeners/snmp*.go   @DataDog/network-device-monitoring
+/comp/core/autodiscovery/listeners/snmp*.go   @DataDog/ndm-core
 /comp/core/autodiscovery/providers/           @DataDog/container-platform
 /comp/core/autodiscovery/providers/file*.go   @DataDog/agent-metrics-logs
 /comp/core/autodiscovery/providers/config_reader*.go @DataDog/container-platform @DataDog/agent-metrics-logs
@@ -383,13 +381,13 @@
 /pkg/collector/corechecks/embed/apm/               @DataDog/agent-apm
 /pkg/collector/corechecks/embed/process/           @DataDog/processes
 /pkg/collector/corechecks/gpu/                     @DataDog/ebpf-platform
-/pkg/collector/corechecks/network-devices/         @DataDog/network-device-monitoring
+/pkg/collector/corechecks/network-devices/         @DataDog/ndm-integrations
 /pkg/collector/corechecks/orchestrator/   @DataDog/container-app
 /pkg/collector/corechecks/net/          @DataDog/platform-integrations
 /pkg/collector/corechecks/oracle        @DataDog/database-monitoring
 /pkg/collector/corechecks/sbom/         @DataDog/container-integrations
 /pkg/collector/corechecks/servicediscovery/ @DataDog/universal-service-monitoring
-/pkg/collector/corechecks/snmp/         @DataDog/network-device-monitoring
+/pkg/collector/corechecks/snmp/         @DataDog/ndm-core
 /pkg/collector/corechecks/system/                 @DataDog/platform-integrations
 /pkg/collector/corechecks/system/**/*_windows*.go @DataDog/platform-integrations @DataDog/windows-agent
 /pkg/collector/corechecks/system/wincrashdetect/  @DataDog/windows-kernel-integrations
@@ -421,6 +419,7 @@
 /pkg/flare/*_win.go                     @Datadog/windows-agent
 /pkg/flare/*_windows.go                 @Datadog/windows-agent
 /pkg/flare/*_windows_test.go            @Datadog/windows-agent
+/pkg/flare/sysprobe                     @DataDog/Networks @Datadog/windows-agent @DataDog/processes @DataDog/agent-security @DataDog/universal-service-monitoring @DataDog/ebpf-platform
 /pkg/fleet/                             @DataDog/fleet @DataDog/windows-agent
 /pkg/pidfile/                           @DataDog/agent-shared-components
 /pkg/persistentcache/                   @DataDog/agent-metrics-logs
@@ -442,10 +441,10 @@
 /pkg/util/ecs/                          @DataDog/container-integrations
 /pkg/util/funcs/                        @DataDog/ebpf-platform
 /pkg/util/kernel/                       @DataDog/ebpf-platform
+/pkg/util/ktime                         @DataDog/agent-security
 /pkg/util/kubernetes/                   @DataDog/container-integrations @DataDog/container-platform @DataDog/container-app
 /pkg/util/podman/                       @DataDog/container-integrations
 /pkg/util/prometheus                    @DataDog/container-integrations
-/pkg/util/tagger                        @DataDog/container-platform
 /pkg/util/trivy/                        @DataDog/container-integrations @DataDog/agent-security
 /pkg/util/uuid/                         @DataDog/agent-shared-components
 /pkg/util/cgroups/                      @DataDog/container-integrations
@@ -521,8 +520,8 @@
 /pkg/databasemonitoring                 @DataDog/database-monitoring
 /pkg/kubestatemetrics                   @DataDog/container-integrations
 /pkg/security/                          @DataDog/agent-security
-/pkg/networkdevice/                     @DataDog/network-device-monitoring
-/pkg/snmp/                              @DataDog/network-device-monitoring
+/pkg/networkdevice/                     @DataDog/ndm-core
+/pkg/snmp/                              @DataDog/ndm-core
 /pkg/tagger/                            @DataDog/container-platform
 /pkg/windowsdriver/                     @DataDog/windows-kernel-integrations
 /comp/core/workloadmeta/collectors/internal/cloudfoundry @DataDog/platform-integrations
@@ -560,6 +559,7 @@
 /tasks/windows_resources.py             @DataDog/windows-agent
 /tasks/components.py                    @DataDog/agent-shared-components
 /tasks/components_templates             @DataDog/agent-shared-components
+/tasks/libs/ciproviders/                @DataDog/agent-devx-infra
 /tasks/libs/common/omnibus.py           @DataDog/agent-delivery
 /tasks/omnibus.py                       @DataDog/agent-delivery
 /tasks/unit_tests/components_tests.py   @DataDog/agent-shared-components
@@ -576,9 +576,6 @@
 /test/kitchen/                          @DataDog/agent-devx-loops
 /test/kitchen/test-definitions/         @DataDog/container-ecosystems @DataDog/agent-delivery
 /test/kitchen/test/integration/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/site-cookbooks/dd-security-agent-check/ @DataDog/agent-security
-/test/kitchen/test/integration/security-agent-stress/ @DataDog/agent-security
-/test/kitchen/test/integration/security-agent-test/ @DataDog/agent-security
 /test/kitchen/site-cookbooks/dd-system-probe-check/ @DataDog/windows-kernel-integrations
 /test/kitchen/test/integration/win-all-subservices/ @DataDog/windows-agent
 /test/kitchen/test/integration/win-alt-dir/ @DataDog/windows-agent
@@ -589,8 +586,8 @@
 /test/kitchen/test/integration/win-repair/ @DataDog/windows-agent
 /test/kitchen/test/integration/win-user/ @DataDog/windows-agent
 /test/fakeintake/                             @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
-/test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/network-device-monitoring
-/test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/network-device-monitoring
+/test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/ndm-integrations
+/test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/ndm-integrations
 /test/fakeintake/aggregator/servicediscovery* @DataDog/universal-service-monitoring
 /test/new-e2e/                                @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/new-e2e/pkg/components/datadog-installer @DataDog/windows-agent
@@ -603,7 +600,8 @@
 /test/new-e2e/tests/containers                @DataDog/container-integrations @DataDog/container-platform
 /test/new-e2e/tests/discovery                 @DataDog/universal-service-monitoring
 /test/new-e2e/tests/language-detection        @DataDog/processes
-/test/new-e2e/tests/ndm                       @DataDog/network-device-monitoring
+/test/new-e2e/tests/ndm                       @DataDog/ndm-core
+/test/new-e2e/tests/ndm/netflow               @DataDog/ndm-integrations
 /test/new-e2e/tests/npm                       @DataDog/Networks
 /test/new-e2e/tests/npm/ec2_1host_wkit_test.go @DataDog/Networks @DataDog/windows-kernel-integrations
 /test/new-e2e/tests/orchestrator              @DataDog/container-app

.github/dependabot.yaml

@@ -37,6 +37,9 @@ updates:
       bun:
         patterns:
           - "github.com/uptrace/bun*"
+      aws-sdk-go-v2:
+        patterns:
+          - "github.com/aws/aws-sdk-go-v2*"
   - package-ecosystem: gomod
     directory: /pkg/trace
     labels:
@@ -157,18 +160,6 @@ updates:
     schedule:
       interval: weekly
     open-pull-requests-limit: 100
-  - package-ecosystem: pip
-    directory: /test/e2e/cws-tests
-    labels:
-      - dependencies
-      - python
-      - team/agent-security
-      - changelog/no-changelog
-      - qa/no-code-change
-      - dev/tooling
-    schedule:
-      interval: monthly
-    open-pull-requests-limit: 100
   - package-ecosystem: docker
     directory: /test/fakeintake
     labels:

.github/workflows/buildimages-update.yml

@@ -78,16 +78,16 @@ jobs:
       - name: Get current buildimage tag
         id: current_buildimage_tag
         run: |
-            echo "BUILDIMAGE_TAG=$(inv pipeline.get-gitlab-config-image-tag)" >> $GITHUB_OUTPUT
+            echo "BUILDIMAGE_TAG=$(inv buildimages.get-tag)" >> $GITHUB_OUTPUT
 
       - name: Update buildimages IDs and Go version
         id: update_build_images
         env:
-          TEST_VERSION_FLAG: ${{ inputs.test_version && '--test-version' || '--no-test-version' }}
+          TEST_VERSION_FLAG: ${{ inputs.test_version && '--test' || '--no-test' }}
           INCLUDE_OTEL_MODULES: ${{ inputs.include_otel_modules && '--include-otel-modules' || '' }}
         run: |
           if [ "${{ steps.current_go_version.outputs.GO_VERSION }}" = "${{ inputs.go_version }}" ]; then
-            inv -e buildimages.update --image-tag ${{ inputs.images_id }} $TEST_VERSION_FLAG
+            inv -e buildimages.update --tag ${{ inputs.images_id }} $TEST_VERSION_FLAG
             echo 'MESSAGE=Update buildimages ID to ${{ inputs.images_id }}' >> $GITHUB_OUTPUT
           else
             inv -e update-go --image-tag ${{ inputs.images_id }} $TEST_VERSION_FLAG $INCLUDE_OTEL_MODULES -v "${{ inputs.go_version }}"
@@ -140,7 +140,7 @@ jobs:
           ${{ inputs.images_id }} \
           ${{ steps.current_go_version.outputs.GO_VERSION }} \
           ${{ inputs.go_version }} \
-          ${{ inputs.test_version && '--test-version' || '' }} > $TMP_PR_BODY_PATH
+          ${{ inputs.test_version && '--test' || '' }} > $TMP_PR_BODY_PATH
 
           # Create the PR
           gh pr create \

.github/workflows/code_review_complexity.yml

@@ -0,0 +1,34 @@
+---
+name: "PR complexity label"
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - main
+      - "[0-9]+.[0-9]+.x"
+  pull_request_review_comment:
+    types: [created, deleted]
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+permissions: {}
+jobs:
+  codereview-complexity:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - name: Setup python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version: 3.12
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
+      - name: Check code review complexity
+        run: inv -e github.assign-codereview-label --pr-id='${{ github.event.pull_request.number }}'