Skip to content

Commit

Permalink
Merge branch 'main' into update-otel-collector-dependencies-0.115.0
Browse files Browse the repository at this point in the history
  • Loading branch information
jackgopack4 authored Dec 10, 2024
2 parents bb1f5ad + 5ff3a47 commit 27a8c30
Show file tree
Hide file tree
Showing 189 changed files with 2,106 additions and 1,377 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@
/.gitlab/binary_build/system_probe.yml @DataDog/ebpf-platform @DataDog/agent-delivery
/.gitlab/binary_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent

/.gitlab/benchmarks/ @DataDog/agent-devx-infra @DataDog/apm-reliability-and-performance @DataDog/agent-apm
/.gitlab/benchmarks/ @DataDog/agent-devx-infra @DataDog/apm-ecosystems-performance @DataDog/agent-apm

/.gitlab/deploy_containers/ @DataDog/container-integrations @DataDog/agent-delivery
/.gitlab/deploy_dca/ @DataDog/container-integrations @DataDog/agent-delivery
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/assign_issue.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,6 @@ jobs:
- name: Assign issue
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_API_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }}
run: |
inv -e issue.assign-owner -i ${{ github.event.issue.number }}
2 changes: 2 additions & 0 deletions .github/workflows/report-merged-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ name: Report Merged PR

on:
pull_request:
branches:
- main
types: [closed]

permissions: {}
Expand Down
27 changes: 22 additions & 5 deletions .gitlab/e2e/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,16 @@
- $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $?
- export AWS_PROFILE=agent-qa-ci
# Now all `aws` commands target the agent-qa profile
- $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_PUBLIC_KEY_PATH || exit $?
- touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_PRIVATE_KEY_PATH || exit $?
# TODO: ADXT-768: Create new secret with different ssh key for the different cloud providers
# SSH Key retrieval for AWS
- $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_AWS_PUBLIC_KEY_PATH || exit $?
- touch $E2E_AWS_PRIVATE_KEY_PATH && chmod 600 $E2E_AWS_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_AWS_PRIVATE_KEY_PATH || exit $?
# SSH Key retrieval for Azure
- $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_AZURE_PUBLIC_KEY_PATH || exit $?
- touch $E2E_AZURE_PRIVATE_KEY_PATH && chmod 600 $E2E_AZURE_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_AZURE_PRIVATE_KEY_PATH || exit $?
# SSH Key retrieval for GCP
- $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_GCP_PUBLIC_KEY_PATH || exit $?
- touch $E2E_GCP_PRIVATE_KEY_PATH && chmod 600 $E2E_GCP_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_GCP_PRIVATE_KEY_PATH || exit $?
# Use S3 backend
- pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
# Setup Azure credentials. https://www.pulumi.com/registry/packages/azure-native/installation-configuration/#set-configuration-using-pulumi-config
Expand All @@ -35,15 +43,23 @@
KUBERNETES_MEMORY_REQUEST: 12Gi
KUBERNETES_MEMORY_LIMIT: 16Gi
KUBERNETES_CPU_REQUEST: 6
E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub
E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
# AWS SSH Key configuration
E2E_AWS_PUBLIC_KEY_PATH: /tmp/agent-qa-aws-ssh-key.pub
E2E_AWS_PRIVATE_KEY_PATH: /tmp/agent-qa-aws-ssh-key
E2E_KEY_PAIR_NAME: datadog-agent-ci-rsa
# Azure SSH Key configuration
E2E_AZURE_PUBLIC_KEY_PATH: /tmp/agent-qa-azure-ssh-key.pub
E2E_AZURE_PRIVATE_KEY_PATH: /tmp/agent-qa-azure-ssh-key
# GCP SSH Key configuration
E2E_GCP_PUBLIC_KEY_PATH: /tmp/agent-qa-gcp-ssh-key.pub
E2E_GCP_PRIVATE_KEY_PATH: /tmp/agent-qa-gcp-ssh-key
E2E_PIPELINE_ID: $CI_PIPELINE_ID
E2E_COMMIT_SHA: $CI_COMMIT_SHORT_SHA
E2E_OUTPUT_DIR: $CI_PROJECT_DIR/e2e-output
EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
E2E_LOGS_PROCESSING_TEST_DEPTH: 1
script:
- inv -e new-e2e-tests.run --targets $TARGETS -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password) --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer
- inv -e new-e2e-tests.run --targets $TARGETS -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password) --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer --logs-folder=$E2E_OUTPUT_DIR/logs --logs-post-processing --logs-post-processing-test-depth=$E2E_LOGS_PROCESSING_TEST_DEPTH
after_script:
- $CI_PROJECT_DIR/tools/ci/junit_upload.sh
artifacts:
Expand Down Expand Up @@ -388,6 +404,7 @@ new-e2e-installer:
TARGETS: ./tests/installer/unix
TEAM: fleet
FLEET_INSTALL_METHOD: "install_script"
E2E_LOGS_PROCESSING_TEST_DEPTH: 2

new-e2e-installer-windows:
extends: .new_e2e_template
Expand Down
5 changes: 5 additions & 0 deletions .gitlab/e2e_install_packages/common.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,23 @@
TARGETS: ./tests/agent-platform/install-script
TEAM: agent-delivery
EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --arch $E2E_ARCH --flavor $FLAVOR --no-verbose
E2E_LOGS_PROCESSING_TEST_DEPTH: 2 # We use a single test suite and run all the platforms test as subtest

.new-e2e_step_by_step:
stage: e2e_install_packages
variables:
TARGETS: ./tests/agent-platform/step-by-step
TEAM: agent-delivery
EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --arch $E2E_ARCH --flavor $FLAVOR
E2E_LOGS_PROCESSING_TEST_DEPTH: 2 # We use a single test suite and run all the platforms test as subtest

.new-e2e_script_upgrade7:
stage: e2e_install_packages
variables:
TARGETS: ./tests/agent-platform/upgrade
TEAM: agent-delivery
EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH --flavor $FLAVOR
E2E_LOGS_PROCESSING_TEST_DEPTH: 2 # We use a single test suite and run all the platforms test as subtest
parallel:
matrix:
- START_MAJOR_VERSION: [5, 6, 7]
Expand All @@ -37,6 +40,7 @@
TARGETS: ./tests/agent-platform/persisting-integrations
TEAM: agent-delivery
EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH --flavor $FLAVOR
E2E_LOGS_PROCESSING_TEST_DEPTH: 2 # We use a single test suite and run all the platforms test as subtest
script:
- DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token) || exit $?; export DATADOG_AGENT_API_KEY
- inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version 7 --test-washer
Expand All @@ -47,6 +51,7 @@
TARGETS: ./tests/agent-platform/rpm
TEAM: agent-delivery
EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH
E2E_LOGS_PROCESSING_TEST_DEPTH: 2 # We use a single test suite and run all the platforms test as subtest
script:
- DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token) || exit $?; export DATADOG_AGENT_API_KEY
- inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --test-washer
41 changes: 32 additions & 9 deletions .gitlab/kernel_matrix_testing/security_agent.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ kmt_run_secagent_tests_x64:
- "oracle_9.3"
- "rocky_8.5"
- "rocky_9.3"
- "rocky_9.4"
- "opensuse_15.3"
- "opensuse_15.5"
- "suse_12.5"
Expand Down Expand Up @@ -268,6 +269,7 @@ kmt_run_secagent_tests_x64_docker:
- "oracle_9.3"
- "rocky_8.5"
- "rocky_9.3"
- "rocky_9.4"
TEST_SET: [cws_docker]
after_script:
- !reference [.collect_outcomes_kmt]
Expand Down Expand Up @@ -301,6 +303,7 @@ kmt_run_secagent_tests_arm64:
- "oracle_9.3"
- "rocky_8.5"
- "rocky_9.3"
- "rocky_9.4"
- "opensuse_15.5"
TEST_SET: [cws_host]
after_script:
Expand Down Expand Up @@ -395,6 +398,7 @@ kmt_run_secagent_tests_arm64_docker:
- "oracle_9.3"
- "rocky_8.5"
- "rocky_9.3"
- "rocky_9.4"
TEST_SET: ["cws_docker"]
after_script:
- !reference [.collect_outcomes_kmt]
Expand All @@ -407,35 +411,54 @@ kmt_run_secagent_tests_arm64_docker:
variables:
TEST_COMPONENT: security-agent

kmt_secagent_cleanup_arm64:
when: always
.kmt_secagent_tests_join:
stage: kernel_matrix_testing_cleanup
rules: !reference [.on_security_agent_changes_or_manual]
image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
tags: ["arch:arm64"]
script:
- echo "nothing to do here"

kmt_secagent_tests_join_arm64:
extends:
- .kmt_secagent_cleanup
- .kmt_secagent_tests_join
needs:
- kmt_setup_env_secagent_arm64
- kmt_run_secagent_tests_arm64
- kmt_run_secagent_tests_arm64_ad
- kmt_run_secagent_tests_arm64_ebpfless
- kmt_run_secagent_tests_arm64_fentry
- kmt_run_secagent_tests_arm64_docker
- upload_dependencies_secagent_arm64

kmt_secagent_cleanup_arm64:
when: always
extends:
- .kmt_secagent_cleanup
needs:
- kmt_setup_env_secagent_arm64
- kmt_secagent_tests_join_arm64
- upload_secagent_tests_arm64
variables:
ARCH: arm64
INSTANCE_TYPE: "m6gd.metal"

kmt_secagent_cleanup_x64:
when: always
kmt_secagent_tests_join_x64:
extends:
- .kmt_secagent_cleanup
- .kmt_secagent_tests_join
needs:
- kmt_setup_env_secagent_x64
- kmt_run_secagent_tests_x64
- kmt_run_secagent_tests_x64_required
- kmt_run_secagent_tests_x64_ad
- kmt_run_secagent_tests_x64_ebpfless
- kmt_run_secagent_tests_x64_fentry
- kmt_run_secagent_tests_x64_docker

kmt_secagent_cleanup_x64:
when: always
extends:
- .kmt_secagent_cleanup
needs:
- kmt_setup_env_secagent_x64
- kmt_secagent_tests_join_x64
- upload_dependencies_secagent_x64
- upload_secagent_tests_x64
variables:
Expand Down
4 changes: 2 additions & 2 deletions cmd/security-agent/subcommands/runtime/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ func checkPoliciesLocal(args *checkPoliciesCliParams, writer io.Writer) error {
},
}

provider, err := rules.NewPoliciesDirProvider(args.dir, false)
provider, err := rules.NewPoliciesDirProvider(args.dir)
if err != nil {
return err
}
Expand Down Expand Up @@ -611,7 +611,7 @@ func evalRule(_ log.Component, _ config.Component, _ secrets.Component, evalArgs
},
}

provider, err := rules.NewPoliciesDirProvider(policiesDir, false)
provider, err := rules.NewPoliciesDirProvider(policiesDir)
if err != nil {
return err
}
Expand Down
7 changes: 1 addition & 6 deletions cmd/system-probe/modules/eventmonitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,9 @@ func createEventMonitorModule(_ *sysconfigtypes.Config, deps module.FactoryDepen

netconfig := netconfig.New()
if netconfig.EnableUSMEventStream {
procmonconsumer, err := createProcessMonitorConsumer(evm, netconfig)
if err != nil {
if err := createProcessMonitorConsumer(evm, netconfig); err != nil {
return nil, err
}
if procmonconsumer != nil {
evm.RegisterEventConsumer(procmonconsumer)
log.Info("USM process monitoring consumer initialized")
}
}

gpucfg := gpuconfig.New()
Expand Down
28 changes: 24 additions & 4 deletions cmd/system-probe/modules/eventmonitor_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@ import (
"github.com/DataDog/datadog-agent/cmd/system-probe/api/module"
"github.com/DataDog/datadog-agent/cmd/system-probe/config"
"github.com/DataDog/datadog-agent/pkg/eventmonitor"
"github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers"
netconfig "github.com/DataDog/datadog-agent/pkg/network/config"
usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config"
usmstate "github.com/DataDog/datadog-agent/pkg/network/usm/state"
procmon "github.com/DataDog/datadog-agent/pkg/process/monitor"
"github.com/DataDog/datadog-agent/pkg/process/monitor"
secconfig "github.com/DataDog/datadog-agent/pkg/security/config"
"github.com/DataDog/datadog-agent/pkg/util/log"
)

// EventMonitor - Event monitor Factory
Expand All @@ -28,10 +30,28 @@ var EventMonitor = module.Factory{
},
}

func createProcessMonitorConsumer(evm *eventmonitor.EventMonitor, config *netconfig.Config) (eventmonitor.EventConsumer, error) {
const (
eventMonitorID = "PROCESS_MONITOR"
eventMonitorChannelSize = 500
)

var (
eventTypes = []consumers.ProcessConsumerEventTypes{
consumers.ExecEventType,
consumers.ExitEventType,
}
)

func createProcessMonitorConsumer(evm *eventmonitor.EventMonitor, config *netconfig.Config) error {
if !usmconfig.IsUSMSupportedAndEnabled(config) || !usmconfig.NeedProcessMonitor(config) || usmstate.Get() != usmstate.Running {
return nil, nil
return nil
}

return procmon.NewProcessMonitorEventConsumer(evm)
consumer, err := consumers.NewProcessConsumer(eventMonitorID, eventMonitorChannelSize, eventTypes, evm)
if err != nil {
return err
}
monitor.InitializeEventConsumer(consumer)
log.Info("USM process monitoring consumer initialized")
return nil
}
4 changes: 2 additions & 2 deletions cmd/system-probe/modules/eventmonitor_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ var EventMonitor = module.Factory{
Fn: createEventMonitorModule,
}

func createProcessMonitorConsumer(_ *eventmonitor.EventMonitor, _ *netconfig.Config) (eventmonitor.EventConsumer, error) {
return nil, nil
func createProcessMonitorConsumer(_ *eventmonitor.EventMonitor, _ *netconfig.Config) error {
return nil
}

func createGPUProcessEventConsumer(_ *eventmonitor.EventMonitor) error {
Expand Down
4 changes: 2 additions & 2 deletions cmd/system-probe/subcommands/runtime/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,7 @@ func checkPoliciesLocal(args *checkPoliciesCliParams, writer io.Writer) error {
},
}

provider, err := rules.NewPoliciesDirProvider(args.dir, false)
provider, err := rules.NewPoliciesDirProvider(args.dir)
if err != nil {
return err
}
Expand Down Expand Up @@ -583,7 +583,7 @@ func evalRule(_ log.Component, _ config.Component, _ secrets.Component, evalArgs
},
}

provider, err := rules.NewPoliciesDirProvider(policiesDir, false)
provider, err := rules.NewPoliciesDirProvider(policiesDir)
if err != nil {
return err
}
Expand Down
6 changes: 3 additions & 3 deletions comp/api/authtoken/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,13 @@ require (
github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect
github.com/DataDog/datadog-agent/pkg/util/log/setup v0.58.0-devel // indirect
github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect
github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect
github.com/DataDog/datadog-agent/pkg/version v0.56.0 // indirect
github.com/DataDog/viper v1.13.5 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
Expand Down
6 changes: 3 additions & 3 deletions comp/core/config/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ require (
github.com/DataDog/datadog-agent/pkg/util/defaultpaths v0.0.0-00010101000000-000000000000
github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3
github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0
github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0
github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1
github.com/DataDog/viper v1.13.5
github.com/stretchr/testify v1.10.0
go.uber.org/fx v1.23.0
Expand All @@ -60,9 +60,9 @@ require (
github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect
github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect
github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
Expand Down
4 changes: 2 additions & 2 deletions comp/core/log/impl-systemprobe/systemprobe_logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2016-present Datadog, Inc.

// Package logimpl implements a component to handle logging internal to the agent for system-probe.
package logimpl
// Package systemprobeimpl implements a component to handle logging internal to the agent for system-probe.
package systemprobeimpl

import (
"context"
Expand Down
2 changes: 1 addition & 1 deletion comp/core/log/impl-systemprobe/systemprobe_logger_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2016-present Datadog, Inc.

package logimpl
package systemprobeimpl

import (
"testing"
Expand Down
Loading

0 comments on commit 27a8c30

Please sign in to comment.