From caf881cfc72ca88f0568c7586bc58aa9dd97de90 Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Fri, 6 Dec 2024 14:16:25 +0100 Subject: [PATCH 01/52] upgrade(installer): Uninstall deb/rpm agent when installing the OCI (#31784) --- pkg/fleet/installer/installer.go | 19 +++++-- pkg/fleet/installer/packages/datadog_agent.go | 20 ++++++-- .../packages/datadog_agent_windows.go | 5 ++ pkg/fleet/installer/packages/pkg_manager.go | 51 +++++++++++++++++++ test/new-e2e/tests/installer/host/host.go | 6 ++- .../installer/unix/package_agent_test.go | 8 ++- 6 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 pkg/fleet/installer/packages/pkg_manager.go diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index 8000be3c5a261..d7cca9819b4da 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -160,7 +160,7 @@ func (i *installerImpl) IsInstalled(_ context.Context, pkg string) (bool, error) func (i *installerImpl) Install(ctx context.Context, url string, args []string) error { i.m.Lock() defer i.m.Unlock() - pkg, err := i.downloader.Download(ctx, url) + pkg, err := i.downloader.Download(ctx, url) // Downloads pkg metadata only if err != nil { return fmt.Errorf("could not download package: %w", err) } @@ -169,6 +169,10 @@ func (i *installerImpl) Install(ctx context.Context, url string, args []string) span.SetTag(ext.ResourceName, pkg.Name) span.SetTag("package_version", pkg.Version) } + err = i.preparePackage(ctx, pkg.Name, args) // Preinst + if err != nil { + return fmt.Errorf("could not prepare package: %w", err) + } dbPkg, err := i.db.GetPackage(pkg.Name) if err != nil && !errors.Is(err, db.ErrPackageNotFound) { return fmt.Errorf("could not get package: %w", err) @@ -203,11 +207,11 @@ func (i *installerImpl) Install(ctx context.Context, url string, args []string) if err != nil { return fmt.Errorf("could not create repository: %w", err) } - err = i.configurePackage(ctx, pkg.Name) + err = i.configurePackage(ctx, pkg.Name) // Config if err != nil { return fmt.Errorf("could not configure package: %w", err) } - err = i.setupPackage(ctx, pkg.Name, args) + err = i.setupPackage(ctx, pkg.Name, args) // Postinst if err != nil { return fmt.Errorf("could not setup package: %w", err) } @@ -615,6 +619,15 @@ func (i *installerImpl) promoteExperiment(ctx context.Context, pkg string) error } } +func (i *installerImpl) preparePackage(ctx context.Context, pkg string, _ []string) error { + switch pkg { + case packageDatadogAgent: + return packages.PrepareAgent(ctx) + default: + return nil + } +} + func (i *installerImpl) setupPackage(ctx context.Context, pkg string, args []string) error { switch pkg { case packageDatadogInstaller: diff --git a/pkg/fleet/installer/packages/datadog_agent.go b/pkg/fleet/installer/packages/datadog_agent.go index ad8b7c4fade4c..7236096bbf1e1 100644 --- a/pkg/fleet/installer/packages/datadog_agent.go +++ b/pkg/fleet/installer/packages/datadog_agent.go @@ -64,6 +64,22 @@ var ( } ) +// PrepareAgent prepares the machine to install the agent +func PrepareAgent(ctx context.Context) (err error) { + span, ctx := tracer.StartSpanFromContext(ctx, "prepare_agent") + defer func() { span.Finish(tracer.WithError(err)) }() + + // Check if the agent has been installed by a package manager, if yes remove it + if !oldAgentInstalled() { + return nil // Nothing to do + } + err = stopOldAgentUnits(ctx) + if err != nil { + return fmt.Errorf("failed to stop old agent units: %w", err) + } + return removeDebRPMPackage(ctx, agentPackage) +} + // SetupAgent installs and starts the agent func SetupAgent(ctx context.Context, _ []string) (err error) { span, ctx := tracer.StartSpanFromContext(ctx, "setup_agent") @@ -75,10 +91,6 @@ func SetupAgent(ctx context.Context, _ []string) (err error) { span.Finish(tracer.WithError(err)) }() - if err = stopOldAgentUnits(ctx); err != nil { - return err - } - for _, unit := range stableUnits { if err = loadUnit(ctx, unit); err != nil { return fmt.Errorf("failed to load %s: %v", unit, err) diff --git a/pkg/fleet/installer/packages/datadog_agent_windows.go b/pkg/fleet/installer/packages/datadog_agent_windows.go index 3000a96fe712d..d330ab434d0e0 100644 --- a/pkg/fleet/installer/packages/datadog_agent_windows.go +++ b/pkg/fleet/installer/packages/datadog_agent_windows.go @@ -21,6 +21,11 @@ const ( datadogAgent = "datadog-agent" ) +// PrepareAgent prepares the machine to install the agent +func PrepareAgent(_ context.Context) error { + return nil // No-op on Windows +} + // SetupAgent installs and starts the agent func SetupAgent(ctx context.Context, args []string) (err error) { span, _ := tracer.StartSpanFromContext(ctx, "setup_agent") diff --git a/pkg/fleet/installer/packages/pkg_manager.go b/pkg/fleet/installer/packages/pkg_manager.go new file mode 100644 index 0000000000000..e74dba210a015 --- /dev/null +++ b/pkg/fleet/installer/packages/pkg_manager.go @@ -0,0 +1,51 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !windows + +package packages + +import ( + "bytes" + "context" + "fmt" + "os/exec" + + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +// removeDebRPMPackage removes a package installed via deb/rpm package manager +// It doesn't remove dependencies or purge as we want to keep existing configuration files +// and reinstall the package using the installer. +// Note: we don't run the pre/post remove scripts as we want to avoid surprises for older agent versions (like removing config) +func removeDebRPMPackage(ctx context.Context, pkg string) (err error) { + span, _ := tracer.StartSpanFromContext(ctx, "remove_deb_rpm_package") + defer func() { span.Finish(tracer.WithError(err)) }() + // Compute the right command depending on the package manager + var cmd *exec.Cmd + if _, pathErr := exec.LookPath("dpkg"); pathErr == nil { + // Doesn't fail if the package isn't installed + cmd = exec.Command("dpkg", "-r", "--no-triggers", agentPackage) + } else if _, pathErr := exec.LookPath("rpm"); pathErr == nil { + // Check if package exist, else the command will fail + pkgErr := exec.Command("rpm", "-q", agentPackage).Run() + if pkgErr == nil { + cmd = exec.Command("rpm", "-e", "--nodeps", "--noscripts", agentPackage) + } + } + + if cmd == nil { + // If we can't find a package manager or the package is not installed, ignore this step + return nil + } + + // Run the command + var buf bytes.Buffer + cmd.Stderr = &buf + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to uninstall deb/rpm package %s (%w): %s", pkg, err, buf.String()) + } + return nil +} diff --git a/test/new-e2e/tests/installer/host/host.go b/test/new-e2e/tests/installer/host/host.go index c98684ab498fa..79b43ef730fec 100644 --- a/test/new-e2e/tests/installer/host/host.go +++ b/test/new-e2e/tests/installer/host/host.go @@ -253,7 +253,9 @@ func (h *Host) AssertPackageNotInstalledByPackageManager(pkgs ...string) { for _, pkg := range pkgs { switch h.pkgManager { case "apt": - h.remote.MustExecute("! dpkg-query -l " + pkg) + // If a package is removed but not purged, it will be in the "rc" state (opposed to "ii" for installed) + // if it's been purged, the command will return an error + h.remote.MustExecute(fmt.Sprintf("dpkg-query -l %[1]s | grep '^rc' || ! dpkg-query -l %[1]s", pkg)) case "yum", "zypper": h.remote.MustExecute("! rpm -q " + pkg) default: @@ -647,7 +649,7 @@ func (s *State) AssertDirExists(path string, perms fs.FileMode, user string, gro func (s *State) AssertPathDoesNotExist(path string) { path = evalSymlinkPath(path, s.FS) _, ok := s.FS[path] - assert.False(s.t, ok, "something exists at path", path) + assert.False(s.t, ok, "something exists at path %s", path) } // AssertFileExistsAnyUser asserts that a file exists on the host with the given perms. diff --git a/test/new-e2e/tests/installer/unix/package_agent_test.go b/test/new-e2e/tests/installer/unix/package_agent_test.go index 571d7a87eae41..70ddeff75c676 100644 --- a/test/new-e2e/tests/installer/unix/package_agent_test.go +++ b/test/new-e2e/tests/installer/unix/package_agent_test.go @@ -113,7 +113,7 @@ func (s *packageAgentSuite) TestUpgrade_AgentDebRPM_to_OCI() { state = s.host.State() s.assertUnits(state, false) s.host.AssertPackageInstalledByInstaller("datadog-agent") - s.host.AssertPackageInstalledByPackageManager("datadog-agent") + s.host.AssertPackageNotInstalledByPackageManager("datadog-agent") } // TestUpgrade_Agent_OCI_then_DebRpm agent deb/rpm install while OCI one is installed @@ -422,7 +422,7 @@ func (s *packageAgentSuite) TestUpgrade_DisabledAgentDebRPM_to_OCI() { state = s.host.State() s.assertUnits(state, false) s.host.AssertPackageInstalledByInstaller("datadog-agent") - s.host.AssertPackageInstalledByPackageManager("datadog-agent") + s.host.AssertPackageNotInstalledByPackageManager("datadog-agent") s.host.Run("sudo systemctl show datadog-agent -p ExecStart | grep /opt/datadog-packages") } @@ -430,6 +430,7 @@ func (s *packageAgentSuite) TestUpgrade_DisabledAgentDebRPM_to_OCI() { func (s *packageAgentSuite) TestInstallWithLeftoverDebDir() { // create /opt/datadog-agent to simulate a disabled agent s.host.Run("sudo mkdir -p /opt/datadog-agent") + defer func() { s.host.Run("sudo rm -rf /opt/datadog-agent") }() // install OCI agent s.RunInstallScript(envForceInstall("datadog-agent")) @@ -451,6 +452,9 @@ func (s *packageAgentSuite) purgeAgentDebInstall() { default: s.T().Fatalf("unsupported package manager: %s", pkgManager) } + // Make sure everything is cleaned up -- there are tests where the package is + // removed but not purged so the directory remains + s.Env().RemoteHost.Execute("sudo rm -rf /opt/datadog-agent") } func (s *packageAgentSuite) installDebRPMAgent() { From 68415f12658292cb1773a0e7998b6712e5abbc0e Mon Sep 17 00:00:00 2001 From: Alexandre Yang Date: Fri, 6 Dec 2024 14:54:31 +0100 Subject: [PATCH 02/52] [HA Agent] Add datadog.agent.ha_agent.running metric (#31782) --- pkg/aggregator/aggregator.go | 13 ++++++ pkg/aggregator/aggregator_test.go | 72 +++++++++++++++++++++++++++++-- pkg/aggregator/sender_test.go | 2 +- 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go index e8ac4fd425d13..653a031194819 100644 --- a/pkg/aggregator/aggregator.go +++ b/pkg/aggregator/aggregator.go @@ -610,6 +610,19 @@ func (agg *BufferedAggregator) appendDefaultSeries(start time.Time, series metri SourceTypeName: "System", }) + if agg.haAgent.Enabled() { + haAgentTags := append(agg.tags(false), "agent_state:"+string(agg.haAgent.GetState())) + // Send along a metric to show if HA Agent is running with agent_state tag. + series.Append(&metrics.Serie{ + Name: fmt.Sprintf("datadog.%s.ha_agent.running", agg.agentName), + Points: []metrics.Point{{Value: float64(1), Ts: float64(start.Unix())}}, + Tags: tagset.CompositeTagsFromSlice(haAgentTags), + Host: agg.hostname, + MType: metrics.APIGaugeType, + SourceTypeName: "System", + }) + } + // Send along a metric that counts the number of times we dropped some payloads because we couldn't split them. series.Append(&metrics.Serie{ Name: fmt.Sprintf("n_o_i_n_d_e_x.datadog.%s.payload.dropped", agg.agentName), diff --git a/pkg/aggregator/aggregator_test.go b/pkg/aggregator/aggregator_test.go index a60e3055f2206..5a0676cbcb507 100644 --- a/pkg/aggregator/aggregator_test.go +++ b/pkg/aggregator/aggregator_test.go @@ -27,6 +27,7 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" orchestratorforwarder "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator" + haagent "github.com/DataDog/datadog-agent/comp/haagent/def" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" @@ -57,8 +58,11 @@ func initF() { tagsetTlm.reset() } -func testNewFlushTrigger(start time.Time, waitForSerializer bool) flushTrigger { - seriesSink := metrics.NewIterableSeries(func(_ *metrics.Serie) {}, 1000, 1000) +func testNewFlushTrigger(start time.Time, waitForSerializer bool, callback func(_ *metrics.Serie)) flushTrigger { + if callback == nil { + callback = func(_ *metrics.Serie) {} + } + seriesSink := metrics.NewIterableSeries(callback, 1000, 1000) flushedSketches := make(metrics.SketchSeriesList, 0) return flushTrigger{ @@ -130,7 +134,7 @@ func TestDeregisterCheckSampler(t *testing.T) { return agg.checkSamplers[checkID1].deregistered && !agg.checkSamplers[checkID2].deregistered }, time.Second, 10*time.Millisecond) - agg.Flush(testNewFlushTrigger(time.Now(), false)) + agg.Flush(testNewFlushTrigger(time.Now(), false, nil)) agg.mu.Lock() require.Len(t, agg.checkSamplers, 1) @@ -265,7 +269,7 @@ func TestDefaultData(t *testing.T) { s.On("SendSeries", series).Return(nil).Times(1) - agg.Flush(testNewFlushTrigger(start, false)) + agg.Flush(testNewFlushTrigger(start, false, nil)) s.AssertNotCalled(t, "SendEvents") s.AssertNotCalled(t, "SendSketch") @@ -273,6 +277,66 @@ func TestDefaultData(t *testing.T) { assert.Equal(t, uint64(0), tagsetTlm.hugeSeriesCount[0].Load()) } +func TestDefaultSeries(t *testing.T) { + s := &MockSerializerIterableSerie{} + taggerComponent := taggerMock.SetupFakeTagger(t) + + mockHaAgent := haagentmock.NewMockHaAgent().(haagentmock.Component) + mockHaAgent.SetEnabled(true) + mockHaAgent.SetState(haagent.Active) + + agg := NewBufferedAggregator(s, nil, mockHaAgent, taggerComponent, "hostname", DefaultFlushInterval) + + start := time.Now() + + // Check only the name for `datadog.agent.up` as the timestamp may not be the same. + agentUpMatcher := mock.MatchedBy(func(m servicecheck.ServiceChecks) bool { + require.Equal(t, 1, len(m)) + require.Equal(t, "datadog.agent.up", m[0].CheckName) + require.Equal(t, servicecheck.ServiceCheckOK, m[0].Status) + require.Equal(t, []string{"agent_group:group01"}, m[0].Tags) + require.Equal(t, agg.hostname, m[0].Host) + + return true + }) + s.On("SendServiceChecks", agentUpMatcher).Return(nil).Times(1) + + expectedSeries := metrics.Series{&metrics.Serie{ + Name: fmt.Sprintf("datadog.%s.running", flavor.GetFlavor()), + Points: []metrics.Point{{Value: 1, Ts: float64(start.Unix())}}, + Tags: tagset.CompositeTagsFromSlice([]string{"version:" + version.AgentVersion, "agent_group:group01"}), + Host: agg.hostname, + MType: metrics.APIGaugeType, + SourceTypeName: "System", + }, &metrics.Serie{ + Name: fmt.Sprintf("datadog.%s.ha_agent.running", agg.agentName), + Points: []metrics.Point{{Value: float64(1), Ts: float64(start.Unix())}}, + Tags: tagset.CompositeTagsFromSlice([]string{"agent_group:group01", "agent_state:standby"}), + Host: agg.hostname, + MType: metrics.APIGaugeType, + SourceTypeName: "System", + }, &metrics.Serie{ + Name: fmt.Sprintf("n_o_i_n_d_e_x.datadog.%s.payload.dropped", flavor.GetFlavor()), + Points: []metrics.Point{{Value: 0, Ts: float64(start.Unix())}}, + Host: agg.hostname, + Tags: tagset.CompositeTagsFromSlice([]string{"agent_group:group01"}), + MType: metrics.APIGaugeType, + SourceTypeName: "System", + NoIndex: true, + }} + + s.On("SendSeries", expectedSeries).Return(nil).Times(1) + + var flushedSeries metrics.Series + triggerInstance := testNewFlushTrigger(start, false, func(serie *metrics.Serie) { + flushedSeries = append(flushedSeries, serie) + }) + + agg.Flush(triggerInstance) + + assert.EqualValues(t, expectedSeries, flushedSeries) +} + func TestSeriesTooManyTags(t *testing.T) { // this test IS USING globals (tagsetTlm and recurrentSeries) but a local aggregator // - diff --git a/pkg/aggregator/sender_test.go b/pkg/aggregator/sender_test.go index 994cc96969d1c..ef7173f8a1bcc 100644 --- a/pkg/aggregator/sender_test.go +++ b/pkg/aggregator/sender_test.go @@ -180,7 +180,7 @@ func TestDestroySender(t *testing.T) { return aggregatorInstance.checkSamplers[checkID1].deregistered }, time.Second, 10*time.Millisecond) - aggregatorInstance.Flush(testNewFlushTrigger(time.Now(), false)) + aggregatorInstance.Flush(testNewFlushTrigger(time.Now(), false, nil)) assertAggSamplersLen(t, aggregatorInstance, 1) } From 35675c30fd1151f1d86a757c4ade99e081e71e13 Mon Sep 17 00:00:00 2001 From: Vickenty Fesunov Date: Fri, 6 Dec 2024 15:06:28 +0100 Subject: [PATCH 03/52] AMLII-2169 Activate BouncyCastle Java FIPS provider for FIPS images (#31827) --- .gitlab/container_build/docker_linux.yml | 4 ++-- Dockerfiles/agent/Dockerfile | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index c94f319627621..517cecdcf3b67 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -154,7 +154,7 @@ docker_build_fips_agent7_jmx: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent TAG_SUFFIX: -7-fips-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz + BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg WITH_JMX_FIPS=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz docker_build_fips_agent7_arm64_jmx: extends: [.docker_build_job_definition_arm64, .docker_build_artifact] @@ -167,7 +167,7 @@ docker_build_fips_agent7_arm64_jmx: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent TAG_SUFFIX: -7-fips-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz + BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg WITH_JMX_FIPS=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz # build agent7 UA image docker_build_ot_agent7: diff --git a/Dockerfiles/agent/Dockerfile b/Dockerfiles/agent/Dockerfile index 58ecd02ae3bb6..f43642f79e819 100644 --- a/Dockerfiles/agent/Dockerfile +++ b/Dockerfiles/agent/Dockerfile @@ -39,6 +39,7 @@ RUN gcc -pipe -Wall -Wextra -O2 -shared -fPIC -Wl,--version-script=/tmp/nosys.sy FROM baseimage AS extract ARG TARGETARCH ARG WITH_JMX +ARG WITH_JMX_FIPS ARG DD_AGENT_ARTIFACT=datadog-agent*-$TARGETARCH.tar.xz ARG GENERAL_ARTIFACTS_CACHE_BUCKET_URL @@ -96,6 +97,7 @@ RUN if [ -n "$WITH_JMX" ]; then cd /opt/bouncycastle-fips && mvn dependency:copy FROM baseimage AS release LABEL maintainer="Datadog " ARG WITH_JMX +ARG WITH_JMX_FIPS ARG DD_GIT_REPOSITORY_URL ARG DD_GIT_COMMIT_SHA ENV DOCKER_DD_AGENT=true \ @@ -196,6 +198,9 @@ COPY --from=extract /opt/bouncycastle-fips/target/dependency/*.jar /opt/bouncyca COPY --chmod=644 bouncycastle-fips/java.security /opt/bouncycastle-fips/ COPY --chmod=644 bouncycastle-fips/bc-fips.policy /opt/bouncycastle-fips/ RUN if [ -z "$WITH_JMX" ]; then rm -rf /opt/bouncycastle-fips; fi +# Configure Java to use BouncyCastle FIPS provider on JMX FIPS images. +# Double equals sign for java.security.properties istructs java to replace system defaults with the contents of the new file. +ENV JAVA_TOOL_OPTIONS="${WITH_JMX_FIPS:+--module-path=/opt/bouncycastle-fips -Djava.security.properties==/opt/bouncycastle-fips/java.security -Dpolicy.url.2=file:/opt/bouncycastle-fips/bc-fips.policy}" # Update if optional OTel Agent process should not run RUN if [ ! -f /opt/datadog-agent/embedded/bin/otel-agent ]; then \ From 4daa586489638c5f6df38499f0f6fa69cb182967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 6 Dec 2024 15:16:42 +0100 Subject: [PATCH 04/52] omnibus: reorder depencencies (#31802) --- omnibus/config/projects/agent.rb | 29 ++++---------- omnibus/config/software/agent-dependencies.rb | 22 ----------- .../software/datadog-agent-dependencies.rb | 38 ++++++++++++++++++- omnibus/config/software/datadog-agent.rb | 1 - omnibus/config/software/snmp-traps.rb | 1 - 5 files changed, 45 insertions(+), 46 deletions(-) delete mode 100644 omnibus/config/software/agent-dependencies.rb diff --git a/omnibus/config/projects/agent.rb b/omnibus/config/projects/agent.rb index 62a6afb6deaf7..b3ab9c10ae1ad 100644 --- a/omnibus/config/projects/agent.rb +++ b/omnibus/config/projects/agent.rb @@ -221,9 +221,17 @@ # ------------------------------------ if do_build + # Include traps db file in snmp.d/traps_db/ + dependency 'snmp-traps' + # Datadog agent dependency 'datadog-agent' + # This depends on the agent and must be added after it + if ENV['WINDOWS_DDPROCMON_DRIVER'] and not ENV['WINDOWS_DDPROCMON_DRIVER'].empty? + dependency 'datadog-security-agent-policies' + end + # System-probe if sysprobe_enabled? dependency 'system-probe' @@ -237,27 +245,6 @@ if linux_target? dependency 'datadog-security-agent-policies' - if fips_mode? - dependency 'openssl-fips-provider' - end - end - - # Include traps db file in snmp.d/traps_db/ - dependency 'snmp-traps' - - # Additional software - if windows_target? - if ENV['WINDOWS_DDNPM_DRIVER'] and not ENV['WINDOWS_DDNPM_DRIVER'].empty? - dependency 'datadog-windows-filter-driver' - end - if ENV['WINDOWS_APMINJECT_MODULE'] and not ENV['WINDOWS_APMINJECT_MODULE'].empty? - dependency 'datadog-windows-apminject' - end - if ENV['WINDOWS_DDPROCMON_DRIVER'] and not ENV['WINDOWS_DDPROCMON_DRIVER'].empty? - dependency 'datadog-windows-procmon-driver' - ## this is a duplicate of the above dependency in linux - dependency 'datadog-security-agent-policies' - end end # this dependency puts few files out of the omnibus install dir and move them diff --git a/omnibus/config/software/agent-dependencies.rb b/omnibus/config/software/agent-dependencies.rb deleted file mode 100644 index 125fd7a9525f0..0000000000000 --- a/omnibus/config/software/agent-dependencies.rb +++ /dev/null @@ -1,22 +0,0 @@ -name 'agent-dependencies' - -# Linux-specific dependencies -if linux_target? - dependency 'procps-ng' - dependency 'curl' -end - -# Bundled cacerts file (is this a good idea?) -dependency 'cacerts' - -# External agents -dependency 'jmxfetch' - -if linux_target? - dependency 'sds' -end - -# version manifest file -dependency 'version-manifest' - - diff --git a/omnibus/config/software/datadog-agent-dependencies.rb b/omnibus/config/software/datadog-agent-dependencies.rb index 3cac114d0578e..fd6712983b10b 100644 --- a/omnibus/config/software/datadog-agent-dependencies.rb +++ b/omnibus/config/software/datadog-agent-dependencies.rb @@ -2,6 +2,28 @@ description "Enforce building dependencies as soon as possible so they can be cached" +# Linux-specific dependencies +if linux_target? + dependency 'procps-ng' + dependency 'curl' + if fips_mode? + dependency 'openssl-fips-provider' + end +end + +# Bundled cacerts file (is this a good idea?) +dependency 'cacerts' + +# External agents +dependency 'jmxfetch' + +if linux_target? + dependency 'sds' +end + +# version manifest file +dependency 'version-manifest' + # Used for memory profiling with the `status py` agent subcommand dependency 'pympler' @@ -9,4 +31,18 @@ dependency "systemd" if linux_target? -dependency 'libpcap' if linux_target? and !heroku_target? # system-probe dependency \ No newline at end of file +dependency 'libpcap' if linux_target? and !heroku_target? # system-probe dependency + +# Additional software +if windows_target? + if ENV['WINDOWS_DDNPM_DRIVER'] and not ENV['WINDOWS_DDNPM_DRIVER'].empty? + dependency 'datadog-windows-filter-driver' + end + if ENV['WINDOWS_APMINJECT_MODULE'] and not ENV['WINDOWS_APMINJECT_MODULE'].empty? + dependency 'datadog-windows-apminject' + end + if ENV['WINDOWS_DDPROCMON_DRIVER'] and not ENV['WINDOWS_DDPROCMON_DRIVER'].empty? + dependency 'datadog-windows-procmon-driver' + end +end + diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index ecfe7cb0d4c7c..8f24178d2c07e 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -20,7 +20,6 @@ # especially at higher thread counts. dependency "libjemalloc" if linux_target? -dependency 'agent-dependencies' dependency 'datadog-agent-dependencies' source path: '..' diff --git a/omnibus/config/software/snmp-traps.rb b/omnibus/config/software/snmp-traps.rb index 1dc01d61b1680..5a021b01ed8d4 100644 --- a/omnibus/config/software/snmp-traps.rb +++ b/omnibus/config/software/snmp-traps.rb @@ -3,7 +3,6 @@ # Needs the configuration folder as created in datadog-agent dependency 'datadog-agent' -always_build true # For cache related purposes, it comes after datadog-agent source :url => "https://s3.amazonaws.com/dd-agent-omnibus/snmp_traps_db/dd_traps_db-#{version}.json.gz", :sha256 => "04fb9d43754c2656edf35f08fbad11ba8dc20d52654962933f3dd8f4d463b42c", From 2ee99e0efe6cc535b7d6272c6f7e0e206c343ab4 Mon Sep 17 00:00:00 2001 From: Jeremy Hanna Date: Fri, 6 Dec 2024 11:04:44 -0500 Subject: [PATCH 05/52] Set GOFIPS via build arg for docker env (#31808) --- .gitlab/container_build/docker_linux.yml | 8 ++++---- Dockerfiles/agent/Dockerfile | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index 517cecdcf3b67..4bd10a6be23ff 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -101,7 +101,7 @@ docker_build_fips_agent7: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent TAG_SUFFIX: -7-fips - BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz + BUILD_ARG: --target test --build-arg FIPS_ENABLED=1 --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz docker_build_fips_agent7_arm64: extends: [.docker_build_job_definition_arm64, .docker_build_artifact] @@ -114,7 +114,7 @@ docker_build_fips_agent7_arm64: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent TAG_SUFFIX: -7-fips - BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz + BUILD_ARG: --target test --build-arg FIPS_ENABLED=1 --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz # build agent7 jmx image docker_build_agent7_jmx: @@ -154,7 +154,7 @@ docker_build_fips_agent7_jmx: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent TAG_SUFFIX: -7-fips-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg WITH_JMX_FIPS=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz + BUILD_ARG: --target test --build-arg FIPS_ENABLED=1 --build-arg WITH_JMX=true --build-arg WITH_JMX_FIPS=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz docker_build_fips_agent7_arm64_jmx: extends: [.docker_build_job_definition_arm64, .docker_build_artifact] @@ -167,7 +167,7 @@ docker_build_fips_agent7_arm64_jmx: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent TAG_SUFFIX: -7-fips-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg WITH_JMX_FIPS=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz + BUILD_ARG: --target test --build-arg FIPS_ENABLED=1 --build-arg WITH_JMX=true --build-arg WITH_JMX_FIPS=true --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz # build agent7 UA image docker_build_ot_agent7: diff --git a/Dockerfiles/agent/Dockerfile b/Dockerfiles/agent/Dockerfile index f43642f79e819..072b832fb3f0a 100644 --- a/Dockerfiles/agent/Dockerfile +++ b/Dockerfiles/agent/Dockerfile @@ -96,6 +96,7 @@ RUN if [ -n "$WITH_JMX" ]; then cd /opt/bouncycastle-fips && mvn dependency:copy FROM baseimage AS release LABEL maintainer="Datadog " +ARG FIPS_ENABLED=0 ARG WITH_JMX ARG WITH_JMX_FIPS ARG DD_GIT_REPOSITORY_URL @@ -185,8 +186,8 @@ RUN [ "$(getent passwd dd-agent | cut -d: -f 3)" -eq 100 ] # Enable FIPS if needed RUN if [ -x /opt/datadog-agent/embedded/bin/fipsinstall.sh ]; then \ /opt/datadog-agent/embedded/bin/fipsinstall.sh; \ - export GOFIPS=1; \ fi +ENV GOFIPS=${FIPS_ENABLED} # Override the exit script by ours to fix --pid=host operations RUN mv /etc/s6/init/init-stage3 /etc/s6/init/init-stage3-original From 81092f177c80c9d7e57d037f48b4dd7b9c437a7e Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Fri, 6 Dec 2024 17:47:44 +0100 Subject: [PATCH 06/52] Fix build without clusterchecks build tag (#31707) Co-authored-by: Cedric Lamoriniere --- pkg/clusteragent/clusterchecks/handler_api_nocompile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/clusteragent/clusterchecks/handler_api_nocompile.go b/pkg/clusteragent/clusterchecks/handler_api_nocompile.go index cd03f0938cecc..d66023e3bf585 100644 --- a/pkg/clusteragent/clusterchecks/handler_api_nocompile.go +++ b/pkg/clusteragent/clusterchecks/handler_api_nocompile.go @@ -13,6 +13,7 @@ import ( "errors" "github.com/DataDog/datadog-agent/comp/core/autodiscovery" + tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" "github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks/types" ) @@ -31,7 +32,7 @@ func (h *Handler) GetState() (types.StateResponse, error) { } // NewHandler not implemented -func NewHandler(_ autodiscovery.Component) (*Handler, error) { +func NewHandler(_ autodiscovery.Component, _ tagger.Component) (*Handler, error) { return nil, ErrNotCompiled } From bf19c87fa6015930a4c36adae17fd1f57d7cafc6 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Fri, 6 Dec 2024 11:58:47 -0500 Subject: [PATCH 07/52] [ebpfless] Fix revive linter warnings in TCP processor (#31817) --- .../connection/ebpfless/tcp_processor.go | 48 ++-- .../ebpfless/tcp_processor_retransmit_test.go | 30 +-- .../ebpfless/tcp_processor_rtt_test.go | 4 +- .../connection/ebpfless/tcp_processor_test.go | 220 +++++++++--------- .../tracer/connection/ebpfless/tcp_utils.go | 36 +-- 5 files changed, 170 insertions(+), 168 deletions(-) diff --git a/pkg/network/tracer/connection/ebpfless/tcp_processor.go b/pkg/network/tracer/connection/ebpfless/tcp_processor.go index 500a89893eeb6..b312f460269ee 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_processor.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_processor.go @@ -21,7 +21,7 @@ import ( ) type connectionState struct { - tcpState ConnStatus + tcpState connStatus // hasSentPacket is whether anything has been sent outgoing (aka whether maxSeqSent exists) hasSentPacket bool @@ -38,9 +38,9 @@ type connectionState struct { lastRemoteAck uint32 // localSynState is the status of the outgoing SYN handshake - localSynState SynState + localSynState synState // remoteSynState is the status of the incoming SYN handshake - remoteSynState SynState + remoteSynState synState // hasLocalFin is whether the outgoing side has FIN'd hasLocalFin bool @@ -54,11 +54,13 @@ type connectionState struct { rttTracker rttTracker } -type TCPProcessor struct { //nolint:revive // TODO +// TCPProcessor encapsulates TCP state tracking for the ebpfless tracer +type TCPProcessor struct { conns map[network.ConnectionTuple]connectionState } -func NewTCPProcessor() *TCPProcessor { //nolint:revive // TODO +// NewTCPProcessor constructs an empty TCPProcessor +func NewTCPProcessor() *TCPProcessor { return &TCPProcessor{ conns: map[network.ConnectionTuple]connectionState{}, } @@ -106,7 +108,7 @@ func checkInvalidTCP(tcp *layers.TCP) bool { return false } -func (t *TCPProcessor) updateSynFlag(conn *network.ConnectionStats, st *connectionState, pktType uint8, tcp *layers.TCP, payloadLen uint16) { //nolint:revive // TODO +func (t *TCPProcessor) updateSynFlag(conn *network.ConnectionStats, st *connectionState, pktType uint8, tcp *layers.TCP, _payloadLen uint16) { if tcp.RST { return } @@ -116,22 +118,22 @@ func (t *TCPProcessor) updateSynFlag(conn *network.ConnectionStats, st *connecti } else { st.remoteSynState.update(tcp.SYN, tcp.ACK) } - // if any SynState has progressed, move to attempted - if st.tcpState == ConnStatClosed && (st.localSynState != SynStateNone || st.remoteSynState != SynStateNone) { - st.tcpState = ConnStatAttempted + // if any synState has progressed, move to attempted + if st.tcpState == connStatClosed && (st.localSynState != synStateNone || st.remoteSynState != synStateNone) { + st.tcpState = connStatAttempted updateConnStatsForOpen(conn) } // if both synStates are ack'd, move to established - if st.tcpState == ConnStatAttempted && st.localSynState == SynStateAcked && st.remoteSynState == SynStateAcked { - st.tcpState = ConnStatEstablished + if st.tcpState == connStatAttempted && st.localSynState == synStateAcked && st.remoteSynState == synStateAcked { + st.tcpState = connStatEstablished conn.Monotonic.TCPEstablished++ } } -// updateTcpStats is designed to mirror the stat tracking in the windows driver's handleFlowProtocolTcp +// updateTCPStats is designed to mirror the stat tracking in the windows driver's handleFlowProtocolTcp // https://github.com/DataDog/datadog-windows-filter/blob/d7560d83eb627117521d631a4c05cd654a01987e/ddfilter/flow/flow_tcp.c#L91 -func (t *TCPProcessor) updateTcpStats(conn *network.ConnectionStats, st *connectionState, pktType uint8, tcp *layers.TCP, payloadLen uint16, timestampNs uint64) { //nolint:revive // TODO +func (t *TCPProcessor) updateTCPStats(conn *network.ConnectionStats, st *connectionState, pktType uint8, tcp *layers.TCP, payloadLen uint16, timestampNs uint64) { nextSeq := calcNextSeq(tcp, payloadLen) if pktType == unix.PACKET_OUTGOING { @@ -152,8 +154,8 @@ func (t *TCPProcessor) updateTcpStats(conn *network.ConnectionStats, st *connect ackOutdated := !st.hasLocalAck || isSeqBefore(st.lastLocalAck, tcp.Ack) if tcp.ACK && ackOutdated { - // wait until data comes in via SynStateAcked - if st.hasLocalAck && st.remoteSynState == SynStateAcked { + // wait until data comes in via synStateAcked + if st.hasLocalAck && st.remoteSynState == synStateAcked { ackDiff := tcp.Ack - st.lastLocalAck isFinAck := st.hasRemoteFin && tcp.Ack == st.remoteFinSeq if isFinAck { @@ -199,31 +201,31 @@ func (t *TCPProcessor) updateFinFlag(conn *network.ConnectionStats, st *connecti // if both fins have been sent and ack'd, then mark the connection closed localFinIsAcked := st.hasLocalFin && isSeqBeforeEq(st.localFinSeq, st.lastRemoteAck) remoteFinIsAcked := st.hasRemoteFin && isSeqBeforeEq(st.remoteFinSeq, st.lastLocalAck) - if st.tcpState == ConnStatEstablished && localFinIsAcked && remoteFinIsAcked { + if st.tcpState == connStatEstablished && localFinIsAcked && remoteFinIsAcked { *st = connectionState{ - tcpState: ConnStatClosed, + tcpState: connStatClosed, } conn.Monotonic.TCPClosed++ updateConnStatsForClose(conn) } } -func (t *TCPProcessor) updateRstFlag(conn *network.ConnectionStats, st *connectionState, pktType uint8, tcp *layers.TCP, payloadLen uint16) { //nolint:revive // TODO - if !tcp.RST || st.tcpState == ConnStatClosed { +func (t *TCPProcessor) updateRstFlag(conn *network.ConnectionStats, st *connectionState, _pktType uint8, tcp *layers.TCP, _payloadLen uint16) { + if !tcp.RST || st.tcpState == connStatClosed { return } reason := syscall.ECONNRESET - if st.tcpState == ConnStatAttempted { + if st.tcpState == connStatAttempted { reason = syscall.ECONNREFUSED } conn.TCPFailures[uint16(reason)]++ - if st.tcpState == ConnStatEstablished { + if st.tcpState == connStatEstablished { conn.Monotonic.TCPClosed++ } *st = connectionState{ - tcpState: ConnStatClosed, + tcpState: connStatClosed, } updateConnStatsForClose(conn) } @@ -251,7 +253,7 @@ func (t *TCPProcessor) Process(conn *network.ConnectionStats, timestampNs uint64 st := t.conns[conn.ConnectionTuple] t.updateSynFlag(conn, &st, pktType, tcp, payloadLen) - t.updateTcpStats(conn, &st, pktType, tcp, payloadLen, timestampNs) + t.updateTCPStats(conn, &st, pktType, tcp, payloadLen, timestampNs) t.updateFinFlag(conn, &st, pktType, tcp, payloadLen) t.updateRstFlag(conn, &st, pktType, tcp, payloadLen) diff --git a/pkg/network/tracer/connection/ebpfless/tcp_processor_retransmit_test.go b/pkg/network/tracer/connection/ebpfless/tcp_processor_retransmit_test.go index ecfdb22e1fc72..9082872964c42 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_processor_retransmit_test.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_processor_retransmit_test.go @@ -67,7 +67,7 @@ func TestAllRetransmitsOutgoing(t *testing.T) { t.Run("retransmit SYN", func(t *testing.T) { traffic := retransmitNth(basicHandshake, 0) - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(traffic) require.Empty(t, f.conn.TCPFailures) @@ -77,7 +77,7 @@ func TestAllRetransmitsOutgoing(t *testing.T) { t.Run("retransmit data", func(t *testing.T) { traffic := retransmitNth(basicHandshake, 3) - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(traffic) require.Empty(t, f.conn.TCPFailures) @@ -87,7 +87,7 @@ func TestAllRetransmitsOutgoing(t *testing.T) { t.Run("retransmit FIN", func(t *testing.T) { traffic := retransmitNth(basicHandshake, 8) - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(traffic) require.Empty(t, f.conn.TCPFailures) @@ -130,7 +130,7 @@ func TestAllRetransmitsIncoming(t *testing.T) { t.Run("retransmit SYNACK", func(t *testing.T) { traffic := retransmitNth(basicHandshake, 1) - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(traffic) require.Empty(t, f.conn.TCPFailures) @@ -140,7 +140,7 @@ func TestAllRetransmitsIncoming(t *testing.T) { t.Run("retransmit data", func(t *testing.T) { traffic := retransmitNth(basicHandshake, 5) - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(traffic) require.Empty(t, f.conn.TCPFailures) @@ -150,7 +150,7 @@ func TestAllRetransmitsIncoming(t *testing.T) { t.Run("retransmit FIN", func(t *testing.T) { traffic := retransmitNth(basicHandshake, 6) - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(traffic) require.Empty(t, f.conn.TCPFailures) @@ -171,16 +171,16 @@ func TestRstTwice(t *testing.T) { pb.outgoing(0, 1, 1, RST|ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, - ConnStatEstablished, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, + connStatEstablished, // reset - ConnStatClosed, - ConnStatClosed, + connStatClosed, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) // should count as a single failure @@ -220,7 +220,7 @@ func TestKeepAlivePacketsArentRetransmits(t *testing.T) { pb.outgoing(0, 2, 2, ACK), } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(basicHandshake) require.Empty(t, f.conn.TCPFailures) @@ -255,7 +255,7 @@ func TestRetransmitMultipleSegments(t *testing.T) { pb.outgoing(0, 2, 2, ACK), } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkts(traffic) require.Empty(t, f.conn.TCPFailures) diff --git a/pkg/network/tracer/connection/ebpfless/tcp_processor_rtt_test.go b/pkg/network/tracer/connection/ebpfless/tcp_processor_rtt_test.go index 7ff2eca99a352..356d1d1a9712b 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_processor_rtt_test.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_processor_rtt_test.go @@ -101,7 +101,7 @@ func TestTcpProcessorRtt(t *testing.T) { // t=300 us, for a round trip of 100us synack.timestampNs = 300 * 1000 - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkt(syn) // round trip has not completed yet @@ -123,7 +123,7 @@ func TestTcpProcessorRttRetransmit(t *testing.T) { // t=300 us, for a round trip of 100us synack.timestampNs = 300 * 1000 - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runPkt(syn) // round trip has not completed yet diff --git a/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go b/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go index a558f073dc4ec..8656a73463d36 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_processor_test.go @@ -21,14 +21,14 @@ import ( "github.com/DataDog/datadog-agent/pkg/process/util" ) -var localhost net.IP = net.ParseIP("127.0.0.1") //nolint:revive // TODO -var remoteIP net.IP = net.ParseIP("12.34.56.78") //nolint:revive // TODO +var localhost = net.ParseIP("127.0.0.1") +var remoteIP = net.ParseIP("12.34.56.78") const ( minIhl = 5 defaultLocalPort = 12345 defaultRemotePort = 8080 - defaultNsId = 123 //nolint:revive // TODO + defaultNsID = 123 ) const ( @@ -79,7 +79,7 @@ type testCapture struct { } // TODO can this be merged with the logic creating scratchConns in ebpfless tracer? -func makeTcpStates(synPkt testCapture) *network.ConnectionStats { //nolint:revive // TODO +func makeTCPStates(synPkt testCapture) *network.ConnectionStats { var family network.ConnectionFamily var srcIP, dstIP net.IP if synPkt.ipv4 != nil && synPkt.ipv6 != nil { @@ -109,8 +109,8 @@ func makeTcpStates(synPkt testCapture) *network.ConnectionStats { //nolint:reviv ConnectionTuple: network.ConnectionTuple{ Source: util.AddressFromNetIP(srcIP), Dest: util.AddressFromNetIP(dstIP), - Pid: 0, // @stu we can't know this right - NetNS: defaultNsId, + Pid: 0, // packet capture does not have PID information. + NetNS: defaultNsID, SPort: uint16(synPkt.tcp.SrcPort), DPort: uint16(synPkt.tcp.DstPort), Type: network.TCP, @@ -165,7 +165,7 @@ func (pb packetBuilder) outgoing(payloadLen uint16, relSeq, relAck uint32, flags } } -func newTcpTestFixture(t *testing.T) *tcpTestFixture { //nolint:revive // TODO +func newTCPTestFixture(t *testing.T) *tcpTestFixture { return &tcpTestFixture{ t: t, tcp: NewTCPProcessor(), @@ -175,7 +175,7 @@ func newTcpTestFixture(t *testing.T) *tcpTestFixture { //nolint:revive // TODO func (fixture *tcpTestFixture) runPkt(pkt testCapture) { if fixture.conn == nil { - fixture.conn = makeTcpStates(pkt) + fixture.conn = makeTCPStates(pkt) } err := fixture.tcp.Process(fixture.conn, pkt.timestampNs, pkt.pktType, pkt.ipv4, pkt.ipv6, pkt.tcp) require.NoError(fixture.t, err) @@ -187,18 +187,18 @@ func (fixture *tcpTestFixture) runPkts(packets []testCapture) { //nolint:unused } } -func (fixture *tcpTestFixture) runAgainstState(packets []testCapture, expected []ConnStatus) { +func (fixture *tcpTestFixture) runAgainstState(packets []testCapture, expected []connStatus) { require.Equal(fixture.t, len(packets), len(expected), "packet length didn't match expected states length") var expectedStrs []string var actualStrs []string for i, pkt := range packets { - expectedStrs = append(expectedStrs, LabelForState(expected[i])) + expectedStrs = append(expectedStrs, labelForState(expected[i])) fixture.runPkt(pkt) connTuple := fixture.conn.ConnectionTuple actual := fixture.tcp.conns[connTuple].tcpState - actualStrs = append(actualStrs, LabelForState(actual)) + actualStrs = append(actualStrs, labelForState(actual)) } require.Equal(fixture.t, expectedStrs, actualStrs) } @@ -223,23 +223,23 @@ func testBasicHandshake(t *testing.T, pb packetBuilder) { pb.incoming(0, 347, 125, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, // three-way handshake finishes here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, // passive close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, // final FIN was ack'd - ConnStatClosed, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Empty(t, f.conn.TCPFailures) @@ -292,22 +292,22 @@ func testReversedBasicHandshake(t *testing.T, pb packetBuilder) { pb.outgoing(0, 347, 125, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, // three-way handshake finishes here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, // active close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatClosed, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Empty(t, f.conn.TCPFailures) @@ -359,21 +359,21 @@ func testCloseWaitState(t *testing.T, pb packetBuilder) { pb.incoming(0, 347, 224+42+1, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, // three-way handshake finishes here - ConnStatEstablished, - ConnStatEstablished, + connStatEstablished, + connStatEstablished, // passive close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatClosed, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Empty(t, f.conn.TCPFailures) @@ -427,23 +427,23 @@ func testFinWait2State(t *testing.T, pb packetBuilder) { pb.outgoing(0, 347, 225, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, // three-way handshake finishes here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, // active close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatClosed, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Empty(t, f.conn.TCPFailures) @@ -486,17 +486,17 @@ func TestImmediateFin(t *testing.T) { pb.outgoing(0, 2, 2, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, - ConnStatEstablished, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, + connStatEstablished, // active close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatClosed, + connStatEstablished, + connStatEstablished, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Empty(t, f.conn.TCPFailures) @@ -520,12 +520,12 @@ func TestConnRefusedSyn(t *testing.T) { pb.outgoing(0, 0, 0, RST|ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatClosed, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Equal(t, map[uint16]uint32{ @@ -552,13 +552,13 @@ func TestConnRefusedSynAck(t *testing.T) { pb.outgoing(0, 0, 0, RST|ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, - ConnStatClosed, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Equal(t, map[uint16]uint32{ @@ -587,15 +587,15 @@ func TestConnReset(t *testing.T) { pb.outgoing(0, 1, 1, RST|ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, - ConnStatEstablished, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, + connStatEstablished, // reset - ConnStatClosed, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Equal(t, map[uint16]uint32{ @@ -628,23 +628,23 @@ func TestConnectTwice(t *testing.T) { pb.outgoing(0, 2, 2, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, - ConnStatEstablished, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, + connStatEstablished, // active close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatClosed, + connStatEstablished, + connStatEstablished, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) state := f.tcp.conns[f.conn.ConnectionTuple] // make sure the TCP state was erased after the connection was closed require.Equal(t, connectionState{ - tcpState: ConnStatClosed, + tcpState: connStatClosed, }, state) // second connection here @@ -677,18 +677,18 @@ func TestSimultaneousClose(t *testing.T) { pb.incoming(0, 2, 2, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, - ConnStatEstablished, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, + connStatEstablished, // active close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatEstablished, - ConnStatClosed, + connStatEstablished, + connStatEstablished, + connStatEstablished, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Empty(t, f.conn.TCPFailures) @@ -720,18 +720,18 @@ func TestUnusualAckSyn(t *testing.T) { pb.outgoing(0, 2, 2, ACK), } - expectedClientStates := []ConnStatus{ - ConnStatAttempted, - ConnStatAttempted, - ConnStatAttempted, - ConnStatEstablished, + expectedClientStates := []connStatus{ + connStatAttempted, + connStatAttempted, + connStatAttempted, + connStatEstablished, // active close begins here - ConnStatEstablished, - ConnStatEstablished, - ConnStatClosed, + connStatEstablished, + connStatEstablished, + connStatClosed, } - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) f.runAgainstState(basicHandshake, expectedClientStates) require.Empty(t, f.conn.TCPFailures) @@ -752,7 +752,7 @@ func TestUnusualAckSyn(t *testing.T) { func TestOpenCloseConn(t *testing.T) { pb := newPacketBuilder(lowerSeq, higherSeq) - f := newTcpTestFixture(t) + f := newTCPTestFixture(t) // send a SYN packet to kick things off f.runPkt(pb.incoming(0, 0, 0, SYN)) diff --git a/pkg/network/tracer/connection/ebpfless/tcp_utils.go b/pkg/network/tracer/connection/ebpfless/tcp_utils.go index 3a02ea3b09e29..1969fd85a5bc1 100644 --- a/pkg/network/tracer/connection/ebpfless/tcp_utils.go +++ b/pkg/network/tracer/connection/ebpfless/tcp_utils.go @@ -37,12 +37,12 @@ var statsTelemetry = struct { const tcpSeqMidpoint = 0x80000000 -type ConnStatus uint8 //nolint:revive // TODO +type connStatus uint8 const ( - ConnStatClosed ConnStatus = iota //nolint:revive // TODO - ConnStatAttempted //nolint:revive // TODO - ConnStatEstablished //nolint:revive // TODO + connStatClosed connStatus = iota + connStatAttempted + connStatEstablished ) var connStatusLabels = []string{ @@ -51,32 +51,32 @@ var connStatusLabels = []string{ "Established", } -type SynState uint8 //nolint:revive // TODO +type synState uint8 const ( - SynStateNone SynState = iota //nolint:revive // TODO - SynStateSent //nolint:revive // TODO - SynStateAcked //nolint:revive // TODO + synStateNone synState = iota + synStateSent + synStateAcked ) -func (ss *SynState) update(synFlag, ackFlag bool) { +func (ss *synState) update(synFlag, ackFlag bool) { // for simplicity, this does not consider the sequence number of the SYNs and ACKs. // if these matter in the future, change this to store SYN seq numbers - if *ss == SynStateNone && synFlag { - *ss = SynStateSent + if *ss == synStateNone && synFlag { + *ss = synStateSent } - if *ss == SynStateSent && ackFlag { - *ss = SynStateAcked + if *ss == synStateSent && ackFlag { + *ss = synStateAcked } // if we see ACK'd traffic but missed the SYN, assume the connection started before // the datadog-agent starts. - if *ss == SynStateNone && ackFlag { + if *ss == synStateNone && ackFlag { statsTelemetry.missedTCPConnections.Inc() - *ss = SynStateAcked + *ss = synStateAcked } } -func LabelForState(tcpState ConnStatus) string { //nolint:revive // TODO +func labelForState(tcpState connStatus) string { idx := int(tcpState) if idx < len(connStatusLabels) { return connStatusLabels[idx] @@ -105,7 +105,7 @@ func debugPacketDir(pktType uint8) string { } } -func debugTcpFlags(tcp *layers.TCP) string { //nolint:revive // TODO +func debugTCPFlags(tcp *layers.TCP) string { var flags []string if tcp.RST { flags = append(flags, "RST") @@ -123,5 +123,5 @@ func debugTcpFlags(tcp *layers.TCP) string { //nolint:revive // TODO } func debugPacketInfo(pktType uint8, tcp *layers.TCP, payloadLen uint16) string { - return fmt.Sprintf("pktType=%+v ports=(%+v, %+v) size=%d seq=%+v ack=%+v flags=%s", debugPacketDir(pktType), uint16(tcp.SrcPort), uint16(tcp.DstPort), payloadLen, tcp.Seq, tcp.Ack, debugTcpFlags(tcp)) + return fmt.Sprintf("pktType=%+v ports=(%+v, %+v) size=%d seq=%+v ack=%+v flags=%s", debugPacketDir(pktType), uint16(tcp.SrcPort), uint16(tcp.DstPort), payloadLen, tcp.Seq, tcp.Ack, debugTCPFlags(tcp)) } From 12815d619573492e1fed6ed7461059beb9dfcc60 Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Fri, 6 Dec 2024 11:59:19 -0500 Subject: [PATCH 08/52] Build Windows FIPS Agent containers (#31727) --- .gitlab/container_build/docker_windows.yml | 16 ++ .../container_build/docker_windows_agent7.yml | 20 ++- .../dev_container_deploy/docker_windows.yml | 155 ++++++------------ Dockerfiles/agent/install-fips.ps1 | 19 ++- Dockerfiles/agent/windows/amd64/Dockerfile | 1 + 5 files changed, 101 insertions(+), 110 deletions(-) diff --git a/.gitlab/container_build/docker_windows.yml b/.gitlab/container_build/docker_windows.yml index 5beeea220c02e..f2402de687d30 100644 --- a/.gitlab/container_build/docker_windows.yml +++ b/.gitlab/container_build/docker_windows.yml @@ -62,5 +62,21 @@ BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:windowsservercore-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=core-${VARIANT}" SERVERCORE: "-servercore" +.docker_build_fips_agent7_windows_common: + extends: + - .docker_build_agent7_windows_common + needs: + ["windows_msi_and_bosh_zip_x64-a7-fips", "build_windows_container_entrypoint"] + variables: + AGENT_ZIP: "datadog-fips-agent-7*-x86_64.zip" + BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:lts-nanoserver-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg WITH_FIPS=true --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=nano-${VARIANT}-fips" + +.docker_build_fips_agent7_windows_servercore_common: + extends: + - .docker_build_fips_agent7_windows_common + variables: + BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:windowsservercore-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg WITH_FIPS=true --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=core-${VARIANT}-fips" + SERVERCORE: "-servercore" + include: - .gitlab/container_build/docker_windows_agent7.yml diff --git a/.gitlab/container_build/docker_windows_agent7.yml b/.gitlab/container_build/docker_windows_agent7.yml index e8bf47cbba812..4a3c9393d5ba1 100644 --- a/.gitlab/container_build/docker_windows_agent7.yml +++ b/.gitlab/container_build/docker_windows_agent7.yml @@ -21,7 +21,6 @@ docker_build_agent7_windows2022_jmx: extends: - .docker_build_agent7_windows_common tags: ["runner:windows-docker", "windowsversion:2022"] - needs: ["windows_msi_and_bosh_zip_x64-a7", "build_windows_container_entrypoint"] variables: VARIANT: ltsc2022 TAG_SUFFIX: -7-jmx @@ -67,8 +66,25 @@ docker_build_agent7_windows2022_core_jmx: extends: - .docker_build_agent7_windows_servercore_common tags: ["runner:windows-docker", "windowsversion:2022"] - needs: ["windows_msi_and_bosh_zip_x64-a7", "build_windows_container_entrypoint"] variables: VARIANT: ltsc2022 TAG_SUFFIX: -7-jmx WITH_JMX: "true" + +docker_build_fips_agent7_windows2022_core: + extends: + - .docker_build_fips_agent7_windows_servercore_common + tags: ["runner:windows-docker", "windowsversion:2022"] + variables: + VARIANT: ltsc2022 + TAG_SUFFIX: "-7-fips" + WITH_JMX: "false" + +docker_build_fips_agent7_windows2022_core_jmx: + extends: + - .docker_build_fips_agent7_windows_servercore_common + tags: ["runner:windows-docker", "windowsversion:2022"] + variables: + VARIANT: ltsc2022 + TAG_SUFFIX: -7-fips-jmx + WITH_JMX: "true" diff --git a/.gitlab/dev_container_deploy/docker_windows.yml b/.gitlab/dev_container_deploy/docker_windows.yml index 734c81cacd36c..c219f4b7a44ec 100644 --- a/.gitlab/dev_container_deploy/docker_windows.yml +++ b/.gitlab/dev_container_deploy/docker_windows.yml @@ -2,11 +2,9 @@ include: - .gitlab/common/container_publish_job_templates.yml -dev_branch-a7-windows: +.dev_a7-windows-common: extends: .docker_publish_job_definition stage: dev_container_deploy - rules: - !reference [.manual] needs: - docker_build_agent7_windows1809 - docker_build_agent7_windows1809_jmx @@ -23,16 +21,16 @@ dev_branch-a7-windows: # Multi-arch - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win + IMG_DESTINATIONS: agent-dev:${IMG_DESTINATION_SLUG}-py3-win - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win + IMG_DESTINATIONS: agent-dev:${IMG_DESTINATION_SLUG}-py3-jmx-win - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-servercore + IMG_DESTINATIONS: agent-dev:${IMG_DESTINATION_SLUG}-py3-win-servercore - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore + IMG_DESTINATIONS: agent-dev:${IMG_DESTINATION_SLUG}-py3-jmx-win-servercore # ltsc2019 - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" IMG_SOURCES: "%BASE%-win1809-amd64" @@ -60,118 +58,63 @@ dev_branch-a7-windows: IMG_SOURCES: "%BASE%-winltsc2022-servercore-amd64" IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore-ltsc2022 +dev_branch-a7-windows: + extends: .dev_a7-windows-common + rules: + !reference [.manual] + variables: + IMG_DESTINATION_SLUG: ${CI_COMMIT_REF_SLUG} + dev_master-a7-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy + extends: .dev_a7-windows-common rules: !reference [.on_main] - needs: - - docker_build_agent7_windows1809 - - docker_build_agent7_windows1809_jmx - - docker_build_agent7_windows1809_core - - docker_build_agent7_windows1809_core_jmx - - docker_build_agent7_windows2022 - - docker_build_agent7_windows2022_jmx - - docker_build_agent7_windows2022_core - - docker_build_agent7_windows2022_core_jmx variables: - IMG_REGISTRIES: dev - parallel: - matrix: - # Multi-arch - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-jmx-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-win-servercore - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:master-py3-jmx-win-servercore - # ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-servercore-ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore-ltsc2019 - # ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-servercore-ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore-ltsc2022 + IMG_DESTINATION_SLUG: master dev_nightly-a7-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy + extends: .dev_a7-windows-common rules: !reference [.on_deploy_nightly_repo_branch] + variables: + IMG_DESTINATION_SLUG: nightly + +.dev_fips-a7-windows-common: + extends: .docker_publish_job_definition + stage: dev_container_deploy needs: - - docker_build_agent7_windows1809 - - docker_build_agent7_windows1809_jmx - - docker_build_agent7_windows1809_core - - docker_build_agent7_windows1809_core_jmx - - docker_build_agent7_windows2022 - - docker_build_agent7_windows2022_jmx - - docker_build_agent7_windows2022_core - - docker_build_agent7_windows2022_core_jmx + - docker_build_fips_agent7_windows2022_core + - docker_build_fips_agent7_windows2022_core_jmx variables: IMG_REGISTRIES: dev + # Only publish ltsc2022 servercore for now, that's all that's used by the integrations testing parallel: matrix: - # Multi-arch - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-jmx-win - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-win-servercore - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-jmx-win-servercore - # ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-servercore-ltsc2019 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore-ltsc2019 # ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" - IMG_SOURCES: "%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" - IMG_SOURCES: "%BASE%-winltsc2022-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7" + - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips" IMG_SOURCES: "%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-win-servercore-ltsc2022 - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" + IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-fips-win-servercore-ltsc2022 + - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx" IMG_SOURCES: "%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore-ltsc2022 + IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-fips-jmx-win-servercore-ltsc2022 + +dev_branch-fips-a7-windows: + extends: .dev_fips-a7-windows-common + rules: + !reference [.manual] + variables: + IMG_DESTINATION_SLUG: ${CI_COMMIT_REF_SLUG} + +dev_master-fips-a7-windows: + extends: .dev_fips-a7-windows-common + rules: + !reference [.on_main] + variables: + IMG_DESTINATION_SLUG: master + +dev_nightly-fips-a7-windows: + extends: .dev_fips-a7-windows-common + rules: + !reference [.on_deploy_nightly_repo_branch] + variables: + IMG_DESTINATION_SLUG: nightly diff --git a/Dockerfiles/agent/install-fips.ps1 b/Dockerfiles/agent/install-fips.ps1 index 70be3b0da44bb..05324776e9dda 100644 --- a/Dockerfiles/agent/install-fips.ps1 +++ b/Dockerfiles/agent/install-fips.ps1 @@ -1,5 +1,16 @@ $ErrorActionPreference = 'Stop' +# Removes temporary files for FIPS setup +function Remove-TempFiles { + Remove-Item -Force -Recurse \fips-build +} + +if ("$env:WITH_FIPS" -ne "true") { + # If FIPS is not enabled, skip the FIPS setup + Remove-TempFiles + exit 0 +} + $maven_sha512 = '8BEAC8D11EF208F1E2A8DF0682B9448A9A363D2AD13CA74AF43705549E72E74C9378823BF689287801CBBFC2F6EA9596201D19CCACFDFB682EE8A2FF4C4418BA' if ("$env:WITH_JMX" -ne "false") { @@ -18,6 +29,10 @@ if ("$env:WITH_JMX" -ne "false") { if (!$?) { Write-Error ("BouncyCastle self check failed with exit code: {0}" -f $LASTEXITCODE) } + cd \ } -cd \ -Remove-Item -Force -Recurse \fips-build + +# TODO: Run openssl fipsinstall command here when embedded Python work is completed +# HERE + +Remove-TempFiles diff --git a/Dockerfiles/agent/windows/amd64/Dockerfile b/Dockerfiles/agent/windows/amd64/Dockerfile index 275801062f047..72c7060810838 100755 --- a/Dockerfiles/agent/windows/amd64/Dockerfile +++ b/Dockerfiles/agent/windows/amd64/Dockerfile @@ -7,6 +7,7 @@ ARG WITH_JMX="false" ARG VARIANT="unknown" ARG INSTALL_INFO="unknown" ARG GENERAL_ARTIFACTS_CACHE_BUCKET_URL +ARG WITH_FIPS="false" LABEL maintainer "Datadog " From af201abc9475d7f37e028f4c03709466a450de3f Mon Sep 17 00:00:00 2001 From: Joachim Date: Fri, 6 Dec 2024 18:00:36 +0000 Subject: [PATCH 09/52] Bugfix - recognize PDB as K8s resource (#31839) --- pkg/orchestrator/model/types.go | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pkg/orchestrator/model/types.go b/pkg/orchestrator/model/types.go index b8046b5ab73af..28687af8c460e 100644 --- a/pkg/orchestrator/model/types.go +++ b/pkg/orchestrator/model/types.go @@ -192,33 +192,34 @@ func (n NodeType) String() string { // Orchestrator returns the orchestrator name for a node type. func (n NodeType) Orchestrator() string { switch n { - case K8sCluster, + case K8sCR, + K8sCRD, + K8sCluster, + K8sClusterRole, + K8sClusterRoleBinding, K8sCronJob, - K8sDeployment, K8sDaemonSet, + K8sDeployment, + K8sHorizontalPodAutoscaler, + K8sIngress, K8sJob, + K8sLimitRange, + K8sNamespace, + K8sNetworkPolicy, K8sNode, - K8sPod, - K8sReplicaSet, - K8sService, - K8sStatefulSet, K8sPersistentVolume, K8sPersistentVolumeClaim, + K8sPod, + K8sPodDisruptionBudget, + K8sReplicaSet, K8sRole, K8sRoleBinding, - K8sClusterRole, - K8sClusterRoleBinding, + K8sService, K8sServiceAccount, - K8sIngress, - K8sCRD, - K8sCR, - K8sNamespace, - K8sVerticalPodAutoscaler, - K8sHorizontalPodAutoscaler, - K8sNetworkPolicy, - K8sLimitRange, + K8sStatefulSet, K8sStorageClass, - K8sUnsetType: + K8sUnsetType, + K8sVerticalPodAutoscaler: return "k8s" case ECSTask: return "ecs" From 4b5c8b9270fe4626702db6d66298a060176251d0 Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Fri, 6 Dec 2024 20:29:28 +0100 Subject: [PATCH 10/52] [serverless] Support trace context propagation for ALB target groups with MultiValueHeaders (#31542) Co-authored-by: quietsato --- pkg/serverless/trace/propagation/carriers.go | 10 ++++ .../trace/propagation/carriers_test.go | 51 ++++++++++++++++++- pkg/serverless/trace/propagation/extractor.go | 2 +- .../trace/propagation/extractor_test.go | 18 +++++++ pkg/serverless/trigger/events/events.go | 9 ++-- pkg/serverless/trigger/extractor.go | 13 ++++- pkg/serverless/trigger/extractor_test.go | 19 +++++++ 7 files changed, 114 insertions(+), 8 deletions(-) diff --git a/pkg/serverless/trace/propagation/carriers.go b/pkg/serverless/trace/propagation/carriers.go index 9cfa1255baabb..ee2062664780e 100644 --- a/pkg/serverless/trace/propagation/carriers.go +++ b/pkg/serverless/trace/propagation/carriers.go @@ -253,6 +253,16 @@ func headersCarrier(hdrs map[string]string) (tracer.TextMapReader, error) { return tracer.TextMapCarrier(hdrs), nil } +// headersOrMultiheadersCarrier returns the tracer.TextMapReader used to extract +// trace context from a Headers field of form map[string]string or MultiValueHeaders +// field of form map[string][]string. +func headersOrMultiheadersCarrier(hdrs map[string]string, multiHdrs map[string][]string) (tracer.TextMapReader, error) { + if len(hdrs) > 0 { + return headersCarrier(hdrs) + } + return tracer.HTTPHeadersCarrier(multiHdrs), nil +} + // extractTraceContextFromStepFunctionContext extracts the execution ARN, state name, and state entered time and uses them to generate Trace ID and Parent ID // The logic is based on the trace context conversion in Logs To Traces, dd-trace-py, dd-trace-js, etc. func extractTraceContextFromStepFunctionContext(event events.StepFunctionPayload) (*TraceContext, error) { diff --git a/pkg/serverless/trace/propagation/carriers_test.go b/pkg/serverless/trace/propagation/carriers_test.go index c58b294b74e39..16e382343f15f 100644 --- a/pkg/serverless/trace/propagation/carriers_test.go +++ b/pkg/serverless/trace/propagation/carriers_test.go @@ -816,7 +816,7 @@ func TestHeadersCarrier(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { tm, err := headersCarrier(tc.event) - t.Logf("rawPayloadCarrier returned TextMapReader=%#v error=%#v", tm, err) + t.Logf("headersCarrier returned TextMapReader=%#v error=%#v", tm, err) assert.Equal(t, tc.expErr != nil, err != nil) if tc.expErr != nil && err != nil { assert.Equal(t, tc.expErr.Error(), err.Error()) @@ -826,6 +826,55 @@ func TestHeadersCarrier(t *testing.T) { } } +func TestHeadersOrMultiheadersCarrier(t *testing.T) { + testcases := []struct { + name string + hdrs map[string]string + multiHdrs map[string][]string + expMap map[string]string + }{ + { + name: "nil-map", + hdrs: headersMapNone, + multiHdrs: toMultiValueHeaders(headersMapNone), + expMap: headersMapEmpty, + }, + { + name: "empty-map", + hdrs: headersMapEmpty, + multiHdrs: toMultiValueHeaders(headersMapEmpty), + expMap: headersMapEmpty, + }, + { + name: "headers-and-multiheaders", + hdrs: headersMapDD, + multiHdrs: toMultiValueHeaders(headersMapW3C), + expMap: headersMapDD, + }, + { + name: "just-headers", + hdrs: headersMapDD, + multiHdrs: toMultiValueHeaders(headersMapEmpty), + expMap: headersMapDD, + }, + { + name: "just-multiheaders", + hdrs: headersMapEmpty, + multiHdrs: toMultiValueHeaders(headersMapW3C), + expMap: headersMapW3C, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + tm, err := headersOrMultiheadersCarrier(tc.hdrs, tc.multiHdrs) + t.Logf("headersOrMultiheadersCarrier returned TextMapReader=%#v error=%#v", tm, err) + assert.Nil(t, err) + assert.Equal(t, tc.expMap, getMapFromCarrier(tm)) + }) + } +} + func Test_stringToDdSpanId(t *testing.T) { type args struct { execArn string diff --git a/pkg/serverless/trace/propagation/extractor.go b/pkg/serverless/trace/propagation/extractor.go index d6c756ae1693f..d9fe9b883275c 100644 --- a/pkg/serverless/trace/propagation/extractor.go +++ b/pkg/serverless/trace/propagation/extractor.go @@ -112,7 +112,7 @@ func (e Extractor) extract(event interface{}) (*TraceContext, error) { case events.APIGatewayCustomAuthorizerRequestTypeRequest: carrier, err = headersCarrier(ev.Headers) case events.ALBTargetGroupRequest: - carrier, err = headersCarrier(ev.Headers) + carrier, err = headersOrMultiheadersCarrier(ev.Headers, ev.MultiValueHeaders) case events.LambdaFunctionURLRequest: carrier, err = headersCarrier(ev.Headers) case events.StepFunctionPayload: diff --git a/pkg/serverless/trace/propagation/extractor_test.go b/pkg/serverless/trace/propagation/extractor_test.go index 8cdd07c6d9027..cdcb4bd39b3ec 100644 --- a/pkg/serverless/trace/propagation/extractor_test.go +++ b/pkg/serverless/trace/propagation/extractor_test.go @@ -176,6 +176,14 @@ var ( } ) +func toMultiValueHeaders(headers map[string]string) map[string][]string { + mvh := make(map[string][]string) + for k, v := range headers { + mvh[k] = []string{v} + } + return mvh +} + func TestNilPropagator(t *testing.T) { var extractor Extractor tc, err := extractor.Extract([]byte(`{"headers":` + headersAll + `}`)) @@ -533,6 +541,16 @@ func TestExtractorExtract(t *testing.T) { expCtx: ddTraceContext, expNoErr: true, }, + { + name: "ALBTargetGroupRequestMultiValueHeaders", + events: []interface{}{ + events.ALBTargetGroupRequest{ + MultiValueHeaders: toMultiValueHeaders(headersMapAll), + }, + }, + expCtx: ddTraceContext, + expNoErr: true, + }, // events.LambdaFunctionURLRequest: { diff --git a/pkg/serverless/trigger/events/events.go b/pkg/serverless/trigger/events/events.go index 03e4760b82044..cb55ea9691ba7 100644 --- a/pkg/serverless/trigger/events/events.go +++ b/pkg/serverless/trigger/events/events.go @@ -122,10 +122,11 @@ type APIGatewayCustomAuthorizerRequestTypeRequestContext struct { // ALBTargetGroupRequest mirrors events.ALBTargetGroupRequest type, removing // unused fields. type ALBTargetGroupRequest struct { - HTTPMethod string - Path string - Headers map[string]string - RequestContext ALBTargetGroupRequestContext + HTTPMethod string + Path string + Headers map[string]string + MultiValueHeaders map[string][]string + RequestContext ALBTargetGroupRequestContext } // ALBTargetGroupRequestContext mirrors events.ALBTargetGroupRequestContext diff --git a/pkg/serverless/trigger/extractor.go b/pkg/serverless/trigger/extractor.go index 1652c176245e3..6632628d69441 100644 --- a/pkg/serverless/trigger/extractor.go +++ b/pkg/serverless/trigger/extractor.go @@ -188,14 +188,23 @@ func GetTagsFromALBTargetGroupRequest(event events.ALBTargetGroupRequest) map[st httpTags := make(map[string]string) httpTags["http.url_details.path"] = event.Path httpTags["http.method"] = event.HTTPMethod + if event.Headers != nil { - if event.Headers["Referer"] != "" { - httpTags["http.referer"] = event.Headers["Referer"] + if r := event.Headers["Referer"]; r != "" { + httpTags["http.referer"] = r } if ua := event.Headers["User-Agent"]; ua != "" { httpTags["http.useragent"] = ua } + } else if event.MultiValueHeaders != nil { + if r := event.MultiValueHeaders["Referer"]; len(r) > 0 && r[0] != "" { + httpTags["http.referer"] = r[0] + } + if ua := event.MultiValueHeaders["User-Agent"]; len(ua) > 0 && ua[0] != "" { + httpTags["http.useragent"] = ua[0] + } } + return httpTags } diff --git a/pkg/serverless/trigger/extractor_test.go b/pkg/serverless/trigger/extractor_test.go index 34cb6f4b3c816..91234c92ed997 100644 --- a/pkg/serverless/trigger/extractor_test.go +++ b/pkg/serverless/trigger/extractor_test.go @@ -353,6 +353,25 @@ func TestGetTagsFromALBTargetGroupRequest(t *testing.T) { }, httpTags) } +func TestGetTagsFromALBTargetGroupRequestMultiValueHeaders(t *testing.T) { + event := events.ALBTargetGroupRequest{ + MultiValueHeaders: map[string][]string{ + "key": {"val"}, + "Referer": {"referer"}, + }, + Path: "path", + HTTPMethod: "http-method", + } + + httpTags := GetTagsFromALBTargetGroupRequest(event) + + assert.Equal(t, map[string]string{ + "http.url_details.path": "path", + "http.method": "http-method", + "http.referer": "referer", + }, httpTags) +} + func TestGetTagsFromFunctionURLRequest(t *testing.T) { event := events.LambdaFunctionURLRequest{ Headers: map[string]string{ From 858cb1cae75f20afd261ed5d4c8bca70bd6e0c3d Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Fri, 6 Dec 2024 14:51:35 -0500 Subject: [PATCH 11/52] Deprecate Windows .bat scripts (#31732) --- .github/CODEOWNERS | 1 + .gitlab/package_build/installer.yml | 2 +- .gitlab/package_build/windows.yml | 7 +- Dockerfiles/agent/windows/README.md | 2 +- docs/dev/agent_omnibus.md | 6 +- tasks/__init__.py | 2 + tasks/winbuild.py | 71 +++++ tasks/winbuildscripts/Build-AgentPackages.ps1 | 83 ++++++ .../Build-InstallerPackages.ps1 | 72 ++++++ tasks/winbuildscripts/Build-OmnibusTarget.ps1 | 90 +++++++ tasks/winbuildscripts/Generate-OCIPackage.ps1 | 11 +- tasks/winbuildscripts/buildinstaller.bat | 59 ----- tasks/winbuildscripts/buildlocal.bat | 14 - tasks/winbuildscripts/buildwin.bat | 39 --- tasks/winbuildscripts/common.ps1 | 242 ++++++++++++++++++ tasks/winbuildscripts/dobuild.bat | 62 ----- tasks/winbuildscripts/libyajl2_install.ps1 | 26 -- 17 files changed, 576 insertions(+), 213 deletions(-) create mode 100644 tasks/winbuild.py create mode 100644 tasks/winbuildscripts/Build-AgentPackages.ps1 create mode 100644 tasks/winbuildscripts/Build-InstallerPackages.ps1 create mode 100644 tasks/winbuildscripts/Build-OmnibusTarget.ps1 delete mode 100644 tasks/winbuildscripts/buildinstaller.bat delete mode 100644 tasks/winbuildscripts/buildlocal.bat delete mode 100644 tasks/winbuildscripts/buildwin.bat create mode 100644 tasks/winbuildscripts/common.ps1 delete mode 100644 tasks/winbuildscripts/dobuild.bat delete mode 100644 tasks/winbuildscripts/libyajl2_install.ps1 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ad6e11974b350..202c143c2da56 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -570,6 +570,7 @@ /tasks/sds.py @DataDog/agent-processing-and-routing /tasks/systray.py @DataDog/windows-agent /tasks/winbuildscripts/ @DataDog/windows-agent +/tasks/winbuild.py @DataDog/windows-agent /tasks/windows_resources.py @DataDog/windows-agent /tasks/components.py @DataDog/agent-shared-components /tasks/components_templates @DataDog/agent-shared-components diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index 443b2fe52318b..6ab10c1a19e62 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -220,7 +220,7 @@ windows-installer-amd64: -e USE_S3_CACHING="$USE_S3_CACHING" -e API_KEY_ORG2=${API_KEY_ORG2} registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildinstaller.bat + powershell -C "c:\mnt\tasks\winbuildscripts\Build-InstallerPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" after_script: - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' artifacts: diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml index 16734f170f1da..cb57ebafa617f 100644 --- a/.gitlab/package_build/windows.yml +++ b/.gitlab/package_build/windows.yml @@ -32,14 +32,13 @@ -e S3_OMNIBUS_CACHE_BUCKET="$S3_OMNIBUS_CACHE_BUCKET" -e USE_S3_CACHING="$USE_S3_CACHING" -e INTEGRATION_WHEELS_CACHE_BUCKET="$INTEGRATION_WHEELS_CACHE_BUCKET" - -e GO_VERSION_CHECK="true" -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG} -e PIP_INDEX_URL=${PIP_INDEX_URL} -e API_KEY_ORG2=${API_KEY_ORG2} -e OMNIBUS_GIT_CACHE_DIR=${Env:TEMP}/${CI_PIPELINE_ID}/omnibus-git-cache -e AGENT_FLAVOR=${AGENT_FLAVOR} registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildwin.bat + powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - get-childitem omnibus\pkg - !reference [.upload_sbom_artifacts_windows] @@ -89,7 +88,7 @@ windows_zip_agent_binaries_x64-a7: needs: ["go_mod_tidy_check", "go_deps"] variables: ARCH: "x64" - OMNIBUS_TARGET: agent_binaries + OMNIBUS_TARGET: agent-binaries before_script: - set RELEASE_VERSION $RELEASE_VERSION_7 script: @@ -122,7 +121,7 @@ windows_zip_agent_binaries_x64-a7: -e PIP_INDEX_URL=${PIP_INDEX_URL} -e API_KEY_ORG2=${API_KEY_ORG2} registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\buildwin.bat + powershell -C "c:\mnt\tasks\winbuildscripts\Build-OmnibusTarget.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - get-childitem omnibus\pkg - !reference [.upload_sbom_artifacts_windows] diff --git a/Dockerfiles/agent/windows/README.md b/Dockerfiles/agent/windows/README.md index e33eeda9bd4e0..7c966fd0294c2 100644 --- a/Dockerfiles/agent/windows/README.md +++ b/Dockerfiles/agent/windows/README.md @@ -6,7 +6,7 @@ How to build the Agent docker image From the root of the repository, run the following command: ``` -docker run --rm -it -v "${pwd}:c:\mnt" -e OMNIBUS_TARGET=main -e MAJOR_VERSION=7 -e RELEASE_VERSION=nightly -e PY_RUNTIMES=3 datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat +docker run --rm -it -v "${pwd}:c:\mnt" -e OMNIBUS_TARGET=main -e MAJOR_VERSION=7 -e RELEASE_VERSION=nightly-a7 -e PY_RUNTIMES=3 datadog/agent-buildimages-windows_x64:1809 powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" ``` The build artifacts will be in `omnibus\pkg`. diff --git a/docs/dev/agent_omnibus.md b/docs/dev/agent_omnibus.md index 68ff17b6e3724..421303cbfc173 100644 --- a/docs/dev/agent_omnibus.md +++ b/docs/dev/agent_omnibus.md @@ -90,7 +90,7 @@ Start a Powershell prompt and navigate to your local clone of the `datadog-agent Run the following command: ```powershell -docker run -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=main -e RELEASE_VERSION=nightly -e MAJOR_VERSION=7 -e TARGET_ARCH=x64 datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat +docker run -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=main -e RELEASE_VERSION=nightly-a7 -e MAJOR_VERSION=7 -e TARGET_ARCH=x64 datadog/agent-buildimages-windows_x64:1809 powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1" ``` Downloading the Docker image may take some time in the first run. @@ -100,7 +100,7 @@ Alternatively here's a small Powershell script to facilitate using the docker im param ( [int]$MAJOR_VERSION=7, $TARGET_ARCH="x64", - $RELEASE_VERSION="nightly", + $RELEASE_VERSION="nightly-a7", [bool]$RM_CONTAINER=$true, [bool]$DEBUG=$false ) @@ -113,7 +113,7 @@ $opts = "-e OMNIBUS_TARGET=main -e RELEASE_VERSION=$RELEASE_VERSION -e MAJOR_VER if ($DEBUG) { $opts += " -e DEBUG_CUSTOMACTION=yes " } -$cmd += " -m 8192M -v ""$(Get-Location):c:\mnt"" $opts datadog/agent-buildimages-windows_x64:1809 c:\mnt\tasks\winbuildscripts\buildwin.bat" +$cmd += " -m 8192M -v ""$(Get-Location):c:\mnt"" $opts datadog/agent-buildimages-windows_x64:1809 powershell -C ""c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1""" Write-Host $cmd Invoke-Expression -Command $cmd ``` diff --git a/tasks/__init__.py b/tasks/__init__.py index 44adcefb1dcad..d59a06de7463b 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -63,6 +63,7 @@ trace_agent, vim, vscode, + winbuild, worktree, ) from tasks.build_tags import audit_tag_impact, print_default_build_tags @@ -212,6 +213,7 @@ ns.add_collection(collector) ns.add_collection(invoke_unit_tests) ns.add_collection(debug) +ns.add_collection(winbuild) ns.add_collection(worktree) ns.configure( { diff --git a/tasks/winbuild.py b/tasks/winbuild.py new file mode 100644 index 0000000000000..66f0cb02b68a1 --- /dev/null +++ b/tasks/winbuild.py @@ -0,0 +1,71 @@ +import os +import shutil + +from invoke.tasks import task + +from tasks.flavor import AgentFlavor +from tasks.libs.common.utils import get_version +from tasks.msi import build as build_agent_msi +from tasks.msi import build_installer as build_installer_msi +from tasks.omnibus import build as omnibus_build + +# Output directory for package files +OUTPUT_PATH = os.path.join(os.getcwd(), "omnibus", "pkg") +# Omnibus stores files here, e.g. C:\opt\datadog-agent, C:\opt\dataog-installer +OPT_SOURCE_DIR = os.path.join('C:\\', 'opt') + + +@task +def agent_package( + ctx, + flavor=AgentFlavor.base.name, + release_version="nightly-a7", + skip_deps=False, +): + # Build agent + omnibus_build( + ctx, + flavor=flavor, + release_version=release_version, + skip_deps=skip_deps, + ) + + # Package Agent into MSI + build_agent_msi(ctx, release_version=release_version) + + # Package MSI into OCI + if AgentFlavor[flavor] == AgentFlavor.base: + ctx.run('powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package datadog-agent"') + + +@task +def installer_package( + ctx, + release_version="nightly-a7", + skip_deps=False, +): + # Build installer + omnibus_build( + ctx, + release_version=release_version, + skip_deps=skip_deps, + target_project="installer", + ) + + # Package Insaller into MSI + build_installer_msi(ctx) + + # Package MSI into OCI + ctx.run('powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package datadog-installer"') + + # Copy installer.exe to the output dir so it can be deployed as the bootstrapper + agent_version = get_version( + ctx, + include_git=True, + url_safe=True, + include_pipeline_id=True, + ) + shutil.copy2( + os.path.join(OPT_SOURCE_DIR, "datadog-installer\\datadog-installer.exe"), + os.path.join(OUTPUT_PATH, f"datadog-installer-{agent_version}-1-x86_64.exe"), + ) diff --git a/tasks/winbuildscripts/Build-AgentPackages.ps1 b/tasks/winbuildscripts/Build-AgentPackages.ps1 new file mode 100644 index 0000000000000..34c2188c8249b --- /dev/null +++ b/tasks/winbuildscripts/Build-AgentPackages.ps1 @@ -0,0 +1,83 @@ +<# +.SYNOPSIS +Builds the Datadog Agent packages for Windows. Builds everything with omnibus and packages the output into MSI, ZIP, and OCI. + +.DESCRIPTION +This script builds the Datadog Agent packages for Windows, with options to configure the build environment. + +.PARAMETER ReleaseVersion +Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION. + +.PARAMETER Flavor +Specifies the flavor of the agent. Default is the value of the environment variable AGENT_FLAVOR. + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.EXAMPLE +.\Build-AgentPackages.ps1 -InstallDeps $false + +.EXAMPLE +.\Build-AgentPackages.ps1 -BuildOutOfSource $true -InstallDeps $true -Flavor "fips" -CheckGoVersion $true + +.NOTES +This script should be run from the root of the repository. + +#> +param( + [bool] $BuildOutOfSource = $false, + [nullable[bool]] $CheckGoVersion, + [bool] $InstallDeps = $true, + [string] $ReleaseVersion = $env:RELEASE_VERSION, + [string] $Flavor = $env:AGENT_FLAVOR +) + +. "$PSScriptRoot\common.ps1" + +Invoke-BuildScript ` + -BuildOutOfSource $BuildOutOfSource ` + -InstallDeps $InstallDeps ` + -CheckGoVersion $CheckGoVersion ` + -Command { + $inv_args = @( + "--skip-deps" + ) + if ($ReleaseVersion) { + $inv_args += "--release-version" + $inv_args += $ReleaseVersion + $env:RELEASE_VERSION=$ReleaseVersion + } + + if ($Flavor) { + $inv_args += "--flavor" + $inv_args += $Flavor + $env:AGENT_FLAVOR=$Flavor + } + + Write-Host "inv -e winbuild.agent-package $inv_args" + inv -e winbuild.agent-package @inv_args + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to build the agent package" + exit 1 + } + + # Show the contents of the output package directories for debugging purposes + Get-ChildItem -Path C:\omnibus-ruby\pkg\ + Get-ChildItem -Path "C:\opt\datadog-agent\bin\agent\" + Get-ChildItem -Path ".\omnibus\pkg\" + + if ($BuildOutOfSource) { + # Copy the resulting package to the mnt directory + mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null + Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop + } +} diff --git a/tasks/winbuildscripts/Build-InstallerPackages.ps1 b/tasks/winbuildscripts/Build-InstallerPackages.ps1 new file mode 100644 index 0000000000000..bf381a8ef5f8a --- /dev/null +++ b/tasks/winbuildscripts/Build-InstallerPackages.ps1 @@ -0,0 +1,72 @@ +<# +.SYNOPSIS +Builds the Datadog Installer packages for Windows. Builds everything with omnibus and packages the output into MSI, ZIP, and OCI. + +.DESCRIPTION +This script builds the Datadog Installer packages for Windows, with options to configure the build environment. + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER ReleaseVersion +Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.EXAMPLE +.\Build-InstallerPackages.ps1 -InstallDeps $false + +.EXAMPLE +.\Build-InstallerPackages.ps1 -BuildOutOfSource $true -InstallDeps $true -CheckGoVersion $true + +.NOTES +This script should be run from the root of the repository. + +#> +param( + [bool] $BuildOutOfSource = $false, + [nullable[bool]] $CheckGoVersion, + [bool] $InstallDeps = $true, + [string] $ReleaseVersion = $env:RELEASE_VERSION +) + +. "$PSScriptRoot\common.ps1" + +Invoke-BuildScript ` + -BuildOutOfSource $BuildOutOfSource ` + -InstallDeps $InstallDeps ` + -CheckGoVersion $CheckGoVersion ` + -Command { + $inv_args = @( + "--skip-deps" + ) + if ($ReleaseVersion) { + $inv_args += "--release-version" + $inv_args += $ReleaseVersion + } + + Write-Host "inv -e winbuild.installer-package $inv_args" + inv -e winbuild.installer-package @inv_args + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to build the agent package" + exit 1 + } + + # Show the contents of the output package directories for debugging purposes + Get-ChildItem -Path C:\omnibus-ruby\pkg\ + Get-ChildItem -Path C:\opt\datadog-installer + Get-ChildItem -Path ".\omnibus\pkg\" + + if ($BuildOutOfSource) { + # Copy the resulting package to the mnt directory + mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null + Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop + } +} diff --git a/tasks/winbuildscripts/Build-OmnibusTarget.ps1 b/tasks/winbuildscripts/Build-OmnibusTarget.ps1 new file mode 100644 index 0000000000000..50dfc63f30e23 --- /dev/null +++ b/tasks/winbuildscripts/Build-OmnibusTarget.ps1 @@ -0,0 +1,90 @@ +<# +.SYNOPSIS +Builds an Omnibus project for Windows. + +.DESCRIPTION +This script builds an Omnibus Project for Windows, with options to configure the build environment. + +.PARAMETER ReleaseVersion +Specifies the release version of the build. Default is the value of the environment variable RELEASE_VERSION. + +.PARAMETER Flavor +Specifies the flavor of the agent. Default is the value of the environment variable AGENT_FLAVOR. + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.PARAMETER TargetProject +Specifies the target project for the build. This parameter is mandatory and defaults to the value of the environment variable OMNIBUS_TARGET. + +.EXAMPLE +.\Build-OmnibusTarget.ps1 -InstallDeps $false + +.EXAMPLE +.\Build-OmnibusTarget.ps1 -BuildOutOfSource $true -InstallDeps $true -Flavor "fips" -CheckGoVersion $true + +.NOTES +This script should be run from the root of the repository. + +#> +param( + [bool] $BuildOutOfSource = $false, + [nullable[bool]] $CheckGoVersion, + [bool] $InstallDeps = $true, + [string] $ReleaseVersion = $env:RELEASE_VERSION, + [string] $TargetProject = $env:OMNIBUS_TARGET +) + +. "$PSScriptRoot\common.ps1" + +Invoke-BuildScript ` + -BuildOutOfSource $BuildOutOfSource ` + -InstallDeps $InstallDeps ` + -CheckGoVersion $CheckGoVersion ` + -Command { + $inv_args = @( + "--skip-deps" + ) + if ($ReleaseVersion) { + $inv_args += "--release-version" + $inv_args += $ReleaseVersion + $env:RELEASE_VERSION=$ReleaseVersion + } + + if ($TargetProject) { + $inv_args += "--target-project" + $inv_args += $TargetProject + $env:OMNIBUS_TARGET=$TargetProject + } else { + Write-Error "Target project is required" + Write-Error "To build the (default) Agent package, use Build-AgentPackages.ps1 instead" + exit 1 + } + + Write-Host "inv -e omnibus.build $inv_args" + inv -e omnibus.build @inv_args + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to build the agent package" + exit 1 + } + + # Show the contents of the output package directories for debugging purposes + Get-ChildItem -Path C:\omnibus-ruby\pkg\ + Get-ChildItem -Path "C:\opt\datadog-agent\bin\agent\" + Get-ChildItem -Path ".\omnibus\pkg\" + + if ($BuildOutOfSource) { + # Copy the resulting package to the mnt directory + mkdir C:\mnt\omnibus\pkg -Force -ErrorAction Stop | Out-Null + Copy-Item -Path ".\omnibus\pkg\*" -Destination "C:\mnt\omnibus\pkg" -Force -ErrorAction Stop + } +} diff --git a/tasks/winbuildscripts/Generate-OCIPackage.ps1 b/tasks/winbuildscripts/Generate-OCIPackage.ps1 index ee48f4badc9fe..8b64223788b3e 100644 --- a/tasks/winbuildscripts/Generate-OCIPackage.ps1 +++ b/tasks/winbuildscripts/Generate-OCIPackage.ps1 @@ -1,11 +1,10 @@ Param( [Parameter(Mandatory=$true)] [string] $package, - [string] $version + [string] $version, + [string] $omnibusOutput = "$(Get-Location)\omnibus\pkg\" ) -$omnibusOutput = "$($Env:REPO_ROOT)\omnibus\pkg\" - if (-not (Test-Path C:\tools\datadog-package.exe)) { Write-Host "Downloading datadog-package.exe" (New-Object System.Net.WebClient).DownloadFile("https://dd-agent-omnibus.s3.amazonaws.com/datadog-package.exe", "C:\\tools\\datadog-package.exe") @@ -26,10 +25,14 @@ if (Test-Path $omnibusOutput\$packageName) { # datadog-package takes a folder as input and will package everything in that, so copy the msi to its own folder Remove-Item -Recurse -Force C:\oci-pkg -ErrorAction SilentlyContinue -New-Item -ItemType Directory C:\oci-pkg +New-Item -ItemType Directory C:\oci-pkg | Out-Null Copy-Item (Get-ChildItem $omnibusOutput\${package}-${version}-x86_64.msi).FullName -Destination C:\oci-pkg\${package}-${version}-x86_64.msi # The argument --archive-path ".\omnibus\pkg\datadog-agent-${version}.tar.gz" is currently broken and has no effects & C:\tools\datadog-package.exe create --package $package --os windows --arch amd64 --archive --version $version C:\oci-pkg +if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to create OCI package" + exit 1 +} Move-Item ${package}-${version}-windows-amd64.tar $omnibusOutput\$packageName diff --git a/tasks/winbuildscripts/buildinstaller.bat b/tasks/winbuildscripts/buildinstaller.bat deleted file mode 100644 index bd41ccad5dc2a..0000000000000 --- a/tasks/winbuildscripts/buildinstaller.bat +++ /dev/null @@ -1,59 +0,0 @@ -if not exist c:\mnt\ goto nomntdir -@echo on -@echo c:\mnt found, continuing -@echo PARAMS %* -@echo RELEASE_VERSION %RELEASE_VERSION% - -set BUILD_ROOT=c:\buildroot -set REPO_ROOT=%BUILD_ROOT%\datadog-agent -mkdir %REPO_ROOT% -if not exist %REPO_ROOT% exit /b 2 -cd %REPO_ROOT% || exit /b 3 -xcopy /e/s/h/q c:\mnt\*.* || exit /b 4 - -call %BUILD_ROOT%\tasks\winbuildscripts\extract-modcache.bat %REPO_ROOT% modcache - -set OMNIBUS_BUILD=omnibus.build -@rem OMNIBUS_TARGET is also used in the C# code to only produce the .cmd for the Datadog Installer (instead of for both the Agent installer and the Datadog installer). -@rem It's not strictly needed, as we will only invoke the .cmd for the Datadog Installer in the invoke task build-installer, but it's a good practice to be consistent. -set OMNIBUS_TARGET=installer -set OMNIBUS_ARGS=%OMNIBUS_ARGS% --target-project %OMNIBUS_TARGET% -@rem Have to use arcane syntax to store AGENT_VERSION, see https://ss64.com/nt/for_cmd.html -FOR /F "tokens=*" %%g IN ('inv agent.version --url-safe --major-version 7') do (SET AGENT_VERSION=%%g) - -if DEFINED GOMODCACHE set OMNIBUS_ARGS=%OMNIBUS_ARGS% --go-mod-cache %GOMODCACHE% -if DEFINED USE_S3_CACHING set OMNIBUS_ARGS=%OMNIBUS_ARGS% %USE_S3_CACHING% - -SET PATH=%PATH%;%GOPATH%/bin -REM AGENT_MSI_OUTDIR is always overridden in msi.py - -@echo GOPATH %GOPATH% -@echo PATH %PATH% -@echo VSTUDIO_ROOT %VSTUDIO_ROOT% - -call ridk enable -pip3 install -r requirements.txt - -@echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION%" -inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION% || exit /b 1 -inv -e msi.build-installer || exit /b 2 - -Powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package 'datadog-installer'" - -REM show output package directories (for debugging) -dir \omnibus-ruby\pkg\ -dir C:\opt\datadog-installer\ -dir %REPO_ROOT%\omnibus\pkg\ - -REM copy resulting packages to expected location for collection by gitlab. -if not exist c:\mnt\omnibus\pkg\ mkdir c:\mnt\omnibus\pkg\ || exit /b 5 -copy %REPO_ROOT%\omnibus\pkg\* c:\mnt\omnibus\pkg\ || exit /b 6 -REM Save the installer.exe for bootstrapping -copy C:\opt\datadog-installer\datadog-installer.exe c:\mnt\omnibus\pkg\datadog-installer-%AGENT_VERSION%-1-x86_64.exe || exit /b 7 - -goto :EOF - -:nomntdir -@echo directory not mounted, parameters incorrect -exit /b 1 -goto :EOF diff --git a/tasks/winbuildscripts/buildlocal.bat b/tasks/winbuildscripts/buildlocal.bat deleted file mode 100644 index c2aaab3b4bfa8..0000000000000 --- a/tasks/winbuildscripts/buildlocal.bat +++ /dev/null @@ -1,14 +0,0 @@ -@echo RELEASE_VERSION %RELEASE_VERSION% -@echo MAJOR_VERSION %MAJOR_VERSION% - -REM set up variables for local build. -REM assumes attempting to build A7/x64 nightly -REM assumes target directory is mounted in the container -REM (vs. copied in as in CI build) -if NOT DEFINED RELEASE_VERSION set RELEASE_VERSION=nightly -if NOT DEFINED MAJOR_VERSION set MAJOR_VERSION=7 -if NOT DEFINED CI_JOB_ID set CI_JOB_ID=1 -if NOT DEFINED TARGET_ARCH set TARGET_ARCH=x64 - -call %~dp0dobuild.bat -if not %ERRORLEVEL% == 0 @echo "Build failed %ERRORLEVEL%" diff --git a/tasks/winbuildscripts/buildwin.bat b/tasks/winbuildscripts/buildwin.bat deleted file mode 100644 index e23c72d579db4..0000000000000 --- a/tasks/winbuildscripts/buildwin.bat +++ /dev/null @@ -1,39 +0,0 @@ -if not exist c:\mnt\ goto nomntdir -@echo on -@echo c:\mnt found, continuing - -set BUILD_ROOT=c:\buildroot -mkdir %BUILD_ROOT%\datadog-agent -if not exist %BUILD_ROOT%\datadog-agent exit /b 2 -cd %BUILD_ROOT%\datadog-agent || exit /b 3 -xcopy /e/s/h/q c:\mnt\*.* || exit /b 4 - -call %BUILD_ROOT%\datadog-agent\tasks\winbuildscripts\extract-modcache.bat %BUILD_ROOT%\datadog-agent modcache - -REM -REM after copying files in from the host, execute the build -REM using `dobuild.bat` -REM -call %BUILD_ROOT%\datadog-agent\tasks\winbuildscripts\dobuild.bat %* -if not %ERRORLEVEL% == 0 exit /b %ERRORLEVEL% - -REM show output package directories (for debugging) -dir \omnibus-ruby\pkg\ - -dir %BUILD_ROOT%\datadog-agent\omnibus\pkg\ - -REM copy resulting packages to expected location for collection by gitlab. -if not exist c:\mnt\omnibus\pkg\ mkdir c:\mnt\omnibus\pkg\ || exit /b 5 -copy %BUILD_ROOT%\datadog-agent\omnibus\pkg\* c:\mnt\omnibus\pkg\ || exit /b 6 - -REM show output binary directories (for debugging) -dir C:\opt\datadog-agent\bin\agent\ - -goto :EOF - -:nomntdir -@echo directory not mounted, parameters incorrect -exit /b 1 -goto :EOF - - diff --git a/tasks/winbuildscripts/common.ps1 b/tasks/winbuildscripts/common.ps1 new file mode 100644 index 0000000000000..35ef863d57535 --- /dev/null +++ b/tasks/winbuildscripts/common.ps1 @@ -0,0 +1,242 @@ +<# +.SYNOPSIS +Copies files from C:\mnt into C:\buildroot\datadog-agent and sets the current directory to the buildroot. + +.PARAMETER buildroot +Specifies the root directory where the files will be copied to. The default value is "c:\buildroot". + +.NOTES +This function is typically used in the context of building and running scripts within a containerized environment to +ensure that the files are copied to the container filesystem before running the build scripts. This is useful for +keeping the source clean and can provide speed improvements for hyper-v based containers. +See also, issue with job cross-contamination due to improperly cancelled jobs: https://datadoghq.atlassian.net/browse/CIEXE-143 +#> +function Enter-BuildRoot() { + param( + [string] $buildroot = "c:\buildroot" + ) + if (-Not (Test-Path -Path "c:\mnt")) { + Write-Error "C:\mnt directory not mounted, parameters incorrect" + exit 1 + } + + # copy to buildroot + mkdir -Force "$buildroot\datadog-agent" -ErrorAction Stop | Out-Null + if (-Not (Test-Path -Path "$buildroot\datadog-agent")) { + Write-Error "Failed to create buildroot directory" + exit 2 + } + + # copy the repository into the container filesystem + Write-Host "Switching to buildroot $buildroot\datadog-agent" + Push-Location "$buildroot\datadog-agent" -ErrorAction Stop -StackName AgentBuildRoot + xcopy /e/s/h/q c:\mnt\*.* +} + +<# +.SYNOPSIS +Leaves the buildroot directory and returns to the original working directory. +#> +function Exit-BuildRoot() { + Write-Host "Leaving buildroot" + Pop-Location -StackName AgentBuildRoot +} + +<# +.SYNOPSIS +Expands the Go module cache from an archive file. + +.DESCRIPTION +This function expands the Go module cache from an archive file located in the specified root directory. +It extracts the contents of the archive file into the Go module cache directory defined by the GOMODCACHE environment variable. + +.PARAMETER root +The root directory where the tar.xz file is located. Defaults to the current location. + +.PARAMETER modcache +The base name (without extension) of the file to be expanded. Expected values are `modcache` and `modcache_tools`. + +.NOTES +If the GOMODCACHE environment variable is not set, the function will skip the expansion process. + +.EXAMPLE +Expand-ModCache -modcache "modcache" +This will expand the modcache file located at "\modcache.tar.xz" into the Go module cache directory. + +#> +function Expand-ModCache() { + param( + [string] $root = (Get-Location).Path, + [ValidateSet('modcache', 'modcache_tools')] + [string] $modcache + ) + + $MODCACHE_ROOT = $root + $MODCACHE_FILE_ROOT = $modcache + $MODCACHE_XZ_FILE = Join-Path $MODCACHE_ROOT "$MODCACHE_FILE_ROOT.tar.xz" + $MODCACHE_TAR_FILE = Join-Path $MODCACHE_ROOT "$MODCACHE_FILE_ROOT.tar" + + if (-not $env:GOMODCACHE) { + Write-Host "GOMODCACHE environment variable not set, skipping expansion of mod cache files" + return + } + + Write-Host "MODCACHE_XZ_FILE $MODCACHE_XZ_FILE MODCACHE_TAR_FILE $MODCACHE_TAR_FILE GOMODCACHE $env:GOMODCACHE" + if (Test-Path $MODCACHE_XZ_FILE) { + Write-Host "Extracting modcache file $MODCACHE_XZ_FILE" + & 7z.exe x $MODCACHE_XZ_FILE -o"$MODCACHE_ROOT" -bt + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to extract $MODCACHE_XZ_FILE" + exit 1 + } + Get-ChildItem $MODCACHE_TAR_FILE + # Use -aoa to allow overwriting existing files + # This shouldn't have any negative impact: since modules are + # stored per version and hash, files that get replaced will + # get replaced by the same files + & 7z.exe x $MODCACHE_TAR_FILE -o"$env:GOMODCACHE\cache" -aoa -bt + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to extract $MODCACHE_XZ_FILE" + exit 1 + } + Write-Host "Modcache extracted" + } else { + Write-Host "Modcache XZ file $MODCACHE_XZ_FILE not found, dependencies will be downloaded" + } + + if (Test-Path $MODCACHE_XZ_FILE) { + Write-Host "Deleting modcache tar.xz $MODCACHE_XZ_FILE" + Remove-Item -Force $MODCACHE_XZ_FILE + } + if (Test-Path $MODCACHE_TAR_FILE) { + Write-Host "Deleting modcache tar $MODCACHE_TAR_FILE" + Remove-Item -Force $MODCACHE_TAR_FILE + } +} + +function Install-Deps() { + Write-Host "Installing python requirements" + pip3.exe install -r .\requirements.txt + Write-Host "Installing go dependencies" + inv -e deps +} + +function Enable-DevEnv() { + # Add go bin to PATH for golangci-lint and other go tools + if (-Not $env:GOPATH) { + Write-Host "GOPATH not set, setting to C:\dev\go" + $env:GOPATH = "C:\dev\go" + } + $env:PATH = "$env:GOPATH\bin;$env:PATH" + + # Enable ruby/msys environment, for mingw, make, etc. + ridk enable +} + +<# +.SYNOPSIS +Converts a string value to a boolean value based on specific conditions. + +.DESCRIPTION +This function takes a string input and a default boolean value. +- If the input string is null or empty, it returns the default boolean value. +- If the input string is "true", "yes", or "1" (case insensitive), it returns $true. +- Otherwise, it returns $false. +#> +function Convert-StringToBool() { + param( + [string] $Value, + [bool] $DefaultValue + ) + + if ([string]::IsNullOrEmpty($Value)) { + return $DefaultValue + } + + if ($Value.ToLower() -eq "true") { + return $true + } + + if ($Value.ToLower() -eq "yes" -or $Value -eq "1") { + return $true + } + + return $false +} + +<# +.SYNOPSIS +Invokes a build script with optional parameters for build environment configuration. + +.DESCRIPTION +The Invoke-BuildScript function sets up the build environment, optionally installs dependencies, checks the Go version, and executes a provided script block. It supports building out of source and restores the original working directory upon completion. + +.PARAMETER buildroot +Specifies the root directory for the build. Default is "c:\buildroot". + +.PARAMETER BuildOutOfSource +Specifies whether to build out of source. Default is $false. + +Use this option in the CI to keep the job directory clean and avoid conflicts/stale data. +Use this option in Hyper-V based containers to improve build performance. + +.PARAMETER InstallDeps +Specifies whether to install dependencies (python requirements, go deps, etc.). Default is $true. + +.PARAMETER CheckGoVersion +Specifies whether to check the Go version. If not provided, it defaults to the value of the environment variable GO_VERSION_CHECK or $true if the environment variable is not set. + +.PARAMETER Command +A script block containing the commands to execute as part of the build process. + +.EXAMPLE +Invoke-BuildScript -buildroot "c:\mybuild" -BuildOutOfSource $true -Command { ./build.ps1 } + +#> +function Invoke-BuildScript { + param( + [string] $buildroot = "c:\buildroot", + [bool] $BuildOutOfSource = $false, + [bool] $InstallDeps = $true, + [nullable[bool]] $CheckGoVersion, + [ScriptBlock] $Command = {$null} + ) + + try { + if ($null -eq $CheckGoVersion) { + $CheckGoVersion = Convert-StringToBool -Value $env:GO_VERSION_CHECK -default $true + } + + if ($BuildOutOfSource) { + Enter-BuildRoot + } + + Expand-ModCache -modcache modcache + + Enable-DevEnv + + if ($InstallDeps) { + Install-Deps + } + + if ($CheckGoVersion) { + inv -e check-go-version + if ($LASTEXITCODE -ne 0) { + Write-Error "Go version check failed" + exit 1 + } + } + + # Execute the provided ScriptBlock/Command + & $Command + + } finally { + # This finally block is executed regardless of whether the try block completes successfully, throws an exception, + # or uses `exit` to terminate the script. + + if ($BuildOutOfSource) { + # Restore the original working directory + Exit-BuildRoot + } + } +} diff --git a/tasks/winbuildscripts/dobuild.bat b/tasks/winbuildscripts/dobuild.bat deleted file mode 100644 index 0ed92222f1f71..0000000000000 --- a/tasks/winbuildscripts/dobuild.bat +++ /dev/null @@ -1,62 +0,0 @@ -@echo PARAMS %* -@echo RELEASE_VERSION %RELEASE_VERSION% -@echo MAJOR_VERSION %MAJOR_VERSION% -@echo GO_VERSION_CHECK %GO_VERSION_CHECK% - -if NOT DEFINED RELEASE_VERSION set RELEASE_VERSION=%~1 -if NOT DEFINED MAJOR_VERSION set MAJOR_VERSION=%~2 -if NOT DEFINED GO_VERSION_CHECK set GO_VERSION_CHECK=%~4 - -set OMNIBUS_BUILD=omnibus.build - -if "%OMNIBUS_TARGET%" == "" set OMNIBUS_TARGET=main -if "%OMNIBUS_TARGET%" == "iot" set OMNIBUS_ARGS=--flavor iot -if "%OMNIBUS_TARGET%" == "dogstatsd" set OMNIBUS_ARGS=--target-project dogstatsd -if "%OMNIBUS_TARGET%" == "agent_binaries" set OMNIBUS_ARGS=%OMNIBUS_ARGS% --target-project agent-binaries -if "%AGENT_FLAVOR%" == "fips" set OMNIBUS_ARGS=--flavor fips - -if DEFINED GOMODCACHE set OMNIBUS_ARGS=%OMNIBUS_ARGS% --go-mod-cache %GOMODCACHE% -if DEFINED USE_S3_CACHING set OMNIBUS_ARGS=%OMNIBUS_ARGS% %USE_S3_CACHING% - -set REPO_ROOT=%~p0\..\.. -pushd . -cd %REPO_ROOT% || exit /b 100 - -SET PATH=%PATH%;%GOPATH%\bin - -@echo GOPATH %GOPATH% -@echo PATH %PATH% -@echo VSTUDIO_ROOT %VSTUDIO_ROOT% - -REM Section to pre-install libyajl2 gem with fix for gcc10 compatibility -Powershell -C "ridk enable; ./tasks/winbuildscripts/libyajl2_install.ps1" - - -if "%TARGET_ARCH%" == "x64" ( - @echo IN x64 BRANCH - call ridk enable -) - - -pip3 install -r requirements.txt || exit /b 120 - -inv -e deps || exit /b 130 -if "%GO_VERSION_CHECK%" == "true" ( - inv -e check-go-version || exit /b 140 -) - -@echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION%" -inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --release-version %RELEASE_VERSION% || exit /b 150 - -REM only build MSI for main targets for now. -if "%OMNIBUS_TARGET%" == "main" ( - @echo "inv -e msi.build --release-version %RELEASE_VERSION% - inv -e msi.build --release-version %RELEASE_VERSION% || exit /b 160 -) - -REM Build the OCI package for the Agent 7 only. -if %MAJOR_VERSION% == 7 ( - Powershell -C "./tasks/winbuildscripts/Generate-OCIPackage.ps1 -package 'datadog-agent'" -) - -popd diff --git a/tasks/winbuildscripts/libyajl2_install.ps1 b/tasks/winbuildscripts/libyajl2_install.ps1 deleted file mode 100644 index a978c004ffcbd..0000000000000 --- a/tasks/winbuildscripts/libyajl2_install.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -pushd . -New-Item -Force -Path c:\tmp -ItemType Directory -cd \tmp - -# Clone the repo; recursive is needed to pick up the submodule inside -git clone --depth 1 --branch 1.2.0 --recursive https://github.com/DataDog/libyajl2-gem libyajl2 -cd libyajl2 - -# Remove existing rake install, we don't need it and its version is too high -gem uninstall -x rake - -# We don't need the development_extras group, we're not running tests -bundle config set --local without 'development_extras' - -# Install dev dependencies - maybe --path should be used to do a local install, but unsure how this will affect the next commands -bundle install - -# Prepare the repo -rake prep -# Install the gem -gem build ./libyajl2.gemspec -gem install ./libyajl2-1.2.0.gem - -# Cleanup -popd -Remove-Item -Recurse -Force c:\tmp\libyajl2 \ No newline at end of file From d255f0cadcbc8a5d657119249f1baa9892f45a8e Mon Sep 17 00:00:00 2001 From: Seth Samuel Date: Fri, 6 Dec 2024 15:25:22 -0500 Subject: [PATCH 12/52] Query Aurora instances per cluster (#31846) --- pkg/databasemonitoring/aws/aurora.go | 77 ++++++++++--------- pkg/databasemonitoring/aws/aurora_test.go | 12 ++- ...nstances-per-cluster-8481d23d9e432e1a.yaml | 12 +++ 3 files changed, 63 insertions(+), 38 deletions(-) create mode 100644 releasenotes/notes/query-aurora-instances-per-cluster-8481d23d9e432e1a.yaml diff --git a/pkg/databasemonitoring/aws/aurora.go b/pkg/databasemonitoring/aws/aurora.go index 590de926ce977..6e5b25198b20d 100644 --- a/pkg/databasemonitoring/aws/aurora.go +++ b/pkg/databasemonitoring/aws/aurora.go @@ -45,45 +45,50 @@ func (c *Client) GetAuroraClusterEndpoints(ctx context.Context, dbClusterIdentif if len(dbClusterIdentifiers) == 0 { return nil, fmt.Errorf("at least one database cluster identifier is required") } - clusterInstances, err := c.client.DescribeDBInstances(ctx, - &rds.DescribeDBInstancesInput{ - Filters: []types.Filter{ - { - Name: aws.String("db-cluster-id"), - Values: dbClusterIdentifiers, - }, - }, - }) - if err != nil { - return nil, fmt.Errorf("error running GetAuroraClusterEndpoints %v", err) - } clusters := make(map[string]*AuroraCluster, 0) - for _, db := range clusterInstances.DBInstances { - if db.Endpoint != nil && db.DBClusterIdentifier != nil { - if db.Endpoint.Address == nil || db.DBInstanceStatus == nil || strings.ToLower(*db.DBInstanceStatus) != "available" { - continue - } - // Add to list of instances for the cluster - instance := &Instance{ - Endpoint: *db.Endpoint.Address, - } - // Set if IAM is configured for the endpoint - if db.IAMDatabaseAuthenticationEnabled != nil { - instance.IamEnabled = *db.IAMDatabaseAuthenticationEnabled - } - // Set the port, if it is known - if db.Endpoint.Port != nil { - instance.Port = *db.Endpoint.Port - } - if db.Engine != nil { - instance.Engine = *db.Engine - } - if _, ok := clusters[*db.DBClusterIdentifier]; !ok { - clusters[*db.DBClusterIdentifier] = &AuroraCluster{ - Instances: make([]*Instance, 0), + + for _, clusterID := range dbClusterIdentifiers { + // TODO: Seth Samuel: This method is not paginated, so if there are more than 100 instances in a cluster, we will only get the first 100 + // We should add pagination support to this method at some point + clusterInstances, err := c.client.DescribeDBInstances(ctx, + &rds.DescribeDBInstancesInput{ + Filters: []types.Filter{ + { + Name: aws.String("db-cluster-id"), + Values: []string{clusterID}, + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("error running GetAuroraClusterEndpoints %v", err) + } + for _, db := range clusterInstances.DBInstances { + if db.Endpoint != nil && db.DBClusterIdentifier != nil { + if db.Endpoint.Address == nil || db.DBInstanceStatus == nil || strings.ToLower(*db.DBInstanceStatus) != "available" { + continue + } + // Add to list of instances for the cluster + instance := &Instance{ + Endpoint: *db.Endpoint.Address, + } + // Set if IAM is configured for the endpoint + if db.IAMDatabaseAuthenticationEnabled != nil { + instance.IamEnabled = *db.IAMDatabaseAuthenticationEnabled + } + // Set the port, if it is known + if db.Endpoint.Port != nil { + instance.Port = *db.Endpoint.Port + } + if db.Engine != nil { + instance.Engine = *db.Engine + } + if _, ok := clusters[*db.DBClusterIdentifier]; !ok { + clusters[*db.DBClusterIdentifier] = &AuroraCluster{ + Instances: make([]*Instance, 0), + } } + clusters[*db.DBClusterIdentifier].Instances = append(clusters[*db.DBClusterIdentifier].Instances, instance) } - clusters[*db.DBClusterIdentifier].Instances = append(clusters[*db.DBClusterIdentifier].Instances, instance) } } if len(clusters) == 0 { diff --git a/pkg/databasemonitoring/aws/aurora_test.go b/pkg/databasemonitoring/aws/aurora_test.go index 64869b7a1d4f6..4fd364ad3da49 100644 --- a/pkg/databasemonitoring/aws/aurora_test.go +++ b/pkg/databasemonitoring/aws/aurora_test.go @@ -211,7 +211,7 @@ func TestGetAuroraClusterEndpoints(t *testing.T) { { name: "multiple cluster ids returns single endpoint from API", configureClient: func(k *MockrdsService) { - k.EXPECT().DescribeDBInstances(gomock.Any(), createDescribeDBInstancesRequest([]string{"test-cluster", "test-cluster-2"})).Return(&rds.DescribeDBInstancesOutput{ + k.EXPECT().DescribeDBInstances(gomock.Any(), createDescribeDBInstancesRequest([]string{"test-cluster"})).Return(&rds.DescribeDBInstancesOutput{ DBInstances: []types.DBInstance{ { Endpoint: &types.Endpoint{ @@ -226,6 +226,9 @@ func TestGetAuroraClusterEndpoints(t *testing.T) { }, }, }, nil).Times(1) + k.EXPECT().DescribeDBInstances(gomock.Any(), createDescribeDBInstancesRequest([]string{"test-cluster-2"})).Return(&rds.DescribeDBInstancesOutput{ + DBInstances: []types.DBInstance{}, + }, nil).Times(1) }, clusterIDs: []string{"test-cluster", "test-cluster-2"}, expectedAuroraClusterEndpoints: map[string]*AuroraCluster{ @@ -244,7 +247,7 @@ func TestGetAuroraClusterEndpoints(t *testing.T) { { name: "multiple cluster ids returns many endpoints from API", configureClient: func(k *MockrdsService) { - k.EXPECT().DescribeDBInstances(gomock.Any(), createDescribeDBInstancesRequest([]string{"test-cluster", "test-cluster-2"})).Return(&rds.DescribeDBInstancesOutput{ + k.EXPECT().DescribeDBInstances(gomock.Any(), createDescribeDBInstancesRequest([]string{"test-cluster"})).Return(&rds.DescribeDBInstancesOutput{ DBInstances: []types.DBInstance{ { Endpoint: &types.Endpoint{ @@ -268,6 +271,11 @@ func TestGetAuroraClusterEndpoints(t *testing.T) { DBInstanceStatus: aws.String("available"), Engine: aws.String("aurora-postgresql"), }, + }, + }, nil).Times(1) + + k.EXPECT().DescribeDBInstances(gomock.Any(), createDescribeDBInstancesRequest([]string{"test-cluster-2"})).Return(&rds.DescribeDBInstancesOutput{ + DBInstances: []types.DBInstance{ { Endpoint: &types.Endpoint{ Address: aws.String("test-endpoint-3"), diff --git a/releasenotes/notes/query-aurora-instances-per-cluster-8481d23d9e432e1a.yaml b/releasenotes/notes/query-aurora-instances-per-cluster-8481d23d9e432e1a.yaml new file mode 100644 index 0000000000000..0bbafb416a868 --- /dev/null +++ b/releasenotes/notes/query-aurora-instances-per-cluster-8481d23d9e432e1a.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Query Aurora instances per cluster to allow up to 100 instances per cluster + rather than 100 instances total. From 6ae9f79ff1d0067e485fb16e607308931a7799e7 Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Fri, 6 Dec 2024 15:52:34 -0500 Subject: [PATCH 13/52] Update CODEOWNERS (#31850) --- .github/CODEOWNERS | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 202c143c2da56..4f6fc8357ea4e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -76,7 +76,7 @@ /.gitlab/deps_fetch/* @DataDog/agent-devx-infra /.gitlab/e2e/* @DataDog/agent-devx-infra @DataDog/agent-devx-loops /.gitlab/e2e_testing_deploy/* @DataDog/agent-devx-loops @DataDog/agent-devx-infra -/.gitlab/e2e_install_packages/* @DataDog/agent-delivery +/.gitlab/e2e_install_packages/* @DataDog/agent-delivery @DataDog/container-ecosystems /.gitlab/e2e_pre_test/* @DataDog/agent-devx-infra @DataDog/agent-devx-loops /.gitlab/kernel_matrix_testing/* @DataDog/agent-devx-infra @DataDog/ebpf-platform /.gitlab/lint/* @DataDog/agent-devx-infra @@ -131,13 +131,12 @@ /.gitlab/deps_build/ @DataDog/ebpf-platform @DataDog/agent-delivery @DataDog/windows-agent -/.gitlab/e2e_install_packages/windows.yml @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/e2e_install_packages/windows.yml @DataDog/agent-delivery @DataDog/container-ecosystems @DataDog/windows-agent /.gitlab/common/ @DataDog/agent-devx-infra /.gitlab/common/test_infra_version.yml @DataDog/agent-devx-loops @DataDog/agent-devx-infra /.gitlab/e2e/e2e.yml @DataDog/container-integrations @DataDog/agent-devx-loops -/.gitlab/e2e_install_packages @DataDog/container-ecosystems @DataDog/agent-delivery /.gitlab/container_build/fakeintake.yml @DataDog/agent-e2e-testing @DataDog/agent-devx-loops /.gitlab/binary_build/fakeintake.yml @DataDog/agent-e2e-testing @DataDog/agent-devx-loops From 07187c133020064fafea30b3451a91b8a00e72cd Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 6 Dec 2024 16:16:24 -0500 Subject: [PATCH 14/52] Add apm_config.additional_endpoints to authorized config paths (#31851) --- comp/api/api/def/component.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comp/api/api/def/component.go b/comp/api/api/def/component.go index 7bd2f7f716670..02c3aae88a231 100644 --- a/comp/api/api/def/component.go +++ b/comp/api/api/def/component.go @@ -48,7 +48,7 @@ type AuthorizedSet map[string]struct{} // config API. var AuthorizedConfigPathsCore = buildAuthorizedSet( "api_key", "site", "dd_url", "logs_config.dd_url", - "additional_endpoints", "logs_config.additional_endpoints", + "additional_endpoints", "logs_config.additional_endpoints", "apm_config.additional_endpoints", ) func buildAuthorizedSet(paths ...string) AuthorizedSet { From 00f8f5b1597ef887a062b8b2ecc3a18cc4be1e5e Mon Sep 17 00:00:00 2001 From: Adel Haj Hassan <41540817+adel121@users.noreply.github.com> Date: Sun, 8 Dec 2024 00:00:41 +0100 Subject: [PATCH 15/52] [CONTP-535] optimise tagger server chunking function by processing chunks in place (#31837) --- comp/core/tagger/server/README.md | 77 +++++++++++++++++++ comp/core/tagger/server/server.go | 27 ++++--- comp/core/tagger/server/util.go | 67 ++++++++++++---- .../core/tagger/server/util_benchmark_test.go | 48 ++++++++++++ comp/core/tagger/server/util_test.go | 75 ++++++++++++------ 5 files changed, 244 insertions(+), 50 deletions(-) create mode 100644 comp/core/tagger/server/README.md create mode 100644 comp/core/tagger/server/util_benchmark_test.go diff --git a/comp/core/tagger/server/README.md b/comp/core/tagger/server/README.md new file mode 100644 index 0000000000000..60e1c0cd90014 --- /dev/null +++ b/comp/core/tagger/server/README.md @@ -0,0 +1,77 @@ +## Introduction + +This package server implements a gRPC server that streams Tagger entities to remote tagger clients. + +## Behaviour + +When a client connects to the tagger grpc server, the server creates a subscription to the local tagger in order to stream tags. + +Before streaming new tag events, the server sends an initial burst to the client over the stream. This initial burst contains a snapshot of the tagger content. After the initial burst has been processed, the server will stream new tag events to the client based on the filters provided in the streaming request. + +### Cutting Events into chunks + +Sending very large messages over the grpc stream can cause the message to be dropped or rejected by the client. The limit is 4MB by default. + +To avoid such scenarios, especially when sending the initial burst, the server cuts each message into small chunks that can be easily transmitted over the stream. + +This logic is implemented in the `util.go` folder. + +We provide 2 implementations: +- `processChunksWithSplit`: splits an event slice into a small chunks where each chunk contains a contiguous slice of events. It then processes the generated chunks sequentially. This will load all chunks into memory (stored in a slice) and then return them. +- `processChunksInPlace`: this has a similar functionality as `processChunksWithSplit`, but it is more optimized in terms of memory and cpu because it processes chunks in place without any extra memory allocation. + +We keep both implementations for at least release candidate to ensure everything works well and be able to quickly revert in case of regressions. + +#### Benchmark Testing + +Benchmark tests show that using lazy chunking results in significant memory and cpu improvement: + +Go to `util_benchmark_test.go`: + +``` +// with processChunksFunc = processChunksWithSplit[int] +go test -bench BenchmarkProcessChunks -benchmem -count 6 -benchtime 100x > old.txt + +// with processChunksFunc = processChunksInPlace[int] +go test -bench BenchmarkProcessChunks -benchmem -count 6 -benchtime 100x > new.txt + +// Compare results +benchstat old.txt new.txt + +goos: linux +goarch: arm64 +pkg: github.com/DataDog/datadog-agent/comp/core/tagger/server + │ old.txt │ new.txt │ + │ sec/op │ sec/op vs base │ +ProcessChunks/100-items-10 2399.5n ± 47% 239.8n ± 4% -90.01% (p=0.002 n=6) +ProcessChunks/1000-items-10 35.072µ ± 13% 2.344µ ± 11% -93.32% (p=0.002 n=6) +ProcessChunks/10000-items-10 365.19µ ± 5% 22.56µ ± 18% -93.82% (p=0.002 n=6) +ProcessChunks/100000-items-10 3435.1µ ± 7% 222.5µ ± 16% -93.52% (p=0.002 n=6) +ProcessChunks/1000000-items-10 29.059m ± 9% 2.219m ± 31% -92.36% (p=0.002 n=6) +geomean 314.3µ 22.87µ -92.72% + + │ old.txt │ new.txt │ + │ B/op │ B/op vs base │ +ProcessChunks/100-items-10 2.969Ki ± 0% 0.000Ki ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/1000-items-10 29.02Ki ± 0% 0.00Ki ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/10000-items-10 370.7Ki ± 0% 0.0Ki ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/100000-items-10 4.165Mi ± 0% 0.000Mi ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/1000000-items-10 44.65Mi ± 0% 0.00Mi ± 0% -100.00% (p=0.002 n=6) +geomean 362.1Ki ? ¹ ² +¹ summaries must be >0 to compute geomean +² ratios must be >0 to compute geomean + + │ old.txt │ new.txt │ + │ allocs/op │ allocs/op vs base │ +ProcessChunks/100-items-10 81.00 ± 0% 0.00 ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/1000-items-10 759.0 ± 0% 0.0 ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/10000-items-10 7.514k ± 0% 0.000k ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/100000-items-10 75.02k ± 0% 0.00k ± 0% -100.00% (p=0.002 n=6) +ProcessChunks/1000000-items-10 750.0k ± 0% 0.0k ± 0% -100.00% (p=0.002 n=6) +geomean 7.638k ? ¹ ² +¹ summaries must be >0 to compute geomean +² ratios must be >0 to compute geomean +``` + + + diff --git a/comp/core/tagger/server/server.go b/comp/core/tagger/server/server.go index 58c4ef2a0667c..5e323c43c7ee2 100644 --- a/comp/core/tagger/server/server.go +++ b/comp/core/tagger/server/server.go @@ -77,6 +77,15 @@ func (s *Server) TaggerStreamEntities(in *pb.StreamTagsRequest, out pb.AgentSecu ticker := time.NewTicker(streamKeepAliveInterval) defer ticker.Stop() + + sendFunc := func(chunk []*pb.StreamTagsEvent) error { + return grpc.DoWithTimeout(func() error { + return out.Send(&pb.StreamTagsResponse{ + Events: chunk, + }) + }, taggerStreamSendTimeout) + } + for { select { case events, ok := <-subscription.EventsChan(): @@ -98,20 +107,10 @@ func (s *Server) TaggerStreamEntities(in *pb.StreamTagsRequest, out pb.AgentSecu responseEvents = append(responseEvents, e) } - // Split events into chunks and send each one - chunks := splitEvents(responseEvents, s.maxEventSize) - for _, chunk := range chunks { - err = grpc.DoWithTimeout(func() error { - return out.Send(&pb.StreamTagsResponse{ - Events: chunk, - }) - }, taggerStreamSendTimeout) - - if err != nil { - log.Warnf("error sending tagger event: %s", err) - s.taggerComponent.GetTaggerTelemetryStore().ServerStreamErrors.Inc() - return err - } + if err := processChunksInPlace(responseEvents, s.maxEventSize, computeTagsEventInBytes, sendFunc); err != nil { + log.Warnf("error sending tagger event: %s", err) + s.taggerComponent.GetTaggerTelemetryStore().ServerStreamErrors.Inc() + return err } case <-out.Context().Done(): diff --git a/comp/core/tagger/server/util.go b/comp/core/tagger/server/util.go index 28fc2c54a1f3d..538c71b4dc05e 100644 --- a/comp/core/tagger/server/util.go +++ b/comp/core/tagger/server/util.go @@ -11,15 +11,44 @@ import ( pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" ) -// splitBySize splits the given slice into contiguous non-overlapping subslices such that -// the size of each sub-slice is at most maxChunkSize. -// The size of each item is calculated using computeSize +type sizeComputerFunc[T any] func(T) int + +type consumeChunkFunc[T any] func(T) error + +// computeProtoEventSize returns the size of a tags stream event in bytes +func computeTagsEventInBytes(event *pb.StreamTagsEvent) int { return proto.Size(event) } + +// processChunksInPlace splits the passed slice into contiguous chunks such that the total size of each chunk is at most maxChunkSize +// and applies the consume function to each of these chunks // -// This function assumes that the size of each single item of the initial slice is not larger than maxChunkSize -func splitBySize[T any](slice []T, maxChunkSize int, computeSize func(T) int) [][]T { +// The size of an item is computed with computeSize +// If an item has a size large than maxChunkSize, it is placed in a singleton chunk (chunk with one item) +// +// The consume function is applied to different chunks in-place, without any need extra memory allocation +func processChunksInPlace[T any](slice []T, maxChunkSize int, computeSize sizeComputerFunc[T], consume consumeChunkFunc[[]T]) error { + idx := 0 + for idx < len(slice) { + chunkSize := computeSize(slice[idx]) + j := idx + 1 + + for j < len(slice) { + eventSize := computeSize(slice[j]) + if chunkSize+eventSize > maxChunkSize { + break + } + chunkSize += eventSize + j++ + } + + if err := consume(slice[idx:j]); err != nil { + return err + } + idx = j + } + return nil +} - // TODO: return an iter.Seq[[]T] instead of [][]T once we upgrade to golang v1.23 - // returning iter.Seq[[]T] has better performance in terms of memory consumption +func splitBySize[T any](slice []T, maxChunkSize int, computeSize func(T) int) [][]T { var chunks [][]T currentChunk := []T{} currentSize := 0 @@ -40,11 +69,21 @@ func splitBySize[T any](slice []T, maxChunkSize int, computeSize func(T) int) [] return chunks } -// splitEvents splits the array of events to chunks with at most maxChunkSize each -func splitEvents(events []*pb.StreamTagsEvent, maxChunkSize int) [][]*pb.StreamTagsEvent { - return splitBySize( - events, - maxChunkSize, - func(event *pb.StreamTagsEvent) int { return proto.Size(event) }, - ) +// processChunksWithSplit splits the passed slice into contiguous chunks such that the total size of each chunk is at most maxChunkSize +// and then applies the consume function to each of these chunks +// +// The size of an item is computed with computeSize +// If an item has a size large than maxChunkSize, it is placed in a singleton chunk (chunk with one item) +// +// Prefer using processChunksInPlace for better CPU and memory performance. This implementation is only kept for benchmarking purposes. +func processChunksWithSplit[T any](slice []T, maxChunkSize int, computeSize sizeComputerFunc[T], consume consumeChunkFunc[[]T]) error { + chunks := splitBySize(slice, maxChunkSize, computeSize) + + for _, chunk := range chunks { + if err := consume(chunk); err != nil { + return err + } + } + + return nil } diff --git a/comp/core/tagger/server/util_benchmark_test.go b/comp/core/tagger/server/util_benchmark_test.go new file mode 100644 index 0000000000000..49ccd243d7365 --- /dev/null +++ b/comp/core/tagger/server/util_benchmark_test.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package server + +import ( + "fmt" + "testing" +) + +var benchmarkSizes = []int{100, 1000, 10000, 100000, 1000000} + +const maxChunkSize = 4 +const mockItemSize = 1 + +var global []int + +func createBaseBenchmarkSlice(size int) []int { + var baseBenchmarkSlice = make([]int, size) + for i := range size { + baseBenchmarkSlice[i] = i + } + return baseBenchmarkSlice +} + +func mockComputeSize(int) int { return mockItemSize } + +func BenchmarkProcessChunks(b *testing.B) { + b.ReportAllocs() + for _, size := range benchmarkSizes { + b.Run(fmt.Sprintf("%d-items", size), func(b *testing.B) { + + // Point this to the implementation you want to benchmark + var processChunksFunc = processChunksInPlace[int] + + items := createBaseBenchmarkSlice(size) + var local []int + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = processChunksFunc(items, maxChunkSize, mockComputeSize, func(t []int) error { local = t; return nil }) + } + global = local + }) + } +} diff --git a/comp/core/tagger/server/util_test.go b/comp/core/tagger/server/util_test.go index 76f94c4988630..7a8b979b30157 100644 --- a/comp/core/tagger/server/util_test.go +++ b/comp/core/tagger/server/util_test.go @@ -6,6 +6,7 @@ package server import ( + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -16,18 +17,35 @@ type mockStreamTagsEvent struct { size int } -func TestSplitEvents(t *testing.T) { +var computeSize = func(e mockStreamTagsEvent) int { + return e.size +} + +func getConsumeFunc(slice *[][]int) consumeChunkFunc[[]mockStreamTagsEvent] { + return func(chunk []mockStreamTagsEvent) error { + ids := make([]int, 0, len(chunk)) + for _, item := range chunk { + ids = append(ids, item.id) + } + + *slice = append(*slice, ids) + return nil + } +} + +func Test(t *testing.T) { + testCases := []struct { name string events []mockStreamTagsEvent maxChunkSize int - expected [][]mockStreamTagsEvent // Expecting indices of events in chunks for easier comparison + expected [][]int // Expecting id's of events in chunks for easier comparison }{ { name: "Empty input", events: []mockStreamTagsEvent{}, maxChunkSize: 100, - expected: nil, // No chunks expected + expected: [][]int{}, }, { name: "Single event within chunk size", @@ -35,9 +53,9 @@ func TestSplitEvents(t *testing.T) { {id: 1, size: 50}, // Mock event with size 50 }, maxChunkSize: 100, - expected: [][]mockStreamTagsEvent{ + expected: [][]int{ { - {id: 1, size: 50}, // One chunk with one event + 1, // One chunk with one event }, }, }, @@ -47,9 +65,9 @@ func TestSplitEvents(t *testing.T) { {id: 1, size: 20}, {id: 2, size: 30}, {id: 3, size: 40}, // Total size = 90 }, maxChunkSize: 100, - expected: [][]mockStreamTagsEvent{ + expected: [][]int{ { - {id: 1, size: 20}, {id: 2, size: 30}, {id: 3, size: 40}, // All events fit in one chunk + 1, 2, 3, // All events fit in one chunk }, }, }, @@ -59,13 +77,12 @@ func TestSplitEvents(t *testing.T) { {id: 1, size: 40}, {id: 2, size: 50}, {id: 3, size: 60}, // Total size = 150 }, maxChunkSize: 100, - expected: [][]mockStreamTagsEvent{ + expected: [][]int{ { - {id: 1, size: 40}, - {id: 2, size: 50}, + 1, 2, }, { - {id: 3, size: 60}, + 3, }, // Last event in second chunk }, }, @@ -75,8 +92,8 @@ func TestSplitEvents(t *testing.T) { {id: 1, size: 50}, {id: 2, size: 50}, // Total size = 100 }, maxChunkSize: 100, - expected: [][]mockStreamTagsEvent{ - {{id: 1, size: 50}, {id: 2, size: 50}}, // Both events fit exactly in one chunk + expected: [][]int{ + {1, 2}, // Both events fit exactly in one chunk }, }, { @@ -85,21 +102,35 @@ func TestSplitEvents(t *testing.T) { {id: 1, size: 100}, {id: 2, size: 101}, // One exactly fits, one exceeds }, maxChunkSize: 100, - expected: [][]mockStreamTagsEvent{ - { - {id: 1, size: 100}, - }, - { - {id: 2, size: 101}, - }, + expected: [][]int{ + {1}, {2}, + }, + }, + { + name: "Multiple items exceeding max chunk size", + events: []mockStreamTagsEvent{ + {id: 1, size: 100}, {id: 2, size: 101}, {id: 3, size: 101}, + }, + maxChunkSize: 100, + expected: [][]int{ + {1}, {2}, {3}, }, }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - chunks := splitBySize(testCase.events, testCase.maxChunkSize, func(e mockStreamTagsEvent) int { return e.size }) - assert.Equal(t, testCase.expected, chunks) + slice := make([][]int, 0, len(testCase.expected)) + processChunksInPlace(testCase.events, testCase.maxChunkSize, computeSize, getConsumeFunc(&slice)) + if len(testCase.expected) > 0 || len(slice) > 0 { + assert.Truef(t, reflect.DeepEqual(testCase.expected, slice), "expected %v, found %v", testCase.expected, slice) + } + + slice = make([][]int, 0, len(testCase.expected)) + processChunksWithSplit(testCase.events, testCase.maxChunkSize, computeSize, getConsumeFunc(&slice)) + if len(testCase.expected) > 0 || len(slice) > 0 { + assert.Truef(t, reflect.DeepEqual(testCase.expected, slice), "expected %v, found %v", testCase.expected, slice) + } }) } } From b08c03dc0cfd3b7c05e3e1225e3a0549b97be712 Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Sun, 8 Dec 2024 13:04:03 +0100 Subject: [PATCH 16/52] pkg/languagedetection: extract mock function from privileged detector (#31860) Co-authored-by: Guy Arbitman --- .../privileged/privileged_detector.go | 8 ------ .../privileged_detector_testutil.go | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 pkg/languagedetection/privileged/privileged_detector_testutil.go diff --git a/pkg/languagedetection/privileged/privileged_detector.go b/pkg/languagedetection/privileged/privileged_detector.go index b6447ea310c14..2e09103a99aa8 100644 --- a/pkg/languagedetection/privileged/privileged_detector.go +++ b/pkg/languagedetection/privileged/privileged_detector.go @@ -18,7 +18,6 @@ import ( "strconv" "sync" "syscall" - "testing" "github.com/hashicorp/golang-lru/v2/simplelru" @@ -107,13 +106,6 @@ func (l *LanguageDetector) DetectWithPrivileges(procs []languagemodels.Process) return languages } -// MockPrivilegedDetectors is used in tests to inject mock tests. It should be called before `DetectWithPrivileges` -func MockPrivilegedDetectors(t *testing.T, newDetectors []languagemodels.Detector) { - oldDetectors := detectorsWithPrivilege - t.Cleanup(func() { detectorsWithPrivilege = oldDetectors }) - detectorsWithPrivilege = newDetectors -} - func (l *LanguageDetector) getBinID(process languagemodels.Process) (binaryID, error) { procPath := filepath.Join(l.hostProc, strconv.Itoa(int(process.GetPid()))) exePath := filepath.Join(procPath, "exe") diff --git a/pkg/languagedetection/privileged/privileged_detector_testutil.go b/pkg/languagedetection/privileged/privileged_detector_testutil.go new file mode 100644 index 0000000000000..1810dd8006164 --- /dev/null +++ b/pkg/languagedetection/privileged/privileged_detector_testutil.go @@ -0,0 +1,25 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux && test + +// Package privileged implements language detection that relies on elevated permissions. +// +// An example of privileged language detection would be binary analysis, where the binary must be +// inspected to determine the language it was compiled from. +package privileged + +import ( + "testing" + + "github.com/DataDog/datadog-agent/pkg/languagedetection/languagemodels" +) + +// MockPrivilegedDetectors is used in tests to inject mock tests. It should be called before `DetectWithPrivileges` +func MockPrivilegedDetectors(t *testing.T, newDetectors []languagemodels.Detector) { + oldDetectors := detectorsWithPrivilege + t.Cleanup(func() { detectorsWithPrivilege = oldDetectors }) + detectorsWithPrivilege = newDetectors +} From 512bca5b18e2481d0048924a3c4c6fbc2756704c Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Sun, 8 Dec 2024 15:22:18 +0200 Subject: [PATCH 17/52] usm: Move helper function to its only usage (#31862) --- pkg/network/usm/sharedlibraries/watcher.go | 28 +++++++++++++++++++++- pkg/process/monitor/process_monitor.go | 26 -------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/pkg/network/usm/sharedlibraries/watcher.go b/pkg/network/usm/sharedlibraries/watcher.go index ca44117d63772..f23673fc6c24e 100644 --- a/pkg/network/usm/sharedlibraries/watcher.go +++ b/pkg/network/usm/sharedlibraries/watcher.go @@ -275,7 +275,7 @@ func (w *Watcher) Start() { return case <-processSync.C: processSet := w.registry.GetRegisteredProcesses() - deletedPids := monitor.FindDeletedProcesses(processSet) + deletedPids := findDeletedProcesses(processSet) for deletedPid := range deletedPids { _ = w.registry.Unregister(deletedPid) } @@ -290,3 +290,29 @@ func (w *Watcher) Start() { utils.AddAttacher(consts.USMModuleName, "native", w) } + +// findDeletedProcesses returns the terminated PIDs from the given map. +func findDeletedProcesses[V any](pids map[uint32]V) map[uint32]struct{} { + existingPids := make(map[uint32]struct{}, len(pids)) + + procIter := func(pid int) error { + if _, exists := pids[uint32(pid)]; exists { + existingPids[uint32(pid)] = struct{}{} + } + return nil + } + // Scanning already running processes + if err := kernel.WithAllProcs(kernel.ProcFSRoot(), procIter); err != nil { + return nil + } + + res := make(map[uint32]struct{}, len(pids)-len(existingPids)) + for pid := range pids { + if _, exists := existingPids[pid]; exists { + continue + } + res[pid] = struct{}{} + } + + return res +} diff --git a/pkg/process/monitor/process_monitor.go b/pkg/process/monitor/process_monitor.go index 1b9e30eb1ec84..8710aa63e324f 100644 --- a/pkg/process/monitor/process_monitor.go +++ b/pkg/process/monitor/process_monitor.go @@ -487,32 +487,6 @@ func (pm *ProcessMonitor) Stop() { pm.processExitCallbacksMutex.Unlock() } -// FindDeletedProcesses returns the terminated PIDs from the given map. -func FindDeletedProcesses[V any](pids map[uint32]V) map[uint32]struct{} { - existingPids := make(map[uint32]struct{}, len(pids)) - - procIter := func(pid int) error { - if _, exists := pids[uint32(pid)]; exists { - existingPids[uint32(pid)] = struct{}{} - } - return nil - } - // Scanning already running processes - if err := kernel.WithAllProcs(kernel.ProcFSRoot(), procIter); err != nil { - return nil - } - - res := make(map[uint32]struct{}, len(pids)-len(existingPids)) - for pid := range pids { - if _, exists := existingPids[pid]; exists { - continue - } - res[pid] = struct{}{} - } - - return res -} - // Event defines the event used by the process monitor type Event struct { Type model.EventType From b027f60e9e1129550c7ee0a272d22a75720f4a78 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Mon, 9 Dec 2024 09:51:18 +0100 Subject: [PATCH 18/52] refactor(package_size): Move code to a python invoke task (#31578) --- .gitlab/JOBOWNERS | 1 + .gitlab/pkg_metrics/pkg_metrics.yml | 29 ++++ tasks/libs/common/git.py | 2 +- tasks/libs/package/size.py | 93 ++++++++++- tasks/libs/package/utils.py | 76 +++++++++ tasks/package.py | 46 ++--- tasks/unit_tests/package_lib_tests.py | 157 +++++++++++++++++- tasks/unit_tests/package_tests.py | 81 +++++++++ tasks/unit_tests/testdata/package_sizes.json | 44 +++++ .../testdata/package_sizes_real.json | 1 + .../packages/datadog-agent-7.aarch64.rpm | 0 .../packages/datadog-agent-7.x86_64.rpm | 0 .../packages/datadog-agent_7_arm64.deb | 0 .../packages/datadog-dogstatsd-7.x86_64.rpm | 0 .../packages/datadog-dogstatsd_7_amd64.deb | 0 .../packages/datadog-dogstatsd_7_arm64.deb | 0 .../packages/datadog-heroku-agent_7_amd64.deb | 0 .../packages/datadog-iot-agent-7.aarch64.rpm | 0 .../packages/datadog-iot-agent-7.x86_64.rpm | 0 .../packages/datadog-iot-agent_7_amd64.deb | 0 .../packages/datadog-iot-agent_7_arm64.deb | 0 21 files changed, 506 insertions(+), 24 deletions(-) create mode 100644 tasks/libs/package/utils.py create mode 100644 tasks/unit_tests/package_tests.py create mode 100644 tasks/unit_tests/testdata/package_sizes.json create mode 100644 tasks/unit_tests/testdata/package_sizes_real.json create mode 100644 tasks/unit_tests/testdata/packages/datadog-agent-7.aarch64.rpm create mode 100644 tasks/unit_tests/testdata/packages/datadog-agent-7.x86_64.rpm create mode 100644 tasks/unit_tests/testdata/packages/datadog-agent_7_arm64.deb create mode 100644 tasks/unit_tests/testdata/packages/datadog-dogstatsd-7.x86_64.rpm create mode 100644 tasks/unit_tests/testdata/packages/datadog-dogstatsd_7_amd64.deb create mode 100644 tasks/unit_tests/testdata/packages/datadog-dogstatsd_7_arm64.deb create mode 100644 tasks/unit_tests/testdata/packages/datadog-heroku-agent_7_amd64.deb create mode 100644 tasks/unit_tests/testdata/packages/datadog-iot-agent-7.aarch64.rpm create mode 100644 tasks/unit_tests/testdata/packages/datadog-iot-agent-7.x86_64.rpm create mode 100644 tasks/unit_tests/testdata/packages/datadog-iot-agent_7_amd64.deb create mode 100644 tasks/unit_tests/testdata/packages/datadog-iot-agent_7_arm64.deb diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index 9cf4e21fc84de..9cb87462dca48 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -67,6 +67,7 @@ iot_agent_suse* @DataDog/agent-delivery dogstatsd_suse* @DataDog/agent-delivery agent_oci* @DataDog/agent-delivery installer_oci* @DataDog/agent-delivery +new_check_pkg_size @DataDog/agent-delivery # Testing package deploy deploy_deb_testing* @DataDog/agent-delivery diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index bb62a727a570a..e8616c61247a2 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -179,3 +179,32 @@ check_pkg_size-arm64-a7: ["datadog-iot-agent"]="10000000" ["datadog-dogstatsd"]="10000000" ) + +new_check_pkg_size: + stage: pkg_metrics + image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + rules: + - !reference [.except_mergequeue] + - when: on_success + allow_failure: true + needs: + - agent_deb-x64-a7 + - iot_agent_deb-x64 + - dogstatsd_deb-x64 + - agent_heroku_deb-x64-a7 + - agent_rpm-x64-a7 + - iot_agent_rpm-x64 + - dogstatsd_rpm-x64 + - agent_suse-x64-a7 + - dogstatsd_suse-x64 + - iot_agent_suse-x64 + - agent_deb-arm64-a7 + - iot_agent_deb-arm64 + - dogstatsd_deb-arm64 + - agent_rpm-arm64-a7 + - iot_agent_rpm-arm64 + script: + - ls -l $OMNIBUS_PACKAGE_DIR + - ls -l $OMNIBUS_PACKAGE_DIR_SUSE + - inv package.check-size diff --git a/tasks/libs/common/git.py b/tasks/libs/common/git.py index 44f5c5a684cfa..dfff77eda03f7 100644 --- a/tasks/libs/common/git.py +++ b/tasks/libs/common/git.py @@ -117,7 +117,7 @@ def get_default_branch(major: int | None = None): def get_common_ancestor(ctx, branch, base=None) -> str: - base = base or get_default_branch() + base = base or f"origin/{get_default_branch()}" return ctx.run(f"git merge-base {branch} {base}", hide=True).stdout.strip() diff --git a/tasks/libs/package/size.py b/tasks/libs/package/size.py index 5f564c91837f2..cae0353453f7d 100644 --- a/tasks/libs/package/size.py +++ b/tasks/libs/package/size.py @@ -1,10 +1,15 @@ import os +import sys import tempfile from datetime import datetime -from tasks.libs.common.color import color_message +from invoke import Exit + +from tasks.libs.common.color import Color, color_message from tasks.libs.common.constants import ORIGIN_CATEGORY, ORIGIN_PRODUCT, ORIGIN_SERVICE +from tasks.libs.common.git import get_common_ancestor, get_current_branch, get_default_branch from tasks.libs.common.utils import get_metric_origin +from tasks.libs.package.utils import get_package_path DEBIAN_OS = "debian" CENTOS_OS = "centos" @@ -31,6 +36,26 @@ }, } +PACKAGE_SIZE_TEMPLATE = { + 'amd64': { + 'datadog-agent': {'deb': 140000000}, + 'datadog-iot-agent': {'deb': 10000000}, + 'datadog-dogstatsd': {'deb': 10000000}, + 'datadog-heroku-agent': {'deb': 70000000}, + }, + 'x86_64': { + 'datadog-agent': {'rpm': 140000000, 'suse': 140000000}, + 'datadog-iot-agent': {'rpm': 10000000, 'suse': 10000000}, + 'datadog-dogstatsd': {'rpm': 10000000, 'suse': 10000000}, + }, + 'arm64': { + 'datadog-agent': {'deb': 140000000}, + 'datadog-iot-agent': {'deb': 10000000}, + 'datadog-dogstatsd': {'deb': 10000000}, + }, + 'aarch64': {'datadog-agent': {'rpm': 140000000}, 'datadog-iot-agent': {'rpm': 10000000}}, +} + def extract_deb_package(ctx, package_path, extract_dir): ctx.run(f"dpkg -x {package_path} {extract_dir} > /dev/null") @@ -132,3 +157,69 @@ def compute_package_size_metrics( ) return series + + +def compare(ctx, package_sizes, arch, flavor, os_name, threshold): + """ + Compare (or update) a package size with the ancestor package size. + """ + mb = 1000000 + if os_name == 'suse': + dir = os.environ['OMNIBUS_PACKAGE_DIR_SUSE'] + path = f'{dir}/{flavor}-7*{arch}.rpm' + else: + dir = os.environ['OMNIBUS_PACKAGE_DIR'] + separator = '_' if os_name == 'deb' else '-' + path = f'{dir}/{flavor}{separator}7*{arch}.{os_name}' + package_size = _get_uncompressed_size(ctx, get_package_path(path), os_name) + branch = get_current_branch(ctx) + ancestor = get_common_ancestor(ctx, branch) + if branch == get_default_branch(): + package_sizes[ancestor][arch][flavor][os_name] = package_size + return + previous_size = get_previous_size(package_sizes, ancestor, arch, flavor, os_name) + diff = package_size - previous_size + + # For printing purposes + new_package_size_mb = package_size / mb + stable_package_size_mb = previous_size / mb + threshold_mb = threshold / mb + diff_mb = diff / mb + message = f"""{flavor}-{arch}-{os_name} size increase is OK: + New package size is {new_package_size_mb:.2f}MB + Ancestor package ({ancestor}) size is {stable_package_size_mb:.2f}MB + Diff is {diff_mb:.2f}MB (max allowed diff: {threshold_mb:.2f}MB)""" + + if diff > threshold: + print(color_message(message.replace('OK', 'too large'), Color.RED), file=sys.stderr) + raise Exit(code=1) + + print(message) + + +def get_previous_size(package_sizes, ancestor, arch, flavor, os_name): + """ + Get the size of the package for the given ancestor, or the earliest ancestor if the given ancestor is not found. + """ + if ancestor in package_sizes: + commit = ancestor + else: + commit = min(package_sizes, key=lambda x: package_sizes[x]['timestamp']) + return package_sizes[commit][arch][flavor][os_name] + + +def _get_uncompressed_size(ctx, package, os_name): + if os_name == 'deb': + return _get_deb_uncompressed_size(ctx, package) + else: + return _get_rpm_uncompressed_size(ctx, package) + + +def _get_deb_uncompressed_size(ctx, package): + # the size returned by dpkg is a number of bytes divided by 1024 + # so we multiply it back to get the same unit as RPM or stat + return int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs').stdout) * 1024 + + +def _get_rpm_uncompressed_size(ctx, package): + return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs').stdout) diff --git a/tasks/libs/package/utils.py b/tasks/libs/package/utils.py new file mode 100644 index 0000000000000..ba5448ad666c7 --- /dev/null +++ b/tasks/libs/package/utils.py @@ -0,0 +1,76 @@ +import glob +import json + +from invoke import Exit, UnexpectedExit + +from tasks.libs.common.color import color_message +from tasks.libs.notify.utils import AWS_S3_CP_CMD + +PACKAGE_SIZE_S3_CI_BUCKET_URL = "s3://dd-ci-artefacts-build-stable/datadog-agent/package_size" + + +def get_package_path(glob_pattern): + package_paths = glob.glob(glob_pattern) + if len(package_paths) > 1: + raise Exit(code=1, message=color_message(f"Too many files matching {glob_pattern}: {package_paths}", "red")) + elif len(package_paths) == 0: + raise Exit(code=1, message=color_message(f"Couldn't find any file matching {glob_pattern}", "red")) + + return package_paths[0] + + +def list_packages(template, parent=""): + """ + Recursively parse the template to generate the argument list for the compare task + """ + packs = [] + parent = parent if parent else [] + for key, value in template.items(): + if isinstance(value, dict): + packs += list_packages(value, parent + [key]) + else: + if key != "timestamp": + packs.append(parent + [key, value]) + return packs + + +def retrieve_package_sizes(ctx, package_size_file: str, distant: bool = True): + """ + Retrieve the stored document in aws s3, or create it + The content of the file is the following: + { + "c0ae34b1": { + "timestamp": "1582230020", + "amd64": { + "datadog-agent": { + "deb": 123456, + "rpm": 123456, + "suse": 123456}}} + """ + try: + if distant: + ctx.run( + f"{AWS_S3_CP_CMD} {PACKAGE_SIZE_S3_CI_BUCKET_URL}/{package_size_file} {package_size_file}", + hide=True, + ) + with open(package_size_file) as f: + package_sizes = json.load(f) + except UnexpectedExit as e: + if "404" in e.result.stderr: + package_sizes = {} + else: + raise e + return package_sizes + + +def upload_package_sizes(ctx, package_sizes: dict, package_size_file: str, distant: bool = True): + """ + Save the package_sizes dict to a file and upload it to the CI bucket + """ + with open(package_size_file, "w") as f: + json.dump(package_sizes, f) + if distant: + ctx.run( + f"{AWS_S3_CP_CMD} {package_size_file} {PACKAGE_SIZE_S3_CI_BUCKET_URL}/{package_size_file}", + hide="stdout", + ) diff --git a/tasks/package.py b/tasks/package.py index ee1dc1adabbf0..4e9a859ae8850 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -1,31 +1,37 @@ -import glob import os +from datetime import datetime from invoke import task from invoke.exceptions import Exit from tasks.libs.common.color import color_message -from tasks.libs.package.size import compute_package_size_metrics +from tasks.libs.common.git import get_common_ancestor, get_current_branch, get_default_branch +from tasks.libs.package.size import ( + PACKAGE_SIZE_TEMPLATE, + _get_deb_uncompressed_size, + _get_rpm_uncompressed_size, + compare, + compute_package_size_metrics, +) +from tasks.libs.package.utils import get_package_path, list_packages, retrieve_package_sizes, upload_package_sizes -def get_package_path(glob_pattern): - package_paths = glob.glob(glob_pattern) - if len(package_paths) > 1: - raise Exit(code=1, message=color_message(f"Too many files matching {glob_pattern}: {package_paths}", "red")) - elif len(package_paths) == 0: - raise Exit(code=1, message=color_message(f"Couldn't find any file matching {glob_pattern}", "red")) - - return package_paths[0] - - -def _get_deb_uncompressed_size(ctx, package): - # the size returned by dpkg is a number of bytes divided by 1024 - # so we multiply it back to get the same unit as RPM or stat - return int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs').stdout) * 1024 - - -def _get_rpm_uncompressed_size(ctx, package): - return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs').stdout) +@task +def check_size(ctx, filename: str = 'package_sizes.json', dry_run: bool = False): + package_sizes = retrieve_package_sizes(ctx, filename, distant=not dry_run) + if get_current_branch(ctx) == get_default_branch(): + # Initialize to default values + ancestor = get_common_ancestor(ctx, get_default_branch()) + if ancestor in package_sizes: + # The test already ran on this commit + return + package_sizes[ancestor] = PACKAGE_SIZE_TEMPLATE + package_sizes[ancestor]['timestamp'] = int(datetime.now().timestamp()) + # Check size of packages + for package_info in list_packages(PACKAGE_SIZE_TEMPLATE): + compare(ctx, package_sizes, *package_info) + if get_current_branch(ctx) == get_default_branch(): + upload_package_sizes(ctx, package_sizes, filename, distant=not dry_run) @task diff --git a/tasks/unit_tests/package_lib_tests.py b/tasks/unit_tests/package_lib_tests.py index ba637931b2ded..0ae56501c3439 100644 --- a/tasks/unit_tests/package_lib_tests.py +++ b/tasks/unit_tests/package_lib_tests.py @@ -1,9 +1,19 @@ +import json +import sys import unittest from unittest.mock import MagicMock, patch -from invoke import MockContext +from invoke import Exit, MockContext, Result -from tasks.libs.package.size import SCANNED_BINARIES, compute_package_size_metrics +from tasks.libs.package.size import ( + PACKAGE_SIZE_TEMPLATE, + SCANNED_BINARIES, + _get_uncompressed_size, + compare, + compute_package_size_metrics, + get_previous_size, +) +from tasks.libs.package.utils import list_packages class TestProduceSizeStats(unittest.TestCase): @@ -94,3 +104,146 @@ def test_compute_size_invalid_flavor(self): bucket_branch=test_branch, arch=test_arch, ) + + +class TestListPackages(unittest.TestCase): + def test_no_package(self): + template = {} + self.assertEqual(list_packages(template), []) + + def test_single_package(self): + template = {"key": "value"} + self.assertEqual(list_packages(template), [["key", "value"]]) + + def test_multiple_packages(self): + template = {"key": {"key2": 42}} + self.assertEqual(list_packages(template), [["key", "key2", 42]]) + + def test_ignore_timestamp_root(self): + template = {"key": {"key2": 42}, "timestamp": 1234567890} + self.assertEqual(list_packages(template), [["key", "key2", 42]]) + + def test_ignore_timestamp_nested(self): + template = {"key": {"key2": 42, "timestamp": 1234567890}} + self.assertEqual(list_packages(template), [["key", "key2", 42]]) + + +class TestGetPreviousSize(unittest.TestCase): + package_sizes = {} + + def setUp(self) -> None: + with open('tasks/unit_tests/testdata/package_sizes.json') as f: + self.package_sizes = json.load(f) + + def test_is_ancestor(self): + self.assertEqual(get_previous_size(self.package_sizes, "grand_ma", "artdeco", "cherry", 'fibula'), 42) + + def test_is_other_ancestor(self): + self.assertEqual(get_previous_size(self.package_sizes, "pa", "artdeco", "cherry", 'fibula'), 3) + + def test_is_not_ancestor(self): + self.assertEqual(get_previous_size(self.package_sizes, "grandPa", "artdeco", "cherry", 'fibula'), 42) + + +class TestGetUncompressedSize(unittest.TestCase): + def test_get_deb_uncompressed_size(self): + flavor = 'datadog-agent.deb' + c = MockContext(run={f"dpkg-deb --info {flavor} | grep Installed-Size | cut -d : -f 2 | xargs": Result(42)}) + self.assertEqual(_get_uncompressed_size(c, flavor, 'deb'), 43008) + + def test_get_rpm_uncompressed_size(self): + flavor = 'datadog-agent.rpm' + c = MockContext(run={f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(42)}) + self.assertEqual(_get_uncompressed_size(c, flavor, 'rpm'), 42) + + def test_get_suse_uncompressed_size(self): + flavor = 'datadog-agent.rpm' + c = MockContext(run={f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(69)}) + self.assertEqual(_get_uncompressed_size(c, flavor, 'suse'), 69) + + +class TestCompare(unittest.TestCase): + package_sizes = {} + pkg_root = 'tasks/unit_tests/testdata/packages' + + def setUp(self) -> None: + with open('tasks/unit_tests/testdata/package_sizes.json') as f: + self.package_sizes = json.load(f) + + @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages'}) + @patch('builtins.print') + def test_on_main(self, mock_print): + flavor, arch, os_name = 'datadog-heroku-agent', 'amd64', 'deb' + c = MockContext( + run={ + 'git rev-parse --abbrev-ref HEAD': Result('main'), + 'git merge-base main origin/main': Result('12345'), + f"dpkg-deb --info {self.pkg_root}/{flavor}_7_{arch}.{os_name} | grep Installed-Size | cut -d : -f 2 | xargs": Result( + 42 + ), + } + ) + self.package_sizes['12345'] = PACKAGE_SIZE_TEMPLATE + self.assertEqual(self.package_sizes['12345'][arch][flavor][os_name], 70000000) + compare(c, self.package_sizes, arch, flavor, os_name, 2001) + self.assertEqual(self.package_sizes['12345'][arch][flavor][os_name], 43008) + mock_print.assert_not_called() + + @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages'}) + @patch('builtins.print') + def test_on_branch_ok(self, mock_print): + flavor, arch, os_name = 'datadog-agent', 'aarch64', 'suse' + c = MockContext( + run={ + 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), + 'git merge-base pikachu origin/main': Result('25'), + f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.rpm | grep Size | cut -d : -f 2 | xargs": Result(69000000), + } + ) + compare(c, self.package_sizes, arch, flavor, os_name, 70000000) + mock_print.assert_called_with(f"""{flavor}-{arch}-{os_name} size increase is OK: + New package size is 69.00MB + Ancestor package (25) size is 68.00MB + Diff is 1.00MB (max allowed diff: 70.00MB)""") + + @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages'}) + @patch('builtins.print') + def test_on_branch_ok_rpm(self, mock_print): + flavor, arch, os_name = 'datadog-iot-agent', 'x86_64', 'rpm' + c = MockContext( + run={ + 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), + 'git merge-base pikachu origin/main': Result('25'), + f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.{os_name} | grep Size | cut -d : -f 2 | xargs": Result( + 69000000 + ), + } + ) + compare(c, self.package_sizes, arch, flavor, os_name, 70000000) + mock_print.assert_called_with(f"""{flavor}-{arch}-{os_name} size increase is OK: + New package size is 69.00MB + Ancestor package (25) size is 78.00MB + Diff is -9.00MB (max allowed diff: 70.00MB)""") + + @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages'}) + @patch('builtins.print') + def test_on_branch_ko(self, mock_print): + flavor, arch, os_name = 'datadog-agent', 'aarch64', 'suse' + c = MockContext( + run={ + 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), + 'git merge-base pikachu origin/main': Result('25'), + f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.rpm | grep Size | cut -d : -f 2 | xargs": Result( + 139000000 + ), + } + ) + with self.assertRaises(Exit): + compare(c, self.package_sizes, arch, flavor, os_name, 70000000) + mock_print.assert_called_with( + """\x1b[91mdatadog-agent-aarch64-suse size increase is too large: + New package size is 139.00MB + Ancestor package (25) size is 68.00MB + Diff is 71.00MB (max allowed diff: 70.00MB)\x1b[0m""", + file=sys.stderr, + ) diff --git a/tasks/unit_tests/package_tests.py b/tasks/unit_tests/package_tests.py new file mode 100644 index 0000000000000..912c1ce53a912 --- /dev/null +++ b/tasks/unit_tests/package_tests.py @@ -0,0 +1,81 @@ +import json +import unittest +from unittest.mock import MagicMock, patch + +from invoke import Context, Exit, MockContext, Result + +from tasks.package import check_size + + +class TestCheckSize(unittest.TestCase): + @patch.dict( + 'os.environ', + { + 'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', + 'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', + }, + ) + @patch('tasks.libs.package.size.get_package_path', new=MagicMock(return_value='datadog-agent')) + def test_dev_branch_ko(self): + flavor = 'datadog-agent' + c = MockContext( + run={ + 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), + 'git merge-base pikachu origin/main': Result('25'), + f"dpkg-deb --info {flavor} | grep Installed-Size | cut -d : -f 2 | xargs": Result(42), + f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(69000000), + } + ) + with self.assertRaises(Exit): + check_size(c, filename='tasks/unit_tests/testdata/package_sizes_real.json', dry_run=True) + + @patch('builtins.print') + @patch.dict( + 'os.environ', + { + 'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', + 'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', + }, + ) + @patch('tasks.libs.package.size.get_package_path', new=MagicMock(return_value='datadog-agent')) + def test_dev_branch_ok(self, print_mock): + flavor = 'datadog-agent' + c = MockContext( + run={ + 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), + 'git merge-base pikachu origin/main': Result('25'), + f"dpkg-deb --info {flavor} | grep Installed-Size | cut -d : -f 2 | xargs": Result(42), + f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(20000000), + } + ) + check_size(c, filename='tasks/unit_tests/testdata/package_sizes_real.json', dry_run=True) + print_mock.assert_called() + self.assertEqual(print_mock.call_count, 15) + + @patch('builtins.print') + @patch.dict( + 'os.environ', + { + 'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', + 'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', + }, + ) + @patch('tasks.libs.package.size.get_package_path', new=MagicMock(return_value='datadog-agent')) + def test_main_branch_ok(self, print_mock): + flavor = 'datadog-agent' + c = MockContext( + run={ + 'git rev-parse --abbrev-ref HEAD': Result('main'), + 'git merge-base main origin/main': Result('25'), + f"dpkg-deb --info {flavor} | grep Installed-Size | cut -d : -f 2 | xargs": Result(42), + f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(20000000), + } + ) + check_size(c, filename='tasks/unit_tests/testdata/package_sizes_real.json', dry_run=True) + with open('tasks/unit_tests/testdata/package_sizes_real.json') as f: + new_sizes = json.load(f) + self.assertIn('25', new_sizes) + self.assertEqual(new_sizes['25']['x86_64']['datadog-agent']['rpm'], 20000000) + self.assertEqual(new_sizes['25']['arm64']['datadog-iot-agent']['deb'], 43008) + ctx = Context() + ctx.run("git checkout -- tasks/unit_tests/testdata/package_sizes_real.json") diff --git a/tasks/unit_tests/testdata/package_sizes.json b/tasks/unit_tests/testdata/package_sizes.json new file mode 100644 index 0000000000000..2aade2e8320fa --- /dev/null +++ b/tasks/unit_tests/testdata/package_sizes.json @@ -0,0 +1,44 @@ +{ + "grand_ma": { + "timestamp": -810907200, + "artdeco": { + "cherry": { + "fibula" : 42, + "ulna" : 2 + }, + "apple": { + "patella" : 4 + } + } + }, + "ma": { + "timestamp": 159192000, + "gothic": { + "vanilla": { + "fabella" : 5 + } + } + }, + "pa": { + "timestamp": -14182940, + "artdeco": { + "cherry": { + "fibula" : 3, + "ulna" : 6 + } + } + }, + "25": { + "timestamp": 0, + "aarch64": { + "datadog-agent": { + "suse" : 68000000 + } + }, + "x86_64": { + "datadog-iot-agent": { + "rpm" : 78000000 + } + } + } +} diff --git a/tasks/unit_tests/testdata/package_sizes_real.json b/tasks/unit_tests/testdata/package_sizes_real.json new file mode 100644 index 0000000000000..e24ed4d2bc98d --- /dev/null +++ b/tasks/unit_tests/testdata/package_sizes_real.json @@ -0,0 +1 @@ +{"12345": {"timestamp": 1732804637, "amd64": {"datadog-agent": {"deb": 140000000}, "datadog-iot-agent": {"deb": 10000000}, "datadog-dogstatsd": {"deb": 10000000}, "datadog-heroku-agent": {"deb": 70000000}}, "x86_64": {"datadog-agent": {"rpm": 140000000, "suse": 140000000}, "datadog-iot-agent": {"rpm": 10000000, "suse": 10000000}, "datadog-dogstatsd": {"rpm": 10000000, "suse": 10000000}}, "arm64": {"datadog-agent": {"deb": 140000000}, "datadog-iot-agent": {"deb": 10000000}, "datadog-dogstatsd": {"deb": 10000000}}, "aarch64": {"datadog-agent": {"rpm": 140000000}, "datadog-iot-agent": {"rpm": 10000000}}}} \ No newline at end of file diff --git a/tasks/unit_tests/testdata/packages/datadog-agent-7.aarch64.rpm b/tasks/unit_tests/testdata/packages/datadog-agent-7.aarch64.rpm new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-agent-7.x86_64.rpm b/tasks/unit_tests/testdata/packages/datadog-agent-7.x86_64.rpm new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-agent_7_arm64.deb b/tasks/unit_tests/testdata/packages/datadog-agent_7_arm64.deb new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-dogstatsd-7.x86_64.rpm b/tasks/unit_tests/testdata/packages/datadog-dogstatsd-7.x86_64.rpm new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-dogstatsd_7_amd64.deb b/tasks/unit_tests/testdata/packages/datadog-dogstatsd_7_amd64.deb new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-dogstatsd_7_arm64.deb b/tasks/unit_tests/testdata/packages/datadog-dogstatsd_7_arm64.deb new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-heroku-agent_7_amd64.deb b/tasks/unit_tests/testdata/packages/datadog-heroku-agent_7_amd64.deb new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-iot-agent-7.aarch64.rpm b/tasks/unit_tests/testdata/packages/datadog-iot-agent-7.aarch64.rpm new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-iot-agent-7.x86_64.rpm b/tasks/unit_tests/testdata/packages/datadog-iot-agent-7.x86_64.rpm new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-iot-agent_7_amd64.deb b/tasks/unit_tests/testdata/packages/datadog-iot-agent_7_amd64.deb new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tasks/unit_tests/testdata/packages/datadog-iot-agent_7_arm64.deb b/tasks/unit_tests/testdata/packages/datadog-iot-agent_7_arm64.deb new file mode 100644 index 0000000000000..e69de29bb2d1d From 2491564a64949ebc3a9bdf2d1f273442f283da1a Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Mon, 9 Dec 2024 09:51:25 +0100 Subject: [PATCH 19/52] [CWS] small optimizations to regex parsing in SECL (#31792) --- pkg/security/secl/compiler/eval/strings.go | 4 +- .../secl/compiler/eval/strings_test.go | 38 ++++++++++++++----- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/security/secl/compiler/eval/strings.go b/pkg/security/secl/compiler/eval/strings.go index 13b2a9a6491ab..751d86db6a015 100644 --- a/pkg/security/secl/compiler/eval/strings.go +++ b/pkg/security/secl/compiler/eval/strings.go @@ -123,7 +123,7 @@ type RegexpStringMatcher struct { re *regexp.Regexp } -var stringBigOrRe = regexp.MustCompile(`^\.\*\(([a-zA-Z_|]+)\)\.\*$`) +var stringBigOrRe = regexp.MustCompile(`^(?:\.\*)?\(([a-zA-Z_|]+)\)(?:\.\*)?$`) // Compile a regular expression based pattern func (r *RegexpStringMatcher) Compile(pattern string, caseInsensitive bool) error { @@ -135,7 +135,7 @@ func (r *RegexpStringMatcher) Compile(pattern string, caseInsensitive bool) erro } } - if caseInsensitive { + if caseInsensitive && !strings.HasPrefix(pattern, "(?i)") { pattern = "(?i)" + pattern } diff --git a/pkg/security/secl/compiler/eval/strings_test.go b/pkg/security/secl/compiler/eval/strings_test.go index 181864bac1bc8..a09be87afbc24 100644 --- a/pkg/security/secl/compiler/eval/strings_test.go +++ b/pkg/security/secl/compiler/eval/strings_test.go @@ -260,17 +260,35 @@ func TestRegexp(t *testing.T) { } func BenchmarkRegexpEvaluator(b *testing.B) { - pattern := ".*(restore|recovery|readme|instruction|how_to|ransom).*" + b.Run("with stars", func(b *testing.B) { + pattern := ".*(restore|recovery|readme|instruction|how_to|ransom).*" - var matcher RegexpStringMatcher - if err := matcher.Compile(pattern, false); err != nil { - b.Fatal(err) - } + var matcher RegexpStringMatcher + if err := matcher.Compile(pattern, false); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !matcher.Matches("123ransom456.txt") { + b.Fatal("unexpected result") + } + } + }) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if !matcher.Matches("123ransom456.txt") { - b.Fatal("unexpected result") + b.Run("without stars", func(b *testing.B) { + pattern := "(restore|recovery|readme|instruction|how_to|ransom)" + + var matcher RegexpStringMatcher + if err := matcher.Compile(pattern, false); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !matcher.Matches("123ransom456.txt") { + b.Fatal("unexpected result") + } } - } + }) } From 88a8e04543920cb3e57b4bf22d477b9396c09456 Mon Sep 17 00:00:00 2001 From: "Duong (Yoon)" <47346352+DDuongNguyen@users.noreply.github.com> Date: Mon, 9 Dec 2024 04:40:54 -0500 Subject: [PATCH 20/52] [AMLII-2184] http1 and http2 regression cases (#31539) --- .../conf.d/disk-listener.d/conf.yaml | 5 +++ .../datadog-agent/datadog.yaml | 21 ++++++++++ .../experiment.yaml | 39 +++++++++++++++++++ .../lading/lading.yaml | 27 +++++++++++++ .../conf.d/disk-listener.d/conf.yaml | 5 +++ .../datadog-agent/datadog.yaml | 21 ++++++++++ .../experiment.yaml | 39 +++++++++++++++++++ .../lading/lading.yaml | 27 +++++++++++++ 8 files changed, 184 insertions(+) create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/conf.d/disk-listener.d/conf.yaml create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/datadog.yaml create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http1/lading/lading.yaml create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/conf.d/disk-listener.d/conf.yaml create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/datadog.yaml create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml create mode 100644 test/regression/cases/file_to_blackhole_0ms_latency_http2/lading/lading.yaml diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/conf.d/disk-listener.d/conf.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/conf.d/disk-listener.d/conf.yaml new file mode 100644 index 0000000000000..ec51a59de1c46 --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/conf.d/disk-listener.d/conf.yaml @@ -0,0 +1,5 @@ +logs: + - type: file + path: "/smp-shared/*.log" + service: "my-service" + source: "my-client-app" diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/datadog.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/datadog.yaml new file mode 100644 index 0000000000000..667ce4d27223f --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http1/datadog-agent/datadog.yaml @@ -0,0 +1,21 @@ +auth_token_file_path: /tmp/agent-auth-token + +# Disable cloud detection. This stops the Agent from poking around the +# execution environment & network. This is particularly important if the target +# has network access. +cloud_provider_metadata: [] + +dd_url: http://127.0.0.1:9091 + +logs_enabled: true +logs_config: + logs_dd_url: 127.0.0.1:9092 + http_protocol: http1 + file_scan_period: 1 + logs_no_ssl: true + force_use_http: true + +process_config.process_dd_url: http://localhost:9093 + +telemetry.enabled: true +telemetry.checks: '*' diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml new file mode 100644 index 0000000000000..c51505add2bb6 --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml @@ -0,0 +1,39 @@ +optimization_goal: egress_throughput +erratic: false + +target: + name: datadog-agent + command: /bin/entrypoint.sh + cpu_allotment: 8 + memory_allotment: 1.5GiB + + environment: + DD_API_KEY: 00000001 + DD_HOSTNAME: smp-regression + + profiling_environment: + DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000 + DD_INTERNAL_PROFILING_CPU_DURATION: 1m + DD_INTERNAL_PROFILING_DELTA_PROFILES: true + DD_INTERNAL_PROFILING_ENABLED: true + DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true + DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10 + DD_INTERNAL_PROFILING_PERIOD: 1m + DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket + DD_PROFILING_EXECUTION_TRACE_ENABLED: true + DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m + DD_PROFILING_WAIT_PROFILE: true + +checks: + - name: memory_usage + description: "Memory usage" + bounds: + series: total_rss_bytes + # The machine has 12GiB free. + upper_bound: 1.2GiB + + - name: lost_bytes + description: "Available bytes not polled by log Agent" + bounds: + series: lost_bytes + upper_bound: 0KiB diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http1/lading/lading.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http1/lading/lading.yaml new file mode 100644 index 0000000000000..639cdb3b546aa --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http1/lading/lading.yaml @@ -0,0 +1,27 @@ +generator: + - file_gen: + logrotate_fs: + seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, + 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + load_profile: + constant: 10MB + concurrent_logs: 1 + maximum_bytes_per_log: 500MB + total_rotations: 5 + max_depth: 0 + variant: "ascii" + maximum_prebuild_cache_size_bytes: 300MB + mount_point: /smp-shared + +blackhole: + - http: + binding_addr: "127.0.0.1:9091" + - http: + binding_addr: "127.0.0.1:9092" + response_delay_millis: 0 + - http: + binding_addr: "127.0.0.1:9093" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:5000/telemetry" diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/conf.d/disk-listener.d/conf.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/conf.d/disk-listener.d/conf.yaml new file mode 100644 index 0000000000000..ec51a59de1c46 --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/conf.d/disk-listener.d/conf.yaml @@ -0,0 +1,5 @@ +logs: + - type: file + path: "/smp-shared/*.log" + service: "my-service" + source: "my-client-app" diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/datadog.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/datadog.yaml new file mode 100644 index 0000000000000..efe55170e4dc9 --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http2/datadog-agent/datadog.yaml @@ -0,0 +1,21 @@ +auth_token_file_path: /tmp/agent-auth-token + +# Disable cloud detection. This stops the Agent from poking around the +# execution environment & network. This is particularly important if the target +# has network access. +cloud_provider_metadata: [] + +dd_url: http://127.0.0.1:9091 + +logs_enabled: true +logs_config: + logs_dd_url: 127.0.0.1:9092 + http_protocol: auto + file_scan_period: 1 + logs_no_ssl: true + force_use_http: true + +process_config.process_dd_url: http://localhost:9093 + +telemetry.enabled: true +telemetry.checks: '*' diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml new file mode 100644 index 0000000000000..c51505add2bb6 --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml @@ -0,0 +1,39 @@ +optimization_goal: egress_throughput +erratic: false + +target: + name: datadog-agent + command: /bin/entrypoint.sh + cpu_allotment: 8 + memory_allotment: 1.5GiB + + environment: + DD_API_KEY: 00000001 + DD_HOSTNAME: smp-regression + + profiling_environment: + DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000 + DD_INTERNAL_PROFILING_CPU_DURATION: 1m + DD_INTERNAL_PROFILING_DELTA_PROFILES: true + DD_INTERNAL_PROFILING_ENABLED: true + DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true + DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10 + DD_INTERNAL_PROFILING_PERIOD: 1m + DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket + DD_PROFILING_EXECUTION_TRACE_ENABLED: true + DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m + DD_PROFILING_WAIT_PROFILE: true + +checks: + - name: memory_usage + description: "Memory usage" + bounds: + series: total_rss_bytes + # The machine has 12GiB free. + upper_bound: 1.2GiB + + - name: lost_bytes + description: "Available bytes not polled by log Agent" + bounds: + series: lost_bytes + upper_bound: 0KiB diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http2/lading/lading.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http2/lading/lading.yaml new file mode 100644 index 0000000000000..639cdb3b546aa --- /dev/null +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http2/lading/lading.yaml @@ -0,0 +1,27 @@ +generator: + - file_gen: + logrotate_fs: + seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, + 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + load_profile: + constant: 10MB + concurrent_logs: 1 + maximum_bytes_per_log: 500MB + total_rotations: 5 + max_depth: 0 + variant: "ascii" + maximum_prebuild_cache_size_bytes: 300MB + mount_point: /smp-shared + +blackhole: + - http: + binding_addr: "127.0.0.1:9091" + - http: + binding_addr: "127.0.0.1:9092" + response_delay_millis: 0 + - http: + binding_addr: "127.0.0.1:9093" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:5000/telemetry" From 5bbb3fc1b22a982f6d264f749b9c630579428179 Mon Sep 17 00:00:00 2001 From: Alexandre Yang Date: Mon, 9 Dec 2024 11:15:25 +0100 Subject: [PATCH 21/52] [HA Agent] e2e test for datadog.agent.ha_agent.running metric (#31835) --- test/new-e2e/tests/ha-agent/haagent_test.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/new-e2e/tests/ha-agent/haagent_test.go b/test/new-e2e/tests/ha-agent/haagent_test.go index bdf4e33790007..2dc700bec2be8 100644 --- a/test/new-e2e/tests/ha-agent/haagent_test.go +++ b/test/new-e2e/tests/ha-agent/haagent_test.go @@ -41,8 +41,9 @@ log_level: debug )) } -func (s *haAgentTestSuite) TestHaAgentGroupTagPresentOnDatadogAgentRunningMetric() { +func (s *haAgentTestSuite) TestHaAgentRunningMetrics() { fakeClient := s.Env().FakeIntake.Client() + s.EventuallyWithT(func(c *assert.CollectT) { s.T().Log("try assert datadog.agent.running metric") metrics, err := fakeClient.FilterMetrics("datadog.agent.running") @@ -57,6 +58,21 @@ func (s *haAgentTestSuite) TestHaAgentGroupTagPresentOnDatadogAgentRunningMetric require.NoError(c, err) assert.NotEmpty(c, metrics) }, 5*time.Minute, 3*time.Second) + + s.EventuallyWithT(func(c *assert.CollectT) { + s.T().Log("try assert datadog.agent.ha_agent.running metric") + metrics, err := fakeClient.FilterMetrics("datadog.agent.ha_agent.running") + require.NoError(c, err) + assert.NotEmpty(c, metrics) + for _, metric := range metrics { + s.T().Logf(" datadog.agent.ha_agent.running metric tags: %+v", metric.Tags) + } + + tags := []string{"agent_group:test-group01", "agent_state:unknown"} + metrics, err = fakeClient.FilterMetrics("datadog.agent.ha_agent.running", fakeintakeclient.WithTags[*aggregator.MetricSeries](tags)) + require.NoError(c, err) + assert.NotEmpty(c, metrics) + }, 5*time.Minute, 3*time.Second) } func (s *haAgentTestSuite) TestHaAgentAddedToRCListeners() { From 3bdaa27c2b3c3809831d167f453f092c4b00d3c9 Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Mon, 9 Dec 2024 11:17:40 +0100 Subject: [PATCH 22/52] add missing `test` build tag on testing utility files (#31859) --- comp/aggregator/diagnosesendermanager/component_mock.go | 2 ++ comp/snmptraps/oidresolver/oidresolverimpl/mock.go | 2 ++ pkg/network/protocols/amqp/server.go | 2 ++ pkg/network/protocols/mongo/server.go | 2 ++ pkg/network/protocols/mysql/server.go | 2 ++ pkg/network/protocols/redis/server.go | 3 +-- pkg/network/usm/monitor_testutil.go | 2 +- pkg/security/secl/rules/testutil.go | 2 ++ pkg/util/dmi/dmi_mock.go | 2 +- pkg/util/dmi/no_dmi_mock.go | 2 +- 10 files changed, 16 insertions(+), 5 deletions(-) diff --git a/comp/aggregator/diagnosesendermanager/component_mock.go b/comp/aggregator/diagnosesendermanager/component_mock.go index 73c87ebe6caee..1d1e2cd662b0b 100644 --- a/comp/aggregator/diagnosesendermanager/component_mock.go +++ b/comp/aggregator/diagnosesendermanager/component_mock.go @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2023-present Datadog, Inc. +//go:build test + // Package diagnosesendermanager defines the sender manager for the local diagnose check package diagnosesendermanager diff --git a/comp/snmptraps/oidresolver/oidresolverimpl/mock.go b/comp/snmptraps/oidresolver/oidresolverimpl/mock.go index 865e74fd56163..e9c259a876a08 100644 --- a/comp/snmptraps/oidresolver/oidresolverimpl/mock.go +++ b/comp/snmptraps/oidresolver/oidresolverimpl/mock.go @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2022-present Datadog, Inc. +//go:build test + package oidresolverimpl import ( diff --git a/pkg/network/protocols/amqp/server.go b/pkg/network/protocols/amqp/server.go index fd343ab5a29ae..31a2fd6286313 100644 --- a/pkg/network/protocols/amqp/server.go +++ b/pkg/network/protocols/amqp/server.go @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +//go:build test + package amqp import ( diff --git a/pkg/network/protocols/mongo/server.go b/pkg/network/protocols/mongo/server.go index cc9bf63b55545..5537f67b6a588 100644 --- a/pkg/network/protocols/mongo/server.go +++ b/pkg/network/protocols/mongo/server.go @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +//go:build test + package mongo import ( diff --git a/pkg/network/protocols/mysql/server.go b/pkg/network/protocols/mysql/server.go index 423ca963943e7..a152e704bd8fd 100644 --- a/pkg/network/protocols/mysql/server.go +++ b/pkg/network/protocols/mysql/server.go @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +//go:build test + package mysql import ( diff --git a/pkg/network/protocols/redis/server.go b/pkg/network/protocols/redis/server.go index 58e645a42b3fd..3c5b4f6ed7b8d 100644 --- a/pkg/network/protocols/redis/server.go +++ b/pkg/network/protocols/redis/server.go @@ -3,10 +3,9 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package redis provides a Redis client to interact with a Redis server. - //go:build test +// Package redis provides a Redis client to interact with a Redis server. package redis import ( diff --git a/pkg/network/usm/monitor_testutil.go b/pkg/network/usm/monitor_testutil.go index bbf01267af343..48f63271ed5ab 100644 --- a/pkg/network/usm/monitor_testutil.go +++ b/pkg/network/usm/monitor_testutil.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build linux_bpf +//go:build linux_bpf && test package usm diff --git a/pkg/security/secl/rules/testutil.go b/pkg/security/secl/rules/testutil.go index 2631662e7b2fe..aea0ab0779d96 100644 --- a/pkg/security/secl/rules/testutil.go +++ b/pkg/security/secl/rules/testutil.go @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +//go:build test + // Package rules holds rules related files package rules diff --git a/pkg/util/dmi/dmi_mock.go b/pkg/util/dmi/dmi_mock.go index 5bdd5325248b2..fa838328ab9c1 100644 --- a/pkg/util/dmi/dmi_mock.go +++ b/pkg/util/dmi/dmi_mock.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows && !serverless +//go:build !windows && !serverless && test package dmi diff --git a/pkg/util/dmi/no_dmi_mock.go b/pkg/util/dmi/no_dmi_mock.go index 947edcc79a393..a28da41a3becd 100644 --- a/pkg/util/dmi/no_dmi_mock.go +++ b/pkg/util/dmi/no_dmi_mock.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build windows || serverless +//go:build (windows || serverless) && test package dmi From feb37a10a39be0f721b150e2f8b0af019349b20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lian=20Raimbault?= <161456554+CelianR@users.noreply.github.com> Date: Mon, 9 Dec 2024 05:19:27 -0500 Subject: [PATCH 23/52] Fix pipeline cancel retries (#31869) --- tasks/libs/ciproviders/gitlab_api.py | 7 +++++++ tasks/libs/pipeline/tools.py | 4 ++-- tasks/pipeline.py | 7 ++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index 03ba1e58c8003..cb4e42644e7c6 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -119,6 +119,13 @@ def refresh_pipeline(pipeline: ProjectPipeline): pipeline.refresh() +@retry_function('cancel pipeline #{0.id}', retry_delay=5) +def cancel_pipeline(pipeline: ProjectPipeline): + """Cancels a pipeline, retries if there is an error.""" + + pipeline.cancel() + + class GitlabCIDiff: def __init__( self, diff --git a/tasks/libs/pipeline/tools.py b/tasks/libs/pipeline/tools.py index 2cab6a42d28b5..ce425eb700317 100644 --- a/tasks/libs/pipeline/tools.py +++ b/tasks/libs/pipeline/tools.py @@ -11,7 +11,7 @@ from gitlab.exceptions import GitlabJobPlayError from gitlab.v4.objects import Project, ProjectJob, ProjectPipeline -from tasks.libs.ciproviders.gitlab_api import refresh_pipeline +from tasks.libs.ciproviders.gitlab_api import cancel_pipeline, refresh_pipeline from tasks.libs.common.color import Color, color_message from tasks.libs.common.git import get_default_branch from tasks.libs.common.user_interactions import yes_no_question @@ -64,7 +64,7 @@ def cancel_pipelines_with_confirmation(repo: Project, pipelines: list[ProjectPip ) if yes_no_question("Do you want to cancel this pipeline?", color="orange", default=True): - pipeline.cancel() + cancel_pipeline(pipeline) print(f"Pipeline {color_message(pipeline.id, 'bold')} has been cancelled.\n") else: print(f"Pipeline {color_message(pipeline.id, 'bold')} will keep running.\n") diff --git a/tasks/pipeline.py b/tasks/pipeline.py index 8d63109fee536..bff7b05b8b1ab 100644 --- a/tasks/pipeline.py +++ b/tasks/pipeline.py @@ -13,6 +13,7 @@ from tasks.libs.ciproviders.github_api import GithubAPI from tasks.libs.ciproviders.gitlab_api import ( + cancel_pipeline, get_gitlab_bot_token, get_gitlab_repo, gitlab_configuration_is_modified, @@ -865,7 +866,7 @@ def test_merge_queue(ctx): # Clean up print("Cleaning up") if success: - pipeline.cancel() + cancel_pipeline(pipeline) pr.edit(state="closed") ctx.run(f"git checkout {current_branch}", hide=True) ctx.run(f"git branch -D {test_default}", hide=True) @@ -929,7 +930,7 @@ def compare_to_itself(ctx): if attempt == max_attempts - 1: # Clean up the branch and possible pipelines for pipeline in pipelines: - pipeline.cancel() + cancel_pipeline(pipeline) ctx.run(f"git checkout {current_branch}", hide=True) ctx.run(f"git branch -D {new_branch}", hide=True) ctx.run(f"git push origin :{new_branch}", hide=True) @@ -946,7 +947,7 @@ def compare_to_itself(ctx): # Clean up print("Cleaning up the pipelines") for pipeline in pipelines: - pipeline.cancel() + cancel_pipeline(pipeline) print("Cleaning up git") ctx.run(f"git checkout {current_branch}", hide=True) ctx.run(f"git branch -D {new_branch}", hide=True) From 61aa61b86ac48a41ab41ecb10b6d7d0f43b751af Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos <91925154+gabedos@users.noreply.github.com> Date: Mon, 9 Dec 2024 12:05:34 +0100 Subject: [PATCH 24/52] [bug] flake pod parse testing (#31871) --- .../collectors/util/kubernetes_resource_parsers/pod_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/comp/core/workloadmeta/collectors/util/kubernetes_resource_parsers/pod_test.go b/comp/core/workloadmeta/collectors/util/kubernetes_resource_parsers/pod_test.go index 2f5e8e0c941a0..5f604e3568220 100644 --- a/comp/core/workloadmeta/collectors/util/kubernetes_resource_parsers/pod_test.go +++ b/comp/core/workloadmeta/collectors/util/kubernetes_resource_parsers/pod_test.go @@ -8,6 +8,7 @@ package kubernetesresourceparsers import ( + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -16,11 +17,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" - "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" ) func TestPodParser_Parse(t *testing.T) { - flake.Mark(t) filterAnnotations := []string{"ignoreAnnotation"} parser, err := NewPodParser(filterAnnotations) @@ -126,5 +125,6 @@ func TestPodParser_Parse(t *testing.T) { QOSClass: "Guaranteed", } - assert.Equal(t, expected, parsed) + assert.True(t, reflect.DeepEqual(expected, parsed), + "Expected: %v, Actual: %v", expected, parsed) } From 92bf0e8cf9db418c17677fc235ea0f367cdc77d2 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Mon, 9 Dec 2024 12:09:55 +0100 Subject: [PATCH 25/52] fix(assign_issue): Use the correct option name (#31867) --- .github/workflows/assign_issue.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml index be9db6209604b..f752788cf64c8 100644 --- a/.github/workflows/assign_issue.yml +++ b/.github/workflows/assign_issue.yml @@ -27,4 +27,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - inv -e issue.assign-owner --issue ${{ github.event.issue.number }} + inv -e issue.assign-owner -i ${{ github.event.issue.number }} From 8619535c19ac43fe3a9b39b78dd3b92348d3ac4c Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Mon, 9 Dec 2024 14:11:59 +0200 Subject: [PATCH 26/52] event monitor: tests: Allow setting cached hostname (#31863) --- pkg/ebpf/uprobes/attacher_test.go | 2 ++ pkg/eventmonitor/consumers/testutil/testutil.go | 4 +++- pkg/network/usm/monitor_tls_test.go | 2 ++ pkg/network/usm/sharedlibraries/watcher_test.go | 5 +++-- pkg/process/monitor/process_monitor_test.go | 2 ++ pkg/security/utils/hostname_testutil.go | 15 +++++++++++++++ 6 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 pkg/security/utils/hostname_testutil.go diff --git a/pkg/ebpf/uprobes/attacher_test.go b/pkg/ebpf/uprobes/attacher_test.go index abad66f8778f2..26428868c1d70 100644 --- a/pkg/ebpf/uprobes/attacher_test.go +++ b/pkg/ebpf/uprobes/attacher_test.go @@ -34,6 +34,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/process/monitor" procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" + secutils "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/util/kernel" ) @@ -799,6 +800,7 @@ func launchProcessMonitor(t *testing.T, useEventStream bool) *monitor.ProcessMon t.Cleanup(pm.Stop) require.NoError(t, pm.Initialize(useEventStream)) if useEventStream { + secutils.SetCachedHostname("test-hostname") eventmonitortestutil.StartEventMonitor(t, procmontestutil.RegisterProcessMonitorEventConsumer) } diff --git a/pkg/eventmonitor/consumers/testutil/testutil.go b/pkg/eventmonitor/consumers/testutil/testutil.go index 4ed3b9fb2a1ea..871cf780b0089 100644 --- a/pkg/eventmonitor/consumers/testutil/testutil.go +++ b/pkg/eventmonitor/consumers/testutil/testutil.go @@ -15,8 +15,8 @@ import ( "github.com/DataDog/datadog-agent/pkg/eventmonitor" "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers" - eventtestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/testutil" + "github.com/DataDog/datadog-agent/pkg/security/utils" ) const defaultChanSize = 100 @@ -25,6 +25,8 @@ const defaultChanSize = 100 // created for testing. This function should be called in tests that require a ProcessConsumer. func NewTestProcessConsumer(t *testing.T) *consumers.ProcessConsumer { var pc *consumers.ProcessConsumer + // Set fake hostname to avoid fetching it from the core agent. + utils.SetCachedHostname("test-hostname") eventtestutil.StartEventMonitor(t, func(t *testing.T, evm *eventmonitor.EventMonitor) { var err error eventTypes := []consumers.ProcessConsumerEventTypes{consumers.ExecEventType, consumers.ExitEventType} diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index a4fd5924bd630..322f9be425da4 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -45,6 +45,7 @@ import ( usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" + secutils "github.com/DataDog/datadog-agent/pkg/security/utils" globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) @@ -869,6 +870,7 @@ func setupUSMTLSMonitor(t *testing.T, cfg *config.Config) *Monitor { require.NoError(t, err) require.NoError(t, usmMonitor.Start()) if cfg.EnableUSMEventStream && usmconfig.NeedProcessMonitor(cfg) { + secutils.SetCachedHostname("test-hostname") eventmonitortestutil.StartEventMonitor(t, procmontestutil.RegisterProcessMonitorEventConsumer) } t.Cleanup(usmMonitor.Stop) diff --git a/pkg/network/usm/sharedlibraries/watcher_test.go b/pkg/network/usm/sharedlibraries/watcher_test.go index ec8eddb5f8edf..ca401e180fc50 100644 --- a/pkg/network/usm/sharedlibraries/watcher_test.go +++ b/pkg/network/usm/sharedlibraries/watcher_test.go @@ -21,9 +21,8 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" "github.com/DataDog/datadog-agent/pkg/ebpf/prebuilt" @@ -33,6 +32,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/process/monitor" procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" + secutils "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -42,6 +42,7 @@ func launchProcessMonitor(t *testing.T, useEventStream bool) { t.Cleanup(pm.Stop) require.NoError(t, pm.Initialize(useEventStream)) if useEventStream { + secutils.SetCachedHostname("test-hostname") eventmonitortestutil.StartEventMonitor(t, procmontestutil.RegisterProcessMonitorEventConsumer) } } diff --git a/pkg/process/monitor/process_monitor_test.go b/pkg/process/monitor/process_monitor_test.go index b6cc085089e74..0fe9e80bd21cb 100644 --- a/pkg/process/monitor/process_monitor_test.go +++ b/pkg/process/monitor/process_monitor_test.go @@ -22,6 +22,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/eventmonitor" eventmonitortestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/testutil" "github.com/DataDog/datadog-agent/pkg/network/protocols/telemetry" + "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/util" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -55,6 +56,7 @@ func waitForProcessMonitor(t *testing.T, pm *ProcessMonitor) { func initializePM(t *testing.T, pm *ProcessMonitor, useEventStream bool) { require.NoError(t, pm.Initialize(useEventStream)) if useEventStream { + utils.SetCachedHostname("test-hostname") eventmonitortestutil.StartEventMonitor(t, func(t *testing.T, evm *eventmonitor.EventMonitor) { // Can't use the implementation in procmontestutil due to import cycles procmonconsumer, err := NewProcessMonitorEventConsumer(evm) diff --git a/pkg/security/utils/hostname_testutil.go b/pkg/security/utils/hostname_testutil.go new file mode 100644 index 0000000000000..e8b180e1665b6 --- /dev/null +++ b/pkg/security/utils/hostname_testutil.go @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build test + +package utils + +// SetCachedHostname test utility to set the cached hostname, to avoid fetching it from the core agent. +func SetCachedHostname(name string) { + hostnameLock.Lock() + cachedHostname = name + hostnameLock.Unlock() +} From 09ef14cb519acddee4102017b9b8c911dd067fcb Mon Sep 17 00:00:00 2001 From: maxime mouial Date: Mon, 9 Dec 2024 14:19:17 +0100 Subject: [PATCH 27/52] Fix diff around default and known settings (#31658) --- pkg/config/model/viper.go | 2 +- pkg/config/nodetreemodel/config.go | 80 +++++++++++++------ pkg/config/nodetreemodel/config_test.go | 14 +++- pkg/config/nodetreemodel/getter.go | 51 +++++++++++- pkg/config/nodetreemodel/getter_test.go | 24 +++++- pkg/config/nodetreemodel/read_config_file.go | 37 ++++++--- .../nodetreemodel/read_config_file_test.go | 2 +- pkg/config/setup/config.go | 11 ++- pkg/config/setup/config_test.go | 7 +- pkg/config/teeconfig/teeconfig.go | 80 ++++++++++--------- 10 files changed, 214 insertions(+), 94 deletions(-) diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index eeb4b65492e70..5742b4c06520b 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -852,7 +852,7 @@ func (c *safeConfig) GetProxies() *Proxy { if c.Viper.GetBool("fips.enabled") { return nil } - if !c.Viper.IsSet("proxy.http") && !c.Viper.IsSet("proxy.https") && !c.Viper.IsSet("proxy.no_proxy") { + if c.Viper.GetString("proxy.http") == "" && c.Viper.GetString("proxy.https") == "" && len(c.Viper.GetStringSlice("proxy.no_proxy")) == 0 { return nil } p := &Proxy{ diff --git a/pkg/config/nodetreemodel/config.go b/pkg/config/nodetreemodel/config.go index 9e293bb9a7592..3d09ddc016eba 100644 --- a/pkg/config/nodetreemodel/config.go +++ b/pkg/config/nodetreemodel/config.go @@ -51,6 +51,18 @@ type ntmConfig struct { // Bellow are all the different configuration layers. Each layers represents a source for our configuration. // They are merge into the 'root' tree following order of importance (see pkg/model/viper.go:sourcesPriority). + // schema holds all the settings with or without value. Settings are added to the schema through BindEnv and + // SetDefault. + // + // This solved the difference between 'AllKeysLowercased' which returns the configuration schema and + // 'AllSettings' which only returns settings with a value. + // + // A setting register with BindEnv without default might not have a value depending on the environment. Such + // settings are part of the schema but won't appear in the configuration (through Get, AllSettings, ...). This + // mimic the behavior from Viper. Once we enfore a default value for all settings we will be able to merge + // 'schema' and 'defaults' fields. + schema InnerNode + // defaults contains the settings with a default value defaults InnerNode // unknown contains the settings set at runtime from unknown source. This should only evey be used by tests. @@ -131,11 +143,11 @@ func (c *ntmConfig) OnUpdate(callback model.NotificationReceiver) { c.notificationReceivers = append(c.notificationReceivers, callback) } -func (c *ntmConfig) setDefault(key string, value interface{}) { +func (c *ntmConfig) addToSchema(key string, source model.Source) { parts := splitKey(key) - // TODO: Ensure that for default tree, setting nil to a node will not override - // an existing value - _, _ = c.defaults.SetAt(parts, value, model.SourceDefault) + _, _ = c.schema.SetAt(parts, nil, source) + + c.addToKnownKeys(key) } func (c *ntmConfig) getTreeBySource(source model.Source) (InnerNode, error) { @@ -179,7 +191,7 @@ func (c *ntmConfig) Set(key string, newValue interface{}, source model.Source) { key = strings.ToLower(key) c.Lock() - previousValue := c.leafAtPath(key).Get() + previousValue := c.leafAtPathFromNode(key, c.root).Get() parts := splitKey(key) @@ -221,8 +233,12 @@ func (c *ntmConfig) SetDefault(key string, value interface{}) { panic("cannot SetDefault() once the config has been marked as ready for use") } key = strings.ToLower(key) - c.knownKeys[key] = struct{}{} - c.setDefault(key, value) + c.addToSchema(key, model.SourceDefault) + + parts := splitKey(key) + // TODO: Ensure that for default tree, setting nil to a node will not override + // an existing value + _, _ = c.defaults.SetAt(parts, value, model.SourceDefault) } // UnsetForSource unsets a config entry for a given source @@ -232,7 +248,19 @@ func (c *ntmConfig) UnsetForSource(_key string, _source model.Source) { c.Unlock() } -// SetKnown adds a key to the set of known valid config keys +func (c *ntmConfig) addToKnownKeys(key string) { + base := "" + keyParts := splitKey(key) + for _, part := range keyParts { + base = joinKey(base, part) + c.knownKeys[base] = struct{}{} + } +} + +// SetKnown adds a key to the set of known valid config keys. +// +// Important: this doesn't add the key to the schema. The "known keys" are a legacy feature we inherited from our Viper +// wrapper. Once all settings have a default we'll be able to remove this concept entirely. func (c *ntmConfig) SetKnown(key string) { c.Lock() defer c.Unlock() @@ -240,8 +268,7 @@ func (c *ntmConfig) SetKnown(key string) { panic("cannot SetKnown() once the config has been marked as ready for use") } - key = strings.ToLower(key) - c.knownKeys[key] = struct{}{} + c.addToKnownKeys(key) } // IsKnown returns whether a key is known @@ -273,6 +300,7 @@ func (c *ntmConfig) checkKnownKey(key string) { } func (c *ntmConfig) mergeAllLayers() error { + // We intentionally don't merge the schema layer as it hold no values treeList := []InnerNode{ c.defaults, c.unknown, @@ -324,7 +352,7 @@ func (c *ntmConfig) BuildSchema() { if err := c.mergeAllLayers(); err != nil { c.warnings = append(c.warnings, err.Error()) } - c.allSettings = computeAllSettings(c.defaults, "") + c.allSettings = computeAllSettings(c.schema, "") } // Stringify stringifies the config, but only with the test build tag @@ -409,7 +437,21 @@ func (c *ntmConfig) IsSet(key string) bool { c.RLock() defer c.RUnlock() - return c.IsKnown(key) + if !c.isReady() { + log.Errorf("attempt to read key before config is constructed: %s", key) + return false + } + + pathParts := splitKey(key) + var curr Node = c.root + for _, part := range pathParts { + next, err := curr.GetChild(part) + if err != nil { + return false + } + curr = next + } + return true } // AllKeysLowercased returns all keys lower-cased from the default tree, but not keys that are merely marked as known @@ -420,16 +462,7 @@ func (c *ntmConfig) AllKeysLowercased() []string { return slices.Clone(c.allSettings) } -func (c *ntmConfig) leafAtPath(key string) LeafNode { - return c.leafAtPathFromNode(key, c.root) -} - func (c *ntmConfig) leafAtPathFromNode(key string, curr Node) LeafNode { - if !c.isReady() { - log.Errorf("attempt to read key before config is constructed: %s", key) - return missingLeaf - } - pathParts := splitKey(key) for _, part := range pathParts { next, err := curr.GetChild(part) @@ -496,8 +529,7 @@ func (c *ntmConfig) BindEnv(key string, envvars ...string) { c.configEnvVars[envvar] = key } - c.knownKeys[key] = struct{}{} - c.setDefault(key, nil) + c.addToSchema(key, model.SourceEnvVar) } // SetEnvKeyReplacer binds a replacer function for keys @@ -706,6 +738,7 @@ func NewConfig(name string, envPrefix string, envKeyReplacer *strings.Replacer) knownKeys: map[string]struct{}{}, allSettings: []string{}, unknownKeys: map[string]struct{}{}, + schema: newInnerNode(nil), defaults: newInnerNode(nil), file: newInnerNode(nil), unknown: newInnerNode(nil), @@ -716,6 +749,7 @@ func NewConfig(name string, envPrefix string, envKeyReplacer *strings.Replacer) fleetPolicies: newInnerNode(nil), cli: newInnerNode(nil), envTransform: make(map[string]func(string) interface{}), + configName: "datadog", } config.SetConfigName(name) diff --git a/pkg/config/nodetreemodel/config_test.go b/pkg/config/nodetreemodel/config_test.go index c9f87747e2345..dfba602113411 100644 --- a/pkg/config/nodetreemodel/config_test.go +++ b/pkg/config/nodetreemodel/config_test.go @@ -64,13 +64,13 @@ secret_backend_command: ./my_secret_fetcher.sh { description: "nested setting from env var works", setting: "network_path.collector.input_chan_size", - expectValue: "23456", + expectValue: 23456, expectSource: model.SourceEnvVar, }, { description: "top-level setting from env var works", setting: "secret_backend_timeout", - expectValue: "60", // TODO: cfg.Get returns string because this is an env var + expectValue: 60, expectSource: model.SourceEnvVar, }, { @@ -326,13 +326,21 @@ func TestIsSet(t *testing.T) { cfg := NewConfig("test", "TEST", nil) cfg.SetDefault("a", 0) cfg.SetDefault("b", 0) + cfg.SetKnown("c") cfg.BuildSchema() cfg.Set("b", 123, model.SourceAgentRuntime) - assert.True(t, cfg.IsSet("b")) assert.True(t, cfg.IsSet("a")) + assert.True(t, cfg.IsSet("b")) + assert.False(t, cfg.IsSet("c")) + + assert.True(t, cfg.IsKnown("a")) + assert.True(t, cfg.IsKnown("b")) + assert.True(t, cfg.IsKnown("c")) + assert.False(t, cfg.IsSet("unknown")) + assert.False(t, cfg.IsKnown("unknown")) } func TestAllKeysLowercased(t *testing.T) { diff --git a/pkg/config/nodetreemodel/getter.go b/pkg/config/nodetreemodel/getter.go index 454bbe2b885aa..450271f001f95 100644 --- a/pkg/config/nodetreemodel/getter.go +++ b/pkg/config/nodetreemodel/getter.go @@ -16,6 +16,15 @@ import ( "golang.org/x/exp/slices" ) +func (c *ntmConfig) leafAtPath(key string) LeafNode { + if !c.isReady() { + log.Errorf("attempt to read key before config is constructed: %s", key) + return missingLeaf + } + + return c.leafAtPathFromNode(key, c.root) +} + // GetKnownKeysLowercased returns all the keys that meet at least one of these criteria: // 1) have a default, 2) have an environment variable binded or 3) have been SetKnown() // Note that it returns the keys lowercased. @@ -71,16 +80,50 @@ func (c *ntmConfig) GetProxies() *model.Proxy { return c.proxies } +func (c *ntmConfig) inferTypeFromDefault(key string, value interface{}) (interface{}, error) { + // Viper infer the type from the default value for Get. This reproduce the same behavior. + // Once all settings have a default value we could move this logic where we load data into the config rather + // than out. + defaultNode := c.leafAtPathFromNode(key, c.defaults) + if defaultNode != missingLeaf { + switch defaultNode.Get().(type) { + case bool: + return cast.ToBoolE(value) + case string: + return cast.ToStringE(value) + case int32, int16, int8, int: + return cast.ToIntE(value) + case int64: + return cast.ToInt64E(value) + case float64, float32: + return cast.ToFloat64E(value) + case time.Time: + return cast.ToTimeE(value) + case time.Duration: + return cast.ToDurationE(value) + case []string: + return cast.ToStringSliceE(value) + } + } + + // NOTE: should only need to deepcopy for `Get`, because it can be an arbitrary value, + // and we shouldn't ever return complex types like maps and slices that could be modified + // by callers accidentally or on purpose. By copying, the caller may modify the result safetly + return deepcopy.Copy(value), nil +} + // Get returns a copy of the value for the given key func (c *ntmConfig) Get(key string) interface{} { c.RLock() defer c.RUnlock() c.checkKnownKey(key) val := c.leafAtPath(key).Get() - // NOTE: should only need to deepcopy for `Get`, because it can be an arbitrary value, - // and we shouldn't ever return complex types like maps and slices that could be modified - // by callers accidentally or on purpose. By copying, the caller may modify the result safetly - return deepcopy.Copy(val) + + val, err := c.inferTypeFromDefault(key, val) + if err != nil { + log.Warnf("failed to get configuration value for key %q: %s", key, err) + } + return val } // GetAllSources returns all values for a key for each source in sorted from lower to higher priority diff --git a/pkg/config/nodetreemodel/getter_test.go b/pkg/config/nodetreemodel/getter_test.go index fc5bf9b3d583e..3c05bffc40d27 100644 --- a/pkg/config/nodetreemodel/getter_test.go +++ b/pkg/config/nodetreemodel/getter_test.go @@ -24,7 +24,10 @@ func TestGetKnownKeysLowercased(t *testing.T) { assert.Equal(t, map[string]interface{}{ "a": struct{}{}, + "b": struct{}{}, "b.c": struct{}{}, + "d": struct{}{}, + "d.e": struct{}{}, "d.e.f": struct{}{}, }, cfg.GetKnownKeysLowercased()) @@ -37,8 +40,25 @@ func TestGet(t *testing.T) { assert.Equal(t, 1234, cfg.Get("a")) - cfg.Set("a", "test", model.SourceAgentRuntime) - assert.Equal(t, "test", cfg.Get("a")) + cfg.Set("a", 9876, model.SourceAgentRuntime) + assert.Equal(t, 9876, cfg.Get("a")) + + assert.Equal(t, nil, cfg.Get("does_not_exists")) +} + +func TestGetCastToDefault(t *testing.T) { + cfg := NewConfig("test", "", nil) + cfg.SetDefault("a", []string{}) + cfg.BuildSchema() + + // This test that we mimic viper's behavior on Get where we convert the value from the config to the same type + // from the default. + + cfg.Set("a", 9876, model.SourceAgentRuntime) + assert.Equal(t, []string{"9876"}, cfg.Get("a")) + + cfg.Set("a", "a b c", model.SourceAgentRuntime) + assert.Equal(t, []string{"a", "b", "c"}, cfg.Get("a")) assert.Equal(t, nil, cfg.Get("does_not_exists")) } diff --git a/pkg/config/nodetreemodel/read_config_file.go b/pkg/config/nodetreemodel/read_config_file.go index c829ff6837aeb..8354a012376b0 100644 --- a/pkg/config/nodetreemodel/read_config_file.go +++ b/pkg/config/nodetreemodel/read_config_file.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "path/filepath" "reflect" "strings" @@ -17,11 +18,16 @@ import ( "gopkg.in/yaml.v2" ) -func (c *ntmConfig) getConfigFile() string { +func (c *ntmConfig) findConfigFile() { if c.configFile == "" { - return "datadog.yaml" + for _, path := range c.configPaths { + configFilePath := filepath.Join(path, c.configName+".yaml") + if _, err := os.Stat(configFilePath); err == nil { + c.configFile = configFilePath + return + } + } } - return c.configFile } // ReadInConfig wraps Viper for concurrent access @@ -33,7 +39,8 @@ func (c *ntmConfig) ReadInConfig() error { c.Lock() defer c.Unlock() - err := c.readInConfig(c.getConfigFile()) + c.findConfigFile() + err := c.readInConfig(c.configFile) if err != nil { return err } @@ -76,10 +83,14 @@ func (c *ntmConfig) readInConfig(filePath string) error { func (c *ntmConfig) readConfigurationContent(target InnerNode, content []byte) error { var obj map[string]interface{} - if err := yaml.Unmarshal(content, &obj); err != nil { - return err + + if strictErr := yaml.UnmarshalStrict(content, &obj); strictErr != nil { + log.Errorf("warning reading config file: %v\n", strictErr) + if err := yaml.Unmarshal(content, &obj); err != nil { + return err + } } - c.warnings = append(c.warnings, loadYamlInto(c.defaults, target, obj, "")...) + c.warnings = append(c.warnings, loadYamlInto(c.schema, target, obj, "")...) return nil } @@ -113,7 +124,7 @@ func toMapStringInterface(data any, path string) (map[string]interface{}, error) // // The function traverses a object loaded from YAML, checking if each node is known within the configuration. // If known, the value from the YAML blob is imported into the 'dest' tree. If unknown, a warning will be created. -func loadYamlInto(defaults InnerNode, dest InnerNode, data map[string]interface{}, path string) []string { +func loadYamlInto(schema InnerNode, dest InnerNode, data map[string]interface{}, path string) []string { if path != "" { path = path + "." } @@ -123,15 +134,15 @@ func loadYamlInto(defaults InnerNode, dest InnerNode, data map[string]interface{ key = strings.ToLower(key) curPath := path + key - // check if the key is know in the defaults - defaultNode, err := defaults.GetChild(key) + // check if the key is know in the schema + schemaNode, err := schema.GetChild(key) if err != nil { warnings = append(warnings, fmt.Sprintf("unknown key from YAML: %s", curPath)) continue } // if the default is a leaf we create a new leaf in dest - if _, isLeaf := defaultNode.(LeafNode); isLeaf { + if _, isLeaf := schemaNode.(LeafNode); isLeaf { // check that dest don't have a inner leaf under that name c, _ := dest.GetChild(key) if _, ok := c.(InnerNode); ok { @@ -148,8 +159,8 @@ func loadYamlInto(defaults InnerNode, dest InnerNode, data map[string]interface{ warnings = append(warnings, err.Error()) } - // by now we know defaultNode is an InnerNode - defaultNext, _ := defaultNode.(InnerNode) + // by now we know schemaNode is an InnerNode + defaultNext, _ := schemaNode.(InnerNode) if !dest.HasChild(key) { destInner := newInnerNode(nil) diff --git a/pkg/config/nodetreemodel/read_config_file_test.go b/pkg/config/nodetreemodel/read_config_file_test.go index 3b68045071be1..4a9281c289065 100644 --- a/pkg/config/nodetreemodel/read_config_file_test.go +++ b/pkg/config/nodetreemodel/read_config_file_test.go @@ -151,7 +151,7 @@ c: cfg.SetDefault("a", "apple") cfg.SetDefault("b", 123) - cfg.SetDefault("c.d", true) + cfg.SetDefault("c.d", 1) cfg.SetDefault("c.e.f", 456) cfg.BuildSchema() diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 73648316c5f4c..a9c74bb2765fa 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -1038,7 +1038,10 @@ func agent(config pkgconfigmodel.Setup) { config.BindEnv("dd_url", "DD_DD_URL", "DD_URL") config.BindEnvAndSetDefault("app_key", "") config.BindEnvAndSetDefault("cloud_provider_metadata", []string{"aws", "gcp", "azure", "alibaba", "oracle", "ibm"}) - config.SetDefault("proxy", nil) + config.SetDefault("proxy.http", "") + config.SetDefault("proxy.https", "") + config.SetDefault("proxy.no_proxy", []string{}) + config.BindEnvAndSetDefault("skip_ssl_validation", false) config.BindEnvAndSetDefault("sslkeylogfile", "") config.BindEnv("tls_handshake_timeout") @@ -1121,10 +1124,6 @@ func agent(config pkgconfigmodel.Setup) { config.BindEnvAndSetDefault("GUI_port", defaultGuiPort) config.BindEnvAndSetDefault("GUI_session_expiration", 0) - config.SetKnown("proxy.http") - config.SetKnown("proxy.https") - config.SetKnown("proxy.no_proxy") - // Core agent (disabled for Error Tracking Standalone, Logs Collection Only) config.BindEnvAndSetDefault("core_agent.enabled", true) pkgconfigmodel.AddOverrideFunc(toggleDefaultPayloads) @@ -1331,7 +1330,7 @@ func serverless(config pkgconfigmodel.Setup) { config.BindEnvAndSetDefault("capture_lambda_payload_max_depth", 10) config.BindEnvAndSetDefault("serverless.trace_enabled", true, "DD_TRACE_ENABLED") config.BindEnvAndSetDefault("serverless.trace_managed_services", true, "DD_TRACE_MANAGED_SERVICES") - config.BindEnvAndSetDefault("serverless.service_mapping", nil, "DD_SERVICE_MAPPING") + config.BindEnvAndSetDefault("serverless.service_mapping", "", "DD_SERVICE_MAPPING") } func forwarder(config pkgconfigmodel.Setup) { diff --git a/pkg/config/setup/config_test.go b/pkg/config/setup/config_test.go index 1421770758180..c9713b2fccd9c 100644 --- a/pkg/config/setup/config_test.go +++ b/pkg/config/setup/config_test.go @@ -243,7 +243,7 @@ func TestProxy(t *testing.T) { { name: "no values", tests: func(t *testing.T, config pkgconfigmodel.Config) { - assert.Nil(t, config.Get("proxy")) + assert.Equal(t, map[string]interface{}{"http": "", "https": "", "no_proxy": []interface{}{}}, config.Get("proxy")) assert.Nil(t, config.GetProxies()) }, proxyForCloudMetadata: true, @@ -373,7 +373,7 @@ func TestProxy(t *testing.T) { proxyForCloudMetadata: true, }, { - name: "proxy withou no_proxy", + name: "proxy without no_proxy", setup: func(t *testing.T, _ pkgconfigmodel.Config) { t.Setenv("DD_PROXY_HTTP", "http_url") t.Setenv("DD_PROXY_HTTPS", "https_url") @@ -385,7 +385,8 @@ func TestProxy(t *testing.T) { HTTPS: "https_url", }, config.GetProxies()) - assert.Equal(t, []interface{}{}, config.Get("proxy.no_proxy")) + fmt.Printf("%#v\n", config.Get("proxy.no_proxy")) + assert.Equal(t, []string(nil), config.Get("proxy.no_proxy")) }, proxyForCloudMetadata: true, }, diff --git a/pkg/config/teeconfig/teeconfig.go b/pkg/config/teeconfig/teeconfig.go index efbc1c44b8e82..ba787dd5db77b 100644 --- a/pkg/config/teeconfig/teeconfig.go +++ b/pkg/config/teeconfig/teeconfig.go @@ -7,9 +7,11 @@ package teeconfig import ( + "fmt" "io" "reflect" "runtime" + "slices" "strings" "time" @@ -25,6 +27,12 @@ type teeConfig struct { compare model.Config } +func getLocation(nbStack int) string { + _, file, line, _ := runtime.Caller(nbStack + 1) + fileParts := strings.Split(file, "DataDog/datadog-agent/") + return fmt.Sprintf("%s:%d", fileParts[len(fileParts)-1], line) +} + // NewTeeConfig constructs a new teeConfig func NewTeeConfig(baseline, compare model.Config) model.Config { return &teeConfig{baseline: baseline, compare: compare} @@ -84,7 +92,7 @@ func (t *teeConfig) IsKnown(key string) bool { func (t *teeConfig) GetKnownKeysLowercased() map[string]interface{} { base := t.baseline.GetKnownKeysLowercased() compare := t.compare.GetKnownKeysLowercased() - compareResult("", "GetKnownKeysLowercased", base, compare) + t.compareResult("", "GetKnownKeysLowercased", base, compare) return base } @@ -125,7 +133,7 @@ func (t *teeConfig) IsSet(key string) bool { base := t.baseline.IsSet(key) compare := t.compare.IsSet(key) if base != compare { - log.Warnf("difference in config: IsSet(%s) -> base: %v | compare %v", key, base, compare) + log.Warnf("difference in config: IsSet(%s) -> base[%s]: %v | compare[%s]: %v | from %s", key, t.baseline.GetSource(key), base, t.compare.GetSource(key), compare, getLocation(1)) } return base } @@ -134,21 +142,22 @@ func (t *teeConfig) AllKeysLowercased() []string { base := t.baseline.AllKeysLowercased() compare := t.compare.AllKeysLowercased() if !reflect.DeepEqual(base, compare) { - log.Warnf("difference in config: allkeyslowercased() -> base len: %d | compare len: %d", len(base), len(compare)) + log.Warnf("difference in config: AllKeysLowercased() -> base len: %d | compare len: %d", len(base), len(compare)) i := 0 j := 0 for i < len(base) && j < len(compare) { - if base[i] != compare[j] { + if base[i] == compare[j] { i++ j++ continue } - log.Warnf("difference in config: allkeyslowercased() -> base[%d]: %v | compare[%d]: %v", i, base[i], j, compare[j]) if strings.Compare(base[i], compare[j]) == -1 { + log.Warnf("difference in config: allkeyslowercased() missing key in compare -> base[%d]: %#v", i, base[i]) i++ } else { + log.Warnf("difference in config: allkeyslowercased() extra key in compare -> --- | compare[%d]: %#v", j, compare[j]) j++ } } @@ -156,27 +165,25 @@ func (t *teeConfig) AllKeysLowercased() []string { return base } -func compareResult(key, method string, base, compare interface{}) interface{} { +func (t *teeConfig) compareResult(key, method string, base, compare interface{}) { if !reflect.DeepEqual(base, compare) { - _, file, line, _ := runtime.Caller(2) - fileParts := strings.Split(file, "DataDog/datadog-agent/") - log.Warnf("difference in config: %s(%s) -> base: %v | compare %v from %s:%d", method, key, base, compare, fileParts[len(fileParts)-1], line) + log.Warnf("difference in config: %s(%s) -> base[%s]: %#v | compare[%s] %#v | from %s", method, key, t.baseline.GetSource(key), base, t.compare.GetSource(key), compare, getLocation(2)) } - return compare } // Get wraps Viper for concurrent access func (t *teeConfig) Get(key string) interface{} { base := t.baseline.Get(key) compare := t.compare.Get(key) - return compareResult(key, "Get", base, compare) + t.compareResult(key, "Get", base, compare) + return base } // GetAllSources returns the value of a key for each source func (t *teeConfig) GetAllSources(key string) []model.ValueWithSource { base := t.baseline.GetAllSources(key) compare := t.compare.GetAllSources(key) - compareResult(key, "GetAllSources", base, compare) + t.compareResult(key, "GetAllSources", base, compare) return base } @@ -184,7 +191,7 @@ func (t *teeConfig) GetAllSources(key string) []model.ValueWithSource { func (t *teeConfig) GetString(key string) string { base := t.baseline.GetString(key) compare := t.compare.GetString(key) - compareResult(key, "GetString", base, compare) + t.compareResult(key, "GetString", base, compare) return base } @@ -193,7 +200,7 @@ func (t *teeConfig) GetString(key string) string { func (t *teeConfig) GetBool(key string) bool { base := t.baseline.GetBool(key) compare := t.compare.GetBool(key) - compareResult(key, "GetBool", base, compare) + t.compareResult(key, "GetBool", base, compare) return base } @@ -202,7 +209,7 @@ func (t *teeConfig) GetBool(key string) bool { func (t *teeConfig) GetInt(key string) int { base := t.baseline.GetInt(key) compare := t.compare.GetInt(key) - compareResult(key, "GetInt", base, compare) + t.compareResult(key, "GetInt", base, compare) return base } @@ -211,7 +218,7 @@ func (t *teeConfig) GetInt(key string) int { func (t *teeConfig) GetInt32(key string) int32 { base := t.baseline.GetInt32(key) compare := t.compare.GetInt32(key) - compareResult(key, "GetInt32", base, compare) + t.compareResult(key, "GetInt32", base, compare) return base } @@ -220,7 +227,7 @@ func (t *teeConfig) GetInt32(key string) int32 { func (t *teeConfig) GetInt64(key string) int64 { base := t.baseline.GetInt64(key) compare := t.compare.GetInt64(key) - compareResult(key, "GetInt64", base, compare) + t.compareResult(key, "GetInt64", base, compare) return base } @@ -229,7 +236,7 @@ func (t *teeConfig) GetInt64(key string) int64 { func (t *teeConfig) GetFloat64(key string) float64 { base := t.baseline.GetFloat64(key) compare := t.compare.GetFloat64(key) - compareResult(key, "GetFloat64", base, compare) + t.compareResult(key, "GetFloat64", base, compare) return base } @@ -238,7 +245,7 @@ func (t *teeConfig) GetFloat64(key string) float64 { func (t *teeConfig) GetDuration(key string) time.Duration { base := t.baseline.GetDuration(key) compare := t.compare.GetDuration(key) - compareResult(key, "GetDuration", base, compare) + t.compareResult(key, "GetDuration", base, compare) return base } @@ -247,7 +254,7 @@ func (t *teeConfig) GetDuration(key string) time.Duration { func (t *teeConfig) GetStringSlice(key string) []string { base := t.baseline.GetStringSlice(key) compare := t.compare.GetStringSlice(key) - compareResult(key, "GetStringSlice", base, compare) + t.compareResult(key, "GetStringSlice", base, compare) return base } @@ -256,7 +263,7 @@ func (t *teeConfig) GetStringSlice(key string) []string { func (t *teeConfig) GetFloat64Slice(key string) []float64 { base := t.baseline.GetFloat64Slice(key) compare := t.compare.GetFloat64Slice(key) - compareResult(key, "GetFloat64Slice", base, compare) + t.compareResult(key, "GetFloat64Slice", base, compare) return base } @@ -265,7 +272,7 @@ func (t *teeConfig) GetFloat64Slice(key string) []float64 { func (t *teeConfig) GetStringMap(key string) map[string]interface{} { base := t.baseline.GetStringMap(key) compare := t.compare.GetStringMap(key) - compareResult(key, "GetStringMap", base, compare) + t.compareResult(key, "GetStringMap", base, compare) return base } @@ -274,7 +281,7 @@ func (t *teeConfig) GetStringMap(key string) map[string]interface{} { func (t *teeConfig) GetStringMapString(key string) map[string]string { base := t.baseline.GetStringMapString(key) compare := t.compare.GetStringMapString(key) - compareResult(key, "GetStringMapString", base, compare) + t.compareResult(key, "GetStringMapString", base, compare) return base } @@ -283,7 +290,7 @@ func (t *teeConfig) GetStringMapString(key string) map[string]string { func (t *teeConfig) GetStringMapStringSlice(key string) map[string][]string { base := t.baseline.GetStringMapStringSlice(key) compare := t.compare.GetStringMapStringSlice(key) - compareResult(key, "GetStringMapStringSlice", base, compare) + t.compareResult(key, "GetStringMapStringSlice", base, compare) return base } @@ -292,7 +299,7 @@ func (t *teeConfig) GetStringMapStringSlice(key string) map[string][]string { func (t *teeConfig) GetSizeInBytes(key string) uint { base := t.baseline.GetSizeInBytes(key) compare := t.compare.GetSizeInBytes(key) - compareResult(key, "GetSizeInBytes", base, compare) + t.compareResult(key, "GetSizeInBytes", base, compare) return base } @@ -301,7 +308,7 @@ func (t *teeConfig) GetSizeInBytes(key string) uint { func (t *teeConfig) GetSource(key string) model.Source { base := t.baseline.GetSource(key) compare := t.compare.GetSource(key) - compareResult(key, "GetSource", base, compare) + t.compareResult(key, "GetSource", base, compare) return base } @@ -386,7 +393,7 @@ func (t *teeConfig) AllSettings() map[string]interface{} { continue } if !reflect.DeepEqual(base[key], compare[key]) { - log.Warnf("\titem %s: %v | %v", key, base[key], compare[key]) + log.Warnf("\titem %s: %#v | %#v", key, base[key], compare[key]) } log.Flush() } @@ -399,7 +406,7 @@ func (t *teeConfig) AllSettings() map[string]interface{} { func (t *teeConfig) AllSettingsWithoutDefault() map[string]interface{} { base := t.baseline.AllSettingsWithoutDefault() compare := t.compare.AllSettingsWithoutDefault() - compareResult("", "AllSettingsWithoutDefault", base, compare) + t.compareResult("", "AllSettingsWithoutDefault", base, compare) return base } @@ -408,7 +415,7 @@ func (t *teeConfig) AllSettingsWithoutDefault() map[string]interface{} { func (t *teeConfig) AllSettingsBySource() map[model.Source]interface{} { base := t.baseline.AllSettingsBySource() compare := t.compare.AllSettingsBySource() - compareResult("", "AllSettingsBySource", base, compare) + t.compareResult("", "AllSettingsBySource", base, compare) return base } @@ -454,21 +461,18 @@ func (t *teeConfig) SetConfigType(in string) { func (t *teeConfig) ConfigFileUsed() string { base := t.baseline.ConfigFileUsed() compare := t.compare.ConfigFileUsed() - compareResult("", "ConfigFileUsed", base, compare) + t.compareResult("", "ConfigFileUsed", base, compare) return base } -//func (t *teeConfig) SetTypeByDefaultValue(in bool) { -// t.baseline.SetTypeByDefaultValue(in) -// t.compare.SetTypeByDefaultValue(in) -//} - // GetEnvVars implements the Config interface func (t *teeConfig) GetEnvVars() []string { base := t.baseline.GetEnvVars() compare := t.compare.GetEnvVars() - compareResult("", "GetEnvVars", base, compare) + slices.Sort(base) + slices.Sort(compare) + t.compareResult("", "GetEnvVars", base, compare) return base } @@ -495,13 +499,13 @@ func (t *teeConfig) Stringify(source model.Source) string { func (t *teeConfig) GetProxies() *model.Proxy { base := t.baseline.GetProxies() compare := t.compare.GetProxies() - compareResult("", "GetProxies", base, compare) + t.compareResult("", "GetProxies", base, compare) return base } func (t *teeConfig) ExtraConfigFilesUsed() []string { base := t.baseline.ExtraConfigFilesUsed() compare := t.compare.ExtraConfigFilesUsed() - compareResult("", "ExtraConfigFilesUsed", base, compare) + t.compareResult("", "ExtraConfigFilesUsed", base, compare) return base } From d9a4c787c54377f67d89a397b4db6ec8f71019a9 Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Mon, 9 Dec 2024 14:55:24 +0100 Subject: [PATCH 28/52] (fleet) downloader flavors (#31840) --- .gitattributes | 47 +++++----- .gitlab/package_build/installer.yml | 3 +- cmd/installer-downloader/main.go | 86 ++++++++++++++++-- .../subcommands/installer/command.go | 26 ++---- .../telemetry/telemetryimpl/telemetry.go | 5 +- pkg/fleet/bootstrapper/bootstrapper.go | 2 +- pkg/fleet/installer/default_packages.go | 2 +- pkg/fleet/installer/default_packages_test.go | 2 +- pkg/fleet/installer/fixtures/README.md | 19 ++++ .../{internal => installer}/fixtures/fs.go | 0 .../oci-layout-simple-v1-linux2-amd128.tar | Bin .../fixtures/oci-layout-simple-v1.tar | Bin .../fixtures/oci-layout-simple-v2.tar | Bin .../fixtures/server.go | 2 +- .../simple-v1-config/datadog.yaml.example | 0 .../fixtures/simple-v1/executable.sh | 0 .../fixtures/simple-v1/file.txt | 0 .../simple-v2-config/datadog.yaml.example | 0 .../fixtures/simple-v2/executable-new.sh | 0 pkg/fleet/installer/installer.go | 2 +- pkg/fleet/installer/installer_test.go | 4 +- .../{internal => installer}/oci/download.go | 2 +- .../oci/download_test.go | 2 +- .../{internal => installer}/oci/mirror.go | 0 .../oci/mirror_test.go | 2 +- pkg/fleet/installer/setup/common/config.go | 2 +- pkg/fleet/installer/setup/setup.go | 20 ++-- pkg/fleet/{internal => installer}/tar/tar.go | 0 pkg/fleet/internal/bootstrap/bootstrap_nix.go | 2 +- .../internal/bootstrap/bootstrap_windows.go | 2 +- pkg/fleet/internal/fixtures/README.md | 19 ---- pkg/fleet/telemetry/telemetry.go | 17 +++- tasks/installer.py | 15 ++- 33 files changed, 184 insertions(+), 99 deletions(-) create mode 100644 pkg/fleet/installer/fixtures/README.md rename pkg/fleet/{internal => installer}/fixtures/fs.go (100%) rename pkg/fleet/{internal => installer}/fixtures/oci-layout-simple-v1-linux2-amd128.tar (100%) rename pkg/fleet/{internal => installer}/fixtures/oci-layout-simple-v1.tar (100%) rename pkg/fleet/{internal => installer}/fixtures/oci-layout-simple-v2.tar (100%) rename pkg/fleet/{internal => installer}/fixtures/server.go (98%) rename pkg/fleet/{internal => installer}/fixtures/simple-v1-config/datadog.yaml.example (100%) rename pkg/fleet/{internal => installer}/fixtures/simple-v1/executable.sh (100%) rename pkg/fleet/{internal => installer}/fixtures/simple-v1/file.txt (100%) rename pkg/fleet/{internal => installer}/fixtures/simple-v2-config/datadog.yaml.example (100%) rename pkg/fleet/{internal => installer}/fixtures/simple-v2/executable-new.sh (100%) rename pkg/fleet/{internal => installer}/oci/download.go (99%) rename pkg/fleet/{internal => installer}/oci/download_test.go (99%) rename pkg/fleet/{internal => installer}/oci/mirror.go (100%) rename pkg/fleet/{internal => installer}/oci/mirror_test.go (97%) rename pkg/fleet/{internal => installer}/tar/tar.go (100%) delete mode 100644 pkg/fleet/internal/fixtures/README.md diff --git a/.gitattributes b/.gitattributes index 0b7a4b14e7cf1..9b3d026858313 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,7 +10,7 @@ go.sum -diff -merge linguist-generated=true pkg/security/probe/constantfetch/btfhub/constants.json -diff -merge linguist-generated=true pkg/security/seclwin/** -diff -merge linguist-generated=true # Fixtures should have LF line endings because they are checked against OCI packages built on Linux -pkg/fleet/internal/fixtures/** text=auto eol=lf +pkg/fleet/installer/fixtures/** text=auto eol=lf # Fix `git diff` when running on the below file formats. # Our windows build image uses MinGit which tries to use the astextplain diff algorithm (https://git-scm.com/docs/gitattributes#_setting_the_internal_diff_algorithm). @@ -27,26 +27,25 @@ pkg/fleet/internal/fixtures/** text=auto eol=lf # textconv = astextplain # ``` - -*.doc diff -*.DOC diff -*.docx diff -*.DOCX diff -*.docm diff -*.DOCM diff -*.dot diff -*.DOT diff -*.dotx diff -*.DOTX diff -*.dotm diff -*.DOTM diff -*.pdf diff -*.PDF diff -*.rtf diff -*.RTF diff -*.ods diff -*.ODS diff -*.odf diff -*.ODF diff -*.odt diff -*.ODT diff +*.doc diff +*.DOC diff +*.docx diff +*.DOCX diff +*.docm diff +*.DOCM diff +*.dot diff +*.DOT diff +*.dotx diff +*.DOTX diff +*.dotm diff +*.DOTM diff +*.pdf diff +*.PDF diff +*.rtf diff +*.RTF diff +*.ods diff +*.ODS diff +*.odf diff +*.ODF diff +*.odt diff +*.ODT diff diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index 6ab10c1a19e62..154392b246469 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -103,7 +103,8 @@ installer-install-scripts: - !reference [.retrieve_linux_go_deps] - echo "About to build for $RELEASE_VERSION" - mkdir -p $OMNIBUS_PACKAGE_DIR - - inv -e installer.build-linux-script && mv ./bin/installer/install.sh $OMNIBUS_PACKAGE_DIR/install-djm.sh + - inv -e installer.build-linux-script "databricks" "$RELEASE_VERSION" + - mv ./bin/installer/install-*.sh $OMNIBUS_PACKAGE_DIR/ - ls -la $OMNIBUS_PACKAGE_DIR artifacts: expire_in: 2 weeks diff --git a/cmd/installer-downloader/main.go b/cmd/installer-downloader/main.go index f964080a4315b..db35a6c337519 100644 --- a/cmd/installer-downloader/main.go +++ b/cmd/installer-downloader/main.go @@ -7,20 +7,94 @@ package main import ( + "context" "fmt" "os" + "os/exec" + "path/filepath" - "github.com/DataDog/datadog-agent/cmd/installer/subcommands/installer" "github.com/DataDog/datadog-agent/cmd/installer/user" - "github.com/DataDog/datadog-agent/cmd/internal/runcmd" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" + "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +const ( + installerPackage = "datadog-installer" + installerBinPath = "bin/installer/installer" +) + +var ( + // Version is the version of the installer to download. + Version string + // Flavor is the flavor of the setup to run. + Flavor string ) func main() { + if Version == "" || Flavor == "" { + fmt.Fprintln(os.Stderr, "Version and Flavor must be set at build time.") + os.Exit(1) + } if !user.IsRoot() { - fmt.Fprintln(os.Stderr, "This command requires root privileges.") + fmt.Fprintln(os.Stderr, "This installer requires root privileges.") os.Exit(1) } - cmd := installer.BootstrapCommand() - cmd.SilenceUsage = true - os.Exit(runcmd.Run(cmd)) + env := env.FromEnv() + ctx := context.Background() + + t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor)) + _ = t.Start(ctx) + defer func() { _ = t.Stop(ctx) }() + var err error + span, ctx := telemetry.StartSpanFromEnv(ctx, "downloader") + defer func() { span.Finish(tracer.WithError(err)) }() + err = runDownloader(ctx, env, Version, Flavor) + if err != nil { + fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err) + os.Exit(1) + } +} + +func runDownloader(ctx context.Context, env *env.Env, version string, flavor string) error { + downloaderPath, err := os.Executable() + if err != nil { + return fmt.Errorf("failed to get executable path: %w", err) + } + tmpDir, err := os.MkdirTemp(filepath.Dir(downloaderPath), "datadog-installer") + if err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + defer os.RemoveAll(tmpDir) + err = downloadInstaller(ctx, env, version, tmpDir) + if err != nil { + return fmt.Errorf("failed to download installer: %w", err) + } + cmd := exec.CommandContext(ctx, filepath.Join(tmpDir, installerBinPath), "setup", "--flavor", flavor) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + err = cmd.Run() + if err != nil { + return fmt.Errorf("failed to run installer: %w", err) + } + return nil +} + +func downloadInstaller(ctx context.Context, env *env.Env, version string, tmpDir string) error { + url := oci.PackageURL(env, installerPackage, version) + downloader := oci.NewDownloader(env, env.HTTPClient()) + downloadedPackage, err := downloader.Download(ctx, url) + if err != nil { + return fmt.Errorf("failed to download installer package: %w", err) + } + if downloadedPackage.Name != installerPackage { + return fmt.Errorf("unexpected package name: %s, expected %s", downloadedPackage.Name, installerPackage) + } + err = downloadedPackage.ExtractLayers(oci.DatadogPackageLayerMediaType, tmpDir) + if err != nil { + return fmt.Errorf("failed to extract layers: %w", err) + } + return nil } diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 7b898b0579cdb..186367fc20b42 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -96,7 +96,7 @@ type cmd struct { func newCmd(operation string) *cmd { env := env.FromEnv() t := newTelemetry(env) - span, ctx := newSpan(operation) + span, ctx := telemetry.StartSpanFromEnv(context.Background(), operation) setInstallerUmask(span) return &cmd{ t: t, @@ -226,12 +226,8 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { if site == "" { site = config.Site } - t, err := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands - if err != nil { - fmt.Printf("failed to initialize telemetry: %v\n", err) - return nil - } - err = t.Start(context.Background()) + t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands + err := t.Start(context.Background()) if err != nil { fmt.Printf("failed to start telemetry: %v\n", err) return nil @@ -239,15 +235,6 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { return t } -func newSpan(operationName string) (ddtrace.Span, context.Context) { - var spanOptions []ddtrace.StartSpanOption - spanContext, ok := telemetry.SpanContextFromEnv() - if ok { - spanOptions = append(spanOptions, tracer.ChildOf(spanContext)) - } - return tracer.StartSpanFromContext(context.Background(), operationName, spanOptions...) -} - func versionCommand() *cobra.Command { return &cobra.Command{ Use: "version", @@ -287,6 +274,7 @@ func bootstrapCommand() *cobra.Command { } func setupCommand() *cobra.Command { + flavor := "" cmd := &cobra.Command{ Use: "setup", Hidden: true, @@ -294,9 +282,13 @@ func setupCommand() *cobra.Command { RunE: func(_ *cobra.Command, _ []string) (err error) { cmd := newCmd("setup") defer func() { cmd.Stop(err) }() - return setup.Setup(cmd.ctx, cmd.env) + if flavor == "" { + return setup.Agent7InstallScript(cmd.ctx, cmd.env) + } + return setup.Setup(cmd.ctx, cmd.env, flavor) }, } + cmd.Flags().StringVar(&flavor, "flavor", "", "The setup flavor") return cmd } diff --git a/comp/updater/telemetry/telemetryimpl/telemetry.go b/comp/updater/telemetry/telemetryimpl/telemetry.go index cf19e681c5708..d8f961e8398e6 100644 --- a/comp/updater/telemetry/telemetryimpl/telemetry.go +++ b/comp/updater/telemetry/telemetryimpl/telemetry.go @@ -38,16 +38,13 @@ func newTelemetry(deps dependencies) (telemetry.Component, error) { client := &http.Client{ Transport: httputils.CreateHTTPTransport(deps.Config), } - telemetry, err := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", + telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", fleettelemetry.WithSamplingRules( tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1), tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05), tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05), ), ) - if err != nil { - return nil, err - } deps.Lc.Append(fx.Hook{OnStart: telemetry.Start, OnStop: telemetry.Stop}) return telemetry, nil } diff --git a/pkg/fleet/bootstrapper/bootstrapper.go b/pkg/fleet/bootstrapper/bootstrapper.go index e935a6cadc649..fed97034722d1 100644 --- a/pkg/fleet/bootstrapper/bootstrapper.go +++ b/pkg/fleet/bootstrapper/bootstrapper.go @@ -11,9 +11,9 @@ import ( "fmt" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" "github.com/DataDog/datadog-agent/pkg/fleet/internal/bootstrap" "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" ) diff --git a/pkg/fleet/installer/default_packages.go b/pkg/fleet/installer/default_packages.go index 66c7017f42bac..04b4828e6eeb9 100644 --- a/pkg/fleet/installer/default_packages.go +++ b/pkg/fleet/installer/default_packages.go @@ -11,7 +11,7 @@ import ( "strings" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" ) // Package represents a package known to the installer diff --git a/pkg/fleet/installer/default_packages_test.go b/pkg/fleet/installer/default_packages_test.go index e17645ec7cefb..f454dbd1ab165 100644 --- a/pkg/fleet/installer/default_packages_test.go +++ b/pkg/fleet/installer/default_packages_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" "github.com/stretchr/testify/assert" ) diff --git a/pkg/fleet/installer/fixtures/README.md b/pkg/fleet/installer/fixtures/README.md new file mode 100644 index 0000000000000..792fb38a15da8 --- /dev/null +++ b/pkg/fleet/installer/fixtures/README.md @@ -0,0 +1,19 @@ +# Datadog Package fixtures + +This directory contains a few examples of Datadog Packages for use in the +updater tests. + +*simple-v1* +```bash +datadog-package create --archive --version "v1" --archive-path "pkg/fleet/installer/fixtures/oci-layout-simple-v1.tar" --package "simple" --configs pkg/fleet/installer/fixtures/simple-v1-config pkg/fleet/installer/fixtures/simple-v1 +``` + +*simple-v2* +```bash +datadog-package create --archive --version "v2" --archive-path "pkg/fleet/installer/fixtures/oci-layout-simple-v2.tar" --package "simple" --configs pkg/fleet/installer/fixtures/simple-v2-config pkg/fleet/installer/fixtures/simple-v2 +``` + +*simple-v1-linux2-amd128* +```bash +datadog-package create --archive --version "v1" --os "linux2" --arch "amd128" --archive-path "pkg/fleet/installer/fixtures/oci-layout-simple-v1-linux2-amd128.tar" --package "simple" pkg/fleet/installer/fixtures/simple-v1 +``` diff --git a/pkg/fleet/internal/fixtures/fs.go b/pkg/fleet/installer/fixtures/fs.go similarity index 100% rename from pkg/fleet/internal/fixtures/fs.go rename to pkg/fleet/installer/fixtures/fs.go diff --git a/pkg/fleet/internal/fixtures/oci-layout-simple-v1-linux2-amd128.tar b/pkg/fleet/installer/fixtures/oci-layout-simple-v1-linux2-amd128.tar similarity index 100% rename from pkg/fleet/internal/fixtures/oci-layout-simple-v1-linux2-amd128.tar rename to pkg/fleet/installer/fixtures/oci-layout-simple-v1-linux2-amd128.tar diff --git a/pkg/fleet/internal/fixtures/oci-layout-simple-v1.tar b/pkg/fleet/installer/fixtures/oci-layout-simple-v1.tar similarity index 100% rename from pkg/fleet/internal/fixtures/oci-layout-simple-v1.tar rename to pkg/fleet/installer/fixtures/oci-layout-simple-v1.tar diff --git a/pkg/fleet/internal/fixtures/oci-layout-simple-v2.tar b/pkg/fleet/installer/fixtures/oci-layout-simple-v2.tar similarity index 100% rename from pkg/fleet/internal/fixtures/oci-layout-simple-v2.tar rename to pkg/fleet/installer/fixtures/oci-layout-simple-v2.tar diff --git a/pkg/fleet/internal/fixtures/server.go b/pkg/fleet/installer/fixtures/server.go similarity index 98% rename from pkg/fleet/internal/fixtures/server.go rename to pkg/fleet/installer/fixtures/server.go index cdb1fb6b7323c..bdeba97c5997e 100644 --- a/pkg/fleet/internal/fixtures/server.go +++ b/pkg/fleet/installer/fixtures/server.go @@ -18,7 +18,7 @@ import ( "strings" "testing" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/tar" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/tar" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" "github.com/google/go-containerregistry/pkg/v1/layout" diff --git a/pkg/fleet/internal/fixtures/simple-v1-config/datadog.yaml.example b/pkg/fleet/installer/fixtures/simple-v1-config/datadog.yaml.example similarity index 100% rename from pkg/fleet/internal/fixtures/simple-v1-config/datadog.yaml.example rename to pkg/fleet/installer/fixtures/simple-v1-config/datadog.yaml.example diff --git a/pkg/fleet/internal/fixtures/simple-v1/executable.sh b/pkg/fleet/installer/fixtures/simple-v1/executable.sh similarity index 100% rename from pkg/fleet/internal/fixtures/simple-v1/executable.sh rename to pkg/fleet/installer/fixtures/simple-v1/executable.sh diff --git a/pkg/fleet/internal/fixtures/simple-v1/file.txt b/pkg/fleet/installer/fixtures/simple-v1/file.txt similarity index 100% rename from pkg/fleet/internal/fixtures/simple-v1/file.txt rename to pkg/fleet/installer/fixtures/simple-v1/file.txt diff --git a/pkg/fleet/internal/fixtures/simple-v2-config/datadog.yaml.example b/pkg/fleet/installer/fixtures/simple-v2-config/datadog.yaml.example similarity index 100% rename from pkg/fleet/internal/fixtures/simple-v2-config/datadog.yaml.example rename to pkg/fleet/installer/fixtures/simple-v2-config/datadog.yaml.example diff --git a/pkg/fleet/internal/fixtures/simple-v2/executable-new.sh b/pkg/fleet/installer/fixtures/simple-v2/executable-new.sh similarity index 100% rename from pkg/fleet/internal/fixtures/simple-v2/executable-new.sh rename to pkg/fleet/installer/fixtures/simple-v2/executable-new.sh diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index d7cca9819b4da..e43e0cfaae4d2 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -22,10 +22,10 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" "github.com/DataDog/datadog-agent/pkg/fleet/installer/packages" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/db" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" diff --git a/pkg/fleet/installer/installer_test.go b/pkg/fleet/installer/installer_test.go index 9ab772b32dc0f..a98deece97ee0 100644 --- a/pkg/fleet/installer/installer_test.go +++ b/pkg/fleet/installer/installer_test.go @@ -17,10 +17,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/fixtures" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/db" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" ) var testCtx = context.TODO() diff --git a/pkg/fleet/internal/oci/download.go b/pkg/fleet/installer/oci/download.go similarity index 99% rename from pkg/fleet/internal/oci/download.go rename to pkg/fleet/installer/oci/download.go index a03f7917d27df..1ddff40e37238 100644 --- a/pkg/fleet/internal/oci/download.go +++ b/pkg/fleet/installer/oci/download.go @@ -35,7 +35,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/tar" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/tar" "github.com/DataDog/datadog-agent/pkg/util/log" ) diff --git a/pkg/fleet/internal/oci/download_test.go b/pkg/fleet/installer/oci/download_test.go similarity index 99% rename from pkg/fleet/internal/oci/download_test.go rename to pkg/fleet/installer/oci/download_test.go index e3c9b61916145..44d6462aaf4f9 100644 --- a/pkg/fleet/internal/oci/download_test.go +++ b/pkg/fleet/installer/oci/download_test.go @@ -19,7 +19,7 @@ import ( "golang.org/x/net/http2" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/fixtures" "github.com/google/go-containerregistry/pkg/authn" oci "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/google" diff --git a/pkg/fleet/internal/oci/mirror.go b/pkg/fleet/installer/oci/mirror.go similarity index 100% rename from pkg/fleet/internal/oci/mirror.go rename to pkg/fleet/installer/oci/mirror.go diff --git a/pkg/fleet/internal/oci/mirror_test.go b/pkg/fleet/installer/oci/mirror_test.go similarity index 97% rename from pkg/fleet/internal/oci/mirror_test.go rename to pkg/fleet/installer/oci/mirror_test.go index cee744c5fcbff..1a59f9d85a6db 100644 --- a/pkg/fleet/internal/oci/mirror_test.go +++ b/pkg/fleet/installer/oci/mirror_test.go @@ -12,7 +12,7 @@ import ( "strings" "testing" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/fixtures" "github.com/stretchr/testify/require" ) diff --git a/pkg/fleet/installer/setup/common/config.go b/pkg/fleet/installer/setup/common/config.go index d025311d5e15e..5ec929682808b 100644 --- a/pkg/fleet/installer/setup/common/config.go +++ b/pkg/fleet/installer/setup/common/config.go @@ -15,9 +15,9 @@ import ( "path/filepath" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" "github.com/DataDog/datadog-agent/pkg/fleet/installer/packages" "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" "gopkg.in/yaml.v2" ) diff --git a/pkg/fleet/installer/setup/setup.go b/pkg/fleet/installer/setup/setup.go index 313b12e2782d5..376abc751eaf5 100644 --- a/pkg/fleet/installer/setup/setup.go +++ b/pkg/fleet/installer/setup/setup.go @@ -9,7 +9,6 @@ package setup import ( "context" "fmt" - "os" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/setup/djm" @@ -17,16 +16,23 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" ) -// Setup is the main function to resolve packages to install and install them -func Setup(ctx context.Context, env *env.Env) error { - if os.Getenv("DD_DJM_FLAVOR") == "databricks" { +const ( + // FlavorDatabricks is the flavor for the Data Jobs Monitoring databricks setup. + FlavorDatabricks = "databricks" +) + +// Setup installs Datadog. +func Setup(ctx context.Context, env *env.Env, flavor string) error { + switch flavor { + case FlavorDatabricks: return djm.SetupDatabricks(ctx, env) + default: + return fmt.Errorf("unknown setup flavor %s", flavor) } - - return defaultSetup(ctx, env) } -func defaultSetup(ctx context.Context, env *env.Env) error { +// Agent7InstallScript is the setup used by the agent7 install script. +func Agent7InstallScript(ctx context.Context, env *env.Env) error { cmd := exec.NewInstallerExec(env, paths.StableInstallerPath) defaultPackages, err := cmd.DefaultPackages(ctx) if err != nil { diff --git a/pkg/fleet/internal/tar/tar.go b/pkg/fleet/installer/tar/tar.go similarity index 100% rename from pkg/fleet/internal/tar/tar.go rename to pkg/fleet/installer/tar/tar.go diff --git a/pkg/fleet/internal/bootstrap/bootstrap_nix.go b/pkg/fleet/internal/bootstrap/bootstrap_nix.go index fbc9817f0783c..cf085c2379e33 100644 --- a/pkg/fleet/internal/bootstrap/bootstrap_nix.go +++ b/pkg/fleet/internal/bootstrap/bootstrap_nix.go @@ -17,8 +17,8 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" ) func install(ctx context.Context, env *env.Env, url string, experiment bool) error { diff --git a/pkg/fleet/internal/bootstrap/bootstrap_windows.go b/pkg/fleet/internal/bootstrap/bootstrap_windows.go index c3d867f959db2..6990ec77cf042 100644 --- a/pkg/fleet/internal/bootstrap/bootstrap_windows.go +++ b/pkg/fleet/internal/bootstrap/bootstrap_windows.go @@ -18,8 +18,8 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" iexec "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" ) func install(ctx context.Context, env *env.Env, url string, experiment bool) error { diff --git a/pkg/fleet/internal/fixtures/README.md b/pkg/fleet/internal/fixtures/README.md deleted file mode 100644 index 4aad4996bed08..0000000000000 --- a/pkg/fleet/internal/fixtures/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Datadog Package fixtures - -This directory contains a few examples of Datadog Packages for use in the -updater tests. - -*simple-v1* -```bash -datadog-package create --archive --version "v1" --archive-path "pkg/fleet/internal/fixtures/oci-layout-simple-v1.tar" --package "simple" --configs pkg/fleet/internal/fixtures/simple-v1-config pkg/fleet/internal/fixtures/simple-v1 -``` - -*simple-v2* -```bash -datadog-package create --archive --version "v2" --archive-path "pkg/fleet/internal/fixtures/oci-layout-simple-v2.tar" --package "simple" --configs pkg/fleet/internal/fixtures/simple-v2-config pkg/fleet/internal/fixtures/simple-v2 -``` - -*simple-v1-linux2-amd128* -```bash -datadog-package create --archive --version "v1" --os "linux2" --arch "amd128" --archive-path "pkg/fleet/internal/fixtures/oci-layout-simple-v1-linux2-amd128.tar" --package "simple" pkg/fleet/internal/fixtures/simple-v1 -``` diff --git a/pkg/fleet/telemetry/telemetry.go b/pkg/fleet/telemetry/telemetry.go index 842ea1b678cbd..b63bdfb0de764 100644 --- a/pkg/fleet/telemetry/telemetry.go +++ b/pkg/fleet/telemetry/telemetry.go @@ -59,7 +59,7 @@ type Telemetry struct { type Option func(*Telemetry) // NewTelemetry creates a new telemetry instance -func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) (*Telemetry, error) { +func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) *Telemetry { endpoint := &traceconfig.Endpoint{ Host: fmt.Sprintf("https://%s.%s", telemetrySubdomain, strings.TrimSpace(site)), APIKey: apiKey, @@ -81,7 +81,7 @@ func NewTelemetry(client *http.Client, apiKey string, site string, service strin opt(t) } t.server.Handler = t.handler() - return t, nil + return t } // Start starts the telemetry @@ -205,8 +205,17 @@ func (addr) String() string { return "local" } -// SpanContextFromEnv injects the traceID and parentID from the environment into the context if available. -func SpanContextFromEnv() (ddtrace.SpanContext, bool) { +// StartSpanFromEnv starts a span using the environment variables to find the parent span. +func StartSpanFromEnv(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (ddtrace.Span, context.Context) { + spanContext, ok := spanContextFromEnv() + if ok { + spanOptions = append([]ddtrace.StartSpanOption{tracer.ChildOf(spanContext)}, spanOptions...) + } + return tracer.StartSpanFromContext(ctx, operationName, spanOptions...) +} + +// spanContextFromEnv injects the traceID and parentID from the environment into the context if available. +func spanContextFromEnv() (ddtrace.SpanContext, bool) { traceID := os.Getenv(EnvTraceID) parentID := os.Getenv(EnvParentID) ctxCarrier := tracer.TextMapCarrier{ diff --git a/tasks/installer.py b/tasks/installer.py index e3554723a5f41..7429a1eaf3148 100644 --- a/tasks/installer.py +++ b/tasks/installer.py @@ -16,8 +16,9 @@ DIR_BIN = os.path.join(".", "bin", "installer") INSTALLER_BIN = os.path.join(DIR_BIN, bin_name("installer")) DOWNLOADER_BIN = os.path.join(DIR_BIN, bin_name("downloader")) -INSTALL_SCRIPT = os.path.join(DIR_BIN, "install.sh") INSTALL_SCRIPT_TEMPLATE = os.path.join("pkg", "fleet", "installer", "setup", "install.sh") +DOWNLOADER_MAIN_PACKAGE = "cmd/installer-downloader" + MAJOR_VERSION = '7' @@ -76,14 +77,18 @@ def build( @task def build_downloader( ctx, + flavor, + version, os="linux", arch="amd64", ): ''' Builds the installer downloader binary. ''' + version_flag = f'-X main.Version={version}' + flavor_flag = f'-X main.Flavor={flavor}' ctx.run( - f'go build -ldflags="-s -w" -o {DOWNLOADER_BIN} {REPO_PATH}/cmd/installer-downloader', + f'go build -ldflags="-s -w {version_flag} {flavor_flag}" -o {DOWNLOADER_BIN} {REPO_PATH}/{DOWNLOADER_MAIN_PACKAGE}', env={'GOOS': os, 'GOARCH': arch, 'CGO_ENABLED': '0'}, ) @@ -91,6 +96,8 @@ def build_downloader( @task def build_linux_script( ctx, + flavor, + version, ): ''' Builds the linux script that is used to install the agent on linux. @@ -101,7 +108,7 @@ def build_linux_script( archs = ['amd64', 'arm64'] for arch in archs: - build_downloader(ctx, os='linux', arch=arch) + build_downloader(ctx, flavor=flavor, version=version, os='linux', arch=arch) with open(DOWNLOADER_BIN, 'rb') as f: encoded_bin = base64.encodebytes(f.read()).decode('utf-8') install_script = install_script.replace(f'DOWNLOADER_BIN_{arch.upper()}', encoded_bin) @@ -109,7 +116,7 @@ def build_linux_script( commit_sha = ctx.run('git rev-parse HEAD', hide=True).stdout.strip() install_script = install_script.replace('INSTALLER_COMMIT', commit_sha) - with open(INSTALL_SCRIPT, 'w') as f: + with open(os.path.join(DIR_BIN, f'install-{flavor}.sh'), 'w') as f: f.write(install_script) From 4f9e172fbeab787906d974ba7855222953a0ea0d Mon Sep 17 00:00:00 2001 From: Usama Saqib Date: Mon, 9 Dec 2024 15:07:57 +0100 Subject: [PATCH 29/52] Add rocky 9.4 to KMT platforms (#31880) --- test/new-e2e/system-probe/config/platforms.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/new-e2e/system-probe/config/platforms.json b/test/new-e2e/system-probe/config/platforms.json index 9249144d9ded9..9229b46f9476e 100644 --- a/test/new-e2e/system-probe/config/platforms.json +++ b/test/new-e2e/system-probe/config/platforms.json @@ -257,6 +257,14 @@ "alt_version_names": [ "oracular" ] + }, + "rocky_9.4": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "5.14.0-503.15.1.el9_5", + "os_version": "9.4", + "image": "rocky-9.4-x86_64.qcow2.xz", + "image_version": "20241209_f6a14db0" } }, "arm64": { @@ -486,6 +494,14 @@ "alt_version_names": [ "oracular" ] + }, + "rocky_9.4": { + "os_name": "Rocky Linux", + "os_id": "rocky", + "kernel": "5.14.0-503.15.1.el9_5", + "os_version": "9.4", + "image": "rocky-9.4-arm64.qcow2.xz", + "image_version": "20241209_f6a14db0" } } } \ No newline at end of file From 495529f72483db90db16d1170240b3c8fb64a941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Mathieu?= Date: Mon, 9 Dec 2024 15:49:25 +0100 Subject: [PATCH 30/52] logs/sds: fallback on default excluded keywords if no included keywords available. (#29981) Co-authored-by: jszwedko --- pkg/logs/sds/scanner.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/logs/sds/scanner.go b/pkg/logs/sds/scanner.go index b0caf689efbfd..9b86f478321c0 100644 --- a/pkg/logs/sds/scanner.go +++ b/pkg/logs/sds/scanner.go @@ -313,13 +313,19 @@ func interpretRCRule(userRule RuleConfig, standardRule StandardRuleConfig, defau } // If the "Use recommended keywords" checkbox has been checked, we use the default - // included keywords available in the rule (curated by Datadog). + // included keywords available in the rule (curated by Datadog), if not included keywords + // exist, fallback on using the default excluded keywords. // Otherwise: // If some included keywords have been manually filled by the user, we use them // Else we start using the default excluded keywords. if userRule.IncludedKeywords.UseRecommendedKeywords { - // default included keywords - extraConfig.ProximityKeywords = sds.CreateProximityKeywordsConfig(defaults.IncludedKeywordsCharCount, defToUse.DefaultIncludedKeywords, nil) + // default included keywords if any + if len(defToUse.DefaultIncludedKeywords) > 0 { + extraConfig.ProximityKeywords = sds.CreateProximityKeywordsConfig(defaults.IncludedKeywordsCharCount, defToUse.DefaultIncludedKeywords, nil) + } else if len(defaults.ExcludedKeywords) > 0 && defaults.ExcludedKeywordsCharCount > 0 { + // otherwise fallback on default excluded keywords + extraConfig.ProximityKeywords = sds.CreateProximityKeywordsConfig(defaults.ExcludedKeywordsCharCount, nil, defaults.ExcludedKeywords) + } } else { if len(userRule.IncludedKeywords.Keywords) > 0 && userRule.IncludedKeywords.CharacterCount > 0 { // user provided included keywords From 40a840dfadd6a4744da510792af666ec36769c6d Mon Sep 17 00:00:00 2001 From: Branden Clark Date: Mon, 9 Dec 2024 10:10:15 -0500 Subject: [PATCH 31/52] Add initial Windows E2E FIPS package test/support (#31771) --- .gitlab/e2e_install_packages/windows.yml | 46 ++++--- .gitlab/e2e_testing_deploy/e2e_deploy.yml | 21 +++ .../windows/base_agent_installer_suite.go | 5 + .../tests/windows/common/agent/package.go | 127 ++++++++++++++---- test/new-e2e/tests/windows/common/policy.go | 46 +++++++ test/new-e2e/tests/windows/common/registry.go | 17 +++ .../tests/windows/fips-test/fips_test.go | 123 +++++++++++++++++ .../tests/windows/install-test/npm_test.go | 2 +- .../windows/install-test/upgrade_test.go | 6 +- 9 files changed, 346 insertions(+), 47 deletions(-) create mode 100644 test/new-e2e/tests/windows/common/policy.go create mode 100644 test/new-e2e/tests/windows/fips-test/fips_test.go diff --git a/.gitlab/e2e_install_packages/windows.yml b/.gitlab/e2e_install_packages/windows.yml index f4575d0159f8c..86958e1a9c9ad 100644 --- a/.gitlab/e2e_install_packages/windows.yml +++ b/.gitlab/e2e_install_packages/windows.yml @@ -1,10 +1,9 @@ # NOTE: If a new job is added, be sure to upate .gitlab/e2e_test_junit_upload.yml -.new-e2e_windows_msi: +.new-e2e_with_version: variables: - TARGETS: ./tests/windows/install-test + WINDOWS_AGENT_ARCH: "x86_64" TEAM: windows-agent - EXTRA_PARAMS: --run "$E2E_MSI_TEST$" extends: - .new_e2e_template before_script: @@ -17,19 +16,18 @@ - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::7" --no-worktree) - !reference [.new_e2e_template, script] +.new-e2e_windows_msi: + variables: + TARGETS: ./tests/windows/install-test + EXTRA_PARAMS: --run "$E2E_MSI_TEST$" + extends: + - .new-e2e_with_version + .new-e2e_windows_domain_test: variables: TARGETS: ./tests/windows/domain-test - TEAM: windows-agent - before_script: - # WINDOWS_AGENT_VERSION is used to verify the installed agent version - # Must run before new_e2e_template changes the aws profile - - WINDOWS_AGENT_VERSION=$(invoke agent.version) || exit $?; export WINDOWS_AGENT_VERSION - - !reference [.new_e2e_template, before_script] - script: - # LAST_STABLE_VERSION is used for upgrade test - - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::7" --no-worktree) - - !reference [.new_e2e_template, script] + extends: + - .new-e2e_with_version .new-e2e_windows_installer_v7_tests: parallel: @@ -72,8 +70,6 @@ new-e2e_windows_powershell_module_test: # Agent 7 .new-e2e_windows_a7_x86_64: - variables: - WINDOWS_AGENT_ARCH: "x86_64" extends: - .new-e2e_windows_msi - .new-e2e_agent_a7 @@ -95,10 +91,7 @@ new-e2e-windows-agent-msi-windows-server-a7-x86_64: new-e2e-windows-agent-domain-tests-a7-x86_64: stage: e2e_install_packages - variables: - WINDOWS_AGENT_ARCH: "x86_64" extends: - - .new_e2e_template - .new-e2e_windows_domain_test - .new-e2e_agent_a7 needs: @@ -110,6 +103,23 @@ new-e2e-windows-agent-domain-tests-a7-x86_64: - !reference [.manual] timeout: 1h15m +new-e2e-windows-agent-a7-x86_64-fips: + stage: e2e_install_packages + variables: + WINDOWS_AGENT_FLAVOR: "fips" + TARGETS: ./tests/windows/fips-test + extends: + - .new-e2e_with_version + - .new-e2e_agent_a7 + needs: + - !reference [.needs_new_e2e_template] + - deploy_windows_testing-a7-fips + rules: + - !reference [.on_deploy] + - !reference [.on_e2e_or_windows_installer_changes] + - !reference [.manual] + timeout: 1h15m + ## single test for PRs ## skipped if the full tests are running new-e2e-windows-agent-msi-upgrade-windows-server-a7-x86_64: diff --git a/.gitlab/e2e_testing_deploy/e2e_deploy.yml b/.gitlab/e2e_testing_deploy/e2e_deploy.yml index 45be38a825a5d..d47fcb6748da3 100644 --- a/.gitlab/e2e_testing_deploy/e2e_deploy.yml +++ b/.gitlab/e2e_testing_deploy/e2e_deploy.yml @@ -192,3 +192,24 @@ deploy_windows_testing-a7: $OMNIBUS_PACKAGE_DIR s3://$WIN_S3_BUCKET/$WINDOWS_TESTING_S3_BUCKET_A7 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 + +deploy_windows_testing-a7-fips: + rules: + - !reference [.except_no_tests_no_deploy] + - !reference [.except_mergequeue] + - when: on_success + stage: e2e_deploy + image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + needs: + ["lint_windows-x64", "windows_msi_and_bosh_zip_x64-a7-fips"] + before_script: + - ls $OMNIBUS_PACKAGE_DIR + script: + - $S3_CP_CMD + --recursive + --exclude "*" + --include "datadog-fips-agent-7.*.msi" + $OMNIBUS_PACKAGE_DIR s3://$WIN_S3_BUCKET/$WINDOWS_TESTING_S3_BUCKET_A7 + --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 diff --git a/test/new-e2e/tests/windows/base_agent_installer_suite.go b/test/new-e2e/tests/windows/base_agent_installer_suite.go index 5f2f6809d33b0..b73ba6d22ace5 100644 --- a/test/new-e2e/tests/windows/base_agent_installer_suite.go +++ b/test/new-e2e/tests/windows/base_agent_installer_suite.go @@ -56,6 +56,11 @@ func (b *BaseAgentInstallerSuite[Env]) SetupSuite() { b.BaseSuite.SetupSuite() var err error + b.OutputDir, err = b.CreateTestOutputDir() + if err != nil { + b.T().Fatalf("should get output dir") + } + b.AgentPackage, err = windowsAgent.GetPackageFromEnv() if err != nil { b.T().Fatalf("failed to get MSI URL from env: %v", err) diff --git a/test/new-e2e/tests/windows/common/agent/package.go b/test/new-e2e/tests/windows/common/agent/package.go index 8e3f867f41a80..5fd6497c2f0b1 100644 --- a/test/new-e2e/tests/windows/common/agent/package.go +++ b/test/new-e2e/tests/windows/common/agent/package.go @@ -18,14 +18,19 @@ import ( ) const ( - defaultMajorVersion = "7" - defaultArch = "x86_64" - agentInstallerListProductName = "datadog-agent" - agentS3BucketRelease = "ddagent-windows-stable" - betaChannel = "beta" - betaURL = "https://s3.amazonaws.com/dd-agent-mstesting/builds/beta/installers_v2.json" - stableChannel = "stable" - stableURL = "https://ddagent-windows-stable.s3.amazonaws.com/installers_v2.json" + defaultMajorVersion = "7" + defaultArch = "x86_64" + defaultFlavor = "base" + agentS3BucketRelease = "ddagent-windows-stable" + betaChannel = "beta" + betaURL = "https://s3.amazonaws.com/dd-agent-mstesting/builds/beta/installers_v2.json" + stableChannel = "stable" + stableURL = "https://ddagent-windows-stable.s3.amazonaws.com/installers_v2.json" +) + +// Environment variable constants +const ( + PackageFlavorEnvVar = "WINDOWS_AGENT_FLAVOR" ) // Package contains identifying information about an Agent MSI package. @@ -40,6 +45,8 @@ type Package struct { Arch string // URL is the URL the MSI can be downloaded from URL string + // Flavor is the Agent Flavor (e.g. `base`, `fips`, `iot``) + Flavor string } // AgentVersion returns a string containing version number and the pre only, e.g. `0.0.0-beta.1` @@ -50,33 +57,61 @@ func (p *Package) AgentVersion() string { } // GetBetaMSIURL returns the URL for the beta agent MSI +// // majorVersion: 6, 7 // arch: x86_64 -func GetBetaMSIURL(version string, arch string) (string, error) { - return GetMSIURL(betaChannel, version, arch) +// flavor: base, fips +func GetBetaMSIURL(version string, arch string, flavor string) (string, error) { + return GetMSIURL(betaChannel, version, arch, flavor) } // GetStableMSIURL returns the URL for the stable agent MSI +// // majorVersion: 6, 7 // arch: x86_64 -func GetStableMSIURL(version string, arch string) (string, error) { - return GetMSIURL(stableChannel, version, arch) +// flavor: base, fips +func GetStableMSIURL(version string, arch string, flavor string) (string, error) { + return GetMSIURL(stableChannel, version, arch, flavor) } // GetMSIURL returns the URL for the agent MSI +// // channel: beta, stable // majorVersion: 6, 7 // arch: x86_64 -func GetMSIURL(channel string, version string, arch string) (string, error) { +// flavor: base, fips +func GetMSIURL(channel string, version string, arch string, flavor string) (string, error) { channelURL, err := GetChannelURL(channel) if err != nil { return "", err } - return installers.GetProductURL(channelURL, agentInstallerListProductName, version, arch) + productName, err := GetFlavorProductName(flavor) + if err != nil { + return "", err + } + + return installers.GetProductURL(channelURL, productName, version, arch) +} + +// GetFlavorProductName returns the product name for the flavor +// +// flavor: base, fips +func GetFlavorProductName(flavor string) (string, error) { + switch flavor { + case "": + return "datadog-agent", nil + case "base": + return "datadog-agent", nil + case "fips": + return "datadog-fips-agent", nil + default: + return "", fmt.Errorf("unknown flavor %v", flavor) + } } // GetChannelURL returns the URL for the channel name +// // channel: beta, stable func GetChannelURL(channel string) (string, error) { if strings.EqualFold(channel, betaChannel) { @@ -89,21 +124,32 @@ func GetChannelURL(channel string) (string, error) { } // GetLatestMSIURL returns the URL for the latest agent MSI +// // majorVersion: 6, 7 // arch: x86_64 -func GetLatestMSIURL(majorVersion string, arch string) string { +func GetLatestMSIURL(majorVersion string, arch string, flavor string) (string, error) { // why do we use amd64 for the latest URL and x86_64 everywhere else? if arch == "x86_64" { arch = "amd64" } - return fmt.Sprintf(`https://s3.amazonaws.com/`+agentS3BucketRelease+`/datadog-agent-%s-latest.%s.msi`, - majorVersion, arch) + productName, err := GetFlavorProductName(flavor) + if err != nil { + return "", err + } + return fmt.Sprintf(`https://s3.amazonaws.com/`+agentS3BucketRelease+`/%s-%s-latest.%s.msi`, + productName, majorVersion, arch), nil } // GetPipelineMSIURL returns the URL for the agent MSI built by the pipeline +// // majorVersion: 6, 7 // arch: x86_64 -func GetPipelineMSIURL(pipelineID string, majorVersion string, arch string) (string, error) { +// flavor: base, fips +func GetPipelineMSIURL(pipelineID string, majorVersion string, arch string, flavor string) (string, error) { + productName, err := GetFlavorProductName(flavor) + if err != nil { + return "", err + } // Manual URL example: https://s3.amazonaws.com/dd-agent-mstesting?prefix=pipelines/A7/25309493 fmt.Printf("Looking for agent MSI for pipeline majorVersion %v %v\n", majorVersion, pipelineID) artifactURL, err := pipeline.GetPipelineArtifact(pipelineID, pipeline.AgentS3BucketTesting, majorVersion, func(artifact string) bool { @@ -114,9 +160,10 @@ func GetPipelineMSIURL(pipelineID string, majorVersion string, arch string) (str // TODO: CIREL-1970 // Example: datadog-agent-7.52.0-1-x86_64.msi // Example: datadog-agent-7.53.0-devel.git.512.41b1225.pipeline.30353507-1-x86_64.msi - if !strings.Contains(artifact, fmt.Sprintf("datadog-agent-%s", majorVersion)) { + if !strings.Contains(artifact, fmt.Sprintf("%s-%s", productName, majorVersion)) { return false } + // Not all pipelines include the pipeline ID in the artifact name, but if it is there then match against it if strings.Contains(artifact, "pipeline.") && !strings.Contains(artifact, fmt.Sprintf("pipeline.%s", pipelineID)) { @@ -129,7 +176,7 @@ func GetPipelineMSIURL(pipelineID string, majorVersion string, arch string) (str return true }) if err != nil { - return "", fmt.Errorf("no agent MSI found for pipeline %v and arch %v: %w", pipelineID, arch, err) + return "", fmt.Errorf("no agent MSI found for pipeline %v arch %v flavor: %v: %w", pipelineID, arch, flavor, err) } return artifactURL, nil } @@ -148,6 +195,20 @@ func LookupChannelFromEnv() (string, bool) { return stableChannel, false } +// LookupFlavorFromEnv looks at environment variables to select the agent flavor, if the value +// is found it is returned along with true, otherwise an empty string and false are returned. +// +// WINDOWS_AGENT_FLAVOR: base, fips +// +// Default Flavor: base +func LookupFlavorFromEnv() (string, bool) { + flavor := os.Getenv(PackageFlavorEnvVar) + if flavor != "" { + return flavor, true + } + return defaultFlavor, false +} + // LookupVersionFromEnv looks at environment variabes to select the agent version, if the value // is found it is returned along with true, otherwise a default value and false are returned. // @@ -217,7 +278,7 @@ func LookupChannelURLFromEnv() (string, bool) { // // The returned Package contains the MSI URL and other identifying information. // Some Package fields will be populated but may not be related to the returned URL. -// For example, if a URL is provided directly, the Channel, Version, and Arch fields +// For example, if a URL is provided directly, the Channel, Version, Arch, and Flavor fields // have no effect on the returned URL. They are returned anyway so they can be used for // other purposes, such as logging, stack name, instance options, test assertions, etc. // @@ -235,6 +296,8 @@ func LookupChannelURLFromEnv() (string, bool) { // // WINDOWS_AGENT_ARCH: The arch of the agent, x86_64 // +// WINDOWS_AGENT_FLAVOR: The flavor of the agent, base or fips +// // If a channel is not provided and the version contains `-rc.`, the beta channel is used. // // See other Lookup*FromEnv functions for more options and details. @@ -245,6 +308,7 @@ func GetPackageFromEnv() (*Package, error) { channel, channelFound := LookupChannelFromEnv() version, _ := LookupVersionFromEnv() arch, _ := LookupArchFromEnv() + flavor, _ := LookupFlavorFromEnv() pipelineID, pipelineIDFound := os.LookupEnv("E2E_PIPELINE_ID") majorVersion := strings.Split(version, ".")[0] @@ -267,12 +331,13 @@ func GetPackageFromEnv() (*Package, error) { Version: version, Arch: arch, URL: url, + Flavor: flavor, }, nil } // check if we should use the URL from a specific CI pipeline if pipelineIDFound { - url, err := GetPipelineMSIURL(pipelineID, majorVersion, arch) + url, err := GetPipelineMSIURL(pipelineID, majorVersion, arch, flavor) if err != nil { return nil, err } @@ -281,6 +346,7 @@ func GetPackageFromEnv() (*Package, error) { Version: version, Arch: arch, URL: url, + Flavor: flavor, }, nil } @@ -295,7 +361,11 @@ func GetPackageFromEnv() (*Package, error) { } } // Get MSI URL - url, err := installers.GetProductURL(channelURL, agentInstallerListProductName, version, arch) + productName, err := GetFlavorProductName(flavor) + if err != nil { + return nil, err + } + url, err := installers.GetProductURL(channelURL, productName, version, arch) if err != nil { return nil, err } @@ -304,16 +374,21 @@ func GetPackageFromEnv() (*Package, error) { Version: version, Arch: arch, URL: url, + Flavor: flavor, }, nil } // Default to latest stable - url = GetLatestMSIURL(majorVersion, arch) + url, err = GetLatestMSIURL(majorVersion, arch, flavor) + if err != nil { + return nil, err + } return &Package{ Channel: stableChannel, Version: version, Arch: arch, URL: url, + Flavor: flavor, }, nil } @@ -329,6 +404,7 @@ func GetPackageFromEnv() (*Package, error) { // invoke release.get-release-json-value "last_stable::$AGENT_MAJOR_VERSION" func GetLastStablePackageFromEnv() (*Package, error) { arch, _ := LookupArchFromEnv() + flavor, _ := LookupFlavorFromEnv() ver := os.Getenv("LAST_STABLE_VERSION") if ver == "" { return nil, fmt.Errorf("LAST_STABLE_VERSION is not set") @@ -341,7 +417,7 @@ func GetLastStablePackageFromEnv() (*Package, error) { url := os.Getenv("LAST_STABLE_WINDOWS_AGENT_MSI_URL") if url == "" { // Manual URL not provided, lookup the URL using the version - url, err = GetStableMSIURL(ver, arch) + url, err = GetStableMSIURL(ver, arch, flavor) if err != nil { return nil, err } @@ -352,5 +428,6 @@ func GetLastStablePackageFromEnv() (*Package, error) { Version: ver, Arch: arch, URL: url, + Flavor: flavor, }, nil } diff --git a/test/new-e2e/tests/windows/common/policy.go b/test/new-e2e/tests/windows/common/policy.go new file mode 100644 index 0000000000000..df50d64375790 --- /dev/null +++ b/test/new-e2e/tests/windows/common/policy.go @@ -0,0 +1,46 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +package common + +import ( + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" +) + +// setFIPSAlgorithmPolicy configures local security policy to enable or disable FIPS mode. +// +// The setting is applied system-wide and does NOT require a reboot. +// This setting may be overridden by group policy. +// +// https://learn.microsoft.com/en-us/previous-versions/windows/it-pro/windows-10/security/threat-protection/security-policy-settings/system-cryptography-use-fips-compliant-algorithms-for-encryption-hashing-and-signing +func setFIPSAlgorithmPolicy(host *components.RemoteHost, enabled bool) error { + path := `HKLM:\SYSTEM\CurrentControlSet\Control\Lsa\FIPSAlgorithmPolicy` + valueName := "Enabled" + value := 0 + if enabled { + value = 1 + } + return SetRegistryDWORDValue(host, path, valueName, value) +} + +// EnableFIPSMode enables FIPS mode on the host. +// +// The setting is applied system-wide and does NOT require a reboot. +// This setting may be overridden by group policy. +// +// https://learn.microsoft.com/en-us/previous-versions/windows/it-pro/windows-10/security/threat-protection/security-policy-settings/system-cryptography-use-fips-compliant-algorithms-for-encryption-hashing-and-signing +func EnableFIPSMode(host *components.RemoteHost) error { + return setFIPSAlgorithmPolicy(host, true) +} + +// DisableFIPSMode disables FIPS mode on the host. +// +// The setting is applied system-wide and does NOT require a reboot. +// This setting may be overridden by group policy. +// +// https://learn.microsoft.com/en-us/previous-versions/windows/it-pro/windows-10/security/threat-protection/security-policy-settings/system-cryptography-use-fips-compliant-algorithms-for-encryption-hashing-and-signing +func DisableFIPSMode(host *components.RemoteHost) error { + return setFIPSAlgorithmPolicy(host, false) +} diff --git a/test/new-e2e/tests/windows/common/registry.go b/test/new-e2e/tests/windows/common/registry.go index 19c44f530f02d..0957c3510251f 100644 --- a/test/new-e2e/tests/windows/common/registry.go +++ b/test/new-e2e/tests/windows/common/registry.go @@ -38,3 +38,20 @@ func DeleteRegistryKey(host *components.RemoteHost, path string) error { _, err := host.Execute(cmd) return err } + +// SetRegistryDWORDValue sets, creating if necessary, a DWORD value at the specified path +func SetRegistryDWORDValue(host *components.RemoteHost, path string, name string, value int) error { + return SetTypedRegistryValue(host, path, name, fmt.Sprintf("%d", value), "DWORD") +} + +// SetTypedRegistryValue sets, creating if necessary, the value at the specified path with the specified type +// +// https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.management/set-itemproperty?view=powershell-7.4#-type +func SetTypedRegistryValue(host *components.RemoteHost, path string, name string, value string, typeName string) error { + cmd := fmt.Sprintf("New-Item -Path '%s' -Force; Set-ItemProperty -Path '%s' -Name '%s' -Value '%s' -Type '%s'", path, path, name, value, typeName) + _, err := host.Execute(cmd) + if err != nil { + return err + } + return nil +} diff --git a/test/new-e2e/tests/windows/fips-test/fips_test.go b/test/new-e2e/tests/windows/fips-test/fips_test.go new file mode 100644 index 0000000000000..e46be55c5a941 --- /dev/null +++ b/test/new-e2e/tests/windows/fips-test/fips_test.go @@ -0,0 +1,123 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package fipstest +package fipstest + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" + "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" + windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" + windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" + + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fipsAgentSuite struct { + windows.BaseAgentInstallerSuite[environments.WindowsHost] + + installPath string +} + +func TestFIPSAgent(t *testing.T) { + opts := []e2e.SuiteOption{e2e.WithProvisioner(awsHostWindows.ProvisionerNoAgentNoFakeIntake())} + s := &fipsAgentSuite{} + e2e.Run(t, s, opts...) +} + +func (s *fipsAgentSuite) SetupSuite() { + // Default to using FIPS Agent package + if _, set := windowsAgent.LookupFlavorFromEnv(); !set { + os.Setenv(windowsAgent.PackageFlavorEnvVar, "fips") + } + + s.BaseAgentInstallerSuite.SetupSuite() + host := s.Env().RemoteHost + var err error + + // Enable FIPS mode before installing the Agent to make sure that works + err = windowsCommon.EnableFIPSMode(host) + require.NoError(s.T(), err) + + // Install Agent (With FIPS mode enabled) + _, err = s.InstallAgent(host, windowsAgent.WithPackage(s.AgentPackage)) + require.NoError(s.T(), err) + + s.installPath, err = windowsAgent.GetInstallPathFromRegistry(host) + require.NoError(s.T(), err) +} + +func (s *fipsAgentSuite) TestWithSystemFIPSDisabled() { + host := s.Env().RemoteHost + windowsCommon.DisableFIPSMode(host) + + s.Run("version command", func() { + s.Run("gofips enabled", func() { + _, err := s.execAgentCommandWithFIPS("version") + assertErrorContainsFIPSPanic(s.T(), err, "agent should panic when GOFIPS=1 but system FIPS is disabled") + }) + + s.Run("gofips disabled", func() { + _, err := s.execAgentCommand("version") + require.NoError(s.T(), err) + }) + }) +} + +func (s *fipsAgentSuite) TestWithSystemFIPSEnabled() { + host := s.Env().RemoteHost + windowsCommon.EnableFIPSMode(host) + + s.Run("version command", func() { + s.Run("gofips enabled", func() { + _, err := s.execAgentCommandWithFIPS("version") + require.NoError(s.T(), err) + }) + + s.Run("gofips disabled", func() { + _, err := s.execAgentCommand("version") + require.NoError(s.T(), err) + }) + }) +} + +func (s *fipsAgentSuite) execAgentCommand(command string, options ...client.ExecuteOption) (string, error) { + host := s.Env().RemoteHost + + require.NotEmpty(s.T(), s.installPath) + agentPath := filepath.Join(s.installPath, "bin", "agent.exe") + + cmd := fmt.Sprintf(`& "%s" %s`, agentPath, command) + return host.Execute(cmd, options...) +} + +func (s *fipsAgentSuite) execAgentCommandWithFIPS(command string) (string, error) { + // There isn't support for appending env vars to client.ExecuteOption, so + // this function doesn't accept any other options. + + // Setting GOFIPS=1 causes the Windows FIPS Agent to panic if the system is not in FIPS mode. + // This setting does NOT control whether the FIPS Agent uses FIPS-compliant crypto libraries, + // the System-level setting determines that. + // https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips#windows-fips-mode-cng + vars := client.EnvVar{ + "GOFIPS": "1", + } + + return s.execAgentCommand(command, client.WithEnvVariables(vars)) +} + +func assertErrorContainsFIPSPanic(t *testing.T, err error, args ...interface{}) bool { + return assert.ErrorContains(t, err, "panic: cngcrypto: not in FIPS mode", args...) +} diff --git a/test/new-e2e/tests/windows/install-test/npm_test.go b/test/new-e2e/tests/windows/install-test/npm_test.go index cefec1f51b626..ecd8ffcd56129 100644 --- a/test/new-e2e/tests/windows/install-test/npm_test.go +++ b/test/new-e2e/tests/windows/install-test/npm_test.go @@ -149,7 +149,7 @@ func (s *testNPMInstallSuite) TearDownSuite() { func (s *testNPMInstallSuite) installPreviousAgentVersion(vm *components.RemoteHost, options ...windowsAgent.InstallAgentOption) { if s.url == "" { - url, err := windowsAgent.GetStableMSIURL(s.previousVersion, "x86_64") + url, err := windowsAgent.GetStableMSIURL(s.previousVersion, "x86_64", "") s.Require().NoError(err, "should get MSI URL for version %s", s.previousVersion) s.url = url } diff --git a/test/new-e2e/tests/windows/install-test/upgrade_test.go b/test/new-e2e/tests/windows/install-test/upgrade_test.go index 851b6a739f376..4f274dbccea1f 100644 --- a/test/new-e2e/tests/windows/install-test/upgrade_test.go +++ b/test/new-e2e/tests/windows/install-test/upgrade_test.go @@ -142,7 +142,7 @@ func (s *testUpgradeRollbackWithoutCWSSuite) SetupSuite() { Version: fmt.Sprintf("%s.51.0-1", majorVersion), Arch: "x86_64", } - s.previousAgentPackage.URL, err = windowsAgent.GetStableMSIURL(s.previousAgentPackage.Version, s.previousAgentPackage.Arch) + s.previousAgentPackage.URL, err = windowsAgent.GetStableMSIURL(s.previousAgentPackage.Version, s.previousAgentPackage.Arch, "") s.Require().NoError(err, "should get stable agent package URL") } @@ -282,7 +282,7 @@ func TestUpgradeFromV5(t *testing.T) { s.agent5Package = &windowsAgent.Package{ Version: "5.32.8-1", } - s.agent5Package.URL, err = windowsAgent.GetStableMSIURL(s.agent5Package.Version, "x86_64") + s.agent5Package.URL, err = windowsAgent.GetStableMSIURL(s.agent5Package.Version, "x86_64", "") require.NoError(t, err) run(t, s) } @@ -375,7 +375,7 @@ func TestUpgradeFromV6(t *testing.T) { Version: "6.53.0-1", Arch: "x86_64", } - s.previousAgentPackge.URL, err = windowsAgent.GetStableMSIURL(s.previousAgentPackge.Version, s.previousAgentPackge.Arch) + s.previousAgentPackge.URL, err = windowsAgent.GetStableMSIURL(s.previousAgentPackge.Version, s.previousAgentPackge.Arch, "") require.NoError(t, err) run(t, s) } From 86aa47af3eb8121086b6b41b029105c9d4d15daf Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Mon, 9 Dec 2024 11:19:08 -0500 Subject: [PATCH 32/52] Only start telemetryForwarder when we start the receiver (APMSP-1633) (#31813) --- pkg/trace/api/api.go | 2 ++ pkg/trace/api/telemetry.go | 1 - pkg/trace/api/telemetry_test.go | 12 ++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/trace/api/api.go b/pkg/trace/api/api.go index 9ad40058faf82..c74ecbb137fb9 100644 --- a/pkg/trace/api/api.go +++ b/pkg/trace/api/api.go @@ -235,6 +235,8 @@ func getConfiguredEVPRequestTimeoutDuration(conf *config.AgentConfig) time.Durat // Start starts doing the HTTP server and is ready to receive traces func (r *HTTPReceiver) Start() { + r.telemetryForwarder.start() + if !r.conf.ReceiverEnabled { log.Debug("HTTP Server is off: HTTPReceiver is disabled.") return diff --git a/pkg/trace/api/telemetry.go b/pkg/trace/api/telemetry.go index a45ac4105042f..5408f3db40e1c 100644 --- a/pkg/trace/api/telemetry.go +++ b/pkg/trace/api/telemetry.go @@ -123,7 +123,6 @@ func NewTelemetryForwarder(conf *config.AgentConfig, containerIDProvider IDProvi statsd: statsd, logger: log.NewThrottled(5, 10*time.Second), } - forwarder.start() return forwarder } diff --git a/pkg/trace/api/telemetry_test.go b/pkg/trace/api/telemetry_test.go index 3608436d83aae..b9a0930507687 100644 --- a/pkg/trace/api/telemetry_test.go +++ b/pkg/trace/api/telemetry_test.go @@ -99,6 +99,7 @@ func TestTelemetryBasicProxyRequest(t *testing.T) { return []string{"key:test\nvalue"}, nil } recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() recv.telemetryForwarder.containerIDProvider = getTestContainerIDProvider() assertSendRequest(t, recv, endpointCalled) @@ -121,6 +122,7 @@ func TestGoogleCloudRun(t *testing.T) { cfg.GlobalTags["service_name"] = "test_service" cfg.GlobalTags["origin"] = "cloudrun" recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() assertSendRequest(t, recv, endpointCalled) } @@ -145,6 +147,7 @@ func TestAzureAppService(t *testing.T) { cfg.GlobalTags["app_name"] = "test_app" cfg.GlobalTags["origin"] = "appservice" recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() assertSendRequest(t, recv, endpointCalled) } @@ -169,6 +172,7 @@ func TestAzureContainerApp(t *testing.T) { cfg.GlobalTags["app_name"] = "test_app" cfg.GlobalTags["origin"] = "containerapp" recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() assertSendRequest(t, recv, endpointCalled) } @@ -202,6 +206,7 @@ func TestAWSFargate(t *testing.T) { return []string{"task_arn:test_ARN"}, nil } recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() recv.telemetryForwarder.containerIDProvider = getTestContainerIDProvider() assertSendRequest(t, recv, endpointCalled) @@ -256,6 +261,7 @@ func TestTelemetryProxyMultipleEndpoints(t *testing.T) { cfg.GlobalTags[functionARNKeyTag] = "test_ARN" recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() req, rec := newRequestRecorder(t) recv.buildMux().ServeHTTP(rec, req) @@ -315,6 +321,7 @@ func TestMaxInflightBytes(t *testing.T) { cfg := getTestConfig(srv.URL) recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() recv.telemetryForwarder.maxInflightBytes = 100 mux := recv.buildMux() @@ -359,6 +366,7 @@ func TestInflightBytesReset(t *testing.T) { cfg := getTestConfig(srv.URL) recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() recv.telemetryForwarder.maxInflightBytes = 100 mux := recv.buildMux() @@ -417,6 +425,7 @@ func TestActualServer(t *testing.T) { cfg := getTestConfig(intakeMockServer.URL) r := newTestReceiverFromConfig(cfg) + r.telemetryForwarder.start() // We call this manually here to avoid starting the entire test receiver logs := bytes.Buffer{} prevLogger := log.SetLogger(log.NewBufferLogger(&logs)) defer log.SetLogger(prevLogger) @@ -445,6 +454,7 @@ func TestTelemetryConfig(t *testing.T) { cfg := config.New() cfg.Endpoints[0].APIKey = "api_key" recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() req, rec := newRequestRecorder(t) recv.buildMux().ServeHTTP(rec, req) @@ -463,6 +473,7 @@ func TestTelemetryConfig(t *testing.T) { Host: "111://malformed.dd_url.com", }} recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() req, rec := newRequestRecorder(t) recv.buildMux().ServeHTTP(rec, req) @@ -486,6 +497,7 @@ func TestTelemetryConfig(t *testing.T) { Host: srv.URL, }} recv := newTestReceiverFromConfig(cfg) + recv.telemetryForwarder.start() req, rec := newRequestRecorder(t) recv.buildMux().ServeHTTP(rec, req) From 952f4b2bc1b854e5e949da235353002cab8979ee Mon Sep 17 00:00:00 2001 From: Dustin Long Date: Mon, 9 Dec 2024 11:31:50 -0500 Subject: [PATCH 33/52] Implement UnsetForSource method (#31733) --- pkg/config/model/viper.go | 15 ++- pkg/config/nodetreemodel/config.go | 80 ++++++++++++- pkg/config/nodetreemodel/config_test.go | 142 +++++++++++++++++++++++ pkg/config/nodetreemodel/inner_node.go | 10 +- pkg/config/nodetreemodel/leaf_node.go | 6 +- pkg/config/nodetreemodel/missing_node.go | 2 +- pkg/config/nodetreemodel/node.go | 3 +- pkg/config/nodetreemodel/struct_node.go | 3 + 8 files changed, 247 insertions(+), 14 deletions(-) diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index 5742b4c06520b..a977863b7368d 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -91,9 +91,18 @@ type ValueWithSource struct { Value interface{} } -// IsGreaterOrEqualThan returns true if the current source is of higher priority than the one given as a parameter -func (s Source) IsGreaterOrEqualThan(x Source) bool { - return sourcesPriority[s] >= sourcesPriority[x] +// IsGreaterThan returns true if the current source is of higher priority than the one given as a parameter +func (s Source) IsGreaterThan(x Source) bool { + return sourcesPriority[s] > sourcesPriority[x] +} + +// PreviousSource returns the source before the current one, or Default (lowest priority) if there isn't one +func (s Source) PreviousSource() Source { + previous := sourcesPriority[s] + if previous == 0 { + return sources[previous] + } + return sources[previous-1] } // String casts Source into a string diff --git a/pkg/config/nodetreemodel/config.go b/pkg/config/nodetreemodel/config.go index 3d09ddc016eba..e2f6c4b8f7139 100644 --- a/pkg/config/nodetreemodel/config.go +++ b/pkg/config/nodetreemodel/config.go @@ -241,11 +241,83 @@ func (c *ntmConfig) SetDefault(key string, value interface{}) { _, _ = c.defaults.SetAt(parts, value, model.SourceDefault) } +func (c *ntmConfig) findPreviousSourceNode(key string, source model.Source) (Node, error) { + iter := source + for iter != model.SourceDefault { + iter = iter.PreviousSource() + tree, err := c.getTreeBySource(iter) + if err != nil { + return nil, err + } + node := c.leafAtPathFromNode(key, tree) + if _, isMissing := node.(*missingLeafImpl); !isMissing { + return node, nil + } + } + return nil, ErrNotFound +} + // UnsetForSource unsets a config entry for a given source -func (c *ntmConfig) UnsetForSource(_key string, _source model.Source) { +func (c *ntmConfig) UnsetForSource(key string, source model.Source) { c.Lock() - c.logErrorNotImplemented("UnsetForSource") - c.Unlock() + defer c.Unlock() + + // Remove it from the original source tree + tree, err := c.getTreeBySource(source) + if err != nil { + log.Errorf("%s", err) + return + } + parentNode, childName, err := c.parentOfNode(tree, key) + if err != nil { + return + } + // Only remove if the setting is a leaf + if child, err := parentNode.GetChild(childName); err == nil { + if _, ok := child.(LeafNode); ok { + parentNode.RemoveChild(childName) + } else { + log.Errorf("cannot remove setting %q, not a leaf", key) + return + } + } + + // If the node in the merged tree doesn't match the source we expect, we're done + if c.leafAtPathFromNode(key, c.root).Source() != source { + return + } + + // Find what the previous value used to be, based upon the previous source + prevNode, err := c.findPreviousSourceNode(key, source) + if err != nil { + return + } + + // Get the parent node of the leaf we're unsetting + parentNode, childName, err = c.parentOfNode(c.root, key) + if err != nil { + return + } + // Replace the child with the node from the previous layer + parentNode.InsertChildNode(childName, prevNode.Clone()) +} + +func (c *ntmConfig) parentOfNode(node Node, key string) (InnerNode, string, error) { + parts := splitKey(key) + lastPart := parts[len(parts)-1] + parts = parts[:len(parts)-1] + var err error + for _, p := range parts { + node, err = node.GetChild(p) + if err != nil { + return nil, "", err + } + } + innerNode, ok := node.(InnerNode) + if !ok { + return nil, "", ErrNotFound + } + return innerNode, lastPart, nil } func (c *ntmConfig) addToKnownKeys(key string) { @@ -609,7 +681,7 @@ func (c *ntmConfig) AllSettingsWithoutDefault() map[string]interface{} { defer c.RUnlock() // We only want to include leaf with a source higher than SourceDefault - return c.root.DumpSettings(func(source model.Source) bool { return source.IsGreaterOrEqualThan(model.SourceUnknown) }) + return c.root.DumpSettings(func(source model.Source) bool { return source.IsGreaterThan(model.SourceDefault) }) } // AllSettingsBySource returns the settings from each source (file, env vars, ...) diff --git a/pkg/config/nodetreemodel/config_test.go b/pkg/config/nodetreemodel/config_test.go index dfba602113411..4738b4e446767 100644 --- a/pkg/config/nodetreemodel/config_test.go +++ b/pkg/config/nodetreemodel/config_test.go @@ -433,3 +433,145 @@ server_timeout val:30, source:default` assert.Equal(t, expect, txt) } + +func TestUnsetForSource(t *testing.T) { + // env source, highest priority + os.Setenv("TEST_NETWORK_PATH_COLLECTOR_INPUT_CHAN_SIZE", "23456") + os.Setenv("TEST_NETWORK_PATH_COLLECTOR_PATHTEST_CONTEXTS_LIMIT", "654321") + os.Setenv("TEST_NETWORK_PATH_COLLECTOR_PROCESSING_CHAN_SIZE", "78900") + // file source, medium priority + configData := `network_path: + collector: + workers: 6 + pathtest_contexts_limit: 43210 + processing_chan_size: 45678` + // default source, lowest priority + cfg := NewConfig("test", "TEST", strings.NewReplacer(".", "_")) + cfg.BindEnvAndSetDefault("network_path.collector.input_chan_size", 100000) + cfg.BindEnvAndSetDefault("network_path.collector.pathtest_contexts_limit", 100000) + cfg.BindEnvAndSetDefault("network_path.collector.processing_chan_size", 100000) + cfg.BindEnvAndSetDefault("network_path.collector.workers", 4) + + cfg.BuildSchema() + err := cfg.ReadConfig(strings.NewReader(configData)) + require.NoError(t, err) + + // The merged config + txt := cfg.(*ntmConfig).Stringify("root") + expect := `network_path + collector + input_chan_size + val:23456, source:environment-variable + pathtest_contexts_limit + val:654321, source:environment-variable + processing_chan_size + val:78900, source:environment-variable + workers + val:6, source:file` + assert.Equal(t, expect, txt) + + // No change if source doesn't match + cfg.UnsetForSource("network_path.collector.input_chan_size", model.SourceFile) + assert.Equal(t, expect, txt) + + // No change if setting is not a leaf + cfg.UnsetForSource("network_path", model.SourceEnvVar) + assert.Equal(t, expect, txt) + + // No change if setting is not found + cfg.UnsetForSource("network_path.unknown", model.SourceEnvVar) + assert.Equal(t, expect, txt) + + // Remove a setting from the env source, nothing in the file source, it goes to default + cfg.UnsetForSource("network_path.collector.input_chan_size", model.SourceEnvVar) + txt = cfg.(*ntmConfig).Stringify("root") + expect = `network_path + collector + input_chan_size + val:100000, source:default + pathtest_contexts_limit + val:654321, source:environment-variable + processing_chan_size + val:78900, source:environment-variable + workers + val:6, source:file` + assert.Equal(t, expect, txt) + + // Remove a setting from the file source, it goes to default + cfg.UnsetForSource("network_path.collector.workers", model.SourceFile) + txt = cfg.(*ntmConfig).Stringify("root") + expect = `network_path + collector + input_chan_size + val:100000, source:default + pathtest_contexts_limit + val:654321, source:environment-variable + processing_chan_size + val:78900, source:environment-variable + workers + val:4, source:default` + assert.Equal(t, expect, txt) + + // Removing a setting from the env source, it goes to file source + cfg.UnsetForSource("network_path.collector.processing_chan_size", model.SourceEnvVar) + txt = cfg.(*ntmConfig).Stringify("root") + expect = `network_path + collector + input_chan_size + val:100000, source:default + pathtest_contexts_limit + val:654321, source:environment-variable + processing_chan_size + val:45678, source:file + workers + val:4, source:default` + assert.Equal(t, expect, txt) + + // Then remove it from the file source as well, leaving the default source + cfg.UnsetForSource("network_path.collector.processing_chan_size", model.SourceFile) + txt = cfg.(*ntmConfig).Stringify("root") + expect = `network_path + collector + input_chan_size + val:100000, source:default + pathtest_contexts_limit + val:654321, source:environment-variable + processing_chan_size + val:100000, source:default + workers + val:4, source:default` + assert.Equal(t, expect, txt) + + // Check the file layer in isolation + fileTxt := cfg.(*ntmConfig).Stringify(model.SourceFile) + fileExpect := `network_path + collector + pathtest_contexts_limit + val:43210, source:file` + assert.Equal(t, fileExpect, fileTxt) + + // Removing from the file source first does not change the merged value, because it uses env layer + cfg.UnsetForSource("network_path.collector.pathtest_contexts_limit", model.SourceFile) + assert.Equal(t, expect, txt) + + // But the file layer itself has been modified + fileTxt = cfg.(*ntmConfig).Stringify(model.SourceFile) + fileExpect = `network_path + collector` + assert.Equal(t, fileExpect, fileTxt) + + // Finally, remove it from the env layer + cfg.UnsetForSource("network_path.collector.pathtest_contexts_limit", model.SourceEnvVar) + txt = cfg.(*ntmConfig).Stringify("root") + expect = `network_path + collector + input_chan_size + val:100000, source:default + pathtest_contexts_limit + val:100000, source:default + processing_chan_size + val:100000, source:default + workers + val:4, source:default` + assert.Equal(t, expect, txt) +} diff --git a/pkg/config/nodetreemodel/inner_node.go b/pkg/config/nodetreemodel/inner_node.go index d531dc1d7facb..dd0c4bee914c1 100644 --- a/pkg/config/nodetreemodel/inner_node.go +++ b/pkg/config/nodetreemodel/inner_node.go @@ -88,7 +88,7 @@ func (n *innerNode) Merge(src InnerNode) error { } if srcIsLeaf { - if srcLeaf.SourceGreaterOrEqual(dstLeaf.Source()) { + if srcLeaf.Source() == dstLeaf.Source() || srcLeaf.SourceGreaterThan(dstLeaf.Source()) { n.children[name] = srcLeaf.Clone() } } else { @@ -132,7 +132,7 @@ func (n *innerNode) SetAt(key []string, value interface{}, source model.Source) } if leaf, ok := node.(LeafNode); ok { - if source.IsGreaterOrEqualThan(leaf.Source()) { + if source == leaf.Source() || source.IsGreaterThan(leaf.Source()) { n.children[part] = newLeafNode(value, source) return true, nil } @@ -163,6 +163,12 @@ func (n *innerNode) InsertChildNode(name string, node Node) { n.makeRemapCase() } +// RemoveChild removes a node from the current node +func (n *innerNode) RemoveChild(name string) { + delete(n.children, name) + n.makeRemapCase() +} + // DumpSettings clone the entire tree starting from the node into a map based on the leaf source. // // The selector will be call with the source of each leaf to determine if it should be included in the dump. diff --git a/pkg/config/nodetreemodel/leaf_node.go b/pkg/config/nodetreemodel/leaf_node.go index 45a6391248bd4..3140145d521f7 100644 --- a/pkg/config/nodetreemodel/leaf_node.go +++ b/pkg/config/nodetreemodel/leaf_node.go @@ -43,10 +43,10 @@ func (n *leafNodeImpl) Clone() Node { return newLeafNode(n.val, n.source) } -// SourceGreaterOrEqual returns true if the source of the current node is greater or equal to the one given as a +// SourceGreaterThan returns true if the source of the current node is greater than the one given as a // parameter -func (n *leafNodeImpl) SourceGreaterOrEqual(source model.Source) bool { - return n.source.IsGreaterOrEqualThan(source) +func (n *leafNodeImpl) SourceGreaterThan(source model.Source) bool { + return n.source.IsGreaterThan(source) } // GetChild returns an error because a leaf has no children diff --git a/pkg/config/nodetreemodel/missing_node.go b/pkg/config/nodetreemodel/missing_node.go index 52d68f0112ef9..d6a2ad640d8af 100644 --- a/pkg/config/nodetreemodel/missing_node.go +++ b/pkg/config/nodetreemodel/missing_node.go @@ -38,6 +38,6 @@ func (m *missingLeafImpl) Clone() Node { return m } -func (m *missingLeafImpl) SourceGreaterOrEqual(model.Source) bool { +func (m *missingLeafImpl) SourceGreaterThan(model.Source) bool { return false } diff --git a/pkg/config/nodetreemodel/node.go b/pkg/config/nodetreemodel/node.go index 97007efa50198..469f0d89cfedd 100644 --- a/pkg/config/nodetreemodel/node.go +++ b/pkg/config/nodetreemodel/node.go @@ -96,6 +96,7 @@ type InnerNode interface { Merge(InnerNode) error SetAt([]string, interface{}, model.Source) (bool, error) InsertChildNode(string, Node) + RemoveChild(string) makeRemapCase() DumpSettings(func(model.Source) bool) map[string]interface{} } @@ -105,5 +106,5 @@ type LeafNode interface { Node Get() interface{} Source() model.Source - SourceGreaterOrEqual(model.Source) bool + SourceGreaterThan(model.Source) bool } diff --git a/pkg/config/nodetreemodel/struct_node.go b/pkg/config/nodetreemodel/struct_node.go index 30fa47ce1c12b..511235381c35a 100644 --- a/pkg/config/nodetreemodel/struct_node.go +++ b/pkg/config/nodetreemodel/struct_node.go @@ -74,6 +74,9 @@ func (n *structNodeImpl) SetAt([]string, interface{}, model.Source) (bool, error // InsertChildNode is not implemented for a leaf node func (n *structNodeImpl) InsertChildNode(string, Node) {} +// RemoveChild is not implemented for struct node +func (n *structNodeImpl) RemoveChild(string) {} + // makeRemapCase not implemented func (n *structNodeImpl) makeRemapCase() {} From 94679d8b5fb02b7e8cd15af96e799e71fab761b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Mon, 9 Dec 2024 18:05:08 +0100 Subject: [PATCH 34/52] fix(podresources): detect API availability client creation (#31806) --- .../env/environment_container_features.go | 2 ++ pkg/config/env/environment_containers.go | 21 +++++++++++++++++-- pkg/util/kubernetes/kubelet/kubelet.go | 12 ++++++----- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/config/env/environment_container_features.go b/pkg/config/env/environment_container_features.go index 1601a47696da5..ffced09b49c61 100644 --- a/pkg/config/env/environment_container_features.go +++ b/pkg/config/env/environment_container_features.go @@ -31,4 +31,6 @@ const ( CloudFoundry Feature = "cloudfoundry" // Podman containers storage path accessible Podman Feature = "podman" + // PodResources socket present + PodResources Feature = "podresources" ) diff --git a/pkg/config/env/environment_containers.go b/pkg/config/env/environment_containers.go index 9ddc1cdfcab62..7dbb0cf9ce58f 100644 --- a/pkg/config/env/environment_containers.go +++ b/pkg/config/env/environment_containers.go @@ -46,6 +46,7 @@ func init() { registerFeature(ECSOrchestratorExplorer) registerFeature(CloudFoundry) registerFeature(Podman) + registerFeature(PodResources) } // IsAnyContainerFeaturePresent checks if any of known container features is present @@ -69,6 +70,7 @@ func detectContainerFeatures(features FeatureMap, cfg model.Reader) { detectAWSEnvironments(features, cfg) detectCloudFoundry(features, cfg) detectPodman(features, cfg) + detectPodResources(features, cfg) } func detectKubernetes(features FeatureMap, cfg model.Reader) { @@ -96,7 +98,7 @@ func detectDocker(features FeatureMap) { // Even though it does not modify configuration, using the OverrideFunc mechanism for uniformity model.AddOverrideFunc(func(model.Config) { - os.Setenv("DOCKER_HOST", getDefaultDockerSocketType()+defaultDockerSocketPath) + os.Setenv("DOCKER_HOST", getDefaultSocketPrefix()+defaultDockerSocketPath) }) break } @@ -226,6 +228,21 @@ func detectPodman(features FeatureMap, cfg model.Reader) { } } +func detectPodResources(features FeatureMap, cfg model.Reader) { + // We only check the path from config. Default socket path is defined in the config + socketPath := getDefaultSocketPrefix() + cfg.GetString("kubernetes_kubelet_podresources_socket") + + exists, reachable := socket.IsAvailable(socketPath, socketTimeout) + if exists && reachable { + log.Infof("Agent found PodResources socket at %s", socketPath) + features[PodResources] = struct{}{} + } else if exists && !reachable { + log.Infof("Agent found PodResources socket at %s but socket not reachable (permissions?)", socketPath) + } else { + log.Infof("Agent did not find PodResources socket at %s", socketPath) + } +} + func getHostMountPrefixes() []string { if IsContainerized() { return []string{"", defaultHostMountPrefix} @@ -233,7 +250,7 @@ func getHostMountPrefixes() []string { return []string{""} } -func getDefaultDockerSocketType() string { +func getDefaultSocketPrefix() string { if runtime.GOOS == "windows" { return winNamedPipePrefix } diff --git a/pkg/util/kubernetes/kubelet/kubelet.go b/pkg/util/kubernetes/kubelet/kubelet.go index 5708841967134..d44e8074be936 100644 --- a/pkg/util/kubernetes/kubelet/kubelet.go +++ b/pkg/util/kubernetes/kubelet/kubelet.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "github.com/DataDog/datadog-agent/pkg/config/env" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/util/cache" @@ -84,9 +85,11 @@ func (ku *KubeUtil) init() error { } } - ku.podResourcesClient, err = NewPodResourcesClient(pkgconfigsetup.Datadog()) - if err != nil { - log.Warnf("Failed to create pod resources client, resource data will not be available: %s", err) + if env.IsFeaturePresent(env.PodResources) { + ku.podResourcesClient, err = NewPodResourcesClient(pkgconfigsetup.Datadog()) + if err != nil { + log.Warnf("Failed to create pod resources client, resource data will not be available: %s", err) + } } return nil @@ -214,8 +217,7 @@ func (ku *KubeUtil) getLocalPodList(ctx context.Context) (*PodList, error) { err = ku.addContainerResourcesData(ctx, pods.Items) if err != nil { - // TODO: Switch back to error level once the socket issue is fixed. - log.Debugf("Error adding container resources data: %s", err) + log.Errorf("Error adding container resources data: %s", err) } // ensure we dont have nil pods From 0b3f726f2a98fd5f3fbf4008399a67314806dbcf Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Mon, 9 Dec 2024 18:17:43 +0100 Subject: [PATCH 35/52] feat(package_size): Display the package size message in the PR (#31832) --- .gitlab/pkg_metrics/pkg_metrics.yml | 3 +- tasks/libs/package/size.py | 72 +++++++-------- tasks/libs/package/utils.py | 30 +++++++ tasks/package.py | 37 ++++++-- tasks/unit_tests/package_lib_tests.py | 87 ++++++++++--------- tasks/unit_tests/package_tests.py | 24 ++--- .../testdata/package_sizes_real.json | 2 +- 7 files changed, 153 insertions(+), 102 deletions(-) diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index e8616c61247a2..ceed774b62f41 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -205,6 +205,5 @@ new_check_pkg_size: - agent_rpm-arm64-a7 - iot_agent_rpm-arm64 script: - - ls -l $OMNIBUS_PACKAGE_DIR - - ls -l $OMNIBUS_PACKAGE_DIR_SUSE + - !reference [.setup_agent_github_app] - inv package.check-size diff --git a/tasks/libs/package/size.py b/tasks/libs/package/size.py index cae0353453f7d..dad67823d3223 100644 --- a/tasks/libs/package/size.py +++ b/tasks/libs/package/size.py @@ -3,11 +3,9 @@ import tempfile from datetime import datetime -from invoke import Exit - from tasks.libs.common.color import Color, color_message from tasks.libs.common.constants import ORIGIN_CATEGORY, ORIGIN_PRODUCT, ORIGIN_SERVICE -from tasks.libs.common.git import get_common_ancestor, get_current_branch, get_default_branch +from tasks.libs.common.git import get_default_branch from tasks.libs.common.utils import get_metric_origin from tasks.libs.package.utils import get_package_path @@ -36,24 +34,25 @@ }, } +# The below template contains the relative increase threshold for each package type PACKAGE_SIZE_TEMPLATE = { 'amd64': { - 'datadog-agent': {'deb': 140000000}, - 'datadog-iot-agent': {'deb': 10000000}, - 'datadog-dogstatsd': {'deb': 10000000}, - 'datadog-heroku-agent': {'deb': 70000000}, + 'datadog-agent': {'deb': 140 * pow(10, 6)}, + 'datadog-iot-agent': {'deb': 10 * pow(10, 6)}, + 'datadog-dogstatsd': {'deb': 10 * pow(10, 6)}, + 'datadog-heroku-agent': {'deb': 70 * pow(10, 6)}, }, 'x86_64': { - 'datadog-agent': {'rpm': 140000000, 'suse': 140000000}, - 'datadog-iot-agent': {'rpm': 10000000, 'suse': 10000000}, - 'datadog-dogstatsd': {'rpm': 10000000, 'suse': 10000000}, + 'datadog-agent': {'rpm': 140 * pow(10, 6), 'suse': 140 * pow(10, 6)}, + 'datadog-iot-agent': {'rpm': 10 * pow(10, 6), 'suse': 10 * pow(10, 6)}, + 'datadog-dogstatsd': {'rpm': 10 * pow(10, 6), 'suse': 10 * pow(10, 6)}, }, 'arm64': { - 'datadog-agent': {'deb': 140000000}, - 'datadog-iot-agent': {'deb': 10000000}, - 'datadog-dogstatsd': {'deb': 10000000}, + 'datadog-agent': {'deb': 140 * pow(10, 6)}, + 'datadog-iot-agent': {'deb': 10 * pow(10, 6)}, + 'datadog-dogstatsd': {'deb': 10 * pow(10, 6)}, }, - 'aarch64': {'datadog-agent': {'rpm': 140000000}, 'datadog-iot-agent': {'rpm': 10000000}}, + 'aarch64': {'datadog-agent': {'rpm': 140 * pow(10, 6)}, 'datadog-iot-agent': {'rpm': 10 * pow(10, 6)}}, } @@ -159,11 +158,10 @@ def compute_package_size_metrics( return series -def compare(ctx, package_sizes, arch, flavor, os_name, threshold): +def compare(ctx, package_sizes, ancestor, arch, flavor, os_name, threshold): """ Compare (or update) a package size with the ancestor package size. """ - mb = 1000000 if os_name == 'suse': dir = os.environ['OMNIBUS_PACKAGE_DIR_SUSE'] path = f'{dir}/{flavor}-7*{arch}.rpm' @@ -172,40 +170,25 @@ def compare(ctx, package_sizes, arch, flavor, os_name, threshold): separator = '_' if os_name == 'deb' else '-' path = f'{dir}/{flavor}{separator}7*{arch}.{os_name}' package_size = _get_uncompressed_size(ctx, get_package_path(path), os_name) - branch = get_current_branch(ctx) - ancestor = get_common_ancestor(ctx, branch) - if branch == get_default_branch(): + if os.environ['CI_COMMIT_REF_NAME'] == get_default_branch(): package_sizes[ancestor][arch][flavor][os_name] = package_size return - previous_size = get_previous_size(package_sizes, ancestor, arch, flavor, os_name) + previous_size = package_sizes[ancestor][arch][flavor][os_name] diff = package_size - previous_size - # For printing purposes - new_package_size_mb = package_size / mb - stable_package_size_mb = previous_size / mb - threshold_mb = threshold / mb - diff_mb = diff / mb - message = f"""{flavor}-{arch}-{os_name} size increase is OK: - New package size is {new_package_size_mb:.2f}MB - Ancestor package ({ancestor}) size is {stable_package_size_mb:.2f}MB - Diff is {diff_mb:.2f}MB (max allowed diff: {threshold_mb:.2f}MB)""" + message = f"{flavor}-{arch}-{os_name} size {mb(package_size)} is OK: {mb(diff)} diff with previous {mb(previous_size)} (max: {mb(threshold)})" if diff > threshold: + emoji = "❌" print(color_message(message.replace('OK', 'too large'), Color.RED), file=sys.stderr) - raise Exit(code=1) - - print(message) + else: + emoji = "✅" if diff <= 0 else "⚠️" + print(message) + return f"|{flavor}-{arch}-{os_name}|{mb(diff)}|{emoji}|{mb(package_size)}|{mb(previous_size)}|{mb(threshold)}|" -def get_previous_size(package_sizes, ancestor, arch, flavor, os_name): - """ - Get the size of the package for the given ancestor, or the earliest ancestor if the given ancestor is not found. - """ - if ancestor in package_sizes: - commit = ancestor - else: - commit = min(package_sizes, key=lambda x: package_sizes[x]['timestamp']) - return package_sizes[commit][arch][flavor][os_name] +def mb(value): + return f"{value / 1000000:.2f}MB" def _get_uncompressed_size(ctx, package, os_name): @@ -218,8 +201,11 @@ def _get_uncompressed_size(ctx, package, os_name): def _get_deb_uncompressed_size(ctx, package): # the size returned by dpkg is a number of bytes divided by 1024 # so we multiply it back to get the same unit as RPM or stat - return int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs').stdout) * 1024 + return ( + int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs', hide=True).stdout) + * 1024 + ) def _get_rpm_uncompressed_size(ctx, package): - return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs').stdout) + return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs', hide=True).stdout) diff --git a/tasks/libs/package/utils.py b/tasks/libs/package/utils.py index ba5448ad666c7..298901732c33e 100644 --- a/tasks/libs/package/utils.py +++ b/tasks/libs/package/utils.py @@ -3,7 +3,9 @@ from invoke import Exit, UnexpectedExit +from tasks.github_tasks import pr_commenter from tasks.libs.common.color import color_message +from tasks.libs.common.git import get_common_ancestor from tasks.libs.notify.utils import AWS_S3_CP_CMD PACKAGE_SIZE_S3_CI_BUCKET_URL = "s3://dd-ci-artefacts-build-stable/datadog-agent/package_size" @@ -74,3 +76,31 @@ def upload_package_sizes(ctx, package_sizes: dict, package_size_file: str, dista f"{AWS_S3_CP_CMD} {package_size_file} {PACKAGE_SIZE_S3_CI_BUCKET_URL}/{package_size_file}", hide="stdout", ) + + +def get_ancestor(ctx, package_sizes, on_main): + """ + Get the common ancestor between HEAD and the default branch + Return the most recent commit if the ancestor is not found in the package_size file + """ + ancestor = get_common_ancestor(ctx, "HEAD") + if not on_main and ancestor not in package_sizes: + return min(package_sizes, key=lambda x: package_sizes[x]['timestamp']) + return ancestor + + +def display_message(ctx, ancestor, rows, decision): + is_open = '' if "Passed" in decision else ' open' + message = f"""Comparison with [ancestor](https://github.com/DataDog/datadog-agent/commit/{ancestor}) `{ancestor}` + + Diff per package + +|package|diff|status|size|ancestor|threshold| +|--|--|--|--|--|--| +{rows} + + +## Decision +{decision} +""" + pr_commenter(ctx, title="Package size comparison", body=message) diff --git a/tasks/package.py b/tasks/package.py index 4e9a859ae8850..307624be2f2f2 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -4,8 +4,8 @@ from invoke import task from invoke.exceptions import Exit -from tasks.libs.common.color import color_message -from tasks.libs.common.git import get_common_ancestor, get_current_branch, get_default_branch +from tasks.libs.common.color import Color, color_message +from tasks.libs.common.git import get_default_branch from tasks.libs.package.size import ( PACKAGE_SIZE_TEMPLATE, _get_deb_uncompressed_size, @@ -13,25 +13,48 @@ compare, compute_package_size_metrics, ) -from tasks.libs.package.utils import get_package_path, list_packages, retrieve_package_sizes, upload_package_sizes +from tasks.libs.package.utils import ( + display_message, + get_ancestor, + get_package_path, + list_packages, + retrieve_package_sizes, + upload_package_sizes, +) @task def check_size(ctx, filename: str = 'package_sizes.json', dry_run: bool = False): package_sizes = retrieve_package_sizes(ctx, filename, distant=not dry_run) - if get_current_branch(ctx) == get_default_branch(): + on_main = os.environ['CI_COMMIT_REF_NAME'] == get_default_branch() + ancestor = get_ancestor(ctx, package_sizes, on_main) + if on_main: # Initialize to default values - ancestor = get_common_ancestor(ctx, get_default_branch()) if ancestor in package_sizes: # The test already ran on this commit return package_sizes[ancestor] = PACKAGE_SIZE_TEMPLATE package_sizes[ancestor]['timestamp'] = int(datetime.now().timestamp()) # Check size of packages + print( + color_message(f"Checking package sizes from {os.environ['CI_COMMIT_REF_NAME']} against {ancestor}", Color.BLUE) + ) + size_table = "" for package_info in list_packages(PACKAGE_SIZE_TEMPLATE): - compare(ctx, package_sizes, *package_info) - if get_current_branch(ctx) == get_default_branch(): + size_table += f"{compare(ctx, package_sizes, ancestor, *package_info)}\n" + + if on_main: upload_package_sizes(ctx, package_sizes, filename, distant=not dry_run) + else: + if "❌" in size_table: + decision = "❌ Failed" + elif "⚠️" in size_table: + decision = "⚠️ Warning" + else: + decision = "✅ Passed" + display_message(ctx, ancestor, size_table, decision) + if "Failed" in decision: + raise Exit(code=1) @task diff --git a/tasks/unit_tests/package_lib_tests.py b/tasks/unit_tests/package_lib_tests.py index 0ae56501c3439..57816701fd07c 100644 --- a/tasks/unit_tests/package_lib_tests.py +++ b/tasks/unit_tests/package_lib_tests.py @@ -3,7 +3,7 @@ import unittest from unittest.mock import MagicMock, patch -from invoke import Exit, MockContext, Result +from invoke import MockContext, Result from tasks.libs.package.size import ( PACKAGE_SIZE_TEMPLATE, @@ -11,9 +11,8 @@ _get_uncompressed_size, compare, compute_package_size_metrics, - get_previous_size, ) -from tasks.libs.package.utils import list_packages +from tasks.libs.package.utils import get_ancestor, list_packages class TestProduceSizeStats(unittest.TestCase): @@ -135,14 +134,20 @@ def setUp(self) -> None: with open('tasks/unit_tests/testdata/package_sizes.json') as f: self.package_sizes = json.load(f) - def test_is_ancestor(self): - self.assertEqual(get_previous_size(self.package_sizes, "grand_ma", "artdeco", "cherry", 'fibula'), 42) + @patch.dict('os.environ', {'CI_COMMIT_REF_NAME': 'puppet'}) + def test_found_on_dev(self): + c = MockContext(run={'git merge-base HEAD origin/main': Result('grand_ma')}) + self.assertEqual(get_ancestor(c, self.package_sizes, False), "grand_ma") - def test_is_other_ancestor(self): - self.assertEqual(get_previous_size(self.package_sizes, "pa", "artdeco", "cherry", 'fibula'), 3) + @patch.dict('os.environ', {'CI_COMMIT_REF_NAME': 'puppet'}) + def test_not_found_on_dev(self): + c = MockContext(run={'git merge-base HEAD origin/main': Result('grand_pa')}) + self.assertEqual(get_ancestor(c, self.package_sizes, False), "grand_ma") - def test_is_not_ancestor(self): - self.assertEqual(get_previous_size(self.package_sizes, "grandPa", "artdeco", "cherry", 'fibula'), 42) + @patch.dict('os.environ', {'CI_COMMIT_REF_NAME': 'main'}) + def test_on_main(self): + c = MockContext(run={'git merge-base HEAD origin/main': Result('kirk')}) + self.assertEqual(get_ancestor(c, self.package_sizes, True), "kirk") class TestGetUncompressedSize(unittest.TestCase): @@ -170,14 +175,15 @@ def setUp(self) -> None: with open('tasks/unit_tests/testdata/package_sizes.json') as f: self.package_sizes = json.load(f) - @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages'}) + @patch.dict( + 'os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'main'} + ) @patch('builtins.print') def test_on_main(self, mock_print): flavor, arch, os_name = 'datadog-heroku-agent', 'amd64', 'deb' c = MockContext( run={ - 'git rev-parse --abbrev-ref HEAD': Result('main'), - 'git merge-base main origin/main': Result('12345'), + 'git merge-base HEAD origin/main': Result('12345'), f"dpkg-deb --info {self.pkg_root}/{flavor}_7_{arch}.{os_name} | grep Installed-Size | cut -d : -f 2 | xargs": Result( 42 ), @@ -185,65 +191,68 @@ def test_on_main(self, mock_print): ) self.package_sizes['12345'] = PACKAGE_SIZE_TEMPLATE self.assertEqual(self.package_sizes['12345'][arch][flavor][os_name], 70000000) - compare(c, self.package_sizes, arch, flavor, os_name, 2001) + res = compare(c, self.package_sizes, '12345', arch, flavor, os_name, 2001) + self.assertIsNone(res) self.assertEqual(self.package_sizes['12345'][arch][flavor][os_name], 43008) mock_print.assert_not_called() - @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages'}) + @patch.dict( + 'os.environ', + {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'pikachu'}, + ) @patch('builtins.print') - def test_on_branch_ok(self, mock_print): + def test_on_branch_warning(self, mock_print): flavor, arch, os_name = 'datadog-agent', 'aarch64', 'suse' c = MockContext( run={ - 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), - 'git merge-base pikachu origin/main': Result('25'), + 'git merge-base HEAD origin/main': Result('25'), f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.rpm | grep Size | cut -d : -f 2 | xargs": Result(69000000), } ) - compare(c, self.package_sizes, arch, flavor, os_name, 70000000) - mock_print.assert_called_with(f"""{flavor}-{arch}-{os_name} size increase is OK: - New package size is 69.00MB - Ancestor package (25) size is 68.00MB - Diff is 1.00MB (max allowed diff: 70.00MB)""") + res = compare(c, self.package_sizes, '25', arch, flavor, os_name, 70000000) + self.assertEqual(res, "|datadog-agent-aarch64-suse|1.00MB|⚠️|69.00MB|68.00MB|70.00MB|") + mock_print.assert_called_with( + f"{flavor}-{arch}-{os_name} size 69.00MB is OK: 1.00MB diff with previous 68.00MB (max: 70.00MB)" + ) - @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages'}) + @patch.dict( + 'os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'pikachu'} + ) @patch('builtins.print') def test_on_branch_ok_rpm(self, mock_print): flavor, arch, os_name = 'datadog-iot-agent', 'x86_64', 'rpm' c = MockContext( run={ - 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), - 'git merge-base pikachu origin/main': Result('25'), + 'git merge-base HEAD origin/main': Result('25'), f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.{os_name} | grep Size | cut -d : -f 2 | xargs": Result( 69000000 ), } ) - compare(c, self.package_sizes, arch, flavor, os_name, 70000000) - mock_print.assert_called_with(f"""{flavor}-{arch}-{os_name} size increase is OK: - New package size is 69.00MB - Ancestor package (25) size is 78.00MB - Diff is -9.00MB (max allowed diff: 70.00MB)""") + res = compare(c, self.package_sizes, '25', arch, flavor, os_name, 70000000) + self.assertEqual(res, "|datadog-iot-agent-x86_64-rpm|-9.00MB|✅|69.00MB|78.00MB|70.00MB|") + mock_print.assert_called_with( + f"{flavor}-{arch}-{os_name} size 69.00MB is OK: -9.00MB diff with previous 78.00MB (max: 70.00MB)" + ) - @patch.dict('os.environ', {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages'}) + @patch.dict( + 'os.environ', + {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'pikachu'}, + ) @patch('builtins.print') def test_on_branch_ko(self, mock_print): flavor, arch, os_name = 'datadog-agent', 'aarch64', 'suse' c = MockContext( run={ - 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), - 'git merge-base pikachu origin/main': Result('25'), + 'git merge-base HEAD origin/main': Result('25'), f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.rpm | grep Size | cut -d : -f 2 | xargs": Result( 139000000 ), } ) - with self.assertRaises(Exit): - compare(c, self.package_sizes, arch, flavor, os_name, 70000000) + res = compare(c, self.package_sizes, '25', arch, flavor, os_name, 70000000) + self.assertEqual(res, "|datadog-agent-aarch64-suse|71.00MB|❌|139.00MB|68.00MB|70.00MB|") mock_print.assert_called_with( - """\x1b[91mdatadog-agent-aarch64-suse size increase is too large: - New package size is 139.00MB - Ancestor package (25) size is 68.00MB - Diff is 71.00MB (max allowed diff: 70.00MB)\x1b[0m""", + "\x1b[91mdatadog-agent-aarch64-suse size 139.00MB is too large: 71.00MB diff with previous 68.00MB (max: 70.00MB)\x1b[0m", file=sys.stderr, ) diff --git a/tasks/unit_tests/package_tests.py b/tasks/unit_tests/package_tests.py index 912c1ce53a912..b1698695b8e5e 100644 --- a/tasks/unit_tests/package_tests.py +++ b/tasks/unit_tests/package_tests.py @@ -13,15 +13,16 @@ class TestCheckSize(unittest.TestCase): { 'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', 'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', + 'CI_COMMIT_REF_NAME': 'pikachu', }, ) @patch('tasks.libs.package.size.get_package_path', new=MagicMock(return_value='datadog-agent')) + @patch('tasks.package.display_message', new=MagicMock()) def test_dev_branch_ko(self): flavor = 'datadog-agent' c = MockContext( run={ - 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), - 'git merge-base pikachu origin/main': Result('25'), + 'git merge-base HEAD origin/main': Result('25'), f"dpkg-deb --info {flavor} | grep Installed-Size | cut -d : -f 2 | xargs": Result(42), f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(69000000), } @@ -35,38 +36,41 @@ def test_dev_branch_ko(self): { 'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', 'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', + 'CI_COMMIT_REF_NAME': 'pikachu', }, ) @patch('tasks.libs.package.size.get_package_path', new=MagicMock(return_value='datadog-agent')) - def test_dev_branch_ok(self, print_mock): + @patch('tasks.package.display_message', new=MagicMock()) + @patch('tasks.package.upload_package_sizes') + def test_dev_branch_ok(self, upload_mock, print_mock): flavor = 'datadog-agent' c = MockContext( run={ - 'git rev-parse --abbrev-ref HEAD': Result('pikachu'), - 'git merge-base pikachu origin/main': Result('25'), + 'git merge-base HEAD origin/main': Result('25'), f"dpkg-deb --info {flavor} | grep Installed-Size | cut -d : -f 2 | xargs": Result(42), f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(20000000), } ) check_size(c, filename='tasks/unit_tests/testdata/package_sizes_real.json', dry_run=True) print_mock.assert_called() - self.assertEqual(print_mock.call_count, 15) + self.assertEqual(print_mock.call_count, 16) + upload_mock.assert_not_called() - @patch('builtins.print') @patch.dict( 'os.environ', { 'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', 'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', + 'CI_COMMIT_REF_NAME': 'main', }, ) @patch('tasks.libs.package.size.get_package_path', new=MagicMock(return_value='datadog-agent')) - def test_main_branch_ok(self, print_mock): + @patch('tasks.package.display_message', new=MagicMock()) + def test_main_branch_ok(self): flavor = 'datadog-agent' c = MockContext( run={ - 'git rev-parse --abbrev-ref HEAD': Result('main'), - 'git merge-base main origin/main': Result('25'), + 'git merge-base HEAD origin/main': Result('25'), f"dpkg-deb --info {flavor} | grep Installed-Size | cut -d : -f 2 | xargs": Result(42), f"rpm -qip {flavor} | grep Size | cut -d : -f 2 | xargs": Result(20000000), } diff --git a/tasks/unit_tests/testdata/package_sizes_real.json b/tasks/unit_tests/testdata/package_sizes_real.json index e24ed4d2bc98d..39f8858b13e0b 100644 --- a/tasks/unit_tests/testdata/package_sizes_real.json +++ b/tasks/unit_tests/testdata/package_sizes_real.json @@ -1 +1 @@ -{"12345": {"timestamp": 1732804637, "amd64": {"datadog-agent": {"deb": 140000000}, "datadog-iot-agent": {"deb": 10000000}, "datadog-dogstatsd": {"deb": 10000000}, "datadog-heroku-agent": {"deb": 70000000}}, "x86_64": {"datadog-agent": {"rpm": 140000000, "suse": 140000000}, "datadog-iot-agent": {"rpm": 10000000, "suse": 10000000}, "datadog-dogstatsd": {"rpm": 10000000, "suse": 10000000}}, "arm64": {"datadog-agent": {"deb": 140000000}, "datadog-iot-agent": {"deb": 10000000}, "datadog-dogstatsd": {"deb": 10000000}}, "aarch64": {"datadog-agent": {"rpm": 140000000}, "datadog-iot-agent": {"rpm": 10000000}}}} \ No newline at end of file +{"12345": {"timestamp": 1732804637, "amd64": {"datadog-agent": {"deb": 140000000}, "datadog-iot-agent": {"deb": 10000000}, "datadog-dogstatsd": {"deb": 10000000}, "datadog-heroku-agent": {"deb": 70000000}}, "x86_64": {"datadog-agent": {"rpm": 140000000, "suse": 140000000}, "datadog-iot-agent": {"rpm": 10000000, "suse": 10000000}, "datadog-dogstatsd": {"rpm": 10000000, "suse": 10000000}}, "arm64": {"datadog-agent": {"deb": 140000000}, "datadog-iot-agent": {"deb": 10000000}, "datadog-dogstatsd": {"deb": 10000000}}, "aarch64": {"datadog-agent": {"rpm": 140000000}, "datadog-iot-agent": {"rpm": 10000000}}}} From c5069b54c0645677da73fcaf0765bbf6c898c8e1 Mon Sep 17 00:00:00 2001 From: Stan Rozenraukh Date: Mon, 9 Dec 2024 12:32:07 -0500 Subject: [PATCH 36/52] Only set INSTALLATION environment variables when doing injection (#31503) Co-authored-by: Rosa Trieu <107086888+rtrieu@users.noreply.github.com> --- .../auto_instrumentation.go | 4 +- .../auto_instrumentation_test.go | 44 ++----------------- ...disabled-no-env-vars-135252f0eb8ddef4.yaml | 5 +++ 3 files changed, 12 insertions(+), 41 deletions(-) create mode 100644 releasenotes-dca/notes/admission-controller-disabled-no-env-vars-135252f0eb8ddef4.yaml diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go index b25289c3a1d31..a3d1aff6b6afa 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go @@ -157,7 +157,6 @@ func (w *Webhook) inject(pod *corev1.Pod, ns string, _ dynamic.Interface) (bool, if pod.Namespace == "" { pod.Namespace = ns } - injectApmTelemetryConfig(pod) if !w.isPodEligible(pod) { return false, nil @@ -371,6 +370,9 @@ func (s libInfoSource) mutatePod(pod *corev1.Pod) error { Name: instrumentationInstallTypeEnvVarName, Value: s.injectionType(), }) + + injectApmTelemetryConfig(pod) + return nil } diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go index 2db44b4acbb6a..97dda98d33174 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go @@ -2307,16 +2307,7 @@ func TestInjectAutoInstrumentation(t *testing.T) { "admission.datadoghq.com/enabled": "false", }, }.Create(), - expectedEnvs: []corev1.EnvVar{ - { - Name: "DD_INSTRUMENTATION_INSTALL_TIME", - Value: installTime, - }, - { - Name: "DD_INSTRUMENTATION_INSTALL_ID", - Value: uuid, - }, - }, + expectedEnvs: nil, expectedInjectedLibraries: map[string]string{}, expectedSecurityContext: &corev1.SecurityContext{}, wantErr: false, @@ -2415,16 +2406,7 @@ func TestInjectAutoInstrumentation(t *testing.T) { ParentKind: "replicaset", ParentName: "test-deployment-123", }.Create(), - expectedEnvs: []corev1.EnvVar{ - { - Name: "DD_INSTRUMENTATION_INSTALL_TIME", - Value: installTime, - }, - { - Name: "DD_INSTRUMENTATION_INSTALL_ID", - Value: uuid, - }, - }, + expectedEnvs: nil, expectedInjectedLibraries: map[string]string{}, expectedSecurityContext: &corev1.SecurityContext{}, wantErr: false, @@ -2495,16 +2477,7 @@ func TestInjectAutoInstrumentation(t *testing.T) { ParentKind: "replicaset", ParentName: "test-deployment-123", }.Create(), - expectedEnvs: []corev1.EnvVar{ - { - Name: "DD_INSTRUMENTATION_INSTALL_TIME", - Value: installTime, - }, - { - Name: "DD_INSTRUMENTATION_INSTALL_ID", - Value: uuid, - }, - }, + expectedEnvs: nil, expectedInjectedLibraries: map[string]string{}, expectedSecurityContext: &corev1.SecurityContext{}, wantErr: false, @@ -2517,16 +2490,7 @@ func TestInjectAutoInstrumentation(t *testing.T) { ParentKind: "replicaset", ParentName: "test-deployment-123", }.Create(), - expectedEnvs: []corev1.EnvVar{ - { - Name: "DD_INSTRUMENTATION_INSTALL_TIME", - Value: installTime, - }, - { - Name: "DD_INSTRUMENTATION_INSTALL_ID", - Value: uuid, - }, - }, + expectedEnvs: nil, expectedInjectedLibraries: map[string]string{}, expectedSecurityContext: &corev1.SecurityContext{}, wantErr: false, diff --git a/releasenotes-dca/notes/admission-controller-disabled-no-env-vars-135252f0eb8ddef4.yaml b/releasenotes-dca/notes/admission-controller-disabled-no-env-vars-135252f0eb8ddef4.yaml new file mode 100644 index 0000000000000..763bb18a91d83 --- /dev/null +++ b/releasenotes-dca/notes/admission-controller-disabled-no-env-vars-135252f0eb8ddef4.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + The auto-instrumentation webhook no longer injects the default environment + variables when disabled. From b2fbed91df80d72c08950b8433e6981c3bc660b0 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Mon, 9 Dec 2024 19:08:39 +0100 Subject: [PATCH 37/52] Fix the Create RC PR Workflow (#31892) --- .github/workflows/create_rc_pr.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 874944a0fe95d..5539dc44532b1 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -62,8 +62,6 @@ jobs: - name: Checkout release branch uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: - ref: ${{ matrix.value }} - fetch-depth: 0 persist-credentials: true - name: Install python From 426d979ee9ba40fe4cb65a28067a4d4fdcf022c1 Mon Sep 17 00:00:00 2001 From: Adel Haj Hassan <41540817+adel121@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:49:12 +0100 Subject: [PATCH 38/52] [CONTP-521] use a hybrid health check for wlm kubeapiserver collector (#31876) --- .../core/healthprobe/impl/healthprobe_test.go | 3 ++- .../internal/kubeapiserver/kubeapiserver.go | 2 +- pkg/status/health/global.go | 16 +++++++------ pkg/status/health/health.go | 23 ++++++++----------- pkg/status/health/health_test.go | 6 ++--- pkg/status/health/options.go | 15 ++++++++++++ 6 files changed, 40 insertions(+), 25 deletions(-) create mode 100644 pkg/status/health/options.go diff --git a/comp/core/healthprobe/impl/healthprobe_test.go b/comp/core/healthprobe/impl/healthprobe_test.go index 0dc9b5d271bad..7774e42043765 100644 --- a/comp/core/healthprobe/impl/healthprobe_test.go +++ b/comp/core/healthprobe/impl/healthprobe_test.go @@ -13,11 +13,12 @@ import ( "net/http/httptest" "testing" + "github.com/stretchr/testify/assert" + healthprobeComponent "github.com/DataDog/datadog-agent/comp/core/healthprobe/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" compdef "github.com/DataDog/datadog-agent/comp/def" "github.com/DataDog/datadog-agent/pkg/status/health" - "github.com/stretchr/testify/assert" ) func TestServer(t *testing.T) { diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go index 0666db1755b75..7def82fc21d9b 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go @@ -224,7 +224,7 @@ func runStartupCheck(ctx context.Context, stores []*reflectorStore) { // There is no way to ensure liveness correctly as it would need to be plugged inside the // inner loop of Reflector. // However, we add Startup when we got at least some data. - startupHealthCheck := health.RegisterStartup(componentName) + startupHealthCheck := health.RegisterReadiness(componentName, health.Once) // Checked synced, in its own scope to cleanly un-reference the syncTimer { diff --git a/pkg/status/health/global.go b/pkg/status/health/global.go index 32af425cbe111..9f6c031f22128 100644 --- a/pkg/status/health/global.go +++ b/pkg/status/health/global.go @@ -12,21 +12,23 @@ import ( var readinessAndLivenessCatalog = newCatalog() var readinessOnlyCatalog = newCatalog() -var startupOnlyCatalog = newStartupCatalog() +var startupOnlyCatalog = newCatalog() // RegisterReadiness registers a component for readiness check with the default 30 seconds timeout, returns a token -func RegisterReadiness(name string) *Handle { - return readinessOnlyCatalog.register(name) +func RegisterReadiness(name string, options ...Option) *Handle { + return readinessOnlyCatalog.register(name, options...) } // RegisterLiveness registers a component for liveness check with the default 30 seconds timeout, returns a token -func RegisterLiveness(name string) *Handle { - return readinessAndLivenessCatalog.register(name) +func RegisterLiveness(name string, options ...Option) *Handle { + return readinessAndLivenessCatalog.register(name, options...) } // RegisterStartup registers a component for startup check, returns a token -func RegisterStartup(name string) *Handle { - return startupOnlyCatalog.register(name) +func RegisterStartup(name string, options ...Option) *Handle { + // Startup health checks are registered with Once option because, by design, they should stop being checked + // once they are marked as healthy once + return startupOnlyCatalog.register(name, append(options, Once)...) } // Deregister a component from the healthcheck diff --git a/pkg/status/health/health.go b/pkg/status/health/health.go index 7e3e3327bb794..0b1a0e8a69c96 100644 --- a/pkg/status/health/health.go +++ b/pkg/status/health/health.go @@ -29,33 +29,25 @@ type component struct { name string healthChan chan time.Time healthy bool + // if set to true, once the check is healthy, we mark it as healthy forever and we stop checking it + once bool } type catalog struct { sync.RWMutex components map[*Handle]*component latestRun time.Time - startup bool } func newCatalog() *catalog { return &catalog{ components: make(map[*Handle]*component), latestRun: time.Now(), // Start healthy - startup: false, - } -} - -func newStartupCatalog() *catalog { - return &catalog{ - components: make(map[*Handle]*component), - latestRun: time.Now(), // Start healthy - startup: true, } } // register a component with the default 30 seconds timeout, returns a token -func (c *catalog) register(name string) *Handle { +func (c *catalog) register(name string, options ...Option) *Handle { c.Lock() defer c.Unlock() @@ -68,6 +60,11 @@ func (c *catalog) register(name string) *Handle { healthChan: make(chan time.Time, bufferSize), healthy: false, } + + for _, option := range options { + option(component) + } + h := &Handle{ C: component.healthChan, } @@ -107,8 +104,8 @@ func (c *catalog) pingComponents(healthDeadline time.Time) bool { c.Lock() defer c.Unlock() for _, component := range c.components { - // In startup mode, we skip already healthy components. - if c.startup && component.healthy { + // We skip components that are registered to be skipped once they pass once + if component.healthy && component.once { continue } select { diff --git a/pkg/status/health/health_test.go b/pkg/status/health/health_test.go index f1d934fb157e5..37a57773a1d73 100644 --- a/pkg/status/health/health_test.go +++ b/pkg/status/health/health_test.go @@ -124,9 +124,9 @@ func TestGetHealthy(t *testing.T) { assert.Len(t, status.Unhealthy, 0) } -func TestStartupCatalog(t *testing.T) { - cat := newStartupCatalog() - token := cat.register("test1") +func TestCatalogWithOnceComponent(t *testing.T) { + cat := newCatalog() + token := cat.register("test1", Once) // Start unhealthy status := cat.getStatus() diff --git a/pkg/status/health/options.go b/pkg/status/health/options.go new file mode 100644 index 0000000000000..c2182a6a3d485 --- /dev/null +++ b/pkg/status/health/options.go @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package health implements the internal healthcheck +package health + +// Option represents the application of an option to a component's health check +type Option func(*component) + +// Once has the effect of not checking the health of a component once it has been marked healthy once +func Once(c *component) { + c.once = true +} From cefb01ce6f8d35672b2fca8dc3cd8939fc1dff0b Mon Sep 17 00:00:00 2001 From: sabrina lu Date: Mon, 9 Dec 2024 14:01:49 -0500 Subject: [PATCH 39/52] add agent onboarding as codeowners to install related files (#31893) --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4f6fc8357ea4e..cd6fe7ade112c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -25,7 +25,7 @@ /CHANGELOG.rst @DataDog/agent-delivery /CHANGELOG-DCA.rst @DataDog/container-integrations @DataDog/container-platform -/CHANGELOG-INSTALLSCRIPT.rst @DataDog/agent-delivery +/CHANGELOG-INSTALLSCRIPT.rst @DataDog/agent-delivery @DataDog/container-ecosystems /*.md @DataDog/agent-devx-infra @DataDog/documentation /NOTICE @DataDog/agent-delivery @DataDog/documentation @@ -91,7 +91,7 @@ /.gitlab/binary_build/include.yml @DataDog/agent-devx-infra /.gitlab/binary_build/linux.yml @DataDog/agent-devx-infra @DataDog/agent-delivery /.gitlab/functional_test/include.yml @DataDog/agent-devx-infra -/.gitlab/install_script_testing/install_script_testing.yml @DataDog/agent-delivery +/.gitlab/install_script_testing/install_script_testing.yml @DataDog/agent-delivery @DataDog/container-ecosystems /.gitlab/integration_test/dogstatsd.yml @DataDog/agent-devx-infra @DataDog/agent-metrics-logs /.gitlab/integration_test/include.yml @DataDog/agent-devx-infra /.gitlab/integration_test/linux.yml @DataDog/agent-devx-infra From e08947533be88e02dba2fcde55055d5495bff9d9 Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Mon, 9 Dec 2024 20:42:01 +0100 Subject: [PATCH 40/52] [CWS] remove watch dir option from directory provider (#31884) Co-authored-by: Sandra (neko) <165049174+neko-dd@users.noreply.github.com> --- .../subcommands/runtime/command.go | 4 +- .../subcommands/runtime/command.go | 4 +- pkg/config/setup/system_probe_cws.go | 1 - pkg/security/config/config.go | 3 - pkg/security/rules/engine.go | 2 +- pkg/security/secl/go.mod | 1 - pkg/security/secl/go.sum | 2 - pkg/security/secl/rules/policy_dir.go | 98 +------------------ pkg/security/secl/rules/policy_test.go | 12 +-- pkg/security/tests/module_tester.go | 2 +- ...ove-watch-dir-option-f8faee4937353316.yaml | 13 +++ 11 files changed, 29 insertions(+), 113 deletions(-) create mode 100644 releasenotes/notes/cws-remove-watch-dir-option-f8faee4937353316.yaml diff --git a/cmd/security-agent/subcommands/runtime/command.go b/cmd/security-agent/subcommands/runtime/command.go index 0b32529c60d36..08bbdcea4c787 100644 --- a/cmd/security-agent/subcommands/runtime/command.go +++ b/cmd/security-agent/subcommands/runtime/command.go @@ -472,7 +472,7 @@ func checkPoliciesLocal(args *checkPoliciesCliParams, writer io.Writer) error { }, } - provider, err := rules.NewPoliciesDirProvider(args.dir, false) + provider, err := rules.NewPoliciesDirProvider(args.dir) if err != nil { return err } @@ -611,7 +611,7 @@ func evalRule(_ log.Component, _ config.Component, _ secrets.Component, evalArgs }, } - provider, err := rules.NewPoliciesDirProvider(policiesDir, false) + provider, err := rules.NewPoliciesDirProvider(policiesDir) if err != nil { return err } diff --git a/cmd/system-probe/subcommands/runtime/command.go b/cmd/system-probe/subcommands/runtime/command.go index e6030500a9b17..e0d17e16d0a2e 100644 --- a/cmd/system-probe/subcommands/runtime/command.go +++ b/cmd/system-probe/subcommands/runtime/command.go @@ -466,7 +466,7 @@ func checkPoliciesLocal(args *checkPoliciesCliParams, writer io.Writer) error { }, } - provider, err := rules.NewPoliciesDirProvider(args.dir, false) + provider, err := rules.NewPoliciesDirProvider(args.dir) if err != nil { return err } @@ -583,7 +583,7 @@ func evalRule(_ log.Component, _ config.Component, _ secrets.Component, evalArgs }, } - provider, err := rules.NewPoliciesDirProvider(policiesDir, false) + provider, err := rules.NewPoliciesDirProvider(policiesDir) if err != nil { return err } diff --git a/pkg/config/setup/system_probe_cws.go b/pkg/config/setup/system_probe_cws.go index b72d72df3c971..d2cc6276907d3 100644 --- a/pkg/config/setup/system_probe_cws.go +++ b/pkg/config/setup/system_probe_cws.go @@ -20,7 +20,6 @@ func initCWSSystemProbeConfig(cfg pkgconfigmodel.Config) { // CWS - general config cfg.BindEnvAndSetDefault("runtime_security_config.enabled", false) cfg.BindEnv("runtime_security_config.fim_enabled") - cfg.BindEnvAndSetDefault("runtime_security_config.policies.watch_dir", false) cfg.BindEnvAndSetDefault("runtime_security_config.policies.monitor.enabled", false) cfg.BindEnvAndSetDefault("runtime_security_config.policies.monitor.per_rule_enabled", false) cfg.BindEnvAndSetDefault("runtime_security_config.policies.monitor.report_internal_policies", false) diff --git a/pkg/security/config/config.go b/pkg/security/config/config.go index fc9cbcecc6a4b..ab6b0b6b591e1 100644 --- a/pkg/security/config/config.go +++ b/pkg/security/config/config.go @@ -45,8 +45,6 @@ type RuntimeSecurityConfig struct { RuntimeEnabled bool // PoliciesDir defines the folder in which the policy files are located PoliciesDir string - // WatchPoliciesDir activate policy dir inotify - WatchPoliciesDir bool // PolicyMonitorEnabled enable policy monitoring PolicyMonitorEnabled bool // PolicyMonitorPerRuleEnabled enabled per-rule policy monitoring @@ -353,7 +351,6 @@ func NewRuntimeSecurityConfig() (*RuntimeSecurityConfig, error) { // policy & ruleset PoliciesDir: pkgconfigsetup.SystemProbe().GetString("runtime_security_config.policies.dir"), - WatchPoliciesDir: pkgconfigsetup.SystemProbe().GetBool("runtime_security_config.policies.watch_dir"), PolicyMonitorEnabled: pkgconfigsetup.SystemProbe().GetBool("runtime_security_config.policies.monitor.enabled"), PolicyMonitorPerRuleEnabled: pkgconfigsetup.SystemProbe().GetBool("runtime_security_config.policies.monitor.per_rule_enabled"), PolicyMonitorReportInternalPolicies: pkgconfigsetup.SystemProbe().GetBool("runtime_security_config.policies.monitor.report_internal_policies"), diff --git a/pkg/security/rules/engine.go b/pkg/security/rules/engine.go index 6b0f98fd15666..ffb73244b6c2d 100644 --- a/pkg/security/rules/engine.go +++ b/pkg/security/rules/engine.go @@ -377,7 +377,7 @@ func (e *RuleEngine) gatherDefaultPolicyProviders() []rules.PolicyProvider { } // directory policy provider - if provider, err := rules.NewPoliciesDirProvider(e.config.PoliciesDir, e.config.WatchPoliciesDir); err != nil { + if provider, err := rules.NewPoliciesDirProvider(e.config.PoliciesDir); err != nil { seclog.Errorf("failed to load local policies: %s", err) } else { policyProviders = append(policyProviders, provider) diff --git a/pkg/security/secl/go.mod b/pkg/security/secl/go.mod index 9c3d225dcdad1..e5f0a3c820483 100644 --- a/pkg/security/secl/go.mod +++ b/pkg/security/secl/go.mod @@ -8,7 +8,6 @@ require ( github.com/alecthomas/participle v0.7.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/fatih/structtag v1.2.0 - github.com/fsnotify/fsnotify v1.8.0 github.com/google/go-cmp v0.6.0 github.com/google/gopacket v1.1.19 github.com/hashicorp/go-multierror v1.1.1 diff --git a/pkg/security/secl/go.sum b/pkg/security/secl/go.sum index 18391dc2e1b6e..0c694b56983cf 100644 --- a/pkg/security/secl/go.sum +++ b/pkg/security/secl/go.sum @@ -17,8 +17,6 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/pkg/security/secl/rules/policy_dir.go b/pkg/security/secl/rules/policy_dir.go index 119f31a40f461..f1c31ca699b46 100644 --- a/pkg/security/secl/rules/policy_dir.go +++ b/pkg/security/secl/rules/policy_dir.go @@ -7,12 +7,10 @@ package rules import ( - "context" "os" "path/filepath" "sort" - "github.com/fsnotify/fsnotify" "github.com/hashicorp/go-multierror" ) @@ -25,17 +23,10 @@ var _ PolicyProvider = (*PoliciesDirProvider)(nil) // PoliciesDirProvider defines a new policy dir provider type PoliciesDirProvider struct { PoliciesDir string - - onNewPoliciesReadyCb func() - cancelFnc func() - watcher *fsnotify.Watcher - watchedFiles []string } // SetOnNewPoliciesReadyCb implements the policy provider interface -func (p *PoliciesDirProvider) SetOnNewPoliciesReadyCb(cb func()) { - p.onNewPoliciesReadyCb = cb -} +func (p *PoliciesDirProvider) SetOnNewPoliciesReadyCb(_ func()) {} // Start starts the policy dir provider func (p *PoliciesDirProvider) Start() {} @@ -92,14 +83,6 @@ func (p *PoliciesDirProvider) LoadPolicies(macroFilters []MacroFilter, ruleFilte errs = multierror.Append(errs, err) } - // remove oldest watched files - if p.watcher != nil { - for _, watched := range p.watchedFiles { - _ = p.watcher.Remove(watched) - } - p.watchedFiles = p.watchedFiles[0:0] - } - // Load and parse policies for _, filename := range policyFiles { policy, err := p.loadPolicy(filename, macroFilters, ruleFilters) @@ -112,14 +95,6 @@ func (p *PoliciesDirProvider) LoadPolicies(macroFilters []MacroFilter, ruleFilte } policies = append(policies, policy) - - if p.watcher != nil { - if err := p.watcher.Add(filename); err != nil { - errs = multierror.Append(errs, err) - } else { - p.watchedFiles = append(p.watchedFiles, filename) - } - } } return policies, errs @@ -127,79 +102,14 @@ func (p *PoliciesDirProvider) LoadPolicies(macroFilters []MacroFilter, ruleFilte // Close stops policy provider interface func (p *PoliciesDirProvider) Close() error { - if p.cancelFnc != nil { - p.cancelFnc() - } - - if p.watcher != nil { - p.watcher.Close() - } return nil } -func filesEqual(a []string, b []string) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} - -func (p *PoliciesDirProvider) watch(ctx context.Context) { - go func() { - for { - select { - case <-ctx.Done(): - return - case event, ok := <-p.watcher.Events: - if !ok { - return - } - - if event.Op&(fsnotify.Create|fsnotify.Remove) > 0 { - files, _ := p.getPolicyFiles() - if !filesEqual(files, p.watchedFiles) { - p.onNewPoliciesReadyCb() - } - } else if event.Op&fsnotify.Write > 0 && filepath.Ext(event.Name) == policyExtension { - p.onNewPoliciesReadyCb() - } - case _, ok := <-p.watcher.Errors: - if !ok { - return - } - } - } - }() -} - // NewPoliciesDirProvider returns providers for the given policies dir -func NewPoliciesDirProvider(policiesDir string, watch bool) (*PoliciesDirProvider, error) { - p := &PoliciesDirProvider{ +func NewPoliciesDirProvider(policiesDir string) (*PoliciesDirProvider, error) { + return &PoliciesDirProvider{ PoliciesDir: policiesDir, - } - - if watch { - var err error - if p.watcher, err = fsnotify.NewWatcher(); err != nil { - return nil, err - } - - if err := p.watcher.Add(policiesDir); err != nil { - p.watcher.Close() - return nil, err - } - - var ctx context.Context - ctx, p.cancelFnc = context.WithCancel(context.Background()) - go p.watch(ctx) - } - - return p, nil + }, nil } // Type returns the type of policy dir provider diff --git a/pkg/security/secl/rules/policy_test.go b/pkg/security/secl/rules/policy_test.go index e6252c10593f1..da1fda9740860 100644 --- a/pkg/security/secl/rules/policy_test.go +++ b/pkg/security/secl/rules/policy_test.go @@ -75,7 +75,7 @@ func TestMacroMerge(t *testing.T) { event.SetFieldValue("open.file.path", "/tmp/test") event.SetFieldValue("process.comm", "/usr/bin/vi") - provider, err := NewPoliciesDirProvider(tmpDir, false) + provider, err := NewPoliciesDirProvider(tmpDir) if err != nil { t.Fatal(err) } @@ -150,7 +150,7 @@ func TestRuleMerge(t *testing.T) { t.Fatal(err) } - provider, err := NewPoliciesDirProvider(tmpDir, false) + provider, err := NewPoliciesDirProvider(tmpDir) if err != nil { t.Fatal(err) } @@ -284,7 +284,7 @@ func TestActionSetVariable(t *testing.T) { t.Fatal(err) } - provider, err := NewPoliciesDirProvider(tmpDir, false) + provider, err := NewPoliciesDirProvider(tmpDir) if err != nil { t.Fatal(err) } @@ -354,7 +354,7 @@ func TestActionSetVariableTTL(t *testing.T) { t.Fatal(err) } - provider, err := NewPoliciesDirProvider(tmpDir, false) + provider, err := NewPoliciesDirProvider(tmpDir) if err != nil { t.Fatal(err) } @@ -420,7 +420,7 @@ func TestActionSetVariableConflict(t *testing.T) { t.Fatal(err) } - provider, err := NewPoliciesDirProvider(tmpDir, false) + provider, err := NewPoliciesDirProvider(tmpDir) if err != nil { t.Fatal(err) } @@ -441,7 +441,7 @@ func loadPolicy(t *testing.T, testPolicy *PolicyDef, policyOpts PolicyLoaderOpts t.Fatal(err) } - provider, err := NewPoliciesDirProvider(tmpDir, false) + provider, err := NewPoliciesDirProvider(tmpDir) if err != nil { t.Fatal(err) } diff --git a/pkg/security/tests/module_tester.go b/pkg/security/tests/module_tester.go index 22711615cc70f..0b01cbcd72d49 100644 --- a/pkg/security/tests/module_tester.go +++ b/pkg/security/tests/module_tester.go @@ -118,7 +118,7 @@ func (tm *testModule) reloadPolicies() error { log.Debugf("reload policies with cfgDir: %s", commonCfgDir) bundledPolicyProvider := bundled.NewPolicyProvider(tm.eventMonitor.Probe.Config.RuntimeSecurity) - policyDirProvider, err := rules.NewPoliciesDirProvider(commonCfgDir, false) + policyDirProvider, err := rules.NewPoliciesDirProvider(commonCfgDir) if err != nil { return err } diff --git a/releasenotes/notes/cws-remove-watch-dir-option-f8faee4937353316.yaml b/releasenotes/notes/cws-remove-watch-dir-option-f8faee4937353316.yaml new file mode 100644 index 0000000000000..320f7e164f786 --- /dev/null +++ b/releasenotes/notes/cws-remove-watch-dir-option-f8faee4937353316.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +deprecations: + - | + CWS: the `runtime_security_config.policies.watch_dir` option has been removed. + Use remote configuration for dynamically updating policies, or send + the `SIGHUP` signal to the `system-probe` process to reload the policies. From 4f0073dbe2585abdc01c50d1188c3609c6bcf3f9 Mon Sep 17 00:00:00 2001 From: Jack Phillips Date: Mon, 9 Dec 2024 16:02:16 -0500 Subject: [PATCH 41/52] Update filesystem permission to restrict access to datadog user (#31770) Co-authored-by: Branden Clark Co-authored-by: Rosa Trieu <107086888+rtrieu@users.noreply.github.com> --- comp/api/authtoken/go.mod | 6 +- comp/core/config/go.mod | 6 +- comp/core/log/impl-trace/go.mod | 6 +- comp/core/log/impl/go.mod | 6 +- comp/core/log/mock/go.mod | 4 +- comp/core/status/statusimpl/go.mod | 6 +- comp/forwarder/defaultforwarder/go.mod | 6 +- .../orchestrator/orchestratorinterface/go.mod | 6 +- comp/logs/agent/config/go.mod | 6 +- comp/otelcol/converter/impl/go.mod | 6 +- comp/otelcol/ddflareextension/impl/go.mod | 6 +- comp/otelcol/logsagentpipeline/go.mod | 6 +- .../logsagentpipelineimpl/go.mod | 6 +- .../exporter/datadogexporter/go.mod | 6 +- .../exporter/logsagentexporter/go.mod | 6 +- .../exporter/serializerexporter/go.mod | 6 +- comp/otelcol/otlp/testutil/go.mod | 6 +- comp/serializer/compression/go.mod | 6 +- go.mod | 6 +- pkg/api/go.mod | 6 +- pkg/config/env/go.mod | 6 +- pkg/config/mock/go.mod | 6 +- pkg/config/remote/go.mod | 6 +- pkg/config/setup/go.mod | 6 +- pkg/config/utils/go.mod | 6 +- pkg/logs/auditor/go.mod | 6 +- pkg/logs/client/go.mod | 6 +- pkg/logs/diagnostic/go.mod | 6 +- pkg/logs/message/go.mod | 6 +- pkg/logs/pipeline/go.mod | 6 +- pkg/logs/processor/go.mod | 6 +- pkg/logs/sds/go.mod | 6 +- pkg/logs/sender/go.mod | 6 +- pkg/logs/sources/go.mod | 6 +- pkg/logs/util/testutils/go.mod | 6 +- pkg/metrics/go.mod | 6 +- pkg/serializer/go.mod | 6 +- pkg/util/filesystem/go.mod | 7 +- pkg/util/filesystem/go.sum | 1 - pkg/util/filesystem/permission_windows.go | 31 +++----- .../filesystem/permission_windows_test.go | 73 +++++++++++++++++++ pkg/util/flavor/go.mod | 6 +- pkg/util/grpc/go.mod | 6 +- pkg/util/http/go.mod | 6 +- pkg/util/log/setup/go.mod | 6 +- pkg/util/system/go.mod | 6 +- pkg/util/winutil/service.go | 17 +++++ pkg/util/winutil/users.go | 51 +++++++++++++ pkg/util/winutil/users_test.go | 26 +++++++ ...ns-to-dduser-windows-6e9940175f9130ff.yaml | 12 +++ test/otel/go.mod | 6 +- 51 files changed, 324 insertions(+), 150 deletions(-) create mode 100644 pkg/util/filesystem/permission_windows_test.go create mode 100644 releasenotes/notes/file-permissions-to-dduser-windows-6e9940175f9130ff.yaml diff --git a/comp/api/authtoken/go.mod b/comp/api/authtoken/go.mod index 532d1be88a5c4..233f326a297d1 100644 --- a/comp/api/authtoken/go.mod +++ b/comp/api/authtoken/go.mod @@ -68,13 +68,13 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/log/setup v0.58.0-devel // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/comp/core/config/go.mod b/comp/core/config/go.mod index 2973aa62895a7..bed15530c21b9 100644 --- a/comp/core/config/go.mod +++ b/comp/core/config/go.mod @@ -42,7 +42,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/defaultpaths v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/DataDog/viper v1.13.5 github.com/stretchr/testify v1.10.0 go.uber.org/fx v1.23.0 @@ -60,9 +60,9 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/comp/core/log/impl-trace/go.mod b/comp/core/log/impl-trace/go.mod index 46dbc7aba09c2..2ee9571d6657a 100644 --- a/comp/core/log/impl-trace/go.mod +++ b/comp/core/log/impl-trace/go.mod @@ -45,7 +45,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/trace v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect; v2.6 github.com/stretchr/testify v1.10.0 go.uber.org/fx v1.23.0 // indirect @@ -72,10 +72,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/comp/core/log/impl/go.mod b/comp/core/log/impl/go.mod index 116cf25e6c584..9d9c721ce2ba8 100644 --- a/comp/core/log/impl/go.mod +++ b/comp/core/log/impl/go.mod @@ -39,7 +39,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/def v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/log/setup v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.10.0 ) @@ -61,10 +61,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/comp/core/log/mock/go.mod b/comp/core/log/mock/go.mod index 96c709d9f2f32..f8515313b702d 100644 --- a/comp/core/log/mock/go.mod +++ b/comp/core/log/mock/go.mod @@ -31,7 +31,7 @@ replace ( require ( github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/log/setup v0.0.0-00010101000000-000000000000 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 ) @@ -40,7 +40,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.60.0-devel // indirect github.com/DataDog/datadog-agent/pkg/config/teeconfig v0.60.0-devel // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect diff --git a/comp/core/status/statusimpl/go.mod b/comp/core/status/statusimpl/go.mod index bc13e9cf4a943..600683ed2cceb 100644 --- a/comp/core/status/statusimpl/go.mod +++ b/comp/core/status/statusimpl/go.mod @@ -70,14 +70,14 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/log/setup v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/comp/forwarder/defaultforwarder/go.mod b/comp/forwarder/defaultforwarder/go.mod index a6e6b5f2297da..55d901ebce30c 100644 --- a/comp/forwarder/defaultforwarder/go.mod +++ b/comp/forwarder/defaultforwarder/go.mod @@ -71,7 +71,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/fxutil v0.57.1 github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.57.1 github.com/golang/protobuf v1.5.4 github.com/hashicorp/go-multierror v1.1.1 @@ -94,12 +94,12 @@ require ( github.com/DataDog/datadog-agent/pkg/config/teeconfig v0.60.0-devel // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/log/setup v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/comp/forwarder/orchestrator/orchestratorinterface/go.mod b/comp/forwarder/orchestrator/orchestratorinterface/go.mod index d5f8d766d32f6..63ef4e9d8c4a8 100644 --- a/comp/forwarder/orchestrator/orchestratorinterface/go.mod +++ b/comp/forwarder/orchestrator/orchestratorinterface/go.mod @@ -94,13 +94,13 @@ require ( github.com/DataDog/datadog-agent/pkg/util/fxutil v0.57.1 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.57.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/comp/logs/agent/config/go.mod b/comp/logs/agent/config/go.mod index 91137e7bcc76c..733a8bf8b17c7 100644 --- a/comp/logs/agent/config/go.mod +++ b/comp/logs/agent/config/go.mod @@ -43,7 +43,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/structure v0.59.0 github.com/DataDog/datadog-agent/pkg/config/utils v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 github.com/DataDog/viper v1.13.5 github.com/stretchr/testify v1.10.0 @@ -64,10 +64,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/comp/otelcol/converter/impl/go.mod b/comp/otelcol/converter/impl/go.mod index b321ff5db70ad..f4c649d45be37 100644 --- a/comp/otelcol/converter/impl/go.mod +++ b/comp/otelcol/converter/impl/go.mod @@ -69,13 +69,13 @@ require ( github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.2 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 6c75776d13338..5f0036a040fb9 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -251,17 +251,17 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/json v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/log/setup v0.58.0-devel // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/sort v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/startstop v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.31.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect diff --git a/comp/otelcol/logsagentpipeline/go.mod b/comp/otelcol/logsagentpipeline/go.mod index 0b8f861795c6c..40a8e60d9efba 100644 --- a/comp/otelcol/logsagentpipeline/go.mod +++ b/comp/otelcol/logsagentpipeline/go.mod @@ -96,15 +96,15 @@ require ( github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/startstop v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect github.com/DataDog/viper v1.13.5 // indirect diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod index ca80a71894cb3..25af1fd191eb1 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod @@ -112,14 +112,14 @@ require ( github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/log/setup v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect github.com/DataDog/viper v1.13.5 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index 109071e12c948..1fbae2bae0cf9 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -186,16 +186,16 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/json v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/sort v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/startstop v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.57.1 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.31.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod index 63514d6b9afdb..b2fc1508016cf 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod @@ -47,7 +47,7 @@ require ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/message v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.21.0 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.21.0 github.com/stormcat24/protodep v0.1.8 @@ -80,13 +80,13 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.31.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.21.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod index 1ca371316a075..f469f0ac3856a 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod @@ -68,7 +68,7 @@ require ( github.com/DataDog/datadog-agent/pkg/proto v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/serializer v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/tagset v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.21.0 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.21.0 github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.21.0 @@ -129,11 +129,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/json v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/sort v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.57.1 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect diff --git a/comp/otelcol/otlp/testutil/go.mod b/comp/otelcol/otlp/testutil/go.mod index 023eadca279c9..08136f64f178e 100644 --- a/comp/otelcol/otlp/testutil/go.mod +++ b/comp/otelcol/otlp/testutil/go.mod @@ -55,13 +55,13 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/comp/serializer/compression/go.mod b/comp/serializer/compression/go.mod index 9e8574fe77f13..7575544de493a 100644 --- a/comp/serializer/compression/go.mod +++ b/comp/serializer/compression/go.mod @@ -36,7 +36,7 @@ replace ( require ( github.com/DataDog/datadog-agent/comp/core/config v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/zstd v1.5.6 ) @@ -58,10 +58,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/go.mod b/go.mod index 2c8473f077cef..c6af83af3b80c 100644 --- a/go.mod +++ b/go.mod @@ -154,9 +154,9 @@ require ( github.com/DataDog/datadog-agent/pkg/security/secl v0.56.0 github.com/DataDog/datadog-agent/pkg/trace v0.59.0 github.com/DataDog/datadog-agent/pkg/util/cgroups v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 github.com/DataDog/datadog-go/v5 v5.5.0 github.com/DataDog/datadog-operator v0.7.1-0.20241024104907-734366f3c0d1 github.com/DataDog/ebpf-manager v0.7.4 @@ -728,7 +728,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 github.com/DataDog/datadog-agent/pkg/util/testutil v0.59.0 github.com/DataDog/datadog-agent/pkg/util/uuid v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.59.0 github.com/DataDog/go-libddwaf/v3 v3.5.1 github.com/DataDog/go-sqllexer v0.0.17 diff --git a/pkg/api/go.mod b/pkg/api/go.mod index b30cc5d081999..28a2032095417 100644 --- a/pkg/api/go.mod +++ b/pkg/api/go.mod @@ -43,7 +43,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/config/utils v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 github.com/stretchr/testify v1.10.0 ) @@ -64,9 +64,9 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/config/env/go.mod b/pkg/config/env/go.mod index 7188be93ca23d..2d26e478fd4e7 100644 --- a/pkg/config/env/go.mod +++ b/pkg/config/env/go.mod @@ -8,18 +8,20 @@ replace ( github.com/DataDog/datadog-agent/pkg/util/log => ../../util/log/ github.com/DataDog/datadog-agent/pkg/util/scrubber => ../../util/scrubber/ github.com/DataDog/datadog-agent/pkg/util/system/socket => ../../util/system/socket/ + github.com/DataDog/datadog-agent/pkg/util/winutil => ../../util/winutil ) require ( github.com/DataDog/datadog-agent/pkg/config/model v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/filesystem v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/system/socket v0.56.0-rc.3 github.com/stretchr/testify v1.10.0 ) require ( - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/pkg/config/mock/go.mod b/pkg/config/mock/go.mod index 041266bf69dff..9fc526fc2c5de 100644 --- a/pkg/config/mock/go.mod +++ b/pkg/config/mock/go.mod @@ -45,13 +45,13 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 5a00e0b18ab42..ab537bc1eb918 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -50,7 +50,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/grpc v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/uuid v0.56.0-rc.3 github.com/Masterminds/semver v1.5.0 github.com/benbjohnson/clock v1.3.5 @@ -83,7 +83,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect @@ -121,7 +121,7 @@ require ( ) require ( - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 github.com/DataDog/viper v1.13.5 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/pkg/config/setup/go.mod b/pkg/config/setup/go.mod index c6855f5379f0f..772c6b0ee8e53 100644 --- a/pkg/config/setup/go.mod +++ b/pkg/config/setup/go.mod @@ -44,11 +44,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/stretchr/testify v1.10.0 go.uber.org/fx v1.23.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/pkg/config/utils/go.mod b/pkg/config/utils/go.mod index 4c39a2d8c2f78..0f4f9537ccb9f 100644 --- a/pkg/config/utils/go.mod +++ b/pkg/config/utils/go.mod @@ -38,7 +38,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/config/setup v0.59.0 github.com/DataDog/datadog-agent/pkg/config/structure v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 github.com/stretchr/testify v1.10.0 ) @@ -54,10 +54,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/pkg/logs/auditor/go.mod b/pkg/logs/auditor/go.mod index 9f000ce69c516..0ea48c9e7895a 100644 --- a/pkg/logs/auditor/go.mod +++ b/pkg/logs/auditor/go.mod @@ -47,7 +47,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/message v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/stretchr/testify v1.10.0 ) @@ -66,11 +66,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/logs/client/go.mod b/pkg/logs/client/go.mod index 9f0ec05367e17..cac2a5ce40f32 100644 --- a/pkg/logs/client/go.mod +++ b/pkg/logs/client/go.mod @@ -60,7 +60,7 @@ require ( github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 github.com/stretchr/testify v1.10.0 golang.org/x/net v0.31.0 @@ -84,11 +84,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/benbjohnson/clock v1.3.5 // indirect diff --git a/pkg/logs/diagnostic/go.mod b/pkg/logs/diagnostic/go.mod index e5350d717dc74..7f3357f60d406 100644 --- a/pkg/logs/diagnostic/go.mod +++ b/pkg/logs/diagnostic/go.mod @@ -67,14 +67,14 @@ require ( github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/logs/message/go.mod b/pkg/logs/message/go.mod index a7acc1d3319f8..b4b0ab2e615ed 100644 --- a/pkg/logs/message/go.mod +++ b/pkg/logs/message/go.mod @@ -42,7 +42,7 @@ replace ( require ( github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/stretchr/testify v1.10.0 ) @@ -62,11 +62,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/logs/pipeline/go.mod b/pkg/logs/pipeline/go.mod index aa3225348e8e9..06f20a79a4199 100644 --- a/pkg/logs/pipeline/go.mod +++ b/pkg/logs/pipeline/go.mod @@ -71,7 +71,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/sender v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/startstop v0.56.0-rc.3 github.com/hashicorp/go-multierror v1.1.1 github.com/stretchr/testify v1.10.0 @@ -100,11 +100,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect github.com/DataDog/viper v1.13.5 // indirect diff --git a/pkg/logs/processor/go.mod b/pkg/logs/processor/go.mod index 42339476936cc..858475c3a8a54 100644 --- a/pkg/logs/processor/go.mod +++ b/pkg/logs/processor/go.mod @@ -57,7 +57,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/metrics v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/sds v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/stretchr/testify v1.10.0 ) @@ -80,11 +80,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect github.com/DataDog/viper v1.13.5 // indirect diff --git a/pkg/logs/sds/go.mod b/pkg/logs/sds/go.mod index b9c7c9706a849..c6f00cb1cf5e7 100644 --- a/pkg/logs/sds/go.mod +++ b/pkg/logs/sds/go.mod @@ -52,7 +52,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/logs/message v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 github.com/stretchr/testify v1.10.0 ) @@ -77,11 +77,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/logs/sender/go.mod b/pkg/logs/sender/go.mod index 085c081f2774e..2f438b2f0cc86 100644 --- a/pkg/logs/sender/go.mod +++ b/pkg/logs/sender/go.mod @@ -59,7 +59,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/benbjohnson/clock v1.3.5 github.com/stretchr/testify v1.10.0 ) @@ -84,11 +84,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/logs/sources/go.mod b/pkg/logs/sources/go.mod index b81f5d609a906..a73c178af0286 100644 --- a/pkg/logs/sources/go.mod +++ b/pkg/logs/sources/go.mod @@ -41,7 +41,7 @@ replace ( require ( github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 github.com/stretchr/testify v1.10.0 ) @@ -61,10 +61,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/logs/util/testutils/go.mod b/pkg/logs/util/testutils/go.mod index d7bbeef948cbb..d2aa1b59f3e82 100644 --- a/pkg/logs/util/testutils/go.mod +++ b/pkg/logs/util/testutils/go.mod @@ -58,14 +58,14 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/metrics/go.mod b/pkg/metrics/go.mod index 2129bb8455c09..198885497f0b9 100644 --- a/pkg/metrics/go.mod +++ b/pkg/metrics/go.mod @@ -46,7 +46,7 @@ require ( github.com/DataDog/datadog-agent/pkg/tagset v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/buf v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.21.0 github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 @@ -67,11 +67,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/sort v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/serializer/go.mod b/pkg/serializer/go.mod index 5e0bd3938c766..4b6353199d174 100644 --- a/pkg/serializer/go.mod +++ b/pkg/serializer/go.mod @@ -77,7 +77,7 @@ require ( github.com/DataDog/datadog-agent/pkg/tagset v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/json v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.57.1 github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.21.0 github.com/gogo/protobuf v1.3.2 @@ -116,11 +116,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/sort v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/sketches-go v1.4.6 // indirect github.com/DataDog/viper v1.13.5 // indirect diff --git a/pkg/util/filesystem/go.mod b/pkg/util/filesystem/go.mod index 1753d7310cddd..d1fbed28429ae 100644 --- a/pkg/util/filesystem/go.mod +++ b/pkg/util/filesystem/go.mod @@ -5,11 +5,13 @@ go 1.22.0 replace ( github.com/DataDog/datadog-agent/pkg/util/log => ../log/ github.com/DataDog/datadog-agent/pkg/util/scrubber => ../scrubber/ + github.com/DataDog/datadog-agent/pkg/util/winutil => ../winutil/ ) require ( - github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 github.com/shirou/gopsutil/v3 v3.24.5 github.com/stretchr/testify v1.10.0 @@ -17,11 +19,10 @@ require ( ) require ( - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/pkg/util/filesystem/go.sum b/pkg/util/filesystem/go.sum index e3cc0b7e589e0..7bde33aa41ad5 100644 --- a/pkg/util/filesystem/go.sum +++ b/pkg/util/filesystem/go.sum @@ -1,6 +1,5 @@ github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= diff --git a/pkg/util/filesystem/permission_windows.go b/pkg/util/filesystem/permission_windows.go index 2ae2929ce1809..43819ee0be97c 100644 --- a/pkg/util/filesystem/permission_windows.go +++ b/pkg/util/filesystem/permission_windows.go @@ -8,15 +8,16 @@ package filesystem import ( "fmt" - "syscall" "github.com/hectane/go-acl" "golang.org/x/sys/windows" + + "github.com/DataDog/datadog-agent/pkg/util/winutil" ) // Permission handles permissions for Unix and Windows type Permission struct { - currentUserSid *windows.SID + ddUserSid *windows.SID administratorSid *windows.SID systemSid *windows.SID } @@ -32,32 +33,24 @@ func NewPermission() (*Permission, error) { return nil, err } - currentUserSid, err := getCurrentUserSid() + ddUserSid, err := getDatadogUserSid() if err != nil { - return nil, fmt.Errorf("Unable to get current user sid %v", err) + return nil, fmt.Errorf("unable to get datadog user sid %v", err) } return &Permission{ - currentUserSid: currentUserSid, + ddUserSid: ddUserSid, administratorSid: administratorSid, systemSid: systemSid, }, nil } -func getCurrentUserSid() (*windows.SID, error) { - token, err := syscall.OpenCurrentProcessToken() - if err != nil { - return nil, fmt.Errorf("Couldn't get process token %v", err) - } - defer token.Close() - user, err := token.GetTokenUser() - if err != nil { - return nil, fmt.Errorf("Couldn't get token user %v", err) - } - sidString, err := user.User.Sid.String() +func getDatadogUserSid() (*windows.SID, error) { + ddUserSid, err := winutil.GetDDAgentUserSID() if err != nil { - return nil, fmt.Errorf("Couldn't get user sid string %v", err) + // falls back to current user on failure + return winutil.GetSidFromUser() } - return windows.StringToSid(sidString) + return ddUserSid, nil } // RestrictAccessToUser update the ACL of a file so only the current user and ADMIN/SYSTEM can access it @@ -68,7 +61,7 @@ func (p *Permission) RestrictAccessToUser(path string) error { false, // don't inherit acl.GrantSid(windows.GENERIC_ALL, p.administratorSid), acl.GrantSid(windows.GENERIC_ALL, p.systemSid), - acl.GrantSid(windows.GENERIC_ALL, p.currentUserSid)) + acl.GrantSid(windows.GENERIC_ALL, p.ddUserSid)) } // RemoveAccessToOtherUsers on Windows this function calls RestrictAccessToUser diff --git a/pkg/util/filesystem/permission_windows_test.go b/pkg/util/filesystem/permission_windows_test.go new file mode 100644 index 0000000000000..61921718cd37f --- /dev/null +++ b/pkg/util/filesystem/permission_windows_test.go @@ -0,0 +1,73 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build windows + +package filesystem + +import ( + "os" + "path/filepath" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +func TestRemoveAccessToOtherUsers(t *testing.T) { + + p, err := NewPermission() + require.NoError(t, err) + + root := t.TempDir() + + t.Log(root) + + testFile := filepath.Join(root, "file") + testDir := filepath.Join(root, "dir") + + err = os.WriteFile(testFile, []byte("test"), 0777) + require.NoError(t, err) + err = os.Mkdir(testDir, 0777) + require.NoError(t, err) + + err = p.RemoveAccessToOtherUsers(testFile) + require.NoError(t, err) + + // Assert the permissions for the file + assertPermissions(t, testFile, p) + + err = p.RemoveAccessToOtherUsers(testDir) + require.NoError(t, err) + + // Assert the permissions for the directory + assertPermissions(t, testDir, p) +} + +func assertPermissions(t *testing.T, path string, p *Permission) { + sd, err := windows.GetNamedSecurityInfo( + path, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, + ) + require.NoError(t, err) + + dacl, _, err := sd.DACL() + require.NoError(t, err) + + aceCount := int(dacl.AceCount) + for i := 0; i < aceCount; i++ { + var ace *windows.ACCESS_ALLOWED_ACE + err := windows.GetAce(dacl, uint32(i), &ace) + require.NoError(t, err) + + var sid *windows.SID = (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + + if !sid.Equals(p.ddUserSid) && !sid.Equals(p.administratorSid) && !sid.Equals(p.systemSid) { + t.Errorf("Unexpected SID with access: %v", sid) + } + } +} diff --git a/pkg/util/flavor/go.mod b/pkg/util/flavor/go.mod index bb7190a4d241c..dabdfd9bcee13 100644 --- a/pkg/util/flavor/go.mod +++ b/pkg/util/flavor/go.mod @@ -46,13 +46,13 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/pkg/util/grpc/go.mod b/pkg/util/grpc/go.mod index b058dd317dd34..bdbb97d385501 100644 --- a/pkg/util/grpc/go.mod +++ b/pkg/util/grpc/go.mod @@ -37,7 +37,7 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/api v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/proto v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/stretchr/testify v1.10.0 @@ -61,10 +61,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/pkg/util/http/go.mod b/pkg/util/http/go.mod index 0d248d6101c9f..e4699e99baf5e 100644 --- a/pkg/util/http/go.mod +++ b/pkg/util/http/go.mod @@ -33,7 +33,7 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/stretchr/testify v1.10.0 golang.org/x/net v0.31.0 ) @@ -51,10 +51,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/pkg/util/log/setup/go.mod b/pkg/util/log/setup/go.mod index 8eb352e5a4570..1d7dfdbbb7c8b 100644 --- a/pkg/util/log/setup/go.mod +++ b/pkg/util/log/setup/go.mod @@ -33,7 +33,7 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 github.com/stretchr/testify v1.10.0 ) @@ -51,10 +51,10 @@ require ( github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/pkg/util/system/go.mod b/pkg/util/system/go.mod index db28f3923c003..8ec61475312c9 100644 --- a/pkg/util/system/go.mod +++ b/pkg/util/system/go.mod @@ -13,10 +13,10 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/util/filesystem v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/testutil v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/winutil v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 github.com/shirou/gopsutil/v3 v3.24.5 github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 @@ -24,7 +24,7 @@ require ( ) require ( - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-ole/go-ole v1.3.0 // indirect diff --git a/pkg/util/winutil/service.go b/pkg/util/winutil/service.go index 11bc407bb0f43..22387fd89ea28 100644 --- a/pkg/util/winutil/service.go +++ b/pkg/util/winutil/service.go @@ -419,3 +419,20 @@ func IsServiceRunning(serviceName string) (running bool, err error) { } return (status.State == windows.SERVICE_RUNNING), nil } + +// GetServiceUser returns the service user for the given service +func GetServiceUser(serviceName string) (string, error) { + manager, service, err := openManagerService(serviceName, windows.SERVICE_QUERY_CONFIG) + if err != nil { + return "", err + } + defer closeManagerService(manager, service) + + serviceConfig, err := service.Config() + if err != nil { + return "", fmt.Errorf("could not retrieve config for %s: %w", serviceName, err) + } + + return serviceConfig.ServiceStartName, nil + +} diff --git a/pkg/util/winutil/users.go b/pkg/util/winutil/users.go index 7026b76507ebb..be17fe6a7cdb2 100644 --- a/pkg/util/winutil/users.go +++ b/pkg/util/winutil/users.go @@ -8,6 +8,7 @@ package winutil import ( "fmt" + "strings" "syscall" "golang.org/x/sys/windows" @@ -75,3 +76,53 @@ func GetLocalSystemSID() (*windows.SID, error) { return localSystem, err } + +// GetServiceUserSID returns the SID of the specified service account +func GetServiceUserSID(service string) (*windows.SID, error) { + // get config for datadogagent service + user, err := GetServiceUser(service) + if err != nil { + return nil, fmt.Errorf("could not get datadogagent service user: %s", err) + } + + username, err := getUserFromServiceUser(user) + if err != nil { + return nil, err + } + + // Manually map some aliases that SCM uses and are not recognized by the + // security subsystem (`LookupAccountName()` will fail) + // https://learn.microsoft.com/en-us/windows/win32/services/service-user-accounts + if username == "LocalSystem" { + return windows.StringToSid("S-1-5-18") + } + + // get the SID for the user account + sid, _, _, err := windows.LookupSID("", username) + return sid, err +} + +func getUserFromServiceUser(user string) (string, error) { + var domain, username string + parts := strings.SplitN(user, "\\", 2) + if len(parts) == 1 { + username = user + } else if len(parts) == 2 { + domain = parts[0] + if domain == "." { + username = parts[1] + } else { + username = user + } + } else { + return "", fmt.Errorf("could not parse user: %s", user) + } + + return username, nil + +} + +// GetDDAgentUserSID returns the SID of the DataDog Agent account +func GetDDAgentUserSID() (*windows.SID, error) { + return GetServiceUserSID("datadogagent") +} diff --git a/pkg/util/winutil/users_test.go b/pkg/util/winutil/users_test.go index d541eeb4e1ff8..d1941003bfda4 100644 --- a/pkg/util/winutil/users_test.go +++ b/pkg/util/winutil/users_test.go @@ -11,6 +11,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" ) func TestGetSidFromUser(t *testing.T) { @@ -19,3 +21,27 @@ func TestGetSidFromUser(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, sid) } + +func TestGetServiceUserSID(t *testing.T) { + // create LocalService SID + serviceSid, err := windows.StringToSid("S-1-5-19") + require.NoError(t, err) + + // get the SID for the EventLog service (has LocalService as its user) + sid, err := GetServiceUserSID("EventLog") + require.NoError(t, err) + assert.NotNil(t, sid) + assert.True(t, windows.EqualSid(sid, serviceSid)) + t.Logf("The SID found was: %v", sid) + + // create LocalSystem SID + systemSid, err := windows.StringToSid("S-1-5-18") + require.NoError(t, err) + + // get the SID for the BITS service (has LocalSystem as its user) + sid, err = GetServiceUserSID("BITS") + require.NoError(t, err) + assert.NotNil(t, sid) + assert.True(t, windows.EqualSid(sid, systemSid)) + t.Logf("The SID found was: %v", sid) +} diff --git a/releasenotes/notes/file-permissions-to-dduser-windows-6e9940175f9130ff.yaml b/releasenotes/notes/file-permissions-to-dduser-windows-6e9940175f9130ff.yaml new file mode 100644 index 0000000000000..45e2515092583 --- /dev/null +++ b/releasenotes/notes/file-permissions-to-dduser-windows-6e9940175f9130ff.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Fix Windows file permissions on authToken to give access to the Datadog user even when privilege processes create it. + diff --git a/test/otel/go.mod b/test/otel/go.mod index 7fb2f3b6d5f1e..dd0663ecebd79 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -164,15 +164,15 @@ require ( github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/util/startstop v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/statstracker v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/system v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.59.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.31.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect From 0744b78e72154436a2b6a533abb5c80be831eea5 Mon Sep 17 00:00:00 2001 From: yuri-lipnesh Date: Mon, 9 Dec 2024 17:46:30 -0500 Subject: [PATCH 42/52] [usm] system-probe, add Postgres message counters to the eBPF module (#31104) --- .../ebpf/c/protocols/postgres/decoding-maps.h | 4 + .../ebpf/c/protocols/postgres/decoding.h | 101 ++++++++-- pkg/network/ebpf/c/protocols/postgres/types.h | 25 ++- pkg/network/protocols/ebpf_types.go | 6 +- pkg/network/protocols/ebpf_types_linux.go | 4 +- pkg/network/protocols/postgres/ebpf/types.go | 8 +- .../protocols/postgres/ebpf/types_linux.go | 11 +- pkg/network/protocols/postgres/protocol.go | 83 +++++++- pkg/network/protocols/postgres/telemetry.go | 47 +++++ pkg/network/protocols/telemetry/metric.go | 14 ++ pkg/network/usm/postgres_monitor_test.go | 177 +++++++++++++++++- 11 files changed, 437 insertions(+), 43 deletions(-) diff --git a/pkg/network/ebpf/c/protocols/postgres/decoding-maps.h b/pkg/network/ebpf/c/protocols/postgres/decoding-maps.h index 67288e1243f9e..3ccbd67e52466 100644 --- a/pkg/network/ebpf/c/protocols/postgres/decoding-maps.h +++ b/pkg/network/ebpf/c/protocols/postgres/decoding-maps.h @@ -15,4 +15,8 @@ BPF_PERCPU_ARRAY_MAP(postgres_scratch_buffer, postgres_event_t, 1) // Maintains the current state of tail calls for each Postgres message. BPF_PERCPU_ARRAY_MAP(postgres_iterations, postgres_tail_call_state_t, 1) +// Postgres telemetry maps in kernel provide empirical statistics on the number of processed Postgres messages. +// Key 0 for plaintext traffic, key 1 for encrypted traffic. +BPF_ARRAY_MAP(postgres_telemetry, postgres_kernel_msg_count_t, 2) + #endif diff --git a/pkg/network/ebpf/c/protocols/postgres/decoding.h b/pkg/network/ebpf/c/protocols/postgres/decoding.h index 300ca6ba949a9..3f5e6d489376e 100644 --- a/pkg/network/ebpf/c/protocols/postgres/decoding.h +++ b/pkg/network/ebpf/c/protocols/postgres/decoding.h @@ -108,6 +108,48 @@ static int __always_inline skip_string(pktbuf_t pkt, int message_len) { return SKIP_STRING_FAILED; } +// Return a pointer to the postgres telemetry record in the corresponding map. +static __always_inline void* get_pg_msg_counts_map(pktbuf_t pkt) { + const __u32 plain_key = 0; + const __u32 tls_key = 1; + + pktbuf_map_lookup_option_t pg_telemetry_lookup_opt[] = { + [PKTBUF_SKB] = { + .map = &postgres_telemetry, + .key = (void*)&plain_key, + }, + [PKTBUF_TLS] = { + .map = &postgres_telemetry, + .key = (void*)&tls_key, + }, + }; + return pktbuf_map_lookup(pkt, pg_telemetry_lookup_opt); +} + +// update_msg_count_telemetry increases the corresponding counter of the telemetry bucket. +static __always_inline void update_msg_count_telemetry(postgres_kernel_msg_count_t* pg_msg_counts, __u8 count) { + // This line can be interpreted as a step function of the difference, multiplied by the difference itself. + // The step function of the difference returns 0 if the difference is negative and 1 if it is positive. + // As a result, if the difference is negative, the output will be 0; if the difference is positive, + // the output will equal the difference. + count = count < PG_KERNEL_MSG_COUNT_FIRST_BUCKET ? 0 : count - PG_KERNEL_MSG_COUNT_FIRST_BUCKET; + + // This line functions as a ceiling operation, ensuring that if the count is not a multiple of the bucket size, + // it is rounded up to the next bucket. Since eBPF does not support floating-point numbers, the implementation + // adds (bucket size - 1) to the count and then divides the result by the bucket size. + // This effectively simulates the ceiling function. + __u8 bucket_idx = (count + PG_KERNEL_MSG_COUNT_BUCKET_SIZE - 1) / PG_KERNEL_MSG_COUNT_BUCKET_SIZE; + + // This line ensures that the bucket index stays within the range of 0 to PG_KERNEL_MSG_COUNT_NUM_BUCKETS. + // While not strictly necessary, we include this check to satisfy the verifier and to explicitly define a lower bound. + bucket_idx = bucket_idx < 0 ? 0 : bucket_idx; + + // This line ensures that the bucket index remains within the range of 0 to PG_KERNEL_MSG_COUNT_NUM_BUCKETS, + // preventing any possibility of exceeding the upper bound. + bucket_idx = bucket_idx >= PG_KERNEL_MSG_COUNT_NUM_BUCKETS ? PG_KERNEL_MSG_COUNT_NUM_BUCKETS-1 : bucket_idx; + __sync_fetch_and_add(&pg_msg_counts->msg_count_buckets[bucket_idx], 1); +} + // Reads the first message header and decides what to do based on the // message tag. If the message is a new query, it stores the query in the in-flight map. // If the message is a parse message, we tail call to the dedicated process_parse_message program. @@ -148,7 +190,7 @@ static __always_inline void postgres_handle_message(pktbuf_t pkt, conn_tuple_t * return; } - iteration_value->iteration = 0; + iteration_value->total_msg_count = 0; iteration_value->data_off = 0; pktbuf_tail_call_option_t handle_response_tail_call_array[] = { [PKTBUF_SKB] = { @@ -197,10 +239,10 @@ static __always_inline void postgres_handle_parse_message(pktbuf_t pkt, conn_tup // Handles Postgres command complete messages by examining packet data for both plaintext and TLS traffic. // This function handles multiple messages within a single packet, processing up to POSTGRES_MAX_MESSAGES_PER_TAIL_CALL -// messages per call. When more messages exist beyond this limit, it uses tail call chaining (up to -// POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES) to continue processing. -static __always_inline bool handle_response(pktbuf_t pkt, conn_tuple_t conn_tuple) { +// messages per call. When more messages exist beyond this limit, it uses tail call chaining to continue processing. +static __always_inline bool handle_response(pktbuf_t pkt, conn_tuple_t conn_tuple, postgres_kernel_msg_count_t* pg_msg_counts) { const __u32 zero = 0; + bool read_result = false; bool found_command_complete = false; struct pg_message_header header; @@ -210,7 +252,7 @@ static __always_inline bool handle_response(pktbuf_t pkt, conn_tuple_t conn_tupl return 0; } - if (iteration_value->iteration >= POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES) { + if (iteration_value->total_msg_count >= (POSTGRES_MAX_TOTAL_MESSAGES - 1)) { return 0; } @@ -225,13 +267,14 @@ static __always_inline bool handle_response(pktbuf_t pkt, conn_tuple_t conn_tupl return 0; } + __u8 messages_count = 0; #pragma unroll(POSTGRES_MAX_MESSAGES_PER_TAIL_CALL) - for (__u32 iteration = 0; iteration < POSTGRES_MAX_MESSAGES_PER_TAIL_CALL; ++iteration) { - if (!read_message_header(pkt, &header)) { + for (; messages_count < POSTGRES_MAX_MESSAGES_PER_TAIL_CALL; ++messages_count) { + read_result = read_message_header(pkt, &header); + if (read_result != true) { break; } if (header.message_tag == POSTGRES_COMMAND_COMPLETE_MAGIC_BYTE) { - handle_command_complete(&conn_tuple, transaction); found_command_complete = true; break; } @@ -240,20 +283,35 @@ static __always_inline bool handle_response(pktbuf_t pkt, conn_tuple_t conn_tupl // the message tag. So we need to add 1 to the message length to jump over the entire message. pktbuf_advance(pkt, header.message_len + 1); } + iteration_value->total_msg_count += messages_count; if (found_command_complete) { + handle_command_complete(&conn_tuple, transaction); + update_msg_count_telemetry(pg_msg_counts, iteration_value->total_msg_count); + return 0; } - // We didn't find a command complete message, so we need to continue processing the packet. - // We save the current data offset and increment the iteration counter. - iteration_value->iteration += 1; - iteration_value->data_off = pktbuf_data_offset(pkt); - // If the maximum number of tail calls has been reached, we can skip invoking the next tail call. - if (iteration_value->iteration >= POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES) { + if (iteration_value->total_msg_count >= (POSTGRES_MAX_TOTAL_MESSAGES - 1)) { + // reached max messages, add counter and stop iterating. + __sync_fetch_and_add(&pg_msg_counts->reached_max_messages, 1); + return 0; + } + if (pktbuf_data_offset(pkt) == pktbuf_data_end(pkt)) { + // stop the iterator if the end of the TCP packet is reached. + update_msg_count_telemetry(pg_msg_counts, iteration_value->total_msg_count); + return 0; + } + if (read_result == false) { + // the packet was fragmented, add counter stop iterating. + __sync_fetch_and_add(&pg_msg_counts->fragmented_packets, 1); return 0; } + // We didn't find a command complete message, so we need to continue processing the packet. + // We save the current data offset. + iteration_value->data_off = pktbuf_data_offset(pkt); + pktbuf_tail_call_option_t handle_response_tail_call_array[] = { [PKTBUF_SKB] = { .prog_array_map = &protocols_progs, @@ -315,7 +373,11 @@ int socket__postgres_handle_response(struct __sk_buff* skb) { normalize_tuple(&conn_tuple); pktbuf_t pkt = pktbuf_from_skb(skb, &skb_info); - handle_response(pkt, conn_tuple); + postgres_kernel_msg_count_t* pg_msg_counts = get_pg_msg_counts_map(pkt); + if (pg_msg_counts == NULL) { + return 0; + } + handle_response(pkt, conn_tuple, pg_msg_counts); return 0; } @@ -406,10 +468,15 @@ int uprobe__postgres_tls_handle_response(struct pt_regs *ctx) { return 0; } + pktbuf_t pkt = pktbuf_from_tls(ctx, args); + postgres_kernel_msg_count_t* pg_msg_counts = get_pg_msg_counts_map(pkt); + if (pg_msg_counts == NULL) { + return 0; + } + // Copying the tuple to the stack to handle verifier issues on kernel 4.14. conn_tuple_t tup = args->tup; - pktbuf_t pkt = pktbuf_from_tls(ctx, args); - handle_response(pkt, tup); + handle_response(pkt, tup, pg_msg_counts); return 0; } diff --git a/pkg/network/ebpf/c/protocols/postgres/types.h b/pkg/network/ebpf/c/protocols/postgres/types.h index 0e88332dc78a7..6f0abf951ddaf 100644 --- a/pkg/network/ebpf/c/protocols/postgres/types.h +++ b/pkg/network/ebpf/c/protocols/postgres/types.h @@ -7,10 +7,13 @@ #define POSTGRES_BUFFER_SIZE 160 // Represents the maximum number of tail calls we can use to process a single message. -#define POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES 1 +#define POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES 3 // Represents the maximum number of messages we process in a single tail call. -#define POSTGRES_MAX_MESSAGES_PER_TAIL_CALL 80 +#define POSTGRES_MAX_MESSAGES_PER_TAIL_CALL 60 + +// maximum number of messages to fetch +#define POSTGRES_MAX_TOTAL_MESSAGES (POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES * POSTGRES_MAX_MESSAGES_PER_TAIL_CALL) // Postgres transaction information we store in the kernel. typedef struct { @@ -30,10 +33,26 @@ typedef struct { } postgres_event_t; typedef struct { - __u8 iteration; + __u8 total_msg_count; // Saving the packet data offset is crucial for maintaining the current read position and ensuring proper utilization // of tail calls. __u32 data_off; } postgres_tail_call_state_t; +// Postgres communication operates via a continuous message stream. +// Gather empirical statistics on the number of messages processed by the program. +// These statistics enable optimization of the monitoring code. +// Collect counters for each subsequent bucket. +// bucket 0: count 0 to 100, bucket 1: 100 to 119, bucket 2: 120 to 139 etc. +#define PG_KERNEL_MSG_COUNT_NUM_BUCKETS 5 +#define PG_KERNEL_MSG_COUNT_BUCKET_SIZE 20 +#define PG_KERNEL_MSG_COUNT_FIRST_BUCKET 100 + +// This structure stores statistics about the number of Postgres messages in a TCP packet. +typedef struct { + __u64 reached_max_messages; + __u64 fragmented_packets; + __u64 msg_count_buckets[PG_KERNEL_MSG_COUNT_NUM_BUCKETS]; +} postgres_kernel_msg_count_t; + #endif diff --git a/pkg/network/protocols/ebpf_types.go b/pkg/network/protocols/ebpf_types.go index 57bdb4539bf5e..c5b90864ee74e 100644 --- a/pkg/network/protocols/ebpf_types.go +++ b/pkg/network/protocols/ebpf_types.go @@ -19,10 +19,8 @@ const ( ) const ( - // PostgresMaxMessagesPerTailCall is the maximum number of messages that can be processed in a single tail call in our Postgres decoding solution - PostgresMaxMessagesPerTailCall = C.POSTGRES_MAX_MESSAGES_PER_TAIL_CALL - // PostgresMaxTailCalls is the maximum number of tail calls that can be made in our Postgres decoding solution - PostgresMaxTailCalls = C.POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES + // PostgresMaxTotalMessages is the maximum number of Postgres messages processed by the eBPF program. + PostgresMaxTotalMessages = C.POSTGRES_MAX_TOTAL_MESSAGES ) // DispatcherProgramType is a C type to represent the eBPF programs used for tail calls. diff --git a/pkg/network/protocols/ebpf_types_linux.go b/pkg/network/protocols/ebpf_types_linux.go index ae855cfe48b77..8cc9c663dc3d5 100644 --- a/pkg/network/protocols/ebpf_types_linux.go +++ b/pkg/network/protocols/ebpf_types_linux.go @@ -10,9 +10,7 @@ const ( ) const ( - PostgresMaxMessagesPerTailCall = 0x50 - - PostgresMaxTailCalls = 0x1 + PostgresMaxTotalMessages = 0xb4 ) type DispatcherProgramType uint32 diff --git a/pkg/network/protocols/postgres/ebpf/types.go b/pkg/network/protocols/postgres/ebpf/types.go index 7c615186958e4..63071b914e5a7 100644 --- a/pkg/network/protocols/postgres/ebpf/types.go +++ b/pkg/network/protocols/postgres/ebpf/types.go @@ -9,6 +9,7 @@ package ebpf /* #include "../../ebpf/c/protocols/postgres/types.h" +#include "../../ebpf/c/protocols/postgres/defs.h" #include "../../ebpf/c/protocols/classification/defs.h" */ import "C" @@ -19,7 +20,12 @@ type ConnTuple = C.conn_tuple_t type EbpfEvent C.postgres_event_t type EbpfTx C.postgres_transaction_t +type PostgresKernelMsgCount C.postgres_kernel_msg_count_t const ( - BufferSize = C.POSTGRES_BUFFER_SIZE + BufferSize = C.POSTGRES_BUFFER_SIZE + MsgCountBucketSize = C.PG_KERNEL_MSG_COUNT_BUCKET_SIZE + MsgCountNumBuckets = C.PG_KERNEL_MSG_COUNT_NUM_BUCKETS + MsgCountFirstBucket = C.PG_KERNEL_MSG_COUNT_FIRST_BUCKET + MsgCountMaxTotal = C.POSTGRES_MAX_TOTAL_MESSAGES ) diff --git a/pkg/network/protocols/postgres/ebpf/types_linux.go b/pkg/network/protocols/postgres/ebpf/types_linux.go index 9b0097a2b5f9d..9413ee269ac20 100644 --- a/pkg/network/protocols/postgres/ebpf/types_linux.go +++ b/pkg/network/protocols/postgres/ebpf/types_linux.go @@ -27,7 +27,16 @@ type EbpfTx struct { Tags uint8 Pad_cgo_0 [3]byte } +type PostgresKernelMsgCount struct { + Reached_max_messages uint64 + Fragmented_packets uint64 + Msg_count_buckets [5]uint64 +} const ( - BufferSize = 0xa0 + BufferSize = 0xa0 + MsgCountBucketSize = 0x14 + MsgCountNumBuckets = 0x5 + MsgCountFirstBucket = 0x64 + MsgCountMaxTotal = 0xb4 ) diff --git a/pkg/network/protocols/postgres/protocol.go b/pkg/network/protocols/postgres/protocol.go index 6bc245896a91a..c0d7aa37a6747 100644 --- a/pkg/network/protocols/postgres/protocol.go +++ b/pkg/network/protocols/postgres/protocol.go @@ -9,6 +9,7 @@ package postgres import ( "io" + "time" "unsafe" "github.com/cilium/ebpf" @@ -28,6 +29,8 @@ import ( ) const ( + // KernelTelemetryMap is the map for getting kernel metrics + KernelTelemetryMap = "postgres_telemetry" // InFlightMap is the name of the in-flight map. InFlightMap = "postgres_in_flight" scratchBufferMap = "postgres_scratch_buffer" @@ -44,11 +47,13 @@ const ( // protocol holds the state of the postgres protocol monitoring. type protocol struct { - cfg *config.Config - telemetry *Telemetry - eventsConsumer *events.Consumer[postgresebpf.EbpfEvent] - mapCleaner *ddebpf.MapCleaner[netebpf.ConnTuple, postgresebpf.EbpfTx] - statskeeper *StatKeeper + cfg *config.Config + telemetry *Telemetry + eventsConsumer *events.Consumer[postgresebpf.EbpfEvent] + mapCleaner *ddebpf.MapCleaner[netebpf.ConnTuple, postgresebpf.EbpfTx] + statskeeper *StatKeeper + kernelTelemetry *kernelTelemetry // retrieves Postgres metrics from kernel + kernelTelemetryStopCh chan struct{} } // Spec is the protocol spec for the postgres protocol. @@ -133,9 +138,11 @@ func newPostgresProtocol(cfg *config.Config) (protocols.Protocol, error) { } return &protocol{ - cfg: cfg, - telemetry: NewTelemetry(cfg), - statskeeper: NewStatkeeper(cfg), + cfg: cfg, + telemetry: NewTelemetry(cfg), + statskeeper: NewStatkeeper(cfg), + kernelTelemetry: newKernelTelemetry(), + kernelTelemetryStopCh: make(chan struct{}), }, nil } @@ -175,6 +182,7 @@ func (p *protocol) PreStart(mgr *manager.Manager) (err error) { func (p *protocol) PostStart(mgr *manager.Manager) error { // Setup map cleaner after manager start. p.setupMapCleaner(mgr) + p.startKernelTelemetry(mgr) return nil } @@ -186,11 +194,16 @@ func (p *protocol) Stop(*manager.Manager) { if p.eventsConsumer != nil { p.eventsConsumer.Stop() } + if p.kernelTelemetryStopCh != nil { + close(p.kernelTelemetryStopCh) + } } // DumpMaps dumps map contents for debugging. func (p *protocol) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map) { - if mapName == InFlightMap { // maps/postgres_in_flight (BPF_MAP_TYPE_HASH), key ConnTuple, value EbpfTx + switch mapName { + case InFlightMap: + // maps/postgres_in_flight (BPF_MAP_TYPE_HASH), key ConnTuple, value EbpfTx var key netebpf.ConnTuple var value postgresebpf.EbpfTx protocols.WriteMapDumpHeader(w, currentMap, mapName, key, value) @@ -198,12 +211,27 @@ func (p *protocol) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map) { for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { spew.Fdump(w, key, value) } + case KernelTelemetryMap: + // postgres_msg_count (BPF_ARRAY_MAP), key 0 and 1, value PostgresKernelMsgCount + plainKey := uint32(0) + tlsKey := uint32(1) + + var value postgresebpf.PostgresKernelMsgCount + protocols.WriteMapDumpHeader(w, currentMap, mapName, plainKey, value) + if err := currentMap.Lookup(unsafe.Pointer(&plainKey), unsafe.Pointer(&value)); err == nil { + spew.Fdump(w, plainKey, value) + } + protocols.WriteMapDumpHeader(w, currentMap, mapName, tlsKey, value) + if err := currentMap.Lookup(unsafe.Pointer(&tlsKey), unsafe.Pointer(&value)); err == nil { + spew.Fdump(w, tlsKey, value) + } } } // GetStats returns a map of Postgres stats. func (p *protocol) GetStats() *protocols.ProtocolStats { p.eventsConsumer.Sync() + p.kernelTelemetry.Log() return &protocols.ProtocolStats{ Type: protocols.Postgres, @@ -250,3 +278,40 @@ func (p *protocol) setupMapCleaner(mgr *manager.Manager) { p.mapCleaner = mapCleaner } + +func (p *protocol) startKernelTelemetry(mgr *manager.Manager) { + telemetryMap, err := protocols.GetMap(mgr, KernelTelemetryMap) + if err != nil { + log.Errorf("couldnt find kernel telemetry map: %s, error: %v", telemetryMap, err) + return + } + + plainKey := uint32(0) + tlsKey := uint32(1) + pgKernelMsgCount := &postgresebpf.PostgresKernelMsgCount{} + ticker := time.NewTicker(30 * time.Second) + + go func() { + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := telemetryMap.Lookup(unsafe.Pointer(&plainKey), unsafe.Pointer(pgKernelMsgCount)); err != nil { + log.Errorf("unable to lookup %q map: %s", KernelTelemetryMap, err) + return + } + p.kernelTelemetry.update(pgKernelMsgCount, false) + + if err := telemetryMap.Lookup(unsafe.Pointer(&tlsKey), unsafe.Pointer(pgKernelMsgCount)); err != nil { + log.Errorf("unable to lookup %q map: %s", KernelTelemetryMap, err) + return + } + p.kernelTelemetry.update(pgKernelMsgCount, true) + + case <-p.kernelTelemetryStopCh: + return + } + } + }() +} diff --git a/pkg/network/protocols/postgres/telemetry.go b/pkg/network/protocols/postgres/telemetry.go index cc17b902c24ea..d8cb08725a80e 100644 --- a/pkg/network/protocols/postgres/telemetry.go +++ b/pkg/network/protocols/postgres/telemetry.go @@ -9,6 +9,7 @@ package postgres import ( "fmt" + "strconv" "github.com/cihub/seelog" @@ -178,3 +179,49 @@ func (t *Telemetry) Log() { log.Debugf("postgres stats summary: %s", t.metricGroup.Summary()) } } + +// kernelTelemetry provides empirical kernel statistics about the number of messages in each TCP packet +type kernelTelemetry struct { + metricGroup *libtelemetry.MetricGroup + reachedMaxMessages *libtelemetry.TLSAwareCounter + fragmentedPackets *libtelemetry.TLSAwareCounter + msgCountBuckets [ebpf.MsgCountNumBuckets]*libtelemetry.TLSAwareCounter // Postgres messages counters divided into buckets +} + +// newKernelTelemetry this is the Postgres message counter store. +func newKernelTelemetry() *kernelTelemetry { + metricGroup := libtelemetry.NewMetricGroup("usm.postgres", libtelemetry.OptStatsd) + kernelTel := &kernelTelemetry{ + metricGroup: metricGroup, + } + kernelTel.reachedMaxMessages = libtelemetry.NewTLSAwareCounter(metricGroup, "max_messages") + kernelTel.fragmentedPackets = libtelemetry.NewTLSAwareCounter(metricGroup, "incomplete_messages") + + for i := range kernelTel.msgCountBuckets { + kernelTel.msgCountBuckets[i] = libtelemetry.NewTLSAwareCounter(metricGroup, "messages_count_bucket_"+strconv.Itoa(i+1)) + } + return kernelTel +} + +// update the postgres message counter store with new counters from the kernel, return immediately if nothing to add. +func (t *kernelTelemetry) update(kernCounts *ebpf.PostgresKernelMsgCount, isTLS bool) { + if kernCounts == nil { + return + } + + t.reachedMaxMessages.Set(int64(kernCounts.Reached_max_messages), isTLS) + t.fragmentedPackets.Set(int64(kernCounts.Fragmented_packets), isTLS) + + for i := range t.msgCountBuckets { + v := kernCounts.Msg_count_buckets[i] + t.msgCountBuckets[i].Set(int64(v), isTLS) + } +} + +// Log logs summary of telemetry +func (t *kernelTelemetry) Log() { + if log.ShouldLog(seelog.DebugLvl) { + s := t.metricGroup.Summary() + log.Debugf("postgres kernel telemetry, summary: %s", s) + } +} diff --git a/pkg/network/protocols/telemetry/metric.go b/pkg/network/protocols/telemetry/metric.go index 46aa226351278..56ce64a1b00b8 100644 --- a/pkg/network/protocols/telemetry/metric.go +++ b/pkg/network/protocols/telemetry/metric.go @@ -26,6 +26,11 @@ func NewCounter(name string, tagsAndOptions ...string) *Counter { return globalRegistry.FindOrCreate(c).(*Counter) } +// Set value atomically +func (c *Counter) Set(v int64) { + c.value.Store(v) +} + // Add value atomically func (c *Counter) Add(v int64) { if v > 0 { @@ -118,6 +123,15 @@ func NewTLSAwareCounter(metricGroup *MetricGroup, metricName string, tags ...str } } +// Set Sets the given value to the counter based on the encryption. +func (c *TLSAwareCounter) Set(v int64, isTLS bool) { + if isTLS { + c.counterTLS.Set(v) + return + } + c.counterPlain.Set(v) +} + // Add adds the given delta to the counter based on the encryption. func (c *TLSAwareCounter) Add(delta int64, isTLS bool) { if isTLS { diff --git a/pkg/network/usm/postgres_monitor_test.go b/pkg/network/usm/postgres_monitor_test.go index 76c616beff5bd..9198448f99468 100644 --- a/pkg/network/usm/postgres_monitor_test.go +++ b/pkg/network/usm/postgres_monitor_test.go @@ -16,8 +16,10 @@ import ( "sync" "testing" "time" + "unsafe" "github.com/jackc/pgx/v5/pgproto3" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -47,7 +49,6 @@ const ( alterTableQuery = "ALTER TABLE dummy ADD test VARCHAR(255);" truncateTableQuery = "TRUNCATE TABLE dummy" showQuery = "SHOW search_path" - maxSupportedMessages = protocols.PostgresMaxMessagesPerTailCall * protocols.PostgresMaxTailCalls ) var ( @@ -591,7 +592,7 @@ func testDecoding(t *testing.T, isTLS bool) { }, // The purpose of this test is to validate the POSTGRES_MAX_MESSAGES_PER_TAIL_CALL * POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES limit. { - name: "validate supporting max supported messages limit", + name: "validate max supported messages limit", preMonitorSetup: func(t *testing.T, ctx pgTestContext) { pg, err := postgres.NewPGXClient(postgres.ConnectionOptions{ ServerAddress: ctx.serverAddress, @@ -607,7 +608,7 @@ func testDecoding(t *testing.T, isTLS bool) { ctx.extras["pg"] = pg require.NoError(t, pg.RunQuery(createTableQuery)) // We reduce the limit by 2 messages because the protocol adds messages at the beginning of the maximum message response. - require.NoError(t, pg.RunQuery(createInsertQuery(generateTestValues(1, maxSupportedMessages-3)...))) + require.NoError(t, pg.RunQuery(createInsertQuery(generateTestValues(1, protocols.PostgresMaxTotalMessages-3)...))) require.NoError(t, pg.RunQuery(selectAllQuery)) }, validation: func(t *testing.T, _ pgTestContext, monitor *Monitor) { @@ -623,7 +624,7 @@ func testDecoding(t *testing.T, isTLS bool) { // This test validates that when we exceed the POSTGRES_MAX_MESSAGES_PER_TAIL_CALL * POSTGRES_MAX_TAIL_CALLS_FOR_MAX_MESSAGES limit, // the request is not captured as we will miss the response.In this case, it applies to the SELECT query. { - name: "validate exceeding max supported messages limit is not supported", + name: "exceeding max supported messages limit", preMonitorSetup: func(t *testing.T, ctx pgTestContext) { pg, err := postgres.NewPGXClient(postgres.ConnectionOptions{ ServerAddress: ctx.serverAddress, @@ -638,7 +639,7 @@ func testDecoding(t *testing.T, isTLS bool) { require.NoError(t, pg.Ping()) ctx.extras["pg"] = pg require.NoError(t, pg.RunQuery(createTableQuery)) - require.NoError(t, pg.RunQuery(createInsertQuery(generateTestValues(1, maxSupportedMessages+1)...))) + require.NoError(t, pg.RunQuery(createInsertQuery(generateTestValues(1, protocols.PostgresMaxTotalMessages+1)...))) require.NoError(t, pg.RunQuery(selectAllQuery)) }, validation: func(t *testing.T, _ pgTestContext, monitor *Monitor) { @@ -890,3 +891,169 @@ func createFragment(fragment []byte) [ebpf.BufferSize]byte { copy(b[:], fragment) return b } + +func (s *postgresProtocolParsingSuite) TestKernelTelemetry() { + t := s.T() + tests := []struct { + name string + isTLS bool + }{ + { + name: "without TLS", + isTLS: false, + }, + { + name: "with TLS", + isTLS: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.isTLS && !gotlstestutil.GoTLSSupported(t, config.New()) { + t.Skip("GoTLS not supported for this setup") + } + testKernelMessagesCount(t, tt.isTLS) + }) + } +} + +// testKernelMessagesCount check postgres kernel messages count. +func testKernelMessagesCount(t *testing.T, isTLS bool) { + serverHost := "127.0.0.1" + serverAddress := net.JoinHostPort(serverHost, postgresPort) + require.NoError(t, postgres.RunServer(t, serverHost, postgresPort, isTLS)) + waitForPostgresServer(t, serverAddress, isTLS) + + monitor := setupUSMTLSMonitor(t, getPostgresDefaultTestConfiguration(isTLS)) + if isTLS { + utils.WaitForProgramsToBeTraced(t, consts.USMModuleName, GoTLSAttacherName, os.Getpid(), utils.ManualTracingFallbackEnabled) + } + pgClient := setupPGClient(t, serverAddress, isTLS) + t.Cleanup(func() { + if pgClient != nil { + _ = pgClient.RunQuery(dropTableQuery) + pgClient.Close() + pgClient = nil + } + }) + + createLargeTable(t, pgClient, ebpf.MsgCountFirstBucket+ebpf.MsgCountBucketSize*ebpf.MsgCountNumBuckets) + expectedBuckets := [ebpf.MsgCountNumBuckets]bool{} + + for i := 0; i < ebpf.MsgCountNumBuckets; i++ { + testName := fmt.Sprintf("kernel messages count bucket[%d]", i) + t.Run(testName, func(t *testing.T) { + + expectedBuckets[i] = true + cleanProtocolMaps(t, "postgres", monitor.ebpfProgram.Manager.Manager) + + require.NoError(t, monitor.Resume()) + if i == 0 { + // first bucket, it counts upto ebpf.MsgCountFirstBucketMax messages + // subtract three messages ('bind', 'row description' and 'ready') + require.NoError(t, pgClient.RunQuery(generateSelectLimitQuery(ebpf.MsgCountFirstBucket-3))) + } else { + limitCount := ebpf.MsgCountFirstBucket + i*ebpf.MsgCountBucketSize - 3 + require.NoError(t, pgClient.RunQuery(generateSelectLimitQuery(limitCount))) + } + require.NoError(t, monitor.Pause()) + + validateKernelBuckets(t, monitor, isTLS, expectedBuckets) + if i > 0 { + // clear current bucket except the first one, which is always non-empty. + expectedBuckets[i] = false + } + }) + } + + t.Run("exceed max buckets", func(t *testing.T) { + + cleanProtocolMaps(t, "postgres", monitor.ebpfProgram.Manager.Manager) + require.NoError(t, monitor.Resume()) + + require.NoError(t, pgClient.RunQuery(generateSelectLimitQuery(ebpf.MsgCountMaxTotal))) + require.NoError(t, monitor.Pause()) + + validateKernelExceedingMax(t, monitor, isTLS) + }) +} + +func setupPGClient(t *testing.T, serverAddress string, isTLS bool) *postgres.PGXClient { + pg, err := postgres.NewPGXClient(postgres.ConnectionOptions{ + ServerAddress: serverAddress, + EnableTLS: isTLS, + }) + require.NoError(t, err) + require.NoError(t, pg.Ping()) + return pg +} + +// createLargeTable runs a postgres query to create a table large enough to retrieve long responses later. +func createLargeTable(t *testing.T, pg *postgres.PGXClient, tableValuesCount int) { + require.NoError(t, pg.RunQuery(createTableQuery)) + require.NoError(t, pg.RunQuery(createInsertQuery(generateTestValues(0, tableValuesCount)...))) +} + +// validateKernel Checking telemetry data received for a postgres query +func validateKernelBuckets(t *testing.T, monitor *Monitor, tls bool, expected [ebpf.MsgCountNumBuckets]bool) { + var actual *ebpf.PostgresKernelMsgCount + assert.Eventually(t, func() bool { + found, err := getKernelTelemetry(monitor, tls) + if err != nil { + return false + } + actual = found + return compareMessagesCount(found, expected) + }, time.Second*2, time.Millisecond*100) + if t.Failed() { + t.Logf("expected telemetry:\n %+v;\nactual telemetry:\n %+v", expected, actual) + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, postgres.KernelTelemetryMap) + } +} + +// getKernelTelemetry returns statistics obtained from the kernel +func getKernelTelemetry(monitor *Monitor, isTLS bool) (*ebpf.PostgresKernelMsgCount, error) { + pgKernelTelemetry := &ebpf.PostgresKernelMsgCount{} + mapName := postgres.KernelTelemetryMap + key := uint32(0) + if isTLS { + key = uint32(1) + } + mp, _, err := monitor.ebpfProgram.GetMap(mapName) + if err != nil { + return nil, fmt.Errorf("unable to get %q map: %s", mapName, err) + } + if err := mp.Lookup(unsafe.Pointer(&key), unsafe.Pointer(pgKernelTelemetry)); err != nil { + return nil, fmt.Errorf("unable to lookup %q map: %s", mapName, err) + } + return pgKernelTelemetry, nil +} + +// compareMessagesCount returns true if the expected bucket is non-empty +func compareMessagesCount(found *ebpf.PostgresKernelMsgCount, expected [ebpf.MsgCountNumBuckets]bool) bool { + for i := range expected { + if expected[i] && found.Msg_count_buckets[i] == 0 { + return false + } + if !expected[i] && found.Msg_count_buckets[i] > 0 { + return false + } + } + return true +} + +// validateKernelExceedingMax check for exceeding the maximum number of buckets +func validateKernelExceedingMax(t *testing.T, monitor *Monitor, tls bool) { + var actual *ebpf.PostgresKernelMsgCount + assert.Eventually(t, func() bool { + found, err := getKernelTelemetry(monitor, tls) + if err != nil { + return false + } + actual = found + return found.Reached_max_messages > 0 + }, time.Second*2, time.Millisecond*100) + if t.Failed() { + t.Logf("expected non-zero max messages, actual telemetry:\n %+v", actual) + } +} From 5e3a9d7cbe4c2d1f6a1ba917b8d45e1c1448b0a8 Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Tue, 10 Dec 2024 08:20:18 +0200 Subject: [PATCH 43/52] Fix require.Eventually gotchas (#31861) --- pkg/network/dns/snooper_test.go | 8 +- .../testutil/testdns/test_dns_server.go | 12 +- pkg/network/tracer/tracer_linux_test.go | 236 ++++++++---------- pkg/network/tracer/tracer_test.go | 97 +++---- pkg/network/usm/kafka_monitor_test.go | 8 +- .../usm/tests/tracer_usm_linux_test.go | 25 +- 6 files changed, 186 insertions(+), 200 deletions(-) diff --git a/pkg/network/dns/snooper_test.go b/pkg/network/dns/snooper_test.go index 3a0d334c80488..a6c1383a9ddcc 100644 --- a/pkg/network/dns/snooper_test.go +++ b/pkg/network/dns/snooper_test.go @@ -230,7 +230,7 @@ func TestDNSFailedResponseCount(t *testing.T) { "nonexistenent.net.com", "missingdomain.com", } - queryIP, queryPort, reps, _ := testdns.SendDNSQueries(t, domains, testdns.GetServerIP(t), "tcp") + queryIP, queryPort, reps, _ := testdns.SendDNSQueries(domains, testdns.GetServerIP(t), "tcp") for _, rep := range reps { require.NotNil(t, rep) require.Equal(t, rep.Rcode, mdns.RcodeNameError) // All the queries should have failed @@ -281,7 +281,7 @@ func TestDNSOverNonPort53(t *testing.T) { shutdown, port := newTestServer(t, localhost, "udp") defer shutdown() - queryIP, queryPort, reps, err := testdns.SendDNSQueriesOnPort(t, domains, net.ParseIP(localhost), strconv.Itoa(int(port)), "udp") + queryIP, queryPort, reps, err := testdns.SendDNSQueriesOnPort(domains, net.ParseIP(localhost), strconv.Itoa(int(port)), "udp") require.NoError(t, err) require.NotNil(t, reps[0]) @@ -334,7 +334,7 @@ func TestDNSOverUDPTimeoutCount(t *testing.T) { invalidServerIP := "8.8.8.90" domainQueried := "agafsdfsdasdfsd" - queryIP, queryPort, reps, err := testdns.SendDNSQueries(t, []string{domainQueried}, net.ParseIP(invalidServerIP), "udp") + queryIP, queryPort, reps, err := testdns.SendDNSQueries([]string{domainQueried}, net.ParseIP(invalidServerIP), "udp") require.ErrorIs(t, err, os.ErrDeadlineExceeded, "error should be i/o timeout") require.Len(t, reps, 1) require.Nil(t, reps[0]) @@ -358,7 +358,7 @@ func TestDNSOverUDPTimeoutCountWithoutDomain(t *testing.T) { invalidServerIP := "8.8.8.90" domainQueried := "agafsdfsdasdfsd" - queryIP, queryPort, reps, err := testdns.SendDNSQueries(t, []string{domainQueried}, net.ParseIP(invalidServerIP), "udp") + queryIP, queryPort, reps, err := testdns.SendDNSQueries([]string{domainQueried}, net.ParseIP(invalidServerIP), "udp") require.ErrorIs(t, err, os.ErrDeadlineExceeded, "error should be i/o timeout") require.Len(t, reps, 1) require.Nil(t, reps[0]) diff --git a/pkg/network/tracer/testutil/testdns/test_dns_server.go b/pkg/network/tracer/testutil/testdns/test_dns_server.go index 48c4e2fe670fe..05745479b4073 100644 --- a/pkg/network/tracer/testutil/testdns/test_dns_server.go +++ b/pkg/network/tracer/testutil/testdns/test_dns_server.go @@ -131,8 +131,7 @@ func respond(req *dns.Msg, writer dns.ResponseWriter, record string) { } // SendDNSQueriesOnPort makes a DNS query for every domain provided, on the given serverIP, port, and protocol. -func SendDNSQueriesOnPort(t *testing.T, domains []string, serverIP net.IP, port string, protocol string) (string, int, []*dns.Msg, error) { - t.Helper() +func SendDNSQueriesOnPort(domains []string, serverIP net.IP, port string, protocol string) (string, int, []*dns.Msg, error) { dnsClient := dns.Client{Net: protocol, Timeout: 3 * time.Second} dnsHost := net.JoinHostPort(serverIP.String(), port) conn, err := dnsClient.Dial(dnsHost) @@ -168,24 +167,21 @@ func SendDNSQueriesOnPort(t *testing.T, domains []string, serverIP net.IP, port // SendDNSQueriesAndCheckError is a simple helper that requires no errors to be present when calling SendDNSQueries func SendDNSQueriesAndCheckError( - t *testing.T, + t require.TestingT, domains []string, serverIP net.IP, protocol string, ) (string, int, []*dns.Msg) { - t.Helper() - ip, port, resp, err := SendDNSQueries(t, domains, serverIP, protocol) + ip, port, resp, err := SendDNSQueries(domains, serverIP, protocol) require.NoError(t, err) return ip, port, resp } // SendDNSQueries is a simple helper that calls SendDNSQueriesOnPort with port 53 func SendDNSQueries( - t *testing.T, domains []string, serverIP net.IP, protocol string, ) (string, int, []*dns.Msg, error) { - t.Helper() - return SendDNSQueriesOnPort(t, domains, serverIP, "53", protocol) + return SendDNSQueriesOnPort(domains, serverIP, "53", protocol) } diff --git a/pkg/network/tracer/tracer_linux_test.go b/pkg/network/tracer/tracer_linux_test.go index 5897839fb2c41..d90928d560da8 100644 --- a/pkg/network/tracer/tracer_linux_test.go +++ b/pkg/network/tracer/tracer_linux_test.go @@ -174,7 +174,7 @@ func (s *TracerSuite) TestTCPRetransmit() { var conn *network.ConnectionStats require.EventuallyWithT(t, func(ct *assert.CollectT) { // Iterate through active connections until we find connection created above, and confirm send + recv counts - connections := getConnections(t, tr) + connections := getConnections(ct, tr) ok := false conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), connections) @@ -185,7 +185,7 @@ func (s *TracerSuite) TestTCPRetransmit() { assert.Equal(ct, 100*clientMessageSize, int(conn.Monotonic.SentBytes)) assert.Equal(ct, serverMessageSize, int(conn.Monotonic.RecvBytes)) if !tr.config.EnableEbpfless { - assert.Equal(t, os.Getpid(), int(conn.Pid)) + assert.Equal(ct, os.Getpid(), int(conn.Pid)) } assert.Equal(ct, addrPort(server.Address()), int(conn.DPort)) @@ -308,7 +308,7 @@ func (s *TracerSuite) TestTCPRTT() { require.EventuallyWithT(t, func(ct *assert.CollectT) { // Fetch connection matching source and target address - allConnections := getConnections(t, tr) + allConnections := getConnections(ct, tr) conn, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), allConnections) if !assert.True(ct, ok) { return @@ -478,7 +478,7 @@ func (s *TracerSuite) TestConntrackExpiration() { return } - connections := getConnections(t, tr) + connections := getConnections(collect, tr) t.Log(connections) // for debugging failures var ok bool conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), connections) @@ -541,10 +541,11 @@ func (s *TracerSuite) TestConntrackDelays() { _, err = c.Write([]byte("ping")) require.NoError(t, err) - require.Eventually(t, func() bool { - connections := getConnections(t, tr) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + connections := getConnections(collect, tr) conn, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), connections) - return ok && tr.conntracker.GetTranslationForConn(&conn.ConnectionTuple) != nil + require.True(collect, ok) + require.NotNil(collect, tr.conntracker.GetTranslationForConn(&conn.ConnectionTuple)) }, 3*time.Second, 100*time.Millisecond, "failed to find connection with translation") // write newline so server connections will exit @@ -629,7 +630,7 @@ func (s *TracerSuite) TestUnconnectedUDPSendIPv6() { require.NoError(t, err) require.EventuallyWithT(t, func(ct *assert.CollectT) { - connections := getConnections(t, tr) + connections := getConnections(ct, tr) outgoing := network.FilterConnections(connections, func(cs network.ConnectionStats) bool { if cs.Type != network.UDP { return false @@ -726,7 +727,7 @@ func (s *TracerSuite) TestGatewayLookupEnabled() { var clientIP string var clientPort int require.EventuallyWithT(t, func(c *assert.CollectT) { - clientIP, clientPort, _, err = testdns.SendDNSQueries(t, []string{"google.com"}, dnsAddr, "udp") + clientIP, clientPort, _, err = testdns.SendDNSQueries([]string{"google.com"}, dnsAddr, "udp") assert.NoError(c, err) }, 6*time.Second, 100*time.Millisecond, "failed to send dns query") @@ -783,7 +784,7 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { var clientIP string var clientPort int require.EventuallyWithT(t, func(c *assert.CollectT) { - clientIP, clientPort, _, err = testdns.SendDNSQueries(t, []string{destDomain}, destAddr, "udp") + clientIP, clientPort, _, err = testdns.SendDNSQueries([]string{destDomain}, destAddr, "udp") assert.NoError(c, err) }, 6*time.Second, 100*time.Millisecond, "failed to send dns query") @@ -798,7 +799,7 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { require.Nil(t, c.Via) require.EventuallyWithT(t, func(c *assert.CollectT) { - clientIP, clientPort, _, err = testdns.SendDNSQueries(t, []string{destDomain}, destAddr, "udp") + clientIP, clientPort, _, err = testdns.SendDNSQueries([]string{destDomain}, destAddr, "udp") assert.NoError(c, err) }, 6*time.Second, 100*time.Millisecond, "failed to send dns query") @@ -906,12 +907,13 @@ func (s *TracerSuite) TestGatewayLookupCrossNamespace() { _, err = c.Write([]byte("foo")) require.NoError(t, err) - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { var ok bool - conns := getConnections(t, tr) + conns := getConnections(collect, tr) t.Log(conns) conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) - return ok && conn.Direction == network.OUTGOING + require.True(collect, ok) + require.Equal(collect, network.OUTGOING, conn.Direction) }, 3*time.Second, 100*time.Millisecond) // conn.Via should be nil, since traffic is local @@ -939,12 +941,13 @@ func (s *TracerSuite) TestGatewayLookupCrossNamespace() { require.NoError(t, err) var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { var ok bool - conns := getConnections(t, tr) + conns := getConnections(collect, tr) t.Log(conns) conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) - return ok && conn.Direction == network.OUTGOING + require.True(collect, ok) + require.Equal(collect, network.OUTGOING, conn.Direction) }, 3*time.Second, 100*time.Millisecond) // traffic is local, so Via field should not be set @@ -957,7 +960,7 @@ func (s *TracerSuite) TestGatewayLookupCrossNamespace() { var clientPort int require.EventuallyWithT(t, func(c *assert.CollectT) { kernel.WithNS(test2Ns, func() error { - clientIP, clientPort, _, err = testdns.SendDNSQueries(t, []string{"google.com"}, dnsAddr, "udp") + clientIP, clientPort, _, err = testdns.SendDNSQueries([]string{"google.com"}, dnsAddr, "udp") return nil }) assert.NoError(c, err) @@ -969,10 +972,11 @@ func (s *TracerSuite) TestGatewayLookupCrossNamespace() { iif := ipRouteGet(t, "", dnsClientAddr.IP.String(), nil) ifi := ipRouteGet(t, dnsClientAddr.IP.String(), dnsAddr.String(), iif) - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { var ok bool - conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok && conn.Direction == network.OUTGOING + conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(collect, tr)) + require.True(collect, ok) + require.Equal(collect, network.OUTGOING, conn.Direction) }, 3*time.Second, 100*time.Millisecond) require.NotNil(t, conn.Via) @@ -1012,16 +1016,15 @@ func (s *TracerSuite) TestConnectionAssured() { require.NoError(t, err) } - var conn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) - var ok bool - conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) - return ok && conn.Monotonic.SentBytes > 0 && conn.Monotonic.RecvBytes > 0 + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) + conn, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), conns) + require.True(collect, ok) + require.Positive(collect, conn.Monotonic.SentBytes) + require.Positive(collect, conn.Monotonic.RecvBytes) + // verify the connection is marked as assured + require.True(collect, conn.IsAssured) }, 3*time.Second, 100*time.Millisecond, "could not find udp connection") - - // verify the connection is marked as assured - require.True(t, conn.IsAssured) } func (s *TracerSuite) TestConnectionNotAssured() { @@ -1047,16 +1050,15 @@ func (s *TracerSuite) TestConnectionNotAssured() { _, err = c.Write(genPayload(clientMessageSize)) require.NoError(t, err) - var conn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) - var ok bool - conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) - return ok && conn.Monotonic.SentBytes > 0 && conn.Monotonic.RecvBytes == 0 + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) + conn, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), conns) + require.True(collect, ok) + require.Positive(collect, conn.Monotonic.SentBytes) + require.Zero(collect, conn.Monotonic.RecvBytes) + // verify the connection is marked as not assured + require.False(collect, conn.IsAssured) }, 3*time.Second, 100*time.Millisecond, "could not find udp connection") - - // verify the connection is marked as not assured - require.False(t, conn.IsAssured) } func (s *TracerSuite) TestUDPConnExpiryTimeout() { @@ -1111,19 +1113,15 @@ func (s *TracerSuite) TestDNATIntraHostIntegration() { }) var incoming, outgoing *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { _, err = conn.Write([]byte("ping")) - if !assert.NoError(t, err, "error writing in client") { - return false - } + require.NoError(collect, err) bs := make([]byte, 4) _, err = conn.Read(bs) - if !assert.NoError(t, err) { - return false - } + require.NoError(collect, err) - conns := getConnections(t, tr) + conns := getConnections(collect, tr) t.Log(conns) outgoing, _ = findConnection(conn.LocalAddr(), conn.RemoteAddr(), conns) @@ -1131,7 +1129,9 @@ func (s *TracerSuite) TestDNATIntraHostIntegration() { t.Logf("incoming: %+v, outgoing: %+v", incoming, outgoing) - return outgoing != nil && incoming != nil && outgoing.IPTranslation != nil + require.NotNil(collect, outgoing) + require.NotNil(collect, incoming) + require.NotNil(collect, outgoing.IPTranslation) }, 3*time.Second, 100*time.Millisecond, "failed to get both incoming and outgoing connection") assert.True(t, outgoing.IntraHost, "did not find outgoing connection classified as local: %v", outgoing) @@ -1175,13 +1175,13 @@ func (s *TracerSuite) TestSelfConnect() { t.Logf("port is %d", port) - require.Eventually(t, func() bool { - conns := network.FilterConnections(getConnections(t, tr), func(cs network.ConnectionStats) bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := network.FilterConnections(getConnections(collect, tr), func(cs network.ConnectionStats) bool { return cs.SPort == uint16(port) && cs.DPort == uint16(port) && cs.Source.IsLoopback() && cs.Dest.IsLoopback() }) t.Logf("connections: %v", conns) - return len(conns) == 2 + require.Len(collect, conns, 2) }, 5*time.Second, 100*time.Millisecond, "could not find expected number of tcp connections, expected: 2") } @@ -1277,8 +1277,8 @@ func testUDPPeekCount(t *testing.T, udpnet, ip string) { var incoming *network.ConnectionStats var outgoing *network.ConnectionStats - require.Eventuallyf(t, func() bool { - conns := getConnections(t, tr) + require.EventuallyWithTf(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) if outgoing == nil { outgoing, _ = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) } @@ -1286,7 +1286,8 @@ func testUDPPeekCount(t *testing.T, udpnet, ip string) { incoming, _ = findConnection(c.RemoteAddr(), c.LocalAddr(), conns) } - return outgoing != nil && incoming != nil + require.NotNil(collect, outgoing) + require.NotNil(collect, incoming) }, 3*time.Second, 100*time.Millisecond, "couldn't find incoming and outgoing connections matching") m := outgoing.Monotonic @@ -1343,8 +1344,8 @@ func testUDPPacketSumming(t *testing.T, udpnet, ip string) { var incoming *network.ConnectionStats var outgoing *network.ConnectionStats - require.Eventuallyf(t, func() bool { - conns := getConnections(t, tr) + require.EventuallyWithTf(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) if outgoing == nil { outgoing, _ = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) } @@ -1352,7 +1353,8 @@ func testUDPPacketSumming(t *testing.T, udpnet, ip string) { incoming, _ = findConnection(c.RemoteAddr(), c.LocalAddr(), conns) } - return outgoing != nil && incoming != nil + require.NotNil(collect, outgoing) + require.NotNil(collect, incoming) }, 3*time.Second, 100*time.Millisecond, "couldn't find incoming and outgoing connections matching") m := outgoing.Monotonic @@ -1407,8 +1409,8 @@ func (s *TracerSuite) TestUDPPythonReusePort() { t.Logf("port is %d", port) conns := map[network.ConnectionTuple]network.ConnectionStats{} - require.Eventually(t, func() bool { - _conns := network.FilterConnections(getConnections(t, tr), func(cs network.ConnectionStats) bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { + _conns := network.FilterConnections(getConnections(collect, tr), func(cs network.ConnectionStats) bool { return cs.Type == network.UDP && cs.Source.IsLoopback() && cs.Dest.IsLoopback() && @@ -1421,7 +1423,7 @@ func (s *TracerSuite) TestUDPPythonReusePort() { t.Log(conns) - return len(conns) == 4 + require.Len(collect, conns, 4) }, 3*time.Second, 100*time.Millisecond, "could not find expected number of udp connections, expected: 4") var incoming, outgoing []network.ConnectionStats @@ -1781,15 +1783,12 @@ func (s *TracerSuite) TestSendfileError() { c.Close() - var conn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) - var ok bool - conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) - return ok + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) + conn, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), conns) + require.True(collect, ok) + require.Equalf(collect, int64(0), int64(conn.Monotonic.SentBytes), "sendfile data wasn't properly traced") }, 3*time.Second, 100*time.Millisecond, "couldn't find connection used by sendfile(2)") - - assert.Equalf(t, int64(0), int64(conn.Monotonic.SentBytes), "sendfile data wasn't properly traced") } func sendFile(t *testing.T, c SyscallConn, f *os.File, offset *int64, count int) (int, error) { @@ -1881,15 +1880,12 @@ func (s *TracerSuite) TestShortWrite() { c, err := net.FileConn(f) require.NoError(t, err) - var conn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) - var ok bool - conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), conns) - return ok + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) + conn, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), conns) + require.True(collect, ok) + require.Equal(collect, sent, conn.Monotonic.SentBytes) }, 3*time.Second, 100*time.Millisecond, "couldn't find connection used by short write") - - assert.Equal(t, sent, conn.Monotonic.SentBytes) } func (s *TracerSuite) TestKprobeAttachWithKprobeEvents() { @@ -1972,14 +1968,11 @@ func (s *TracerSuite) TestBlockingReadCounts() { assert.Equal(collect, 6, read) }, 10*time.Second, 100*time.Millisecond, "failed to get required bytes") - var conn *network.ConnectionStats - require.Eventually(t, func() bool { - var found bool - conn, found = findConnection(c.(*net.TCPConn).LocalAddr(), c.(*net.TCPConn).RemoteAddr(), getConnections(t, tr)) - return found + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conn, found := findConnection(c.(*net.TCPConn).LocalAddr(), c.(*net.TCPConn).RemoteAddr(), getConnections(collect, tr)) + require.True(collect, found) + require.Equal(collect, uint64(read), conn.Monotonic.RecvBytes) }, 3*time.Second, 100*time.Millisecond) - - assert.Equal(t, uint64(read), conn.Monotonic.RecvBytes) } func (s *TracerSuite) TestPreexistingConnectionDirection() { @@ -2020,15 +2013,16 @@ func (s *TracerSuite) TestPreexistingConnectionDirection() { c.Close() var incoming, outgoing *network.ConnectionStats - require.Eventually(t, func() bool { - connections := getConnections(t, tr) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + connections := getConnections(collect, tr) if outgoing == nil { outgoing, _ = findConnection(c.LocalAddr(), c.RemoteAddr(), connections) } if incoming == nil { incoming, _ = findConnection(c.RemoteAddr(), c.LocalAddr(), connections) } - return incoming != nil && outgoing != nil + require.NotNil(collect, outgoing) + require.NotNil(collect, incoming) }, 3*time.Second, 100*time.Millisecond, "could not find connection incoming and outgoing connections") m := outgoing.Monotonic @@ -2136,14 +2130,12 @@ func (s *TracerSuite) TestUDPIncomingDirectionFix() { raddr, err := net.ResolveUDPAddr("udp", server.address) require.NoError(t, err) - var conn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) - conn, _ = findConnection(net.UDPAddrFromAddrPort(ap), raddr, conns) - return conn != nil + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) + conn, _ := findConnection(net.UDPAddrFromAddrPort(ap), raddr, conns) + require.NotNil(collect, conn) + require.Equal(collect, network.OUTGOING, conn.Direction) }, 3*time.Second, 100*time.Millisecond) - - assert.Equal(t, network.OUTGOING, conn.Direction) } func TestEbpfConntrackerFallback(t *testing.T) { @@ -2413,31 +2405,24 @@ LOOP: // get connections, the client connection will still // not be in the closed state, so duration will the // timestamp of when it was created - var conn *network.ConnectionStats require.EventuallyWithT(t, func(collect *assert.CollectT) { - conns := getConnections(t, tr) - var found bool - conn, found = findConnection(c.LocalAddr(), srv.Addr(), conns) - assert.True(collect, found, "could not find connection") - + conns := getConnections(collect, tr) + conn, found := findConnection(c.LocalAddr(), srv.Addr(), conns) + require.True(collect, found, "could not find connection") + // all we can do is verify it is > 0 + require.Greater(collect, conn.Duration, time.Duration(0)) }, 3*time.Second, 100*time.Millisecond, "could not find connection") - // all we can do is verify it is > 0 - assert.Greater(t, conn.Duration, time.Duration(0)) require.NoError(t, c.Close(), "error closing client connection") require.EventuallyWithT(t, func(collect *assert.CollectT) { - var found bool - conn, found = findConnection(c.LocalAddr(), srv.Addr(), getConnections(t, tr)) - if !assert.True(collect, found, "could not find connection") { - return - } - assert.True(collect, conn.IsClosed, "connection should be closed") + conn, found := findConnection(c.LocalAddr(), srv.Addr(), getConnections(t, tr)) + require.True(collect, found, "could not find connection") + require.True(collect, conn.IsClosed, "connection should be closed") + // after closing the client connection, the duration should be + // updated to a value between 1s and 2s + require.Greater(collect, conn.Duration, time.Second, "connection duration should be between 1 and 2 seconds") + require.Less(collect, conn.Duration, 2*time.Second, "connection duration should be between 1 and 2 seconds") }, 3*time.Second, 100*time.Millisecond, "could not find closed connection") - - // after closing the client connection, the duration should be - // updated to a value between 1s and 2s - assert.Greater(t, conn.Duration, time.Second, "connection duration should be between 1 and 2 seconds") - assert.Less(t, conn.Duration, 2*time.Second, "connection duration should be between 1 and 2 seconds") } var failedConnectionsBuildModes = map[ebpftest.BuildMode]struct{}{ @@ -2503,19 +2488,17 @@ func (s *TracerSuite) TestTCPFailureConnectionTimeout() { localAddr := fmt.Sprintf("127.0.0.1:%d", port) // Check if the connection was recorded as failed due to timeout - var conn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) // 110 is the errno for ETIMEDOUT - conn = findFailedConnection(t, localAddr, srvAddr, conns, 110) - return conn != nil + conn := findFailedConnection(t, localAddr, srvAddr, conns, 110) + require.NotNil(collect, conn) + assert.Equal(collect, uint32(0), conn.TCPFailures[104], "expected 0 connection reset") + assert.Equal(collect, uint32(0), conn.TCPFailures[111], "expected 0 connection refused") + assert.Equal(collect, uint32(1), conn.TCPFailures[110], "expected 1 connection timeout") + assert.Equal(collect, uint64(0), conn.Monotonic.SentBytes, "expected 0 bytes sent") + assert.Equal(collect, uint64(0), conn.Monotonic.RecvBytes, "expected 0 bytes received") }, 3*time.Second, 100*time.Millisecond, "Failed connection not recorded properly") - - assert.Equal(t, uint32(0), conn.TCPFailures[104], "expected 0 connection reset") - assert.Equal(t, uint32(0), conn.TCPFailures[111], "expected 0 connection refused") - assert.Equal(t, uint32(1), conn.TCPFailures[110], "expected 1 connection timeout") - assert.Equal(t, uint64(0), conn.Monotonic.SentBytes, "expected 0 bytes sent") - assert.Equal(t, uint64(0), conn.Monotonic.RecvBytes, "expected 0 bytes received") } func (s *TracerSuite) TestTCPFailureConnectionResetWithDNAT() { @@ -2561,10 +2544,11 @@ func (s *TracerSuite) TestTCPFailureConnectionResetWithDNAT() { // Check if the connection was recorded as reset var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { // 104 is the errno for ECONNRESET - conn = findFailedConnection(t, c.LocalAddr().String(), serverAddr, getConnections(t, tr), 104) - return conn != nil + // findFailedConnection gets `t` as it needs to log, it does not assert so no conversion is needed. + conn = findFailedConnection(t, c.LocalAddr().String(), serverAddr, getConnections(collect, tr), 104) + require.NotNil(collect, conn) }, 3*time.Second, 100*time.Millisecond, "Failed connection not recorded properly") require.NoError(t, c.Close(), "error closing client connection") diff --git a/pkg/network/tracer/tracer_test.go b/pkg/network/tracer/tracer_test.go index b032aabe44c5f..a108e11363fef 100644 --- a/pkg/network/tracer/tracer_test.go +++ b/pkg/network/tracer/tracer_test.go @@ -198,12 +198,13 @@ func (s *TracerSuite) TestTCPSendAndReceive() { require.NoError(t, err) var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { // Iterate through active connections until we find connection created above, and confirm send + recv counts - connections := getConnections(t, tr) + connections := getConnections(collect, tr) var ok bool conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), connections) - return conn != nil && ok + require.True(collect, ok) + require.NotNil(collect, conn) }, 3*time.Second, 100*time.Millisecond, "failed to find connection") m := conn.Monotonic @@ -246,10 +247,10 @@ func (s *TracerSuite) TestTCPShortLived() { c.Close() var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { var ok bool - conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(t, tr)) - return ok + conn, ok = findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(collect, tr)) + require.True(collect, ok) }, 3*time.Second, 100*time.Millisecond, "connection not found") m := conn.Monotonic @@ -399,14 +400,15 @@ func (s *TracerSuite) TestTCPConnsReported() { var reverse *network.ConnectionStats var okForward, okReverse bool // for ebpfless, it takes time for the packet capture to arrive, so poll - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { // Test - connections := getConnections(t, tr) + connections := getConnections(collect, tr) // Server-side forward, okForward = findConnection(c.RemoteAddr(), c.LocalAddr(), connections) + require.True(collect, okForward) // Client-side reverse, okReverse = findConnection(c.LocalAddr(), c.RemoteAddr(), connections) - return okForward && okReverse + require.True(collect, okReverse) }, 3*time.Second, 100*time.Millisecond, "connection not found") assert.Equal(t, network.INCOMING, forward.Direction) @@ -470,7 +472,7 @@ func testUDPSendAndReceive(t *testing.T, tr *Tracer, addr string) { // Iterate through active connections until we find connection created above, and confirm send + recv counts require.EventuallyWithT(t, func(ct *assert.CollectT) { // use t instead of ct because getConnections uses require (not assert), and we get a better error message - connections := getConnections(t, tr) + connections := getConnections(ct, tr) incoming, ok := findConnection(c.RemoteAddr(), c.LocalAddr(), connections) if assert.True(ct, ok, "unable to find incoming connection") { assert.Equal(ct, network.INCOMING, incoming.Direction) @@ -482,12 +484,12 @@ func testUDPSendAndReceive(t *testing.T, tr *Tracer, addr string) { } outgoing, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), connections) - if assert.True(t, ok, "unable to find outgoing connection") { - assert.Equal(t, network.OUTGOING, outgoing.Direction) + if assert.True(ct, ok, "unable to find outgoing connection") { + assert.Equal(ct, network.OUTGOING, outgoing.Direction) - assert.Equal(t, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") - assert.Equal(t, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") - assert.True(t, outgoing.IntraHost, "outgoing intrahost") + assert.Equal(ct, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") + assert.Equal(ct, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") + assert.True(ct, outgoing.IntraHost, "outgoing intrahost") } }, 3*time.Second, 100*time.Millisecond) @@ -579,14 +581,14 @@ func (s *TracerSuite) TestLocalDNSCollectionEnabled() { assert.NoError(t, err) // Iterate through active connections making sure theres at least one connection - require.Eventually(t, func() bool { - for _, c := range getConnections(t, tr).Conns { + require.EventuallyWithT(t, func(collect *assert.CollectT) { + for _, c := range getConnections(collect, tr).Conns { if isLocalDNS(c) { - return true + return } } - return false + require.Fail(collect, "could not find connection") }, 3*time.Second, 100*time.Millisecond, "could not find connection") } @@ -651,15 +653,15 @@ func (s *TracerSuite) TestShouldExcludeEmptyStatsConnection() { assert.NoError(t, err) var zeroConn network.ConnectionStats - require.Eventually(t, func() bool { - cxs := getConnections(t, tr) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + cxs := getConnections(collect, tr) for _, c := range cxs.Conns { if c.Dest.String() == "127.0.0.1" && c.DPort == 80 { zeroConn = c - return true + return } } - return false + require.Fail(collect, "could not find connection") }, 2*time.Second, 100*time.Millisecond) // next call should not have the same connection @@ -1094,9 +1096,9 @@ func (s *TracerSuite) TestTCPEstablished() { var ok bool // for ebpfless, wait for the packet capture to appear - require.Eventually(t, func() bool { - conn, ok = findConnection(laddr, raddr, getConnections(t, tr)) - return ok + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conn, ok = findConnection(laddr, raddr, getConnections(collect, tr)) + require.True(collect, ok) }, 3*time.Second, 100*time.Millisecond, "couldn't find connection") require.True(t, ok) @@ -1106,10 +1108,10 @@ func (s *TracerSuite) TestTCPEstablished() { c.Close() // Wait for the connection to be sent from the perf buffer - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { var ok bool - conn, ok = findConnection(laddr, raddr, getConnections(t, tr)) - return ok + conn, ok = findConnection(laddr, raddr, getConnections(collect, tr)) + require.True(collect, ok) }, 3*time.Second, 100*time.Millisecond, "couldn't find connection") require.True(t, ok) @@ -1140,10 +1142,10 @@ func (s *TracerSuite) TestTCPEstablishedPreExistingConn() { // Wait for the connection to be sent from the perf buffer var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { var ok bool - conn, ok = findConnection(laddr, raddr, getConnections(t, tr)) - return ok + conn, ok = findConnection(laddr, raddr, getConnections(collect, tr)) + require.True(collect, ok) }, 3*time.Second, 100*time.Millisecond, "couldn't find connection") m := conn.Monotonic @@ -1167,7 +1169,7 @@ func (s *TracerSuite) TestUnconnectedUDPSendIPv4() { require.NoError(t, err) require.EventuallyWithT(t, func(ct *assert.CollectT) { - connections := getConnections(t, tr) + connections := getConnections(ct, tr) outgoing := network.FilterConnections(connections, func(cs network.ConnectionStats) bool { return cs.DPort == uint16(remotePort) }) @@ -1198,7 +1200,7 @@ func (s *TracerSuite) TestConnectedUDPSendIPv6() { var outgoing []network.ConnectionStats require.EventuallyWithT(t, func(ct *assert.CollectT) { - connections := getConnections(t, tr) + connections := getConnections(ct, tr) outgoing = network.FilterConnections(connections, func(cs network.ConnectionStats) bool { return cs.DPort == uint16(remotePort) }) @@ -1247,8 +1249,8 @@ func (s *TracerSuite) TestTCPDirection() { // Iterate through active connections until we find connection created above var outgoingConns []network.ConnectionStats var incomingConns []network.ConnectionStats - require.Eventuallyf(t, func() bool { - conns := getConnections(t, tr) + require.EventuallyWithTf(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) if len(outgoingConns) == 0 { outgoingConns = network.FilterConnections(conns, func(cs network.ConnectionStats) bool { return fmt.Sprintf("%s:%d", cs.Dest, cs.DPort) == serverAddr @@ -1260,7 +1262,8 @@ func (s *TracerSuite) TestTCPDirection() { }) } - return len(outgoingConns) == 1 && len(incomingConns) == 1 + require.Len(collect, outgoingConns, 1) + require.Len(collect, incomingConns, 1) }, 3*time.Second, 10*time.Millisecond, "couldn't find incoming and outgoing http connections matching: %s", serverAddr) // Verify connection directions @@ -1290,11 +1293,11 @@ func (s *TracerSuite) TestTCPFailureConnectionRefused() { // Check if the connection was recorded as refused var foundConn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) // Check for the refusal record foundConn = findFailedConnectionByRemoteAddr(srvAddr, conns, 111) - return foundConn != nil + require.NotNil(collect, foundConn) }, 3*time.Second, 100*time.Millisecond, "Failed connection not recorded properly") assert.Equal(t, uint32(1), foundConn.TCPFailures[111], "expected 1 connection refused") @@ -1342,10 +1345,11 @@ func (s *TracerSuite) TestTCPFailureConnectionResetWithData() { // Check if the connection was recorded as reset var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { // 104 is the errno for ECONNRESET - conn = findFailedConnection(t, c.LocalAddr().String(), serverAddr, getConnections(t, tr), 104) - return conn != nil + // findFailedConnection needs `t` for logging, hence no need to pass `collect`. + conn = findFailedConnection(t, c.LocalAddr().String(), serverAddr, getConnections(collect, tr), 104) + require.NotNil(collect, conn) }, 3*time.Second, 100*time.Millisecond, "Failed connection not recorded properly") require.NoError(t, c.Close(), "error closing client connection") @@ -1391,11 +1395,12 @@ func (s *TracerSuite) TestTCPFailureConnectionResetNoData() { // Check if the connection was recorded as reset var conn *network.ConnectionStats - require.Eventually(t, func() bool { - conns := getConnections(t, tr) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + conns := getConnections(collect, tr) // 104 is the errno for ECONNRESET + // findFailedConnection needs `t` for logging, hence no need to pass `collect`. conn = findFailedConnection(t, c.LocalAddr().String(), serverAddr, conns, 104) - return conn != nil + require.NotNil(collect, conn) }, 3*time.Second, 100*time.Millisecond, "Failed connection not recorded properly") require.NoError(t, c.Close(), "error closing client connection") diff --git a/pkg/network/usm/kafka_monitor_test.go b/pkg/network/usm/kafka_monitor_test.go index 645db1e3bc064..c4f3e30c08ad4 100644 --- a/pkg/network/usm/kafka_monitor_test.go +++ b/pkg/network/usm/kafka_monitor_test.go @@ -521,14 +521,14 @@ func (s *KafkaProtocolParsingSuite) testKafkaProtocolParsing(t *testing.T, tls b require.NoError(t, client.Client.ProduceSync(ctxTimeout, record).FirstErr(), "record had a produce error while synchronously producing") var telemetryMap *kafka.RawKernelTelemetry - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(collect *assert.CollectT) { telemetryMap, err = kafka.GetKernelTelemetryMap(monitor.ebpfProgram.Manager.Manager) - require.NoError(t, err) + require.NoError(collect, err) // Ensure that the other buckets remain unchanged before verifying the expected bucket. for idx := 0; idx < kafka.TopicNameBuckets; idx++ { if idx != tt.expectedBucketIndex { - require.Equal(t, currentRawKernelTelemetry.Topic_name_size_buckets[idx], + require.Equal(collect, currentRawKernelTelemetry.Topic_name_size_buckets[idx], telemetryMap.Topic_name_size_buckets[idx], "Expected bucket (%d) to remain unchanged", idx) } @@ -536,7 +536,7 @@ func (s *KafkaProtocolParsingSuite) testKafkaProtocolParsing(t *testing.T, tls b // Verify that the expected bucket contains the correct number of occurrences. expectedNumberOfOccurrences := fixCount(2) // (1 produce request + 1 fetch request) - return uint64(expectedNumberOfOccurrences)+currentRawKernelTelemetry.Topic_name_size_buckets[tt.expectedBucketIndex] == telemetryMap.Topic_name_size_buckets[tt.expectedBucketIndex] + require.Equal(collect, uint64(expectedNumberOfOccurrences)+currentRawKernelTelemetry.Topic_name_size_buckets[tt.expectedBucketIndex], telemetryMap.Topic_name_size_buckets[tt.expectedBucketIndex]) }, time.Second*3, time.Millisecond*100) // Update the current raw kernel telemetry for the next iteration diff --git a/pkg/network/usm/tests/tracer_usm_linux_test.go b/pkg/network/usm/tests/tracer_usm_linux_test.go index cb7cf47e30e4a..5cbbef2763777 100644 --- a/pkg/network/usm/tests/tracer_usm_linux_test.go +++ b/pkg/network/usm/tests/tracer_usm_linux_test.go @@ -429,15 +429,15 @@ func (s *USMSuite) TestIgnoreTLSClassificationIfApplicationProtocolWasDetected() // Perform the TLS handshake require.NoError(t, tlsConn.Handshake()) - - require.Eventually(t, func() bool { - payload := getConnections(t, tr) + require.EventuallyWithT(t, func(collect *assert.CollectT) { + payload := getConnections(collect, tr) for _, c := range payload.Conns { if c.DPort == srvPortU16 || c.SPort == srvPortU16 { - return c.ProtocolStack.Contains(protocols.TLS) == tt.shouldBeTLS + require.Equal(collect, c.ProtocolStack.Contains(protocols.TLS), tt.shouldBeTLS) + return } } - return false + require.Fail(collect, "") }, 10*time.Second, 100*time.Millisecond) }) } @@ -502,14 +502,14 @@ func (s *USMSuite) TestTLSClassification() { }, validation: func(t *testing.T, tr *tracer.Tracer) { // Iterate through active connections until we find connection created above - require.Eventuallyf(t, func() bool { - payload := getConnections(t, tr) + require.EventuallyWithTf(t, func(collect *assert.CollectT) { + payload := getConnections(collect, tr) for _, c := range payload.Conns { if c.DPort == port && c.ProtocolStack.Contains(protocols.TLS) { - return true + return } } - return false + require.Fail(collect, "") }, 4*time.Second, 100*time.Millisecond, "couldn't find TLS connection matching: dst port %v", portAsString) }, }) @@ -574,8 +574,8 @@ func (s *USMSuite) TestTLSClassificationAlreadyRunning() { // Iterate through active connections until we find connection created above var foundIncoming, foundOutgoing bool - require.Eventuallyf(t, func() bool { - payload := getConnections(t, tr) + require.EventuallyWithTf(t, func(collect *assert.CollectT) { + payload := getConnections(collect, tr) for _, c := range payload.Conns { if !foundIncoming && c.DPort == uint16(portAsValue) && c.ProtocolStack.Contains(protocols.TLS) { @@ -586,7 +586,8 @@ func (s *USMSuite) TestTLSClassificationAlreadyRunning() { foundOutgoing = true } } - return foundIncoming && foundOutgoing + require.True(collect, foundIncoming) + require.True(collect, foundOutgoing) }, 4*time.Second, 100*time.Millisecond, "couldn't find matching TLS connection") } From 6615d8d6d1ba839ed20fce6448aa65c1911d79f6 Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Tue, 10 Dec 2024 08:56:16 +0100 Subject: [PATCH 44/52] [CWS] add `rocky_9.4` to CWS KMT tested platforms (#31886) --- .../kernel_matrix_testing/security_agent.yml | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index 26e9293844e9e..b7f4b80cc6ed1 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -147,6 +147,7 @@ kmt_run_secagent_tests_x64: - "oracle_9.3" - "rocky_8.5" - "rocky_9.3" + - "rocky_9.4" - "opensuse_15.3" - "opensuse_15.5" - "suse_12.5" @@ -268,6 +269,7 @@ kmt_run_secagent_tests_x64_docker: - "oracle_9.3" - "rocky_8.5" - "rocky_9.3" + - "rocky_9.4" TEST_SET: [cws_docker] after_script: - !reference [.collect_outcomes_kmt] @@ -301,6 +303,7 @@ kmt_run_secagent_tests_arm64: - "oracle_9.3" - "rocky_8.5" - "rocky_9.3" + - "rocky_9.4" - "opensuse_15.5" TEST_SET: [cws_host] after_script: @@ -395,6 +398,7 @@ kmt_run_secagent_tests_arm64_docker: - "oracle_9.3" - "rocky_8.5" - "rocky_9.3" + - "rocky_9.4" TEST_SET: ["cws_docker"] after_script: - !reference [.collect_outcomes_kmt] @@ -407,35 +411,54 @@ kmt_run_secagent_tests_arm64_docker: variables: TEST_COMPONENT: security-agent -kmt_secagent_cleanup_arm64: - when: always +.kmt_secagent_tests_join: + stage: kernel_matrix_testing_cleanup + rules: !reference [.on_security_agent_changes_or_manual] + image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES + tags: ["arch:arm64"] + script: + - echo "nothing to do here" + +kmt_secagent_tests_join_arm64: extends: - - .kmt_secagent_cleanup + - .kmt_secagent_tests_join needs: - - kmt_setup_env_secagent_arm64 - kmt_run_secagent_tests_arm64 - kmt_run_secagent_tests_arm64_ad - kmt_run_secagent_tests_arm64_ebpfless - kmt_run_secagent_tests_arm64_fentry - kmt_run_secagent_tests_arm64_docker - - upload_dependencies_secagent_arm64 + +kmt_secagent_cleanup_arm64: + when: always + extends: + - .kmt_secagent_cleanup + needs: + - kmt_setup_env_secagent_arm64 + - kmt_secagent_tests_join_arm64 - upload_secagent_tests_arm64 variables: ARCH: arm64 INSTANCE_TYPE: "m6gd.metal" -kmt_secagent_cleanup_x64: - when: always +kmt_secagent_tests_join_x64: extends: - - .kmt_secagent_cleanup + - .kmt_secagent_tests_join needs: - - kmt_setup_env_secagent_x64 - kmt_run_secagent_tests_x64 - kmt_run_secagent_tests_x64_required - kmt_run_secagent_tests_x64_ad - kmt_run_secagent_tests_x64_ebpfless - kmt_run_secagent_tests_x64_fentry - kmt_run_secagent_tests_x64_docker + +kmt_secagent_cleanup_x64: + when: always + extends: + - .kmt_secagent_cleanup + needs: + - kmt_setup_env_secagent_x64 + - kmt_secagent_tests_join_x64 - upload_dependencies_secagent_x64 - upload_secagent_tests_x64 variables: From 27d7ef8cfc37bdf9801b4bf8922c43ada21d500e Mon Sep 17 00:00:00 2001 From: pducolin <45568537+pducolin@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:01:31 +0100 Subject: [PATCH 45/52] [tasks] add download link to update python (#31831) --- tasks/libs/common/utils.py | 12 ++++++++++++ tasks/setup.py | 15 ++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tasks/libs/common/utils.py b/tasks/libs/common/utils.py index 175b6d848d4d7..56e96ba23403f 100644 --- a/tasks/libs/common/utils.py +++ b/tasks/libs/common/utils.py @@ -661,3 +661,15 @@ def agent_working_directory(): from tasks.libs.common.worktree import LOCAL_DIRECTORY, WORKTREE_DIRECTORY, is_worktree return WORKTREE_DIRECTORY if is_worktree() else LOCAL_DIRECTORY + + +def is_macos(): + return sys.platform == 'darwin' + + +def is_linux(): + return sys.platform.startswith('linux') + + +def is_windows(): + return sys.platform == 'win32' diff --git a/tasks/setup.py b/tasks/setup.py index 5e1eeee98d1f1..cf75c396f5779 100644 --- a/tasks/setup.py +++ b/tasks/setup.py @@ -19,7 +19,7 @@ from tasks.libs.common.color import Color, color_message from tasks.libs.common.git import get_default_branch from tasks.libs.common.status import Status -from tasks.libs.common.utils import running_in_pyapp +from tasks.libs.common.utils import is_linux, is_windows, running_in_pyapp if TYPE_CHECKING: from collections.abc import Generator @@ -145,10 +145,15 @@ def check_python_version(_ctx) -> SetupResult: status = Status.OK if tuple(sys.version_info)[:2] != tuple(int(d) for d in expected_version.split(".")): status = Status.FAIL - message = ( - f"Python version is {sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}. " - "Please update your environment: https://datadoghq.dev/datadog-agent/setup/#python-dependencies" - ) + install_message = f"Please install Python {expected_version} with 'brew install python@{expected_version}'" + if is_windows(): + install_message = f"Please install Python {expected_version} from https://www.python.org/downloads/windows/" + elif is_linux(): + install_message = ( + f"Please install Python {expected_version} with 'sudo apt-get install python{expected_version}-dev'" + ) + + message = f"Python version out of date, current is {sys.version_info[0]}.{sys.version_info[1]} while expected is {expected_version}.\n{install_message}" return SetupResult("Check Python version", status, message) From d8125d7aa9970af55741c7425af86bb348562586 Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Tue, 10 Dec 2024 10:28:14 +0200 Subject: [PATCH 46/52] usm: process monitor: Increase number of runners (#31878) --- pkg/process/monitor/process_monitor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/process/monitor/process_monitor.go b/pkg/process/monitor/process_monitor.go index 8710aa63e324f..57c174840b66e 100644 --- a/pkg/process/monitor/process_monitor.go +++ b/pkg/process/monitor/process_monitor.go @@ -221,7 +221,10 @@ func (pm *ProcessMonitor) initNetlinkProcessEventMonitor() error { // initCallbackRunner runs multiple workers that run tasks sent over a queue. func (pm *ProcessMonitor) initCallbackRunner() { - cpuNum := runtime.NumVCPU() + cpuNum, err := kernel.PossibleCPUs() + if err != nil { + cpuNum = runtime.NumVCPU() + } pm.callbackRunner = make(chan func(), pendingCallbacksQueueSize) pm.callbackRunnerStopChannel = make(chan struct{}) pm.callbackRunnersWG.Add(cpuNum) From e70c007014b58d371c4494566eb39361d57dd872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 10 Dec 2024 09:28:22 +0100 Subject: [PATCH 47/52] omnibus: snmp-traps: drop dependency on datadog-agent (#31868) --- omnibus/config/projects/agent.rb | 3 --- omnibus/config/software/datadog-agent-dependencies.rb | 3 +++ omnibus/config/software/datadog-agent.rb | 3 ++- omnibus/config/software/snmp-traps.rb | 3 --- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/omnibus/config/projects/agent.rb b/omnibus/config/projects/agent.rb index b3ab9c10ae1ad..51bacc89a45d0 100644 --- a/omnibus/config/projects/agent.rb +++ b/omnibus/config/projects/agent.rb @@ -221,9 +221,6 @@ # ------------------------------------ if do_build - # Include traps db file in snmp.d/traps_db/ - dependency 'snmp-traps' - # Datadog agent dependency 'datadog-agent' diff --git a/omnibus/config/software/datadog-agent-dependencies.rb b/omnibus/config/software/datadog-agent-dependencies.rb index fd6712983b10b..fc34796e2fa4b 100644 --- a/omnibus/config/software/datadog-agent-dependencies.rb +++ b/omnibus/config/software/datadog-agent-dependencies.rb @@ -33,6 +33,9 @@ dependency 'libpcap' if linux_target? and !heroku_target? # system-probe dependency +# Include traps db file in snmp.d/traps_db/ +dependency 'snmp-traps' + # Additional software if windows_target? if ENV['WINDOWS_DDNPM_DRIVER'] and not ENV['WINDOWS_DDNPM_DRIVER'].empty? diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index 8f24178d2c07e..a05e3b71bd877 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -120,7 +120,8 @@ # move around bin and config files move 'bin/agent/dist/datadog.yaml', "#{conf_dir}/datadog.yaml.example" - move 'bin/agent/dist/conf.d', "#{conf_dir}/" + copy 'bin/agent/dist/conf.d/.', "#{conf_dir}" + delete 'bin/agent/dist/conf.d' unless windows_target? copy 'bin/agent', "#{install_dir}/bin/" diff --git a/omnibus/config/software/snmp-traps.rb b/omnibus/config/software/snmp-traps.rb index 5a021b01ed8d4..bb13fc542fd3b 100644 --- a/omnibus/config/software/snmp-traps.rb +++ b/omnibus/config/software/snmp-traps.rb @@ -1,9 +1,6 @@ name "snmp-traps" default_version "0.4.0" -# Needs the configuration folder as created in datadog-agent -dependency 'datadog-agent' - source :url => "https://s3.amazonaws.com/dd-agent-omnibus/snmp_traps_db/dd_traps_db-#{version}.json.gz", :sha256 => "04fb9d43754c2656edf35f08fbad11ba8dc20d52654962933f3dd8f4d463b42c", :target_filename => "dd_traps_db.json.gz" From 5ad43ca4c34e476d4757045fabdcf5ed9288dd52 Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Tue, 10 Dec 2024 10:55:55 +0200 Subject: [PATCH 48/52] usm: process monitor: reuse consumers.NewProcessConsumer (#31864) --- cmd/system-probe/modules/eventmonitor.go | 7 +- .../modules/eventmonitor_linux.go | 28 ++++++- .../modules/eventmonitor_windows.go | 4 +- pkg/ebpf/uprobes/attacher_test.go | 7 +- pkg/network/usm/monitor_tls_test.go | 8 +- .../usm/sharedlibraries/watcher_test.go | 7 +- pkg/process/monitor/process_monitor.go | 79 +++---------------- pkg/process/monitor/process_monitor_test.go | 30 +++---- pkg/process/monitor/testutil/testutil.go | 27 ------- 9 files changed, 55 insertions(+), 142 deletions(-) delete mode 100644 pkg/process/monitor/testutil/testutil.go diff --git a/cmd/system-probe/modules/eventmonitor.go b/cmd/system-probe/modules/eventmonitor.go index f74b13bec36af..41707ad513dec 100644 --- a/cmd/system-probe/modules/eventmonitor.go +++ b/cmd/system-probe/modules/eventmonitor.go @@ -85,14 +85,9 @@ func createEventMonitorModule(_ *sysconfigtypes.Config, deps module.FactoryDepen netconfig := netconfig.New() if netconfig.EnableUSMEventStream { - procmonconsumer, err := createProcessMonitorConsumer(evm, netconfig) - if err != nil { + if err := createProcessMonitorConsumer(evm, netconfig); err != nil { return nil, err } - if procmonconsumer != nil { - evm.RegisterEventConsumer(procmonconsumer) - log.Info("USM process monitoring consumer initialized") - } } gpucfg := gpuconfig.New() diff --git a/cmd/system-probe/modules/eventmonitor_linux.go b/cmd/system-probe/modules/eventmonitor_linux.go index 498e3b9feb92d..a3ee2f33666da 100644 --- a/cmd/system-probe/modules/eventmonitor_linux.go +++ b/cmd/system-probe/modules/eventmonitor_linux.go @@ -11,11 +11,13 @@ import ( "github.com/DataDog/datadog-agent/cmd/system-probe/api/module" "github.com/DataDog/datadog-agent/cmd/system-probe/config" "github.com/DataDog/datadog-agent/pkg/eventmonitor" + "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers" netconfig "github.com/DataDog/datadog-agent/pkg/network/config" usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config" usmstate "github.com/DataDog/datadog-agent/pkg/network/usm/state" - procmon "github.com/DataDog/datadog-agent/pkg/process/monitor" + "github.com/DataDog/datadog-agent/pkg/process/monitor" secconfig "github.com/DataDog/datadog-agent/pkg/security/config" + "github.com/DataDog/datadog-agent/pkg/util/log" ) // EventMonitor - Event monitor Factory @@ -28,10 +30,28 @@ var EventMonitor = module.Factory{ }, } -func createProcessMonitorConsumer(evm *eventmonitor.EventMonitor, config *netconfig.Config) (eventmonitor.EventConsumer, error) { +const ( + eventMonitorID = "PROCESS_MONITOR" + eventMonitorChannelSize = 500 +) + +var ( + eventTypes = []consumers.ProcessConsumerEventTypes{ + consumers.ExecEventType, + consumers.ExitEventType, + } +) + +func createProcessMonitorConsumer(evm *eventmonitor.EventMonitor, config *netconfig.Config) error { if !usmconfig.IsUSMSupportedAndEnabled(config) || !usmconfig.NeedProcessMonitor(config) || usmstate.Get() != usmstate.Running { - return nil, nil + return nil } - return procmon.NewProcessMonitorEventConsumer(evm) + consumer, err := consumers.NewProcessConsumer(eventMonitorID, eventMonitorChannelSize, eventTypes, evm) + if err != nil { + return err + } + monitor.InitializeEventConsumer(consumer) + log.Info("USM process monitoring consumer initialized") + return nil } diff --git a/cmd/system-probe/modules/eventmonitor_windows.go b/cmd/system-probe/modules/eventmonitor_windows.go index 870c707f8f3e7..7ec627c076d67 100644 --- a/cmd/system-probe/modules/eventmonitor_windows.go +++ b/cmd/system-probe/modules/eventmonitor_windows.go @@ -21,8 +21,8 @@ var EventMonitor = module.Factory{ Fn: createEventMonitorModule, } -func createProcessMonitorConsumer(_ *eventmonitor.EventMonitor, _ *netconfig.Config) (eventmonitor.EventConsumer, error) { - return nil, nil +func createProcessMonitorConsumer(_ *eventmonitor.EventMonitor, _ *netconfig.Config) error { + return nil } func createGPUProcessEventConsumer(_ *eventmonitor.EventMonitor) error { diff --git a/pkg/ebpf/uprobes/attacher_test.go b/pkg/ebpf/uprobes/attacher_test.go index 26428868c1d70..be848bb5c6afb 100644 --- a/pkg/ebpf/uprobes/attacher_test.go +++ b/pkg/ebpf/uprobes/attacher_test.go @@ -27,14 +27,12 @@ import ( "github.com/DataDog/datadog-agent/pkg/ebpf/bytecode" "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" "github.com/DataDog/datadog-agent/pkg/ebpf/prebuilt" - eventmonitortestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/testutil" + "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers/testutil" "github.com/DataDog/datadog-agent/pkg/network/go/bininspect" "github.com/DataDog/datadog-agent/pkg/network/usm/sharedlibraries" fileopener "github.com/DataDog/datadog-agent/pkg/network/usm/sharedlibraries/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/process/monitor" - procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" - secutils "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/util/kernel" ) @@ -800,8 +798,7 @@ func launchProcessMonitor(t *testing.T, useEventStream bool) *monitor.ProcessMon t.Cleanup(pm.Stop) require.NoError(t, pm.Initialize(useEventStream)) if useEventStream { - secutils.SetCachedHostname("test-hostname") - eventmonitortestutil.StartEventMonitor(t, procmontestutil.RegisterProcessMonitorEventConsumer) + monitor.InitializeEventConsumer(testutil.NewTestProcessConsumer(t)) } return pm diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 322f9be425da4..44ca4244b07a1 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -31,7 +31,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" "github.com/DataDog/datadog-agent/pkg/ebpf/prebuilt" - eventmonitortestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/testutil" + consumerstestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers/testutil" "github.com/DataDog/datadog-agent/pkg/network" "github.com/DataDog/datadog-agent/pkg/network/config" "github.com/DataDog/datadog-agent/pkg/network/protocols" @@ -44,8 +44,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/usm/consts" usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" - procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" - secutils "github.com/DataDog/datadog-agent/pkg/security/utils" + "github.com/DataDog/datadog-agent/pkg/process/monitor" globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) @@ -870,8 +869,7 @@ func setupUSMTLSMonitor(t *testing.T, cfg *config.Config) *Monitor { require.NoError(t, err) require.NoError(t, usmMonitor.Start()) if cfg.EnableUSMEventStream && usmconfig.NeedProcessMonitor(cfg) { - secutils.SetCachedHostname("test-hostname") - eventmonitortestutil.StartEventMonitor(t, procmontestutil.RegisterProcessMonitorEventConsumer) + monitor.InitializeEventConsumer(consumerstestutil.NewTestProcessConsumer(t)) } t.Cleanup(usmMonitor.Stop) t.Cleanup(utils.ResetDebugger) diff --git a/pkg/network/usm/sharedlibraries/watcher_test.go b/pkg/network/usm/sharedlibraries/watcher_test.go index ca401e180fc50..7af5b82782fe3 100644 --- a/pkg/network/usm/sharedlibraries/watcher_test.go +++ b/pkg/network/usm/sharedlibraries/watcher_test.go @@ -26,13 +26,11 @@ import ( "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" "github.com/DataDog/datadog-agent/pkg/ebpf/prebuilt" - eventmonitortestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/testutil" + "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers/testutil" usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config" fileopener "github.com/DataDog/datadog-agent/pkg/network/usm/sharedlibraries/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/process/monitor" - procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" - secutils "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -42,8 +40,7 @@ func launchProcessMonitor(t *testing.T, useEventStream bool) { t.Cleanup(pm.Stop) require.NoError(t, pm.Initialize(useEventStream)) if useEventStream { - secutils.SetCachedHostname("test-hostname") - eventmonitortestutil.StartEventMonitor(t, procmontestutil.RegisterProcessMonitorEventConsumer) + monitor.InitializeEventConsumer(testutil.NewTestProcessConsumer(t)) } } diff --git a/pkg/process/monitor/process_monitor.go b/pkg/process/monitor/process_monitor.go index 57c174840b66e..8dbdefc99a822 100644 --- a/pkg/process/monitor/process_monitor.go +++ b/pkg/process/monitor/process_monitor.go @@ -18,10 +18,9 @@ import ( "github.com/vishvananda/netlink" "go.uber.org/atomic" - "github.com/DataDog/datadog-agent/pkg/eventmonitor" + "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers" "github.com/DataDog/datadog-agent/pkg/network/protocols/telemetry" "github.com/DataDog/datadog-agent/pkg/runtime" - "github.com/DataDog/datadog-agent/pkg/security/secl/model" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -490,76 +489,20 @@ func (pm *ProcessMonitor) Stop() { pm.processExitCallbacksMutex.Unlock() } -// Event defines the event used by the process monitor -type Event struct { - Type model.EventType - Pid uint32 -} - -// EventConsumer defines an event consumer to handle event monitor events in the -// process monitor -type EventConsumer struct{} - -// NewProcessMonitorEventConsumer returns a new process monitor event consumer -func NewProcessMonitorEventConsumer(em *eventmonitor.EventMonitor) (*EventConsumer, error) { - consumer := &EventConsumer{} - err := em.AddEventConsumerHandler(consumer) - return consumer, err -} - -// ChanSize returns the channel size used by this consumer -func (ec *EventConsumer) ChanSize() int { - return 500 -} - -// ID returns the ID of this consumer -func (ec *EventConsumer) ID() string { - return "PROCESS_MONITOR" -} - -// Start the consumer -func (ec *EventConsumer) Start() error { - return nil -} - -// Stop the consumer -func (ec *EventConsumer) Stop() { -} - -// EventTypes returns the event types handled by this consumer -func (ec *EventConsumer) EventTypes() []model.EventType { - return []model.EventType{ - model.ExecEventType, - model.ExitEventType, - } -} - -// HandleEvent handles events received from the event monitor -func (ec *EventConsumer) HandleEvent(event any) { - sevent, ok := event.(*Event) - if !ok { - return - } - - processMonitor.tel.events.Add(1) - switch sevent.Type { - case model.ExecEventType: +// InitializeEventConsumer initializes the event consumer with the event handling. +func InitializeEventConsumer(consumer *consumers.ProcessConsumer) { + consumer.SubscribeExec(func(pid uint32) { + processMonitor.tel.events.Add(1) processMonitor.tel.exec.Add(1) if processMonitor.hasExecCallbacks.Load() { - processMonitor.handleProcessExec(sevent.Pid) + processMonitor.handleProcessExec(pid) } - case model.ExitEventType: + }) + consumer.SubscribeExit(func(pid uint32) { + processMonitor.tel.events.Add(1) processMonitor.tel.exit.Add(1) if processMonitor.hasExitCallbacks.Load() { - processMonitor.handleProcessExit(sevent.Pid) + processMonitor.handleProcessExit(pid) } - } -} - -// Copy should copy the given event or return nil to discard it -func (ec *EventConsumer) Copy(event *model.Event) any { - return &Event{ - Type: event.GetEventType(), - Pid: event.GetProcessPid(), - } + }) } diff --git a/pkg/process/monitor/process_monitor_test.go b/pkg/process/monitor/process_monitor_test.go index 0fe9e80bd21cb..c03cf721aaafc 100644 --- a/pkg/process/monitor/process_monitor_test.go +++ b/pkg/process/monitor/process_monitor_test.go @@ -19,13 +19,10 @@ import ( "github.com/vishvananda/netns" "go.uber.org/atomic" - "github.com/DataDog/datadog-agent/pkg/eventmonitor" - eventmonitortestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/testutil" + "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers/testutil" "github.com/DataDog/datadog-agent/pkg/network/protocols/telemetry" - "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/util" "github.com/DataDog/datadog-agent/pkg/util/kernel" - "github.com/DataDog/datadog-agent/pkg/util/log" ) func getProcessMonitor(t *testing.T) *ProcessMonitor { @@ -40,12 +37,12 @@ func getProcessMonitor(t *testing.T) *ProcessMonitor { func waitForProcessMonitor(t *testing.T, pm *ProcessMonitor) { execCounter := atomic.NewInt32(0) execCallback := func(_ uint32) { execCounter.Inc() } - registerCallback(t, pm, true, (*ProcessCallback)(&execCallback)) + registerCallback(t, pm, true, &execCallback) exitCounter := atomic.NewInt32(0) // Sanity subscribing a callback. exitCallback := func(_ uint32) { exitCounter.Inc() } - registerCallback(t, pm, false, (*ProcessCallback)(&exitCallback)) + registerCallback(t, pm, false, &exitCallback) require.Eventually(t, func() bool { _ = exec.Command("/bin/echo").Run() @@ -56,14 +53,7 @@ func waitForProcessMonitor(t *testing.T, pm *ProcessMonitor) { func initializePM(t *testing.T, pm *ProcessMonitor, useEventStream bool) { require.NoError(t, pm.Initialize(useEventStream)) if useEventStream { - utils.SetCachedHostname("test-hostname") - eventmonitortestutil.StartEventMonitor(t, func(t *testing.T, evm *eventmonitor.EventMonitor) { - // Can't use the implementation in procmontestutil due to import cycles - procmonconsumer, err := NewProcessMonitorEventConsumer(evm) - require.NoError(t, err) - evm.RegisterEventConsumer(procmonconsumer) - log.Info("process monitoring test consumer initialized") - }) + InitializeEventConsumer(testutil.NewTestProcessConsumer(t)) } waitForProcessMonitor(t, pm) } @@ -113,7 +103,7 @@ func (s *processMonitorSuite) TestProcessMonitorSanity() { defer execsMutex.Unlock() execs[pid] = struct{}{} } - registerCallback(t, pm, true, (*ProcessCallback)(&callback)) + registerCallback(t, pm, true, &callback) exitMutex := sync.RWMutex{} exits := make(map[uint32]struct{}) @@ -122,7 +112,7 @@ func (s *processMonitorSuite) TestProcessMonitorSanity() { defer exitMutex.Unlock() exits[pid] = struct{}{} } - registerCallback(t, pm, false, (*ProcessCallback)(&exitCallback)) + registerCallback(t, pm, false, &exitCallback) initializePM(t, pm, s.useEventStream) cmd := exec.Command(testBinaryPath, "test") @@ -182,7 +172,7 @@ func (s *processMonitorSuite) TestProcessRegisterMultipleCallbacks() { defer execCountersMutexes[i].Unlock() c[pid] = struct{}{} } - registerCallback(t, pm, true, (*ProcessCallback)(&callback)) + registerCallback(t, pm, true, &callback) exitCountersMutexes[i] = sync.RWMutex{} exitCounters[i] = make(map[uint32]struct{}) @@ -193,7 +183,7 @@ func (s *processMonitorSuite) TestProcessRegisterMultipleCallbacks() { defer exitCountersMutexes[i].Unlock() exitc[pid] = struct{}{} } - registerCallback(t, pm, false, (*ProcessCallback)(&exitCallback)) + registerCallback(t, pm, false, &exitCallback) } initializePM(t, pm, s.useEventStream) @@ -252,10 +242,10 @@ func (s *processMonitorSuite) TestProcessMonitorInNamespace() { pm := getProcessMonitor(t) callback := func(pid uint32) { execSet.Store(pid, struct{}{}) } - registerCallback(t, pm, true, (*ProcessCallback)(&callback)) + registerCallback(t, pm, true, &callback) exitCallback := func(pid uint32) { exitSet.Store(pid, struct{}{}) } - registerCallback(t, pm, false, (*ProcessCallback)(&exitCallback)) + registerCallback(t, pm, false, &exitCallback) monNs, err := netns.New() require.NoError(t, err, "could not create network namespace for process monitor") diff --git a/pkg/process/monitor/testutil/testutil.go b/pkg/process/monitor/testutil/testutil.go deleted file mode 100644 index d270faec3968f..0000000000000 --- a/pkg/process/monitor/testutil/testutil.go +++ /dev/null @@ -1,27 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux && test - -// Package testutil provides utilities for testing the process monitor -package testutil - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/DataDog/datadog-agent/pkg/eventmonitor" - procmon "github.com/DataDog/datadog-agent/pkg/process/monitor" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -// RegisterProcessMonitorEventConsumer registers the process monitor consumer to an EventMonitor -func RegisterProcessMonitorEventConsumer(t *testing.T, evm *eventmonitor.EventMonitor) { - procmonconsumer, err := procmon.NewProcessMonitorEventConsumer(evm) - require.NoError(t, err) - evm.RegisterEventConsumer(procmonconsumer) - log.Info("process monitoring test consumer initialized") -} From 7c1a13b48c98a3782396bd2390edaf199a67ca77 Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Tue, 10 Dec 2024 11:09:50 +0200 Subject: [PATCH 49/52] usm: sowatcher: Cut instruction cout by 90% (#31947) --- pkg/network/ebpf/c/shared-libraries/probes.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/network/ebpf/c/shared-libraries/probes.h b/pkg/network/ebpf/c/shared-libraries/probes.h index 7a71650cdba5e..a33e228a63b26 100644 --- a/pkg/network/ebpf/c/shared-libraries/probes.h +++ b/pkg/network/ebpf/c/shared-libraries/probes.h @@ -23,10 +23,9 @@ static __always_inline void do_sys_open_helper_enter(const char *filename) { // Find the null character and clean up the garbage following it #pragma unroll for (int i = 0; i < LIB_PATH_MAX_SIZE; i++) { - if (path.len) { - path.buf[i] = 0; - } else if (path.buf[i] == 0) { + if (path.buf[i] == 0) { path.len = i; + break; } } } else { From 4f89621f4eb0e48640f666ddcff92845e31ab6bc Mon Sep 17 00:00:00 2001 From: Steven Blumenthal Date: Tue, 10 Dec 2024 10:26:24 +0100 Subject: [PATCH 50/52] Increase default kube_cache_sync_timeout_seconds from 5 to 10 seconds (#31877) --- flakes.yaml | 2 ++ pkg/config/setup/config.go | 2 +- test/new-e2e/tests/containers/dump_cluster_state.go | 5 +++-- test/new-e2e/tests/containers/k8s_test.go | 3 --- test/new-e2e/tests/containers/kindvm_test.go | 3 --- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/flakes.yaml b/flakes.yaml index ab2af6b55f3ea..49c403b802517 100644 --- a/flakes.yaml +++ b/flakes.yaml @@ -11,3 +11,5 @@ test/new-e2e/tests/containers: - TestECSSuite/TestCPU/metric___container.cpu.usage{^ecs_container_name:stress-ng$} - TestEKSSuite/TestCPU/metric___container.cpu.usage{^kube_deployment:stress-ng$,^kube_namespace:workload-cpustress$} - TestKindSuite/TestCPU/metric___container.cpu.usage{^kube_deployment:stress-ng$,^kube_namespace:workload-cpustress$} + - TestKindSuite/TestAdmissionControllerWithAutoDetectedLanguage + - TestKindSuite diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index a9c74bb2765fa..563f4230a866e 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -490,7 +490,7 @@ func InitConfig(config pkgconfigmodel.Setup) { config.BindEnvAndSetDefault("leader_election_default_resource", "configmap") config.BindEnvAndSetDefault("leader_election_release_on_shutdown", true) config.BindEnvAndSetDefault("kube_resources_namespace", "") - config.BindEnvAndSetDefault("kube_cache_sync_timeout_seconds", 5) + config.BindEnvAndSetDefault("kube_cache_sync_timeout_seconds", 10) // Datadog cluster agent config.BindEnvAndSetDefault("cluster_agent.enabled", false) diff --git a/test/new-e2e/tests/containers/dump_cluster_state.go b/test/new-e2e/tests/containers/dump_cluster_state.go index ddf546d889d09..a1d0eeb97928c 100644 --- a/test/new-e2e/tests/containers/dump_cluster_state.go +++ b/test/new-e2e/tests/containers/dump_cluster_state.go @@ -17,7 +17,6 @@ import ( "strings" "sync" - "github.com/DataDog/datadog-agent/pkg/util/pointer" awsconfig "github.com/aws/aws-sdk-go-v2/config" awsec2 "github.com/aws/aws-sdk-go-v2/service/ec2" awsec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" @@ -34,6 +33,8 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" kubectlget "k8s.io/kubectl/pkg/cmd/get" kubectlutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func dumpEKSClusterState(ctx context.Context, name string) (ret string) { @@ -287,7 +288,7 @@ func dumpK8sClusterState(ctx context.Context, kubeconfig *clientcmdapi.Config, o getCmd.SetErr(out) getCmd.SetContext(ctx) getCmd.SetArgs([]string{ - "nodes,all", + "nodes,mutatingwebhookconfiguration,validatingwebhookconfiguration,all", "--all-namespaces", "-o", "wide", diff --git a/test/new-e2e/tests/containers/k8s_test.go b/test/new-e2e/tests/containers/k8s_test.go index fbb1195a976c3..fb235f979c951 100644 --- a/test/new-e2e/tests/containers/k8s_test.go +++ b/test/new-e2e/tests/containers/k8s_test.go @@ -19,7 +19,6 @@ import ( "gopkg.in/zorkian/go-datadog-api.v2" "github.com/DataDog/datadog-agent/pkg/util/pointer" - "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" fakeintake "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" @@ -993,8 +992,6 @@ func (suite *k8sSuite) TestAdmissionControllerWithLibraryAnnotation() { } func (suite *k8sSuite) TestAdmissionControllerWithAutoDetectedLanguage() { - // CONTINT-4009 - flake.Mark(suite.T()) suite.testAdmissionControllerPod("workload-mutated-lib-injection", "mutated-with-auto-detected-language", "python", true) } diff --git a/test/new-e2e/tests/containers/kindvm_test.go b/test/new-e2e/tests/containers/kindvm_test.go index 08e327fced435..5282e6fd65e34 100644 --- a/test/new-e2e/tests/containers/kindvm_test.go +++ b/test/new-e2e/tests/containers/kindvm_test.go @@ -10,8 +10,6 @@ import ( "encoding/json" "testing" - "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" - "github.com/DataDog/test-infra-definitions/scenarios/aws/kindvm" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" @@ -28,7 +26,6 @@ type kindSuite struct { } func TestKindSuite(t *testing.T) { - flake.Mark(t) suite.Run(t, &kindSuite{}) } From a7ba9110c9023ca31f2c5e913a7ff98ab31a9a5b Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Tue, 10 Dec 2024 10:51:13 +0100 Subject: [PATCH 51/52] fix(assign_issue): Add the required slack token (#31949) --- .github/workflows/assign_issue.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml index f752788cf64c8..0230f9d7c4d6f 100644 --- a/.github/workflows/assign_issue.yml +++ b/.github/workflows/assign_issue.yml @@ -26,5 +26,6 @@ jobs: - name: Assign issue env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_API_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }} run: | inv -e issue.assign-owner -i ${{ github.event.issue.number }} From bcfd6abb95c101caf619f5b2290a27b489ab3056 Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Tue, 10 Dec 2024 11:35:36 +0100 Subject: [PATCH 52/52] Make # of log pipelines configurable and default to GOMAXPROCS (#31190) Co-authored-by: Srdjan Grubor --- comp/logs/agent/agentimpl/agent_core_init.go | 2 +- comp/logs/agent/agentimpl/agent_serverless_init.go | 2 +- comp/logs/agent/config/constants.go | 5 ----- .../otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go | 2 +- pkg/compliance/reporter.go | 2 +- pkg/config/setup/config.go | 4 ++++ pkg/security/reporter/reporter.go | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/comp/logs/agent/agentimpl/agent_core_init.go b/comp/logs/agent/agentimpl/agent_core_init.go index 37d5c029cf0f6..fae9804bae9b8 100644 --- a/comp/logs/agent/agentimpl/agent_core_init.go +++ b/comp/logs/agent/agentimpl/agent_core_init.go @@ -46,7 +46,7 @@ func (a *logAgent) SetupPipeline(processingRules []*config.ProcessingRule, wmeta diagnosticMessageReceiver := diagnostic.NewBufferedMessageReceiver(nil, a.hostname) // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(config.NumberOfPipelines, auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) + pipelineProvider := pipeline.NewProvider(a.config.GetInt("logs_config.pipelines"), auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) // setup the launchers lnchrs := launchers.NewLaunchers(a.sources, pipelineProvider, auditor, a.tracker) diff --git a/comp/logs/agent/agentimpl/agent_serverless_init.go b/comp/logs/agent/agentimpl/agent_serverless_init.go index 31dbf3e41d2dc..67711def14029 100644 --- a/comp/logs/agent/agentimpl/agent_serverless_init.go +++ b/comp/logs/agent/agentimpl/agent_serverless_init.go @@ -49,7 +49,7 @@ func (a *logAgent) SetupPipeline( destinationsCtx := client.NewDestinationsContext() // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewServerlessProvider(config.NumberOfPipelines, a.auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) + pipelineProvider := pipeline.NewServerlessProvider(a.config.GetInt("logs_config.pipelines"), a.auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) lnchrs := launchers.NewLaunchers(a.sources, pipelineProvider, a.auditor, a.tracker) lnchrs.AddLauncher(channel.NewLauncher()) diff --git a/comp/logs/agent/config/constants.go b/comp/logs/agent/config/constants.go index ae9a0d74680f0..a66b989366370 100644 --- a/comp/logs/agent/config/constants.go +++ b/comp/logs/agent/config/constants.go @@ -5,11 +5,6 @@ package config -// Pipeline constraints -const ( - NumberOfPipelines = 4 -) - const ( // DateFormat is the default date format. DateFormat = "2006-01-02T15:04:05.000000000Z" diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go index d1910d28db034..b014d5b31689e 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go @@ -210,7 +210,7 @@ func (a *Agent) SetupPipeline( destinationsCtx := client.NewDestinationsContext() // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(config.NumberOfPipelines, auditor, &diagnostic.NoopMessageReceiver{}, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) + pipelineProvider := pipeline.NewProvider(a.config.GetInt("logs_config.pipelines"), auditor, &diagnostic.NoopMessageReceiver{}, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) a.auditor = auditor a.destinationsCtx = destinationsCtx diff --git a/pkg/compliance/reporter.go b/pkg/compliance/reporter.go index 501ca6f10bc43..cf59ee9043489 100644 --- a/pkg/compliance/reporter.go +++ b/pkg/compliance/reporter.go @@ -44,7 +44,7 @@ func NewLogReporter(hostname string, sourceName, sourceType string, endpoints *c auditor.Start() // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(config.NumberOfPipelines, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, dstcontext, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog()) + pipelineProvider := pipeline.NewProvider(4, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, dstcontext, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog()) pipelineProvider.Start() logSource := sources.NewLogSource( diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 563f4230a866e..a19de1ebed171 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -1572,6 +1572,10 @@ func logsagent(config pkgconfigmodel.Setup) { // Add a tag to logs that are truncated by the agent config.BindEnvAndSetDefault("logs_config.tag_truncated_logs", false) + // Number of logs pipeline instances. Defaults to number of logical CPU cores as defined by GOMAXPROCS or 4, whichever is lower. + logsPipelines := min(4, runtime.GOMAXPROCS(0)) + config.BindEnvAndSetDefault("logs_config.pipelines", logsPipelines) + // If true, the agent looks for container logs in the location used by podman, rather // than docker. This is a temporary configuration parameter to support podman logs until // a more substantial refactor of autodiscovery is made to determine this automatically. diff --git a/pkg/security/reporter/reporter.go b/pkg/security/reporter/reporter.go index 1c8a4f71bf5a0..227f94ac456ed 100644 --- a/pkg/security/reporter/reporter.go +++ b/pkg/security/reporter/reporter.go @@ -52,7 +52,7 @@ func newReporter(hostname string, stopper startstop.Stopper, sourceName, sourceT stopper.Add(auditor) // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(logsconfig.NumberOfPipelines, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, context, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog()) + pipelineProvider := pipeline.NewProvider(4, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, context, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog()) pipelineProvider.Start() stopper.Add(pipelineProvider)