fix(ci): [agent6] Fix ASC new-e2e tests (#31277)

Co-authored-by: Pierre Gimalac <[email protected]>

test/new-e2e/tests/agent-subcommands/check/check_common_test.go

@@ -7,19 +7,26 @@
 package check
 
 import (
+	_ "embed"
 	"fmt"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 type baseCheckSuite struct {
 	e2e.BaseSuite[environments.Host]
 }
 
+//go:embed fixtures/hello.yaml
+var customCheckYaml []byte
+
+//go:embed fixtures/hello.py
+var customCheckPython []byte
+
 func (v *baseCheckSuite) TestCheckDisk() {
 	check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"disk"}))
 
@@ -43,8 +50,7 @@ func (v *baseCheckSuite) TestCustomCheck() {
 
 func (v *baseCheckSuite) TestCheckRate() {
 	check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"hello", "--check-rate", "--json"}))
-	data := parseCheckOutput([]byte(check))
-	require.NotNil(v.T(), data)
+	data := parseCheckOutput(v.T(), []byte(check))
 
 	metrics := data[0].Aggregator.Metrics
 
@@ -59,8 +65,7 @@ func (v *baseCheckSuite) TestCheckTimes() {
 	times := 10
 	check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"hello", "--check-times", fmt.Sprint(times), "--json"}))
 
-	data := parseCheckOutput([]byte(check))
-	require.NotNil(v.T(), data)
+	data := parseCheckOutput(v.T(), []byte(check))
 
 	metrics := data[0].Aggregator.Metrics
 

test/new-e2e/tests/agent-subcommands/check/check_nix_test.go

@@ -7,7 +7,6 @@
 package check
 
 import (
-	_ "embed"
 	"testing"
 
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
@@ -21,12 +20,6 @@ type linuxCheckSuite struct {
 	baseCheckSuite
 }
 
-//go:embed fixtures/hello.yaml
-var customCheckYaml []byte
-
-//go:embed fixtures/hello.py
-var customCheckPython []byte
-
 func TestLinuxCheckSuite(t *testing.T) {
 	e2e.Run(t, &linuxCheckSuite{}, e2e.WithProvisioner(awshost.ProvisionerNoFakeIntake(awshost.WithAgentOptions(
 		agentparams.WithFile("/etc/datadog-agent/conf.d/hello.yaml", string(customCheckYaml), true),

test/new-e2e/tests/agent-subcommands/check/check_win_test.go

@@ -21,13 +21,14 @@ type windowsCheckSuite struct {
 }
 
 func TestWindowsCheckSuite(t *testing.T) {
-	t.Skip("not working because of the following error: unable to import module 'hello': source code string cannot contain null bytes")
-
+	t.Parallel()
 	e2e.Run(t, &windowsCheckSuite{}, e2e.WithProvisioner(
 		awshost.ProvisionerNoFakeIntake(
 			awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault)),
 			awshost.WithAgentOptions(
 				agentparams.WithFile("C:/ProgramData/Datadog/conf.d/hello.d/conf.yaml", string(customCheckYaml), true),
 				agentparams.WithFile("C:/ProgramData/Datadog/checks.d/hello.py", string(customCheckPython), true),
-			))))
+			),
+		),
+	))
 }

test/new-e2e/tests/agent-subcommands/check/fixtures/hello.py

@@ -13,7 +13,7 @@
 # flake8: noqa
 class HelloCheck(AgentCheck):
     def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+        super(HelloCheck, self).__init__(*args, **kwargs)
         self.value = 123
 
     def check(self, instance):

test/new-e2e/tests/agent-subcommands/check/parse.go

@@ -6,7 +6,11 @@
 package check
 
 import (
+	"bytes"
 	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 type root struct {
@@ -27,11 +31,17 @@ type metric struct {
 	Type           string   `json:"type"`
 }
 
-func parseCheckOutput(check []byte) []root {
+func parseCheckOutput(t *testing.T, check []byte) []root {
+	// On Windows a warning is printed when running the check command with the wrong user
+	// This warning is not part of the JSON output and needs to be ignored when parsing
+	startIdx := bytes.IndexAny(check, "[{")
+	require.NotEqual(t, -1, startIdx, "Failed to find start of JSON output in check output: %v", string(check))
+
+	check = check[startIdx:]
+
 	var data []root
-	if err := json.Unmarshal([]byte(check), &data); err != nil {
-		return nil
-	}
+	err := json.Unmarshal([]byte(check), &data)
+	require.NoErrorf(t, err, "Failed to unmarshal check output: %v", string(check))
 
 	return data
 }

test/new-e2e/tests/agent-subcommands/health/health_nix_test.go

@@ -6,13 +6,16 @@
 package health
 
 import (
+	"net/http"
 	"testing"
+	"time"
 
 	"github.com/DataDog/datadog-agent/test/fakeintake/api"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
 	awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host"
 	"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type linuxHealthSuite struct {
@@ -26,23 +29,23 @@ func TestLinuxHealthSuite(t *testing.T) {
 func (v *linuxHealthSuite) TestDefaultInstallUnhealthy() {
 	// the fakeintake says that any API key is invalid by sending a 403 code
 	override := api.ResponseOverride{
-		Endpoint:    "/api/v1/validate",
-		StatusCode:  403,
-		ContentType: "text/plain",
-		Body:        []byte("invalid API key"),
+		Endpoint:   "/api/v1/validate",
+		StatusCode: 403,
+		Method:     http.MethodGet,
+		Body:       []byte("invalid API key"),
 	}
-	v.Env().FakeIntake.Client().ConfigureOverride(override)
+	err := v.Env().FakeIntake.Client().ConfigureOverride(override)
+	require.NoError(v.T(), err)
 
 	// restart the agent, which validates the key using the fakeintake at startup
 	v.UpdateEnv(awshost.Provisioner(
 		awshost.WithAgentOptions(agentparams.WithAgentConfig("log_level: info\n")),
 	))
 
-	// agent should be unhealthy because the key is invalid
-	_, err := v.Env().Agent.Client.Health()
-	if err == nil {
-		assert.Fail(v.T(), "agent expected to be unhealthy, but no error found!")
-		return
-	}
-	assert.Contains(v.T(), err.Error(), "Agent health: FAIL")
+	require.EventuallyWithT(v.T(), func(collect *assert.CollectT) {
+		// forwarder should be unhealthy because the key is invalid
+		_, err = v.Env().Agent.Client.Health()
+		assert.ErrorContains(collect, err, "Agent health: FAIL")
+		assert.ErrorContains(collect, err, "=== 1 unhealthy components ===\nforwarder")
+	}, time.Second*30, time.Second)
 }

test/new-e2e/tests/agent-subcommands/health/health_win_test.go

@@ -6,7 +6,9 @@
 package health
 
 import (
+	"net/http"
 	"testing"
+	"time"
 
 	"github.com/DataDog/datadog-agent/test/fakeintake/api"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
@@ -15,6 +17,7 @@ import (
 	"github.com/DataDog/test-infra-definitions/components/os"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/ec2"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type windowsHealthSuite struct {
@@ -28,21 +31,21 @@ func TestWindowsHealthSuite(t *testing.T) {
 func (v *windowsHealthSuite) TestDefaultInstallUnhealthy() {
 	// the fakeintake says that any API key is invalid by sending a 403 code
 	override := api.ResponseOverride{
-		Endpoint:    "/api/v1/validate",
-		StatusCode:  403,
-		ContentType: "text/plain",
-		Body:        []byte("invalid API key"),
+		Endpoint:   "/api/v1/validate",
+		StatusCode: 403,
+		Method:     http.MethodGet,
+		Body:       []byte("invalid API key"),
 	}
-	v.Env().FakeIntake.Client().ConfigureOverride(override)
+	err := v.Env().FakeIntake.Client().ConfigureOverride(override)
+	require.NoError(v.T(), err)
 	// restart the agent, which validates the key using the fakeintake at startup
 	v.UpdateEnv(awshost.Provisioner(awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault)),
 		awshost.WithAgentOptions(agentparams.WithAgentConfig("log_level: info\n"))))
 
-	// agent should be unhealthy because the key is invalid
-	_, err := v.Env().Agent.Client.Health()
-	if err == nil {
-		assert.Fail(v.T(), "agent expected to be unhealthy, but no error found!")
-		return
-	}
-	assert.Contains(v.T(), err.Error(), "Agent health: FAIL")
+	require.EventuallyWithT(v.T(), func(collect *assert.CollectT) {
+		// forwarder should be unhealthy because the key is invalid
+		_, err := v.Env().Agent.Client.Health()
+		assert.ErrorContains(collect, err, "Agent health: FAIL")
+		assert.ErrorContains(collect, err, "=== 1 unhealthy components ===\nforwarder")
+	}, time.Second*30, time.Second)
 }