diff --git a/collector/lib/ConfigLoader.cpp b/collector/lib/ConfigLoader.cpp index 075f6e62c1..14d7c3c26f 100644 --- a/collector/lib/ConfigLoader.cpp +++ b/collector/lib/ConfigLoader.cpp @@ -55,12 +55,13 @@ bool ConfigLoader::LoadConfiguration(CollectorConfig& config) { bool ConfigLoader::LoadConfiguration(CollectorConfig& config, const YAML::Node& node) { const auto& config_file = CONFIG_FILE.value(); - if (node.IsNull() || !node.IsDefined()) { + if (node.IsNull() || !node.IsDefined() || !node.IsMap()) { CLOG(ERROR) << "Unable to read config from " << config_file; return false; } + YAML::Node networking_node = node["networking"]; - if (!networking_node) { + if (!networking_node || networking_node.IsNull()) { CLOG(DEBUG) << "No networking in " << config_file; return true; } diff --git a/collector/test/ConfigLoaderTest.cpp b/collector/test/ConfigLoaderTest.cpp index 811586abfa..8a86f00992 100644 --- a/collector/test/ConfigLoaderTest.cpp +++ b/collector/test/ConfigLoaderTest.cpp @@ -66,14 +66,22 @@ TEST(CollectorConfigTest, TestYamlConfigToConfigInvalid) { } } -TEST(CollectorConfigTest, TestYamlConfigToConfigEmpty) { - YAML::Node yamlNode = YAML::Load(""); - CollectorConfig config; - ASSERT_FALSE(ConfigLoader::LoadConfiguration(config, yamlNode)); +TEST(CollectorConfigTest, TestYamlConfigToConfigEmptyOrMalformed) { + std::vector tests = { + R"( + asdf + )", + R"()"}; + + for (const auto& yamlStr : tests) { + YAML::Node yamlNode = YAML::Load(yamlStr); + CollectorConfig config; + ASSERT_FALSE(ConfigLoader::LoadConfiguration(config, yamlNode)); - auto runtime_config = config.GetRuntimeConfig(); + auto runtime_config = config.GetRuntimeConfig(); - EXPECT_FALSE(runtime_config.has_value()); + EXPECT_FALSE(runtime_config.has_value()); + } } TEST(CollectorConfigTest, TestPerContainerRateLimit) { diff --git a/integration-tests/integration_test.go b/integration-tests/integration_test.go index 6bf60cb616..c34e8977ff 100644 --- a/integration-tests/integration_test.go +++ b/integration-tests/integration_test.go @@ -529,3 +529,7 @@ func TestUdpNetworkFlow(t *testing.T) { } suite.Run(t, new(suites.UdpNetworkFlow)) } + +func TestRuntimeConfigFile(t *testing.T) { + suite.Run(t, new(suites.RuntimeConfigFileTestSuite)) +} diff --git a/integration-tests/pkg/assert/assert.go b/integration-tests/pkg/assert/assert.go index 3c561f8d22..c39fba1813 100644 --- a/integration-tests/pkg/assert/assert.go +++ b/integration-tests/pkg/assert/assert.go @@ -1,13 +1,67 @@ package assert import ( + "encoding/json" "fmt" + "strings" "testing" + "time" "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/assert" + + "github.com/stackrox/collector/integration-tests/pkg/collector" + "github.com/stackrox/collector/integration-tests/pkg/log" + "github.com/stackrox/collector/integration-tests/pkg/types" +) + +var ( + runtimeConfigErrorMsg = "Runtime configuration was not updated" ) +func AssertExternalIps(t *testing.T, enabled bool, collectorIP string) { + tickTime := 1 * time.Second + timeout := 3 * time.Minute + AssertRepeated(t, tickTime, timeout, runtimeConfigErrorMsg, func() bool { + body, err := collector.IntrospectionQuery(collectorIP, "/state/config") + assert.NoError(t, err) + var response types.RuntimeConfig + err = json.Unmarshal(body, &response) + assert.NoError(t, err) + + return response.Networking.ExternalIps.Enable == enabled + }) +} + +func AssertNoRuntimeConfig(t *testing.T, collectorIP string) { + tickTime := 1 * time.Second + timeout := 3 * time.Minute + AssertRepeated(t, tickTime, timeout, runtimeConfigErrorMsg, func() bool { + body, err := collector.IntrospectionQuery(collectorIP, "/state/config") + assert.NoError(t, err) + return strings.TrimSpace(string(body)) == "{}" + }) +} + +func AssertRepeated(t *testing.T, tickTime time.Duration, timeout time.Duration, msg string, condition func() bool) { + tick := time.NewTicker(tickTime) + timer := time.After(timeout) + + for { + select { + case <-tick.C: + if condition() { + // Condition has been met + return + } + + case <-timer: + log.Error("Timeout reached: " + msg) + t.FailNow() + } + } +} + func ElementsMatchFunc[N any](expected []N, actual []N, equal func(a, b N) bool) bool { if len(expected) != len(actual) { return false diff --git a/integration-tests/pkg/collector/collector_docker.go b/integration-tests/pkg/collector/collector_docker.go index 732913f9bc..b0d3df86ab 100644 --- a/integration-tests/pkg/collector/collector_docker.go +++ b/integration-tests/pkg/collector/collector_docker.go @@ -41,6 +41,7 @@ func NewDockerCollectorManager(e executor.Executor, name string) *DockerCollecto "/host/etc:ro": "/etc", "/host/usr/lib:ro": "/usr/lib", "/host/sys/kernel/debug:ro": "/sys/kernel/debug", + "/etc/stackrox:ro": "/tmp/collector-test", } return &DockerCollectorManager{ diff --git a/integration-tests/pkg/collector/introspection.go b/integration-tests/pkg/collector/introspection.go index 2290f292d5..f371d4b1a4 100644 --- a/integration-tests/pkg/collector/introspection.go +++ b/integration-tests/pkg/collector/introspection.go @@ -8,8 +8,8 @@ import ( "github.com/stackrox/collector/integration-tests/pkg/log" ) -func (k *K8sCollectorManager) IntrospectionQuery(endpoint string) ([]byte, error) { - uri := fmt.Sprintf("http://%s:8080%s", k.IP(), endpoint) +func IntrospectionQuery(collectorIP string, endpoint string) ([]byte, error) { + uri := fmt.Sprintf("http://%s:8080%s", collectorIP, endpoint) resp, err := http.Get(uri) if err != nil { return nil, err diff --git a/integration-tests/pkg/mock_sensor/expect_conn.go b/integration-tests/pkg/mock_sensor/expect_conn.go index 92c24ec8ad..9411197f88 100644 --- a/integration-tests/pkg/mock_sensor/expect_conn.go +++ b/integration-tests/pkg/mock_sensor/expect_conn.go @@ -90,7 +90,7 @@ func (s *MockSensor) ExpectSameElementsConnections(t *testing.T, containerID str } connections := s.Connections(containerID) - if collectorAssert.ElementsMatchFunc(connections, expected, equal) { + if collectorAssert.ElementsMatchFunc(expected, connections, equal) { return true } @@ -100,13 +100,13 @@ func (s *MockSensor) ExpectSameElementsConnections(t *testing.T, containerID str select { case <-timer: connections := s.Connections(containerID) - return collectorAssert.AssertElementsMatchFunc(t, connections, expected, equal) + return collectorAssert.AssertElementsMatchFunc(t, expected, connections, equal) case conn := <-s.LiveConnections(): if conn.GetContainerId() != containerID { continue } connections := s.Connections(containerID) - if collectorAssert.ElementsMatchFunc(connections, expected, equal) { + if collectorAssert.ElementsMatchFunc(expected, connections, equal) { return true } } diff --git a/integration-tests/pkg/mock_sensor/server.go b/integration-tests/pkg/mock_sensor/server.go index 9bdf018a14..f8e1e66be3 100644 --- a/integration-tests/pkg/mock_sensor/server.go +++ b/integration-tests/pkg/mock_sensor/server.go @@ -432,8 +432,8 @@ func (m *MockSensor) pushConnection(containerID string, connection *sensorAPI.Ne CloseTimestamp: connection.GetCloseTimestamp().String(), } - if _, ok := m.connections[containerID]; ok { - m.connections[containerID] = append(m.connections[containerID], conn) + if c, ok := m.connections[containerID]; ok { + m.connections[containerID] = append(c, conn) } else { m.connections[containerID] = []types.NetworkInfo{conn} } diff --git a/integration-tests/pkg/types/runtime_config.go b/integration-tests/pkg/types/runtime_config.go new file mode 100644 index 0000000000..8cb26a964c --- /dev/null +++ b/integration-tests/pkg/types/runtime_config.go @@ -0,0 +1,27 @@ +package types + +import ( + "gopkg.in/yaml.v3" +) + +type RuntimeConfig struct { + Networking struct { + ExternalIps struct { + Enable bool `yaml:"enable"` + } `yaml:"externalIps"` + } `yaml:"networking"` +} + +func (n *RuntimeConfig) Equal(other RuntimeConfig) bool { + return n.Networking.ExternalIps.Enable == other.Networking.ExternalIps.Enable +} + +func (n *RuntimeConfig) GetRuntimeConfigStr() (string, error) { + yamlBytes, err := yaml.Marshal(n) + + if err != nil { + return "", err + } + + return string(yamlBytes), err +} diff --git a/integration-tests/suites/base.go b/integration-tests/suites/base.go index 1e41e48934..b61c3212d0 100644 --- a/integration-tests/suites/base.go +++ b/integration-tests/suites/base.go @@ -458,6 +458,22 @@ func (s *IntegrationTestSuiteBase) execShellCommand(command string) error { return err } +func (s *IntegrationTestSuiteBase) createDirectory(dir string) { + if _, err := os.Stat(dir); err == nil { + return + } + err := os.Mkdir(dir, os.ModePerm) + s.Require().NoError(err) +} + +func (s *IntegrationTestSuiteBase) deleteFile(file string) { + if _, err := os.Stat(file); os.IsNotExist(err) { + return + } + err := os.Remove(file) + s.Require().NoError(err) +} + func (s *IntegrationTestSuiteBase) waitForFileToBeDeleted(file string) error { timer := time.After(10 * time.Second) ticker := time.NewTicker(time.Second) diff --git a/integration-tests/suites/k8s/config_reload.go b/integration-tests/suites/k8s/config_reload.go index a4d83b8ecf..0be89c5b09 100644 --- a/integration-tests/suites/k8s/config_reload.go +++ b/integration-tests/suites/k8s/config_reload.go @@ -1,10 +1,7 @@ package k8s import ( - "encoding/json" - "strings" - "time" - + "github.com/stackrox/collector/integration-tests/pkg/assert" "github.com/stackrox/collector/integration-tests/pkg/collector" "github.com/stackrox/collector/integration-tests/pkg/log" @@ -28,14 +25,6 @@ networking: CONFIG_MAP_NAME = "collector-config" ) -type ConfigQueryResponse struct { - Networking struct { - ExternalIps struct { - Enable bool - } - } -} - type K8sConfigReloadTestSuite struct { K8sTestSuiteBase } @@ -62,7 +51,7 @@ func (k *K8sConfigReloadTestSuite) TestCreateConfigurationAfterStart() { }) log.Info("Checking runtime configuration is not in use") - k.assertNoRuntimeConfig() + assert.AssertNoRuntimeConfig(k.T(), k.Collector().IP()) log.Info("Checking external IPs is enabled") configMap := coreV1.ConfigMap{ @@ -75,21 +64,21 @@ func (k *K8sConfigReloadTestSuite) TestCreateConfigurationAfterStart() { }, } k.createConfigMap(&configMap) - k.assertExternalIps(true) + assert.AssertExternalIps(k.T(), true, k.Collector().IP()) log.Info("Checking external IPs is disabled") configMap.Data["runtime_config.yaml"] = EXT_IP_DISABLE k.updateConfigMap(&configMap) - k.assertExternalIps(false) + assert.AssertExternalIps(k.T(), false, k.Collector().IP()) log.Info("Checking runtime configuration is not in use") k.deleteConfigMap(CONFIG_MAP_NAME) - k.assertNoRuntimeConfig() + assert.AssertNoRuntimeConfig(k.T(), k.Collector().IP()) log.Info("Checking external IPs is enabled again") configMap.Data["runtime_config.yaml"] = EXT_IP_ENABLE k.createConfigMap(&configMap) - k.assertExternalIps(true) + assert.AssertExternalIps(k.T(), true, k.Collector().IP()) } func (k *K8sConfigReloadTestSuite) TestConfigurationReload() { @@ -110,54 +99,10 @@ func (k *K8sConfigReloadTestSuite) TestConfigurationReload() { "ROX_COLLECTOR_INTROSPECTION_ENABLE": "true", }, }) - k.assertExternalIps(true) + assert.AssertExternalIps(k.T(), true, k.Collector().IP()) log.Info("Checking external IPs is disabled") configMap.Data["runtime_config.yaml"] = EXT_IP_DISABLE k.updateConfigMap(&configMap) - k.assertExternalIps(false) -} - -func (k *K8sConfigReloadTestSuite) queryConfig() []byte { - log.Info("Querying: /state/config") - body, err := k.Collector().IntrospectionQuery("/state/config") - k.Require().NoError(err) - log.Info("Response: %q", body) - return body -} - -func (k *K8sConfigReloadTestSuite) assertRepeated(condition func() bool) { - tick := time.Tick(10 * time.Second) - timer := time.After(3 * time.Minute) - - for { - select { - case <-tick: - if condition() { - // Condition has been met - return - } - - case <-timer: - k.FailNow("Runtime configuration was not updated") - } - } -} - -func (k *K8sConfigReloadTestSuite) assertExternalIps(enable bool) { - k.assertRepeated(func() bool { - body := k.queryConfig() - var response ConfigQueryResponse - err := json.Unmarshal(body, &response) - k.Require().NoError(err) - - return response.Networking.ExternalIps.Enable == enable - }) -} - -func (k *K8sConfigReloadTestSuite) assertNoRuntimeConfig() { - k.assertRepeated(func() bool { - body := k.queryConfig() - return strings.TrimSpace(string(body)) == "{}" - }) + assert.AssertExternalIps(k.T(), false, k.Collector().IP()) } diff --git a/integration-tests/suites/k8s/namespace.go b/integration-tests/suites/k8s/namespace.go index 810a39ad72..5b94df3a5e 100644 --- a/integration-tests/suites/k8s/namespace.go +++ b/integration-tests/suites/k8s/namespace.go @@ -63,7 +63,7 @@ func (k *K8sNamespaceTestSuite) TestK8sNamespace() { for _, tt := range k.tests { endpoint := fmt.Sprintf("/state/containers/%s", tt.containerID) log.Info("Querying: %s", endpoint) - raw, err := k.Collector().IntrospectionQuery(endpoint) + raw, err := collector.IntrospectionQuery(k.Collector().IP(), endpoint) k.Require().NoError(err) log.Info("Response: %s", raw) diff --git a/integration-tests/suites/runtime_config_file.go b/integration-tests/suites/runtime_config_file.go new file mode 100644 index 0000000000..525e1de2d1 --- /dev/null +++ b/integration-tests/suites/runtime_config_file.go @@ -0,0 +1,193 @@ +package suites + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/stackrox/collector/integration-tests/pkg/assert" + "github.com/stackrox/collector/integration-tests/pkg/collector" + "github.com/stackrox/collector/integration-tests/pkg/common" + "github.com/stackrox/collector/integration-tests/pkg/config" + "github.com/stackrox/collector/integration-tests/pkg/types" +) + +var ( + normalizedIp = "255.255.255.255" + externalIp = "8.8.8.8" + serverPort = 53 + externalUrl = fmt.Sprintf("http://%s:%d", externalIp, serverPort) + + activeNormalizedConnection = types.NetworkInfo{ + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", normalizedIp, serverPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + } + + activeUnnormalizedConnection = types.NetworkInfo{ + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", externalIp, serverPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + } + + inactiveNormalizedConnection = types.NetworkInfo{ + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", normalizedIp, serverPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: "Not nill time", + } + + inactiveUnnormalizedConnection = types.NetworkInfo{ + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", externalIp, serverPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: "Not nill time", + } + + runtimeConfigDir = "/tmp/collector-test" + runtimeConfigFile = filepath.Join(runtimeConfigDir, "/runtime_config.yaml") + collectorIP = "localhost" +) + +type RuntimeConfigFileTestSuite struct { + IntegrationTestSuiteBase + ClientContainer string +} + +func (s *RuntimeConfigFileTestSuite) setRuntimeConfig(runtimeConfigFile string, configStr string) { + err := os.WriteFile(runtimeConfigFile, []byte(configStr), 0666) + s.Require().NoError(err) +} + +func (s *RuntimeConfigFileTestSuite) getRuntimeConfigEnabledStr(enabled bool) string { + var runtimeConfig types.RuntimeConfig + runtimeConfig.Networking.ExternalIps.Enable = enabled + + configStr, err := runtimeConfig.GetRuntimeConfigStr() + s.Require().NoError(err) + + return configStr +} + +func (s *RuntimeConfigFileTestSuite) setExternalIpsEnabled(runtimeConfigFile string, enabled bool) { + runtimeConfigStr := s.getRuntimeConfigEnabledStr(enabled) + s.setRuntimeConfig(runtimeConfigFile, runtimeConfigStr) +} + +// Launches collector and creates the directory for runtime configuration. +func (s *RuntimeConfigFileTestSuite) SetupTest() { + s.RegisterCleanup("external-connection") + + s.StartContainerStats() + + containerID, err := s.Executor().StartContainer( + config.ContainerStartConfig{ + Name: "external-connection", + Image: config.Images().QaImageByKey("qa-alpine-curl"), + Command: []string{"sh", "-c", "while true; do curl " + externalUrl + "; sleep 1; done"}, + }) + s.Require().NoError(err) + s.ClientContainer = common.ContainerShortID(containerID) + + collectorOptions := collector.StartupOptions{ + Env: map[string]string{ + "ROX_AFTERGLOW_PERIOD": "6", + "ROX_COLLECTOR_INTROSPECTION_ENABLE": "true", + }, + } + + s.createDirectory(runtimeConfigDir) + s.deleteFile(runtimeConfigFile) + s.StartCollector(false, &collectorOptions) +} + +func (s *RuntimeConfigFileTestSuite) AfterTest() { + s.StopCollector() + s.cleanupContainers("external-connection") + s.WritePerfResults() + s.deleteFile(runtimeConfigFile) +} + +func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigFileEnable() { + // The runtime config file was deleted before starting collector. + // Default configuration is external IPs disabled. + // We expect normalized connections. + assert.AssertNoRuntimeConfig(s.T(), collectorIP) + expectedConnections := []types.NetworkInfo{activeNormalizedConnection} + connectionSuccess := s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) + + // External IPs enabled. + // Normalized connection must be reported as inactive + // Unnormalized connection will now be reported. + s.setExternalIpsEnabled(runtimeConfigFile, true) + assert.AssertExternalIps(s.T(), true, collectorIP) + expectedConnections = append(expectedConnections, activeUnnormalizedConnection, inactiveNormalizedConnection) + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) + + // The runtime config file is deleted. This disables external IPs. The normalized connection should be active + // and the unnormalized connection shoul be inactive. + s.deleteFile(runtimeConfigFile) + assert.AssertNoRuntimeConfig(s.T(), collectorIP) + expectedConnections = append(expectedConnections, activeNormalizedConnection, inactiveUnnormalizedConnection) + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) + + // Back to having external IPs enabled. + s.setExternalIpsEnabled(runtimeConfigFile, true) + assert.AssertExternalIps(s.T(), true, collectorIP) + expectedConnections = append(expectedConnections, activeUnnormalizedConnection, inactiveNormalizedConnection) + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) +} + +func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigFileDisable() { + // The runtime config file was deleted before starting collector. + // Default configuration is external IPs disabled. + // We expect normalized connections. + assert.AssertNoRuntimeConfig(s.T(), collectorIP) + expectedConnections := []types.NetworkInfo{activeNormalizedConnection} + connectionSuccess := s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) + + // The runtime config file is created, but external IPs is disables. + // There is no change in the state, so there are no changes to the connections + s.setExternalIpsEnabled(runtimeConfigFile, false) + assert.AssertExternalIps(s.T(), false, collectorIP) + common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) + + // Back to using default behavior of external IPs disabled with no file. + s.deleteFile(runtimeConfigFile) + assert.AssertNoRuntimeConfig(s.T(), collectorIP) + common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) +} + +func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigFileInvalid() { + // The runtime config file was deleted before starting collector. + // Default configuration is external IPs disabled. + // We expect normalized connections. + assert.AssertNoRuntimeConfig(s.T(), collectorIP) + expectedConnections := []types.NetworkInfo{activeNormalizedConnection} + connectionSuccess := s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) + + // Testing an invalid configuration. There should not be a change in the configuration or reported connections + invalidConfig := "asdf" + s.setRuntimeConfig(runtimeConfigFile, invalidConfig) + assert.AssertExternalIps(s.T(), false, collectorIP) + common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + s.Require().True(connectionSuccess) +}