Add option to run tests on Windows (#47)

* Add option to run tests on Windows

* Improve error handling

* Update README.md

* Fix local certificate from within the collection not being used

* Fix missmatch between embedded certificate. In case the installed collection had proper certificate, generated installers would fail the validation as the embedded one would be transfered, instead of being downloaded from mock server

* Move certificate download URL to pytest.ini file

* Update README.md

roles/oneagent/tasks/provide-installer/signature-unix.yml

@@ -10,7 +10,7 @@
     src: "{{ oneagent_ca_cert_src_path }}"
     dest: "{{ oneagent_ca_cert_dest_path }}"
     mode: "0644"
-  when: _oneagent_ca_cert_state.stat.exists
+  when: not (oneagent_force_cert_download | default(true)) and _oneagent_ca_cert_state.stat.exists
 
 - name: Download CA certificate
   ansible.builtin.get_url:
@@ -20,7 +20,7 @@
     validate_certs: "{{ oneagent_validate_certs | default(true) }}"
   environment:
     SSL_CERT_FILE: "{{ oneagent_ca_cert_download_cert | default(omit) }}"
-  when: not _oneagent_ca_cert_state.stat.exists
+  when: oneagent_force_cert_download | default(true) or not _oneagent_ca_cert_state.stat.exists
 
 - name: Validate installer signature
   ansible.builtin.shell: >

roles/oneagent/tests/README.md

@@ -1,7 +1,24 @@
 # Component tests
-The tests are using mocked version of the OneAgent installer, which simulates its basic behavior - returning version, 
-deploying `uninstall.sh` script and creating `oneagentctl`, used for configuring installation. The tests can run on 
-the same machine as main node.
+The tests support two types of deployment:
+- local  - the tests are run on the same Unix machine as main node;
+- remote - the tests are run on a remote Windows (Unix is not supported at the moment) machine;
+Currently, there is no option to mix these two types of deployment and the tests must be run for one platform at a time.
+
+## Remote deployment
+For remote deployment, regular OneAgent installers are used, which are downloaded from the Dynatrace environment during
+the tests. To use this type of deployment, the following parameters must be provided:
+- `--user` - username for the remote machine;
+- `--password` - password for the remote machine;
+- `--tenant` - The environment URL from which the installer will be downloaded, in form of `https://abc123456.com`;
+- `--tenant_token` - Token for downloading the installer, generated in Deployment UI;
+- `--windows_x86=<IP>` - IP address of the remote Windows machine;
+Failing to provide any of these parameters will result in failure.
+
+## Local deployment
+For local deployment, the tests are using mocked version of the OneAgent installer, which simulates its basic behavior -
+returning version, deploying `uninstall.sh` script and creating `oneagentctl`, used for configuring installation.
+To use this type of deployment, the only required parameter is `--linux_x86=localhost`. In case, multiple platforms for
+local deployment are specified or any other platforms is used along with local one, only the first local platform is used.
 
 ## Requirements
 - Python 3.10+
@@ -27,6 +44,10 @@ $ mkdir -p roles/oneagent/files && wget https://ca.dynatrace.com/dt-root.cert.pe
 $ ansible-galaxy collection build . -vvv
 $ sudo bash -c "source venv/bin/activate && ansible-galaxy collection install -vvv dynatrace-oneagent*"
 
-# Run tests (eg. For linux_x86 platform)
+# Run tests for any platform (except from Windows) on local machine
 $ sudo bash -c "source venv/bin/activate && pytest roles/oneagent/tests --linux_x86=localhost"
+
+# Run tests with regular installer on remote Windows machine
+$ sudo bash -c "source venv/bin/activate && pytest roles/oneagent/tests --user=<USER> --password=<password> \
+--tenant=https://abc123456.com --tenant_token=<TOKEN> --windows_x86=<IP>"
 ```

roles/oneagent/tests/ansible/config.py

@@ -28,6 +28,7 @@ def _prepare_collection() -> None:
     shutil.rmtree(TEST_COLLECTIONS_DIR, ignore_errors=True)
     shutil.copytree(INSTALLED_COLLECTIONS_DIR, TEST_COLLECTIONS_DIR)
 
+
 def _prepare_playbook_file() -> None:
     shutil.copy(
         str(ANSIBLE_RESOURCE_DIR / PLAYBOOK_TEMPLATE_FILE_NAME), str(TEST_DIRECTORY / PLAYBOOK_TEMPLATE_FILE_NAME)
@@ -71,11 +72,13 @@ class AnsibleConfig:
     VERIFY_SIGNATURE_KEY = "oneagent_verify_signature"
     INSTALLER_VERSION_KEY = "oneagent_version"
     PRESERVE_INSTALLER_KEY = "oneagent_preserve_installer"
-    CA_CERT_DOWNLOAD_URL_KEY = "oneagent_ca_cert_download_url"
-    CA_CERT_DOWNLOAD_CERT_KEY = "oneagent_ca_cert_download_cert"
+
     # Internal parameters
+    FORCE_CERT_DOWNLOAD_KEY = "oneagent_force_cert_download"
+    CA_CERT_DOWNLOAD_CERT_KEY = "oneagent_ca_cert_download_cert"
     VALIDATE_DOWNLOAD_CERTS_KEY = "oneagent_validate_certs"
     INSTALLER_DOWNLOAD_CERT_KEY = "oneagent_installer_download_cert"
+    CA_CERT_DOWNLOAD_URL_KEY = "oneagent_ca_cert_download_url"
 
     # Platform-specific
     DOWNLOAD_DIR_KEY = "oneagent_download_dir"

roles/oneagent/tests/conftest.py

@@ -7,22 +7,28 @@
 import os
 import socket
 from pathlib import Path
+from typing import Any
 
 from command.platform_command_wrapper import PlatformCommandWrapper
 from ansible.config import AnsibleConfig
 from ansible.runner import AnsibleRunner
+from util.installer_provider import generate_installers, download_signature, download_installers
 from util.common_utils import prepare_test_dirs
 from util.test_data_types import DeploymentPlatform, PlatformCollection, DeploymentResult
 from util.test_helpers import check_agent_state, perform_operation_on_platforms
-from util.constants.common_constants import (TEST_DIRECTORY, INSTALLERS_DIRECTORY, INSTALLER_CERTIFICATE_FILE_NAME,
-                                             INSTALLER_PRIVATE_KEY_FILE_NAME, INSTALLERS_RESOURCE_DIR,
-                                             COMPONENT_TEST_BASE, SERVER_DIRECTORY, LOG_DIRECTORY, InstallerVersion)
-from util.ssl_certificate_generator import SSLCertificateGenerator
-
+from util.constants.common_constants import (TEST_DIRECTORY, INSTALLERS_DIRECTORY, COMPONENT_TEST_BASE,
+                                             SERVER_DIRECTORY, LOG_DIRECTORY)
 
 
+# Command line options
 USER_KEY = "user"
 PASS_KEY = "password"
+TENANT_KEY = "tenant"
+TENANT_TOKEN_KEY = "tenant_token"
+PRESERVE_INSTALLERS_KEY = "preserve_installers"
+
+# Ini file configuration
+CA_CERT_URL_KEY = "dynatrace_ca_cert_url"
 
 RUNNER_KEY = "runner"
 WRAPPER_KEY = "wrapper"
@@ -31,85 +37,62 @@
 CONFIGURATOR_KEY = "configurator"
 
 
-@pytest.fixture(scope="session", autouse=True)
-def create_test_directories() -> None:
-    shutil.rmtree(COMPONENT_TEST_BASE, ignore_errors=True)
-    os.makedirs(INSTALLERS_DIRECTORY, exist_ok=True)
-    os.makedirs(SERVER_DIRECTORY, exist_ok=True)
-    os.makedirs(LOG_DIRECTORY, exist_ok=True)
-
-
-def get_file_content(path: Path) -> list[str]:
-    with path.open("r") as f:
-        return f.readlines()
-
-
-def replace_tag(source: list[str], old: str, new: str) -> list[str]:
-    return [line.replace(old, new) for line in source]
-
+def is_local_deployment(platforms: PlatformCollection) -> bool:
+    return any("localhost" in hosts for _, hosts in platforms.items())
 
-def sign_installer(installer: list[str]) -> list[str]:
-    cmd = ["openssl", "cms", "-sign",
-           "-signer", f"{INSTALLERS_DIRECTORY / INSTALLER_CERTIFICATE_FILE_NAME}",
-           "-inkey", f"{INSTALLERS_DIRECTORY / INSTALLER_PRIVATE_KEY_FILE_NAME}"]
 
-    proc = subprocess.run(cmd, input=f"{''.join(installer)}", encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    if proc.returncode != 0:
-        logging.error(f"Failed to sign installer: {proc.stdout}")
-        sys.exit(1)
-
-    signed_installer = proc.stdout.splitlines()
-    delimiter = next(l for l in signed_installer if l.startswith("----"))
-    index = signed_installer.index(delimiter)
-    signed_installer = signed_installer[index + 1:]
-
-    custom_delimiter = "----SIGNED-INSTALLER"
-    return [ f"{l}\n" if not l.startswith(delimiter) else f"{l.replace(delimiter, custom_delimiter)}\n" for l in signed_installer]
+def parse_platforms_from_options(options: dict[str, Any]) -> PlatformCollection:
+    platforms: PlatformCollection = {}
+    deployment_platforms = [e.value for e in DeploymentPlatform]
 
+    for key, hosts in options.items():
+        if key in deployment_platforms and hosts:
+            if "localhost" in hosts:
+                logging.info(f"Local deployment detected for {key}, only this host will be used")
+                return {DeploymentPlatform.from_str(key): hosts}
+            platforms[DeploymentPlatform.from_str(key)] = hosts
+    return platforms
 
-def prepend(filename, line) -> None:
-    with open(filename, 'r+') as f:
-        content = f.read()
-        f.seek(0, 0)
-        f.write(line + '\n' + content)
 
+@pytest.fixture(scope="session", autouse=True)
+def create_test_directories(request) -> None:
+    if request.config.getoption(PRESERVE_INSTALLERS_KEY):
+        logging.info("Installers will be preserved, no installers will be generated")
+        shutil.rmtree(SERVER_DIRECTORY, ignore_errors=True)
+        shutil.rmtree(LOG_DIRECTORY, ignore_errors=True)
+        shutil.rmtree(TEST_DIRECTORY, ignore_errors=True)
+    else:
+        shutil.rmtree(COMPONENT_TEST_BASE, ignore_errors=True)
 
-def save_file(data: list[str], path: Path) -> None:
-    with path.open("w") as log:
-        log.writelines(data)
+    os.makedirs(INSTALLERS_DIRECTORY, exist_ok=True)
+    os.makedirs(SERVER_DIRECTORY, exist_ok=True)
+    os.makedirs(LOG_DIRECTORY, exist_ok=True)
 
 
 @pytest.fixture(scope="session", autouse=True)
-def prepare_installers() -> None:
+def prepare_installers(request) -> None:
     logging.info("Preparing installers...")
+    tenant = request.config.getoption(TENANT_KEY)
+    tenant_token = request.config.getoption(TENANT_TOKEN_KEY)
+    preserve_installers = request.config.getoption(PRESERVE_INSTALLERS_KEY)
+    platforms = parse_platforms_from_options(vars(request.config.option))
 
-    uninstall_template = get_file_content(INSTALLERS_RESOURCE_DIR / "uninstall.sh")
-    uninstall_code = replace_tag(uninstall_template, "$", r"\$")
-
-    oneagentctl_template = get_file_content(INSTALLERS_RESOURCE_DIR / "oneagentctl.sh")
-    oneagentctl_code = replace_tag(oneagentctl_template, "$", r"\$")
+    cert_url = request.config.getini(CA_CERT_URL_KEY)
 
-    installer_partial_name = "Dynatrace-OneAgent-Linux"
-    installer_template = get_file_content(INSTALLERS_RESOURCE_DIR / f"{installer_partial_name}.sh")
-    installer_template = replace_tag(installer_template, "##UNINSTALL_CODE##", "".join(uninstall_code))
-    installer_template = replace_tag(installer_template, "##ONEAGENTCTL_CODE##", "".join(oneagentctl_code))
+    if preserve_installers:
+        logging.info("Skipping installers preparation...")
+        return
 
-    generator = SSLCertificateGenerator(
-        country_name="US",
-        state_name="California",
-        locality_name="San Francisco",
-        organization_name="Dynatrace",
-        common_name="127.0.0.1"
-    )
-    generator.generate_and_save(f"{INSTALLERS_DIRECTORY / INSTALLER_PRIVATE_KEY_FILE_NAME}",
-                                f"{INSTALLERS_DIRECTORY / INSTALLER_CERTIFICATE_FILE_NAME}")
-
-    for version in InstallerVersion:
-        installer_code = replace_tag(installer_template, "##VERSION##", version.value)
-        installer_code = sign_installer(installer_code)
-        save_file(installer_code, INSTALLERS_DIRECTORY / f"{installer_partial_name}-{version.value}.sh")
-
-    prepend(INSTALLERS_DIRECTORY / f"{installer_partial_name}-{InstallerVersion.MALFORMED.value}.sh", "Malformed line")
+    if is_local_deployment(platforms):
+        logging.info("Generating installers...")
+        if not generate_installers():
+            pytest.exit("Generating installers failed")
+    elif tenant and tenant_token:
+        logging.info("Downloading installers and signature...")
+        if not download_signature(cert_url) or not download_installers(tenant, tenant_token, platforms):
+            pytest.exit("Downloading installers and signature failed")
+    else:
+        pytest.exit("No tenant or tenant token provided, cannot download installers")
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -160,8 +143,13 @@ def handle_test_environment(runner, configurator, platforms, wrapper) -> None:
 
 
 def pytest_addoption(parser) -> None:
+    parser.addini(CA_CERT_URL_KEY, "Url to CA certificate for downloading installers")
+
     parser.addoption(f"--{USER_KEY}", type=str, help="Name of the user", required=False)
     parser.addoption(f"--{PASS_KEY}", type=str, help="Password of the user", required=False)
+    parser.addoption(f"--{TENANT_KEY}", type=str, help="Tenant URL for downloading installer", required=False)
+    parser.addoption(f"--{TENANT_TOKEN_KEY}", type=str, help="API key for downloading installer", required=False)
+    parser.addoption(f"--{PRESERVE_INSTALLERS_KEY}", type=bool, default=False, help="Preserve installers after test run", required=False)
 
     for platform in DeploymentPlatform:
         parser.addoption(
@@ -185,12 +173,7 @@ def pytest_generate_tests(metafunc) -> None:
 
     user = options[USER_KEY]
     password = options[PASS_KEY]
-
-    platforms: PlatformCollection = {}
-    deployment_platforms = [e.value for e in DeploymentPlatform]
-    for platform, hosts in options.items():
-        if platform in deployment_platforms and hosts:
-            platforms[DeploymentPlatform.from_str(platform)] = hosts
+    platforms = parse_platforms_from_options(options)
 
     wrapper = PlatformCommandWrapper(user, password)
     configurator = AnsibleConfig(user, password, platforms)
@@ -206,4 +189,4 @@ def pytest_generate_tests(metafunc) -> None:
         metafunc.parametrize(PLATFORMS_KEY, [platforms])
 
     if WRAPPER_KEY in metafunc.fixturenames:
-        metafunc.parametrize(WRAPPER_KEY, [wrapper])
+        metafunc.parametrize(WRAPPER_KEY, [wrapper])

roles/oneagent/tests/pytest.ini

@@ -1,2 +1,3 @@
 [pytest]
 pythonpath = .
+dynatrace_ca_cert_url = "https://ca.dynatrace.com/dt-root.cert.pem"

roles/oneagent/tests/test_installAndConfig.py

@@ -114,6 +114,3 @@ def test_basic_installation(runner, configurator, platforms, wrapper, installer_
 
     logging.info("Check if installer args were passed correctly")
     perform_operation_on_platforms(platforms, _check_install_args, wrapper, TECH_NAME)
-
-
-

roles/oneagent/tests/test_resilience.py

@@ -7,7 +7,6 @@
 from util.common_utils import read_yaml_file
 from util.test_data_types import DeploymentResult
 from util.test_helpers import run_deployment, set_installer_download_params, enable_for_system_family
-from util.constants.common_constants import InstallerVersion
 
 MISSING_REQUIRED_PARAMETERS_KEY = "missing_mandatory_params"
 UNKNOWN_ARCHITECTURE_KEY = "unknown_arch"
@@ -157,7 +156,8 @@ def test_failed_signature_verification(_error_messages, runner, configurator, pl
     logging.info("Running failed signature verification test")
 
     set_installer_download_params(configurator, installer_server_url)
-    configurator.set_common_parameter(configurator.INSTALLER_VERSION_KEY, InstallerVersion.MALFORMED.value)
+    configurator.set_common_parameter(configurator.FORCE_CERT_DOWNLOAD_KEY, False)
+    configurator.set_common_parameter(configurator.INSTALLER_VERSION_KEY, "latest")
 
     with TEST_SIGNATURE_FILE.open("w") as signature:
         signature.write("break signature by writing some text")

roles/oneagent/tests/test_upgrade.py

@@ -10,13 +10,20 @@
     set_installer_download_params,
     run_deployment,
 )
-from util.constants.common_constants import InstallerVersion
 
+def _get_versions_for_platforms(platforms: PlatformCollection, latest: bool) -> dict[DeploymentPlatform, str]:
+    versions: Dict[DeploymentPlatform, str] = {}
+    for platform, _ in platforms.items():
+        installers = get_installers(platform.system(), platform.arch())
+        versioned_installer = installers[-1 if latest else 0]
+        versions[platform] = re.search(r"\d.\d+.\d+.\d+-\d+", str(versioned_installer)).group()
+    return versions
 
 def _check_agent_version(
-    platform: DeploymentPlatform, address: str, wrapper: PlatformCommandWrapper, version:  str) -> None:
+        platform: DeploymentPlatform, address: str, wrapper: PlatformCommandWrapper, versions: dict[DeploymentPlatform, str]
+) -> None:
     installed_version = wrapper.run_command(platform, address, f"{get_oneagentctl_path(platform)}", "--version")
-    assert installed_version.stdout.strip() == version
+    assert installed_version.stdout.strip() == versions[platform]
 
 
 def test_upgrade(runner, configurator, platforms, wrapper, installer_server_url):
@@ -25,16 +32,17 @@ def test_upgrade(runner, configurator, platforms, wrapper, installer_server_url)
     set_installer_download_params(configurator, installer_server_url)
     configurator.set_common_parameter(configurator.VALIDATE_DOWNLOAD_CERTS_KEY, False)
 
-    for platform in platforms:
-        configurator.set_platform_parameter(platform, configurator.INSTALLER_VERSION_KEY, InstallerVersion.OLD.value)
+    old_versions = _get_versions_for_platforms(platforms, False)
+    for platform, version in old_versions.items():
+        configurator.set_platform_parameter(platform, configurator.INSTALLER_VERSION_KEY, version)
 
     run_deployment(runner)
 
     logging.info("Check if agent is installed")
     perform_operation_on_platforms(platforms, check_agent_state, wrapper, True)
 
     logging.info("Check if agent has proper version")
-    perform_operation_on_platforms(platforms, _check_agent_version, wrapper, InstallerVersion.OLD.value)
+    perform_operation_on_platforms(platforms, _check_agent_version, wrapper, old_versions)
 
     configurator.set_common_parameter(configurator.INSTALLER_VERSION_KEY, "latest")
 
@@ -44,4 +52,5 @@ def test_upgrade(runner, configurator, platforms, wrapper, installer_server_url)
     perform_operation_on_platforms(platforms, check_agent_state, wrapper, True)
 
     logging.info("Check if agent has proper version")
-    perform_operation_on_platforms(platforms, _check_agent_version, wrapper, InstallerVersion.LATEST.value)
+    new_versions = _get_versions_for_platforms(platforms, True)
+    perform_operation_on_platforms(platforms, _check_agent_version, wrapper, new_versions)

roles/oneagent/tests/util/constants/common_constants.py

@@ -20,8 +20,6 @@
 
 INSTALLER_SERVER_TOKEN = "abcdefghijk1234567890"
 
-
 class InstallerVersion(Enum):
     OLD = "1.199.0.20241008-150308"
-    MALFORMED = "1.259.0.20241008-150308"
     LATEST = "1.300.0.20241008-150308"