From 1c593682b66a0a496ed4c86dcec4d68469d6fa6d Mon Sep 17 00:00:00 2001 From: Pedro Valois Date: Sat, 28 Oct 2023 19:50:27 +0900 Subject: [PATCH 1/3] as sweeper tests mostly working --- .../hydra_ax_sweeper/example/conf/config.yaml | 4 ++- .../hydra_plugins/hydra_ax_sweeper/_core.py | 9 ++++-- .../hydra_plugins/hydra_ax_sweeper/config.py | 7 ++--- plugins/hydra_ax_sweeper/setup.py | 9 +++--- .../tests/apps/polynomial.yaml | 4 ++- .../apps/polynomial_with_coefficients.yaml | 4 ++- .../apps/polynomial_with_constraint.yaml | 5 ++-- .../polynomial_with_dict_coefficients.yaml | 4 ++- .../hydra_ax_sweeper/tests/config/config.yaml | 4 ++- .../tests/test_ax_sweeper_plugin.py | 29 ++++++++++++------- 10 files changed, 50 insertions(+), 29 deletions(-) diff --git a/plugins/hydra_ax_sweeper/example/conf/config.yaml b/plugins/hydra_ax_sweeper/example/conf/config.yaml index 4bbd69bfb55..5f65a9991a3 100644 --- a/plugins/hydra_ax_sweeper/example/conf/config.yaml +++ b/plugins/hydra_ax_sweeper/example/conf/config.yaml @@ -18,7 +18,9 @@ hydra: experiment: # Default to minimize, set to false to maximize - minimize: true + objectives: + result: + minimize: true early_stop: # Number of epochs without a significant improvement from diff --git a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py index 4967626c3be..58c2d75c448 100644 --- a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py +++ b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py @@ -184,7 +184,12 @@ def sweep(self, arguments: List[str]) -> None: num_trials_so_far += len(list_of_trials_to_launch) num_trials_left -= len(list_of_trials_to_launch) - best_parameters, predictions = ax_client.get_best_parameters() + result = ax_client.get_best_parameters() + if result is None: + log.info("Ax has not yet found any parameters") + continue + + best_parameters, predictions = result metric = predictions[0][ax_client.objective_name] if self.early_stopper.should_stop(metric, best_parameters): @@ -336,8 +341,8 @@ def create_choice_param_from_range_override(override: Override) -> Dict[str, Any "name": key, "type": "choice", "values": [val for val in override.sweep_iterator()], + "is_ordered": False, } - return param diff --git a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py index 0aeff862f93..48f58a2f604 100644 --- a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py +++ b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py @@ -4,6 +4,7 @@ from hydra.core.config_store import ConfigStore +from ax.service.utils.instantiation import ObjectiveProperties @dataclass class EarlyStopConfig: @@ -20,11 +21,7 @@ class ExperimentConfig: # Experiment name name: Optional[str] = None - # Name of metric to optimize or null if you only return one metric - objective_name: str = "objective" - - # Defaults to minimize, set to false to maximize - minimize: bool = True + objectives: Optional[Dict[str, ObjectiveProperties]] = None # For the remaining parameters, refer the Ax documentation: https://ax.dev/api/core.html#experiment parameter_constraints: Optional[List[str]] = None diff --git a/plugins/hydra_ax_sweeper/setup.py b/plugins/hydra_ax_sweeper/setup.py index b49ffff8f71..01e07190f13 100644 --- a/plugins/hydra_ax_sweeper/setup.py +++ b/plugins/hydra_ax_sweeper/setup.py @@ -21,15 +21,16 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: POSIX :: Linux", "Operating System :: MacOS", "Development Status :: 4 - Beta", ], install_requires=[ - "hydra-core>=1.1.0.dev7", - "ax-platform>=0.1.20,<0.2.1", # https://github.com/facebookresearch/hydra/issues/1767 - "torch", - "gpytorch<=1.8.1", # avoid deprecation warnings. This can probably be removed when ax-platform is unpinned. + "hydra-core>=1.2", + "ax-platform>=0.3.4,<0.4", ], include_package_data=True, ) + + diff --git a/plugins/hydra_ax_sweeper/tests/apps/polynomial.yaml b/plugins/hydra_ax_sweeper/tests/apps/polynomial.yaml index 1148851f015..26666f34141 100644 --- a/plugins/hydra_ax_sweeper/tests/apps/polynomial.yaml +++ b/plugins/hydra_ax_sweeper/tests/apps/polynomial.yaml @@ -12,7 +12,9 @@ hydra: max_trials: 10 experiment: - minimize: true + objectives: + result: + minimize: true early_stop: max_epochs_without_improvement: 2 diff --git a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml index bb247e37ef8..6d8416b7c5b 100644 --- a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml +++ b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml @@ -10,7 +10,9 @@ hydra: max_trials: 10 experiment: - minimize: true + objectives: + result: + minimize: true early_stop: max_epochs_without_improvement: 2 diff --git a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml index 03f903f1282..b62e26fc357 100644 --- a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml +++ b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml @@ -12,8 +12,9 @@ hydra: max_trials: 10 experiment: - minimize: true - objective_name: result + objectives: + result: + minimize: true outcome_constraints: ['constraint_metric >= 2.1'] # Optional. early_stop: diff --git a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml index bb247e37ef8..6d8416b7c5b 100644 --- a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml +++ b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml @@ -10,7 +10,9 @@ hydra: max_trials: 10 experiment: - minimize: true + objectives: + result: + minimize: true early_stop: max_epochs_without_improvement: 2 diff --git a/plugins/hydra_ax_sweeper/tests/config/config.yaml b/plugins/hydra_ax_sweeper/tests/config/config.yaml index af01fbd8984..ce364aa20c7 100644 --- a/plugins/hydra_ax_sweeper/tests/config/config.yaml +++ b/plugins/hydra_ax_sweeper/tests/config/config.yaml @@ -17,7 +17,9 @@ hydra: max_trials: 5 experiment: - minimize: true + objectives: + result: + minimize: true early_stop: max_epochs_without_improvement: 2 diff --git a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py index e8dda25e281..7a7b95f254f 100644 --- a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py +++ b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import Any, List +import pytest + from hydra.core.plugins import Plugins from hydra.plugins.sweeper import Sweeper from hydra.test_utils.test_utils import ( @@ -18,6 +20,10 @@ chdir_plugin_root() +os.environ["HYDRA_FULL_ERROR"] = "1" + +filter_ax_internal_warnings = pytest.mark.filterwarnings("ignore::DeprecationWarning", "ignore::FutureWarning", "ignore::RuntimeWarning", "ignore::UserWarning", "ignore::linear_operator.utils.warnings.NumericalWarning") + def test_discovery() -> None: """ @@ -61,7 +67,7 @@ def test_chunk_method_for_invalid_inputs(n: int) -> None: with raises(ValueError): list(chunk_func(batch, n)) - +@filter_ax_internal_warnings def test_jobs_dirs(hydra_sweep_runner: TSweepRunner) -> None: # Verify that the spawned jobs are not overstepping the directories of one another. sweep = hydra_sweep_runner( @@ -143,7 +149,7 @@ def test_jobs_configured_via_cmd( assert math.isclose(best_parameters["quadratic.x"], expected_x, abs_tol=1e-4) assert math.isclose(best_parameters["quadratic.y"], -2.0, abs_tol=1e-4) - +@filter_ax_internal_warnings def test_jobs_configured_via_cmd_and_config(hydra_sweep_runner: TSweepRunner) -> None: sweep = hydra_sweep_runner( calling_file="tests/test_ax_sweeper_plugin.py", @@ -168,7 +174,7 @@ def test_jobs_configured_via_cmd_and_config(hydra_sweep_runner: TSweepRunner) -> assert math.isclose(best_parameters["quadratic.x"], -2.0, abs_tol=1e-4) assert math.isclose(best_parameters["quadratic.y"], 1.0, abs_tol=1e-4) - +@filter_ax_internal_warnings def test_configuration_set_via_cmd_and_default_config( hydra_sweep_runner: TSweepRunner, ) -> None: @@ -195,7 +201,7 @@ def test_configuration_set_via_cmd_and_default_config( assert "quadratic.x" in best_parameters assert "quadratic.y" in best_parameters - +@filter_ax_internal_warnings @mark.parametrize( "cmd_arg, expected_str", [ @@ -222,12 +228,12 @@ def test_ax_logging(tmpdir: Path, cmd_arg: str, expected_str: str) -> None: "polynomial.z=10", "hydra.sweeper.ax_config.max_trials=2", ] + [cmd_arg] - result, _ = run_python_script(cmd) + result, _ = run_python_script(cmd, allow_warnings=True) assert "polynomial.x: range=[-5.0, -2.0]" in result assert expected_str in result assert "polynomial.z: fixed=10" in result - +@filter_ax_internal_warnings @mark.parametrize( "cmd_args", [ @@ -264,7 +270,7 @@ def test_search_space_with_constraint_metric(tmpdir: Path, cmd_args: List[str]) "hydra.job.chdir=True", "hydra.sweeper.ax_config.max_trials=2", ] + cmd_args - results, _ = run_python_script(cmd) + results, _ = run_python_script(cmd, allow_warnings=True) @mark.parametrize( @@ -298,12 +304,11 @@ def test_jobs_using_choice_between_lists( "hydra.job.chdir=True", "hydra.sweeper.ax_config.max_trials=3", ] + [cmd_arg] - result, _ = run_python_script(cmd) + result, _ = run_python_script(cmd, allow_warnings=True) assert f"polynomial.coefficients: {serialized_encoding}" in result assert f"'polynomial.coefficients': {best_coefficients}" in result assert f"New best value: {best_value}" in result - @mark.parametrize( "cmd_arg, serialized_encoding, best_coefficients, best_value", [ @@ -321,6 +326,7 @@ def test_jobs_using_choice_between_lists( ), ], ) +@filter_ax_internal_warnings def test_jobs_using_choice_between_dicts( tmpdir: Path, cmd_arg: str, @@ -335,12 +341,13 @@ def test_jobs_using_choice_between_dicts( "hydra.job.chdir=True", "hydra.sweeper.ax_config.max_trials=3", ] + [cmd_arg] - result, _ = run_python_script(cmd) + result, _ = run_python_script(cmd, allow_warnings=True) assert f"polynomial.coefficients: {serialized_encoding}" in result assert f"'+polynomial.coefficients': {best_coefficients}" in result assert f"New best value: {best_value}" in result +@filter_ax_internal_warnings def test_example_app(tmpdir: Path) -> None: cmd = [ "example/banana.py", @@ -351,6 +358,6 @@ def test_example_app(tmpdir: Path) -> None: "banana.y=interval(-5, 10.1)", "hydra.sweeper.ax_config.max_trials=2", ] - result, _ = run_python_script(cmd) + result, _ = run_python_script(cmd, allow_warnings=True) assert "banana.x: range=[-5, 5]" in result assert "banana.y: range=[-5.0, 10.1]" in result From 053a88c1a328c1fe46af2b33f62b9608fd0fba62 Mon Sep 17 00:00:00 2001 From: Pedro Valois Date: Sat, 28 Oct 2023 20:04:36 +0900 Subject: [PATCH 2/3] tests are passing when warning are ignored --- plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py index 7a7b95f254f..e03bd693c50 100644 --- a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py +++ b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py @@ -22,7 +22,7 @@ os.environ["HYDRA_FULL_ERROR"] = "1" -filter_ax_internal_warnings = pytest.mark.filterwarnings("ignore::DeprecationWarning", "ignore::FutureWarning", "ignore::RuntimeWarning", "ignore::UserWarning", "ignore::linear_operator.utils.warnings.NumericalWarning") +filter_ax_internal_warnings = pytest.mark.filterwarnings("ignore::DeprecationWarning", "ignore::FutureWarning", "ignore::RuntimeWarning", "ignore::linear_operator.utils.warnings.NumericalWarning") def test_discovery() -> None: @@ -201,7 +201,6 @@ def test_configuration_set_via_cmd_and_default_config( assert "quadratic.x" in best_parameters assert "quadratic.y" in best_parameters -@filter_ax_internal_warnings @mark.parametrize( "cmd_arg, expected_str", [ @@ -233,7 +232,6 @@ def test_ax_logging(tmpdir: Path, cmd_arg: str, expected_str: str) -> None: assert expected_str in result assert "polynomial.z: fixed=10" in result -@filter_ax_internal_warnings @mark.parametrize( "cmd_args", [ @@ -249,7 +247,7 @@ def test_search_space_exhausted_exception(tmpdir: Path, cmd_args: List[str]) -> "hydra.job.chdir=True", "hydra.sweeper.ax_config.max_trials=2", ] + cmd_args - run_python_script(cmd) + run_python_script(cmd, allow_warnings=True) @mark.parametrize( @@ -326,7 +324,6 @@ def test_jobs_using_choice_between_lists( ), ], ) -@filter_ax_internal_warnings def test_jobs_using_choice_between_dicts( tmpdir: Path, cmd_arg: str, @@ -347,7 +344,6 @@ def test_jobs_using_choice_between_dicts( assert f"New best value: {best_value}" in result -@filter_ax_internal_warnings def test_example_app(tmpdir: Path) -> None: cmd = [ "example/banana.py", From 2573339ab429864b0d30bd942a8bf40699ae5cb5 Mon Sep 17 00:00:00 2001 From: Pedro Valois Date: Sat, 30 Dec 2023 16:21:34 +0900 Subject: [PATCH 3/3] lint fixes --- .../hydra_ax_sweeper/example/conf/config.yaml | 2 +- .../hydra_plugins/hydra_ax_sweeper/config.py | 2 +- plugins/hydra_ax_sweeper/setup.py | 2 - .../apps/polynomial_with_coefficients.yaml | 23 ++++++------ .../apps/polynomial_with_constraint.yaml | 37 ++++++++++--------- .../polynomial_with_dict_coefficients.yaml | 23 ++++++------ .../hydra_ax_sweeper/tests/config/config.yaml | 2 +- .../tests/test_ax_sweeper_plugin.py | 14 ++++++- 8 files changed, 58 insertions(+), 47 deletions(-) diff --git a/plugins/hydra_ax_sweeper/example/conf/config.yaml b/plugins/hydra_ax_sweeper/example/conf/config.yaml index 5f65a9991a3..e08989f670e 100644 --- a/plugins/hydra_ax_sweeper/example/conf/config.yaml +++ b/plugins/hydra_ax_sweeper/example/conf/config.yaml @@ -19,7 +19,7 @@ hydra: experiment: # Default to minimize, set to false to maximize objectives: - result: + result: minimize: true early_stop: diff --git a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py index 48f58a2f604..58f7bdbe890 100644 --- a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py +++ b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py @@ -2,9 +2,9 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional +from ax.service.utils.instantiation import ObjectiveProperties from hydra.core.config_store import ConfigStore -from ax.service.utils.instantiation import ObjectiveProperties @dataclass class EarlyStopConfig: diff --git a/plugins/hydra_ax_sweeper/setup.py b/plugins/hydra_ax_sweeper/setup.py index 01e07190f13..6a2f0d3e707 100644 --- a/plugins/hydra_ax_sweeper/setup.py +++ b/plugins/hydra_ax_sweeper/setup.py @@ -32,5 +32,3 @@ ], include_package_data=True, ) - - diff --git a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml index 6d8416b7c5b..ef3b3e9b7f1 100644 --- a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml +++ b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_coefficients.yaml @@ -1,18 +1,19 @@ +--- defaults: - - override hydra/sweeper: ax + - override hydra/sweeper: ax polynomial: - coefficients: ??? + coefficients: ??? hydra: - sweeper: - ax_config: - max_trials: 10 + sweeper: + ax_config: + max_trials: 10 - experiment: - objectives: - result: - minimize: true + experiment: + objectives: + result: + minimize: true - early_stop: - max_epochs_without_improvement: 2 + early_stop: + max_epochs_without_improvement: 2 diff --git a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml index b62e26fc357..8485d6c64ed 100644 --- a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml +++ b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_constraint.yaml @@ -1,26 +1,27 @@ +--- defaults: - - override hydra/sweeper: ax + - override hydra/sweeper: ax polynomial: - x: ??? - y: ??? - z: ??? + x: ??? + y: ??? + z: ??? hydra: - sweeper: - ax_config: - max_trials: 10 + sweeper: + ax_config: + max_trials: 10 - experiment: - objectives: - result: - minimize: true - outcome_constraints: ['constraint_metric >= 2.1'] # Optional. + experiment: + objectives: + result: + minimize: true + outcome_constraints: [constraint_metric >= 2.1] # Optional. - early_stop: - max_epochs_without_improvement: 2 + early_stop: + max_epochs_without_improvement: 2 - params: - polynomial.x: - type: range - bounds: [-1, 1] + params: + polynomial.x: + type: range + bounds: [-1, 1] diff --git a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml index 6d8416b7c5b..ef3b3e9b7f1 100644 --- a/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml +++ b/plugins/hydra_ax_sweeper/tests/apps/polynomial_with_dict_coefficients.yaml @@ -1,18 +1,19 @@ +--- defaults: - - override hydra/sweeper: ax + - override hydra/sweeper: ax polynomial: - coefficients: ??? + coefficients: ??? hydra: - sweeper: - ax_config: - max_trials: 10 + sweeper: + ax_config: + max_trials: 10 - experiment: - objectives: - result: - minimize: true + experiment: + objectives: + result: + minimize: true - early_stop: - max_epochs_without_improvement: 2 + early_stop: + max_epochs_without_improvement: 2 diff --git a/plugins/hydra_ax_sweeper/tests/config/config.yaml b/plugins/hydra_ax_sweeper/tests/config/config.yaml index ce364aa20c7..7eaf0c3eca8 100644 --- a/plugins/hydra_ax_sweeper/tests/config/config.yaml +++ b/plugins/hydra_ax_sweeper/tests/config/config.yaml @@ -18,7 +18,7 @@ hydra: experiment: objectives: - result: + result: minimize: true early_stop: diff --git a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py index e03bd693c50..b1dd743529b 100644 --- a/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py +++ b/plugins/hydra_ax_sweeper/tests/test_ax_sweeper_plugin.py @@ -5,7 +5,6 @@ from typing import Any, List import pytest - from hydra.core.plugins import Plugins from hydra.plugins.sweeper import Sweeper from hydra.test_utils.test_utils import ( @@ -22,7 +21,12 @@ os.environ["HYDRA_FULL_ERROR"] = "1" -filter_ax_internal_warnings = pytest.mark.filterwarnings("ignore::DeprecationWarning", "ignore::FutureWarning", "ignore::RuntimeWarning", "ignore::linear_operator.utils.warnings.NumericalWarning") +filter_ax_internal_warnings = pytest.mark.filterwarnings( + "ignore::DeprecationWarning", + "ignore::FutureWarning", + "ignore::RuntimeWarning", + "ignore::linear_operator.utils.warnings.NumericalWarning", +) def test_discovery() -> None: @@ -67,6 +71,7 @@ def test_chunk_method_for_invalid_inputs(n: int) -> None: with raises(ValueError): list(chunk_func(batch, n)) + @filter_ax_internal_warnings def test_jobs_dirs(hydra_sweep_runner: TSweepRunner) -> None: # Verify that the spawned jobs are not overstepping the directories of one another. @@ -149,6 +154,7 @@ def test_jobs_configured_via_cmd( assert math.isclose(best_parameters["quadratic.x"], expected_x, abs_tol=1e-4) assert math.isclose(best_parameters["quadratic.y"], -2.0, abs_tol=1e-4) + @filter_ax_internal_warnings def test_jobs_configured_via_cmd_and_config(hydra_sweep_runner: TSweepRunner) -> None: sweep = hydra_sweep_runner( @@ -174,6 +180,7 @@ def test_jobs_configured_via_cmd_and_config(hydra_sweep_runner: TSweepRunner) -> assert math.isclose(best_parameters["quadratic.x"], -2.0, abs_tol=1e-4) assert math.isclose(best_parameters["quadratic.y"], 1.0, abs_tol=1e-4) + @filter_ax_internal_warnings def test_configuration_set_via_cmd_and_default_config( hydra_sweep_runner: TSweepRunner, @@ -201,6 +208,7 @@ def test_configuration_set_via_cmd_and_default_config( assert "quadratic.x" in best_parameters assert "quadratic.y" in best_parameters + @mark.parametrize( "cmd_arg, expected_str", [ @@ -232,6 +240,7 @@ def test_ax_logging(tmpdir: Path, cmd_arg: str, expected_str: str) -> None: assert expected_str in result assert "polynomial.z: fixed=10" in result + @mark.parametrize( "cmd_args", [ @@ -307,6 +316,7 @@ def test_jobs_using_choice_between_lists( assert f"'polynomial.coefficients': {best_coefficients}" in result assert f"New best value: {best_value}" in result + @mark.parametrize( "cmd_arg, serialized_encoding, best_coefficients, best_value", [