Merge pull request #657 from automl/development

Release 0.12.2

.travis.yml

@@ -55,17 +55,10 @@ before_cache:
   - rm -f $HOME/.cache/pip/log/debug.log
 
 before_install:
-  - wget $MINICONDA_URL -O miniconda.sh
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - if [[ `which conda` ]]; then echo 'Conda installation successful'; else exit 1; fi
-  - conda create -n testenv --yes python=$PYTHON_VERSION pip wheel pytest gxx_linux-64 gcc_linux-64 swig
-  - source activate testenv
+  - source ci_scripts/install_env.sh
 
 install:
-  - pip install pep8 codecov mypy flake8 pytest-cov
-  - cat requirements.txt | xargs -n 1 -L 1 pip install
-  - pip install .[all]
+  - source ci_scripts/install.sh
 
 script:
   - ci_scripts/$TESTSUITE

changelog.md

@@ -1,6 +1,13 @@
-# 0.12.1
+# 0.12.2
 
-## Major Changes
+## Bug Fixes
+
+* Fixes the docstring of SMAC's default acquisition function optimizer (#653)
+* Correctly attributes the configurations' origin if using the `FixedSet` acquisition function optimizer (#653)
+* Fixes an infinite loop which could occur if using only a single configuration per iteration (#654)
+* Fixes a bug in the kernel construction of the `BOFacade` (#655)
+
+# 0.12.1
 
 ## Minor Changes
 

ci_scripts/install.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+pip install pep8 codecov mypy flake8 pytest-cov
+cat requirements.txt | xargs -n 1 -L 1 pip install
+pip install .[all]

ci_scripts/install_env.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+python --version
+
+wget $MINICONDA_URL -O miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
+export PATH="$HOME/miniconda/bin:$PATH"
+if [[ `which conda` ]]; then echo 'Conda installation successful'; else exit 1; fi
+conda create -n testenv --yes python=$PYTHON_VERSION pip wheel pytest gxx_linux-64 gcc_linux-64 swig
+source activate testenv

smac/__init__.py

@@ -5,7 +5,7 @@
 import lazy_import
 from smac.utils import dependencies
 
-__version__ = '0.12.1'
+__version__ = '0.12.2'
 __author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein,'\
     'Stefan Falkner and Frank Hutter'
 

smac/facade/smac_ac_facade.py

@@ -34,7 +34,7 @@
 # optimizer
 from smac.optimizer.smbo import SMBO
 from smac.optimizer.acquisition import EI, LogEI, AbstractAcquisitionFunction, IntegratedAcquisitionFunction
-from smac.optimizer.ei_optimization import InterleavedLocalAndRandomSearch, \
+from smac.optimizer.ei_optimization import LocalAndSortedRandomSearch, \
     AcquisitionFunctionMaximizer
 from smac.optimizer.random_configuration_chooser import RandomConfigurationChooser, ChooserProb
 # epm
@@ -135,7 +135,7 @@ def __init__(self,
             hyperparameters (i.e. GaussianProcessMCMC).
         acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer
             Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`.
-            Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set.
+            Will use :class:`smac.optimizer.ei_optimization.LocalAndSortedRandomSearch` if not set.
         acquisition_function_optimizer_kwargs: Optional[Dict]
             Arguments passed to constructor of '~acquisition_function_optimizer'
         model : AbstractEPM
@@ -332,7 +332,7 @@ def __init__(self,
                 if key not in acq_func_opt_kwargs:
                     acq_func_opt_kwargs[key] = value
             acquisition_function_optimizer_instance = (
-                InterleavedLocalAndRandomSearch(**acq_func_opt_kwargs)  # type: ignore[arg-type] # noqa F821
+                LocalAndSortedRandomSearch(**acq_func_opt_kwargs)  # type: ignore[arg-type] # noqa F821
             )  # type: AcquisitionFunctionMaximizer
         elif inspect.isclass(acquisition_function_optimizer):
             acquisition_function_optimizer_instance = acquisition_function_optimizer(**acq_func_opt_kwargs)  # type: ignore[arg-type] # noqa F821

smac/facade/smac_bo_facade.py

@@ -86,8 +86,8 @@ def __init__(self, model_type: str = 'gp_mcmc', **kwargs: typing.Any):
                 prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
             )
 
-            cont_dims = np.nonzero(types == 0)[0]
-            cat_dims = np.nonzero(types != 0)[0]
+            cont_dims = np.where(np.array(types) == 0)[0]
+            cat_dims = np.where(np.array(types) != 0)[0]
 
             if len(cont_dims) > 0:
                 exp_kernel = Matern(
@@ -104,6 +104,8 @@ def __init__(self, model_type: str = 'gp_mcmc', **kwargs: typing.Any):
                     operate_on=cat_dims,
                 )
 
+            assert len(cont_dims + len(cat_dims)) == len(scenario.cs.get_hyperparameters())
+
             noise_kernel = WhiteKernel(
                 noise_level=1e-8,
                 noise_level_bounds=(np.exp(-25), np.exp(2)),

smac/intensification/intensification.py

@@ -126,6 +126,7 @@ def __init__(self, tae_runner: ExecuteTARun,
 
         # challenger related variables
         self._chall_indx = 0
+        self.num_chall_run = 0
         self.current_challenger = None
         self.continue_challenger = False
         self.configs_to_run = iter([])  # type: _config_to_run_type
@@ -225,7 +226,13 @@ def eval_challenger(self,
         self.elapsed_time += time.time() - start_time
 
         # check if 1 intensification run is complete - line 18
-        if self.stage == IntensifierStage.RUN_INCUMBENT and self._chall_indx >= self.min_chall:
+        # this is different to regular SMAC as it requires at least successful challenger run,
+        # which is necessary to work on a fixed grid of configurations.
+        if (
+            self.stage == IntensifierStage.RUN_INCUMBENT
+            and self._chall_indx >= self.min_chall
+            and self.num_chall_run > 0
+        ):
             if self.num_run > self.run_limit:
                 self.logger.info("Maximum #runs for intensification reached")
                 self._next_iteration()
@@ -294,6 +301,7 @@ def _add_inc_run(self,
                 next_seed = self.rs.randint(low=0, high=MAXINT, size=1)[0]
 
             if available_insts:
+                self.num_run += 1
                 # Line 5 (here for easier code)
                 next_instance = self.rs.choice(available_insts)
                 # Line 7
@@ -305,7 +313,6 @@ def _add_inc_run(self,
                     cutoff=self.cutoff,
                     instance_specific=self.instance_specifics.get(next_instance, "0"))
                 self._ta_time += dur
-                self.num_run += 1
             else:
                 self.logger.debug("No further instance-seed pairs for "
                                   "incumbent available.")
@@ -392,6 +399,9 @@ def _race_challenger(self,
 
             self.logger.debug("Add run of challenger")
             try:
+                self.num_run += 1
+                self.num_chall_run += 1
+
                 status, cost, dur, res = self.tae_runner.start(
                     config=challenger,
                     instance=instance,
@@ -401,7 +411,6 @@ def _race_challenger(self,
                     # Cutoff might be None if self.cutoff is None, but then the first if statement prevents
                     # evaluation of the second if statement
                     capped=(self.cutoff is not None) and (cutoff < self.cutoff))  # type: ignore[operator] # noqa F821
-                self.num_run += 1
                 self._ta_time += dur
 
             except CappedRunException:
@@ -625,6 +634,7 @@ def _next_iteration(self) -> None:
 
         # reset for a new iteration
         self.num_run = 0
+        self.num_chall_run = 0
         self._chall_indx = 0
         self.elapsed_time = 0
         self._ta_time = 0.0

smac/optimizer/ei_optimization.py

@@ -574,13 +574,13 @@ def _maximize(
             return [(0, rand_configs[i]) for i in range(len(rand_configs))]
 
 
-class InterleavedLocalAndRandomSearch(AcquisitionFunctionMaximizer):
+class LocalAndSortedRandomSearch(AcquisitionFunctionMaximizer):
     """Implements SMAC's default acquisition function optimization.
 
     This optimizer performs local search from the previous best points
     according, to the acquisition function, uses the acquisition function to
-    sort randomly sampled configurations and interleaves unsorted, randomly
-    sampled configurations in between.
+    sort randomly sampled configurations. Random configurations are
+    interleaved by the main SMAC code.
 
     Parameters
     ----------
@@ -751,4 +751,6 @@ def _maximize(
         num_points: int,
     ) -> List[Tuple[float, Configuration]]:
         configurations = copy.deepcopy(self.configurations)
+        for config in configurations:
+            config.origin = 'Fixed Set'
         return self._sort_configs_by_acq_value(configurations)

smac/runhistory/runhistory.py

@@ -168,7 +168,7 @@ def add(
         status: StatusType,
         instance_id: typing.Optional[str] = None,
         seed: typing.Optional[int] = None,
-        budget: float = 0,
+        budget: float = 0.0,
         additional_info: typing.Optional[typing.Dict] = None,
         origin: DataOrigin = DataOrigin.INTERNAL,
     ) -> None:

smac/utils/io/traj_logging.py

@@ -323,7 +323,7 @@ def _convert_dict_to_config(config_list: typing.List[str], cs: ConfigurationSpac
                             continue
 
                 # Second, check if it's in the choices / the correct type.
-                legal = {l for l in interpretations if hp.is_legal(l)}
+                legal = {interpretation for interpretation in interpretations if hp.is_legal(interpretation)}
 
                 # Third, issue warnings if the interpretation is ambigious
                 if len(legal) != 1:

test/test_intensify/test_intensify.py

@@ -78,6 +78,8 @@ def target(x):
                                            run_history=self.rh)
 
         self.assertEqual(inc, self.config2)
+        self.assertEqual(intensifier.num_run, 1)
+        self.assertEqual(intensifier.num_chall_run, 1)
 
     def test_race_challenger_2(self):
         """
@@ -108,8 +110,9 @@ def target(x):
                                            incumbent=self.config1,
                                            run_history=self.rh,)
 
-        # self.assertTrue(False)
         self.assertEqual(inc, self.config1)
+        self.assertEqual(intensifier.num_run, 1)
+        self.assertEqual(intensifier.num_chall_run, 1)
 
     def test_race_challenger_3(self):
         """
@@ -176,6 +179,9 @@ def target(config: Configuration, seed: int, instance: str):
         self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2)
         self.assertFalse(intensifier.continue_challenger)
 
+        self.assertEqual(intensifier.num_run, 3)
+        self.assertEqual(intensifier.num_chall_run, 3)
+
     def test_race_challenger_large(self):
         """
            test _race_challenger using solution_quality
@@ -221,6 +227,9 @@ def target(x):
         runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)
         self.assertEqual(len(runs), 10)
 
+        self.assertEqual(intensifier.num_run, 10)
+        self.assertEqual(intensifier.num_chall_run, 10)
+
     def test_race_challenger_large_blocked_seed(self):
         """
            test _race_challenger whether seeds are blocked for challenger runs
@@ -269,6 +278,9 @@ def target(x):
         seeds = sorted([r.seed for r in runs])
         self.assertEqual(seeds, list(range(10)), seeds)
 
+        self.assertEqual(intensifier.num_run, 10)
+        self.assertEqual(intensifier.num_chall_run, 10)
+
     def test_add_inc_run_det(self):
         """
             test _add_inc_run()
@@ -295,6 +307,11 @@ def target(x):
         intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
         self.assertEqual(len(self.rh.data), 1, self.rh.data)
 
+        # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run
+        # as it is the first evaluation of this intensifier
+        self.assertEqual(intensifier.num_run, 0)
+        self.assertEqual(intensifier.num_chall_run, 0)
+
     def test_add_inc_run_nondet(self):
         """
             test _add_inc_run()
@@ -324,6 +341,9 @@ def target(x):
         intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
         self.assertEqual(len(self.rh.data), 3, self.rh.data)
 
+        self.assertEqual(intensifier.num_run, 2)
+        self.assertEqual(intensifier.num_chall_run, 0)
+
     def test_get_next_challenger(self):
         """
             test get_next_challenger()
@@ -604,3 +624,76 @@ def target(x):
 
         self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
         self.assertEqual(len(self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2)
+
+    def test_no_new_intensification_wo_challenger_run(self):
+        """
+        This test ensures that no new iteration is started if no challenger run was conducted
+        """
+        def target(x):
+            return 2 * x['a'] + x['b']
+
+        taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality")
+        taf.runhistory = self.rh
+
+        intensifier = Intensifier(
+            tae_runner=taf, stats=self.stats,
+            traj_logger=TrajLogger(output_dir=None, stats=self.stats),
+            rng=np.random.RandomState(12345),
+            instances=[1], run_obj_time=False,
+            deterministic=True, always_race_against=None, run_limit=1,
+            min_chall=1,
+        )
+
+        self.assertEqual(intensifier.n_iters, 0)
+        self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
+
+        config, _ = intensifier.get_next_challenger(challengers=[self.config3],
+                                                    chooser=None)
+        self.assertEqual(config, self.config3)
+        self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
+        inc, _ = intensifier.eval_challenger(challenger=config, incumbent=None, run_history=self.rh, )
+        self.assertEqual(inc, self.config3)
+        self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
+        self.assertEqual(intensifier.n_iters, 1)  # 1 intensification run complete!
+
+        # regular intensification begins - run incumbent
+        config, _ = intensifier.get_next_challenger(challengers=None,  # since incumbent is run, no configs required
+                                                    chooser=None)
+        self.assertEqual(config, inc)
+        self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
+        inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, )
+        self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
+        self.assertEqual(intensifier.n_iters, 1)
+
+        # Check that we don't walk into the next iteration if the challenger is passed again
+        config, _ = intensifier.get_next_challenger(challengers=[self.config3],
+                                                    chooser=None)
+        inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, )
+        self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
+        self.assertEqual(intensifier.n_iters, 1)
+
+        intensifier._next_iteration()
+
+        # Add a configuration, then try to execute it afterwards
+        self.assertEqual(intensifier.n_iters, 2)
+        self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS,
+                    instance_id=1, seed=0, additional_info=None)
+        intensifier.stage = IntensifierStage.RUN_CHALLENGER
+        config, _ = intensifier.get_next_challenger(challengers=[self.config1],
+                                                    chooser=None)
+        inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, )
+        self.assertEqual(intensifier.n_iters, 2)
+        self.assertEqual(intensifier.num_chall_run, 0)
+
+        # This returns the config evaluating the incumbent again
+        config, _ = intensifier.get_next_challenger(challengers=None, chooser=None)
+        inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, )
+        # This doesn't return a config because the array of configs is exhausted
+        config, _ = intensifier.get_next_challenger(challengers=None, chooser=None)
+        self.assertIsNone(config)
+        # This finally gives a runable configuration
+        config, _ = intensifier.get_next_challenger(challengers=[self.config2],
+                                                    chooser=None)
+        inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, )
+        self.assertEqual(intensifier.n_iters, 3)
+        self.assertEqual(intensifier.num_chall_run, 1)

test/test_utils/io/test_traj_logging.py

@@ -96,9 +96,9 @@ def test_add_entries(self, mock_stats):
             with open(os.path.join(tmpdir, 'traj_old.csv')) as to:
                 data = to.read().split('\n')
             with open(os.path.join(tmpdir, 'traj_aclib2.json')) as js_aclib:
-                json_dicts_aclib2 = [json.loads(l) for l in js_aclib.read().splitlines()]
+                json_dicts_aclib2 = [json.loads(line) for line in js_aclib.read().splitlines()]
             with open(os.path.join(tmpdir, 'traj.json')) as js:
-                json_dicts_alljson = [json.loads(l) for l in js.read().splitlines()]
+                json_dicts_alljson = [json.loads(line) for line in js.read().splitlines()]
 
         # Check old format
         header = data[0].split(',')