From 64543c237426a8aaf3b325d190f008e4229007df Mon Sep 17 00:00:00 2001 From: Pierre Delaunay Date: Fri, 24 Sep 2021 13:28:34 -0400 Subject: [PATCH 01/12] Add Orion Extension concept [OC-343] --- src/orion/ext/__init__.py | 112 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 src/orion/ext/__init__.py diff --git a/src/orion/ext/__init__.py b/src/orion/ext/__init__.py new file mode 100644 index 000000000..1849d5f11 --- /dev/null +++ b/src/orion/ext/__init__.py @@ -0,0 +1,112 @@ +"""Defines extension mechanism for third party to hook into Orion""" + + +class EventDelegate: + """Allow extensions to listen to incoming events from Orion. + Orion broadcasts events which trigger extensions callbacks. + + Parameters + ---------- + name: str + name of the event we are creating, this is useful for error reporting + + deferred: bool + if false events are triggered as soon as broadcast is called + if true the events will need to be triggered manually + + parent: optional + Used to specify a hierachy of callbacks for debugging + """ + def __init__(self, name, deferred=False, parent=None) -> None: + self.handlers = [] + self.deferred_calls = [] + self.name = name + self.parent = parent + self.deferred = deferred + self.bad_handlers = [] + self.manager = None + + def remove(self, function) -> bool: + try: + self.handlers.remove(function) + return True + except ValueError: + return False + + def add(self, function): + self.handlers.append(function) + + def broadcast(self, *args, **kwargs): + if not self.deferred: + self._execute(args, kwargs) + return + + self.deferred_calls.append((args, kwargs)) + + def _execute(self, args, kwargs): + for fun in self.handlers: + try: + fun(*args, _parent=self.parent, **kwargs) + except Exception as err: + if self.manager: + self.manager.broadcast(self.name, fun, err, args=(args, kwargs)) + + def execute(self): + self.bad_handlers = [] + + for args, kwargs in self.deferred_calls: + self._execute(args, kwargs) + + +class OrionExtensionManager: + """Manages third party extensions for Orion""" + + def __init__(self): + self._events = {} + + self._get_event('error') + self._get_event('start_experiment') + self._get_event('new_trial') + self._get_event('end_trial') + self._get_event('end_experiment') + + + def _get_event(self, key): + """Retrieve or generate a new event delegate""" + delegate = self._events.get(key) + + if delegate is None: + delegate = EventDelegate(key) + delegate.manager = self + self._events[key] = delegate + + return delegate + + def register(self, ext): + """Register a new extensions""" + for name, delegate in self._events.items(): + if hasattr(ext, name): + delegate.add(getattr(ext, name)) + + def unregister(self, ext): + """Remove an extensions if it was already registered""" + for name, delegate in self._events.items(): + if hasattr(ext, name): + delegate.remove(getattr(ext, name)) + + +class OrionExtension: + """Base orion extension interface you need to implement""" + + def error(self, *args, **kwargs): + return + + def start_experiment(self, *args, **kwargs): + return + + def new_trial(self, *args, **kwargs): + return + + def end_experiment(self, *args, **kwargs): + return + From c6191f3b42951d0d8710962ef4e0c1f1d961d465 Mon Sep 17 00:00:00 2001 From: Pierre Delaunay Date: Fri, 1 Oct 2021 12:10:04 -0400 Subject: [PATCH 02/12] Add the callbacks --- src/orion/client/experiment.py | 8 ++++++- src/orion/ext/{__init__.py => extensions.py} | 23 ++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) rename src/orion/ext/{__init__.py => extensions.py} (78%) diff --git a/src/orion/client/experiment.py b/src/orion/client/experiment.py index 8e2180d11..d5435984e 100644 --- a/src/orion/client/experiment.py +++ b/src/orion/client/experiment.py @@ -28,6 +28,7 @@ from orion.executor.base import Executor from orion.plotting.base import PlotAccessor from orion.storage.base import FailedUpdate +from orion.ext.extensions import OrionExtensionManager log = logging.getLogger(__name__) @@ -87,6 +88,7 @@ def __init__(self, experiment, producer, executor=None, heartbeat=None): **orion.core.config.worker.executor_configuration, ) self.plot = PlotAccessor(self) + self.extensions = OrionExtensionManager() ### # Attributes @@ -753,6 +755,7 @@ def workon( self._experiment.max_trials = max_trials self._experiment.algorithms.algorithm.max_trials = max_trials + self.extensions.start_experiment.broadcast(self) trials = self.executor.wait( self.executor.submit( self._optimize, @@ -766,7 +769,7 @@ def workon( ) for _ in range(n_workers) ) - + self.extensions.end_experiment.broadcast(self) return sum(trials) def _optimize( @@ -786,13 +789,16 @@ def _optimize( kwargs[trial_arg] = trial try: + self.extensions.start_trial.broadcast(trial) results = self.executor.wait( [self.executor.submit(fct, **unflatten(kwargs))] )[0] self.observe(trial, results=results) + self.extensions.end_trial.broadcast(trial) except (KeyboardInterrupt, InvalidResult): raise except BaseException as e: + self.extensions.error(e) if on_error is None or on_error( self, trial, e, worker_broken_trials ): diff --git a/src/orion/ext/__init__.py b/src/orion/ext/extensions.py similarity index 78% rename from src/orion/ext/__init__.py rename to src/orion/ext/extensions.py index 1849d5f11..7477671e5 100644 --- a/src/orion/ext/__init__.py +++ b/src/orion/ext/extensions.py @@ -27,6 +27,7 @@ def __init__(self, name, deferred=False, parent=None) -> None: self.manager = None def remove(self, function) -> bool: + """Remove an event handler from the handler list""" try: self.handlers.remove(function) return True @@ -34,9 +35,11 @@ def remove(self, function) -> bool: return False def add(self, function): + """Add an event handler to our handler list""" self.handlers.append(function) def broadcast(self, *args, **kwargs): + """Broadcast and event to all our handlers""" if not self.deferred: self._execute(args, kwargs) return @@ -52,8 +55,7 @@ def _execute(self, args, kwargs): self.manager.broadcast(self.name, fun, err, args=(args, kwargs)) def execute(self): - self.bad_handlers = [] - + """Execute all our deferred handlers if any""" for args, kwargs in self.deferred_calls: self._execute(args, kwargs) @@ -70,6 +72,11 @@ def __init__(self): self._get_event('end_trial') self._get_event('end_experiment') + def __getattribute__(self, name): + if name not in ('register', 'unregister'): + return self._get_event(name) + + return super().__getattribute__(name) def _get_event(self, key): """Retrieve or generate a new event delegate""" @@ -79,7 +86,7 @@ def _get_event(self, key): delegate = EventDelegate(key) delegate.manager = self self._events[key] = delegate - + return delegate def register(self, ext): @@ -91,7 +98,7 @@ def register(self, ext): def unregister(self, ext): """Remove an extensions if it was already registered""" for name, delegate in self._events.items(): - if hasattr(ext, name): + if hasattr(ext, name): delegate.remove(getattr(ext, name)) @@ -99,14 +106,22 @@ class OrionExtension: """Base orion extension interface you need to implement""" def error(self, *args, **kwargs): + """Called when a error occur during the optimization process""" return def start_experiment(self, *args, **kwargs): + """Called at the begin of the optimization process before the worker starts""" return def new_trial(self, *args, **kwargs): + """Called when the trial starts with a new configuration""" + return + + def end_trial(self, *args, **kwargs): + """Called when the trial finished""" return def end_experiment(self, *args, **kwargs): + """Called at the end of the optimization process after the worker exits""" return From 7e8bd78bd7b8b7fa09faab5e113ffb946c758f9e Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Tue, 14 Sep 2021 15:56:55 -0400 Subject: [PATCH 03/12] Add Orion extensions --- src/orion/client/experiment.py | 113 ++++++++------ src/orion/ext/extensions.py | 209 ++++++++++++++++++++++++++ tests/unittests/ext/test_extension.py | 140 +++++++++++++++++ 3 files changed, 414 insertions(+), 48 deletions(-) create mode 100644 src/orion/ext/extensions.py create mode 100644 tests/unittests/ext/test_extension.py diff --git a/src/orion/client/experiment.py b/src/orion/client/experiment.py index 8e2180d11..d8724e60a 100644 --- a/src/orion/client/experiment.py +++ b/src/orion/client/experiment.py @@ -28,6 +28,7 @@ from orion.executor.base import Executor from orion.plotting.base import PlotAccessor from orion.storage.base import FailedUpdate +from orion.ext.extensions import OrionExtensionManager log = logging.getLogger(__name__) @@ -87,6 +88,7 @@ def __init__(self, experiment, producer, executor=None, heartbeat=None): **orion.core.config.worker.executor_configuration, ) self.plot = PlotAccessor(self) + self.extensions = OrionExtensionManager() ### # Attributes @@ -753,22 +755,54 @@ def workon( self._experiment.max_trials = max_trials self._experiment.algorithms.algorithm.max_trials = max_trials - trials = self.executor.wait( - self.executor.submit( - self._optimize, - fct, - pool_size, - max_trials_per_worker, - max_broken, - trial_arg, - on_error, - **kwargs, + with self.extensions.experiment(self._experiment): + trials = self.executor.wait( + self.executor.submit( + self._optimize, + fct, + pool_size, + max_trials_per_worker, + max_broken, + trial_arg, + on_error, + **kwargs, + ) + for _ in range(n_workers) ) - for _ in range(n_workers) - ) return sum(trials) + def _optimize_trial(self, fct, trial, trial_arg, kwargs, worker_broken_trials, max_broken, on_error): + kwargs.update(flatten(trial.params)) + + if trial_arg: + kwargs[trial_arg] = trial + + try: + with self.extensions.trial(trial): + results = self.executor.wait( + [self.executor.submit(fct, **unflatten(kwargs))] + )[0] + self.observe(trial, results=results) + except (KeyboardInterrupt, InvalidResult): + raise + except BaseException as e: + if on_error is None or on_error(self, trial, e, worker_broken_trials): + log.error(traceback.format_exc()) + worker_broken_trials += 1 + else: + log.error(str(e)) + log.debug(traceback.format_exc()) + + if worker_broken_trials >= max_broken: + raise BrokenExperiment( + "Worker has reached broken trials threshold" + ) + else: + self.release(trial, status="broken") + + return worker_broken_trials + def _optimize( self, fct, pool_size, max_trials, max_broken, trial_arg, on_error, **kwargs ): @@ -776,43 +810,26 @@ def _optimize( trials = 0 kwargs = flatten(kwargs) max_trials = min(max_trials, self.max_trials) + while not self.is_done and trials - worker_broken_trials < max_trials: - try: - with self.suggest(pool_size=pool_size) as trial: - - kwargs.update(flatten(trial.params)) - - if trial_arg: - kwargs[trial_arg] = trial - - try: - results = self.executor.wait( - [self.executor.submit(fct, **unflatten(kwargs))] - )[0] - self.observe(trial, results=results) - except (KeyboardInterrupt, InvalidResult): - raise - except BaseException as e: - if on_error is None or on_error( - self, trial, e, worker_broken_trials - ): - log.error(traceback.format_exc()) - worker_broken_trials += 1 - else: - log.error(str(e)) - log.debug(traceback.format_exc()) - - if worker_broken_trials >= max_broken: - raise BrokenExperiment( - "Worker has reached broken trials threshold" - ) - else: - self.release(trial, status="broken") - except CompletedExperiment as e: - log.warning(e) - break - - trials += 1 + try: + with self.suggest(pool_size=pool_size) as trial: + + worker_broken_trials = self._optimize_trial( + fct, + trial, + trial_arg, + kwargs, + worker_broken_trials, + max_broken, + on_error + ) + + except CompletedExperiment as e: + log.warning(e) + break + + trials += 1 return trials diff --git a/src/orion/ext/extensions.py b/src/orion/ext/extensions.py new file mode 100644 index 000000000..05abbd92d --- /dev/null +++ b/src/orion/ext/extensions.py @@ -0,0 +1,209 @@ +"""Defines extension mechanism for third party to hook into Orion""" + + +class EventDelegate: + """Allow extensions to listen to incoming events from Orion. + Orion broadcasts events which trigger extensions callbacks. + + Parameters + ---------- + name: str + name of the event we are creating, this is useful for error reporting + + deferred: bool + if false events are triggered as soon as broadcast is called + if true the events will need to be triggered manually + """ + def __init__(self, name, deferred=False) -> None: + self.handlers = [] + self.deferred_calls = [] + self.name = name + self.deferred = deferred + self.bad_handlers = [] + self.manager = None + + def remove(self, function) -> bool: + """Remove an event handler from the handler list""" + try: + self.handlers.remove(function) + return True + except ValueError: + return False + + def add(self, function): + """Add an event handler to our handler list""" + self.handlers.append(function) + + def broadcast(self, *args, **kwargs): + """Broadcast and event to all our handlers""" + if not self.deferred: + self._execute(args, kwargs) + return + + self.deferred_calls.append((args, kwargs)) + + def _execute(self, args, kwargs): + for fun in self.handlers: + try: + fun(*args, **kwargs) + except Exception as err: + if self.manager: + self.manager.on_extension_error.broadcast(self.name, fun, err, args=(args, kwargs)) + + def execute(self): + """Execute all our deferred handlers if any""" + for args, kwargs in self.deferred_calls: + self._execute(args, kwargs) + + +class _DelegateStartEnd: + def __init__(self, start, error, end, *args, **kwargs): + self.args = args + self.kwargs = kwargs + self.start = start + self.end = end + self.error = error + + def __enter__(self): + self.start.broadcast(*self.args, **self.kwargs) + return self + + def __exit__(self, exception_type, exception_value, exception_traceback): + self.end.broadcast(*self.args, **self.kwargs) + + if exception_value is not None: + self.error.broadcast( + *self.args, + exception_type, + exception_value, + exception_traceback, + **self.kwargs + ) + + +class OrionExtensionManager: + """Manages third party extensions for Orion""" + + def __init__(self): + self._events = {} + self._get_event('on_extension_error') + + # -- Trials + self._get_event('new_trial') + self._get_event('on_trial_error') + self._get_event('end_trial') + + # -- Experiments + self._get_event('start_experiment') + self._get_event('on_experiment_error') + self._get_event('end_experiment') + + def experiment(self, *args, **kwargs): + """Initialize a context manager that will call start/error/end events automatically""" + return _DelegateStartEnd( + self.start_experiment, + self.on_experiment_error, + self.end_experiment, + *args, + **kwargs + ) + + def trial(self, *args, **kwargs): + """Initialize a context manager that will call start/error/end events automatically""" + return _DelegateStartEnd( + self.new_trial, + self.on_trial_error, + self.end_trial, + *args, + **kwargs + ) + + def __getattr__(self, name): + if name in self._events: + return self._get_event(name) + + def _get_event(self, key): + """Retrieve or generate a new event delegate""" + delegate = self._events.get(key) + + if delegate is None: + delegate = EventDelegate(key) + delegate.manager = self + self._events[key] = delegate + + return delegate + + def register(self, ext): + """Register a new extensions + + Parameters + ---------- + ext + object implementing :class`OrionExtension` methods + + Returns + ------- + the number of calls that was registered + """ + registered_callbacks = 0 + for name, delegate in self._events.items(): + if hasattr(ext, name): + delegate.add(getattr(ext, name)) + registered_callbacks += 1 + + return registered_callbacks + + def unregister(self, ext): + """Remove an extensions if it was already registered""" + unregistered_callbacks = 0 + for name, delegate in self._events.items(): + if hasattr(ext, name): + delegate.remove(getattr(ext, name)) + unregistered_callbacks += 1 + + return unregistered_callbacks + + +class OrionExtension: + """Base orion extension interface you need to implement""" + + def on_extension_error(self, name, fun, exception, args): + """Called when an extension callbakc raise an exception + + Parameters + ---------- + fun: callable + handler that raised the error + + exception: + raised exception + + args: tuple + tuple of the arguments that were used + """ + return + + def on_trial_error(self, trial, exception_type, exception_value, exception_traceback): + """Called when a error occur during the optimization process""" + return + + def new_trial(self, trial): + """Called when the trial starts with a new configuration""" + return + + def end_trial(self, trial): + """Called when the trial finished""" + return + + def on_experiment_error(self, experiment, exception_type, exception_value, exception_traceback): + """Called when a error occur during the optimization process""" + return + + def start_experiment(self, experiment): + """Called at the begin of the optimization process before the worker starts""" + return + + def end_experiment(self, experiment): + """Called at the end of the optimization process after the worker exits""" + return + diff --git a/tests/unittests/ext/test_extension.py b/tests/unittests/ext/test_extension.py new file mode 100644 index 000000000..7d205b942 --- /dev/null +++ b/tests/unittests/ext/test_extension.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Example usage and tests for :mod:`orion.client.experiment`.""" +from collections import defaultdict + +import pytest + +from orion.core.utils.exceptions import BrokenExperiment +from orion.testing import create_experiment + +config = dict( + name="supernaekei", + space={"x": "uniform(0, 200)"}, + metadata={ + "user": "tsirif", + "orion_version": "XYZ", + "VCS": { + "type": "git", + "is_dirty": False, + "HEAD_sha": "test", + "active_branch": None, + "diff_sha": "diff", + }, + }, + version=1, + max_trials=10, + max_broken=5, + working_dir="", + algorithms={"random": {"seed": 1}}, + producer={"strategy": "NoParallelStrategy"}, + refers=dict(root_id="supernaekei", parent_id=None, adapter=[]), +) + +base_trial = { + "experiment": 0, + "status": "new", # new, reserved, suspended, completed, broken + "worker": None, + "start_time": None, + "end_time": None, + "heartbeat": None, + "results": [], + "params": [], +} + +class OrionExtensionTest: + """Base orion extension interface you need to implement""" + def __init__(self) -> None: + self.calls = defaultdict(int) + + def on_experiment_error(self, *args, **kwargs): + self.calls['on_experiment_error'] += 1 + + def on_trial_error(self, *args, **kwargs): + self.calls['on_trial_error'] += 1 + + def start_experiment(self, *args, **kwargs): + self.calls['start_experiment'] += 1 + + def new_trial(self, *args, **kwargs): + self.calls['new_trial'] += 1 + + def end_trial(self, *args, **kwargs): + self.calls['end_trial'] += 1 + + def end_experiment(self, *args, **kwargs): + self.calls['end_experiment'] += 1 + + +def test_client_extension(): + ext = OrionExtensionTest() + with create_experiment(config, base_trial) as (cfg, experiment, client): + registered_callback = client.extensions.register(ext) + assert registered_callback == 6, "All ext callbacks got registered" + + def foo(x): + if len(client.fetch_trials()) > 5: + raise RuntimeError() + return [dict(name="result", type="objective", value=x * 2)] + + MAX_TRIALS = 10 + MAX_BROKEN = 5 + assert client.max_trials == MAX_TRIALS + + with pytest.raises(BrokenExperiment): + client.workon(foo, max_trials=MAX_TRIALS, max_broken=MAX_BROKEN) + + n_trials = len(experiment.fetch_trials_by_status("completed")) + n_broken = len(experiment.fetch_trials_by_status("broken")) + n_reserved = len(experiment.fetch_trials_by_status("reserved")) + + assert ext.calls['new_trial'] == n_trials + n_broken - n_reserved, 'all trials should have triggered callbacks' + assert ext.calls['end_trial'] == n_trials + n_broken - n_reserved, 'all trials should have triggered callbacks' + assert ext.calls['on_trial_error'] == n_broken, 'failed trial should be reported ' + + assert ext.calls['start_experiment'] == 1, 'experiment should have started' + assert ext.calls['end_experiment'] == 1, 'experiment should have ended' + assert ext.calls['on_experiment_error'] == 1, 'failed experiment ' + + unregistered_callback = client.extensions.unregister(ext) + assert unregistered_callback == 6, "All ext callbacks got unregistered" + + +class BadOrionExtensionTest: + """Base orion extension interface you need to implement""" + def __init__(self) -> None: + self.calls = defaultdict(int) + + def on_extension_error(self, name, fun, exception, args): + self.calls['on_extension_error'] += 1 + + def on_experiment_error(self, *args, **kwargs): + self.calls['on_experiment_error'] += 1 + + def on_trial_error(self, *args, **kwargs): + self.calls['on_trial_error'] += 1 + + def new_trial(self, *args, **kwargs): + raise RuntimeError() + + +def test_client_bad_extension(): + ext = BadOrionExtensionTest() + with create_experiment(config, base_trial) as (cfg, experiment, client): + registered_callback = client.extensions.register(ext) + assert registered_callback == 4, "All ext callbacks got registered" + + def foo(x): + return [dict(name="result", type="objective", value=x * 2)] + + MAX_TRIALS = 10 + MAX_BROKEN = 5 + assert client.max_trials == MAX_TRIALS + client.workon(foo, max_trials=MAX_TRIALS, max_broken=MAX_BROKEN) + + assert ext.calls['on_trial_error'] == 0, 'Orion worked as expected' + assert ext.calls['on_experiment_error'] == 0, 'Orion worked as expected' + assert ext.calls['on_extension_error'] == 9, 'Extension error got reported' + + unregistered_callback = client.extensions.unregister(ext) + assert unregistered_callback == 4, "All ext callbacks got unregistered" From 60b630db90f7b99073a90993b6de5960ad77fec2 Mon Sep 17 00:00:00 2001 From: Setepenre Date: Tue, 5 Oct 2021 12:55:17 -0400 Subject: [PATCH 04/12] revert change --- src/orion/client/experiment.py | 89 ++++++++++++--------------- src/orion/ext/extensions.py | 34 +++++----- tests/unittests/ext/test_extension.py | 45 ++++++++------ 3 files changed, 84 insertions(+), 84 deletions(-) diff --git a/src/orion/client/experiment.py b/src/orion/client/experiment.py index d8724e60a..eee905d53 100644 --- a/src/orion/client/experiment.py +++ b/src/orion/client/experiment.py @@ -26,9 +26,9 @@ from orion.core.worker.trial import Trial, TrialCM from orion.core.worker.trial_pacemaker import TrialPacemaker from orion.executor.base import Executor +from orion.ext.extensions import OrionExtensionManager from orion.plotting.base import PlotAccessor from orion.storage.base import FailedUpdate -from orion.ext.extensions import OrionExtensionManager log = logging.getLogger(__name__) @@ -772,37 +772,6 @@ def workon( return sum(trials) - def _optimize_trial(self, fct, trial, trial_arg, kwargs, worker_broken_trials, max_broken, on_error): - kwargs.update(flatten(trial.params)) - - if trial_arg: - kwargs[trial_arg] = trial - - try: - with self.extensions.trial(trial): - results = self.executor.wait( - [self.executor.submit(fct, **unflatten(kwargs))] - )[0] - self.observe(trial, results=results) - except (KeyboardInterrupt, InvalidResult): - raise - except BaseException as e: - if on_error is None or on_error(self, trial, e, worker_broken_trials): - log.error(traceback.format_exc()) - worker_broken_trials += 1 - else: - log.error(str(e)) - log.debug(traceback.format_exc()) - - if worker_broken_trials >= max_broken: - raise BrokenExperiment( - "Worker has reached broken trials threshold" - ) - else: - self.release(trial, status="broken") - - return worker_broken_trials - def _optimize( self, fct, pool_size, max_trials, max_broken, trial_arg, on_error, **kwargs ): @@ -812,24 +781,44 @@ def _optimize( max_trials = min(max_trials, self.max_trials) while not self.is_done and trials - worker_broken_trials < max_trials: - try: - with self.suggest(pool_size=pool_size) as trial: - - worker_broken_trials = self._optimize_trial( - fct, - trial, - trial_arg, - kwargs, - worker_broken_trials, - max_broken, - on_error - ) - - except CompletedExperiment as e: - log.warning(e) - break - - trials += 1 + try: + with self.suggest(pool_size=pool_size) as trial: + + kwargs.update(flatten(trial.params)) + + if trial_arg: + kwargs[trial_arg] = trial + + try: + with self.extensions.trial(trial): + results = self.executor.wait( + [self.executor.submit(fct, **unflatten(kwargs))] + )[0] + self.observe(trial, results=results) + except (KeyboardInterrupt, InvalidResult): + raise + except BaseException as e: + if on_error is None or on_error( + self, trial, e, worker_broken_trials + ): + log.error(traceback.format_exc()) + worker_broken_trials += 1 + else: + log.error(str(e)) + log.debug(traceback.format_exc()) + + if worker_broken_trials >= max_broken: + raise BrokenExperiment( + "Worker has reached broken trials threshold" + ) + else: + self.release(trial, status="broken") + + except CompletedExperiment as e: + log.warning(e) + break + + trials += 1 return trials diff --git a/src/orion/ext/extensions.py b/src/orion/ext/extensions.py index 05abbd92d..863f955c2 100644 --- a/src/orion/ext/extensions.py +++ b/src/orion/ext/extensions.py @@ -14,6 +14,7 @@ class EventDelegate: if false events are triggered as soon as broadcast is called if true the events will need to be triggered manually """ + def __init__(self, name, deferred=False) -> None: self.handlers = [] self.deferred_calls = [] @@ -48,7 +49,9 @@ def _execute(self, args, kwargs): fun(*args, **kwargs) except Exception as err: if self.manager: - self.manager.on_extension_error.broadcast(self.name, fun, err, args=(args, kwargs)) + self.manager.on_extension_error.broadcast( + self.name, fun, err, args=(args, kwargs) + ) def execute(self): """Execute all our deferred handlers if any""" @@ -86,17 +89,17 @@ class OrionExtensionManager: def __init__(self): self._events = {} - self._get_event('on_extension_error') + self._get_event("on_extension_error") # -- Trials - self._get_event('new_trial') - self._get_event('on_trial_error') - self._get_event('end_trial') + self._get_event("new_trial") + self._get_event("on_trial_error") + self._get_event("end_trial") # -- Experiments - self._get_event('start_experiment') - self._get_event('on_experiment_error') - self._get_event('end_experiment') + self._get_event("start_experiment") + self._get_event("on_experiment_error") + self._get_event("end_experiment") def experiment(self, *args, **kwargs): """Initialize a context manager that will call start/error/end events automatically""" @@ -111,11 +114,7 @@ def experiment(self, *args, **kwargs): def trial(self, *args, **kwargs): """Initialize a context manager that will call start/error/end events automatically""" return _DelegateStartEnd( - self.new_trial, - self.on_trial_error, - self.end_trial, - *args, - **kwargs + self.new_trial, self.on_trial_error, self.end_trial, *args, **kwargs ) def __getattr__(self, name): @@ -183,7 +182,9 @@ def on_extension_error(self, name, fun, exception, args): """ return - def on_trial_error(self, trial, exception_type, exception_value, exception_traceback): + def on_trial_error( + self, trial, exception_type, exception_value, exception_traceback + ): """Called when a error occur during the optimization process""" return @@ -195,7 +196,9 @@ def end_trial(self, trial): """Called when the trial finished""" return - def on_experiment_error(self, experiment, exception_type, exception_value, exception_traceback): + def on_experiment_error( + self, experiment, exception_type, exception_value, exception_traceback + ): """Called when a error occur during the optimization process""" return @@ -206,4 +209,3 @@ def start_experiment(self, experiment): def end_experiment(self, experiment): """Called at the end of the optimization process after the worker exits""" return - diff --git a/tests/unittests/ext/test_extension.py b/tests/unittests/ext/test_extension.py index 7d205b942..3f217990e 100644 --- a/tests/unittests/ext/test_extension.py +++ b/tests/unittests/ext/test_extension.py @@ -42,28 +42,30 @@ "params": [], } + class OrionExtensionTest: """Base orion extension interface you need to implement""" + def __init__(self) -> None: self.calls = defaultdict(int) def on_experiment_error(self, *args, **kwargs): - self.calls['on_experiment_error'] += 1 + self.calls["on_experiment_error"] += 1 def on_trial_error(self, *args, **kwargs): - self.calls['on_trial_error'] += 1 + self.calls["on_trial_error"] += 1 def start_experiment(self, *args, **kwargs): - self.calls['start_experiment'] += 1 + self.calls["start_experiment"] += 1 def new_trial(self, *args, **kwargs): - self.calls['new_trial'] += 1 + self.calls["new_trial"] += 1 def end_trial(self, *args, **kwargs): - self.calls['end_trial'] += 1 + self.calls["end_trial"] += 1 def end_experiment(self, *args, **kwargs): - self.calls['end_experiment'] += 1 + self.calls["end_experiment"] += 1 def test_client_extension(): @@ -88,13 +90,19 @@ def foo(x): n_broken = len(experiment.fetch_trials_by_status("broken")) n_reserved = len(experiment.fetch_trials_by_status("reserved")) - assert ext.calls['new_trial'] == n_trials + n_broken - n_reserved, 'all trials should have triggered callbacks' - assert ext.calls['end_trial'] == n_trials + n_broken - n_reserved, 'all trials should have triggered callbacks' - assert ext.calls['on_trial_error'] == n_broken, 'failed trial should be reported ' + assert ( + ext.calls["new_trial"] == n_trials + n_broken - n_reserved + ), "all trials should have triggered callbacks" + assert ( + ext.calls["end_trial"] == n_trials + n_broken - n_reserved + ), "all trials should have triggered callbacks" + assert ( + ext.calls["on_trial_error"] == n_broken + ), "failed trial should be reported " - assert ext.calls['start_experiment'] == 1, 'experiment should have started' - assert ext.calls['end_experiment'] == 1, 'experiment should have ended' - assert ext.calls['on_experiment_error'] == 1, 'failed experiment ' + assert ext.calls["start_experiment"] == 1, "experiment should have started" + assert ext.calls["end_experiment"] == 1, "experiment should have ended" + assert ext.calls["on_experiment_error"] == 1, "failed experiment " unregistered_callback = client.extensions.unregister(ext) assert unregistered_callback == 6, "All ext callbacks got unregistered" @@ -102,17 +110,18 @@ def foo(x): class BadOrionExtensionTest: """Base orion extension interface you need to implement""" + def __init__(self) -> None: self.calls = defaultdict(int) def on_extension_error(self, name, fun, exception, args): - self.calls['on_extension_error'] += 1 + self.calls["on_extension_error"] += 1 def on_experiment_error(self, *args, **kwargs): - self.calls['on_experiment_error'] += 1 + self.calls["on_experiment_error"] += 1 def on_trial_error(self, *args, **kwargs): - self.calls['on_trial_error'] += 1 + self.calls["on_trial_error"] += 1 def new_trial(self, *args, **kwargs): raise RuntimeError() @@ -132,9 +141,9 @@ def foo(x): assert client.max_trials == MAX_TRIALS client.workon(foo, max_trials=MAX_TRIALS, max_broken=MAX_BROKEN) - assert ext.calls['on_trial_error'] == 0, 'Orion worked as expected' - assert ext.calls['on_experiment_error'] == 0, 'Orion worked as expected' - assert ext.calls['on_extension_error'] == 9, 'Extension error got reported' + assert ext.calls["on_trial_error"] == 0, "Orion worked as expected" + assert ext.calls["on_experiment_error"] == 0, "Orion worked as expected" + assert ext.calls["on_extension_error"] == 9, "Extension error got reported" unregistered_callback = client.extensions.unregister(ext) assert unregistered_callback == 4, "All ext callbacks got unregistered" From 0d9411af132625e5001463452d1b8c336134f933 Mon Sep 17 00:00:00 2001 From: Pierre Delaunay Date: Wed, 6 Oct 2021 14:44:46 -0400 Subject: [PATCH 05/12] - --- .gitignore | 2 ++ src/orion/client/experiment.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ceb8008f2..d54b7612c 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,5 @@ target/ # Notebooks tests/**.ipynb +dask-worker-space/ +tests/functional/commands/*.json \ No newline at end of file diff --git a/src/orion/client/experiment.py b/src/orion/client/experiment.py index 4e2ffbf7a..4c6a13a0d 100644 --- a/src/orion/client/experiment.py +++ b/src/orion/client/experiment.py @@ -29,7 +29,6 @@ from orion.ext.extensions import OrionExtensionManager from orion.plotting.base import PlotAccessor from orion.storage.base import FailedUpdate -from orion.ext.extensions import OrionExtensionManager log = logging.getLogger(__name__) From 4c3fa27e3ecda1158c07b2802a1ceb14780166e1 Mon Sep 17 00:00:00 2001 From: Pierre Delaunay Date: Wed, 6 Oct 2021 15:22:05 -0400 Subject: [PATCH 06/12] - --- src/orion/client/experiment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/orion/client/experiment.py b/src/orion/client/experiment.py index 4c6a13a0d..eee905d53 100644 --- a/src/orion/client/experiment.py +++ b/src/orion/client/experiment.py @@ -798,7 +798,6 @@ def _optimize( except (KeyboardInterrupt, InvalidResult): raise except BaseException as e: - self.extensions.error(e) if on_error is None or on_error( self, trial, e, worker_broken_trials ): From 58e5bce067b1f52f8f4f9de1731c955f9b482288 Mon Sep 17 00:00:00 2001 From: Setepenre Date: Tue, 5 Oct 2021 12:55:17 -0400 Subject: [PATCH 07/12] revert change --- src/orion/client/experiment.py | 89 ++++++++++++--------------- src/orion/ext/extensions.py | 45 ++++++++------ tests/unittests/ext/test_extension.py | 45 ++++++++------ 3 files changed, 91 insertions(+), 88 deletions(-) diff --git a/src/orion/client/experiment.py b/src/orion/client/experiment.py index d8724e60a..eee905d53 100644 --- a/src/orion/client/experiment.py +++ b/src/orion/client/experiment.py @@ -26,9 +26,9 @@ from orion.core.worker.trial import Trial, TrialCM from orion.core.worker.trial_pacemaker import TrialPacemaker from orion.executor.base import Executor +from orion.ext.extensions import OrionExtensionManager from orion.plotting.base import PlotAccessor from orion.storage.base import FailedUpdate -from orion.ext.extensions import OrionExtensionManager log = logging.getLogger(__name__) @@ -772,37 +772,6 @@ def workon( return sum(trials) - def _optimize_trial(self, fct, trial, trial_arg, kwargs, worker_broken_trials, max_broken, on_error): - kwargs.update(flatten(trial.params)) - - if trial_arg: - kwargs[trial_arg] = trial - - try: - with self.extensions.trial(trial): - results = self.executor.wait( - [self.executor.submit(fct, **unflatten(kwargs))] - )[0] - self.observe(trial, results=results) - except (KeyboardInterrupt, InvalidResult): - raise - except BaseException as e: - if on_error is None or on_error(self, trial, e, worker_broken_trials): - log.error(traceback.format_exc()) - worker_broken_trials += 1 - else: - log.error(str(e)) - log.debug(traceback.format_exc()) - - if worker_broken_trials >= max_broken: - raise BrokenExperiment( - "Worker has reached broken trials threshold" - ) - else: - self.release(trial, status="broken") - - return worker_broken_trials - def _optimize( self, fct, pool_size, max_trials, max_broken, trial_arg, on_error, **kwargs ): @@ -812,24 +781,44 @@ def _optimize( max_trials = min(max_trials, self.max_trials) while not self.is_done and trials - worker_broken_trials < max_trials: - try: - with self.suggest(pool_size=pool_size) as trial: - - worker_broken_trials = self._optimize_trial( - fct, - trial, - trial_arg, - kwargs, - worker_broken_trials, - max_broken, - on_error - ) - - except CompletedExperiment as e: - log.warning(e) - break - - trials += 1 + try: + with self.suggest(pool_size=pool_size) as trial: + + kwargs.update(flatten(trial.params)) + + if trial_arg: + kwargs[trial_arg] = trial + + try: + with self.extensions.trial(trial): + results = self.executor.wait( + [self.executor.submit(fct, **unflatten(kwargs))] + )[0] + self.observe(trial, results=results) + except (KeyboardInterrupt, InvalidResult): + raise + except BaseException as e: + if on_error is None or on_error( + self, trial, e, worker_broken_trials + ): + log.error(traceback.format_exc()) + worker_broken_trials += 1 + else: + log.error(str(e)) + log.debug(traceback.format_exc()) + + if worker_broken_trials >= max_broken: + raise BrokenExperiment( + "Worker has reached broken trials threshold" + ) + else: + self.release(trial, status="broken") + + except CompletedExperiment as e: + log.warning(e) + break + + trials += 1 return trials diff --git a/src/orion/ext/extensions.py b/src/orion/ext/extensions.py index 05abbd92d..7d124d11a 100644 --- a/src/orion/ext/extensions.py +++ b/src/orion/ext/extensions.py @@ -14,6 +14,7 @@ class EventDelegate: if false events are triggered as soon as broadcast is called if true the events will need to be triggered manually """ + def __init__(self, name, deferred=False) -> None: self.handlers = [] self.deferred_calls = [] @@ -48,7 +49,9 @@ def _execute(self, args, kwargs): fun(*args, **kwargs) except Exception as err: if self.manager: - self.manager.on_extension_error.broadcast(self.name, fun, err, args=(args, kwargs)) + self.manager.on_extension_error.broadcast( + self.name, fun, err, args=(args, kwargs) + ) def execute(self): """Execute all our deferred handlers if any""" @@ -86,24 +89,24 @@ class OrionExtensionManager: def __init__(self): self._events = {} - self._get_event('on_extension_error') + self._get_event("on_extension_error") # -- Trials - self._get_event('new_trial') - self._get_event('on_trial_error') - self._get_event('end_trial') + self._get_event("new_trial") + self._get_event("on_trial_error") + self._get_event("end_trial") # -- Experiments - self._get_event('start_experiment') - self._get_event('on_experiment_error') - self._get_event('end_experiment') + self._get_event("start_experiment") + self._get_event("on_experiment_error") + self._get_event("end_experiment") def experiment(self, *args, **kwargs): """Initialize a context manager that will call start/error/end events automatically""" return _DelegateStartEnd( - self.start_experiment, - self.on_experiment_error, - self.end_experiment, + self._get_event("start_experiment"), + self._get_event("on_experiment_error"), + self._get_event("end_experiment"), *args, **kwargs ) @@ -111,16 +114,15 @@ def experiment(self, *args, **kwargs): def trial(self, *args, **kwargs): """Initialize a context manager that will call start/error/end events automatically""" return _DelegateStartEnd( - self.new_trial, - self.on_trial_error, - self.end_trial, + self._get_event("new_trial"), + self._get_event("on_trial_error"), + self._get_event("end_trial"), *args, **kwargs ) - def __getattr__(self, name): - if name in self._events: - return self._get_event(name) + def broadcast(self, name, *args, **kwargs): + return self._get_event(name).broadcast(*args, **kwargs) def _get_event(self, key): """Retrieve or generate a new event delegate""" @@ -183,7 +185,9 @@ def on_extension_error(self, name, fun, exception, args): """ return - def on_trial_error(self, trial, exception_type, exception_value, exception_traceback): + def on_trial_error( + self, trial, exception_type, exception_value, exception_traceback + ): """Called when a error occur during the optimization process""" return @@ -195,7 +199,9 @@ def end_trial(self, trial): """Called when the trial finished""" return - def on_experiment_error(self, experiment, exception_type, exception_value, exception_traceback): + def on_experiment_error( + self, experiment, exception_type, exception_value, exception_traceback + ): """Called when a error occur during the optimization process""" return @@ -206,4 +212,3 @@ def start_experiment(self, experiment): def end_experiment(self, experiment): """Called at the end of the optimization process after the worker exits""" return - diff --git a/tests/unittests/ext/test_extension.py b/tests/unittests/ext/test_extension.py index 7d205b942..3f217990e 100644 --- a/tests/unittests/ext/test_extension.py +++ b/tests/unittests/ext/test_extension.py @@ -42,28 +42,30 @@ "params": [], } + class OrionExtensionTest: """Base orion extension interface you need to implement""" + def __init__(self) -> None: self.calls = defaultdict(int) def on_experiment_error(self, *args, **kwargs): - self.calls['on_experiment_error'] += 1 + self.calls["on_experiment_error"] += 1 def on_trial_error(self, *args, **kwargs): - self.calls['on_trial_error'] += 1 + self.calls["on_trial_error"] += 1 def start_experiment(self, *args, **kwargs): - self.calls['start_experiment'] += 1 + self.calls["start_experiment"] += 1 def new_trial(self, *args, **kwargs): - self.calls['new_trial'] += 1 + self.calls["new_trial"] += 1 def end_trial(self, *args, **kwargs): - self.calls['end_trial'] += 1 + self.calls["end_trial"] += 1 def end_experiment(self, *args, **kwargs): - self.calls['end_experiment'] += 1 + self.calls["end_experiment"] += 1 def test_client_extension(): @@ -88,13 +90,19 @@ def foo(x): n_broken = len(experiment.fetch_trials_by_status("broken")) n_reserved = len(experiment.fetch_trials_by_status("reserved")) - assert ext.calls['new_trial'] == n_trials + n_broken - n_reserved, 'all trials should have triggered callbacks' - assert ext.calls['end_trial'] == n_trials + n_broken - n_reserved, 'all trials should have triggered callbacks' - assert ext.calls['on_trial_error'] == n_broken, 'failed trial should be reported ' + assert ( + ext.calls["new_trial"] == n_trials + n_broken - n_reserved + ), "all trials should have triggered callbacks" + assert ( + ext.calls["end_trial"] == n_trials + n_broken - n_reserved + ), "all trials should have triggered callbacks" + assert ( + ext.calls["on_trial_error"] == n_broken + ), "failed trial should be reported " - assert ext.calls['start_experiment'] == 1, 'experiment should have started' - assert ext.calls['end_experiment'] == 1, 'experiment should have ended' - assert ext.calls['on_experiment_error'] == 1, 'failed experiment ' + assert ext.calls["start_experiment"] == 1, "experiment should have started" + assert ext.calls["end_experiment"] == 1, "experiment should have ended" + assert ext.calls["on_experiment_error"] == 1, "failed experiment " unregistered_callback = client.extensions.unregister(ext) assert unregistered_callback == 6, "All ext callbacks got unregistered" @@ -102,17 +110,18 @@ def foo(x): class BadOrionExtensionTest: """Base orion extension interface you need to implement""" + def __init__(self) -> None: self.calls = defaultdict(int) def on_extension_error(self, name, fun, exception, args): - self.calls['on_extension_error'] += 1 + self.calls["on_extension_error"] += 1 def on_experiment_error(self, *args, **kwargs): - self.calls['on_experiment_error'] += 1 + self.calls["on_experiment_error"] += 1 def on_trial_error(self, *args, **kwargs): - self.calls['on_trial_error'] += 1 + self.calls["on_trial_error"] += 1 def new_trial(self, *args, **kwargs): raise RuntimeError() @@ -132,9 +141,9 @@ def foo(x): assert client.max_trials == MAX_TRIALS client.workon(foo, max_trials=MAX_TRIALS, max_broken=MAX_BROKEN) - assert ext.calls['on_trial_error'] == 0, 'Orion worked as expected' - assert ext.calls['on_experiment_error'] == 0, 'Orion worked as expected' - assert ext.calls['on_extension_error'] == 9, 'Extension error got reported' + assert ext.calls["on_trial_error"] == 0, "Orion worked as expected" + assert ext.calls["on_experiment_error"] == 0, "Orion worked as expected" + assert ext.calls["on_extension_error"] == 9, "Extension error got reported" unregistered_callback = client.extensions.unregister(ext) assert unregistered_callback == 4, "All ext callbacks got unregistered" From db1dcd45cf07d290113cd637f6294162d084bab3 Mon Sep 17 00:00:00 2001 From: Pierre Delaunay Date: Tue, 12 Oct 2021 12:59:52 -0400 Subject: [PATCH 08/12] Add doc --- docs/src/code/client.rst | 1 + docs/src/code/client/extensions.rst | 7 +++++++ setup.py | 1 + src/orion/client/experiment.py | 15 +++++++++++++++ 4 files changed, 24 insertions(+) create mode 100644 docs/src/code/client/extensions.rst diff --git a/docs/src/code/client.rst b/docs/src/code/client.rst index 82dc287ab..d3d960472 100644 --- a/docs/src/code/client.rst +++ b/docs/src/code/client.rst @@ -9,6 +9,7 @@ Client helper functions client/cli client/experiment client/manual + client/extensions .. automodule:: orion.client :members: diff --git a/docs/src/code/client/extensions.rst b/docs/src/code/client/extensions.rst new file mode 100644 index 000000000..140887af7 --- /dev/null +++ b/docs/src/code/client/extensions.rst @@ -0,0 +1,7 @@ +Extensions +========== + +.. automodule:: orion.ext.extensions + :members: + + diff --git a/setup.py b/setup.py index 07f67bcaf..819d7f99b 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ "orion.client", "orion.core", "orion.executor", + "orion.ext", "orion.plotting", "orion.serving", "orion.storage", diff --git a/src/orion/client/experiment.py b/src/orion/client/experiment.py index eee905d53..d242751ae 100644 --- a/src/orion/client/experiment.py +++ b/src/orion/client/experiment.py @@ -73,6 +73,11 @@ class ExperimentClient: producer: `orion.core.worker.producer.Producer` Producer object used to produce new trials. + Notes + ----- + + Users can write generic extensions to ExperimentClient through + `orion.client.experiment.OrionExtension`. """ def __init__(self, experiment, producer, executor=None, heartbeat=None): @@ -322,6 +327,16 @@ def fetch_noncompleted_trials(self, with_evc_tree=False): ### # Actions ### + def register_extension(self, ext): + """Register a third party extension + + Parameters + ---------- + ext: OrionExtension + object that implements the OrionExtension interface + + """ + return self.extensions.register(ext) # pylint: disable=unused-argument def insert(self, params, results=None, reserve=False): From f1f1351a16940dd1185f2f7f2e82ca59847b8696 Mon Sep 17 00:00:00 2001 From: Setepenre Date: Fri, 22 Oct 2021 15:15:41 -0400 Subject: [PATCH 09/12] Update src/orion/ext/extensions.py Co-authored-by: Xavier Bouthillier --- src/orion/ext/extensions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/orion/ext/extensions.py b/src/orion/ext/extensions.py index cccc910b6..3c4db97e1 100644 --- a/src/orion/ext/extensions.py +++ b/src/orion/ext/extensions.py @@ -130,7 +130,10 @@ def broadcast(self, name, *args, **kwargs): return self._get_event(name).broadcast(*args, **kwargs) def _get_event(self, key): - """Retrieve or generate a new event delegate""" + """Retrieve event delegate + + Will generate one if not defined already. + """ delegate = self._events.get(key) if delegate is None: From dc5067e321457d79ee7d9e828564f6f7793a8e94 Mon Sep 17 00:00:00 2001 From: Setepenre Date: Fri, 22 Oct 2021 15:15:58 -0400 Subject: [PATCH 10/12] Update src/orion/ext/extensions.py Co-authored-by: Xavier Bouthillier --- src/orion/ext/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orion/ext/extensions.py b/src/orion/ext/extensions.py index 3c4db97e1..38f2fa7c7 100644 --- a/src/orion/ext/extensions.py +++ b/src/orion/ext/extensions.py @@ -12,7 +12,7 @@ class EventDelegate: deferred: bool if false events are triggered as soon as broadcast is called - if true the events will need to be triggered manually + If true, the events will need to be triggered manually. """ def __init__(self, name, deferred=False) -> None: From eb585296e52178ec4fc457decfe2864bbc21322c Mon Sep 17 00:00:00 2001 From: Setepenre Date: Fri, 22 Oct 2021 15:16:08 -0400 Subject: [PATCH 11/12] Update src/orion/ext/extensions.py Co-authored-by: Xavier Bouthillier --- src/orion/ext/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orion/ext/extensions.py b/src/orion/ext/extensions.py index 38f2fa7c7..5723a8de1 100644 --- a/src/orion/ext/extensions.py +++ b/src/orion/ext/extensions.py @@ -148,7 +148,7 @@ def register(self, ext): Parameters ---------- - ext + ext: ``OrionExtension`` object implementing :class`OrionExtension` methods Returns From 778e3564d24d31467680eca8f6559ef4b5a08a0d Mon Sep 17 00:00:00 2001 From: Setepenre Date: Fri, 22 Oct 2021 15:17:16 -0400 Subject: [PATCH 12/12] Update src/orion/ext/extensions.py Co-authored-by: Xavier Bouthillier --- src/orion/ext/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orion/ext/extensions.py b/src/orion/ext/extensions.py index 5723a8de1..21bdb7dbf 100644 --- a/src/orion/ext/extensions.py +++ b/src/orion/ext/extensions.py @@ -149,7 +149,7 @@ def register(self, ext): Parameters ---------- ext: ``OrionExtension`` - object implementing :class`OrionExtension` methods + object implementing :class:`OrionExtension` methods Returns -------