From 1e1883ad1c9a5472cea27181d0ef52d5915fd161 Mon Sep 17 00:00:00 2001
From: Janos Gabler <janos.gabler@gmail.com>
Date: Sat, 19 Nov 2022 15:34:49 +0100
Subject: [PATCH] Several small improvements (#412)

---
 codecov.yml                                   |  6 +-
 .../how_to_specify_constraints.md             |  4 ++
 src/estimagic/inference/shared.py             |  4 +-
 .../optimization/fides_optimizers.py          | 19 ++++++
 ...nt_quadratic.py => _conjugate_gradient.py} |  6 +-
 ...ic_fast.py => _conjugate_gradient_fast.py} |  6 +-
 ..._toint_quadratic.py => _steihaug_toint.py} |  0
 ...dratic_fast.py => _steihaug_toint_fast.py} |  0
 .../{_trsbox_quadratic.py => _trsbox.py}      |  0
 ...sbox_quadratic_fast.py => _trsbox_fast.py} |  0
 src/estimagic/optimization/subsolvers/bntr.py |  6 +-
 .../optimization/subsolvers/bntr_fast.py      |  6 +-
 src/estimagic/optimization/tiktak.py          |  6 ++
 .../test_bntr_fast.py                         |  0
 .../test_gqtpar_fast.py                       |  0
 .../test_gqtpar_lambdas.py                    |  0
 .../test_minimize_trust_region.py             | 62 +++++++++----------
 .../optimization/test_quadratic_subsolvers.py |  6 +-
 18 files changed, 85 insertions(+), 46 deletions(-)
 rename src/estimagic/optimization/subsolvers/{_conjugate_gradient_quadratic.py => _conjugate_gradient.py} (95%)
 rename src/estimagic/optimization/subsolvers/{_conjugate_gradient_quadratic_fast.py => _conjugate_gradient_fast.py} (96%)
 rename src/estimagic/optimization/subsolvers/{_steihaug_toint_quadratic.py => _steihaug_toint.py} (100%)
 rename src/estimagic/optimization/subsolvers/{_steihaug_toint_quadratic_fast.py => _steihaug_toint_fast.py} (100%)
 rename src/estimagic/optimization/subsolvers/{_trsbox_quadratic.py => _trsbox.py} (100%)
 rename src/estimagic/optimization/subsolvers/{_trsbox_quadratic_fast.py => _trsbox_fast.py} (100%)
 rename tests/optimization/{tranquilo => subsolvers}/test_bntr_fast.py (100%)
 rename tests/optimization/{tranquilo => subsolvers}/test_gqtpar_fast.py (100%)
 rename tests/optimization/{tranquilo => subsolvers}/test_gqtpar_lambdas.py (100%)
 rename tests/optimization/{tranquilo => subsolvers}/test_minimize_trust_region.py (85%)

diff --git a/codecov.yml b/codecov.yml
index c8b0cc426..36eb45c99 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -20,7 +20,7 @@ ignore:
   - "setup.py"
   - "estimagic/tests/**/*"
   - "src/estimagic/optimization/subsolvers/bntr_fast.py"
-  - "src/estimagic/optimization/subsolvers/_trsbox_quadratic_fast.py"
-  - "src/estimagic/optimization/subsolvers/_conjugate_gradient_quadratic_fast.py"
-  - "src/estimagic/optimization/subsolvers/_steihaug_toint_quadratic_fast.py"
+  - "src/estimagic/optimization/subsolvers/_trsbox_fast.py"
+  - "src/estimagic/optimization/subsolvers/_conjugate_gradient_fast.py"
+  - "src/estimagic/optimization/subsolvers/_steihaug_toint_fast.py"
   - "src/estimagic/optimization/subsolvers/gqtpar_fast.py"
diff --git a/docs/source/how_to_guides/optimization/how_to_specify_constraints.md b/docs/source/how_to_guides/optimization/how_to_specify_constraints.md
index ec07164ca..d6aa5be5e 100644
--- a/docs/source/how_to_guides/optimization/how_to_specify_constraints.md
+++ b/docs/source/how_to_guides/optimization/how_to_specify_constraints.md
@@ -406,6 +406,10 @@ flat numpy array are explained in the next section.
 
     Where the product of all but the last parameters is equal to 1.
 
+    If you have a function that calculates the derivative of your constraint, you can
+    add this under the key `"derivative"` to the constraint dictionary. Otherwise,
+    numerical derivatives are calculated for you if needed.
+
 ```
 
 ## Imposing multiple constraints at once
diff --git a/src/estimagic/inference/shared.py b/src/estimagic/inference/shared.py
index f8169003a..8e9a29d5b 100644
--- a/src/estimagic/inference/shared.py
+++ b/src/estimagic/inference/shared.py
@@ -153,12 +153,14 @@ def calculate_estimation_summary(
 
     df = pd.DataFrame(flat_data, index=names)
 
-    df.loc[free_names, "stars"] = pd.cut(
+    stars = pd.cut(
         df.loc[free_names, "p_value"],
         bins=[-1, 0.01, 0.05, 0.1, 2],
         labels=["***", "**", "*", ""],
     )
 
+    df["stars"] = stars
+
     # ==================================================================================
     # Map summary data into params tree structure
     # ==================================================================================
diff --git a/src/estimagic/optimization/fides_optimizers.py b/src/estimagic/optimization/fides_optimizers.py
index 9c8ae2df0..27a9637aa 100644
--- a/src/estimagic/optimization/fides_optimizers.py
+++ b/src/estimagic/optimization/fides_optimizers.py
@@ -108,10 +108,29 @@ def _process_fides_res(raw_res, opt):
         "solution_hessian": hess,
         "success": opt.converged,
         "n_iterations": opt.iteration,
+        "message": _process_exitflag(opt.exitflag),
     }
     return res
 
 
+def _process_exitflag(exitflag):
+    messages = {
+        "DID_NOT_RUN": "The optimizer did not run",
+        "MAXITER": "Reached maximum number of allowed iterations",
+        "MAXTIME": "Expected to reach maximum allowed time in next iteration",
+        "NOT_FINITE": "Encountered non-finite fval/grad/hess",
+        "EXCEEDED_BOUNDARY": "Exceeded specified boundaries",
+        "DELTA_TOO_SMALL": "Trust Region Radius too small to proceed",
+        "FTOL": "Converged according to fval difference",
+        "XTOL": "Converged according to x difference",
+        "GTOL": "Converged according to gradient norm",
+    }
+
+    out = messages.get(exitflag.name)
+
+    return out
+
+
 def _create_hessian_updater_from_user_input(hessian_update_strategy):
     hessians_needing_residuals = (
         hessian_approximation.FX,
diff --git a/src/estimagic/optimization/subsolvers/_conjugate_gradient_quadratic.py b/src/estimagic/optimization/subsolvers/_conjugate_gradient.py
similarity index 95%
rename from src/estimagic/optimization/subsolvers/_conjugate_gradient_quadratic.py
rename to src/estimagic/optimization/subsolvers/_conjugate_gradient.py
index 67ef53f0b..6d805e55d 100644
--- a/src/estimagic/optimization/subsolvers/_conjugate_gradient_quadratic.py
+++ b/src/estimagic/optimization/subsolvers/_conjugate_gradient.py
@@ -47,7 +47,11 @@ def minimize_trust_cg(
             x_candidate, direction, trustregion_radius
         )
 
-        step_size = (residual @ residual) / square_terms
+        # avoid divide by zero warning
+        if square_terms > 0:
+            step_size = (residual @ residual) / square_terms
+        else:
+            step_size = np.inf
 
         if square_terms <= 0 or step_size > distance_to_boundary:
             x_candidate = x_candidate + distance_to_boundary * direction
diff --git a/src/estimagic/optimization/subsolvers/_conjugate_gradient_quadratic_fast.py b/src/estimagic/optimization/subsolvers/_conjugate_gradient_fast.py
similarity index 96%
rename from src/estimagic/optimization/subsolvers/_conjugate_gradient_quadratic_fast.py
rename to src/estimagic/optimization/subsolvers/_conjugate_gradient_fast.py
index 5ff30d520..7a75bc82c 100644
--- a/src/estimagic/optimization/subsolvers/_conjugate_gradient_quadratic_fast.py
+++ b/src/estimagic/optimization/subsolvers/_conjugate_gradient_fast.py
@@ -49,7 +49,11 @@ def minimize_trust_cg_fast(
             x_candidate, direction, trustregion_radius
         )
 
-        step_size = (residual @ residual) / square_terms
+        # avoid divide by zero warning
+        if square_terms > 0:
+            step_size = (residual @ residual) / square_terms
+        else:
+            step_size = np.inf
 
         if square_terms <= 0 or step_size > distance_to_boundary:
             x_candidate = x_candidate + distance_to_boundary * direction
diff --git a/src/estimagic/optimization/subsolvers/_steihaug_toint_quadratic.py b/src/estimagic/optimization/subsolvers/_steihaug_toint.py
similarity index 100%
rename from src/estimagic/optimization/subsolvers/_steihaug_toint_quadratic.py
rename to src/estimagic/optimization/subsolvers/_steihaug_toint.py
diff --git a/src/estimagic/optimization/subsolvers/_steihaug_toint_quadratic_fast.py b/src/estimagic/optimization/subsolvers/_steihaug_toint_fast.py
similarity index 100%
rename from src/estimagic/optimization/subsolvers/_steihaug_toint_quadratic_fast.py
rename to src/estimagic/optimization/subsolvers/_steihaug_toint_fast.py
diff --git a/src/estimagic/optimization/subsolvers/_trsbox_quadratic.py b/src/estimagic/optimization/subsolvers/_trsbox.py
similarity index 100%
rename from src/estimagic/optimization/subsolvers/_trsbox_quadratic.py
rename to src/estimagic/optimization/subsolvers/_trsbox.py
diff --git a/src/estimagic/optimization/subsolvers/_trsbox_quadratic_fast.py b/src/estimagic/optimization/subsolvers/_trsbox_fast.py
similarity index 100%
rename from src/estimagic/optimization/subsolvers/_trsbox_quadratic_fast.py
rename to src/estimagic/optimization/subsolvers/_trsbox_fast.py
diff --git a/src/estimagic/optimization/subsolvers/bntr.py b/src/estimagic/optimization/subsolvers/bntr.py
index 88c085a2d..1b6076738 100644
--- a/src/estimagic/optimization/subsolvers/bntr.py
+++ b/src/estimagic/optimization/subsolvers/bntr.py
@@ -4,13 +4,13 @@
 from typing import Union
 
 import numpy as np
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic import (
+from estimagic.optimization.subsolvers._conjugate_gradient import (
     minimize_trust_cg,
 )
-from estimagic.optimization.subsolvers._steihaug_toint_quadratic import (
+from estimagic.optimization.subsolvers._steihaug_toint import (
     minimize_trust_stcg,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import minimize_trust_trsbox
+from estimagic.optimization.subsolvers._trsbox import minimize_trust_trsbox
 
 EPSILON = np.finfo(float).eps ** (2 / 3)
 
diff --git a/src/estimagic/optimization/subsolvers/bntr_fast.py b/src/estimagic/optimization/subsolvers/bntr_fast.py
index d6fd5ce17..fbbbe9257 100644
--- a/src/estimagic/optimization/subsolvers/bntr_fast.py
+++ b/src/estimagic/optimization/subsolvers/bntr_fast.py
@@ -1,12 +1,12 @@
 """Auxiliary functions for the quadratic BNTR trust-region subsolver."""
 import numpy as np
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic_fast import (
+from estimagic.optimization.subsolvers._conjugate_gradient_fast import (
     minimize_trust_cg_fast,
 )
-from estimagic.optimization.subsolvers._steihaug_toint_quadratic_fast import (
+from estimagic.optimization.subsolvers._steihaug_toint_fast import (
     minimize_trust_stcg_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     minimize_trust_trsbox_fast,
 )
 from numba import njit
diff --git a/src/estimagic/optimization/tiktak.py b/src/estimagic/optimization/tiktak.py
index 31f511f0e..3e2844e21 100644
--- a/src/estimagic/optimization/tiktak.py
+++ b/src/estimagic/optimization/tiktak.py
@@ -454,6 +454,9 @@ def update_convergence_state(
         bool: A bool that indicates if the optimizer has converged.
 
     """
+    # ==================================================================================
+    # unpack some variables
+    # ==================================================================================
     xtol = convergence_criteria["xtol"]
     max_discoveries = convergence_criteria["max_discoveries"]
 
@@ -461,6 +464,9 @@ def update_convergence_state(
     best_y = current_state["best_y"]
     best_res = current_state["best_res"]
 
+    # ==================================================================================
+    # filter out optimizations that raised errors
+    # ==================================================================================
     # get indices of local optimizations that did not fail
     valid_indices = [i for i, res in enumerate(results) if not isinstance(res, str)]
 
diff --git a/tests/optimization/tranquilo/test_bntr_fast.py b/tests/optimization/subsolvers/test_bntr_fast.py
similarity index 100%
rename from tests/optimization/tranquilo/test_bntr_fast.py
rename to tests/optimization/subsolvers/test_bntr_fast.py
diff --git a/tests/optimization/tranquilo/test_gqtpar_fast.py b/tests/optimization/subsolvers/test_gqtpar_fast.py
similarity index 100%
rename from tests/optimization/tranquilo/test_gqtpar_fast.py
rename to tests/optimization/subsolvers/test_gqtpar_fast.py
diff --git a/tests/optimization/tranquilo/test_gqtpar_lambdas.py b/tests/optimization/subsolvers/test_gqtpar_lambdas.py
similarity index 100%
rename from tests/optimization/tranquilo/test_gqtpar_lambdas.py
rename to tests/optimization/subsolvers/test_gqtpar_lambdas.py
diff --git a/tests/optimization/tranquilo/test_minimize_trust_region.py b/tests/optimization/subsolvers/test_minimize_trust_region.py
similarity index 85%
rename from tests/optimization/tranquilo/test_minimize_trust_region.py
rename to tests/optimization/subsolvers/test_minimize_trust_region.py
index 3c4c41865..b7a9041bd 100644
--- a/tests/optimization/tranquilo/test_minimize_trust_region.py
+++ b/tests/optimization/subsolvers/test_minimize_trust_region.py
@@ -1,93 +1,93 @@
 import numpy as np
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic import (
+from estimagic.optimization.subsolvers._conjugate_gradient import (
     _get_distance_to_trustregion_boundary as gdtb,
 )
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic import (
+from estimagic.optimization.subsolvers._conjugate_gradient import (
     _update_vectors_for_next_iteration as uvnr,
 )
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic import (
+from estimagic.optimization.subsolvers._conjugate_gradient import (
     minimize_trust_cg,
 )
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic_fast import (
+from estimagic.optimization.subsolvers._conjugate_gradient_fast import (
     _get_distance_to_trustregion_boundary as gdtb_fast,
 )
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic_fast import (
+from estimagic.optimization.subsolvers._conjugate_gradient_fast import (
     _update_vectors_for_next_iteration as uvnr_fast,
 )
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic_fast import (
+from estimagic.optimization.subsolvers._conjugate_gradient_fast import (
     minimize_trust_cg_fast,
 )
-from estimagic.optimization.subsolvers._steihaug_toint_quadratic import (
+from estimagic.optimization.subsolvers._steihaug_toint import (
     minimize_trust_stcg,
 )
-from estimagic.optimization.subsolvers._steihaug_toint_quadratic_fast import (
+from estimagic.optimization.subsolvers._steihaug_toint_fast import (
     minimize_trust_stcg_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _apply_bounds_to_candidate_vector,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _calc_greatest_criterion_reduction as greatest_reduction_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _calc_new_reduction as new_reduction_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _calc_upper_bound_on_tangent as upper_bound_tangent_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _compute_new_search_direction_and_norm as new_dir_and_norm_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _take_constrained_step_up_to_boundary as step_constrained_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _take_unconstrained_step_up_to_boundary as step_unconstrained_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _update_candidate_vectors_and_reduction as update_candidate_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _update_candidate_vectors_and_reduction_alt_step as update_candidate_alt_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import (
+from estimagic.optimization.subsolvers._trsbox import (
     _update_tangent as update_tanget_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import minimize_trust_trsbox
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox import minimize_trust_trsbox
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _calc_greatest_criterion_reduction as greatest_reduction_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _calc_new_reduction as new_reduction_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _calc_upper_bound_on_tangent as upper_bound_tangent_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _compute_new_search_direction_and_norm as new_dir_and_norm_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _perform_alternative_trustregion_step as perform_step_alt_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _perform_alternative_trustregion_step as perform_step_alt_orig,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _take_constrained_step_up_to_boundary as step_constrained_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _take_unconstrained_step_up_to_boundary as step_unconstrained_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _update_candidate_vectors_and_reduction as update_candidate_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _update_candidate_vectors_and_reduction_alt_step as update_candidate_alt_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     _update_tangent as update_tanget_fast,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic_fast import (
+from estimagic.optimization.subsolvers._trsbox_fast import (
     minimize_trust_trsbox_fast,
 )
 from numpy.testing import assert_array_almost_equal as aaae
diff --git a/tests/optimization/test_quadratic_subsolvers.py b/tests/optimization/test_quadratic_subsolvers.py
index af2b2d2f6..50c3f4f4e 100644
--- a/tests/optimization/test_quadratic_subsolvers.py
+++ b/tests/optimization/test_quadratic_subsolvers.py
@@ -2,13 +2,13 @@
 import numpy as np
 import pytest
 from estimagic.optimization.pounders_auxiliary import MainModel
-from estimagic.optimization.subsolvers._conjugate_gradient_quadratic import (
+from estimagic.optimization.subsolvers._conjugate_gradient import (
     minimize_trust_cg,
 )
-from estimagic.optimization.subsolvers._steihaug_toint_quadratic import (
+from estimagic.optimization.subsolvers._steihaug_toint import (
     minimize_trust_stcg,
 )
-from estimagic.optimization.subsolvers._trsbox_quadratic import minimize_trust_trsbox
+from estimagic.optimization.subsolvers._trsbox import minimize_trust_trsbox
 from estimagic.optimization.subsolvers.bntr import (
     bntr,
 )