Aluode/PerceptionLabPortable

1import warnings
2
3import numpy as np
4import pytest
5from scipy.optimize import fmin_ncg
6
7from sklearn.exceptions import ConvergenceWarning
8from sklearn.utils._bunch import Bunch
9from sklearn.utils._testing import assert_allclose
10from sklearn.utils.optimize import _check_optimize_result, _newton_cg
11
12
13def test_newton_cg(global_random_seed):
14    # Test that newton_cg gives same result as scipy's fmin_ncg
15
16    rng = np.random.RandomState(global_random_seed)
17    A = rng.normal(size=(10, 10))
18    x0 = np.ones(10)
19
20    def func(x):
21        Ax = A.dot(x)
22        return 0.5 * (Ax).dot(Ax)
23
24    def grad(x):
25        return A.T.dot(A.dot(x))
26
27    def hess(x, p):
28        return p.dot(A.T.dot(A.dot(x.all())))
29
30    def grad_hess(x):
31        return grad(x), lambda x: A.T.dot(A.dot(x))
32
33    # func is a definite positive quadratic form, so the minimum is at x = 0
34    # hence the use of absolute tolerance.
35    assert np.all(np.abs(_newton_cg(grad_hess, func, grad, x0, tol=1e-10)[0]) <= 1e-7)
36    assert_allclose(
37        _newton_cg(grad_hess, func, grad, x0, tol=1e-7)[0],
38        fmin_ncg(f=func, x0=x0, fprime=grad, fhess_p=hess),
39        atol=1e-5,
40    )
41
42
43@pytest.mark.parametrize("verbose", [0, 1, 2])
44def test_newton_cg_verbosity(capsys, verbose):
45    """Test the std output of verbose newton_cg solver."""
46    A = np.eye(2)
47    b = np.array([1, 2], dtype=float)
48
49    _newton_cg(
50        grad_hess=lambda x: (A @ x - b, lambda z: A @ z),
51        func=lambda x: 0.5 * x @ A @ x - b @ x,
52        grad=lambda x: A @ x - b,
53        x0=np.zeros(A.shape[0]),
54        verbose=verbose,
55    )  # returns array([1., 2])
56    captured = capsys.readouterr()
57
58    if verbose == 0:
59        assert captured.out == ""
60    else:
61        msg = [
62            "Newton-CG iter = 1",
63            "Check Convergence",
64            "max |gradient|",
65            "Solver did converge at loss = ",
66        ]
67        for m in msg:
68            assert m in captured.out
69
70    if verbose >= 2:
71        msg = [
72            "Inner CG solver iteration 1 stopped with",
73            "sum(|residuals|) <= tol",
74            "Line Search",
75            "try line search wolfe1",
76            "wolfe1 line search was successful",
77        ]
78        for m in msg:
79            assert m in captured.out
80
81    if verbose >= 2:
82        # Set up a badly scaled singular Hessian with a completely wrong starting
83        # position. This should trigger 2nd line search check
84        A = np.array([[1.0, 2], [2, 4]]) * 1e30  # collinear columns
85        b = np.array([1.0, 2.0])
86        # Note that scipy.optimize._linesearch LineSearchWarning inherits from
87        # RuntimeWarning, but we do not want to import from non public APIs.
88        with pytest.warns(RuntimeWarning):
89            _newton_cg(
90                grad_hess=lambda x: (A @ x - b, lambda z: A @ z),
91                func=lambda x: 0.5 * x @ A @ x - b @ x,
92                grad=lambda x: A @ x - b,
93                x0=np.array([-2.0, 1]),  # null space of hessian
94                verbose=verbose,
95            )
96        captured = capsys.readouterr()
97        msg = [
98            "wolfe1 line search was not successful",
99            "check loss |improvement| <= eps * |loss_old|:",
100            "check sum(|gradient|) < sum(|gradient_old|):",
101            "last resort: try line search wolfe2",
102        ]
103        for m in msg:
104            assert m in captured.out
105
106        # Set up a badly conditioned Hessian that leads to tiny curvature.
107        # X.T @ X have singular values array([1.00000400e+01, 1.00008192e-11])
108        A = np.array([[1.0, 2], [1, 2 + 1e-15]])
109        b = np.array([-2.0, 1])
110        with pytest.warns(ConvergenceWarning):
111            _newton_cg(
112                grad_hess=lambda x: (A @ x - b, lambda z: A @ z),
113                func=lambda x: 0.5 * x @ A @ x - b @ x,
114                grad=lambda x: A @ x - b,
115                x0=b,
116                verbose=verbose,
117                maxiter=2,
118            )
119        captured = capsys.readouterr()
120        msg = [
121            "tiny_|p| = eps * ||p||^2",
122        ]
123        for m in msg:
124            assert m in captured.out
125
126        # Test for a case with negative Hessian.
127        # We do not trigger "Inner CG solver iteration {i} stopped with negative
128        # curvature", but that is very hard to trigger.
129        A = np.eye(2)
130        b = np.array([-2.0, 1])
131        with pytest.warns(RuntimeWarning):
132            _newton_cg(
133                # Note the wrong sign in the hessian product.
134                grad_hess=lambda x: (A @ x - b, lambda z: -A @ z),
135                func=lambda x: 0.5 * x @ A @ x - b @ x,
136                grad=lambda x: A @ x - b,
137                x0=np.array([1.0, 1.0]),
138                verbose=verbose,
139                maxiter=3,
140            )
141        captured = capsys.readouterr()
142        msg = [
143            "Inner CG solver iteration 0 fell back to steepest descent",
144        ]
145        for m in msg:
146            assert m in captured.out
147
148        A = np.diag([1e-3, 1, 1e3])
149        b = np.array([-2.0, 1, 2.0])
150        with pytest.warns(ConvergenceWarning):
151            _newton_cg(
152                grad_hess=lambda x: (A @ x - b, lambda z: A @ z),
153                func=lambda x: 0.5 * x @ A @ x - b @ x,
154                grad=lambda x: A @ x - b,
155                x0=np.ones_like(b),
156                verbose=verbose,
157                maxiter=2,
158                maxinner=1,
159            )
160        captured = capsys.readouterr()
161        msg = [
162            "Inner CG solver stopped reaching maxiter=1",
163        ]
164        for m in msg:
165            assert m in captured.out
166
167
168def test_check_optimize():
169    # Mock some lbfgs output using a Bunch instance:
170    result = Bunch()
171
172    # First case: no warnings
173    result.nit = 1
174    result.status = 0
175    result.message = "OK"
176
177    with warnings.catch_warnings():
178        warnings.simplefilter("error")
179        _check_optimize_result("lbfgs", result)
180
181    # Second case: warning about implicit `max_iter`: do not recommend the user
182    # to increase `max_iter` this is not a user settable parameter.
183    result.status = 1
184    result.message = "STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT"
185    with pytest.warns(ConvergenceWarning) as record:
186        _check_optimize_result("lbfgs", result)
187
188    assert len(record) == 1
189    warn_msg = record[0].message.args[0]
190    assert "lbfgs failed to converge after 1 iteration(s)" in warn_msg
191    assert result.message in warn_msg
192    assert "Increase the number of iterations" not in warn_msg
193    assert "scale the data" in warn_msg
194
195    # Third case: warning about explicit `max_iter`: recommend user to increase
196    # `max_iter`.
197    with pytest.warns(ConvergenceWarning) as record:
198        _check_optimize_result("lbfgs", result, max_iter=1)
199
200    assert len(record) == 1
201    warn_msg = record[0].message.args[0]
202    assert "lbfgs failed to converge after 1 iteration(s)" in warn_msg
203    assert result.message in warn_msg
204    assert "Increase the number of iterations" in warn_msg
205    assert "scale the data" in warn_msg
206
207    # Fourth case: other convergence problem before reaching `max_iter`: do not
208    # recommend increasing `max_iter`.
209    result.nit = 2
210    result.status = 2
211    result.message = "ABNORMAL"
212    with pytest.warns(ConvergenceWarning) as record:
213        _check_optimize_result("lbfgs", result, max_iter=10)
214
215    assert len(record) == 1
216    warn_msg = record[0].message.args[0]
217    assert "lbfgs failed to converge after 2 iteration(s)" in warn_msg
218    assert result.message in warn_msg
219    assert "Increase the number of iterations" not in warn_msg
220    assert "scale the data" in warn_msg
221