Aluode/PerceptionLabPortable
0
1import numpy as np
2import pytest
3
4from sklearn.linear_model import LogisticRegression
5from sklearn.utils._plotting import (
6 _BinaryClassifierCurveDisplayMixin,
7 _deprecate_estimator_name,
8 _despine,
9 _interval_max_min_ratio,
10 _validate_score_name,
11 _validate_style_kwargs,
12)
13from sklearn.utils._response import _get_response_values_binary
14from sklearn.utils._testing import assert_allclose
15
16
17@pytest.mark.parametrize("ax", [None, "Ax"])
18@pytest.mark.parametrize(
19 "name, expected_name_out", [(None, "TestEstimator"), ("CustomName", "CustomName")]
20)
21def test_validate_plot_params(pyplot, ax, name, expected_name_out):
22 """Check `_validate_plot_params` returns the correct values."""
23 display = _BinaryClassifierCurveDisplayMixin()
24 display.estimator_name = "TestEstimator"
25 if ax:
26 _, ax = pyplot.subplots()
27 ax_out, _, name_out = display._validate_plot_params(ax=ax, name=name)
28
29 assert name_out == expected_name_out
30
31 if ax:
32 assert ax == ax_out
33
34
35@pytest.mark.parametrize("pos_label", [None, 0])
36@pytest.mark.parametrize("name", [None, "CustomName"])
37@pytest.mark.parametrize(
38 "response_method", ["auto", "predict_proba", "decision_function"]
39)
40def test_validate_and_get_response_values(pyplot, pos_label, name, response_method):
41 """Check `_validate_and_get_response_values` returns the correct values."""
42 X = np.array([[0, 0], [1, 1], [2, 2], [3, 3]])
43 y = np.array([0, 0, 2, 2])
44 estimator = LogisticRegression().fit(X, y)
45
46 y_pred, pos_label, name_out = (
47 _BinaryClassifierCurveDisplayMixin._validate_and_get_response_values(
48 estimator,
49 X,
50 y,
51 response_method=response_method,
52 pos_label=pos_label,
53 name=name,
54 )
55 )
56
57 expected_y_pred, expected_pos_label = _get_response_values_binary(
58 estimator, X, response_method=response_method, pos_label=pos_label
59 )
60
61 assert_allclose(y_pred, expected_y_pred)
62 assert pos_label == expected_pos_label
63
64 # Check name is handled correctly
65 expected_name = name if name is not None else "LogisticRegression"
66 assert name_out == expected_name
67
68
69@pytest.mark.parametrize(
70 "y_true, error_message",
71 [
72 (np.array([0, 1, 2]), "The target y is not binary."),
73 (np.array([0, 1]), "Found input variables with inconsistent"),
74 (np.array([0, 2, 0, 2]), r"y_true takes value in \{0, 2\} and pos_label"),
75 ],
76)
77def test_validate_from_predictions_params_errors(pyplot, y_true, error_message):
78 """Check `_validate_from_predictions_params` raises the correct errors."""
79 y_pred = np.array([0.1, 0.2, 0.3, 0.4])
80 sample_weight = np.ones(4)
81
82 with pytest.raises(ValueError, match=error_message):
83 _BinaryClassifierCurveDisplayMixin._validate_from_predictions_params(
84 y_true=y_true,
85 y_pred=y_pred,
86 sample_weight=sample_weight,
87 pos_label=None,
88 )
89
90
91@pytest.mark.parametrize("name", [None, "CustomName"])
92@pytest.mark.parametrize(
93 "pos_label, y_true",
94 [
95 (None, np.array([0, 1, 0, 1])),
96 (2, np.array([0, 2, 0, 2])),
97 ],
98)
99def test_validate_from_predictions_params_returns(pyplot, name, pos_label, y_true):
100 """Check `_validate_from_predictions_params` returns the correct values."""
101 y_pred = np.array([0.1, 0.2, 0.3, 0.4])
102 pos_label_out, name_out = (
103 _BinaryClassifierCurveDisplayMixin._validate_from_predictions_params(
104 y_true=y_true,
105 y_pred=y_pred,
106 sample_weight=None,
107 pos_label=pos_label,
108 name=name,
109 )
110 )
111
112 # Check name is handled correctly
113 expected_name = name if name is not None else "Classifier"
114 assert name_out == expected_name
115
116 # Check pos_label is handled correctly
117 expected_pos_label = pos_label if pos_label is not None else 1
118 assert pos_label_out == expected_pos_label
119
120
121@pytest.mark.parametrize(
122 "params, err_msg",
123 [
124 (
125 {
126 # Missing "indices" key
127 "cv_results": {"estimator": "dummy"},
128 "X": np.array([[1, 2], [3, 4]]),
129 "y": np.array([0, 1]),
130 "sample_weight": None,
131 "pos_label": None,
132 },
133 "`cv_results` does not contain one of the following",
134 ),
135 (
136 {
137 "cv_results": {
138 "estimator": "dummy",
139 "indices": {"test": [[1, 2], [1, 2]], "train": [[3, 4], [3, 4]]},
140 },
141 # `X` wrong length
142 "X": np.array([[1, 2]]),
143 "y": np.array([0, 1]),
144 "sample_weight": None,
145 "pos_label": None,
146 },
147 "`X` does not contain the correct number of",
148 ),
149 (
150 {
151 "cv_results": {
152 "estimator": "dummy",
153 "indices": {"test": [[1, 2], [1, 2]], "train": [[3, 4], [3, 4]]},
154 },
155 "X": np.array([1, 2, 3, 4]),
156 # `y` not binary
157 "y": np.array([0, 2, 1, 3]),
158 "sample_weight": None,
159 "pos_label": None,
160 },
161 "The target `y` is not binary",
162 ),
163 (
164 {
165 "cv_results": {
166 "estimator": "dummy",
167 "indices": {"test": [[1, 2], [1, 2]], "train": [[3, 4], [3, 4]]},
168 },
169 "X": np.array([1, 2, 3, 4]),
170 "y": np.array([0, 1, 0, 1]),
171 # `sample_weight` wrong length
172 "sample_weight": np.array([0.5]),
173 "pos_label": None,
174 },
175 "Found input variables with inconsistent",
176 ),
177 (
178 {
179 "cv_results": {
180 "estimator": "dummy",
181 "indices": {"test": [[1, 2], [1, 2]], "train": [[3, 4], [3, 4]]},
182 },
183 "X": np.array([1, 2, 3, 4]),
184 "y": np.array([2, 3, 2, 3]),
185 "sample_weight": None,
186 # Not specified when `y` not in {0, 1} or {-1, 1}
187 "pos_label": None,
188 },
189 "y takes value in {2, 3} and pos_label is not specified",
190 ),
191 ],
192)
193def test_validate_from_cv_results_params(pyplot, params, err_msg):
194 """Check parameter validation is performed correctly."""
195 with pytest.raises(ValueError, match=err_msg):
196 _BinaryClassifierCurveDisplayMixin()._validate_from_cv_results_params(**params)
197
198
199@pytest.mark.parametrize(
200 "curve_legend_metric, curve_name, expected_label",
201 [
202 (0.85, None, "AUC = 0.85"),
203 (None, "Model A", "Model A"),
204 (0.95, "Random Forest", "Random Forest (AUC = 0.95)"),
205 (None, None, None),
206 ],
207)
208def test_get_legend_label(curve_legend_metric, curve_name, expected_label):
209 """Check `_get_legend_label` returns the correct label."""
210 legend_metric_name = "AUC"
211 label = _BinaryClassifierCurveDisplayMixin._get_legend_label(
212 curve_legend_metric, curve_name, legend_metric_name
213 )
214 assert label == expected_label
215
216
217# TODO(1.9) : Remove
218@pytest.mark.parametrize("curve_kwargs", [{"alpha": 1.0}, None])
219@pytest.mark.parametrize("kwargs", [{}, {"alpha": 1.0}])
220def test_validate_curve_kwargs_deprecate_kwargs(curve_kwargs, kwargs):
221 """Check `_validate_curve_kwargs` deprecates kwargs correctly."""
222 n_curves = 1
223 name = None
224 legend_metric = {"mean": 0.8, "std": 0.1}
225 legend_metric_name = "AUC"
226
227 if curve_kwargs and kwargs:
228 with pytest.raises(ValueError, match="Cannot provide both `curve_kwargs`"):
229 _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
230 n_curves,
231 name,
232 legend_metric,
233 legend_metric_name,
234 curve_kwargs,
235 **kwargs,
236 )
237 elif kwargs:
238 with pytest.warns(FutureWarning, match=r"`\*\*kwargs` is deprecated and"):
239 _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
240 n_curves,
241 name,
242 legend_metric,
243 legend_metric_name,
244 curve_kwargs,
245 **kwargs,
246 )
247 else:
248 # No warning or error should be raised
249 _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
250 n_curves, name, legend_metric, legend_metric_name, curve_kwargs, **kwargs
251 )
252
253
254def test_validate_curve_kwargs_error():
255 """Check `_validate_curve_kwargs` performs parameter validation correctly."""
256 n_curves = 3
257 legend_metric = {"mean": 0.8, "std": 0.1}
258 legend_metric_name = "AUC"
259 with pytest.raises(ValueError, match="`curve_kwargs` must be None"):
260 _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
261 n_curves=n_curves,
262 name=None,
263 legend_metric=legend_metric,
264 legend_metric_name=legend_metric_name,
265 curve_kwargs=[{"alpha": 1.0}],
266 )
267 with pytest.raises(ValueError, match="To avoid labeling individual curves"):
268 name = ["one", "two", "three"]
269 _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
270 n_curves=n_curves,
271 name=name,
272 legend_metric=legend_metric,
273 legend_metric_name=legend_metric_name,
274 curve_kwargs=None,
275 )
276 _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
277 n_curves=n_curves,
278 name=name,
279 legend_metric=legend_metric,
280 legend_metric_name=legend_metric_name,
281 curve_kwargs={"alpha": 1.0},
282 )
283
284
285@pytest.mark.parametrize("name", [None, "curve_name", ["curve_name"]])
286@pytest.mark.parametrize(
287 "legend_metric",
288 [
289 {"mean": 0.8, "std": 0.2},
290 {"mean": None, "std": None},
291 ],
292)
293@pytest.mark.parametrize("legend_metric_name", ["AUC", "AP"])
294@pytest.mark.parametrize(
295 "curve_kwargs",
296 [
297 None,
298 {"color": "red"},
299 ],
300)
301def test_validate_curve_kwargs_single_legend(
302 name, legend_metric, legend_metric_name, curve_kwargs
303):
304 """Check `_validate_curve_kwargs` returns correct kwargs for single legend entry."""
305 n_curves = 3
306 curve_kwargs_out = _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
307 n_curves=n_curves,
308 name=name,
309 legend_metric=legend_metric,
310 legend_metric_name=legend_metric_name,
311 curve_kwargs=curve_kwargs,
312 )
313
314 assert isinstance(curve_kwargs_out, list)
315 assert len(curve_kwargs_out) == n_curves
316
317 expected_label = None
318 if isinstance(name, list):
319 name = name[0]
320 if name is not None:
321 expected_label = name
322 if legend_metric["mean"] is not None:
323 expected_label = expected_label + f" ({legend_metric_name} = 0.80 +/- 0.20)"
324 # `name` is None
325 elif legend_metric["mean"] is not None:
326 expected_label = f"{legend_metric_name} = 0.80 +/- 0.20"
327
328 assert curve_kwargs_out[0]["label"] == expected_label
329 # All remaining curves should have None as "label"
330 assert curve_kwargs_out[1]["label"] is None
331 assert curve_kwargs_out[2]["label"] is None
332
333 # Default multi-curve kwargs
334 if curve_kwargs is None:
335 assert all(len(kwargs) == 4 for kwargs in curve_kwargs_out)
336 assert all(kwargs["alpha"] == 0.5 for kwargs in curve_kwargs_out)
337 assert all(kwargs["linestyle"] == "--" for kwargs in curve_kwargs_out)
338 assert all(kwargs["color"] == "blue" for kwargs in curve_kwargs_out)
339 else:
340 assert all(len(kwargs) == 2 for kwargs in curve_kwargs_out)
341 assert all(kwargs["color"] == "red" for kwargs in curve_kwargs_out)
342
343
344@pytest.mark.parametrize("name", [None, "curve_name", ["one", "two", "three"]])
345@pytest.mark.parametrize(
346 "legend_metric", [{"metric": [1.0, 1.0, 1.0]}, {"metric": [None, None, None]}]
347)
348@pytest.mark.parametrize("legend_metric_name", ["AUC", "AP"])
349def test_validate_curve_kwargs_multi_legend(name, legend_metric, legend_metric_name):
350 """Check `_validate_curve_kwargs` returns correct kwargs for multi legend entry."""
351 n_curves = 3
352 curve_kwargs = [{"color": "red"}, {"color": "yellow"}, {"color": "blue"}]
353 curve_kwargs_out = _BinaryClassifierCurveDisplayMixin._validate_curve_kwargs(
354 n_curves=n_curves,
355 name=name,
356 legend_metric=legend_metric,
357 legend_metric_name=legend_metric_name,
358 curve_kwargs=curve_kwargs,
359 )
360
361 assert isinstance(curve_kwargs_out, list)
362 assert len(curve_kwargs_out) == n_curves
363
364 expected_labels = [None, None, None]
365 if isinstance(name, str):
366 expected_labels = "curve_name"
367 if legend_metric["metric"][0] is not None:
368 expected_labels = expected_labels + f" ({legend_metric_name} = 1.00)"
369 expected_labels = [expected_labels] * n_curves
370 elif isinstance(name, list) and legend_metric["metric"][0] is None:
371 expected_labels = name
372 elif isinstance(name, list) and legend_metric["metric"][0] is not None:
373 expected_labels = [
374 f"{name_single} ({legend_metric_name} = 1.00)" for name_single in name
375 ]
376 # `name` is None
377 elif legend_metric["metric"][0] is not None:
378 expected_labels = [f"{legend_metric_name} = 1.00"] * n_curves
379
380 for idx, expected_label in enumerate(expected_labels):
381 assert curve_kwargs_out[idx]["label"] == expected_label
382
383 assert all(len(kwargs) == 2 for kwargs in curve_kwargs_out)
384 for curve_kwarg, curve_kwarg_out in zip(curve_kwargs, curve_kwargs_out):
385 assert curve_kwarg_out["color"] == curve_kwarg["color"]
386
387
388def metric():
389 pass # pragma: no cover
390
391
392def neg_metric():
393 pass # pragma: no cover
394
395
396@pytest.mark.parametrize(
397 "score_name, scoring, negate_score, expected_score_name",
398 [
399 ("accuracy", None, False, "accuracy"), # do not transform the name
400 (None, "accuracy", False, "Accuracy"), # capitalize the name
401 (None, "accuracy", True, "Negative accuracy"), # add "Negative"
402 (None, "neg_mean_absolute_error", False, "Negative mean absolute error"),
403 (None, "neg_mean_absolute_error", True, "Mean absolute error"), # remove "neg_"
404 ("MAE", "neg_mean_absolute_error", True, "MAE"), # keep score_name
405 (None, None, False, "Score"), # default name
406 (None, None, True, "Negative score"), # default name but negated
407 ("Some metric", metric, False, "Some metric"), # do not transform the name
408 ("Some metric", metric, True, "Some metric"), # do not transform the name
409 (None, metric, False, "Metric"), # default name
410 (None, metric, True, "Negative metric"), # default name but negated
411 ("Some metric", neg_metric, False, "Some metric"), # do not transform the name
412 ("Some metric", neg_metric, True, "Some metric"), # do not transform the name
413 (None, neg_metric, False, "Negative metric"), # default name
414 (None, neg_metric, True, "Metric"), # default name but negated
415 ],
416)
417def test_validate_score_name(score_name, scoring, negate_score, expected_score_name):
418 """Check that we return the right score name."""
419 assert (
420 _validate_score_name(score_name, scoring, negate_score) == expected_score_name
421 )
422
423
424# In the following test, we check the value of the max to min ratio
425# for parameter value intervals to check that using a decision threshold
426# of 5. is a good heuristic to decide between linear and log scales on
427# common ranges of parameter values.
428@pytest.mark.parametrize(
429 "data, lower_bound, upper_bound",
430 [
431 # Such a range could be clearly displayed with either log scale or linear
432 # scale.
433 (np.geomspace(0.1, 1, 5), 5, 6),
434 # Checking that the ratio is still positive on a negative log scale.
435 (-np.geomspace(0.1, 1, 10), 7, 8),
436 # Evenly spaced parameter values lead to a ratio of 1.
437 (np.linspace(0, 1, 5), 0.9, 1.1),
438 # This is not exactly spaced on a log scale but we will benefit from treating
439 # it as such for visualization.
440 ([1, 2, 5, 10, 20, 50], 20, 40),
441 ],
442)
443def test_inverval_max_min_ratio(data, lower_bound, upper_bound):
444 assert lower_bound < _interval_max_min_ratio(data) < upper_bound
445
446
447@pytest.mark.parametrize(
448 "default_kwargs, user_kwargs, expected",
449 [
450 (
451 {"color": "blue", "linewidth": 2},
452 {"linestyle": "dashed"},
453 {"color": "blue", "linewidth": 2, "linestyle": "dashed"},
454 ),
455 (
456 {"color": "blue", "linestyle": "solid"},
457 {"c": "red", "ls": "dashed"},
458 {"color": "red", "linestyle": "dashed"},
459 ),
460 (
461 {"label": "xxx", "color": "k", "linestyle": "--"},
462 {"ls": "-."},
463 {"label": "xxx", "color": "k", "linestyle": "-."},
464 ),
465 ({}, {}, {}),
466 (
467 {},
468 {
469 "ls": "dashed",
470 "c": "red",
471 "ec": "black",
472 "fc": "yellow",
473 "lw": 2,
474 "mec": "green",
475 "mfcalt": "blue",
476 "ms": 5,
477 },
478 {
479 "linestyle": "dashed",
480 "color": "red",
481 "edgecolor": "black",
482 "facecolor": "yellow",
483 "linewidth": 2,
484 "markeredgecolor": "green",
485 "markerfacecoloralt": "blue",
486 "markersize": 5,
487 },
488 ),
489 ],
490)
491def test_validate_style_kwargs(default_kwargs, user_kwargs, expected):
492 """Check the behaviour of `validate_style_kwargs` with various type of entries."""
493 result = _validate_style_kwargs(default_kwargs, user_kwargs)
494 assert result == expected, (
495 "The validation of style keywords does not provide the expected results: "
496 f"Got {result} instead of {expected}."
497 )
498
499
500@pytest.mark.parametrize(
501 "default_kwargs, user_kwargs",
502 [({}, {"ls": 2, "linestyle": 3}), ({}, {"c": "r", "color": "blue"})],
503)
504def test_validate_style_kwargs_error(default_kwargs, user_kwargs):
505 """Check that `validate_style_kwargs` raises TypeError"""
506 with pytest.raises(TypeError):
507 _validate_style_kwargs(default_kwargs, user_kwargs)
508
509
510def test_despine(pyplot):
511 ax = pyplot.gca()
512 _despine(ax)
513 assert ax.spines["top"].get_visible() is False
514 assert ax.spines["right"].get_visible() is False
515 assert ax.spines["bottom"].get_bounds() == (0, 1)
516 assert ax.spines["left"].get_bounds() == (0, 1)
517
518
519@pytest.mark.parametrize("estimator_name", ["my_est_name", "deprecated"])
520@pytest.mark.parametrize("name", [None, "my_name"])
521def test_deprecate_estimator_name(estimator_name, name):
522 """Check `_deprecate_estimator_name` behaves correctly"""
523 version = "1.7"
524 version_remove = "1.9"
525
526 if estimator_name == "deprecated":
527 name_out = _deprecate_estimator_name(estimator_name, name, version)
528 assert name_out == name
529 # `estimator_name` is provided and `name` is:
530 elif name is None:
531 warning_message = (
532 f"`estimator_name` is deprecated in {version} and will be removed in "
533 f"{version_remove}. Use `name` instead."
534 )
535 with pytest.warns(FutureWarning, match=warning_message):
536 result = _deprecate_estimator_name(estimator_name, name, version)
537 assert result == estimator_name
538 elif name is not None:
539 error_message = (
540 f"Cannot provide both `estimator_name` and `name`. `estimator_name` "
541 f"is deprecated in {version} and will be removed in {version_remove}. "
542 )
543 with pytest.raises(ValueError, match=error_message):
544 _deprecate_estimator_name(estimator_name, name, version)
545