Aluode/PerceptionLabPortable
0
1import numpy as np
2import pytest
3
4from sklearn import config_context
5from sklearn.base import clone
6from sklearn.datasets import (
7 load_breast_cancer,
8 load_iris,
9 make_classification,
10 make_multilabel_classification,
11)
12from sklearn.dummy import DummyClassifier
13from sklearn.ensemble import GradientBoostingClassifier
14from sklearn.exceptions import NotFittedError
15from sklearn.linear_model import LogisticRegression
16from sklearn.metrics import (
17 balanced_accuracy_score,
18 f1_score,
19 fbeta_score,
20 make_scorer,
21)
22from sklearn.metrics._scorer import _CurveScorer
23from sklearn.model_selection import (
24 FixedThresholdClassifier,
25 StratifiedShuffleSplit,
26 TunedThresholdClassifierCV,
27)
28from sklearn.model_selection._classification_threshold import (
29 _fit_and_score_over_thresholds,
30)
31from sklearn.pipeline import make_pipeline
32from sklearn.preprocessing import StandardScaler
33from sklearn.svm import SVC
34from sklearn.tree import DecisionTreeClassifier
35from sklearn.utils._mocking import CheckingClassifier
36from sklearn.utils._testing import (
37 _convert_container,
38 assert_allclose,
39 assert_array_equal,
40)
41
42
43def test_fit_and_score_over_thresholds_curve_scorers():
44 """Check that `_fit_and_score_over_thresholds` returns thresholds in ascending order
45 for the different accepted curve scorers."""
46 X, y = make_classification(n_samples=100, random_state=0)
47 train_idx, val_idx = np.arange(50), np.arange(50, 100)
48 classifier = LogisticRegression()
49
50 curve_scorer = _CurveScorer(
51 score_func=balanced_accuracy_score,
52 sign=1,
53 response_method="predict_proba",
54 thresholds=10,
55 kwargs={},
56 )
57 scores, thresholds = _fit_and_score_over_thresholds(
58 classifier,
59 X,
60 y,
61 fit_params={},
62 train_idx=train_idx,
63 val_idx=val_idx,
64 curve_scorer=curve_scorer,
65 score_params={},
66 )
67
68 assert np.all(thresholds[:-1] <= thresholds[1:])
69 assert isinstance(scores, np.ndarray)
70 assert np.logical_and(scores >= 0, scores <= 1).all()
71
72
73def test_fit_and_score_over_thresholds_prefit():
74 """Check the behaviour with a prefit classifier."""
75 X, y = make_classification(n_samples=100, random_state=0)
76
77 # `train_idx is None` to indicate that the classifier is prefit
78 train_idx, val_idx = None, np.arange(50, 100)
79 classifier = DecisionTreeClassifier(random_state=0).fit(X, y)
80 # make sure that the classifier memorized the full dataset such that
81 # we get perfect predictions and thus match the expected score
82 assert classifier.score(X[val_idx], y[val_idx]) == pytest.approx(1.0)
83
84 curve_scorer = _CurveScorer(
85 score_func=balanced_accuracy_score,
86 sign=1,
87 response_method="predict_proba",
88 thresholds=2,
89 kwargs={},
90 )
91 scores, thresholds = _fit_and_score_over_thresholds(
92 classifier,
93 X,
94 y,
95 fit_params={},
96 train_idx=train_idx,
97 val_idx=val_idx,
98 curve_scorer=curve_scorer,
99 score_params={},
100 )
101 assert np.all(thresholds[:-1] <= thresholds[1:])
102 assert_allclose(scores, [0.5, 1.0])
103
104
105@config_context(enable_metadata_routing=True)
106def test_fit_and_score_over_thresholds_sample_weight():
107 """Check that we dispatch the sample-weight to fit and score the classifier."""
108 X, y = load_iris(return_X_y=True)
109 X, y = X[:100], y[:100] # only 2 classes
110
111 # create a dataset and repeat twice the sample of class #0
112 X_repeated, y_repeated = np.vstack([X, X[y == 0]]), np.hstack([y, y[y == 0]])
113 # create a sample weight vector that is equivalent to the repeated dataset
114 sample_weight = np.ones_like(y)
115 sample_weight[:50] *= 2
116
117 classifier = LogisticRegression()
118 train_repeated_idx = np.arange(X_repeated.shape[0])
119 val_repeated_idx = np.arange(X_repeated.shape[0])
120 curve_scorer = _CurveScorer(
121 score_func=balanced_accuracy_score,
122 sign=1,
123 response_method="predict_proba",
124 thresholds=10,
125 kwargs={},
126 )
127 scores_repeated, thresholds_repeated = _fit_and_score_over_thresholds(
128 classifier,
129 X_repeated,
130 y_repeated,
131 fit_params={},
132 train_idx=train_repeated_idx,
133 val_idx=val_repeated_idx,
134 curve_scorer=curve_scorer,
135 score_params={},
136 )
137
138 train_idx, val_idx = np.arange(X.shape[0]), np.arange(X.shape[0])
139 scores, thresholds = _fit_and_score_over_thresholds(
140 classifier.set_fit_request(sample_weight=True),
141 X,
142 y,
143 fit_params={"sample_weight": sample_weight},
144 train_idx=train_idx,
145 val_idx=val_idx,
146 curve_scorer=curve_scorer.set_score_request(sample_weight=True),
147 score_params={"sample_weight": sample_weight},
148 )
149
150 assert_allclose(thresholds_repeated, thresholds)
151 assert_allclose(scores_repeated, scores)
152
153
154@pytest.mark.parametrize("fit_params_type", ["list", "array"])
155@config_context(enable_metadata_routing=True)
156def test_fit_and_score_over_thresholds_fit_params(fit_params_type):
157 """Check that we pass `fit_params` to the classifier when calling `fit`."""
158 X, y = make_classification(n_samples=100, random_state=0)
159 fit_params = {
160 "a": _convert_container(y, fit_params_type),
161 "b": _convert_container(y, fit_params_type),
162 }
163
164 classifier = CheckingClassifier(expected_fit_params=["a", "b"], random_state=0)
165 classifier.set_fit_request(a=True, b=True)
166 train_idx, val_idx = np.arange(50), np.arange(50, 100)
167
168 curve_scorer = _CurveScorer(
169 score_func=balanced_accuracy_score,
170 sign=1,
171 response_method="predict_proba",
172 thresholds=10,
173 kwargs={},
174 )
175 _fit_and_score_over_thresholds(
176 classifier,
177 X,
178 y,
179 fit_params=fit_params,
180 train_idx=train_idx,
181 val_idx=val_idx,
182 curve_scorer=curve_scorer,
183 score_params={},
184 )
185
186
187@pytest.mark.parametrize(
188 "data",
189 [
190 make_classification(n_classes=3, n_clusters_per_class=1, random_state=0),
191 make_multilabel_classification(random_state=0),
192 ],
193)
194def test_tuned_threshold_classifier_no_binary(data):
195 """Check that we raise an informative error message for non-binary problem."""
196 err_msg = "Only binary classification is supported."
197 with pytest.raises(ValueError, match=err_msg):
198 TunedThresholdClassifierCV(LogisticRegression()).fit(*data)
199
200
201@pytest.mark.parametrize(
202 "params, err_type, err_msg",
203 [
204 (
205 {"cv": "prefit", "refit": True},
206 ValueError,
207 "When cv='prefit', refit cannot be True.",
208 ),
209 (
210 {"cv": 10, "refit": False},
211 ValueError,
212 "When cv has several folds, refit cannot be False.",
213 ),
214 (
215 {"cv": "prefit", "refit": False},
216 NotFittedError,
217 "`estimator` must be fitted.",
218 ),
219 ],
220)
221def test_tuned_threshold_classifier_conflict_cv_refit(params, err_type, err_msg):
222 """Check that we raise an informative error message when `cv` and `refit`
223 cannot be used together.
224 """
225 X, y = make_classification(n_samples=100, random_state=0)
226 with pytest.raises(err_type, match=err_msg):
227 TunedThresholdClassifierCV(LogisticRegression(), **params).fit(X, y)
228
229
230@pytest.mark.parametrize(
231 "estimator",
232 [LogisticRegression(), SVC(), GradientBoostingClassifier(n_estimators=4)],
233)
234@pytest.mark.parametrize(
235 "response_method", ["predict_proba", "predict_log_proba", "decision_function"]
236)
237@pytest.mark.parametrize(
238 "ThresholdClassifier", [FixedThresholdClassifier, TunedThresholdClassifierCV]
239)
240def test_threshold_classifier_estimator_response_methods(
241 ThresholdClassifier, estimator, response_method
242):
243 """Check that `TunedThresholdClassifierCV` exposes the same response methods as the
244 underlying estimator.
245 """
246 X, y = make_classification(n_samples=100, random_state=0)
247
248 model = ThresholdClassifier(estimator=estimator)
249 assert hasattr(model, response_method) == hasattr(estimator, response_method)
250
251 model.fit(X, y)
252 assert hasattr(model, response_method) == hasattr(estimator, response_method)
253
254 if hasattr(model, response_method):
255 y_pred_cutoff = getattr(model, response_method)(X)
256 y_pred_underlying_estimator = getattr(model.estimator_, response_method)(X)
257
258 assert_allclose(y_pred_cutoff, y_pred_underlying_estimator)
259
260
261@pytest.mark.parametrize(
262 "response_method", ["auto", "decision_function", "predict_proba"]
263)
264def test_tuned_threshold_classifier_without_constraint_value(response_method):
265 """Check that `TunedThresholdClassifierCV` is optimizing a given objective
266 metric."""
267 X, y = load_breast_cancer(return_X_y=True)
268 # remove feature to degrade performances
269 X = X[:, :5]
270
271 # make the problem completely imbalanced such that the balanced accuracy is low
272 indices_pos = np.flatnonzero(y == 1)
273 indices_pos = indices_pos[: indices_pos.size // 50]
274 indices_neg = np.flatnonzero(y == 0)
275
276 X = np.vstack([X[indices_neg], X[indices_pos]])
277 y = np.hstack([y[indices_neg], y[indices_pos]])
278
279 lr = make_pipeline(StandardScaler(), LogisticRegression()).fit(X, y)
280 thresholds = 100
281 model = TunedThresholdClassifierCV(
282 estimator=lr,
283 scoring="balanced_accuracy",
284 response_method=response_method,
285 thresholds=thresholds,
286 store_cv_results=True,
287 )
288 score_optimized = balanced_accuracy_score(y, model.fit(X, y).predict(X))
289 score_baseline = balanced_accuracy_score(y, lr.predict(X))
290 assert score_optimized > score_baseline
291 assert model.cv_results_["thresholds"].shape == (thresholds,)
292 assert model.cv_results_["scores"].shape == (thresholds,)
293
294
295def test_tuned_threshold_classifier_metric_with_parameter():
296 """Check that we can pass a metric with a parameter in addition check that
297 `f_beta` with `beta=1` is equivalent to `f1` and different from `f_beta` with
298 `beta=2`.
299 """
300 X, y = load_breast_cancer(return_X_y=True)
301 lr = make_pipeline(StandardScaler(), LogisticRegression()).fit(X, y)
302 model_fbeta_1 = TunedThresholdClassifierCV(
303 estimator=lr, scoring=make_scorer(fbeta_score, beta=1)
304 ).fit(X, y)
305 model_fbeta_2 = TunedThresholdClassifierCV(
306 estimator=lr, scoring=make_scorer(fbeta_score, beta=2)
307 ).fit(X, y)
308 model_f1 = TunedThresholdClassifierCV(
309 estimator=lr, scoring=make_scorer(f1_score)
310 ).fit(X, y)
311
312 assert model_fbeta_1.best_threshold_ == pytest.approx(model_f1.best_threshold_)
313 assert model_fbeta_1.best_threshold_ != pytest.approx(model_fbeta_2.best_threshold_)
314
315
316@pytest.mark.parametrize(
317 "response_method", ["auto", "decision_function", "predict_proba"]
318)
319@pytest.mark.parametrize(
320 "metric",
321 [
322 make_scorer(balanced_accuracy_score),
323 make_scorer(f1_score, pos_label="cancer"),
324 ],
325)
326def test_tuned_threshold_classifier_with_string_targets(response_method, metric):
327 """Check that targets represented by str are properly managed.
328 Also, check with several metrics to be sure that `pos_label` is properly
329 dispatched.
330 """
331 X, y = load_breast_cancer(return_X_y=True)
332 # Encode numeric targets by meaningful strings. We purposely designed the class
333 # names such that the `pos_label` is the first alphabetically sorted class and thus
334 # encoded as 0.
335 classes = np.array(["cancer", "healthy"], dtype=object)
336 y = classes[y]
337 model = TunedThresholdClassifierCV(
338 estimator=make_pipeline(StandardScaler(), LogisticRegression()),
339 scoring=metric,
340 response_method=response_method,
341 thresholds=100,
342 ).fit(X, y)
343 assert_array_equal(model.classes_, np.sort(classes))
344 y_pred = model.predict(X)
345 assert_array_equal(np.unique(y_pred), np.sort(classes))
346
347
348@pytest.mark.parametrize("with_sample_weight", [True, False])
349@config_context(enable_metadata_routing=True)
350def test_tuned_threshold_classifier_refit(with_sample_weight, global_random_seed):
351 """Check the behaviour of the `refit` parameter."""
352 rng = np.random.RandomState(global_random_seed)
353 X, y = make_classification(n_samples=100, random_state=0)
354 if with_sample_weight:
355 sample_weight = rng.randn(X.shape[0])
356 sample_weight = np.abs(sample_weight, out=sample_weight)
357 else:
358 sample_weight = None
359
360 # check that `estimator_` if fitted on the full dataset when `refit=True`
361 estimator = LogisticRegression().set_fit_request(sample_weight=True)
362 model = TunedThresholdClassifierCV(estimator, refit=True).fit(
363 X, y, sample_weight=sample_weight
364 )
365
366 assert model.estimator_ is not estimator
367 estimator.fit(X, y, sample_weight=sample_weight)
368 assert_allclose(model.estimator_.coef_, estimator.coef_)
369 assert_allclose(model.estimator_.intercept_, estimator.intercept_)
370
371 # check that `estimator_` was not altered when `refit=False` and `cv="prefit"`
372 estimator = LogisticRegression().set_fit_request(sample_weight=True)
373 estimator.fit(X, y, sample_weight=sample_weight)
374 coef = estimator.coef_.copy()
375 model = TunedThresholdClassifierCV(estimator, cv="prefit", refit=False).fit(
376 X, y, sample_weight=sample_weight
377 )
378
379 assert model.estimator_ is estimator
380 assert_allclose(model.estimator_.coef_, coef)
381
382 # check that we train `estimator_` on the training split of a given cross-validation
383 estimator = LogisticRegression().set_fit_request(sample_weight=True)
384 cv = [
385 (np.arange(50), np.arange(50, 100)),
386 ] # single split
387 model = TunedThresholdClassifierCV(estimator, cv=cv, refit=False).fit(
388 X, y, sample_weight=sample_weight
389 )
390
391 assert model.estimator_ is not estimator
392 if with_sample_weight:
393 sw_train = sample_weight[cv[0][0]]
394 else:
395 sw_train = None
396 estimator.fit(X[cv[0][0]], y[cv[0][0]], sample_weight=sw_train)
397 assert_allclose(model.estimator_.coef_, estimator.coef_)
398
399
400@pytest.mark.parametrize("fit_params_type", ["list", "array"])
401@config_context(enable_metadata_routing=True)
402def test_tuned_threshold_classifier_fit_params(fit_params_type):
403 """Check that we pass `fit_params` to the classifier when calling `fit`."""
404 X, y = make_classification(n_samples=100, random_state=0)
405 fit_params = {
406 "a": _convert_container(y, fit_params_type),
407 "b": _convert_container(y, fit_params_type),
408 }
409
410 classifier = CheckingClassifier(expected_fit_params=["a", "b"], random_state=0)
411 classifier.set_fit_request(a=True, b=True)
412 model = TunedThresholdClassifierCV(classifier)
413 model.fit(X, y, **fit_params)
414
415
416@config_context(enable_metadata_routing=True)
417def test_tuned_threshold_classifier_cv_zeros_sample_weights_equivalence():
418 """Check that passing removing some sample from the dataset `X` is
419 equivalent to passing a `sample_weight` with a factor 0."""
420 X, y = load_iris(return_X_y=True)
421 # Scale the data to avoid any convergence issue
422 X = StandardScaler().fit_transform(X)
423 # Only use 2 classes and select samples such that 2-fold cross-validation
424 # split will lead to an equivalence with a `sample_weight` of 0
425 X = np.vstack((X[:40], X[50:90]))
426 y = np.hstack((y[:40], y[50:90]))
427 sample_weight = np.zeros_like(y)
428 sample_weight[::2] = 1
429
430 estimator = LogisticRegression().set_fit_request(sample_weight=True)
431 model_without_weights = TunedThresholdClassifierCV(estimator, cv=2)
432 model_with_weights = clone(model_without_weights)
433
434 model_with_weights.fit(X, y, sample_weight=sample_weight)
435 model_without_weights.fit(X[::2], y[::2])
436
437 assert_allclose(
438 model_with_weights.estimator_.coef_, model_without_weights.estimator_.coef_
439 )
440
441 y_pred_with_weights = model_with_weights.predict_proba(X)
442 y_pred_without_weights = model_without_weights.predict_proba(X)
443 assert_allclose(y_pred_with_weights, y_pred_without_weights)
444
445
446def test_tuned_threshold_classifier_thresholds_array():
447 """Check that we can pass an array to `thresholds` and it is used as candidate
448 threshold internally."""
449 X, y = make_classification(random_state=0)
450 estimator = LogisticRegression()
451 thresholds = np.linspace(0, 1, 11)
452 tuned_model = TunedThresholdClassifierCV(
453 estimator,
454 thresholds=thresholds,
455 response_method="predict_proba",
456 store_cv_results=True,
457 ).fit(X, y)
458 assert_allclose(tuned_model.cv_results_["thresholds"], thresholds)
459
460
461@pytest.mark.parametrize("store_cv_results", [True, False])
462def test_tuned_threshold_classifier_store_cv_results(store_cv_results):
463 """Check that if `cv_results_` exists depending on `store_cv_results`."""
464 X, y = make_classification(random_state=0)
465 estimator = LogisticRegression()
466 tuned_model = TunedThresholdClassifierCV(
467 estimator, store_cv_results=store_cv_results
468 ).fit(X, y)
469 if store_cv_results:
470 assert hasattr(tuned_model, "cv_results_")
471 else:
472 assert not hasattr(tuned_model, "cv_results_")
473
474
475def test_tuned_threshold_classifier_cv_float():
476 """Check the behaviour when `cv` is set to a float."""
477 X, y = make_classification(random_state=0)
478
479 # case where `refit=False` and cv is a float: the underlying estimator will be fit
480 # on the training set given by a ShuffleSplit. We check that we get the same model
481 # coefficients.
482 test_size = 0.3
483 estimator = LogisticRegression()
484 tuned_model = TunedThresholdClassifierCV(
485 estimator, cv=test_size, refit=False, random_state=0
486 ).fit(X, y)
487 tuned_model.fit(X, y)
488
489 cv = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=0)
490 train_idx, val_idx = next(cv.split(X, y))
491 cloned_estimator = clone(estimator).fit(X[train_idx], y[train_idx])
492
493 assert_allclose(tuned_model.estimator_.coef_, cloned_estimator.coef_)
494
495 # case where `refit=True`, then the underlying estimator is fitted on the full
496 # dataset.
497 tuned_model.set_params(refit=True).fit(X, y)
498 cloned_estimator = clone(estimator).fit(X, y)
499
500 assert_allclose(tuned_model.estimator_.coef_, cloned_estimator.coef_)
501
502
503def test_tuned_threshold_classifier_error_constant_predictor():
504 """Check that we raise a ValueError if the underlying classifier returns constant
505 probabilities such that we cannot find any threshold.
506 """
507 X, y = make_classification(random_state=0)
508 estimator = DummyClassifier(strategy="constant", constant=1)
509 tuned_model = TunedThresholdClassifierCV(estimator, response_method="predict_proba")
510 err_msg = "The provided estimator makes constant predictions"
511 with pytest.raises(ValueError, match=err_msg):
512 tuned_model.fit(X, y)
513
514
515@pytest.mark.parametrize(
516 "response_method", ["auto", "predict_proba", "decision_function"]
517)
518def test_fixed_threshold_classifier_equivalence_default(response_method):
519 """Check that `FixedThresholdClassifier` has the same behaviour as the vanilla
520 classifier.
521 """
522 X, y = make_classification(random_state=0)
523 classifier = LogisticRegression().fit(X, y)
524 classifier_default_threshold = FixedThresholdClassifier(
525 estimator=clone(classifier), response_method=response_method
526 )
527 classifier_default_threshold.fit(X, y)
528
529 # emulate the response method that should take into account the `pos_label`
530 if response_method in ("auto", "predict_proba"):
531 y_score = classifier_default_threshold.predict_proba(X)[:, 1]
532 threshold = 0.5
533 else: # response_method == "decision_function"
534 y_score = classifier_default_threshold.decision_function(X)
535 threshold = 0.0
536
537 y_pred_lr = (y_score >= threshold).astype(int)
538 assert_allclose(classifier_default_threshold.predict(X), y_pred_lr)
539
540
541@pytest.mark.parametrize(
542 "response_method, threshold", [("predict_proba", 0.7), ("decision_function", 2.0)]
543)
544@pytest.mark.parametrize("pos_label", [0, 1])
545def test_fixed_threshold_classifier(response_method, threshold, pos_label):
546 """Check that applying `predict` lead to the same prediction as applying the
547 threshold to the output of the response method.
548 """
549 X, y = make_classification(n_samples=50, random_state=0)
550 logistic_regression = LogisticRegression().fit(X, y)
551 model = FixedThresholdClassifier(
552 estimator=clone(logistic_regression),
553 threshold=threshold,
554 response_method=response_method,
555 pos_label=pos_label,
556 ).fit(X, y)
557
558 # check that the underlying estimator is the same
559 assert_allclose(model.estimator_.coef_, logistic_regression.coef_)
560
561 # emulate the response method that should take into account the `pos_label`
562 if response_method == "predict_proba":
563 y_score = model.predict_proba(X)[:, pos_label]
564 else: # response_method == "decision_function"
565 y_score = model.decision_function(X)
566 y_score = y_score if pos_label == 1 else -y_score
567
568 # create a mapping from boolean values to class labels
569 map_to_label = np.array([0, 1]) if pos_label == 1 else np.array([1, 0])
570 y_pred_lr = map_to_label[(y_score >= threshold).astype(int)]
571 assert_allclose(model.predict(X), y_pred_lr)
572
573 for method in ("predict_proba", "predict_log_proba", "decision_function"):
574 assert_allclose(
575 getattr(model, method)(X), getattr(logistic_regression, method)(X)
576 )
577 assert_allclose(
578 getattr(model.estimator_, method)(X),
579 getattr(logistic_regression, method)(X),
580 )
581
582
583@config_context(enable_metadata_routing=True)
584def test_fixed_threshold_classifier_metadata_routing():
585 """Check that everything works with metadata routing."""
586 X, y = make_classification(random_state=0)
587 sample_weight = np.ones_like(y)
588 sample_weight[::2] = 2
589 classifier = LogisticRegression().set_fit_request(sample_weight=True)
590 classifier.fit(X, y, sample_weight=sample_weight)
591 classifier_default_threshold = FixedThresholdClassifier(estimator=clone(classifier))
592 classifier_default_threshold.fit(X, y, sample_weight=sample_weight)
593 assert_allclose(classifier_default_threshold.estimator_.coef_, classifier.coef_)
594
595
596@pytest.mark.parametrize(
597 "method", ["predict_proba", "decision_function", "predict", "predict_log_proba"]
598)
599def test_fixed_threshold_classifier_fitted_estimator(method):
600 """Check that if the underlying estimator is already fitted, no fit is required."""
601 X, y = make_classification(random_state=0)
602 classifier = LogisticRegression().fit(X, y)
603 fixed_threshold_classifier = FixedThresholdClassifier(estimator=classifier)
604 # This should not raise an error
605 getattr(fixed_threshold_classifier, method)(X)
606
607
608def test_fixed_threshold_classifier_classes_():
609 """Check that the classes_ attribute is properly set."""
610 X, y = make_classification(random_state=0)
611 with pytest.raises(
612 AttributeError, match="The underlying estimator is not fitted yet."
613 ):
614 FixedThresholdClassifier(estimator=LogisticRegression()).classes_
615
616 classifier = LogisticRegression().fit(X, y)
617 fixed_threshold_classifier = FixedThresholdClassifier(estimator=classifier)
618 assert_array_equal(fixed_threshold_classifier.classes_, classifier.classes_)
619