Aluode/PerceptionLabPortable
0
1# Authors: The scikit-learn developers
2# SPDX-License-Identifier: BSD-3-Clause
3
4from collections.abc import MutableMapping
5from numbers import Integral, Real
6
7import numpy as np
8
9from ..base import (
10 BaseEstimator,
11 ClassifierMixin,
12 MetaEstimatorMixin,
13 _fit_context,
14 clone,
15)
16from ..exceptions import NotFittedError
17from ..metrics import (
18 check_scoring,
19 get_scorer_names,
20)
21from ..metrics._scorer import (
22 _CurveScorer,
23 _threshold_scores_to_class_labels,
24)
25from ..utils import _safe_indexing, get_tags
26from ..utils._param_validation import HasMethods, Interval, RealNotInt, StrOptions
27from ..utils._response import _get_response_values_binary
28from ..utils.metadata_routing import (
29 MetadataRouter,
30 MethodMapping,
31 _raise_for_params,
32 process_routing,
33)
34from ..utils.metaestimators import available_if
35from ..utils.multiclass import type_of_target
36from ..utils.parallel import Parallel, delayed
37from ..utils.validation import (
38 _check_method_params,
39 _estimator_has,
40 _num_samples,
41 check_is_fitted,
42 indexable,
43)
44from ._split import StratifiedShuffleSplit, check_cv
45
46
47def _check_is_fitted(estimator):
48 try:
49 check_is_fitted(estimator.estimator)
50 except NotFittedError:
51 check_is_fitted(estimator, "estimator_")
52
53
54class BaseThresholdClassifier(ClassifierMixin, MetaEstimatorMixin, BaseEstimator):
55 """Base class for binary classifiers that set a non-default decision threshold.
56
57 In this base class, we define the following interface:
58
59 - the validation of common parameters in `fit`;
60 - the different prediction methods that can be used with the classifier.
61
62 .. versionadded:: 1.5
63
64 Parameters
65 ----------
66 estimator : estimator instance
67 The binary classifier, fitted or not, for which we want to optimize
68 the decision threshold used during `predict`.
69
70 response_method : {"auto", "decision_function", "predict_proba"}, default="auto"
71 Methods by the classifier `estimator` corresponding to the
72 decision function for which we want to find a threshold. It can be:
73
74 * if `"auto"`, it will try to invoke, for each classifier,
75 `"predict_proba"` or `"decision_function"` in that order.
76 * otherwise, one of `"predict_proba"` or `"decision_function"`.
77 If the method is not implemented by the classifier, it will raise an
78 error.
79 """
80
81 _parameter_constraints: dict = {
82 "estimator": [
83 HasMethods(["fit", "predict_proba"]),
84 HasMethods(["fit", "decision_function"]),
85 ],
86 "response_method": [StrOptions({"auto", "predict_proba", "decision_function"})],
87 }
88
89 def __init__(self, estimator, *, response_method="auto"):
90 self.estimator = estimator
91 self.response_method = response_method
92
93 def _get_response_method(self):
94 """Define the response method."""
95 if self.response_method == "auto":
96 response_method = ["predict_proba", "decision_function"]
97 else:
98 response_method = self.response_method
99 return response_method
100
101 @_fit_context(
102 # *ThresholdClassifier*.estimator is not validated yet
103 prefer_skip_nested_validation=False
104 )
105 def fit(self, X, y, **params):
106 """Fit the classifier.
107
108 Parameters
109 ----------
110 X : {array-like, sparse matrix} of shape (n_samples, n_features)
111 Training data.
112
113 y : array-like of shape (n_samples,)
114 Target values.
115
116 **params : dict
117 Parameters to pass to the `fit` method of the underlying
118 classifier.
119
120 Returns
121 -------
122 self : object
123 Returns an instance of self.
124 """
125 _raise_for_params(params, self, None)
126
127 X, y = indexable(X, y)
128
129 y_type = type_of_target(y, input_name="y")
130 if y_type != "binary":
131 raise ValueError(
132 f"Only binary classification is supported. Unknown label type: {y_type}"
133 )
134
135 self._fit(X, y, **params)
136
137 if hasattr(self.estimator_, "n_features_in_"):
138 self.n_features_in_ = self.estimator_.n_features_in_
139 if hasattr(self.estimator_, "feature_names_in_"):
140 self.feature_names_in_ = self.estimator_.feature_names_in_
141
142 return self
143
144 @property
145 def classes_(self):
146 """Classes labels."""
147 return self.estimator_.classes_
148
149 @available_if(_estimator_has("predict_proba"))
150 def predict_proba(self, X):
151 """Predict class probabilities for `X` using the fitted estimator.
152
153 Parameters
154 ----------
155 X : {array-like, sparse matrix} of shape (n_samples, n_features)
156 Training vectors, where `n_samples` is the number of samples and
157 `n_features` is the number of features.
158
159 Returns
160 -------
161 probabilities : ndarray of shape (n_samples, n_classes)
162 The class probabilities of the input samples.
163 """
164 _check_is_fitted(self)
165 estimator = getattr(self, "estimator_", self.estimator)
166 return estimator.predict_proba(X)
167
168 @available_if(_estimator_has("predict_log_proba"))
169 def predict_log_proba(self, X):
170 """Predict logarithm class probabilities for `X` using the fitted estimator.
171
172 Parameters
173 ----------
174 X : {array-like, sparse matrix} of shape (n_samples, n_features)
175 Training vectors, where `n_samples` is the number of samples and
176 `n_features` is the number of features.
177
178 Returns
179 -------
180 log_probabilities : ndarray of shape (n_samples, n_classes)
181 The logarithm class probabilities of the input samples.
182 """
183 _check_is_fitted(self)
184 estimator = getattr(self, "estimator_", self.estimator)
185 return estimator.predict_log_proba(X)
186
187 @available_if(_estimator_has("decision_function"))
188 def decision_function(self, X):
189 """Decision function for samples in `X` using the fitted estimator.
190
191 Parameters
192 ----------
193 X : {array-like, sparse matrix} of shape (n_samples, n_features)
194 Training vectors, where `n_samples` is the number of samples and
195 `n_features` is the number of features.
196
197 Returns
198 -------
199 decisions : ndarray of shape (n_samples,)
200 The decision function computed the fitted estimator.
201 """
202 _check_is_fitted(self)
203 estimator = getattr(self, "estimator_", self.estimator)
204 return estimator.decision_function(X)
205
206 def __sklearn_tags__(self):
207 tags = super().__sklearn_tags__()
208 tags.classifier_tags.multi_class = False
209 tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
210 return tags
211
212
213class FixedThresholdClassifier(BaseThresholdClassifier):
214 """Binary classifier that manually sets the decision threshold.
215
216 This classifier allows to change the default decision threshold used for
217 converting posterior probability estimates (i.e. output of `predict_proba`) or
218 decision scores (i.e. output of `decision_function`) into a class label.
219
220 Here, the threshold is not optimized and is set to a constant value.
221
222 Read more in the :ref:`User Guide <FixedThresholdClassifier>`.
223
224 .. versionadded:: 1.5
225
226 Parameters
227 ----------
228 estimator : estimator instance
229 The binary classifier, fitted or not, for which we want to optimize
230 the decision threshold used during `predict`.
231
232 threshold : {"auto"} or float, default="auto"
233 The decision threshold to use when converting posterior probability estimates
234 (i.e. output of `predict_proba`) or decision scores (i.e. output of
235 `decision_function`) into a class label. When `"auto"`, the threshold is set
236 to 0.5 if `predict_proba` is used as `response_method`, otherwise it is set to
237 0 (i.e. the default threshold for `decision_function`).
238
239 pos_label : int, float, bool or str, default=None
240 The label of the positive class. Used to process the output of the
241 `response_method` method. When `pos_label=None`, if `y_true` is in `{-1, 1}` or
242 `{0, 1}`, `pos_label` is set to 1, otherwise an error will be raised.
243
244 response_method : {"auto", "decision_function", "predict_proba"}, default="auto"
245 Methods by the classifier `estimator` corresponding to the
246 decision function for which we want to find a threshold. It can be:
247
248 * if `"auto"`, it will try to invoke `"predict_proba"` or `"decision_function"`
249 in that order.
250 * otherwise, one of `"predict_proba"` or `"decision_function"`.
251 If the method is not implemented by the classifier, it will raise an
252 error.
253
254 Attributes
255 ----------
256 estimator_ : estimator instance
257 The fitted classifier used when predicting.
258
259 classes_ : ndarray of shape (n_classes,)
260 The class labels.
261
262 n_features_in_ : int
263 Number of features seen during :term:`fit`. Only defined if the
264 underlying estimator exposes such an attribute when fit.
265
266 feature_names_in_ : ndarray of shape (`n_features_in_`,)
267 Names of features seen during :term:`fit`. Only defined if the
268 underlying estimator exposes such an attribute when fit.
269
270 See Also
271 --------
272 sklearn.model_selection.TunedThresholdClassifierCV : Classifier that post-tunes
273 the decision threshold based on some metrics and using cross-validation.
274 sklearn.calibration.CalibratedClassifierCV : Estimator that calibrates
275 probabilities.
276
277 Examples
278 --------
279 >>> from sklearn.datasets import make_classification
280 >>> from sklearn.linear_model import LogisticRegression
281 >>> from sklearn.metrics import confusion_matrix
282 >>> from sklearn.model_selection import FixedThresholdClassifier, train_test_split
283 >>> X, y = make_classification(
284 ... n_samples=1_000, weights=[0.9, 0.1], class_sep=0.8, random_state=42
285 ... )
286 >>> X_train, X_test, y_train, y_test = train_test_split(
287 ... X, y, stratify=y, random_state=42
288 ... )
289 >>> classifier = LogisticRegression(random_state=0).fit(X_train, y_train)
290 >>> print(confusion_matrix(y_test, classifier.predict(X_test)))
291 [[217 7]
292 [ 19 7]]
293 >>> classifier_other_threshold = FixedThresholdClassifier(
294 ... classifier, threshold=0.1, response_method="predict_proba"
295 ... ).fit(X_train, y_train)
296 >>> print(confusion_matrix(y_test, classifier_other_threshold.predict(X_test)))
297 [[184 40]
298 [ 6 20]]
299 """
300
301 _parameter_constraints: dict = {
302 **BaseThresholdClassifier._parameter_constraints,
303 "threshold": [StrOptions({"auto"}), Real],
304 "pos_label": [Real, str, "boolean", None],
305 }
306
307 def __init__(
308 self,
309 estimator,
310 *,
311 threshold="auto",
312 pos_label=None,
313 response_method="auto",
314 ):
315 super().__init__(estimator=estimator, response_method=response_method)
316 self.pos_label = pos_label
317 self.threshold = threshold
318
319 @property
320 def classes_(self):
321 if estimator := getattr(self, "estimator_", None):
322 return estimator.classes_
323 try:
324 check_is_fitted(self.estimator)
325 return self.estimator.classes_
326 except NotFittedError:
327 raise AttributeError(
328 "The underlying estimator is not fitted yet."
329 ) from NotFittedError
330
331 def _fit(self, X, y, **params):
332 """Fit the classifier.
333
334 Parameters
335 ----------
336 X : {array-like, sparse matrix} of shape (n_samples, n_features)
337 Training data.
338
339 y : array-like of shape (n_samples,)
340 Target values.
341
342 **params : dict
343 Parameters to pass to the `fit` method of the underlying
344 classifier.
345
346 Returns
347 -------
348 self : object
349 Returns an instance of self.
350 """
351 routed_params = process_routing(self, "fit", **params)
352 self.estimator_ = clone(self.estimator).fit(X, y, **routed_params.estimator.fit)
353 return self
354
355 def predict(self, X):
356 """Predict the target of new samples.
357
358 Parameters
359 ----------
360 X : {array-like, sparse matrix} of shape (n_samples, n_features)
361 The samples, as accepted by `estimator.predict`.
362
363 Returns
364 -------
365 class_labels : ndarray of shape (n_samples,)
366 The predicted class.
367 """
368 _check_is_fitted(self)
369
370 estimator = getattr(self, "estimator_", self.estimator)
371
372 y_score, _, response_method_used = _get_response_values_binary(
373 estimator,
374 X,
375 self._get_response_method(),
376 pos_label=self.pos_label,
377 return_response_method_used=True,
378 )
379
380 if self.threshold == "auto":
381 decision_threshold = 0.5 if response_method_used == "predict_proba" else 0.0
382 else:
383 decision_threshold = self.threshold
384
385 return _threshold_scores_to_class_labels(
386 y_score, decision_threshold, self.classes_, self.pos_label
387 )
388
389 def get_metadata_routing(self):
390 """Get metadata routing of this object.
391
392 Please check :ref:`User Guide <metadata_routing>` on how the routing
393 mechanism works.
394
395 Returns
396 -------
397 routing : MetadataRouter
398 A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
399 routing information.
400 """
401 router = MetadataRouter(owner=self.__class__.__name__).add(
402 estimator=self.estimator,
403 method_mapping=MethodMapping().add(callee="fit", caller="fit"),
404 )
405 return router
406
407
408def _fit_and_score_over_thresholds(
409 classifier,
410 X,
411 y,
412 *,
413 fit_params,
414 train_idx,
415 val_idx,
416 curve_scorer,
417 score_params,
418):
419 """Fit a classifier and compute the scores for different decision thresholds.
420
421 Parameters
422 ----------
423 classifier : estimator instance
424 The classifier to fit and use for scoring. If `classifier` is already fitted,
425 it will be used as is.
426
427 X : {array-like, sparse matrix} of shape (n_samples, n_features)
428 The entire dataset.
429
430 y : array-like of shape (n_samples,)
431 The entire target vector.
432
433 fit_params : dict
434 Parameters to pass to the `fit` method of the underlying classifier.
435
436 train_idx : ndarray of shape (n_train_samples,) or None
437 The indices of the training set. If `None`, `classifier` is expected to be
438 already fitted.
439
440 val_idx : ndarray of shape (n_val_samples,)
441 The indices of the validation set used to score `classifier`. If `train_idx`,
442 the entire set will be used.
443
444 curve_scorer : scorer instance
445 The scorer taking `classifier` and the validation set as input and outputting
446 decision thresholds and scores as a curve. Note that this is different from
447 the usual scorer that outputs a single score value as `curve_scorer`
448 outputs a single score value for each threshold.
449
450 score_params : dict
451 Parameters to pass to the `score` method of the underlying scorer.
452
453 Returns
454 -------
455 scores : ndarray of shape (thresholds,) or tuple of such arrays
456 The scores computed for each decision threshold. When TPR/TNR or precision/
457 recall are computed, `scores` is a tuple of two arrays.
458
459 potential_thresholds : ndarray of shape (thresholds,)
460 The decision thresholds used to compute the scores. They are returned in
461 ascending order.
462 """
463
464 if train_idx is not None:
465 X_train, X_val = _safe_indexing(X, train_idx), _safe_indexing(X, val_idx)
466 y_train, y_val = _safe_indexing(y, train_idx), _safe_indexing(y, val_idx)
467 fit_params_train = _check_method_params(X, fit_params, indices=train_idx)
468 score_params_val = _check_method_params(X, score_params, indices=val_idx)
469 classifier.fit(X_train, y_train, **fit_params_train)
470 else: # prefit estimator, only a validation set is provided
471 X_val, y_val, score_params_val = X, y, score_params
472
473 return curve_scorer(classifier, X_val, y_val, **score_params_val)
474
475
476def _mean_interpolated_score(target_thresholds, cv_thresholds, cv_scores):
477 """Compute the mean interpolated score across folds by defining common thresholds.
478
479 Parameters
480 ----------
481 target_thresholds : ndarray of shape (thresholds,)
482 The thresholds to use to compute the mean score.
483
484 cv_thresholds : ndarray of shape (n_folds, thresholds_fold)
485 The thresholds used to compute the scores for each fold.
486
487 cv_scores : ndarray of shape (n_folds, thresholds_fold)
488 The scores computed for each threshold for each fold.
489
490 Returns
491 -------
492 mean_score : ndarray of shape (thresholds,)
493 The mean score across all folds for each target threshold.
494 """
495 return np.mean(
496 [
497 np.interp(target_thresholds, split_thresholds, split_score)
498 for split_thresholds, split_score in zip(cv_thresholds, cv_scores)
499 ],
500 axis=0,
501 )
502
503
504class TunedThresholdClassifierCV(BaseThresholdClassifier):
505 """Classifier that post-tunes the decision threshold using cross-validation.
506
507 This estimator post-tunes the decision threshold (cut-off point) that is
508 used for converting posterior probability estimates (i.e. output of
509 `predict_proba`) or decision scores (i.e. output of `decision_function`)
510 into a class label. The tuning is done by optimizing a binary metric,
511 potentially constrained by a another metric.
512
513 Read more in the :ref:`User Guide <TunedThresholdClassifierCV>`.
514
515 .. versionadded:: 1.5
516
517 Parameters
518 ----------
519 estimator : estimator instance
520 The classifier, fitted or not, for which we want to optimize
521 the decision threshold used during `predict`.
522
523 scoring : str or callable, default="balanced_accuracy"
524 The objective metric to be optimized. Can be one of:
525
526 - str: string associated to a scoring function for binary classification,
527 see :ref:`scoring_string_names` for options.
528 - callable: a scorer callable object (e.g., function) with signature
529 ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
530
531 response_method : {"auto", "decision_function", "predict_proba"}, default="auto"
532 Methods by the classifier `estimator` corresponding to the
533 decision function for which we want to find a threshold. It can be:
534
535 * if `"auto"`, it will try to invoke, for each classifier,
536 `"predict_proba"` or `"decision_function"` in that order.
537 * otherwise, one of `"predict_proba"` or `"decision_function"`.
538 If the method is not implemented by the classifier, it will raise an
539 error.
540
541 thresholds : int or array-like, default=100
542 The number of decision threshold to use when discretizing the output of the
543 classifier `method`. Pass an array-like to manually specify the thresholds
544 to use.
545
546 cv : int, float, cross-validation generator, iterable or "prefit", default=None
547 Determines the cross-validation splitting strategy to train classifier.
548 Possible inputs for cv are:
549
550 * `None`, to use the default 5-fold stratified K-fold cross validation;
551 * An integer number, to specify the number of folds in a stratified k-fold;
552 * A float number, to specify a single shuffle split. The floating number should
553 be in (0, 1) and represent the size of the validation set;
554 * An object to be used as a cross-validation generator;
555 * An iterable yielding train, test splits;
556 * `"prefit"`, to bypass the cross-validation.
557
558 Refer :ref:`User Guide <cross_validation>` for the various
559 cross-validation strategies that can be used here.
560
561 .. warning::
562 Using `cv="prefit"` and passing the same dataset for fitting `estimator`
563 and tuning the cut-off point is subject to undesired overfitting. You can
564 refer to :ref:`TunedThresholdClassifierCV_no_cv` for an example.
565
566 This option should only be used when the set used to fit `estimator` is
567 different from the one used to tune the cut-off point (by calling
568 :meth:`TunedThresholdClassifierCV.fit`).
569
570 refit : bool, default=True
571 Whether or not to refit the classifier on the entire training set once
572 the decision threshold has been found.
573 Note that forcing `refit=False` on cross-validation having more
574 than a single split will raise an error. Similarly, `refit=True` in
575 conjunction with `cv="prefit"` will raise an error.
576
577 n_jobs : int, default=None
578 The number of jobs to run in parallel. When `cv` represents a
579 cross-validation strategy, the fitting and scoring on each data split
580 is done in parallel. ``None`` means 1 unless in a
581 :obj:`joblib.parallel_backend` context. ``-1`` means using all
582 processors. See :term:`Glossary <n_jobs>` for more details.
583
584 random_state : int, RandomState instance or None, default=None
585 Controls the randomness of cross-validation when `cv` is a float.
586 See :term:`Glossary <random_state>`.
587
588 store_cv_results : bool, default=False
589 Whether to store all scores and thresholds computed during the cross-validation
590 process.
591
592 Attributes
593 ----------
594 estimator_ : estimator instance
595 The fitted classifier used when predicting.
596
597 best_threshold_ : float
598 The new decision threshold.
599
600 best_score_ : float or None
601 The optimal score of the objective metric, evaluated at `best_threshold_`.
602
603 cv_results_ : dict or None
604 A dictionary containing the scores and thresholds computed during the
605 cross-validation process. Only exist if `store_cv_results=True`. The
606 keys are `"thresholds"` and `"scores"`.
607
608 classes_ : ndarray of shape (n_classes,)
609 The class labels.
610
611 n_features_in_ : int
612 Number of features seen during :term:`fit`. Only defined if the
613 underlying estimator exposes such an attribute when fit.
614
615 feature_names_in_ : ndarray of shape (`n_features_in_`,)
616 Names of features seen during :term:`fit`. Only defined if the
617 underlying estimator exposes such an attribute when fit.
618
619 See Also
620 --------
621 sklearn.model_selection.FixedThresholdClassifier : Classifier that uses a
622 constant threshold.
623 sklearn.calibration.CalibratedClassifierCV : Estimator that calibrates
624 probabilities.
625
626 Examples
627 --------
628 >>> from sklearn.datasets import make_classification
629 >>> from sklearn.ensemble import RandomForestClassifier
630 >>> from sklearn.metrics import classification_report
631 >>> from sklearn.model_selection import TunedThresholdClassifierCV, train_test_split
632 >>> X, y = make_classification(
633 ... n_samples=1_000, weights=[0.9, 0.1], class_sep=0.8, random_state=42
634 ... )
635 >>> X_train, X_test, y_train, y_test = train_test_split(
636 ... X, y, stratify=y, random_state=42
637 ... )
638 >>> classifier = RandomForestClassifier(random_state=0).fit(X_train, y_train)
639 >>> print(classification_report(y_test, classifier.predict(X_test)))
640 precision recall f1-score support
641 <BLANKLINE>
642 0 0.94 0.99 0.96 224
643 1 0.80 0.46 0.59 26
644 <BLANKLINE>
645 accuracy 0.93 250
646 macro avg 0.87 0.72 0.77 250
647 weighted avg 0.93 0.93 0.92 250
648 <BLANKLINE>
649 >>> classifier_tuned = TunedThresholdClassifierCV(
650 ... classifier, scoring="balanced_accuracy"
651 ... ).fit(X_train, y_train)
652 >>> print(
653 ... f"Cut-off point found at {classifier_tuned.best_threshold_:.3f}"
654 ... )
655 Cut-off point found at 0.342
656 >>> print(classification_report(y_test, classifier_tuned.predict(X_test)))
657 precision recall f1-score support
658 <BLANKLINE>
659 0 0.96 0.95 0.96 224
660 1 0.61 0.65 0.63 26
661 <BLANKLINE>
662 accuracy 0.92 250
663 macro avg 0.78 0.80 0.79 250
664 weighted avg 0.92 0.92 0.92 250
665 <BLANKLINE>
666 """
667
668 _parameter_constraints: dict = {
669 **BaseThresholdClassifier._parameter_constraints,
670 "scoring": [
671 StrOptions(set(get_scorer_names())),
672 callable,
673 MutableMapping,
674 ],
675 "thresholds": [Interval(Integral, 1, None, closed="left"), "array-like"],
676 "cv": [
677 "cv_object",
678 StrOptions({"prefit"}),
679 Interval(RealNotInt, 0.0, 1.0, closed="neither"),
680 ],
681 "refit": ["boolean"],
682 "n_jobs": [Integral, None],
683 "random_state": ["random_state"],
684 "store_cv_results": ["boolean"],
685 }
686
687 def __init__(
688 self,
689 estimator,
690 *,
691 scoring="balanced_accuracy",
692 response_method="auto",
693 thresholds=100,
694 cv=None,
695 refit=True,
696 n_jobs=None,
697 random_state=None,
698 store_cv_results=False,
699 ):
700 super().__init__(estimator=estimator, response_method=response_method)
701 self.scoring = scoring
702 self.thresholds = thresholds
703 self.cv = cv
704 self.refit = refit
705 self.n_jobs = n_jobs
706 self.random_state = random_state
707 self.store_cv_results = store_cv_results
708
709 def _fit(self, X, y, **params):
710 """Fit the classifier and post-tune the decision threshold.
711
712 Parameters
713 ----------
714 X : {array-like, sparse matrix} of shape (n_samples, n_features)
715 Training data.
716
717 y : array-like of shape (n_samples,)
718 Target values.
719
720 **params : dict
721 Parameters to pass to the `fit` method of the underlying
722 classifier and to the `scoring` scorer.
723
724 Returns
725 -------
726 self : object
727 Returns an instance of self.
728 """
729 if isinstance(self.cv, Real) and 0 < self.cv < 1:
730 cv = StratifiedShuffleSplit(
731 n_splits=1, test_size=self.cv, random_state=self.random_state
732 )
733 elif self.cv == "prefit":
734 if self.refit is True:
735 raise ValueError("When cv='prefit', refit cannot be True.")
736 try:
737 check_is_fitted(self.estimator, "classes_")
738 except NotFittedError as exc:
739 raise NotFittedError(
740 """When cv='prefit', `estimator` must be fitted."""
741 ) from exc
742 cv = self.cv
743 else:
744 cv = check_cv(self.cv, y=y, classifier=True)
745 if self.refit is False and cv.get_n_splits() > 1:
746 raise ValueError("When cv has several folds, refit cannot be False.")
747
748 routed_params = process_routing(self, "fit", **params)
749 self._curve_scorer = self._get_curve_scorer()
750
751 # in the following block, we:
752 # - define the final classifier `self.estimator_` and train it if necessary
753 # - define `classifier` to be used to post-tune the decision threshold
754 # - define `split` to be used to fit/score `classifier`
755 if cv == "prefit":
756 self.estimator_ = self.estimator
757 classifier = self.estimator_
758 splits = [(None, range(_num_samples(X)))]
759 else:
760 self.estimator_ = clone(self.estimator)
761 classifier = clone(self.estimator)
762 splits = cv.split(X, y, **routed_params.splitter.split)
763
764 if self.refit:
765 # train on the whole dataset
766 X_train, y_train, fit_params_train = X, y, routed_params.estimator.fit
767 else:
768 # single split cross-validation
769 train_idx, _ = next(cv.split(X, y, **routed_params.splitter.split))
770 X_train = _safe_indexing(X, train_idx)
771 y_train = _safe_indexing(y, train_idx)
772 fit_params_train = _check_method_params(
773 X, routed_params.estimator.fit, indices=train_idx
774 )
775
776 self.estimator_.fit(X_train, y_train, **fit_params_train)
777
778 cv_scores, cv_thresholds = zip(
779 *Parallel(n_jobs=self.n_jobs)(
780 delayed(_fit_and_score_over_thresholds)(
781 clone(classifier) if cv != "prefit" else classifier,
782 X,
783 y,
784 fit_params=routed_params.estimator.fit,
785 train_idx=train_idx,
786 val_idx=val_idx,
787 curve_scorer=self._curve_scorer,
788 score_params=routed_params.scorer.score,
789 )
790 for train_idx, val_idx in splits
791 )
792 )
793
794 if any(np.isclose(th[0], th[-1]) for th in cv_thresholds):
795 raise ValueError(
796 "The provided estimator makes constant predictions. Therefore, it is "
797 "impossible to optimize the decision threshold."
798 )
799
800 # find the global min and max thresholds across all folds
801 min_threshold = min(
802 split_thresholds.min() for split_thresholds in cv_thresholds
803 )
804 max_threshold = max(
805 split_thresholds.max() for split_thresholds in cv_thresholds
806 )
807 if isinstance(self.thresholds, Integral):
808 decision_thresholds = np.linspace(
809 min_threshold, max_threshold, num=self.thresholds
810 )
811 else:
812 decision_thresholds = np.asarray(self.thresholds)
813
814 objective_scores = _mean_interpolated_score(
815 decision_thresholds, cv_thresholds, cv_scores
816 )
817 best_idx = objective_scores.argmax()
818 self.best_score_ = objective_scores[best_idx]
819 self.best_threshold_ = decision_thresholds[best_idx]
820 if self.store_cv_results:
821 self.cv_results_ = {
822 "thresholds": decision_thresholds,
823 "scores": objective_scores,
824 }
825
826 return self
827
828 def predict(self, X):
829 """Predict the target of new samples.
830
831 Parameters
832 ----------
833 X : {array-like, sparse matrix} of shape (n_samples, n_features)
834 The samples, as accepted by `estimator.predict`.
835
836 Returns
837 -------
838 class_labels : ndarray of shape (n_samples,)
839 The predicted class.
840 """
841 check_is_fitted(self, "estimator_")
842 pos_label = self._curve_scorer._get_pos_label()
843 y_score, _ = _get_response_values_binary(
844 self.estimator_,
845 X,
846 self._get_response_method(),
847 pos_label=pos_label,
848 )
849
850 return _threshold_scores_to_class_labels(
851 y_score, self.best_threshold_, self.classes_, pos_label
852 )
853
854 def get_metadata_routing(self):
855 """Get metadata routing of this object.
856
857 Please check :ref:`User Guide <metadata_routing>` on how the routing
858 mechanism works.
859
860 Returns
861 -------
862 routing : MetadataRouter
863 A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
864 routing information.
865 """
866 router = (
867 MetadataRouter(owner=self.__class__.__name__)
868 .add(
869 estimator=self.estimator,
870 method_mapping=MethodMapping().add(callee="fit", caller="fit"),
871 )
872 .add(
873 splitter=self.cv,
874 method_mapping=MethodMapping().add(callee="split", caller="fit"),
875 )
876 .add(
877 scorer=self._get_curve_scorer(),
878 method_mapping=MethodMapping().add(callee="score", caller="fit"),
879 )
880 )
881 return router
882
883 def _get_curve_scorer(self):
884 """Get the curve scorer based on the objective metric used."""
885 scoring = check_scoring(self.estimator, scoring=self.scoring)
886 curve_scorer = _CurveScorer.from_scorer(
887 scoring, self._get_response_method(), self.thresholds
888 )
889 return curve_scorer
890