Aluode/PerceptionLabPortable
0
1"""Multioutput regression and classification.
2
3The estimators provided in this module are meta-estimators: they require
4a base estimator to be provided in their constructor. The meta-estimator
5extends single output estimators to multioutput estimators.
6"""
7
8# Authors: The scikit-learn developers
9# SPDX-License-Identifier: BSD-3-Clause
10
11import warnings
12from abc import ABCMeta, abstractmethod
13from numbers import Integral
14
15import numpy as np
16import scipy.sparse as sp
17
18from .base import (
19 BaseEstimator,
20 ClassifierMixin,
21 MetaEstimatorMixin,
22 RegressorMixin,
23 _fit_context,
24 clone,
25 is_classifier,
26)
27from .model_selection import cross_val_predict
28from .utils import Bunch, check_random_state, get_tags
29from .utils._param_validation import (
30 HasMethods,
31 Hidden,
32 StrOptions,
33)
34from .utils._response import _get_response_values
35from .utils._user_interface import _print_elapsed_time
36from .utils.metadata_routing import (
37 MetadataRouter,
38 MethodMapping,
39 _raise_for_params,
40 _routing_enabled,
41 process_routing,
42)
43from .utils.metaestimators import available_if
44from .utils.multiclass import check_classification_targets
45from .utils.parallel import Parallel, delayed
46from .utils.validation import (
47 _check_method_params,
48 _check_response_method,
49 check_is_fitted,
50 has_fit_parameter,
51 validate_data,
52)
53
54__all__ = [
55 "ClassifierChain",
56 "MultiOutputClassifier",
57 "MultiOutputRegressor",
58 "RegressorChain",
59]
60
61
62def _fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
63 estimator = clone(estimator)
64 if sample_weight is not None:
65 estimator.fit(X, y, sample_weight=sample_weight, **fit_params)
66 else:
67 estimator.fit(X, y, **fit_params)
68 return estimator
69
70
71def _partial_fit_estimator(
72 estimator, X, y, classes=None, partial_fit_params=None, first_time=True
73):
74 partial_fit_params = {} if partial_fit_params is None else partial_fit_params
75 if first_time:
76 estimator = clone(estimator)
77
78 if classes is not None:
79 estimator.partial_fit(X, y, classes=classes, **partial_fit_params)
80 else:
81 estimator.partial_fit(X, y, **partial_fit_params)
82 return estimator
83
84
85def _available_if_estimator_has(attr):
86 """Return a function to check if the sub-estimator(s) has(have) `attr`.
87
88 Helper for Chain implementations.
89 """
90
91 def _check(self):
92 if hasattr(self, "estimators_"):
93 return all(hasattr(est, attr) for est in self.estimators_)
94
95 if hasattr(self.estimator, attr):
96 return True
97
98 return False
99
100 return available_if(_check)
101
102
103class _MultiOutputEstimator(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta):
104 _parameter_constraints: dict = {
105 "estimator": [HasMethods(["fit", "predict"])],
106 "n_jobs": [Integral, None],
107 }
108
109 @abstractmethod
110 def __init__(self, estimator, *, n_jobs=None):
111 self.estimator = estimator
112 self.n_jobs = n_jobs
113
114 @_available_if_estimator_has("partial_fit")
115 @_fit_context(
116 # MultiOutput*.estimator is not validated yet
117 prefer_skip_nested_validation=False
118 )
119 def partial_fit(self, X, y, classes=None, sample_weight=None, **partial_fit_params):
120 """Incrementally fit a separate model for each class output.
121
122 Parameters
123 ----------
124 X : {array-like, sparse matrix} of shape (n_samples, n_features)
125 The input data.
126
127 y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
128 Multi-output targets.
129
130 classes : list of ndarray of shape (n_outputs,), default=None
131 Each array is unique classes for one output in str/int.
132 Can be obtained via
133 ``[np.unique(y[:, i]) for i in range(y.shape[1])]``, where `y`
134 is the target matrix of the entire dataset.
135 This argument is required for the first call to partial_fit
136 and can be omitted in the subsequent calls.
137 Note that `y` doesn't need to contain all labels in `classes`.
138
139 sample_weight : array-like of shape (n_samples,), default=None
140 Sample weights. If `None`, then samples are equally weighted.
141 Only supported if the underlying regressor supports sample
142 weights.
143
144 **partial_fit_params : dict of str -> object
145 Parameters passed to the ``estimator.partial_fit`` method of each
146 sub-estimator.
147
148 Only available if `enable_metadata_routing=True`. See the
149 :ref:`User Guide <metadata_routing>`.
150
151 .. versionadded:: 1.3
152
153 Returns
154 -------
155 self : object
156 Returns a fitted instance.
157 """
158 _raise_for_params(partial_fit_params, self, "partial_fit")
159
160 first_time = not hasattr(self, "estimators_")
161
162 y = validate_data(self, X="no_validation", y=y, multi_output=True)
163
164 if y.ndim == 1:
165 raise ValueError(
166 "y must have at least two dimensions for "
167 "multi-output regression but has only one."
168 )
169
170 if _routing_enabled():
171 if sample_weight is not None:
172 partial_fit_params["sample_weight"] = sample_weight
173 routed_params = process_routing(
174 self,
175 "partial_fit",
176 **partial_fit_params,
177 )
178 else:
179 if sample_weight is not None and not has_fit_parameter(
180 self.estimator, "sample_weight"
181 ):
182 raise ValueError(
183 "Underlying estimator does not support sample weights."
184 )
185
186 if sample_weight is not None:
187 routed_params = Bunch(
188 estimator=Bunch(partial_fit=Bunch(sample_weight=sample_weight))
189 )
190 else:
191 routed_params = Bunch(estimator=Bunch(partial_fit=Bunch()))
192
193 self.estimators_ = Parallel(n_jobs=self.n_jobs)(
194 delayed(_partial_fit_estimator)(
195 self.estimators_[i] if not first_time else self.estimator,
196 X,
197 y[:, i],
198 classes[i] if classes is not None else None,
199 partial_fit_params=routed_params.estimator.partial_fit,
200 first_time=first_time,
201 )
202 for i in range(y.shape[1])
203 )
204
205 if first_time and hasattr(self.estimators_[0], "n_features_in_"):
206 self.n_features_in_ = self.estimators_[0].n_features_in_
207 if first_time and hasattr(self.estimators_[0], "feature_names_in_"):
208 self.feature_names_in_ = self.estimators_[0].feature_names_in_
209
210 return self
211
212 @_fit_context(
213 # MultiOutput*.estimator is not validated yet
214 prefer_skip_nested_validation=False
215 )
216 def fit(self, X, y, sample_weight=None, **fit_params):
217 """Fit the model to data, separately for each output variable.
218
219 Parameters
220 ----------
221 X : {array-like, sparse matrix} of shape (n_samples, n_features)
222 The input data.
223
224 y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
225 Multi-output targets. An indicator matrix turns on multilabel
226 estimation.
227
228 sample_weight : array-like of shape (n_samples,), default=None
229 Sample weights. If `None`, then samples are equally weighted.
230 Only supported if the underlying regressor supports sample
231 weights.
232
233 **fit_params : dict of string -> object
234 Parameters passed to the ``estimator.fit`` method of each step.
235
236 .. versionadded:: 0.23
237
238 Returns
239 -------
240 self : object
241 Returns a fitted instance.
242 """
243 if not hasattr(self.estimator, "fit"):
244 raise ValueError("The base estimator should implement a fit method")
245
246 y = validate_data(self, X="no_validation", y=y, multi_output=True)
247
248 if is_classifier(self):
249 check_classification_targets(y)
250
251 if y.ndim == 1:
252 raise ValueError(
253 "y must have at least two dimensions for "
254 "multi-output regression but has only one."
255 )
256
257 if _routing_enabled():
258 if sample_weight is not None:
259 fit_params["sample_weight"] = sample_weight
260 routed_params = process_routing(
261 self,
262 "fit",
263 **fit_params,
264 )
265 else:
266 if sample_weight is not None and not has_fit_parameter(
267 self.estimator, "sample_weight"
268 ):
269 raise ValueError(
270 "Underlying estimator does not support sample weights."
271 )
272
273 fit_params_validated = _check_method_params(X, params=fit_params)
274 routed_params = Bunch(estimator=Bunch(fit=fit_params_validated))
275 if sample_weight is not None:
276 routed_params.estimator.fit["sample_weight"] = sample_weight
277
278 self.estimators_ = Parallel(n_jobs=self.n_jobs)(
279 delayed(_fit_estimator)(
280 self.estimator, X, y[:, i], **routed_params.estimator.fit
281 )
282 for i in range(y.shape[1])
283 )
284
285 if hasattr(self.estimators_[0], "n_features_in_"):
286 self.n_features_in_ = self.estimators_[0].n_features_in_
287 if hasattr(self.estimators_[0], "feature_names_in_"):
288 self.feature_names_in_ = self.estimators_[0].feature_names_in_
289
290 return self
291
292 def predict(self, X):
293 """Predict multi-output variable using model for each target variable.
294
295 Parameters
296 ----------
297 X : {array-like, sparse matrix} of shape (n_samples, n_features)
298 The input data.
299
300 Returns
301 -------
302 y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
303 Multi-output targets predicted across multiple predictors.
304 Note: Separate models are generated for each predictor.
305 """
306 check_is_fitted(self)
307 if not hasattr(self.estimators_[0], "predict"):
308 raise ValueError("The base estimator should implement a predict method")
309
310 y = Parallel(n_jobs=self.n_jobs)(
311 delayed(e.predict)(X) for e in self.estimators_
312 )
313
314 return np.asarray(y).T
315
316 def __sklearn_tags__(self):
317 tags = super().__sklearn_tags__()
318 tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
319 tags.target_tags.single_output = False
320 tags.target_tags.multi_output = True
321 return tags
322
323 def get_metadata_routing(self):
324 """Get metadata routing of this object.
325
326 Please check :ref:`User Guide <metadata_routing>` on how the routing
327 mechanism works.
328
329 .. versionadded:: 1.3
330
331 Returns
332 -------
333 routing : MetadataRouter
334 A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
335 routing information.
336 """
337 router = MetadataRouter(owner=self.__class__.__name__).add(
338 estimator=self.estimator,
339 method_mapping=MethodMapping()
340 .add(caller="partial_fit", callee="partial_fit")
341 .add(caller="fit", callee="fit"),
342 )
343 return router
344
345
346class MultiOutputRegressor(RegressorMixin, _MultiOutputEstimator):
347 """Multi target regression.
348
349 This strategy consists of fitting one regressor per target. This is a
350 simple strategy for extending regressors that do not natively support
351 multi-target regression.
352
353 .. versionadded:: 0.18
354
355 Parameters
356 ----------
357 estimator : estimator object
358 An estimator object implementing :term:`fit` and :term:`predict`.
359
360 n_jobs : int or None, optional (default=None)
361 The number of jobs to run in parallel.
362 :meth:`fit`, :meth:`predict` and :meth:`partial_fit` (if supported
363 by the passed estimator) will be parallelized for each target.
364
365 When individual estimators are fast to train or predict,
366 using ``n_jobs > 1`` can result in slower performance due
367 to the parallelism overhead.
368
369 ``None`` means `1` unless in a :obj:`joblib.parallel_backend` context.
370 ``-1`` means using all available processes / threads.
371 See :term:`Glossary <n_jobs>` for more details.
372
373 .. versionchanged:: 0.20
374 `n_jobs` default changed from `1` to `None`.
375
376 Attributes
377 ----------
378 estimators_ : list of ``n_output`` estimators
379 Estimators used for predictions.
380
381 n_features_in_ : int
382 Number of features seen during :term:`fit`. Only defined if the
383 underlying `estimator` exposes such an attribute when fit.
384
385 .. versionadded:: 0.24
386
387 feature_names_in_ : ndarray of shape (`n_features_in_`,)
388 Names of features seen during :term:`fit`. Only defined if the
389 underlying estimators expose such an attribute when fit.
390
391 .. versionadded:: 1.0
392
393 See Also
394 --------
395 RegressorChain : A multi-label model that arranges regressions into a
396 chain.
397 MultiOutputClassifier : Classifies each output independently rather than
398 chaining.
399
400 Examples
401 --------
402 >>> import numpy as np
403 >>> from sklearn.datasets import load_linnerud
404 >>> from sklearn.multioutput import MultiOutputRegressor
405 >>> from sklearn.linear_model import Ridge
406 >>> X, y = load_linnerud(return_X_y=True)
407 >>> regr = MultiOutputRegressor(Ridge(random_state=123)).fit(X, y)
408 >>> regr.predict(X[[0]])
409 array([[176, 35.1, 57.1]])
410 """
411
412 def __init__(self, estimator, *, n_jobs=None):
413 super().__init__(estimator, n_jobs=n_jobs)
414
415 @_available_if_estimator_has("partial_fit")
416 def partial_fit(self, X, y, sample_weight=None, **partial_fit_params):
417 """Incrementally fit the model to data, for each output variable.
418
419 Parameters
420 ----------
421 X : {array-like, sparse matrix} of shape (n_samples, n_features)
422 The input data.
423
424 y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
425 Multi-output targets.
426
427 sample_weight : array-like of shape (n_samples,), default=None
428 Sample weights. If `None`, then samples are equally weighted.
429 Only supported if the underlying regressor supports sample
430 weights.
431
432 **partial_fit_params : dict of str -> object
433 Parameters passed to the ``estimator.partial_fit`` method of each
434 sub-estimator.
435
436 Only available if `enable_metadata_routing=True`. See the
437 :ref:`User Guide <metadata_routing>`.
438
439 .. versionadded:: 1.3
440
441 Returns
442 -------
443 self : object
444 Returns a fitted instance.
445 """
446 super().partial_fit(X, y, sample_weight=sample_weight, **partial_fit_params)
447
448
449class MultiOutputClassifier(ClassifierMixin, _MultiOutputEstimator):
450 """Multi target classification.
451
452 This strategy consists of fitting one classifier per target. This is a
453 simple strategy for extending classifiers that do not natively support
454 multi-target classification.
455
456 Parameters
457 ----------
458 estimator : estimator object
459 An estimator object implementing :term:`fit` and :term:`predict`.
460 A :term:`predict_proba` method will be exposed only if `estimator` implements
461 it.
462
463 n_jobs : int or None, optional (default=None)
464 The number of jobs to run in parallel.
465 :meth:`fit`, :meth:`predict` and :meth:`partial_fit` (if supported
466 by the passed estimator) will be parallelized for each target.
467
468 When individual estimators are fast to train or predict,
469 using ``n_jobs > 1`` can result in slower performance due
470 to the parallelism overhead.
471
472 ``None`` means `1` unless in a :obj:`joblib.parallel_backend` context.
473 ``-1`` means using all available processes / threads.
474 See :term:`Glossary <n_jobs>` for more details.
475
476 .. versionchanged:: 0.20
477 `n_jobs` default changed from `1` to `None`.
478
479 Attributes
480 ----------
481 classes_ : ndarray of shape (n_classes,)
482 Class labels.
483
484 estimators_ : list of ``n_output`` estimators
485 Estimators used for predictions.
486
487 n_features_in_ : int
488 Number of features seen during :term:`fit`. Only defined if the
489 underlying `estimator` exposes such an attribute when fit.
490
491 .. versionadded:: 0.24
492
493 feature_names_in_ : ndarray of shape (`n_features_in_`,)
494 Names of features seen during :term:`fit`. Only defined if the
495 underlying estimators expose such an attribute when fit.
496
497 .. versionadded:: 1.0
498
499 See Also
500 --------
501 ClassifierChain : A multi-label model that arranges binary classifiers
502 into a chain.
503 MultiOutputRegressor : Fits one regressor per target variable.
504
505 Examples
506 --------
507 >>> import numpy as np
508 >>> from sklearn.datasets import make_multilabel_classification
509 >>> from sklearn.multioutput import MultiOutputClassifier
510 >>> from sklearn.linear_model import LogisticRegression
511 >>> X, y = make_multilabel_classification(n_classes=3, random_state=0)
512 >>> clf = MultiOutputClassifier(LogisticRegression()).fit(X, y)
513 >>> clf.predict(X[-2:])
514 array([[1, 1, 1],
515 [1, 0, 1]])
516 """
517
518 def __init__(self, estimator, *, n_jobs=None):
519 super().__init__(estimator, n_jobs=n_jobs)
520
521 def fit(self, X, Y, sample_weight=None, **fit_params):
522 """Fit the model to data matrix X and targets Y.
523
524 Parameters
525 ----------
526 X : {array-like, sparse matrix} of shape (n_samples, n_features)
527 The input data.
528
529 Y : array-like of shape (n_samples, n_classes)
530 The target values.
531
532 sample_weight : array-like of shape (n_samples,), default=None
533 Sample weights. If `None`, then samples are equally weighted.
534 Only supported if the underlying classifier supports sample
535 weights.
536
537 **fit_params : dict of string -> object
538 Parameters passed to the ``estimator.fit`` method of each step.
539
540 .. versionadded:: 0.23
541
542 Returns
543 -------
544 self : object
545 Returns a fitted instance.
546 """
547 super().fit(X, Y, sample_weight=sample_weight, **fit_params)
548 self.classes_ = [estimator.classes_ for estimator in self.estimators_]
549 return self
550
551 def _check_predict_proba(self):
552 if hasattr(self, "estimators_"):
553 # raise an AttributeError if `predict_proba` does not exist for
554 # each estimator
555 [getattr(est, "predict_proba") for est in self.estimators_]
556 return True
557 # raise an AttributeError if `predict_proba` does not exist for the
558 # unfitted estimator
559 getattr(self.estimator, "predict_proba")
560 return True
561
562 @available_if(_check_predict_proba)
563 def predict_proba(self, X):
564 """Return prediction probabilities for each class of each output.
565
566 This method will raise a ``ValueError`` if any of the
567 estimators do not have ``predict_proba``.
568
569 Parameters
570 ----------
571 X : array-like of shape (n_samples, n_features)
572 The input data.
573
574 Returns
575 -------
576 p : array of shape (n_samples, n_classes), or a list of n_outputs \
577 such arrays if n_outputs > 1.
578 The class probabilities of the input samples. The order of the
579 classes corresponds to that in the attribute :term:`classes_`.
580
581 .. versionchanged:: 0.19
582 This function now returns a list of arrays where the length of
583 the list is ``n_outputs``, and each array is (``n_samples``,
584 ``n_classes``) for that particular output.
585 """
586 check_is_fitted(self)
587 results = [estimator.predict_proba(X) for estimator in self.estimators_]
588 return results
589
590 def score(self, X, y):
591 """Return the mean accuracy on the given test data and labels.
592
593 Parameters
594 ----------
595 X : array-like of shape (n_samples, n_features)
596 Test samples.
597
598 y : array-like of shape (n_samples, n_outputs)
599 True values for X.
600
601 Returns
602 -------
603 scores : float
604 Mean accuracy of predicted target versus true target.
605 """
606 check_is_fitted(self)
607 n_outputs_ = len(self.estimators_)
608 if y.ndim == 1:
609 raise ValueError(
610 "y must have at least two dimensions for "
611 "multi target classification but has only one"
612 )
613 if y.shape[1] != n_outputs_:
614 raise ValueError(
615 "The number of outputs of Y for fit {0} and"
616 " score {1} should be same".format(n_outputs_, y.shape[1])
617 )
618 y_pred = self.predict(X)
619 return np.mean(np.all(y == y_pred, axis=1))
620
621 def __sklearn_tags__(self):
622 tags = super().__sklearn_tags__()
623 # FIXME
624 tags._skip_test = True
625 return tags
626
627
628def _available_if_base_estimator_has(attr):
629 """Return a function to check if `base_estimator` or `estimators_` has `attr`.
630
631 Helper for Chain implementations.
632 """
633
634 def _check(self):
635 return hasattr(self._get_estimator(), attr) or all(
636 hasattr(est, attr) for est in self.estimators_
637 )
638
639 return available_if(_check)
640
641
642class _BaseChain(BaseEstimator, metaclass=ABCMeta):
643 _parameter_constraints: dict = {
644 "base_estimator": [
645 HasMethods(["fit", "predict"]),
646 StrOptions({"deprecated"}),
647 ],
648 "estimator": [
649 HasMethods(["fit", "predict"]),
650 Hidden(None),
651 ],
652 "order": ["array-like", StrOptions({"random"}), None],
653 "cv": ["cv_object", StrOptions({"prefit"})],
654 "random_state": ["random_state"],
655 "verbose": ["boolean"],
656 }
657
658 # TODO(1.9): Remove base_estimator
659 def __init__(
660 self,
661 estimator=None,
662 *,
663 order=None,
664 cv=None,
665 random_state=None,
666 verbose=False,
667 base_estimator="deprecated",
668 ):
669 self.estimator = estimator
670 self.base_estimator = base_estimator
671 self.order = order
672 self.cv = cv
673 self.random_state = random_state
674 self.verbose = verbose
675
676 # TODO(1.8): This is a temporary getter method to validate input wrt deprecation.
677 # It was only included to avoid relying on the presence of self.estimator_
678 def _get_estimator(self):
679 """Get and validate estimator."""
680
681 if self.estimator is not None and (self.base_estimator != "deprecated"):
682 raise ValueError(
683 "Both `estimator` and `base_estimator` are provided. You should only"
684 " pass `estimator`. `base_estimator` as a parameter is deprecated in"
685 " version 1.7, and will be removed in version 1.9."
686 )
687
688 if self.base_estimator != "deprecated":
689 warning_msg = (
690 "`base_estimator` as an argument was deprecated in 1.7 and will be"
691 " removed in 1.9. Use `estimator` instead."
692 )
693 warnings.warn(warning_msg, FutureWarning)
694 return self.base_estimator
695 else:
696 return self.estimator
697
698 def _log_message(self, *, estimator_idx, n_estimators, processing_msg):
699 if not self.verbose:
700 return None
701 return f"({estimator_idx} of {n_estimators}) {processing_msg}"
702
703 def _get_predictions(self, X, *, output_method):
704 """Get predictions for each model in the chain."""
705 check_is_fitted(self)
706 X = validate_data(self, X, accept_sparse=True, reset=False)
707 Y_output_chain = np.zeros((X.shape[0], len(self.estimators_)))
708 Y_feature_chain = np.zeros((X.shape[0], len(self.estimators_)))
709
710 # `RegressorChain` does not have a `chain_method_` parameter so we
711 # default to "predict"
712 chain_method = getattr(self, "chain_method_", "predict")
713 hstack = sp.hstack if sp.issparse(X) else np.hstack
714 for chain_idx, estimator in enumerate(self.estimators_):
715 previous_predictions = Y_feature_chain[:, :chain_idx]
716 # if `X` is a scipy sparse dok_array, we convert it to a sparse
717 # coo_array format before hstacking, it's faster; see
718 # https://github.com/scipy/scipy/issues/20060#issuecomment-1937007039:
719 if sp.issparse(X) and not sp.isspmatrix(X) and X.format == "dok":
720 X = sp.coo_array(X)
721 X_aug = hstack((X, previous_predictions))
722
723 feature_predictions, _ = _get_response_values(
724 estimator,
725 X_aug,
726 response_method=chain_method,
727 )
728 Y_feature_chain[:, chain_idx] = feature_predictions
729
730 output_predictions, _ = _get_response_values(
731 estimator,
732 X_aug,
733 response_method=output_method,
734 )
735 Y_output_chain[:, chain_idx] = output_predictions
736
737 inv_order = np.empty_like(self.order_)
738 inv_order[self.order_] = np.arange(len(self.order_))
739 Y_output = Y_output_chain[:, inv_order]
740
741 return Y_output
742
743 @abstractmethod
744 def fit(self, X, Y, **fit_params):
745 """Fit the model to data matrix X and targets Y.
746
747 Parameters
748 ----------
749 X : {array-like, sparse matrix} of shape (n_samples, n_features)
750 The input data.
751
752 Y : array-like of shape (n_samples, n_classes)
753 The target values.
754
755 **fit_params : dict of string -> object
756 Parameters passed to the `fit` method of each step.
757
758 .. versionadded:: 0.23
759
760 Returns
761 -------
762 self : object
763 Returns a fitted instance.
764 """
765 X, Y = validate_data(self, X, Y, multi_output=True, accept_sparse=True)
766
767 random_state = check_random_state(self.random_state)
768 self.order_ = self.order
769 if isinstance(self.order_, tuple):
770 self.order_ = np.array(self.order_)
771
772 if self.order_ is None:
773 self.order_ = np.array(range(Y.shape[1]))
774 elif isinstance(self.order_, str):
775 if self.order_ == "random":
776 self.order_ = random_state.permutation(Y.shape[1])
777 elif sorted(self.order_) != list(range(Y.shape[1])):
778 raise ValueError("invalid order")
779
780 self.estimators_ = [clone(self._get_estimator()) for _ in range(Y.shape[1])]
781
782 if self.cv is None:
783 Y_pred_chain = Y[:, self.order_]
784 if sp.issparse(X):
785 X_aug = sp.hstack((X, Y_pred_chain), format="lil")
786 X_aug = X_aug.tocsr()
787 else:
788 X_aug = np.hstack((X, Y_pred_chain))
789
790 elif sp.issparse(X):
791 # TODO: remove this condition check when the minimum supported scipy version
792 # doesn't support sparse matrices anymore
793 if not sp.isspmatrix(X):
794 # if `X` is a scipy sparse dok_array, we convert it to a sparse
795 # coo_array format before hstacking, it's faster; see
796 # https://github.com/scipy/scipy/issues/20060#issuecomment-1937007039:
797 if X.format == "dok":
798 X = sp.coo_array(X)
799 # in case that `X` is a sparse array we create `Y_pred_chain` as a
800 # sparse array format:
801 Y_pred_chain = sp.coo_array((X.shape[0], Y.shape[1]))
802 else:
803 Y_pred_chain = sp.coo_matrix((X.shape[0], Y.shape[1]))
804 X_aug = sp.hstack((X, Y_pred_chain), format="lil")
805
806 else:
807 Y_pred_chain = np.zeros((X.shape[0], Y.shape[1]))
808 X_aug = np.hstack((X, Y_pred_chain))
809
810 del Y_pred_chain
811
812 if _routing_enabled():
813 routed_params = process_routing(self, "fit", **fit_params)
814 else:
815 routed_params = Bunch(estimator=Bunch(fit=fit_params))
816
817 if hasattr(self, "chain_method"):
818 chain_method = _check_response_method(
819 self._get_estimator(),
820 self.chain_method,
821 ).__name__
822 self.chain_method_ = chain_method
823 else:
824 # `RegressorChain` does not have a `chain_method` parameter
825 chain_method = "predict"
826
827 for chain_idx, estimator in enumerate(self.estimators_):
828 message = self._log_message(
829 estimator_idx=chain_idx + 1,
830 n_estimators=len(self.estimators_),
831 processing_msg=f"Processing order {self.order_[chain_idx]}",
832 )
833 y = Y[:, self.order_[chain_idx]]
834 with _print_elapsed_time("Chain", message):
835 estimator.fit(
836 X_aug[:, : (X.shape[1] + chain_idx)],
837 y,
838 **routed_params.estimator.fit,
839 )
840
841 if self.cv is not None and chain_idx < len(self.estimators_) - 1:
842 col_idx = X.shape[1] + chain_idx
843 cv_result = cross_val_predict(
844 self._get_estimator(),
845 X_aug[:, :col_idx],
846 y=y,
847 cv=self.cv,
848 method=chain_method,
849 )
850 # `predict_proba` output is 2D, we use only output for classes[-1]
851 if cv_result.ndim > 1:
852 cv_result = cv_result[:, 1]
853 if sp.issparse(X_aug):
854 X_aug[:, col_idx] = np.expand_dims(cv_result, 1)
855 else:
856 X_aug[:, col_idx] = cv_result
857
858 return self
859
860 def predict(self, X):
861 """Predict on the data matrix X using the ClassifierChain model.
862
863 Parameters
864 ----------
865 X : {array-like, sparse matrix} of shape (n_samples, n_features)
866 The input data.
867
868 Returns
869 -------
870 Y_pred : array-like of shape (n_samples, n_classes)
871 The predicted values.
872 """
873 return self._get_predictions(X, output_method="predict")
874
875 def __sklearn_tags__(self):
876 tags = super().__sklearn_tags__()
877 tags.input_tags.sparse = get_tags(self._get_estimator()).input_tags.sparse
878 return tags
879
880
881class ClassifierChain(MetaEstimatorMixin, ClassifierMixin, _BaseChain):
882 """A multi-label model that arranges binary classifiers into a chain.
883
884 Each model makes a prediction in the order specified by the chain using
885 all of the available features provided to the model plus the predictions
886 of models that are earlier in the chain.
887
888 For an example of how to use ``ClassifierChain`` and benefit from its
889 ensemble, see
890 :ref:`ClassifierChain on a yeast dataset
891 <sphx_glr_auto_examples_multioutput_plot_classifier_chain_yeast.py>` example.
892
893 Read more in the :ref:`User Guide <classifierchain>`.
894
895 .. versionadded:: 0.19
896
897 Parameters
898 ----------
899 estimator : estimator
900 The base estimator from which the classifier chain is built.
901
902 order : array-like of shape (n_outputs,) or 'random', default=None
903 If `None`, the order will be determined by the order of columns in
904 the label matrix Y.::
905
906 order = [0, 1, 2, ..., Y.shape[1] - 1]
907
908 The order of the chain can be explicitly set by providing a list of
909 integers. For example, for a chain of length 5.::
910
911 order = [1, 3, 2, 4, 0]
912
913 means that the first model in the chain will make predictions for
914 column 1 in the Y matrix, the second model will make predictions
915 for column 3, etc.
916
917 If order is `random` a random ordering will be used.
918
919 cv : int, cross-validation generator or an iterable, default=None
920 Determines whether to use cross validated predictions or true
921 labels for the results of previous estimators in the chain.
922 Possible inputs for cv are:
923
924 - None, to use true labels when fitting,
925 - integer, to specify the number of folds in a (Stratified)KFold,
926 - :term:`CV splitter`,
927 - An iterable yielding (train, test) splits as arrays of indices.
928
929 chain_method : {'predict', 'predict_proba', 'predict_log_proba', \
930 'decision_function'} or list of such str's, default='predict'
931
932 Prediction method to be used by estimators in the chain for
933 the 'prediction' features of previous estimators in the chain.
934
935 - if `str`, name of the method;
936 - if a list of `str`, provides the method names in order of
937 preference. The method used corresponds to the first method in
938 the list that is implemented by `base_estimator`.
939
940 .. versionadded:: 1.5
941
942 random_state : int, RandomState instance or None, optional (default=None)
943 If ``order='random'``, determines random number generation for the
944 chain order.
945 In addition, it controls the random seed given at each `base_estimator`
946 at each chaining iteration. Thus, it is only used when `base_estimator`
947 exposes a `random_state`.
948 Pass an int for reproducible output across multiple function calls.
949 See :term:`Glossary <random_state>`.
950
951 verbose : bool, default=False
952 If True, chain progress is output as each model is completed.
953
954 .. versionadded:: 1.2
955
956 base_estimator : estimator, default="deprecated"
957 Use `estimator` instead.
958
959 .. deprecated:: 1.7
960 `base_estimator` is deprecated and will be removed in 1.9.
961 Use `estimator` instead.
962
963 Attributes
964 ----------
965 classes_ : list
966 A list of arrays of length ``len(estimators_)`` containing the
967 class labels for each estimator in the chain.
968
969 estimators_ : list
970 A list of clones of base_estimator.
971
972 order_ : list
973 The order of labels in the classifier chain.
974
975 chain_method_ : str
976 Prediction method used by estimators in the chain for the prediction
977 features.
978
979 n_features_in_ : int
980 Number of features seen during :term:`fit`. Only defined if the
981 underlying `base_estimator` exposes such an attribute when fit.
982
983 .. versionadded:: 0.24
984
985 feature_names_in_ : ndarray of shape (`n_features_in_`,)
986 Names of features seen during :term:`fit`. Defined only when `X`
987 has feature names that are all strings.
988
989 .. versionadded:: 1.0
990
991 See Also
992 --------
993 RegressorChain : Equivalent for regression.
994 MultiOutputClassifier : Classifies each output independently rather than
995 chaining.
996
997 References
998 ----------
999 Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier
1000 Chains for Multi-label Classification", 2009.
1001
1002 Examples
1003 --------
1004 >>> from sklearn.datasets import make_multilabel_classification
1005 >>> from sklearn.linear_model import LogisticRegression
1006 >>> from sklearn.model_selection import train_test_split
1007 >>> from sklearn.multioutput import ClassifierChain
1008 >>> X, Y = make_multilabel_classification(
1009 ... n_samples=12, n_classes=3, random_state=0
1010 ... )
1011 >>> X_train, X_test, Y_train, Y_test = train_test_split(
1012 ... X, Y, random_state=0
1013 ... )
1014 >>> base_lr = LogisticRegression(solver='lbfgs', random_state=0)
1015 >>> chain = ClassifierChain(base_lr, order='random', random_state=0)
1016 >>> chain.fit(X_train, Y_train).predict(X_test)
1017 array([[1., 1., 0.],
1018 [1., 0., 0.],
1019 [0., 1., 0.]])
1020 >>> chain.predict_proba(X_test)
1021 array([[0.8387, 0.9431, 0.4576],
1022 [0.8878, 0.3684, 0.2640],
1023 [0.0321, 0.9935, 0.0626]])
1024 """
1025
1026 _parameter_constraints: dict = {
1027 **_BaseChain._parameter_constraints,
1028 "chain_method": [
1029 list,
1030 tuple,
1031 StrOptions(
1032 {"predict", "predict_proba", "predict_log_proba", "decision_function"}
1033 ),
1034 ],
1035 }
1036
1037 # TODO(1.9): Remove base_estimator from __init__
1038 def __init__(
1039 self,
1040 estimator=None,
1041 *,
1042 order=None,
1043 cv=None,
1044 chain_method="predict",
1045 random_state=None,
1046 verbose=False,
1047 base_estimator="deprecated",
1048 ):
1049 super().__init__(
1050 estimator,
1051 order=order,
1052 cv=cv,
1053 random_state=random_state,
1054 verbose=verbose,
1055 base_estimator=base_estimator,
1056 )
1057 self.chain_method = chain_method
1058
1059 @_fit_context(
1060 # ClassifierChain.base_estimator is not validated yet
1061 prefer_skip_nested_validation=False
1062 )
1063 def fit(self, X, Y, **fit_params):
1064 """Fit the model to data matrix X and targets Y.
1065
1066 Parameters
1067 ----------
1068 X : {array-like, sparse matrix} of shape (n_samples, n_features)
1069 The input data.
1070
1071 Y : array-like of shape (n_samples, n_classes)
1072 The target values.
1073
1074 **fit_params : dict of string -> object
1075 Parameters passed to the `fit` method of each step.
1076
1077 Only available if `enable_metadata_routing=True`. See the
1078 :ref:`User Guide <metadata_routing>`.
1079
1080 .. versionadded:: 1.3
1081
1082 Returns
1083 -------
1084 self : object
1085 Class instance.
1086 """
1087 _raise_for_params(fit_params, self, "fit")
1088
1089 super().fit(X, Y, **fit_params)
1090 self.classes_ = [estimator.classes_ for estimator in self.estimators_]
1091 return self
1092
1093 @_available_if_base_estimator_has("predict_proba")
1094 def predict_proba(self, X):
1095 """Predict probability estimates.
1096
1097 Parameters
1098 ----------
1099 X : {array-like, sparse matrix} of shape (n_samples, n_features)
1100 The input data.
1101
1102 Returns
1103 -------
1104 Y_prob : array-like of shape (n_samples, n_classes)
1105 The predicted probabilities.
1106 """
1107 return self._get_predictions(X, output_method="predict_proba")
1108
1109 def predict_log_proba(self, X):
1110 """Predict logarithm of probability estimates.
1111
1112 Parameters
1113 ----------
1114 X : {array-like, sparse matrix} of shape (n_samples, n_features)
1115 The input data.
1116
1117 Returns
1118 -------
1119 Y_log_prob : array-like of shape (n_samples, n_classes)
1120 The predicted logarithm of the probabilities.
1121 """
1122 return np.log(self.predict_proba(X))
1123
1124 @_available_if_base_estimator_has("decision_function")
1125 def decision_function(self, X):
1126 """Evaluate the decision_function of the models in the chain.
1127
1128 Parameters
1129 ----------
1130 X : array-like of shape (n_samples, n_features)
1131 The input data.
1132
1133 Returns
1134 -------
1135 Y_decision : array-like of shape (n_samples, n_classes)
1136 Returns the decision function of the sample for each model
1137 in the chain.
1138 """
1139 return self._get_predictions(X, output_method="decision_function")
1140
1141 def get_metadata_routing(self):
1142 """Get metadata routing of this object.
1143
1144 Please check :ref:`User Guide <metadata_routing>` on how the routing
1145 mechanism works.
1146
1147 .. versionadded:: 1.3
1148
1149 Returns
1150 -------
1151 routing : MetadataRouter
1152 A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
1153 routing information.
1154 """
1155
1156 router = MetadataRouter(owner=self.__class__.__name__).add(
1157 estimator=self._get_estimator(),
1158 method_mapping=MethodMapping().add(caller="fit", callee="fit"),
1159 )
1160 return router
1161
1162 def __sklearn_tags__(self):
1163 tags = super().__sklearn_tags__()
1164 # FIXME
1165 tags._skip_test = True
1166 tags.target_tags.single_output = False
1167 tags.target_tags.multi_output = True
1168 return tags
1169
1170
1171class RegressorChain(MetaEstimatorMixin, RegressorMixin, _BaseChain):
1172 """A multi-label model that arranges regressions into a chain.
1173
1174 Each model makes a prediction in the order specified by the chain using
1175 all of the available features provided to the model plus the predictions
1176 of models that are earlier in the chain.
1177
1178 Read more in the :ref:`User Guide <regressorchain>`.
1179
1180 .. versionadded:: 0.20
1181
1182 Parameters
1183 ----------
1184 estimator : estimator
1185 The base estimator from which the regressor chain is built.
1186
1187 order : array-like of shape (n_outputs,) or 'random', default=None
1188 If `None`, the order will be determined by the order of columns in
1189 the label matrix Y.::
1190
1191 order = [0, 1, 2, ..., Y.shape[1] - 1]
1192
1193 The order of the chain can be explicitly set by providing a list of
1194 integers. For example, for a chain of length 5.::
1195
1196 order = [1, 3, 2, 4, 0]
1197
1198 means that the first model in the chain will make predictions for
1199 column 1 in the Y matrix, the second model will make predictions
1200 for column 3, etc.