Aluode/PerceptionLabPortable
0
1import copy
2import re
3
4import numpy as np
5import pytest
6
7from sklearn import config_context
8from sklearn.base import BaseEstimator, is_classifier
9from sklearn.calibration import CalibratedClassifierCV
10from sklearn.compose import TransformedTargetRegressor
11from sklearn.covariance import GraphicalLassoCV
12from sklearn.ensemble import (
13 AdaBoostClassifier,
14 AdaBoostRegressor,
15 BaggingClassifier,
16 BaggingRegressor,
17)
18from sklearn.exceptions import UnsetMetadataPassedError
19from sklearn.experimental import (
20 enable_halving_search_cv, # noqa: F401
21 enable_iterative_imputer, # noqa: F401
22)
23from sklearn.feature_selection import (
24 RFE,
25 RFECV,
26 SelectFromModel,
27 SequentialFeatureSelector,
28)
29from sklearn.impute import IterativeImputer
30from sklearn.linear_model import (
31 ElasticNetCV,
32 LarsCV,
33 LassoCV,
34 LassoLarsCV,
35 LogisticRegressionCV,
36 MultiTaskElasticNetCV,
37 MultiTaskLassoCV,
38 OrthogonalMatchingPursuitCV,
39 RANSACRegressor,
40 RidgeClassifierCV,
41 RidgeCV,
42)
43from sklearn.metrics._regression import mean_squared_error
44from sklearn.metrics._scorer import make_scorer
45from sklearn.model_selection import (
46 FixedThresholdClassifier,
47 GridSearchCV,
48 GroupKFold,
49 HalvingGridSearchCV,
50 HalvingRandomSearchCV,
51 RandomizedSearchCV,
52 TunedThresholdClassifierCV,
53 cross_validate,
54)
55from sklearn.multiclass import (
56 OneVsOneClassifier,
57 OneVsRestClassifier,
58 OutputCodeClassifier,
59)
60from sklearn.multioutput import (
61 ClassifierChain,
62 MultiOutputClassifier,
63 MultiOutputRegressor,
64 RegressorChain,
65)
66from sklearn.semi_supervised import SelfTrainingClassifier
67from sklearn.tests.metadata_routing_common import (
68 ConsumingClassifier,
69 ConsumingRegressor,
70 ConsumingScorer,
71 ConsumingSplitter,
72 NonConsumingClassifier,
73 NonConsumingRegressor,
74 _Registry,
75 assert_request_is_empty,
76 check_recorded_metadata,
77)
78from sklearn.utils.metadata_routing import MetadataRouter
79
80rng = np.random.RandomState(42)
81N, M = 100, 4
82X = rng.rand(N, M)
83y = rng.randint(0, 3, size=N)
84y_binary = (y >= 1).astype(int)
85classes = np.unique(y)
86y_multi = rng.randint(0, 3, size=(N, 3))
87classes_multi = [np.unique(y_multi[:, i]) for i in range(y_multi.shape[1])]
88metadata = rng.randint(0, 10, size=N)
89sample_weight = rng.rand(N)
90groups = rng.randint(0, 10, size=len(y))
91
92
93METAESTIMATORS: list = [
94 {
95 "metaestimator": MultiOutputRegressor,
96 "estimator_name": "estimator",
97 "estimator": "regressor",
98 "X": X,
99 "y": y_multi,
100 "estimator_routing_methods": ["fit", "partial_fit"],
101 },
102 {
103 "metaestimator": MultiOutputClassifier,
104 "estimator_name": "estimator",
105 "estimator": "classifier",
106 "X": X,
107 "y": y_multi,
108 "estimator_routing_methods": ["fit", "partial_fit"],
109 "method_args": {"partial_fit": {"classes": classes_multi}},
110 },
111 {
112 "metaestimator": CalibratedClassifierCV,
113 "estimator_name": "estimator",
114 "estimator": "classifier",
115 "X": X,
116 "y": y,
117 "estimator_routing_methods": ["fit"],
118 "preserves_metadata": "subset",
119 },
120 {
121 "metaestimator": ClassifierChain,
122 "estimator_name": "estimator",
123 "estimator": "classifier",
124 "X": X,
125 "y": y_multi,
126 "estimator_routing_methods": ["fit"],
127 },
128 {
129 "metaestimator": RegressorChain,
130 "estimator_name": "estimator",
131 "estimator": "regressor",
132 "X": X,
133 "y": y_multi,
134 "estimator_routing_methods": ["fit"],
135 },
136 {
137 "metaestimator": LogisticRegressionCV,
138 "X": X,
139 "y": y,
140 "scorer_name": "scoring",
141 "scorer_routing_methods": ["fit", "score"],
142 "cv_name": "cv",
143 "cv_routing_methods": ["fit"],
144 },
145 {
146 "metaestimator": GridSearchCV,
147 "estimator_name": "estimator",
148 "estimator": "classifier",
149 "init_args": {"param_grid": {"alpha": [0.1, 0.2]}},
150 "X": X,
151 "y": y,
152 "estimator_routing_methods": ["fit"],
153 "preserves_metadata": "subset",
154 "scorer_name": "scoring",
155 "scorer_routing_methods": ["fit", "score"],
156 "cv_name": "cv",
157 "cv_routing_methods": ["fit"],
158 },
159 {
160 "metaestimator": RandomizedSearchCV,
161 "estimator_name": "estimator",
162 "estimator": "classifier",
163 "init_args": {"param_distributions": {"alpha": [0.1, 0.2]}},
164 "X": X,
165 "y": y,
166 "estimator_routing_methods": ["fit"],
167 "preserves_metadata": "subset",
168 "scorer_name": "scoring",
169 "scorer_routing_methods": ["fit", "score"],
170 "cv_name": "cv",
171 "cv_routing_methods": ["fit"],
172 },
173 {
174 "metaestimator": HalvingGridSearchCV,
175 "estimator_name": "estimator",
176 "estimator": "classifier",
177 "init_args": {"param_grid": {"alpha": [0.1, 0.2]}},
178 "X": X,
179 "y": y,
180 "estimator_routing_methods": ["fit"],
181 "preserves_metadata": "subset",
182 "scorer_name": "scoring",
183 "scorer_routing_methods": ["fit", "score"],
184 "cv_name": "cv",
185 "cv_routing_methods": ["fit"],
186 },
187 {
188 "metaestimator": HalvingRandomSearchCV,
189 "estimator_name": "estimator",
190 "estimator": "classifier",
191 "init_args": {"param_distributions": {"alpha": [0.1, 0.2]}},
192 "X": X,
193 "y": y,
194 "estimator_routing_methods": ["fit"],
195 "preserves_metadata": "subset",
196 "scorer_name": "scoring",
197 "scorer_routing_methods": ["fit", "score"],
198 "cv_name": "cv",
199 "cv_routing_methods": ["fit"],
200 },
201 {
202 "metaestimator": FixedThresholdClassifier,
203 "estimator_name": "estimator",
204 "estimator": "classifier",
205 "X": X,
206 "y": y_binary,
207 "estimator_routing_methods": ["fit"],
208 "preserves_metadata": "subset",
209 },
210 {
211 "metaestimator": TunedThresholdClassifierCV,
212 "estimator_name": "estimator",
213 "estimator": "classifier",
214 "X": X,
215 "y": y_binary,
216 "estimator_routing_methods": ["fit"],
217 "preserves_metadata": "subset",
218 },
219 {
220 "metaestimator": OneVsRestClassifier,
221 "estimator_name": "estimator",
222 "estimator": "classifier",
223 "X": X,
224 "y": y,
225 "estimator_routing_methods": ["fit", "partial_fit"],
226 "method_args": {"partial_fit": {"classes": classes}},
227 },
228 {
229 "metaestimator": OneVsOneClassifier,
230 "estimator_name": "estimator",
231 "estimator": "classifier",
232 "X": X,
233 "y": y,
234 "estimator_routing_methods": ["fit", "partial_fit"],
235 "preserves_metadata": "subset",
236 "method_args": {"partial_fit": {"classes": classes}},
237 },
238 {
239 "metaestimator": OutputCodeClassifier,
240 "estimator_name": "estimator",
241 "estimator": "classifier",
242 "init_args": {"random_state": 42},
243 "X": X,
244 "y": y,
245 "estimator_routing_methods": ["fit"],
246 },
247 {
248 "metaestimator": SelectFromModel,
249 "estimator_name": "estimator",
250 "estimator": "classifier",
251 "X": X,
252 "y": y,
253 "estimator_routing_methods": ["fit", "partial_fit"],
254 "method_args": {"partial_fit": {"classes": classes}},
255 },
256 {
257 "metaestimator": OrthogonalMatchingPursuitCV,
258 "X": X,
259 "y": y,
260 "cv_name": "cv",
261 "cv_routing_methods": ["fit"],
262 },
263 {
264 "metaestimator": ElasticNetCV,
265 "X": X,
266 "y": y,
267 "cv_name": "cv",
268 "cv_routing_methods": ["fit"],
269 },
270 {
271 "metaestimator": LassoCV,
272 "X": X,
273 "y": y,
274 "cv_name": "cv",
275 "cv_routing_methods": ["fit"],
276 },
277 {
278 "metaestimator": MultiTaskElasticNetCV,
279 "X": X,
280 "y": y_multi,
281 "cv_name": "cv",
282 "cv_routing_methods": ["fit"],
283 },
284 {
285 "metaestimator": MultiTaskLassoCV,
286 "X": X,
287 "y": y_multi,
288 "cv_name": "cv",
289 "cv_routing_methods": ["fit"],
290 },
291 {
292 "metaestimator": LarsCV,
293 "X": X,
294 "y": y,
295 "cv_name": "cv",
296 "cv_routing_methods": ["fit"],
297 },
298 {
299 "metaestimator": LassoLarsCV,
300 "X": X,
301 "y": y,
302 "cv_name": "cv",
303 "cv_routing_methods": ["fit"],
304 },
305 {
306 "metaestimator": RANSACRegressor,
307 "estimator_name": "estimator",
308 "estimator": "regressor",
309 "init_args": {"min_samples": 0.5},
310 "X": X,
311 "y": y,
312 "preserves_metadata": "subset",
313 "estimator_routing_methods": ["fit", "predict", "score"],
314 "method_mapping": {"fit": ["fit", "score"]},
315 },
316 {
317 "metaestimator": IterativeImputer,
318 "estimator_name": "estimator",
319 "estimator": "regressor",
320 "init_args": {"skip_complete": False},
321 "X": X,
322 "y": y,
323 "estimator_routing_methods": ["fit"],
324 },
325 {
326 "metaestimator": BaggingClassifier,
327 "estimator_name": "estimator",
328 "estimator": "classifier",
329 "X": X,
330 "y": y,
331 "preserves_metadata": False,
332 "estimator_routing_methods": [
333 "fit",
334 "predict",
335 "predict_proba",
336 "predict_log_proba",
337 "decision_function",
338 ],
339 "method_mapping": {
340 "predict": ["predict", "predict_proba"],
341 "predict_proba": ["predict", "predict_proba"],
342 "predict_log_proba": ["predict", "predict_proba", "predict_log_proba"],
343 },
344 },
345 {
346 "metaestimator": BaggingRegressor,
347 "estimator_name": "estimator",
348 "estimator": "regressor",
349 "X": X,
350 "y": y,
351 "preserves_metadata": False,
352 "estimator_routing_methods": ["fit", "predict"],
353 },
354 {
355 "metaestimator": RidgeCV,
356 "X": X,
357 "y": y,
358 "scorer_name": "scoring",
359 "scorer_routing_methods": ["fit"],
360 },
361 {
362 "metaestimator": RidgeClassifierCV,
363 "X": X,
364 "y": y,
365 "scorer_name": "scoring",
366 "scorer_routing_methods": ["fit"],
367 },
368 {
369 "metaestimator": RidgeCV,
370 "X": X,
371 "y": y,
372 "scorer_name": "scoring",
373 "scorer_routing_methods": ["fit"],
374 "cv_name": "cv",
375 "cv_routing_methods": ["fit"],
376 },
377 {
378 "metaestimator": RidgeClassifierCV,
379 "X": X,
380 "y": y,
381 "scorer_name": "scoring",
382 "scorer_routing_methods": ["fit"],
383 "cv_name": "cv",
384 "cv_routing_methods": ["fit"],
385 },
386 {
387 "metaestimator": GraphicalLassoCV,
388 "X": X,
389 "y": y,
390 "cv_name": "cv",
391 "cv_routing_methods": ["fit"],
392 },
393 {
394 "metaestimator": TransformedTargetRegressor,
395 "estimator": "regressor",
396 "estimator_name": "regressor",
397 "X": X,
398 "y": y,
399 "estimator_routing_methods": ["fit", "predict"],
400 },
401 {
402 "metaestimator": SelfTrainingClassifier,
403 "estimator_name": "estimator",
404 "estimator": "classifier",
405 "X": X,
406 "y": y,
407 "preserves_metadata": True,
408 "estimator_routing_methods": [
409 "fit",
410 "predict",
411 "predict_proba",
412 "predict_log_proba",
413 "decision_function",
414 "score",
415 ],
416 "method_mapping": {"fit": ["fit", "score"]},
417 },
418 {
419 "metaestimator": SequentialFeatureSelector,
420 "estimator_name": "estimator",
421 "estimator": "classifier",
422 "X": X,
423 "y": y,
424 "estimator_routing_methods": ["fit"],
425 "scorer_name": "scoring",
426 "scorer_routing_methods": ["fit"],
427 "cv_name": "cv",
428 "cv_routing_methods": ["fit"],
429 },
430 {
431 "metaestimator": RFE,
432 "estimator": "classifier",
433 "estimator_name": "estimator",
434 "X": X,
435 "y": y,
436 "estimator_routing_methods": ["fit", "predict", "score"],
437 },
438 {
439 "metaestimator": RFECV,
440 "estimator": "classifier",
441 "estimator_name": "estimator",
442 "estimator_routing_methods": ["fit"],
443 "cv_name": "cv",
444 "cv_routing_methods": ["fit"],
445 "scorer_name": "scoring",
446 "scorer_routing_methods": ["fit", "score"],
447 "X": X,
448 "y": y,
449 },
450]
451"""List containing all metaestimators to be tested and their settings
452
453The keys are as follows:
454
455- metaestimator: The metaestimator to be tested
456- estimator_name: The name of the argument for the sub-estimator
457- estimator: The sub-estimator type, either "regressor" or "classifier"
458- init_args: The arguments to be passed to the metaestimator's constructor
459- X: X-data to fit and predict
460- y: y-data to fit
461- estimator_routing_methods: list of all methods to check for routing metadata
462 to the sub-estimator
463- preserves_metadata:
464 - True (default): the metaestimator passes the metadata to the
465 sub-estimator without modification. We check that the values recorded by
466 the sub-estimator are identical to what we've passed to the
467 metaestimator.
468 - False: no check is performed regarding values, we only check that a
469 metadata with the expected names/keys are passed.
470 - "subset": we check that the recorded metadata by the sub-estimator is a
471 subset of what is passed to the metaestimator.
472- scorer_name: The name of the argument for the scorer
473- scorer_routing_methods: list of all methods to check for routing metadata
474 to the scorer
475- cv_name: The name of the argument for the CV splitter
476- cv_routing_methods: list of all methods to check for routing metadata
477 to the splitter
478- method_args: a dict of dicts, defining extra arguments needed to be passed to
479 methods, such as passing `classes` to `partial_fit`.
480- method_mapping: a dict of the form `{caller: [callee1, ...]}` which signals
481 which `.set_{method}_request` methods should be called to set request values.
482 If not present, a one-to-one mapping is assumed.
483"""
484
485# IDs used by pytest to get meaningful verbose messages when running the tests
486METAESTIMATOR_IDS = [str(row["metaestimator"].__name__) for row in METAESTIMATORS]
487
488UNSUPPORTED_ESTIMATORS = [
489 AdaBoostClassifier(),
490 AdaBoostRegressor(),
491]
492
493
494def get_init_args(metaestimator_info, sub_estimator_consumes):
495 """Get the init args for a metaestimator
496
497 This is a helper function to get the init args for a metaestimator from
498 the METAESTIMATORS list. It returns an empty dict if no init args are
499 required.
500
501 Parameters
502 ----------
503 metaestimator_info : dict
504 The metaestimator info from METAESTIMATORS
505
506 sub_estimator_consumes : bool
507 Whether the sub-estimator consumes metadata or not.
508
509 Returns
510 -------
511 kwargs : dict
512 The init args for the metaestimator.
513
514 (estimator, estimator_registry) : (estimator, registry)
515 The sub-estimator and the corresponding registry.
516
517 (scorer, scorer_registry) : (scorer, registry)
518 The scorer and the corresponding registry.
519
520 (cv, cv_registry) : (CV splitter, registry)
521 The CV splitter and the corresponding registry.
522 """
523 kwargs = metaestimator_info.get("init_args", {})
524 estimator, estimator_registry = None, None
525 scorer, scorer_registry = None, None
526 cv, cv_registry = None, None
527 if "estimator" in metaestimator_info:
528 estimator_name = metaestimator_info["estimator_name"]
529 estimator_registry = _Registry()
530 sub_estimator_type = metaestimator_info["estimator"]
531 if sub_estimator_consumes:
532 if sub_estimator_type == "regressor":
533 estimator = ConsumingRegressor(estimator_registry)
534 elif sub_estimator_type == "classifier":
535 estimator = ConsumingClassifier(estimator_registry)
536 else:
537 raise ValueError("Unpermitted `sub_estimator_type`.") # pragma: nocover
538 else:
539 if sub_estimator_type == "regressor":
540 estimator = NonConsumingRegressor()
541 elif sub_estimator_type == "classifier":
542 estimator = NonConsumingClassifier()
543 else:
544 raise ValueError("Unpermitted `sub_estimator_type`.") # pragma: nocover
545 kwargs[estimator_name] = estimator
546 if "scorer_name" in metaestimator_info:
547 scorer_name = metaestimator_info["scorer_name"]
548 scorer_registry = _Registry()
549 scorer = ConsumingScorer(registry=scorer_registry)
550 kwargs[scorer_name] = scorer
551 if "cv_name" in metaestimator_info:
552 cv_name = metaestimator_info["cv_name"]
553 cv_registry = _Registry()
554 cv = ConsumingSplitter(registry=cv_registry)
555 kwargs[cv_name] = cv
556
557 return (
558 kwargs,
559 (estimator, estimator_registry),
560 (scorer, scorer_registry),
561 (cv, cv_registry),
562 )
563
564
565def set_requests(obj, *, method_mapping, methods, metadata_name, value=True):
566 """Call `set_{method}_request` on a list of methods from the sub-estimator.
567
568 Parameters
569 ----------
570 obj : BaseEstimator
571 The object for which `set_{method}_request` methods are called.
572
573 method_mapping : dict
574 The method mapping in the form of `{caller: [callee, ...]}`.
575 If a "caller" is not present in the method mapping, a one-to-one mapping is
576 assumed.
577
578 methods : list of str
579 The list of methods as "caller"s for which the request for the child should
580 be set.
581
582 metadata_name : str
583 The name of the metadata to be routed, usually either `"metadata"` or
584 `"sample_weight"` in our tests.
585
586 value : None, bool, or str
587 The request value to be set, by default it's `True`
588 """
589 for caller in methods:
590 for callee in method_mapping.get(caller, [caller]):
591 set_request_for_method = getattr(obj, f"set_{callee}_request")
592 set_request_for_method(**{metadata_name: value})
593 if (
594 isinstance(obj, BaseEstimator)
595 and is_classifier(obj)
596 and callee == "partial_fit"
597 ):
598 set_request_for_method(classes=True)
599
600
601@pytest.mark.parametrize("estimator", UNSUPPORTED_ESTIMATORS)
602@config_context(enable_metadata_routing=True)
603def test_unsupported_estimators_get_metadata_routing(estimator):
604 """Test that get_metadata_routing is not implemented on meta-estimators for
605 which we haven't implemented routing yet."""
606 with pytest.raises(NotImplementedError):
607 estimator.get_metadata_routing()
608
609
610@pytest.mark.parametrize("estimator", UNSUPPORTED_ESTIMATORS)
611@config_context(enable_metadata_routing=True)
612def test_unsupported_estimators_fit_with_metadata(estimator):
613 """Test that fit raises NotImplementedError when metadata routing is
614 enabled and a metadata is passed on meta-estimators for which we haven't
615 implemented routing yet."""
616 with pytest.raises(NotImplementedError):
617 try:
618 estimator.fit([[1]], [1], sample_weight=[1])
619 except TypeError:
620 # not all meta-estimators in the list support sample_weight,
621 # and for those we skip this test.
622 raise NotImplementedError
623
624
625@config_context(enable_metadata_routing=True)
626def test_registry_copy():
627 # test that _Registry is not copied into a new instance.
628 a = _Registry()
629 b = _Registry()
630 assert a is not b
631 assert a is copy.copy(a)
632 assert a is copy.deepcopy(a)
633
634
635@pytest.mark.parametrize("metaestimator", METAESTIMATORS, ids=METAESTIMATOR_IDS)
636@config_context(enable_metadata_routing=True)
637def test_default_request(metaestimator):
638 # Check that by default request is empty and the right type
639 metaestimator_class = metaestimator["metaestimator"]
640 kwargs, *_ = get_init_args(metaestimator, sub_estimator_consumes=True)
641 instance = metaestimator_class(**kwargs)
642 if "cv_name" in metaestimator:
643 # Our GroupCV splitters request groups by default, which we should
644 # ignore in this test.
645 exclude = {"splitter": ["split"]}
646 else:
647 exclude = None
648 assert_request_is_empty(instance.get_metadata_routing(), exclude=exclude)
649 assert isinstance(instance.get_metadata_routing(), MetadataRouter)
650
651
652@pytest.mark.parametrize("metaestimator", METAESTIMATORS, ids=METAESTIMATOR_IDS)
653@config_context(enable_metadata_routing=True)
654def test_error_on_missing_requests_for_sub_estimator(metaestimator):
655 # Test that a UnsetMetadataPassedError is raised when the sub-estimator's
656 # requests are not set
657 if "estimator" not in metaestimator:
658 # This test only makes sense for metaestimators which have a
659 # sub-estimator, e.g. MyMetaEstimator(estimator=MySubEstimator())
660 return
661
662 metaestimator_class = metaestimator["metaestimator"]
663 X = metaestimator["X"]
664 y = metaestimator["y"]
665 routing_methods = metaestimator["estimator_routing_methods"]
666
667 for method_name in routing_methods:
668 for key in ["sample_weight", "metadata"]:
669 kwargs, (estimator, _), (scorer, _), *_ = get_init_args(
670 metaestimator, sub_estimator_consumes=True
671 )
672 if scorer:
673 scorer.set_score_request(**{key: True})
674 val = {"sample_weight": sample_weight, "metadata": metadata}[key]
675 method_kwargs = {key: val}
676 instance = metaestimator_class(**kwargs)
677 msg = (
678 f"[{key}] are passed but are not explicitly set as requested or not"
679 f" requested for {estimator.__class__.__name__}.{method_name}"
680 )
681 with pytest.raises(UnsetMetadataPassedError, match=re.escape(msg)):
682 method = getattr(instance, method_name)
683 if "fit" not in method_name:
684 # set request on fit
685 set_requests(
686 estimator,
687 method_mapping=metaestimator.get("method_mapping", {}),
688 methods=["fit"],
689 metadata_name=key,
690 )
691 instance.fit(X, y, **method_kwargs)
692 # making sure the requests are unset, in case they were set as a
693 # side effect of setting them for fit. For instance, if method
694 # mapping for fit is: `"fit": ["fit", "score"]`, that would mean
695 # calling `.score` here would not raise, because we have already
696 # set request value for child estimator's `score`.
697 set_requests(
698 estimator,
699 method_mapping=metaestimator.get("method_mapping", {}),
700 methods=["fit"],
701 metadata_name=key,
702 value=None,
703 )
704 try:
705 # `fit`, `partial_fit`, 'score' accept y, others don't.
706 method(X, y, **method_kwargs)
707 except TypeError:
708 method(X, **method_kwargs)
709
710
711@pytest.mark.parametrize("metaestimator", METAESTIMATORS, ids=METAESTIMATOR_IDS)
712@config_context(enable_metadata_routing=True)
713def test_setting_request_on_sub_estimator_removes_error(metaestimator):
714 # When the metadata is explicitly requested on the sub-estimator, there
715 # should be no errors.
716 if "estimator" not in metaestimator:
717 # This test only makes sense for metaestimators which have a
718 # sub-estimator, e.g. MyMetaEstimator(estimator=MySubEstimator())
719 return
720
721 metaestimator_class = metaestimator["metaestimator"]
722 X = metaestimator["X"]
723 y = metaestimator["y"]
724 routing_methods = metaestimator["estimator_routing_methods"]
725 method_mapping = metaestimator.get("method_mapping", {})
726 preserves_metadata = metaestimator.get("preserves_metadata", True)
727
728 for method_name in routing_methods:
729 for key in ["sample_weight", "metadata"]:
730 val = {"sample_weight": sample_weight, "metadata": metadata}[key]
731 method_kwargs = {key: val}
732
733 kwargs, (estimator, registry), (scorer, _), (cv, _) = get_init_args(
734 metaestimator, sub_estimator_consumes=True
735 )
736 if scorer:
737 set_requests(
738 scorer, method_mapping={}, methods=["score"], metadata_name=key
739 )
740 if cv:
741 cv.set_split_request(groups=True, metadata=True)
742
743 # `set_{method}_request({metadata}==True)` on the underlying objects
744 set_requests(
745 estimator,
746 method_mapping=method_mapping,
747 methods=[method_name],
748 metadata_name=key,
749 )
750
751 instance = metaestimator_class(**kwargs)
752 method = getattr(instance, method_name)
753 extra_method_args = metaestimator.get("method_args", {}).get(
754 method_name, {}
755 )
756 if "fit" not in method_name:
757 # fit before calling method
758 instance.fit(X, y)
759 try:
760 # `fit` and `partial_fit` accept y, others don't.
761 method(X, y, **method_kwargs, **extra_method_args)
762 except TypeError:
763 method(X, **method_kwargs, **extra_method_args)
764
765 # sanity check that registry is not empty, or else the test passes
766 # trivially
767 assert registry
768 split_params = (
769 method_kwargs.keys() if preserves_metadata == "subset" else ()
770 )
771 for estimator in registry:
772 check_recorded_metadata(
773 estimator,
774 method=method_name,
775 parent=method_name,
776 split_params=split_params,
777 **method_kwargs,
778 )
779
780
781@pytest.mark.parametrize("metaestimator", METAESTIMATORS, ids=METAESTIMATOR_IDS)
782@config_context(enable_metadata_routing=True)
783def test_non_consuming_estimator_works(metaestimator):
784 # Test that when a non-consuming estimator is given, the meta-estimator
785 # works w/o setting any requests.
786 # Regression test for https://github.com/scikit-learn/scikit-learn/issues/28239
787 if "estimator" not in metaestimator:
788 # This test only makes sense for metaestimators which have a
789 # sub-estimator, e.g. MyMetaEstimator(estimator=MySubEstimator())
790 return
791
792 def set_request(estimator, method_name):
793 # e.g. call set_fit_request on estimator
794 if is_classifier(estimator) and method_name == "partial_fit":
795 estimator.set_partial_fit_request(classes=True)
796
797 metaestimator_class = metaestimator["metaestimator"]
798 X = metaestimator["X"]
799 y = metaestimator["y"]
800 routing_methods = metaestimator["estimator_routing_methods"]
801
802 for method_name in routing_methods:
803 kwargs, (estimator, _), (_, _), (_, _) = get_init_args(
804 metaestimator, sub_estimator_consumes=False
805 )
806 instance = metaestimator_class(**kwargs)
807 set_request(estimator, method_name)
808 method = getattr(instance, method_name)
809 extra_method_args = metaestimator.get("method_args", {}).get(method_name, {})
810 if "fit" not in method_name:
811 instance.fit(X, y, **extra_method_args)
812 # The following should pass w/o raising a routing error.
813 try:
814 # `fit` and `partial_fit` accept y, others don't.
815 method(X, y, **extra_method_args)
816 except TypeError:
817 method(X, **extra_method_args)
818
819
820@pytest.mark.parametrize("metaestimator", METAESTIMATORS, ids=METAESTIMATOR_IDS)
821@config_context(enable_metadata_routing=True)
822def test_metadata_is_routed_correctly_to_scorer(metaestimator):
823 """Test that any requested metadata is correctly routed to the underlying
824 scorers in CV estimators.
825 """
826 if "scorer_name" not in metaestimator:
827 # This test only makes sense for CV estimators
828 return
829
830 metaestimator_class = metaestimator["metaestimator"]
831 routing_methods = metaestimator["scorer_routing_methods"]
832 method_mapping = metaestimator.get("method_mapping", {})
833
834 for method_name in routing_methods:
835 kwargs, (estimator, _), (scorer, registry), (cv, _) = get_init_args(
836 metaestimator, sub_estimator_consumes=True
837 )
838 scorer.set_score_request(sample_weight=True)
839 if cv:
840 cv.set_split_request(groups=True, metadata=True)
841 if estimator is not None:
842 set_requests(
843 estimator,
844 method_mapping=method_mapping,
845 methods=[method_name],
846 metadata_name="sample_weight",
847 )
848 instance = metaestimator_class(**kwargs)
849 method = getattr(instance, method_name)
850 method_kwargs = {"sample_weight": sample_weight}
851 if "fit" not in method_name:
852 instance.fit(X, y)
853 method(X, y, **method_kwargs)
854
855 assert registry
856 for _scorer in registry:
857 check_recorded_metadata(
858 obj=_scorer,
859 method="score",
860 parent=method_name,
861 split_params=("sample_weight",),
862 **method_kwargs,
863 )
864
865
866@pytest.mark.parametrize("metaestimator", METAESTIMATORS, ids=METAESTIMATOR_IDS)
867@config_context(enable_metadata_routing=True)
868def test_metadata_is_routed_correctly_to_splitter(metaestimator):
869 """Test that any requested metadata is correctly routed to the underlying
870 splitters in CV estimators.
871 """
872 if "cv_routing_methods" not in metaestimator:
873 # This test is only for metaestimators accepting a CV splitter
874 return
875
876 metaestimator_class = metaestimator["metaestimator"]
877 routing_methods = metaestimator["cv_routing_methods"]
878 X_ = metaestimator["X"]
879 y_ = metaestimator["y"]
880
881 for method_name in routing_methods:
882 kwargs, (estimator, _), (scorer, _), (cv, registry) = get_init_args(
883 metaestimator, sub_estimator_consumes=True
884 )
885 if estimator:
886 estimator.set_fit_request(sample_weight=False, metadata=False)
887 if scorer:
888 scorer.set_score_request(sample_weight=False, metadata=False)
889 cv.set_split_request(groups=True, metadata=True)
890 instance = metaestimator_class(**kwargs)
891 method_kwargs = {"groups": groups, "metadata": metadata}
892 method = getattr(instance, method_name)
893 method(X_, y_, **method_kwargs)
894 assert registry
895 for _splitter in registry:
896 check_recorded_metadata(
897 obj=_splitter, method="split", parent=method_name, **method_kwargs
898 )
899
900
901@pytest.mark.parametrize("metaestimator", METAESTIMATORS, ids=METAESTIMATOR_IDS)
902@config_context(enable_metadata_routing=True)
903def test_metadata_routed_to_group_splitter(metaestimator):
904 """Test that groups are routed correctly if group splitter of CV estimator is used
905 within cross_validate. Regression test for issue described in PR #29634 to test that
906 `ValueError: The 'groups' parameter should not be None.` is not raised."""
907
908 if "cv_routing_methods" not in metaestimator:
909 # This test is only for metaestimators accepting a CV splitter
910 return
911
912 metaestimator_class = metaestimator["metaestimator"]
913 X_ = metaestimator["X"]
914 y_ = metaestimator["y"]
915
916 kwargs, *_ = get_init_args(metaestimator, sub_estimator_consumes=True)
917 # remove `ConsumingSplitter` from kwargs, so 'cv' param isn't passed twice:
918 kwargs.pop("cv", None)
919 instance = metaestimator_class(cv=GroupKFold(n_splits=2), **kwargs)
920 cross_validate(
921 instance,
922 X_,
923 y_,
924 params={"groups": groups},
925 cv=GroupKFold(n_splits=2),
926 scoring=make_scorer(mean_squared_error, response_method="predict"),
927 )
928