Aluode/PerceptionLabPortable
0
1# We can not use pytest here, because we run
2# build_tools/azure/test_pytest_soft_dependency.sh on these
3# tests to make sure estimator_checks works without pytest.
4
5import importlib
6import re
7import sys
8import unittest
9import warnings
10from inspect import isgenerator
11from numbers import Integral, Real
12
13import joblib
14import numpy as np
15import scipy.sparse as sp
16
17from sklearn import config_context, get_config
18from sklearn.base import BaseEstimator, ClassifierMixin, OutlierMixin, TransformerMixin
19from sklearn.cluster import MiniBatchKMeans
20from sklearn.datasets import (
21 load_iris,
22 make_multilabel_classification,
23)
24from sklearn.decomposition import PCA
25from sklearn.exceptions import (
26 ConvergenceWarning,
27 EstimatorCheckFailedWarning,
28 SkipTestWarning,
29)
30from sklearn.linear_model import (
31 LinearRegression,
32 LogisticRegression,
33 MultiTaskElasticNet,
34 SGDClassifier,
35)
36from sklearn.mixture import GaussianMixture
37from sklearn.neighbors import KNeighborsRegressor
38from sklearn.preprocessing import StandardScaler
39from sklearn.svm import SVC, NuSVC
40from sklearn.utils import _array_api, all_estimators, deprecated
41from sklearn.utils._param_validation import Interval, StrOptions
42from sklearn.utils._test_common.instance_generator import (
43 _construct_instances,
44 _get_expected_failed_checks,
45)
46from sklearn.utils._testing import (
47 MinimalClassifier,
48 MinimalRegressor,
49 MinimalTransformer,
50 SkipTest,
51 ignore_warnings,
52 raises,
53)
54from sklearn.utils.estimator_checks import (
55 _check_name,
56 _NotAnArray,
57 _yield_all_checks,
58 check_array_api_input,
59 check_class_weight_balanced_linear_classifier,
60 check_classifier_data_not_an_array,
61 check_classifier_not_supporting_multiclass,
62 check_classifiers_multilabel_output_format_decision_function,
63 check_classifiers_multilabel_output_format_predict,
64 check_classifiers_multilabel_output_format_predict_proba,
65 check_classifiers_one_label_sample_weights,
66 check_dataframe_column_names_consistency,
67 check_decision_proba_consistency,
68 check_dict_unchanged,
69 check_dont_overwrite_parameters,
70 check_estimator,
71 check_estimator_cloneable,
72 check_estimator_repr,
73 check_estimator_sparse_array,
74 check_estimator_sparse_matrix,
75 check_estimator_sparse_tag,
76 check_estimator_tags_renamed,
77 check_estimators_nan_inf,
78 check_estimators_overwrite_params,
79 check_estimators_unfitted,
80 check_fit_check_is_fitted,
81 check_fit_score_takes_y,
82 check_methods_sample_order_invariance,
83 check_methods_subset_invariance,
84 check_mixin_order,
85 check_no_attributes_set_in_init,
86 check_outlier_contamination,
87 check_outlier_corruption,
88 check_parameters_default_constructible,
89 check_positive_only_tag_during_fit,
90 check_regressor_data_not_an_array,
91 check_requires_y_none,
92 check_sample_weights_pandas_series,
93 check_set_params,
94 estimator_checks_generator,
95 set_random_state,
96)
97from sklearn.utils.fixes import CSR_CONTAINERS, SPARRAY_PRESENT
98from sklearn.utils.metaestimators import available_if
99from sklearn.utils.multiclass import type_of_target
100from sklearn.utils.validation import (
101 check_array,
102 check_is_fitted,
103 check_X_y,
104 validate_data,
105)
106
107
108class CorrectNotFittedError(ValueError):
109 """Exception class to raise if estimator is used before fitting.
110
111 Like NotFittedError, it inherits from ValueError, but not from
112 AttributeError. Used for testing only.
113 """
114
115
116class BaseBadClassifier(ClassifierMixin, BaseEstimator):
117 def fit(self, X, y):
118 return self
119
120 def predict(self, X):
121 return np.ones(X.shape[0])
122
123
124class ChangesDict(BaseEstimator):
125 def __init__(self, key=0):
126 self.key = key
127
128 def fit(self, X, y=None):
129 X, y = validate_data(self, X, y)
130 return self
131
132 def predict(self, X):
133 X = check_array(X)
134 self.key = 1000
135 return np.ones(X.shape[0])
136
137
138class SetsWrongAttribute(BaseEstimator):
139 def __init__(self, acceptable_key=0):
140 self.acceptable_key = acceptable_key
141
142 def fit(self, X, y=None):
143 self.wrong_attribute = 0
144 X, y = validate_data(self, X, y)
145 return self
146
147
148class ChangesWrongAttribute(BaseEstimator):
149 def __init__(self, wrong_attribute=0):
150 self.wrong_attribute = wrong_attribute
151
152 def fit(self, X, y=None):
153 self.wrong_attribute = 1
154 X, y = validate_data(self, X, y)
155 return self
156
157
158class ChangesUnderscoreAttribute(BaseEstimator):
159 def fit(self, X, y=None):
160 self._good_attribute = 1
161 X, y = validate_data(self, X, y)
162 return self
163
164
165class RaisesErrorInSetParams(BaseEstimator):
166 def __init__(self, p=0):
167 self.p = p
168
169 def set_params(self, **kwargs):
170 if "p" in kwargs:
171 p = kwargs.pop("p")
172 if p < 0:
173 raise ValueError("p can't be less than 0")
174 self.p = p
175 return super().set_params(**kwargs)
176
177 def fit(self, X, y=None):
178 X, y = validate_data(self, X, y)
179 return self
180
181
182class HasMutableParameters(BaseEstimator):
183 def __init__(self, p=object()):
184 self.p = p
185
186 def fit(self, X, y=None):
187 X, y = validate_data(self, X, y)
188 return self
189
190
191class HasImmutableParameters(BaseEstimator):
192 # Note that object is an uninitialized class, thus immutable.
193 def __init__(self, p=42, q=np.int32(42), r=object):
194 self.p = p
195 self.q = q
196 self.r = r
197
198 def fit(self, X, y=None):
199 X, y = validate_data(self, X, y)
200 return self
201
202
203class ModifiesValueInsteadOfRaisingError(BaseEstimator):
204 def __init__(self, p=0):
205 self.p = p
206
207 def set_params(self, **kwargs):
208 if "p" in kwargs:
209 p = kwargs.pop("p")
210 if p < 0:
211 p = 0
212 self.p = p
213 return super().set_params(**kwargs)
214
215 def fit(self, X, y=None):
216 X, y = validate_data(self, X, y)
217 return self
218
219
220class ModifiesAnotherValue(BaseEstimator):
221 def __init__(self, a=0, b="method1"):
222 self.a = a
223 self.b = b
224
225 def set_params(self, **kwargs):
226 if "a" in kwargs:
227 a = kwargs.pop("a")
228 self.a = a
229 if a is None:
230 kwargs.pop("b")
231 self.b = "method2"
232 return super().set_params(**kwargs)
233
234 def fit(self, X, y=None):
235 X, y = validate_data(self, X, y)
236 return self
237
238
239class NoCheckinPredict(BaseBadClassifier):
240 def fit(self, X, y):
241 X, y = validate_data(self, X, y)
242 return self
243
244
245class NoSparseClassifier(BaseBadClassifier):
246 def __init__(self, raise_for_type=None):
247 # raise_for_type : str, expects "sparse_array" or "sparse_matrix"
248 self.raise_for_type = raise_for_type
249
250 def fit(self, X, y):
251 X, y = validate_data(self, X, y, accept_sparse=["csr", "csc"])
252 if self.raise_for_type == "sparse_array":
253 correct_type = isinstance(X, sp.sparray)
254 elif self.raise_for_type == "sparse_matrix":
255 correct_type = isinstance(X, sp.spmatrix)
256 if correct_type:
257 raise ValueError("Nonsensical Error")
258 return self
259
260 def predict(self, X):
261 X = check_array(X)
262 return np.ones(X.shape[0])
263
264
265class CorrectNotFittedErrorClassifier(BaseBadClassifier):
266 def fit(self, X, y):
267 X, y = validate_data(self, X, y)
268 self.coef_ = np.ones(X.shape[1])
269 return self
270
271 def predict(self, X):
272 check_is_fitted(self)
273 X = check_array(X)
274 return np.ones(X.shape[0])
275
276
277class NoSampleWeightPandasSeriesType(BaseEstimator):
278 def fit(self, X, y, sample_weight=None):
279 # Convert data
280 X, y = validate_data(
281 self, X, y, accept_sparse=("csr", "csc"), multi_output=True, y_numeric=True
282 )
283 # Function is only called after we verify that pandas is installed
284 from pandas import Series
285
286 if isinstance(sample_weight, Series):
287 raise ValueError(
288 "Estimator does not accept 'sample_weight'of type pandas.Series"
289 )
290 return self
291
292 def predict(self, X):
293 X = check_array(X)
294 return np.ones(X.shape[0])
295
296
297class BadBalancedWeightsClassifier(BaseBadClassifier):
298 def __init__(self, class_weight=None):
299 self.class_weight = class_weight
300
301 def fit(self, X, y):
302 from sklearn.preprocessing import LabelEncoder
303 from sklearn.utils import compute_class_weight
304
305 label_encoder = LabelEncoder().fit(y)
306 classes = label_encoder.classes_
307 class_weight = compute_class_weight(self.class_weight, classes=classes, y=y)
308
309 # Intentionally modify the balanced class_weight
310 # to simulate a bug and raise an exception
311 if self.class_weight == "balanced":
312 class_weight += 1.0
313
314 # Simply assigning coef_ to the class_weight
315 self.coef_ = class_weight
316 return self
317
318
319class BadTransformerWithoutMixin(BaseEstimator):
320 def fit(self, X, y=None):
321 X = validate_data(self, X)
322 return self
323
324 def transform(self, X):
325 check_is_fitted(self)
326 X = validate_data(self, X, reset=False)
327 return X
328
329
330class NotInvariantPredict(BaseEstimator):
331 def fit(self, X, y):
332 # Convert data
333 X, y = validate_data(
334 self, X, y, accept_sparse=("csr", "csc"), multi_output=True, y_numeric=True
335 )
336 return self
337
338 def predict(self, X):
339 # return 1 if X has more than one element else return 0
340 X = check_array(X)
341 if X.shape[0] > 1:
342 return np.ones(X.shape[0])
343 return np.zeros(X.shape[0])
344
345
346class NotInvariantSampleOrder(BaseEstimator):
347 def fit(self, X, y):
348 X, y = validate_data(
349 self, X, y, accept_sparse=("csr", "csc"), multi_output=True, y_numeric=True
350 )
351 # store the original X to check for sample order later
352 self._X = X
353 return self
354
355 def predict(self, X):
356 X = check_array(X)
357 # if the input contains the same elements but different sample order,
358 # then just return zeros.
359 if (
360 np.array_equiv(np.sort(X, axis=0), np.sort(self._X, axis=0))
361 and (X != self._X).any()
362 ):
363 return np.zeros(X.shape[0])
364 return X[:, 0]
365
366
367class OneClassSampleErrorClassifier(BaseBadClassifier):
368 """Classifier allowing to trigger different behaviors when `sample_weight` reduces
369 the number of classes to 1."""
370
371 def __init__(self, raise_when_single_class=False):
372 self.raise_when_single_class = raise_when_single_class
373
374 def fit(self, X, y, sample_weight=None):
375 X, y = check_X_y(
376 X, y, accept_sparse=("csr", "csc"), multi_output=True, y_numeric=True
377 )
378
379 self.has_single_class_ = False
380 self.classes_, y = np.unique(y, return_inverse=True)
381 n_classes_ = self.classes_.shape[0]
382 if n_classes_ < 2 and self.raise_when_single_class:
383 self.has_single_class_ = True
384 raise ValueError("normal class error")
385
386 # find the number of class after trimming
387 if sample_weight is not None:
388 if isinstance(sample_weight, np.ndarray) and len(sample_weight) > 0:
389 n_classes_ = np.count_nonzero(np.bincount(y, sample_weight))
390 if n_classes_ < 2:
391 self.has_single_class_ = True
392 raise ValueError("Nonsensical Error")
393
394 return self
395
396 def predict(self, X):
397 check_is_fitted(self)
398 X = check_array(X)
399 if self.has_single_class_:
400 return np.zeros(X.shape[0])
401 return np.ones(X.shape[0])
402
403
404class LargeSparseNotSupportedClassifier(BaseEstimator):
405 """Estimator that claims to support large sparse data
406 (accept_large_sparse=True), but doesn't"""
407
408 def __init__(self, raise_for_type=None):
409 # raise_for_type : str, expects "sparse_array" or "sparse_matrix"
410 self.raise_for_type = raise_for_type
411
412 def fit(self, X, y):
413 X, y = validate_data(
414 self,
415 X,
416 y,
417 accept_sparse=("csr", "csc", "coo"),
418 accept_large_sparse=True,
419 multi_output=True,
420 y_numeric=True,
421 )
422 if self.raise_for_type == "sparse_array":
423 correct_type = isinstance(X, sp.sparray)
424 elif self.raise_for_type == "sparse_matrix":
425 correct_type = isinstance(X, sp.spmatrix)
426 if correct_type:
427 if X.format == "coo":
428 if X.row.dtype == "int64" or X.col.dtype == "int64":
429 raise ValueError("Estimator doesn't support 64-bit indices")
430 elif X.format in ["csc", "csr"]:
431 assert "int64" not in (
432 X.indices.dtype,
433 X.indptr.dtype,
434 ), "Estimator doesn't support 64-bit indices"
435
436 return self
437
438
439class SparseTransformer(TransformerMixin, BaseEstimator):
440 def __init__(self, sparse_container=None):
441 self.sparse_container = sparse_container
442
443 def fit(self, X, y=None):
444 validate_data(self, X)
445 return self
446
447 def fit_transform(self, X, y=None):
448 return self.fit(X, y).transform(X)
449
450 def transform(self, X):
451 check_is_fitted(self)
452 X = validate_data(self, X, accept_sparse=True, reset=False)
453 return self.sparse_container(X)
454
455
456class EstimatorInconsistentForPandas(BaseEstimator):
457 def fit(self, X, y):
458 try:
459 from pandas import DataFrame
460
461 if isinstance(X, DataFrame):
462 self.value_ = X.iloc[0, 0]
463 else:
464 X = check_array(X)
465 self.value_ = X[1, 0]
466 return self
467
468 except ImportError:
469 X = check_array(X)
470 self.value_ = X[1, 0]
471 return self
472
473 def predict(self, X):
474 X = check_array(X)
475 return np.array([self.value_] * X.shape[0])
476
477
478class UntaggedBinaryClassifier(SGDClassifier):
479 # Toy classifier that only supports binary classification, will fail tests.
480 def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None):
481 super().fit(X, y, coef_init, intercept_init, sample_weight)
482 if len(self.classes_) > 2:
483 raise ValueError("Only 2 classes are supported")
484 return self
485
486 def partial_fit(self, X, y, classes=None, sample_weight=None):
487 super().partial_fit(X=X, y=y, classes=classes, sample_weight=sample_weight)
488 if len(self.classes_) > 2:
489 raise ValueError("Only 2 classes are supported")
490 return self
491
492
493class TaggedBinaryClassifier(UntaggedBinaryClassifier):
494 def fit(self, X, y):
495 y_type = type_of_target(y, input_name="y", raise_unknown=True)
496 if y_type != "binary":
497 raise ValueError(
498 "Only binary classification is supported. The type of the target "
499 f"is {y_type}."
500 )
501 return super().fit(X, y)
502
503 # Toy classifier that only supports binary classification.
504 def __sklearn_tags__(self):
505 tags = super().__sklearn_tags__()
506 tags.classifier_tags.multi_class = False
507 return tags
508
509
510class RequiresPositiveXRegressor(LinearRegression):
511 def fit(self, X, y):
512 # reject sparse X to be able to call (X < 0).any()
513 X, y = validate_data(self, X, y, accept_sparse=False, multi_output=True)
514 if (X < 0).any():
515 raise ValueError("Negative values in data passed to X.")
516 return super().fit(X, y)
517
518 def __sklearn_tags__(self):
519 tags = super().__sklearn_tags__()
520 tags.input_tags.positive_only = True
521 # reject sparse X to be able to call (X < 0).any()
522 tags.input_tags.sparse = False
523 return tags
524
525
526class RequiresPositiveYRegressor(LinearRegression):
527 def fit(self, X, y):
528 X, y = validate_data(self, X, y, accept_sparse=True, multi_output=True)
529 if (y <= 0).any():
530 raise ValueError("negative y values not supported!")
531 return super().fit(X, y)
532
533 def __sklearn_tags__(self):
534 tags = super().__sklearn_tags__()
535 tags.target_tags.positive_only = True
536 return tags
537
538
539class PoorScoreLogisticRegression(LogisticRegression):
540 def decision_function(self, X):
541 return super().decision_function(X) + 1
542
543 def __sklearn_tags__(self):
544 tags = super().__sklearn_tags__()
545 tags.classifier_tags.poor_score = True
546 return tags
547
548
549class PartialFitChecksName(BaseEstimator):
550 def fit(self, X, y):
551 validate_data(self, X, y)
552 return self
553
554 def partial_fit(self, X, y):
555 reset = not hasattr(self, "_fitted")
556 validate_data(self, X, y, reset=reset)
557 self._fitted = True
558 return self
559
560
561class BrokenArrayAPI(BaseEstimator):
562 """Make different predictions when using Numpy and the Array API"""
563
564 def fit(self, X, y):
565 return self
566
567 def predict(self, X):
568 enabled = get_config()["array_api_dispatch"]
569 xp, _ = _array_api.get_namespace(X)
570 if enabled:
571 return xp.asarray([1, 2, 3])
572 else:
573 return np.array([3, 2, 1])
574
575
576def test_check_array_api_input():
577 try:
578 importlib.import_module("array_api_strict")
579 except ModuleNotFoundError: # pragma: nocover
580 raise SkipTest("array-api-strict is required to run this test")
581
582 with raises(AssertionError, match="Not equal to tolerance"):
583 check_array_api_input(
584 "BrokenArrayAPI",
585 BrokenArrayAPI(),
586 array_namespace="array_api_strict",
587 check_values=True,
588 )
589
590
591def test_not_an_array_array_function():
592 not_array = _NotAnArray(np.ones(10))
593 msg = "Don't want to call array_function sum!"
594 with raises(TypeError, match=msg):
595 np.sum(not_array)
596 # always returns True
597 assert np.may_share_memory(not_array, None)
598
599
600def test_check_fit_score_takes_y_works_on_deprecated_fit():
601 # Tests that check_fit_score_takes_y works on a class with
602 # a deprecated fit method
603
604 class TestEstimatorWithDeprecatedFitMethod(BaseEstimator):
605 @deprecated("Deprecated for the purpose of testing check_fit_score_takes_y")
606 def fit(self, X, y):
607 return self
608
609 check_fit_score_takes_y("test", TestEstimatorWithDeprecatedFitMethod())
610
611
612def test_check_estimator_with_class_removed():
613 """Test that passing a class instead of an instance fails."""
614 msg = "Passing a class was deprecated"
615 with raises(TypeError, match=msg):
616 check_estimator(LogisticRegression)
617
618
619def test_mutable_default_params():
620 """Test that constructor cannot have mutable default parameters."""
621 msg = (
622 "Parameter 'p' of estimator 'HasMutableParameters' is of type "
623 "object which is not allowed"
624 )
625 # check that the "default_constructible" test checks for mutable parameters
626 check_parameters_default_constructible(
627 "Immutable", HasImmutableParameters()
628 ) # should pass
629 with raises(AssertionError, match=msg):
630 check_parameters_default_constructible("Mutable", HasMutableParameters())
631
632
633def test_check_set_params():
634 """Check set_params doesn't fail and sets the right values."""
635 # check that values returned by get_params match set_params
636 msg = "get_params result does not match what was passed to set_params"
637 with raises(AssertionError, match=msg):
638 check_set_params("test", ModifiesValueInsteadOfRaisingError())
639
640 with warnings.catch_warnings(record=True) as records:
641 check_set_params("test", RaisesErrorInSetParams())
642 assert UserWarning in [rec.category for rec in records]
643
644 with raises(AssertionError, match=msg):
645 check_set_params("test", ModifiesAnotherValue())
646
647
648def test_check_estimators_nan_inf():
649 # check that predict does input validation (doesn't accept dicts in input)
650 msg = "Estimator NoCheckinPredict doesn't check for NaN and inf in predict"
651 with raises(AssertionError, match=msg):
652 check_estimators_nan_inf("NoCheckinPredict", NoCheckinPredict())
653
654
655def test_check_dict_unchanged():
656 # check that estimator state does not change
657 # at transform/predict/predict_proba time
658 msg = "Estimator changes __dict__ during predict"
659 with raises(AssertionError, match=msg):
660 check_dict_unchanged("test", ChangesDict())
661
662
663def test_check_sample_weights_pandas_series():
664 # check that sample_weights in fit accepts pandas.Series type
665 try:
666 from pandas import Series # noqa: F401
667
668 msg = (
669 "Estimator NoSampleWeightPandasSeriesType raises error if "
670 "'sample_weight' parameter is of type pandas.Series"
671 )
672 with raises(ValueError, match=msg):
673 check_sample_weights_pandas_series(
674 "NoSampleWeightPandasSeriesType", NoSampleWeightPandasSeriesType()
675 )
676 except ImportError:
677 pass
678
679
680def test_check_estimators_overwrite_params():
681 # check that `fit` only changes attributes that
682 # are private (start with an _ or end with a _).
683 msg = (
684 "Estimator ChangesWrongAttribute should not change or mutate "
685 "the parameter wrong_attribute from 0 to 1 during fit."
686 )
687 with raises(AssertionError, match=msg):
688 check_estimators_overwrite_params(
689 "ChangesWrongAttribute", ChangesWrongAttribute()
690 )
691 check_estimators_overwrite_params("test", ChangesUnderscoreAttribute())
692
693
694def test_check_dont_overwrite_parameters():
695 # check that `fit` doesn't add any public attribute
696 msg = (
697 r"Estimator adds public attribute\(s\) during the fit method."
698 " Estimators are only allowed to add private attributes"
699 " either started with _ or ended"
700 " with _ but wrong_attribute added"
701 )
702 with raises(AssertionError, match=msg):
703 check_dont_overwrite_parameters("test", SetsWrongAttribute())
704
705
706def test_check_methods_sample_order_invariance():
707 # check for sample order invariance
708 name = NotInvariantSampleOrder.__name__
709 method = "predict"
710 msg = (
711 "{method} of {name} is not invariant when applied to a dataset"
712 "with different sample order."
713 ).format(method=method, name=name)
714 with raises(AssertionError, match=msg):
715 check_methods_sample_order_invariance(
716 "NotInvariantSampleOrder", NotInvariantSampleOrder()
717 )
718
719
720def test_check_methods_subset_invariance():
721 # check for invariant method
722 name = NotInvariantPredict.__name__
723 method = "predict"
724 msg = ("{method} of {name} is not invariant when applied to a subset.").format(
725 method=method, name=name
726 )
727 with raises(AssertionError, match=msg):
728 check_methods_subset_invariance("NotInvariantPredict", NotInvariantPredict())
729
730
731def test_check_estimator_sparse_data():
732 # check for sparse data input handling
733 name = NoSparseClassifier.__name__
734 msg = "Estimator %s doesn't seem to fail gracefully on sparse data" % name
735 with raises(AssertionError, match=msg):
736 check_estimator_sparse_matrix(name, NoSparseClassifier("sparse_matrix"))
737
738 if SPARRAY_PRESENT:
739 with raises(AssertionError, match=msg):
740 check_estimator_sparse_array(name, NoSparseClassifier("sparse_array"))
741
742 # Large indices test on bad estimator
743 msg = (
744 "Estimator LargeSparseNotSupportedClassifier doesn't seem to "
745 r"support \S{3}_64 matrix, and is not failing gracefully.*"
746 )
747 with raises(AssertionError, match=msg):
748 check_estimator_sparse_matrix(
749 "LargeSparseNotSupportedClassifier",
750 LargeSparseNotSupportedClassifier("sparse_matrix"),
751 )
752
753 if SPARRAY_PRESENT:
754 with raises(AssertionError, match=msg):
755 check_estimator_sparse_array(
756 "LargeSparseNotSupportedClassifier",
757 LargeSparseNotSupportedClassifier("sparse_array"),
758 )
759
760
761def test_check_classifiers_one_label_sample_weights():
762 # check for classifiers reducing to less than two classes via sample weights
763 name = OneClassSampleErrorClassifier.__name__
764 msg = (
765 f"{name} failed when fitted on one label after sample_weight "
766 "trimming. Error message is not explicit, it should have "
767 "'class'."
768 )
769 with raises(AssertionError, match=msg):
770 check_classifiers_one_label_sample_weights(
771 "OneClassSampleErrorClassifier", OneClassSampleErrorClassifier()
772 )
773
774
775def test_check_estimator_not_fail_fast():
776 """Check the contents of the results returned with on_fail!="raise".
777
778 This results should contain details about the observed failures, expected
779 or not.
780 """
781 check_results = check_estimator(BaseEstimator(), on_fail=None)
782 assert isinstance(check_results, list)
783 assert len(check_results) > 0
784 assert all(
785 isinstance(item, dict)
786 and set(item.keys())
787 == {
788 "estimator",
789 "check_name",
790 "exception",
791 "status",
792 "expected_to_fail",
793 "expected_to_fail_reason",
794 }
795 for item in check_results
796 )
797 # Some tests are expected to fail, some are expected to pass.
798 assert any(item["status"] == "failed" for item in check_results)
799 assert any(item["status"] == "passed" for item in check_results)
800
801
802def test_check_estimator():
803 # tests that the estimator actually fails on "bad" estimators.
804 # not a complete test of all checks, which are very extensive.
805
806 # check that we have a fit method
807 msg = "object has no attribute 'fit'"
808 with raises(AttributeError, match=msg):
809 check_estimator(BaseEstimator())
810
811 # does error on binary_only untagged estimator
812 msg = "Only 2 classes are supported"
813 with raises(ValueError, match=msg):
814 check_estimator(UntaggedBinaryClassifier())
815
816 for csr_container in CSR_CONTAINERS:
817 # non-regression test for estimators transforming to sparse data
818 check_estimator(SparseTransformer(sparse_container=csr_container))
819
820 # doesn't error on actual estimator
821 check_estimator(LogisticRegression())
822 check_estimator(LogisticRegression(C=0.01))
823 check_estimator(MultiTaskElasticNet())
824
825 # doesn't error on binary_only tagged estimator
826 check_estimator(TaggedBinaryClassifier())
827 check_estimator(RequiresPositiveXRegressor())
828
829 # Check regressor with requires_positive_y estimator tag
830 msg = "negative y values not supported!"
831 with raises(ValueError, match=msg):
832 check_estimator(RequiresPositiveYRegressor())
833
834 # Does not raise error on classifier with poor_score tag
835 check_estimator(PoorScoreLogisticRegression())
836
837
838def test_check_outlier_corruption():
839 # should raise AssertionError
840 decision = np.array([0.0, 1.0, 1.5, 2.0])
841 with raises(AssertionError):
842 check_outlier_corruption(1, 2, decision)
843 # should pass
844 decision = np.array([0.0, 1.0, 1.0, 2.0])
845 check_outlier_corruption(1, 2, decision)
846
847
848def test_check_estimator_sparse_tag():
849 """Test that check_estimator_sparse_tag raises error when sparse tag is
850 misaligned."""
851
852 class EstimatorWithSparseConfig(BaseEstimator):
853 def __init__(self, tag_sparse, accept_sparse, fit_error=None):
854 self.tag_sparse = tag_sparse
855 self.accept_sparse = accept_sparse
856 self.fit_error = fit_error
857
858 def fit(self, X, y=None):
859 if self.fit_error:
860 raise self.fit_error
861 validate_data(self, X, y, accept_sparse=self.accept_sparse)
862 return self
863
864 def __sklearn_tags__(self):
865 tags = super().__sklearn_tags__()
866 tags.input_tags.sparse = self.tag_sparse
867 return tags
868
869 test_cases = [
870 {"tag_sparse": True, "accept_sparse": True, "error_type": None},
871 {"tag_sparse": False, "accept_sparse": False, "error_type": None},
872 {"tag_sparse": False, "accept_sparse": True, "error_type": AssertionError},
873 {"tag_sparse": True, "accept_sparse": False, "error_type": AssertionError},
874 ]
875
876 for test_case in test_cases:
877 estimator = EstimatorWithSparseConfig(
878 test_case["tag_sparse"],
879 test_case["accept_sparse"],
880 )
881 if test_case["error_type"] is None:
882 check_estimator_sparse_tag(estimator.__class__.__name__, estimator)
883 else:
884 with raises(test_case["error_type"]):
885 check_estimator_sparse_tag(estimator.__class__.__name__, estimator)
886
887 # estimator `tag_sparse=accept_sparse=False` fails on sparse data
888 # but does not raise the appropriate error
889 for fit_error in [TypeError("unexpected error"), KeyError("other error")]:
890 estimator = EstimatorWithSparseConfig(False, False, fit_error)
891 with raises(AssertionError):
892 check_estimator_sparse_tag(estimator.__class__.__name__, estimator)
893
894
895def test_check_estimator_transformer_no_mixin():
896 # check that TransformerMixin is not required for transformer tests to run
897 # but it fails since the tag is not set
898 with raises(RuntimeError, "the `transformer_tags` tag is not set"):
899 check_estimator(BadTransformerWithoutMixin())
900
901
902def test_check_estimator_clones():
903 # check that check_estimator doesn't modify the estimator it receives
904
905 iris = load_iris()
906
907 for Estimator in [
908 GaussianMixture,
909 LinearRegression,
910 SGDClassifier,
911 PCA,
912 MiniBatchKMeans,
913 ]:
914 # without fitting
915 with ignore_warnings(category=ConvergenceWarning):
916 est = Estimator()
917 set_random_state(est)
918 old_hash = joblib.hash(est)
919 check_estimator(
920 est, expected_failed_checks=_get_expected_failed_checks(est)
921 )
922 assert old_hash == joblib.hash(est)
923
924 # with fitting
925 with ignore_warnings(category=ConvergenceWarning):
926 est = Estimator()
927 set_random_state(est)
928 est.fit(iris.data, iris.target)
929 old_hash = joblib.hash(est)
930 check_estimator(
931 est, expected_failed_checks=_get_expected_failed_checks(est)
932 )
933 assert old_hash == joblib.hash(est)
934
935
936def test_check_estimators_unfitted():
937 # check that a ValueError/AttributeError is raised when calling predict
938 # on an unfitted estimator
939 msg = "Estimator should raise a NotFittedError when calling"
940 with raises(AssertionError, match=msg):
941 check_estimators_unfitted("estimator", NoSparseClassifier())
942
943 # check that CorrectNotFittedError inherit from either ValueError
944 # or AttributeError
945 check_estimators_unfitted("estimator", CorrectNotFittedErrorClassifier())
946
947
948def test_check_no_attributes_set_in_init():
949 class NonConformantEstimatorPrivateSet(BaseEstimator):
950 def __init__(self):
951 self.you_should_not_set_this_ = None
952
953 class NonConformantEstimatorNoParamSet(BaseEstimator):
954 def __init__(self, you_should_set_this_=None):
955 pass
956
957 class ConformantEstimatorClassAttribute(BaseEstimator):
958 # making sure our __metadata_request__* class attributes are okay!
959 __metadata_request__fit = {"foo": True}
960
961 msg = (
962 "Estimator estimator_name should not set any"
963 " attribute apart from parameters during init."
964 r" Found attributes \['you_should_not_set_this_'\]."
965 )
966 with raises(AssertionError, match=msg):
967 check_no_attributes_set_in_init(
968 "estimator_name", NonConformantEstimatorPrivateSet()
969 )
970
971 msg = (
972 "Estimator estimator_name should store all parameters as an attribute"
973 " during init"
974 )
975 with raises(AttributeError, match=msg):
976 check_no_attributes_set_in_init(
977 "estimator_name", NonConformantEstimatorNoParamSet()
978 )
979
980 # a private class attribute is okay!
981 check_no_attributes_set_in_init(
982 "estimator_name", ConformantEstimatorClassAttribute()
983 )
984 # also check if cloning an estimator which has non-default set requests is
985 # fine. Setting a non-default value via `set_{method}_request` sets the
986 # private _metadata_request instance attribute which is copied in `clone`.
987 with config_context(enable_metadata_routing=True):
988 check_no_attributes_set_in_init(
989 "estimator_name",
990 ConformantEstimatorClassAttribute().set_fit_request(foo=True),
991 )
992
993
994def test_check_estimator_pairwise():
995 # check that check_estimator() works on estimator with _pairwise
996 # kernel or metric
997
998 # test precomputed kernel
999 est = SVC(kernel="precomputed")
1000 check_estimator(est)
1001
1002 # test precomputed metric
1003 est = KNeighborsRegressor(metric="precomputed")
1004 check_estimator(est, expected_failed_checks=_get_expected_failed_checks(est))
1005
1006
1007def test_check_classifier_data_not_an_array():
1008 with raises(AssertionError, match="Not equal to tolerance"):
1009 check_classifier_data_not_an_array(
1010 "estimator_name", EstimatorInconsistentForPandas()
1011 )
1012
1013
1014def test_check_regressor_data_not_an_array():
1015 with raises(AssertionError, match="Not equal to tolerance"):
1016 check_regressor_data_not_an_array(
1017 "estimator_name", EstimatorInconsistentForPandas()
1018 )
1019
1020
1021def test_check_dataframe_column_names_consistency():
1022 err_msg = "Estimator does not have a feature_names_in_"
1023 with raises(ValueError, match=err_msg):
1024 check_dataframe_column_names_consistency("estimator_name", BaseBadClassifier())
1025 check_dataframe_column_names_consistency("estimator_name", PartialFitChecksName())
1026
1027 lr = LogisticRegression()
1028 check_dataframe_column_names_consistency(lr.__class__.__name__, lr)
1029 lr.__doc__ = "Docstring that does not document the estimator's attributes"
1030 err_msg = (
1031 "Estimator LogisticRegression does not document its feature_names_in_ attribute"
1032 )
1033 with raises(ValueError, match=err_msg):
1034 check_dataframe_column_names_consistency(lr.__class__.__name__, lr)
1035
1036
1037class _BaseMultiLabelClassifierMock(ClassifierMixin, BaseEstimator):
1038 def __init__(self, response_output):
1039 self.response_output = response_output
1040
1041 def fit(self, X, y):
1042 return self
1043
1044 def __sklearn_tags__(self):
1045 tags = super().__sklearn_tags__()
1046 tags.classifier_tags.multi_label = True
1047 return tags
1048
1049
1050def test_check_classifiers_multilabel_output_format_predict():
1051 n_samples, test_size, n_outputs = 100, 25, 5
1052 _, y = make_multilabel_classification(
1053 n_samples=n_samples,
1054 n_features=2,
1055 n_classes=n_outputs,
1056 n_labels=3,
1057 length=50,
1058 allow_unlabeled=True,
1059 random_state=0,
1060 )
1061 y_test = y[-test_size:]
1062
1063 class MultiLabelClassifierPredict(_BaseMultiLabelClassifierMock):
1064 def predict(self, X):
1065 return self.response_output
1066
1067 # 1. inconsistent array type
1068 clf = MultiLabelClassifierPredict(response_output=y_test.tolist())
1069 err_msg = (
1070 r"MultiLabelClassifierPredict.predict is expected to output a "
1071 r"NumPy array. Got <class 'list'> instead."
1072 )
1073 with raises(AssertionError, match=err_msg):
1074 check_classifiers_multilabel_output_format_predict(clf.__class__.__name__, clf)
1075 # 2. inconsistent shape
1076 clf = MultiLabelClassifierPredict(response_output=y_test[:, :-1])
1077 err_msg = (
1078 r"MultiLabelClassifierPredict.predict outputs a NumPy array of "
1079 r"shape \(25, 4\) instead of \(25, 5\)."
1080 )
1081 with raises(AssertionError, match=err_msg):
1082 check_classifiers_multilabel_output_format_predict(clf.__class__.__name__, clf)
1083 # 3. inconsistent dtype
1084 clf = MultiLabelClassifierPredict(response_output=y_test.astype(np.float64))
1085 err_msg = (
1086 r"MultiLabelClassifierPredict.predict does not output the same "
1087 r"dtype than the targets."
1088 )
1089 with raises(AssertionError, match=err_msg):
1090 check_classifiers_multilabel_output_format_predict(clf.__class__.__name__, clf)
1091
1092
1093def test_check_classifiers_multilabel_output_format_predict_proba():
1094 n_samples, test_size, n_outputs = 100, 25, 5
1095 _, y = make_multilabel_classification(
1096 n_samples=n_samples,
1097 n_features=2,
1098 n_classes=n_outputs,
1099 n_labels=3,
1100 length=50,
1101 allow_unlabeled=True,
1102 random_state=0,
1103 )
1104 y_test = y[-test_size:]
1105
1106 class MultiLabelClassifierPredictProba(_BaseMultiLabelClassifierMock):
1107 def predict_proba(self, X):
1108 return self.response_output
1109
1110 for csr_container in CSR_CONTAINERS:
1111 # 1. unknown output type
1112 clf = MultiLabelClassifierPredictProba(response_output=csr_container(y_test))
1113 err_msg = (
1114 f"Unknown returned type .*{csr_container.__name__}.* by "
1115 r"MultiLabelClassifierPredictProba.predict_proba. A list or a Numpy "
1116 r"array is expected."
1117 )
1118 with raises(ValueError, match=err_msg):
1119 check_classifiers_multilabel_output_format_predict_proba(
1120 clf.__class__.__name__,
1121 clf,
1122 )
1123 # 2. for list output
1124 # 2.1. inconsistent length
1125 clf = MultiLabelClassifierPredictProba(response_output=y_test.tolist())
1126 err_msg = (
1127 "When MultiLabelClassifierPredictProba.predict_proba returns a list, "
1128 "the list should be of length n_outputs and contain NumPy arrays. Got "
1129 f"length of {test_size} instead of {n_outputs}."
1130 )
1131 with raises(AssertionError, match=err_msg):
1132 check_classifiers_multilabel_output_format_predict_proba(
1133 clf.__class__.__name__,
1134 clf,
1135 )
1136 # 2.2. array of inconsistent shape
1137 response_output = [np.ones_like(y_test) for _ in range(n_outputs)]
1138 clf = MultiLabelClassifierPredictProba(response_output=response_output)
1139 err_msg = (
1140 r"When MultiLabelClassifierPredictProba.predict_proba returns a list, "
1141 r"this list should contain NumPy arrays of shape \(n_samples, 2\). Got "
1142 r"NumPy arrays of shape \(25, 5\) instead of \(25, 2\)."
1143 )
1144 with raises(AssertionError, match=err_msg):
1145 check_classifiers_multilabel_output_format_predict_proba(
1146 clf.__class__.__name__,
1147 clf,
1148 )
1149 # 2.3. array of inconsistent dtype
1150 response_output = [
1151 np.ones(shape=(y_test.shape[0], 2), dtype=np.int64) for _ in range(n_outputs)
1152 ]
1153 clf = MultiLabelClassifierPredictProba(response_output=response_output)
1154 err_msg = (
1155 "When MultiLabelClassifierPredictProba.predict_proba returns a list, "
1156 "it should contain NumPy arrays with floating dtype."
1157 )
1158 with raises(AssertionError, match=err_msg):
1159 check_classifiers_multilabel_output_format_predict_proba(
1160 clf.__class__.__name__,
1161 clf,
1162 )
1163 # 2.4. array does not contain probability (each row should sum to 1)
1164 response_output = [
1165 np.ones(shape=(y_test.shape[0], 2), dtype=np.float64) for _ in range(n_outputs)
1166 ]
1167 clf = MultiLabelClassifierPredictProba(response_output=response_output)
1168 err_msg = (
1169 r"When MultiLabelClassifierPredictProba.predict_proba returns a list, "
1170 r"each NumPy array should contain probabilities for each class and "
1171 r"thus each row should sum to 1"
1172 )
1173 with raises(AssertionError, match=err_msg):
1174 check_classifiers_multilabel_output_format_predict_proba(
1175 clf.__class__.__name__,
1176 clf,
1177 )
1178 # 3 for array output
1179 # 3.1. array of inconsistent shape
1180 clf = MultiLabelClassifierPredictProba(response_output=y_test[:, :-1])
1181 err_msg = (
1182 r"When MultiLabelClassifierPredictProba.predict_proba returns a NumPy "
1183 r"array, the expected shape is \(n_samples, n_outputs\). Got \(25, 4\)"
1184 r" instead of \(25, 5\)."
1185 )
1186 with raises(AssertionError, match=err_msg):
1187 check_classifiers_multilabel_output_format_predict_proba(
1188 clf.__class__.__name__,
1189 clf,
1190 )
1191 # 3.2. array of inconsistent dtype
1192 response_output = np.zeros_like(y_test, dtype=np.int64)
1193 clf = MultiLabelClassifierPredictProba(response_output=response_output)
1194 err_msg = (
1195 r"When MultiLabelClassifierPredictProba.predict_proba returns a NumPy "
1196 r"array, the expected data type is floating."
1197 )
1198 with raises(AssertionError, match=err_msg):
1199 check_classifiers_multilabel_output_format_predict_proba(
1200 clf.__class__.__name__,