Aluode/PerceptionLabPortable
0
1"""Test the search module"""
2
3import pickle
4import re
5import sys
6import warnings
7from collections.abc import Iterable, Sized
8from functools import partial
9from io import StringIO
10from itertools import chain, product
11from types import GeneratorType
12
13import numpy as np
14import pytest
15from scipy.stats import bernoulli, expon, uniform
16
17from sklearn import config_context
18from sklearn.base import BaseEstimator, ClassifierMixin, clone, is_classifier
19from sklearn.cluster import KMeans
20from sklearn.compose import ColumnTransformer
21from sklearn.datasets import (
22 make_blobs,
23 make_classification,
24 make_multilabel_classification,
25)
26from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
27from sklearn.dummy import DummyClassifier
28from sklearn.ensemble import HistGradientBoostingClassifier
29from sklearn.exceptions import FitFailedWarning
30from sklearn.experimental import enable_halving_search_cv # noqa: F401
31from sklearn.feature_extraction.text import TfidfVectorizer
32from sklearn.impute import SimpleImputer
33from sklearn.linear_model import (
34 LinearRegression,
35 LogisticRegression,
36 Ridge,
37 SGDClassifier,
38)
39from sklearn.metrics import (
40 accuracy_score,
41 confusion_matrix,
42 f1_score,
43 make_scorer,
44 r2_score,
45 recall_score,
46 roc_auc_score,
47)
48from sklearn.metrics.pairwise import euclidean_distances
49from sklearn.model_selection import (
50 GridSearchCV,
51 GroupKFold,
52 GroupShuffleSplit,
53 HalvingGridSearchCV,
54 KFold,
55 LeaveOneGroupOut,
56 LeavePGroupsOut,
57 ParameterGrid,
58 ParameterSampler,
59 RandomizedSearchCV,
60 StratifiedKFold,
61 StratifiedShuffleSplit,
62 train_test_split,
63)
64from sklearn.model_selection._search import (
65 BaseSearchCV,
66 _yield_masked_array_for_each_param,
67)
68from sklearn.model_selection.tests.common import OneTimeSplitter
69from sklearn.naive_bayes import ComplementNB
70from sklearn.neighbors import KernelDensity, KNeighborsClassifier, LocalOutlierFactor
71from sklearn.pipeline import Pipeline, make_pipeline
72from sklearn.preprocessing import (
73 OneHotEncoder,
74 OrdinalEncoder,
75 SplineTransformer,
76 StandardScaler,
77)
78from sklearn.svm import SVC, LinearSVC
79from sklearn.tests.metadata_routing_common import (
80 ConsumingScorer,
81 _Registry,
82 check_recorded_metadata,
83)
84from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
85from sklearn.utils._array_api import (
86 _get_namespace_device_dtype_ids,
87 yield_namespace_device_dtype_combinations,
88)
89from sklearn.utils._mocking import CheckingClassifier, MockDataFrame
90from sklearn.utils._testing import (
91 MinimalClassifier,
92 MinimalRegressor,
93 MinimalTransformer,
94 _array_api_for_tests,
95 assert_allclose,
96 assert_allclose_dense_sparse,
97 assert_almost_equal,
98 assert_array_almost_equal,
99 assert_array_equal,
100 set_random_state,
101)
102from sklearn.utils.estimator_checks import _enforce_estimator_tags_y
103from sklearn.utils.fixes import CSR_CONTAINERS
104from sklearn.utils.validation import _num_samples
105
106
107# Neither of the following two estimators inherit from BaseEstimator,
108# to test hyperparameter search on user-defined classifiers.
109class MockClassifier(ClassifierMixin, BaseEstimator):
110 """Dummy classifier to test the parameter search algorithms"""
111
112 def __init__(self, foo_param=0):
113 self.foo_param = foo_param
114
115 def fit(self, X, Y):
116 assert len(X) == len(Y)
117 self.classes_ = np.unique(Y)
118 return self
119
120 def predict(self, T):
121 return T.shape[0]
122
123 def transform(self, X):
124 return X + self.foo_param
125
126 def inverse_transform(self, X):
127 return X - self.foo_param
128
129 predict_proba = predict
130 predict_log_proba = predict
131 decision_function = predict
132
133 def score(self, X=None, Y=None):
134 if self.foo_param > 1:
135 score = 1.0
136 else:
137 score = 0.0
138 return score
139
140 def get_params(self, deep=False):
141 return {"foo_param": self.foo_param}
142
143 def set_params(self, **params):
144 self.foo_param = params["foo_param"]
145 return self
146
147
148class LinearSVCNoScore(LinearSVC):
149 """A LinearSVC classifier that has no score method."""
150
151 @property
152 def score(self):
153 raise AttributeError
154
155
156X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
157y = np.array([1, 1, 2, 2])
158
159
160def assert_grid_iter_equals_getitem(grid):
161 assert list(grid) == [grid[i] for i in range(len(grid))]
162
163
164@pytest.mark.parametrize("klass", [ParameterGrid, partial(ParameterSampler, n_iter=10)])
165@pytest.mark.parametrize(
166 "input, error_type, error_message",
167 [
168 (0, TypeError, r"Parameter .* a dict or a list, got: 0 of type int"),
169 ([{"foo": [0]}, 0], TypeError, r"Parameter .* is not a dict \(0\)"),
170 (
171 {"foo": 0},
172 TypeError,
173 r"Parameter (grid|distribution) for parameter 'foo' (is not|needs to be) "
174 r"(a list or a numpy array|iterable or a distribution).*",
175 ),
176 ],
177)
178def test_validate_parameter_input(klass, input, error_type, error_message):
179 with pytest.raises(error_type, match=error_message):
180 klass(input)
181
182
183def test_parameter_grid():
184 # Test basic properties of ParameterGrid.
185 params1 = {"foo": [1, 2, 3]}
186 grid1 = ParameterGrid(params1)
187 assert isinstance(grid1, Iterable)
188 assert isinstance(grid1, Sized)
189 assert len(grid1) == 3
190 assert_grid_iter_equals_getitem(grid1)
191
192 params2 = {"foo": [4, 2], "bar": ["ham", "spam", "eggs"]}
193 grid2 = ParameterGrid(params2)
194 assert len(grid2) == 6
195
196 # loop to assert we can iterate over the grid multiple times
197 for i in range(2):
198 # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
199 points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
200 assert points == set(
201 ("bar", x, "foo", y) for x, y in product(params2["bar"], params2["foo"])
202 )
203 assert_grid_iter_equals_getitem(grid2)
204
205 # Special case: empty grid (useful to get default estimator settings)
206 empty = ParameterGrid({})
207 assert len(empty) == 1
208 assert list(empty) == [{}]
209 assert_grid_iter_equals_getitem(empty)
210 with pytest.raises(IndexError):
211 empty[1]
212
213 has_empty = ParameterGrid([{"C": [1, 10]}, {}, {"C": [0.5]}])
214 assert len(has_empty) == 4
215 assert list(has_empty) == [{"C": 1}, {"C": 10}, {}, {"C": 0.5}]
216 assert_grid_iter_equals_getitem(has_empty)
217
218
219def test_grid_search():
220 # Test that the best estimator contains the right value for foo_param
221 clf = MockClassifier()
222 grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=2, verbose=3)
223 # make sure it selects the smallest parameter in case of ties
224 old_stdout = sys.stdout
225 sys.stdout = StringIO()
226 grid_search.fit(X, y)
227 sys.stdout = old_stdout
228 assert grid_search.best_estimator_.foo_param == 2
229
230 assert_array_equal(grid_search.cv_results_["param_foo_param"].data, [1, 2, 3])
231
232 # Smoke test the score etc:
233 grid_search.score(X, y)
234 grid_search.predict_proba(X)
235 grid_search.decision_function(X)
236 grid_search.transform(X)
237
238 # Test exception handling on scoring
239 grid_search.scoring = "sklearn"
240 with pytest.raises(ValueError):
241 grid_search.fit(X, y)
242
243
244def test_grid_search_pipeline_steps():
245 # check that parameters that are estimators are cloned before fitting
246 pipe = Pipeline([("regressor", LinearRegression())])
247 param_grid = {"regressor": [LinearRegression(), Ridge()]}
248 grid_search = GridSearchCV(pipe, param_grid, cv=2)
249 grid_search.fit(X, y)
250 regressor_results = grid_search.cv_results_["param_regressor"]
251 assert isinstance(regressor_results[0], LinearRegression)
252 assert isinstance(regressor_results[1], Ridge)
253 assert not hasattr(regressor_results[0], "coef_")
254 assert not hasattr(regressor_results[1], "coef_")
255 assert regressor_results[0] is not grid_search.best_estimator_
256 assert regressor_results[1] is not grid_search.best_estimator_
257 # check that we didn't modify the parameter grid that was passed
258 assert not hasattr(param_grid["regressor"][0], "coef_")
259 assert not hasattr(param_grid["regressor"][1], "coef_")
260
261
262@pytest.mark.parametrize("SearchCV", [GridSearchCV, RandomizedSearchCV])
263def test_SearchCV_with_fit_params(SearchCV):
264 X = np.arange(100).reshape(10, 10)
265 y = np.array([0] * 5 + [1] * 5)
266 clf = CheckingClassifier(expected_fit_params=["spam", "eggs"])
267 searcher = SearchCV(clf, {"foo_param": [1, 2, 3]}, cv=2, error_score="raise")
268
269 # The CheckingClassifier generates an assertion error if
270 # a parameter is missing or has length != len(X).
271 err_msg = r"Expected fit parameter\(s\) \['eggs'\] not seen."
272 with pytest.raises(AssertionError, match=err_msg):
273 searcher.fit(X, y, spam=np.ones(10))
274
275 err_msg = "Fit parameter spam has length 1; expected"
276 with pytest.raises(AssertionError, match=err_msg):
277 searcher.fit(X, y, spam=np.ones(1), eggs=np.zeros(10))
278 searcher.fit(X, y, spam=np.ones(10), eggs=np.zeros(10))
279
280
281def test_grid_search_no_score():
282 # Test grid-search on classifier that has no score function.
283 clf = LinearSVC(random_state=0)
284 X, y = make_blobs(random_state=0, centers=2)
285 Cs = [0.1, 1, 10]
286 clf_no_score = LinearSVCNoScore(random_state=0)
287 grid_search = GridSearchCV(clf, {"C": Cs}, scoring="accuracy")
288 grid_search.fit(X, y)
289
290 grid_search_no_score = GridSearchCV(clf_no_score, {"C": Cs}, scoring="accuracy")
291 # smoketest grid search
292 grid_search_no_score.fit(X, y)
293
294 # check that best params are equal
295 assert grid_search_no_score.best_params_ == grid_search.best_params_
296 # check that we can call score and that it gives the correct result
297 assert grid_search.score(X, y) == grid_search_no_score.score(X, y)
298
299 # giving no scoring function raises an error
300 grid_search_no_score = GridSearchCV(clf_no_score, {"C": Cs})
301 with pytest.raises(TypeError, match="no scoring"):
302 grid_search_no_score.fit([[1]])
303
304
305def test_grid_search_score_method():
306 X, y = make_classification(n_samples=100, n_classes=2, flip_y=0.2, random_state=0)
307 clf = LinearSVC(random_state=0)
308 grid = {"C": [0.1]}
309
310 search_no_scoring = GridSearchCV(clf, grid, scoring=None).fit(X, y)
311 search_accuracy = GridSearchCV(clf, grid, scoring="accuracy").fit(X, y)
312 search_no_score_method_auc = GridSearchCV(
313 LinearSVCNoScore(), grid, scoring="roc_auc"
314 ).fit(X, y)
315 search_auc = GridSearchCV(clf, grid, scoring="roc_auc").fit(X, y)
316
317 # Check warning only occurs in situation where behavior changed:
318 # estimator requires score method to compete with scoring parameter
319 score_no_scoring = search_no_scoring.score(X, y)
320 score_accuracy = search_accuracy.score(X, y)
321 score_no_score_auc = search_no_score_method_auc.score(X, y)
322 score_auc = search_auc.score(X, y)
323
324 # ensure the test is sane
325 assert score_auc < 1.0
326 assert score_accuracy < 1.0
327 assert score_auc != score_accuracy
328
329 assert_almost_equal(score_accuracy, score_no_scoring)
330 assert_almost_equal(score_auc, score_no_score_auc)
331
332
333def test_grid_search_groups():
334 # Check if ValueError (when groups is None) propagates to GridSearchCV
335 # And also check if groups is correctly passed to the cv object
336 rng = np.random.RandomState(0)
337
338 X, y = make_classification(n_samples=15, n_classes=2, random_state=0)
339 groups = rng.randint(0, 3, 15)
340
341 clf = LinearSVC(random_state=0)
342 grid = {"C": [1]}
343
344 group_cvs = [
345 LeaveOneGroupOut(),
346 LeavePGroupsOut(2),
347 GroupKFold(n_splits=3),
348 GroupShuffleSplit(),
349 ]
350 error_msg = "The 'groups' parameter should not be None."
351 for cv in group_cvs:
352 gs = GridSearchCV(clf, grid, cv=cv)
353 with pytest.raises(ValueError, match=error_msg):
354 gs.fit(X, y)
355 gs.fit(X, y, groups=groups)
356
357 non_group_cvs = [StratifiedKFold(), StratifiedShuffleSplit()]
358 for cv in non_group_cvs:
359 gs = GridSearchCV(clf, grid, cv=cv)
360 # Should not raise an error
361 gs.fit(X, y)
362
363
364def test_classes__property():
365 # Test that classes_ property matches best_estimator_.classes_
366 X = np.arange(100).reshape(10, 10)
367 y = np.array([0] * 5 + [1] * 5)
368 Cs = [0.1, 1, 10]
369
370 grid_search = GridSearchCV(LinearSVC(random_state=0), {"C": Cs})
371 grid_search.fit(X, y)
372 assert_array_equal(grid_search.best_estimator_.classes_, grid_search.classes_)
373
374 # Test that regressors do not have a classes_ attribute
375 grid_search = GridSearchCV(Ridge(), {"alpha": [1.0, 2.0]})
376 grid_search.fit(X, y)
377 assert not hasattr(grid_search, "classes_")
378
379 # Test that the grid searcher has no classes_ attribute before it's fit
380 grid_search = GridSearchCV(LinearSVC(random_state=0), {"C": Cs})
381 assert not hasattr(grid_search, "classes_")
382
383 # Test that the grid searcher has no classes_ attribute without a refit
384 grid_search = GridSearchCV(LinearSVC(random_state=0), {"C": Cs}, refit=False)
385 grid_search.fit(X, y)
386 assert not hasattr(grid_search, "classes_")
387
388
389def test_trivial_cv_results_attr():
390 # Test search over a "grid" with only one point.
391 clf = MockClassifier()
392 grid_search = GridSearchCV(clf, {"foo_param": [1]}, cv=2)
393 grid_search.fit(X, y)
394 assert hasattr(grid_search, "cv_results_")
395
396 random_search = RandomizedSearchCV(clf, {"foo_param": [0]}, n_iter=1, cv=2)
397 random_search.fit(X, y)
398 assert hasattr(grid_search, "cv_results_")
399
400
401def test_no_refit():
402 # Test that GSCV can be used for model selection alone without refitting
403 clf = MockClassifier()
404 for scoring in [None, ["accuracy", "precision"]]:
405 grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, refit=False, cv=2)
406 grid_search.fit(X, y)
407 assert (
408 not hasattr(grid_search, "best_estimator_")
409 and hasattr(grid_search, "best_index_")
410 and hasattr(grid_search, "best_params_")
411 )
412
413 # Make sure the functions predict/transform etc. raise meaningful
414 # error messages
415 for fn_name in (
416 "predict",
417 "predict_proba",
418 "predict_log_proba",
419 "transform",
420 "inverse_transform",
421 ):
422 outer_msg = f"has no attribute '{fn_name}'"
423 inner_msg = (
424 f"`refit=False`. {fn_name} is available only after "
425 "refitting on the best parameters"
426 )
427 with pytest.raises(AttributeError, match=outer_msg) as exec_info:
428 getattr(grid_search, fn_name)(X)
429
430 assert isinstance(exec_info.value.__cause__, AttributeError)
431 assert inner_msg in str(exec_info.value.__cause__)
432
433 # Test that an invalid refit param raises appropriate error messages
434 error_msg = (
435 "For multi-metric scoring, the parameter refit must be set to a scorer key"
436 )
437 for refit in [True, "recall", "accuracy"]:
438 with pytest.raises(ValueError, match=error_msg):
439 GridSearchCV(
440 clf, {}, refit=refit, scoring={"acc": "accuracy", "prec": "precision"}
441 ).fit(X, y)
442
443
444def test_grid_search_error():
445 # Test that grid search will capture errors on data with different length
446 X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
447
448 clf = LinearSVC()
449 cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
450 with pytest.raises(ValueError):
451 cv.fit(X_[:180], y_)
452
453
454def test_grid_search_one_grid_point():
455 X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
456 param_dict = {"C": [1.0], "kernel": ["rbf"], "gamma": [0.1]}
457
458 clf = SVC(gamma="auto")
459 cv = GridSearchCV(clf, param_dict)
460 cv.fit(X_, y_)
461
462 clf = SVC(C=1.0, kernel="rbf", gamma=0.1)
463 clf.fit(X_, y_)
464
465 assert_array_equal(clf.dual_coef_, cv.best_estimator_.dual_coef_)
466
467
468def test_grid_search_when_param_grid_includes_range():
469 # Test that the best estimator contains the right value for foo_param
470 clf = MockClassifier()
471 grid_search = None
472 grid_search = GridSearchCV(clf, {"foo_param": range(1, 4)}, cv=2)
473 grid_search.fit(X, y)
474 assert grid_search.best_estimator_.foo_param == 2
475
476
477def test_grid_search_bad_param_grid():
478 X, y = make_classification(n_samples=10, n_features=5, random_state=0)
479 param_dict = {"C": 1}
480 clf = SVC(gamma="auto")
481 error_msg = re.escape(
482 "Parameter grid for parameter 'C' needs to be a list or "
483 "a numpy array, but got 1 (of type int) instead. Single "
484 "values need to be wrapped in a list with one element."
485 )
486 search = GridSearchCV(clf, param_dict)
487 with pytest.raises(TypeError, match=error_msg):
488 search.fit(X, y)
489
490 param_dict = {"C": []}
491 clf = SVC()
492 error_msg = re.escape(
493 "Parameter grid for parameter 'C' need to be a non-empty sequence, got: []"
494 )
495 search = GridSearchCV(clf, param_dict)
496 with pytest.raises(ValueError, match=error_msg):
497 search.fit(X, y)
498
499 param_dict = {"C": "1,2,3"}
500 clf = SVC(gamma="auto")
501 error_msg = re.escape(
502 "Parameter grid for parameter 'C' needs to be a list or a numpy array, "
503 "but got '1,2,3' (of type str) instead. Single values need to be "
504 "wrapped in a list with one element."
505 )
506 search = GridSearchCV(clf, param_dict)
507 with pytest.raises(TypeError, match=error_msg):
508 search.fit(X, y)
509
510 param_dict = {"C": np.ones((3, 2))}
511 clf = SVC()
512 search = GridSearchCV(clf, param_dict)
513 with pytest.raises(ValueError):
514 search.fit(X, y)
515
516
517@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
518def test_grid_search_sparse(csr_container):
519 # Test that grid search works with both dense and sparse matrices
520 X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
521
522 clf = LinearSVC()
523 cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
524 cv.fit(X_[:180], y_[:180])
525 y_pred = cv.predict(X_[180:])
526 C = cv.best_estimator_.C
527
528 X_ = csr_container(X_)
529 clf = LinearSVC()
530 cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
531 cv.fit(X_[:180].tocoo(), y_[:180])
532 y_pred2 = cv.predict(X_[180:])
533 C2 = cv.best_estimator_.C
534
535 assert np.mean(y_pred == y_pred2) >= 0.9
536 assert C == C2
537
538
539@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
540def test_grid_search_sparse_scoring(csr_container):
541 X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
542
543 clf = LinearSVC()
544 cv = GridSearchCV(clf, {"C": [0.1, 1.0]}, scoring="f1")
545 cv.fit(X_[:180], y_[:180])
546 y_pred = cv.predict(X_[180:])
547 C = cv.best_estimator_.C
548
549 X_ = csr_container(X_)
550 clf = LinearSVC()
551 cv = GridSearchCV(clf, {"C": [0.1, 1.0]}, scoring="f1")
552 cv.fit(X_[:180], y_[:180])
553 y_pred2 = cv.predict(X_[180:])
554 C2 = cv.best_estimator_.C
555
556 assert_array_equal(y_pred, y_pred2)
557 assert C == C2
558 # Smoke test the score
559 # np.testing.assert_allclose(f1_score(cv.predict(X_[:180]), y[:180]),
560 # cv.score(X_[:180], y[:180]))
561
562 # test loss where greater is worse
563 def f1_loss(y_true_, y_pred_):
564 return -f1_score(y_true_, y_pred_)
565
566 F1Loss = make_scorer(f1_loss, greater_is_better=False)
567 cv = GridSearchCV(clf, {"C": [0.1, 1.0]}, scoring=F1Loss)
568 cv.fit(X_[:180], y_[:180])
569 y_pred3 = cv.predict(X_[180:])
570 C3 = cv.best_estimator_.C
571
572 assert C == C3
573 assert_array_equal(y_pred, y_pred3)
574
575
576def test_grid_search_precomputed_kernel():
577 # Test that grid search works when the input features are given in the
578 # form of a precomputed kernel matrix
579 X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
580
581 # compute the training kernel matrix corresponding to the linear kernel
582 K_train = np.dot(X_[:180], X_[:180].T)
583 y_train = y_[:180]
584
585 clf = SVC(kernel="precomputed")
586 cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
587 cv.fit(K_train, y_train)
588
589 assert cv.best_score_ >= 0
590
591 # compute the test kernel matrix
592 K_test = np.dot(X_[180:], X_[:180].T)
593 y_test = y_[180:]
594
595 y_pred = cv.predict(K_test)
596
597 assert np.mean(y_pred == y_test) >= 0
598
599 # test error is raised when the precomputed kernel is not array-like
600 # or sparse
601 with pytest.raises(ValueError):
602 cv.fit(K_train.tolist(), y_train)
603
604
605def test_grid_search_precomputed_kernel_error_nonsquare():
606 # Test that grid search returns an error with a non-square precomputed
607 # training kernel matrix
608 K_train = np.zeros((10, 20))
609 y_train = np.ones((10,))
610 clf = SVC(kernel="precomputed")
611 cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
612 with pytest.raises(ValueError):
613 cv.fit(K_train, y_train)
614
615
616class BrokenClassifier(BaseEstimator):
617 """Broken classifier that cannot be fit twice"""
618
619 def __init__(self, parameter=None):
620 self.parameter = parameter
621
622 def fit(self, X, y):
623 assert not hasattr(self, "has_been_fit_")
624 self.has_been_fit_ = True
625
626 def predict(self, X):
627 return np.zeros(X.shape[0])
628
629
630@pytest.mark.filterwarnings("ignore::sklearn.exceptions.UndefinedMetricWarning")
631def test_refit():
632 # Regression test for bug in refitting
633 # Simulates re-fitting a broken estimator; this used to break with
634 # sparse SVMs.
635 X = np.arange(100).reshape(10, 10)
636 y = np.array([0] * 5 + [1] * 5)
637
638 clf = GridSearchCV(
639 BrokenClassifier(), [{"parameter": [0, 1]}], scoring="precision", refit=True
640 )
641 clf.fit(X, y)
642
643
644def test_refit_callable():
645 """
646 Test refit=callable, which adds flexibility in identifying the
647 "best" estimator.
648 """
649
650 def refit_callable(cv_results):
651 """
652 A dummy function tests `refit=callable` interface.
653 Return the index of a model that has the least
654 `mean_test_score`.
655 """
656 # Fit a dummy clf with `refit=True` to get a list of keys in
657 # clf.cv_results_.
658 X, y = make_classification(n_samples=100, n_features=4, random_state=42)
659 clf = GridSearchCV(
660 LinearSVC(random_state=42),
661 {"C": [0.01, 0.1, 1]},
662 scoring="precision",
663 refit=True,
664 )
665 clf.fit(X, y)
666 # Ensure that `best_index_ != 0` for this dummy clf
667 assert clf.best_index_ != 0
668
669 # Assert every key matches those in `cv_results`
670 for key in clf.cv_results_.keys():
671 assert key in cv_results
672
673 return cv_results["mean_test_score"].argmin()
674
675 X, y = make_classification(n_samples=100, n_features=4, random_state=42)
676 clf = GridSearchCV(
677 LinearSVC(random_state=42),
678 {"C": [0.01, 0.1, 1]},
679 scoring="precision",
680 refit=refit_callable,
681 )
682 clf.fit(X, y)
683
684 assert clf.best_index_ == 0
685 # Ensure `best_score_` is disabled when using `refit=callable`
686 assert not hasattr(clf, "best_score_")
687
688
689def test_refit_callable_invalid_type():
690 """
691 Test implementation catches the errors when 'best_index_' returns an
692 invalid result.
693 """
694
695 def refit_callable_invalid_type(cv_results):
696 """
697 A dummy function tests when returned 'best_index_' is not integer.
698 """
699 return None
700
701 X, y = make_classification(n_samples=100, n_features=4, random_state=42)
702
703 clf = GridSearchCV(
704 LinearSVC(random_state=42),
705 {"C": [0.1, 1]},
706 scoring="precision",
707 refit=refit_callable_invalid_type,
708 )
709 with pytest.raises(TypeError, match="best_index_ returned is not an integer"):
710 clf.fit(X, y)
711
712
713@pytest.mark.parametrize("out_bound_value", [-1, 2])
714@pytest.mark.parametrize("search_cv", [RandomizedSearchCV, GridSearchCV])
715def test_refit_callable_out_bound(out_bound_value, search_cv):
716 """
717 Test implementation catches the errors when 'best_index_' returns an
718 out of bound result.
719 """
720
721 def refit_callable_out_bound(cv_results):
722 """
723 A dummy function tests when returned 'best_index_' is out of bounds.
724 """
725 return out_bound_value
726
727 X, y = make_classification(n_samples=100, n_features=4, random_state=42)
728
729 clf = search_cv(
730 LinearSVC(random_state=42),
731 {"C": [0.1, 1]},
732 scoring="precision",
733 refit=refit_callable_out_bound,
734 )
735 with pytest.raises(IndexError, match="best_index_ index out of range"):
736 clf.fit(X, y)
737
738
739def test_refit_callable_multi_metric():
740 """
741 Test refit=callable in multiple metric evaluation setting
742 """
743
744 def refit_callable(cv_results):
745 """
746 A dummy function tests `refit=callable` interface.
747 Return the index of a model that has the least
748 `mean_test_prec`.
749 """
750 assert "mean_test_prec" in cv_results
751 return cv_results["mean_test_prec"].argmin()
752
753 X, y = make_classification(n_samples=100, n_features=4, random_state=42)
754 scoring = {"Accuracy": make_scorer(accuracy_score), "prec": "precision"}
755 clf = GridSearchCV(
756 LinearSVC(random_state=42),
757 {"C": [0.01, 0.1, 1]},
758 scoring=scoring,
759 refit=refit_callable,
760 )
761 clf.fit(X, y)
762
763 assert clf.best_index_ == 0
764 # Ensure `best_score_` is disabled when using `refit=callable`
765 assert not hasattr(clf, "best_score_")
766
767
768def test_gridsearch_nd():
769 # Pass X as list in GridSearchCV
770 X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
771 y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
772
773 def check_X(x):
774 return x.shape[1:] == (5, 3, 2)
775
776 def check_y(x):
777 return x.shape[1:] == (7, 11)
778
779 clf = CheckingClassifier(
780 check_X=check_X,
781 check_y=check_y,
782 methods_to_check=["fit"],
783 )
784 grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]})
785 grid_search.fit(X_4d, y_3d).score(X, y)
786 assert hasattr(grid_search, "cv_results_")
787
788
789def test_X_as_list():
790 # Pass X as list in GridSearchCV
791 X = np.arange(100).reshape(10, 10)
792 y = np.array([0] * 5 + [1] * 5)
793
794 clf = CheckingClassifier(
795 check_X=lambda x: isinstance(x, list),
796 methods_to_check=["fit"],
797 )
798 cv = KFold(n_splits=3)
799 grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=cv)
800 grid_search.fit(X.tolist(), y).score(X, y)
801 assert hasattr(grid_search, "cv_results_")
802
803
804def test_y_as_list():
805 # Pass y as list in GridSearchCV
806 X = np.arange(100).reshape(10, 10)
807 y = np.array([0] * 5 + [1] * 5)
808
809 clf = CheckingClassifier(
810 check_y=lambda x: isinstance(x, list),
811 methods_to_check=["fit"],
812 )
813 cv = KFold(n_splits=3)
814 grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=cv)
815 grid_search.fit(X, y.tolist()).score(X, y)
816 assert hasattr(grid_search, "cv_results_")
817
818
819def test_pandas_input():
820 # check cross_val_score doesn't destroy pandas dataframe
821 types = [(MockDataFrame, MockDataFrame)]
822 try:
823 from pandas import DataFrame, Series
824
825 types.append((DataFrame, Series))
826 except ImportError:
827 pass
828
829 X = np.arange(100).reshape(10, 10)
830 y = np.array([0] * 5 + [1] * 5)
831
832 for InputFeatureType, TargetType in types:
833 # X dataframe, y series
834 X_df, y_ser = InputFeatureType(X), TargetType(y)
835
836 def check_df(x):
837 return isinstance(x, InputFeatureType)
838
839 def check_series(x):
840 return isinstance(x, TargetType)
841
842 clf = CheckingClassifier(check_X=check_df, check_y=check_series)
843
844 grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]})
845 grid_search.fit(X_df, y_ser).score(X_df, y_ser)
846 grid_search.predict(X_df)
847 assert hasattr(grid_search, "cv_results_")
848
849
850def test_unsupervised_grid_search():
851 # test grid-search with unsupervised estimator
852 X, y = make_blobs(n_samples=50, random_state=0)
853 km = KMeans(random_state=0, init="random", n_init=1)
854
855 # Multi-metric evaluation unsupervised
856 scoring = ["adjusted_rand_score", "fowlkes_mallows_score"]
857 for refit in ["adjusted_rand_score", "fowlkes_mallows_score"]:
858 grid_search = GridSearchCV(
859 km, param_grid=dict(n_clusters=[2, 3, 4]), scoring=scoring, refit=refit
860 )
861 grid_search.fit(X, y)
862 # Both ARI and FMS can find the right number :)
863 assert grid_search.best_params_["n_clusters"] == 3
864
865 # Single metric evaluation unsupervised
866 grid_search = GridSearchCV(
867 km, param_grid=dict(n_clusters=[2, 3, 4]), scoring="fowlkes_mallows_score"
868 )
869 grid_search.fit(X, y)
870 assert grid_search.best_params_["n_clusters"] == 3
871
872 # Now without a score, and without y
873 grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]))
874 grid_search.fit(X)
875 assert grid_search.best_params_["n_clusters"] == 4
876
877
878def test_gridsearch_no_predict():
879 # test grid-search with an estimator without predict.
880 # slight duplication of a test from KDE
881 def custom_scoring(estimator, X):
882 return 42 if estimator.bandwidth == 0.1 else 0
883
884 X, _ = make_blobs(cluster_std=0.1, random_state=1, centers=[[0, 1], [1, 0], [0, 0]])
885 search = GridSearchCV(
886 KernelDensity(),
887 param_grid=dict(bandwidth=[0.01, 0.1, 1]),
888 scoring=custom_scoring,
889 )
890 search.fit(X)
891 assert search.best_params_["bandwidth"] == 0.1
892 assert search.best_score_ == 42
893
894
895def test_param_sampler():
896 # test basic properties of param sampler
897 param_distributions = {"kernel": ["rbf", "linear"], "C": uniform(0, 1)}
898 sampler = ParameterSampler(
899 param_distributions=param_distributions, n_iter=10, random_state=0
900 )
901 samples = [x for x in sampler]
902 assert len(samples) == 10
903 for sample in samples:
904 assert sample["kernel"] in ["rbf", "linear"]
905 assert 0 <= sample["C"] <= 1
906
907 # test that repeated calls yield identical parameters
908 param_distributions = {"C": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}
909 sampler = ParameterSampler(
910 param_distributions=param_distributions, n_iter=3, random_state=0
911 )
912 assert [x for x in sampler] == [x for x in sampler]
913
914 param_distributions = {"C": uniform(0, 1)}
915 sampler = ParameterSampler(
916 param_distributions=param_distributions, n_iter=10, random_state=0
917 )
918 assert [x for x in sampler] == [x for x in sampler]
919
920
921def check_cv_results_array_types(
922 search, param_keys, score_keys, expected_cv_results_kinds
923):
924 # Check if the search `cv_results`'s array are of correct types
925 cv_results = search.cv_results_
926 assert all(isinstance(cv_results[param], np.ma.MaskedArray) for param in param_keys)
927 assert {
928 key: cv_results[key].dtype.kind for key in param_keys
929 } == expected_cv_results_kinds
930 assert not any(isinstance(cv_results[key], np.ma.MaskedArray) for key in score_keys)
931 assert all(
932 cv_results[key].dtype == np.float64
933 for key in score_keys
934 if not key.startswith("rank")
935 )
936
937 scorer_keys = search.scorer_.keys() if search.multimetric_ else ["score"]
938
939 for key in scorer_keys:
940 assert cv_results["rank_test_%s" % key].dtype == np.int32
941
942
943def check_cv_results_keys(cv_results, param_keys, score_keys, n_cand, extra_keys=()):
944 # Test the search.cv_results_ contains all the required results
945 all_keys = param_keys + score_keys + extra_keys
946 assert_array_equal(sorted(cv_results.keys()), sorted(all_keys + ("params",)))
947 assert all(cv_results[key].shape == (n_cand,) for key in param_keys + score_keys)
948
949
950def test_grid_search_cv_results():
951 X, y = make_classification(n_samples=50, n_features=4, random_state=42)
952
953 n_grid_points = 6
954 params = [
955 dict(
956 kernel=[
957 "rbf",
958 ],
959 C=[1, 10],
960 gamma=[0.1, 1],
961 ),
962 dict(
963 kernel=[
964 "poly",
965 ],
966 degree=[1, 2],
967 ),
968 ]
969
970 param_keys = ("param_C", "param_degree", "param_gamma", "param_kernel")
971 score_keys = (
972 "mean_test_score",
973 "mean_train_score",
974 "rank_test_score",
975 "split0_test_score",
976 "split1_test_score",
977 "split2_test_score",
978 "split0_train_score",
979 "split1_train_score",
980 "split2_train_score",
981 "std_test_score",
982 "std_train_score",
983 "mean_fit_time",
984 "std_fit_time",
985 "mean_score_time",
986 "std_score_time",
987 )
988 n_candidates = n_grid_points
989
990 search = GridSearchCV(SVC(), cv=3, param_grid=params, return_train_score=True)
991 search.fit(X, y)
992 cv_results = search.cv_results_
993 # Check if score and timing are reasonable
994 assert all(cv_results["rank_test_score"] >= 1)
995 assert (all(cv_results[k] >= 0) for k in score_keys if k != "rank_test_score")
996 assert (
997 all(cv_results[k] <= 1)
998 for k in score_keys
999 if "time" not in k and k != "rank_test_score"
1000 )
1001 # Check cv_results structure
1002 expected_cv_results_kinds = {
1003 "param_C": "i",
1004 "param_degree": "i",
1005 "param_gamma": "f",
1006 "param_kernel": "O",
1007 }
1008 check_cv_results_array_types(
1009 search, param_keys, score_keys, expected_cv_results_kinds
1010 )
1011 check_cv_results_keys(cv_results, param_keys, score_keys, n_candidates)
1012 # Check masking
1013 cv_results = search.cv_results_
1014
1015 poly_results = [
1016 (
1017 cv_results["param_C"].mask[i]
1018 and cv_results["param_gamma"].mask[i]
1019 and not cv_results["param_degree"].mask[i]
1020 )
1021 for i in range(n_candidates)
1022 if cv_results["param_kernel"][i] == "poly"
1023 ]
1024 assert all(poly_results)
1025 assert len(poly_results) == 2
1026
1027 rbf_results = [
1028 (
1029 not cv_results["param_C"].mask[i]
1030 and not cv_results["param_gamma"].mask[i]
1031 and cv_results["param_degree"].mask[i]
1032 )
1033 for i in range(n_candidates)
1034 if cv_results["param_kernel"][i] == "rbf"
1035 ]
1036 assert all(rbf_results)
1037 assert len(rbf_results) == 4
1038
1039
1040def test_random_search_cv_results():
1041 X, y = make_classification(n_samples=50, n_features=4, random_state=42)
1042
1043 n_search_iter = 30
1044
1045 params = [
1046 {"kernel": ["rbf"], "C": expon(scale=10), "gamma": expon(scale=0.1)},
1047 {"kernel": ["poly"], "degree": [2, 3]},
1048 ]
1049 param_keys = ("param_C", "param_degree", "param_gamma", "param_kernel")
1050 score_keys = (
1051 "mean_test_score",
1052 "mean_train_score",
1053 "rank_test_score",
1054 "split0_test_score",
1055 "split1_test_score",
1056 "split2_test_score",
1057 "split0_train_score",
1058 "split1_train_score",
1059 "split2_train_score",
1060 "std_test_score",
1061 "std_train_score",
1062 "mean_fit_time",
1063 "std_fit_time",
1064 "mean_score_time",
1065 "std_score_time",
1066 )
1067 n_candidates = n_search_iter
1068
1069 search = RandomizedSearchCV(
1070 SVC(),
1071 n_iter=n_search_iter,
1072 cv=3,
1073 param_distributions=params,
1074 return_train_score=True,
1075 )
1076 search.fit(X, y)
1077 cv_results = search.cv_results_
1078 # Check results structure
1079 expected_cv_results_kinds = {
1080 "param_C": "f",
1081 "param_degree": "i",
1082 "param_gamma": "f",
1083 "param_kernel": "O",
1084 }
1085 check_cv_results_array_types(
1086 search, param_keys, score_keys, expected_cv_results_kinds
1087 )
1088 check_cv_results_keys(cv_results, param_keys, score_keys, n_candidates)
1089 assert all(
1090 (
1091 cv_results["param_C"].mask[i]
1092 and cv_results["param_gamma"].mask[i]
1093 and not cv_results["param_degree"].mask[i]
1094 )
1095 for i in range(n_candidates)
1096 if cv_results["param_kernel"][i] == "poly"
1097 )
1098 assert all(
1099 (
1100 not cv_results["param_C"].mask[i]
1101 and not cv_results["param_gamma"].mask[i]
1102 and cv_results["param_degree"].mask[i]
1103 )
1104 for i in range(n_candidates)
1105 if cv_results["param_kernel"][i] == "rbf"
1106 )
1107
1108
1109@pytest.mark.parametrize(
1110 "SearchCV, specialized_params",
1111 [
1112 (GridSearchCV, {"param_grid": {"C": [1, 10]}}),
1113 (RandomizedSearchCV, {"param_distributions": {"C": [1, 10]}, "n_iter": 2}),
1114 ],
1115)
1116def test_search_default_iid(SearchCV, specialized_params):
1117 # Test the IID parameter TODO: Clearly this test does something else???
1118 # noise-free simple 2d-data
1119 X, y = make_blobs(
1120 centers=[[0, 0], [1, 0], [0, 1], [1, 1]],
1121 random_state=0,
1122 cluster_std=0.1,
1123 shuffle=False,
1124 n_samples=80,
1125 )
1126 # split dataset into two folds that are not iid
1127 # first one contains data of all 4 blobs, second only from two.
1128 mask = np.ones(X.shape[0], dtype=bool)
1129 mask[np.where(y == 1)[0][::2]] = 0
1130 mask[np.where(y == 2)[0][::2]] = 0
1131 # this leads to perfect classification on one fold and a score of 1/3 on
1132 # the other
1133 # create "cv" for splits
1134 cv = [[mask, ~mask], [~mask, mask]]
1135
1136 common_params = {"estimator": SVC(), "cv": cv, "return_train_score": True}
1137 search = SearchCV(**common_params, **specialized_params)
1138 search.fit(X, y)
1139
1140 test_cv_scores = np.array(
1141 [
1142 search.cv_results_["split%d_test_score" % s][0]
1143 for s in range(search.n_splits_)
1144 ]
1145 )
1146 test_mean = search.cv_results_["mean_test_score"][0]
1147 test_std = search.cv_results_["std_test_score"][0]
1148
1149 train_cv_scores = np.array(
1150 [
1151 search.cv_results_["split%d_train_score" % s][0]
1152 for s in range(search.n_splits_)
1153 ]
1154 )
1155 train_mean = search.cv_results_["mean_train_score"][0]
1156 train_std = search.cv_results_["std_train_score"][0]
1157
1158 assert search.cv_results_["param_C"][0] == 1
1159 # scores are the same as above
1160 assert_allclose(test_cv_scores, [1, 1.0 / 3.0])
1161 assert_allclose(train_cv_scores, [1, 1])
1162 # Unweighted mean/std is used
1163 assert test_mean == pytest.approx(np.mean(test_cv_scores))
1164 assert test_std == pytest.approx(np.std(test_cv_scores))
1165
1166 # For the train scores, we do not take a weighted mean irrespective of
1167 # i.i.d. or not
1168 assert train_mean == pytest.approx(1)
1169 assert train_std == pytest.approx(0)
1170
1171
1172def test_grid_search_cv_results_multimetric():
1173 X, y = make_classification(n_samples=50, n_features=4, random_state=42)
1174
1175 n_splits = 3
1176 params = [
1177 dict(
1178 kernel=[
1179 "rbf",
1180 ],
1181 C=[1, 10],
1182 gamma=[0.1, 1],
1183 ),
1184 dict(
1185 kernel=[
1186 "poly",
1187 ],
1188 degree=[1, 2],
1189 ),
1190 ]
1191
1192 grid_searches = []
1193 for scoring in (
1194 {"accuracy": make_scorer(accuracy_score), "recall": make_scorer(recall_score)},
1195 "accuracy",
1196 "recall",
1197 ):
1198 grid_search = GridSearchCV(
1199 SVC(), cv=n_splits, param_grid=params, scoring=scoring, refit=False
1200 )