Aluode/PerceptionLabPortable
0
1"""Test the split module"""
2
3import re
4import warnings
5from itertools import combinations, combinations_with_replacement, permutations
6
7import numpy as np
8import pytest
9from scipy import stats
10from scipy.sparse import issparse
11from scipy.special import comb
12
13from sklearn import config_context
14from sklearn.datasets import load_digits, make_classification
15from sklearn.dummy import DummyClassifier
16from sklearn.model_selection import (
17 GridSearchCV,
18 GroupKFold,
19 GroupShuffleSplit,
20 KFold,
21 LeaveOneGroupOut,
22 LeaveOneOut,
23 LeavePGroupsOut,
24 LeavePOut,
25 PredefinedSplit,
26 RepeatedKFold,
27 RepeatedStratifiedKFold,
28 ShuffleSplit,
29 StratifiedGroupKFold,
30 StratifiedKFold,
31 StratifiedShuffleSplit,
32 TimeSeriesSplit,
33 check_cv,
34 cross_val_score,
35 train_test_split,
36)
37from sklearn.model_selection._split import (
38 _build_repr,
39 _validate_shuffle_split,
40 _yields_constant_splits,
41)
42from sklearn.svm import SVC
43from sklearn.tests.metadata_routing_common import assert_request_is_empty
44from sklearn.utils._array_api import (
45 _convert_to_numpy,
46 _get_namespace_device_dtype_ids,
47 get_namespace,
48 yield_namespace_device_dtype_combinations,
49)
50from sklearn.utils._array_api import (
51 device as array_api_device,
52)
53from sklearn.utils._mocking import MockDataFrame
54from sklearn.utils._testing import (
55 assert_allclose,
56 assert_array_almost_equal,
57 assert_array_equal,
58 ignore_warnings,
59)
60from sklearn.utils.estimator_checks import (
61 _array_api_for_tests,
62)
63from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS
64from sklearn.utils.validation import _num_samples
65
66NO_GROUP_SPLITTERS = [
67 KFold(),
68 StratifiedKFold(),
69 TimeSeriesSplit(),
70 LeaveOneOut(),
71 LeavePOut(p=2),
72 ShuffleSplit(),
73 StratifiedShuffleSplit(test_size=0.5),
74 PredefinedSplit([1, 1, 2, 2]),
75 RepeatedKFold(),
76 RepeatedStratifiedKFold(),
77]
78
79GROUP_SPLITTERS = [
80 GroupKFold(),
81 LeavePGroupsOut(n_groups=1),
82 StratifiedGroupKFold(),
83 LeaveOneGroupOut(),
84 GroupShuffleSplit(),
85]
86GROUP_SPLITTER_NAMES = set(splitter.__class__.__name__ for splitter in GROUP_SPLITTERS)
87
88ALL_SPLITTERS = NO_GROUP_SPLITTERS + GROUP_SPLITTERS # type: ignore[list-item]
89
90SPLITTERS_REQUIRING_TARGET = [
91 StratifiedKFold(),
92 StratifiedShuffleSplit(),
93 RepeatedStratifiedKFold(),
94]
95
96X = np.ones(10)
97y = np.arange(10) // 2
98test_groups = (
99 np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
100 np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
101 np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
102 np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
103 [1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3],
104 ["1", "1", "1", "1", "2", "2", "2", "3", "3", "3", "3", "3"],
105)
106digits = load_digits()
107
108pytestmark = pytest.mark.filterwarnings(
109 "error:The groups parameter:UserWarning:sklearn.*"
110)
111
112
113def _split(splitter, X, y, groups):
114 if splitter.__class__.__name__ in GROUP_SPLITTER_NAMES:
115 return splitter.split(X, y, groups=groups)
116 else:
117 return splitter.split(X, y)
118
119
120def test_cross_validator_with_default_params():
121 n_samples = 4
122 n_unique_groups = 4
123 n_splits = 2
124 p = 2
125 n_shuffle_splits = 10 # (the default value)
126
127 X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
128 X_1d = np.array([1, 2, 3, 4])
129 y = np.array([1, 1, 2, 2])
130 groups = np.array([1, 2, 3, 4])
131 loo = LeaveOneOut()
132 lpo = LeavePOut(p)
133 kf = KFold(n_splits)
134 skf = StratifiedKFold(n_splits)
135 lolo = LeaveOneGroupOut()
136 lopo = LeavePGroupsOut(p)
137 ss = ShuffleSplit(random_state=0)
138 ps = PredefinedSplit([1, 1, 2, 2]) # n_splits = np of unique folds = 2
139 sgkf = StratifiedGroupKFold(n_splits)
140
141 loo_repr = "LeaveOneOut()"
142 lpo_repr = "LeavePOut(p=2)"
143 kf_repr = "KFold(n_splits=2, random_state=None, shuffle=False)"
144 skf_repr = "StratifiedKFold(n_splits=2, random_state=None, shuffle=False)"
145 lolo_repr = "LeaveOneGroupOut()"
146 lopo_repr = "LeavePGroupsOut(n_groups=2)"
147 ss_repr = (
148 "ShuffleSplit(n_splits=10, random_state=0, test_size=None, train_size=None)"
149 )
150 ps_repr = "PredefinedSplit(test_fold=array([1, 1, 2, 2]))"
151 sgkf_repr = "StratifiedGroupKFold(n_splits=2, random_state=None, shuffle=False)"
152
153 n_splits_expected = [
154 n_samples,
155 comb(n_samples, p),
156 n_splits,
157 n_splits,
158 n_unique_groups,
159 comb(n_unique_groups, p),
160 n_shuffle_splits,
161 2,
162 n_splits,
163 ]
164
165 for i, (cv, cv_repr) in enumerate(
166 zip(
167 [loo, lpo, kf, skf, lolo, lopo, ss, ps, sgkf],
168 [
169 loo_repr,
170 lpo_repr,
171 kf_repr,
172 skf_repr,
173 lolo_repr,
174 lopo_repr,
175 ss_repr,
176 ps_repr,
177 sgkf_repr,
178 ],
179 )
180 ):
181 # Test if get_n_splits works correctly
182 assert n_splits_expected[i] == cv.get_n_splits(X, y, groups)
183
184 # Test if the cross-validator works as expected even if
185 # the data is 1d
186 np.testing.assert_equal(
187 list(_split(cv, X, y, groups)), list(_split(cv, X_1d, y, groups))
188 )
189 # Test that train, test indices returned are integers
190 for train, test in _split(cv, X, y, groups):
191 assert np.asarray(train).dtype.kind == "i"
192 assert np.asarray(test).dtype.kind == "i"
193
194 # Test if the repr works without any errors
195 assert cv_repr == repr(cv)
196
197 # ValueError for get_n_splits methods
198 msg = "The 'X' parameter should not be None."
199 with pytest.raises(ValueError, match=msg):
200 loo.get_n_splits(None, y, groups)
201 with pytest.raises(ValueError, match=msg):
202 lpo.get_n_splits(None, y, groups)
203
204
205def test_2d_y():
206 # smoke test for 2d y and multi-label
207 n_samples = 30
208 rng = np.random.RandomState(1)
209 X = rng.randint(0, 3, size=(n_samples, 2))
210 y = rng.randint(0, 3, size=(n_samples,))
211 y_2d = y.reshape(-1, 1)
212 y_multilabel = rng.randint(0, 2, size=(n_samples, 3))
213 groups = rng.randint(0, 3, size=(n_samples,))
214 splitters = [
215 LeaveOneOut(),
216 LeavePOut(p=2),
217 KFold(),
218 StratifiedKFold(),
219 RepeatedKFold(),
220 RepeatedStratifiedKFold(),
221 StratifiedGroupKFold(),
222 ShuffleSplit(),
223 StratifiedShuffleSplit(test_size=0.5),
224 GroupShuffleSplit(),
225 LeaveOneGroupOut(),
226 LeavePGroupsOut(n_groups=2),
227 GroupKFold(n_splits=3),
228 TimeSeriesSplit(),
229 PredefinedSplit(test_fold=groups),
230 ]
231 for splitter in splitters:
232 list(_split(splitter, X, y, groups=groups))
233 list(_split(splitter, X, y_2d, groups=groups))
234 try:
235 list(_split(splitter, X, y_multilabel, groups=groups))
236 except ValueError as e:
237 allowed_target_types = ("binary", "multiclass")
238 msg = "Supported target types are: {}. Got 'multilabel".format(
239 allowed_target_types
240 )
241 assert msg in str(e)
242
243
244def check_valid_split(train, test, n_samples=None):
245 # Use python sets to get more informative assertion failure messages
246 train, test = set(train), set(test)
247
248 # Train and test split should not overlap
249 assert train.intersection(test) == set()
250
251 if n_samples is not None:
252 # Check that the union of train an test split cover all the indices
253 assert train.union(test) == set(range(n_samples))
254
255
256def check_cv_coverage(cv, X, y, groups, expected_n_splits):
257 n_samples = _num_samples(X)
258 # Check that a all the samples appear at least once in a test fold
259 assert cv.get_n_splits(X, y, groups) == expected_n_splits
260
261 collected_test_samples = set()
262 iterations = 0
263 for train, test in cv.split(X, y, groups):
264 check_valid_split(train, test, n_samples=n_samples)
265 iterations += 1
266 collected_test_samples.update(test)
267
268 # Check that the accumulated test samples cover the whole dataset
269 assert iterations == expected_n_splits
270 if n_samples is not None:
271 assert collected_test_samples == set(range(n_samples))
272
273
274def test_kfold_valueerrors():
275 X1 = np.array([[1, 2], [3, 4], [5, 6]])
276 X2 = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]])
277 # Check that errors are raised if there is not enough samples
278 (ValueError, next, KFold(4).split(X1))
279
280 # Check that a warning is raised if the least populated class has too few
281 # members.
282 y = np.array([3, 3, -1, -1, 3])
283
284 skf_3 = StratifiedKFold(3)
285 with pytest.warns(Warning, match="The least populated class"):
286 next(skf_3.split(X2, y))
287
288 sgkf_3 = StratifiedGroupKFold(3)
289 naive_groups = np.arange(len(y))
290 with pytest.warns(Warning, match="The least populated class"):
291 next(sgkf_3.split(X2, y, naive_groups))
292
293 # Check that despite the warning the folds are still computed even
294 # though all the classes are not necessarily represented at on each
295 # side of the split at each split
296 with warnings.catch_warnings():
297 warnings.simplefilter("ignore")
298 check_cv_coverage(skf_3, X2, y, groups=None, expected_n_splits=3)
299
300 with warnings.catch_warnings():
301 warnings.simplefilter("ignore")
302 check_cv_coverage(sgkf_3, X2, y, groups=naive_groups, expected_n_splits=3)
303
304 # Check that errors are raised if all n_groups for individual
305 # classes are less than n_splits.
306 y = np.array([3, 3, -1, -1, 2])
307
308 with pytest.raises(ValueError):
309 next(skf_3.split(X2, y))
310 with pytest.raises(ValueError):
311 next(sgkf_3.split(X2, y))
312
313 # Error when number of folds is <= 1
314 with pytest.raises(ValueError):
315 KFold(0)
316 with pytest.raises(ValueError):
317 KFold(1)
318 error_string = "k-fold cross-validation requires at least one train/test split"
319 with pytest.raises(ValueError, match=error_string):
320 StratifiedKFold(0)
321 with pytest.raises(ValueError, match=error_string):
322 StratifiedKFold(1)
323 with pytest.raises(ValueError, match=error_string):
324 StratifiedGroupKFold(0)
325 with pytest.raises(ValueError, match=error_string):
326 StratifiedGroupKFold(1)
327
328 # When n_splits is not integer:
329 with pytest.raises(ValueError):
330 KFold(1.5)
331 with pytest.raises(ValueError):
332 KFold(2.0)
333 with pytest.raises(ValueError):
334 StratifiedKFold(1.5)
335 with pytest.raises(ValueError):
336 StratifiedKFold(2.0)
337 with pytest.raises(ValueError):
338 StratifiedGroupKFold(1.5)
339 with pytest.raises(ValueError):
340 StratifiedGroupKFold(2.0)
341
342 # When shuffle is not a bool:
343 with pytest.raises(TypeError):
344 KFold(n_splits=4, shuffle=None)
345
346
347def test_kfold_indices():
348 # Check all indices are returned in the test folds
349 X1 = np.ones(18)
350 kf = KFold(3)
351 check_cv_coverage(kf, X1, y=None, groups=None, expected_n_splits=3)
352
353 # Check all indices are returned in the test folds even when equal-sized
354 # folds are not possible
355 X2 = np.ones(17)
356 kf = KFold(3)
357 check_cv_coverage(kf, X2, y=None, groups=None, expected_n_splits=3)
358
359 # Check if get_n_splits returns the number of folds
360 assert 5 == KFold(5).get_n_splits(X2)
361
362
363def test_kfold_no_shuffle():
364 # Manually check that KFold preserves the data ordering on toy datasets
365 X2 = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
366
367 splits = KFold(2).split(X2[:-1])
368 train, test = next(splits)
369 assert_array_equal(test, [0, 1])
370 assert_array_equal(train, [2, 3])
371
372 train, test = next(splits)
373 assert_array_equal(test, [2, 3])
374 assert_array_equal(train, [0, 1])
375
376 splits = KFold(2).split(X2)
377 train, test = next(splits)
378 assert_array_equal(test, [0, 1, 2])
379 assert_array_equal(train, [3, 4])
380
381 train, test = next(splits)
382 assert_array_equal(test, [3, 4])
383 assert_array_equal(train, [0, 1, 2])
384
385
386def test_stratified_kfold_no_shuffle():
387 # Manually check that StratifiedKFold preserves the data ordering as much
388 # as possible on toy datasets in order to avoid hiding sample dependencies
389 # when possible
390 X, y = np.ones(4), [1, 1, 0, 0]
391 splits = StratifiedKFold(2).split(X, y)
392 train, test = next(splits)
393 assert_array_equal(test, [0, 2])
394 assert_array_equal(train, [1, 3])
395
396 train, test = next(splits)
397 assert_array_equal(test, [1, 3])
398 assert_array_equal(train, [0, 2])
399
400 X, y = np.ones(7), [1, 1, 1, 0, 0, 0, 0]
401 splits = StratifiedKFold(2).split(X, y)
402 train, test = next(splits)
403 assert_array_equal(test, [0, 1, 3, 4])
404 assert_array_equal(train, [2, 5, 6])
405
406 train, test = next(splits)
407 assert_array_equal(test, [2, 5, 6])
408 assert_array_equal(train, [0, 1, 3, 4])
409
410 # Check if get_n_splits returns the number of folds
411 assert 5 == StratifiedKFold(5).get_n_splits(X, y)
412
413 # Make sure string labels are also supported
414 X = np.ones(7)
415 y1 = ["1", "1", "1", "0", "0", "0", "0"]
416 y2 = [1, 1, 1, 0, 0, 0, 0]
417 np.testing.assert_equal(
418 list(StratifiedKFold(2).split(X, y1)), list(StratifiedKFold(2).split(X, y2))
419 )
420
421 # Check equivalence to KFold
422 y = [0, 1, 0, 1, 0, 1, 0, 1]
423 X = np.ones_like(y)
424 np.testing.assert_equal(
425 list(StratifiedKFold(3).split(X, y)), list(KFold(3).split(X, y))
426 )
427
428
429@pytest.mark.parametrize("shuffle", [False, True])
430@pytest.mark.parametrize("k", [4, 5, 6, 7, 8, 9, 10])
431@pytest.mark.parametrize("kfold", [StratifiedKFold, StratifiedGroupKFold])
432def test_stratified_kfold_ratios(k, shuffle, kfold):
433 # Check that stratified kfold preserves class ratios in individual splits
434 # Repeat with shuffling turned off and on
435 n_samples = 1000
436 X = np.ones(n_samples)
437 y = np.array(
438 [4] * int(0.10 * n_samples)
439 + [0] * int(0.89 * n_samples)
440 + [1] * int(0.01 * n_samples)
441 )
442 # ensure perfect stratification with StratifiedGroupKFold
443 groups = np.arange(len(y))
444 distr = np.bincount(y) / len(y)
445
446 test_sizes = []
447 random_state = None if not shuffle else 0
448 skf = kfold(k, random_state=random_state, shuffle=shuffle)
449 for train, test in _split(skf, X, y, groups=groups):
450 assert_allclose(np.bincount(y[train]) / len(train), distr, atol=0.02)
451 assert_allclose(np.bincount(y[test]) / len(test), distr, atol=0.02)
452 test_sizes.append(len(test))
453 assert np.ptp(test_sizes) <= 1
454
455
456@pytest.mark.parametrize("shuffle", [False, True])
457@pytest.mark.parametrize("k", [4, 6, 7])
458@pytest.mark.parametrize("kfold", [StratifiedKFold, StratifiedGroupKFold])
459def test_stratified_kfold_label_invariance(k, shuffle, kfold):
460 # Check that stratified kfold gives the same indices regardless of labels
461 n_samples = 100
462 y = np.array(
463 [2] * int(0.10 * n_samples)
464 + [0] * int(0.89 * n_samples)
465 + [1] * int(0.01 * n_samples)
466 )
467 X = np.ones(len(y))
468 # ensure perfect stratification with StratifiedGroupKFold
469 groups = np.arange(len(y))
470
471 def get_splits(y):
472 random_state = None if not shuffle else 0
473 return [
474 (list(train), list(test))
475 for train, test in _split(
476 kfold(k, random_state=random_state, shuffle=shuffle),
477 X,
478 y,
479 groups=groups,
480 )
481 ]
482
483 splits_base = get_splits(y)
484 for perm in permutations([0, 1, 2]):
485 y_perm = np.take(perm, y)
486 splits_perm = get_splits(y_perm)
487 assert splits_perm == splits_base
488
489
490def test_kfold_balance():
491 # Check that KFold returns folds with balanced sizes
492 for i in range(11, 17):
493 kf = KFold(5).split(X=np.ones(i))
494 sizes = [len(test) for _, test in kf]
495
496 assert (np.max(sizes) - np.min(sizes)) <= 1
497 assert np.sum(sizes) == i
498
499
500@pytest.mark.parametrize("kfold", [StratifiedKFold, StratifiedGroupKFold])
501def test_stratifiedkfold_balance(kfold):
502 # Check that KFold returns folds with balanced sizes (only when
503 # stratification is possible)
504 # Repeat with shuffling turned off and on
505 X = np.ones(17)
506 y = [0] * 3 + [1] * 14
507 # ensure perfect stratification with StratifiedGroupKFold
508 groups = np.arange(len(y))
509
510 for shuffle in (True, False):
511 cv = kfold(3, shuffle=shuffle)
512 for i in range(11, 17):
513 skf = _split(cv, X[:i], y[:i], groups[:i])
514 sizes = [len(test) for _, test in skf]
515
516 assert (np.max(sizes) - np.min(sizes)) <= 1
517 assert np.sum(sizes) == i
518
519
520def test_shuffle_kfold():
521 # Check the indices are shuffled properly
522 kf = KFold(3)
523 kf2 = KFold(3, shuffle=True, random_state=0)
524 kf3 = KFold(3, shuffle=True, random_state=1)
525
526 X = np.ones(300)
527
528 all_folds = np.zeros(300)
529 for (tr1, te1), (tr2, te2), (tr3, te3) in zip(
530 kf.split(X), kf2.split(X), kf3.split(X)
531 ):
532 for tr_a, tr_b in combinations((tr1, tr2, tr3), 2):
533 # Assert that there is no complete overlap
534 assert len(np.intersect1d(tr_a, tr_b)) != len(tr1)
535
536 # Set all test indices in successive iterations of kf2 to 1
537 all_folds[te2] = 1
538
539 # Check that all indices are returned in the different test folds
540 assert sum(all_folds) == 300
541
542
543@pytest.mark.parametrize("kfold", [KFold, StratifiedKFold, StratifiedGroupKFold])
544def test_shuffle_kfold_stratifiedkfold_reproducibility(kfold):
545 X = np.ones(15) # Divisible by 3
546 y = [0] * 7 + [1] * 8
547 groups_1 = np.arange(len(y))
548 X2 = np.ones(16) # Not divisible by 3
549 y2 = [0] * 8 + [1] * 8
550 groups_2 = np.arange(len(y2))
551
552 # Check that when the shuffle is True, multiple split calls produce the
553 # same split when random_state is int
554 kf = kfold(3, shuffle=True, random_state=0)
555
556 np.testing.assert_equal(
557 list(_split(kf, X, y, groups_1)), list(_split(kf, X, y, groups_1))
558 )
559
560 # Check that when the shuffle is True, multiple split calls often
561 # (not always) produce different splits when random_state is
562 # RandomState instance or None
563 kf = kfold(3, shuffle=True, random_state=np.random.RandomState(0))
564 for data in zip((X, X2), (y, y2), (groups_1, groups_2)):
565 # Test if the two splits are different cv
566 for (_, test_a), (_, test_b) in zip(_split(kf, *data), _split(kf, *data)):
567 # cv.split(...) returns an array of tuples, each tuple
568 # consisting of an array with train indices and test indices
569 # Ensure that the splits for data are not same
570 # when random state is not set
571 with pytest.raises(AssertionError):
572 np.testing.assert_array_equal(test_a, test_b)
573
574
575def test_shuffle_stratifiedkfold():
576 # Check that shuffling is happening when requested, and for proper
577 # sample coverage
578 X_40 = np.ones(40)
579 y = [0] * 20 + [1] * 20
580 kf0 = StratifiedKFold(5, shuffle=True, random_state=0)
581 kf1 = StratifiedKFold(5, shuffle=True, random_state=1)
582 for (_, test0), (_, test1) in zip(kf0.split(X_40, y), kf1.split(X_40, y)):
583 assert set(test0) != set(test1)
584 check_cv_coverage(kf0, X_40, y, groups=None, expected_n_splits=5)
585
586 # Ensure that we shuffle each class's samples with different
587 # random_state in StratifiedKFold
588 # See https://github.com/scikit-learn/scikit-learn/pull/13124
589 X = np.arange(10)
590 y = [0] * 5 + [1] * 5
591 kf1 = StratifiedKFold(5, shuffle=True, random_state=0)
592 kf2 = StratifiedKFold(5, shuffle=True, random_state=1)
593 test_set1 = sorted([tuple(s[1]) for s in kf1.split(X, y)])
594 test_set2 = sorted([tuple(s[1]) for s in kf2.split(X, y)])
595 assert test_set1 != test_set2
596
597
598def test_shuffle_groupkfold():
599 # Check that shuffling is happening when requested, and for proper
600 # sample coverage
601 X = np.ones(40)
602 y = [0] * 20 + [1] * 20
603 groups = np.arange(40) // 3
604 gkf0 = GroupKFold(4, shuffle=True, random_state=0)
605 gkf1 = GroupKFold(4, shuffle=True, random_state=1)
606
607 # Check that the groups are shuffled differently
608 test_groups0 = [
609 set(groups[test_idx]) for _, test_idx in gkf0.split(X, None, groups)
610 ]
611 test_groups1 = [
612 set(groups[test_idx]) for _, test_idx in gkf1.split(X, None, groups)
613 ]
614 for g0, g1 in zip(test_groups0, test_groups1):
615 assert g0 != g1, "Test groups should differ with different random states"
616
617 # Check coverage and splits
618 check_cv_coverage(gkf0, X, y, groups, expected_n_splits=4)
619 check_cv_coverage(gkf1, X, y, groups, expected_n_splits=4)
620
621
622def test_kfold_can_detect_dependent_samples_on_digits(): # see #2372
623 # The digits samples are dependent: they are apparently grouped by authors
624 # although we don't have any information on the groups segment locations
625 # for this data. We can highlight this fact by computing k-fold cross-
626 # validation with and without shuffling: we observe that the shuffling case
627 # wrongly makes the IID assumption and is therefore too optimistic: it
628 # estimates a much higher accuracy (around 0.93) than that the non
629 # shuffling variant (around 0.81).
630
631 X, y = digits.data[:600], digits.target[:600]
632 model = SVC(C=10, gamma=0.005)
633
634 n_splits = 3
635
636 cv = KFold(n_splits=n_splits, shuffle=False)
637 mean_score = cross_val_score(model, X, y, cv=cv).mean()
638 assert 0.92 > mean_score
639 assert mean_score > 0.80
640
641 # Shuffling the data artificially breaks the dependency and hides the
642 # overfitting of the model with regards to the writing style of the authors
643 # by yielding a seriously overestimated score:
644
645 cv = KFold(n_splits, shuffle=True, random_state=0)
646 mean_score = cross_val_score(model, X, y, cv=cv).mean()
647 assert mean_score > 0.92
648
649 cv = KFold(n_splits, shuffle=True, random_state=1)
650 mean_score = cross_val_score(model, X, y, cv=cv).mean()
651 assert mean_score > 0.92
652
653 # Similarly, StratifiedKFold should try to shuffle the data as little
654 # as possible (while respecting the balanced class constraints)
655 # and thus be able to detect the dependency by not overestimating
656 # the CV score either. As the digits dataset is approximately balanced
657 # the estimated mean score is close to the score measured with
658 # non-shuffled KFold
659
660 cv = StratifiedKFold(n_splits)
661 mean_score = cross_val_score(model, X, y, cv=cv).mean()
662 assert 0.94 > mean_score
663 assert mean_score > 0.80
664
665
666def test_stratified_group_kfold_trivial():
667 sgkf = StratifiedGroupKFold(n_splits=3)
668 # Trivial example - groups with the same distribution
669 y = np.array([1] * 6 + [0] * 12)
670 X = np.ones_like(y).reshape(-1, 1)
671 groups = np.asarray((1, 2, 3, 4, 5, 6, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6))
672 distr = np.bincount(y) / len(y)
673 test_sizes = []
674 for train, test in sgkf.split(X, y, groups):
675 # check group constraint
676 assert np.intersect1d(groups[train], groups[test]).size == 0
677 # check y distribution
678 assert_allclose(np.bincount(y[train]) / len(train), distr, atol=0.02)
679 assert_allclose(np.bincount(y[test]) / len(test), distr, atol=0.02)
680 test_sizes.append(len(test))
681 assert np.ptp(test_sizes) <= 1
682
683
684def test_stratified_group_kfold_approximate():
685 # Not perfect stratification (even though it is possible) because of
686 # iteration over groups
687 sgkf = StratifiedGroupKFold(n_splits=3)
688 y = np.array([1] * 6 + [0] * 12)
689 X = np.ones_like(y).reshape(-1, 1)
690 groups = np.array([1, 2, 3, 3, 4, 4, 1, 1, 2, 2, 3, 4, 5, 5, 5, 6, 6, 6])
691 expected = np.asarray([[0.833, 0.166], [0.666, 0.333], [0.5, 0.5]])
692 test_sizes = []
693 for (train, test), expect_dist in zip(sgkf.split(X, y, groups), expected):
694 # check group constraint
695 assert np.intersect1d(groups[train], groups[test]).size == 0
696 split_dist = np.bincount(y[test]) / len(test)
697 assert_allclose(split_dist, expect_dist, atol=0.001)
698 test_sizes.append(len(test))
699 assert np.ptp(test_sizes) <= 1
700
701
702@pytest.mark.parametrize(
703 "y, groups, expected",
704 [
705 (
706 np.array([0] * 6 + [1] * 6),
707 np.array([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6]),
708 np.asarray([[0.5, 0.5], [0.5, 0.5], [0.5, 0.5]]),
709 ),
710 (
711 np.array([0] * 9 + [1] * 3),
712 np.array([1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 5, 6]),
713 np.asarray([[0.75, 0.25], [0.75, 0.25], [0.75, 0.25]]),
714 ),
715 ],
716)
717def test_stratified_group_kfold_homogeneous_groups(y, groups, expected):
718 sgkf = StratifiedGroupKFold(n_splits=3)
719 X = np.ones_like(y).reshape(-1, 1)
720 for (train, test), expect_dist in zip(sgkf.split(X, y, groups), expected):
721 # check group constraint
722 assert np.intersect1d(groups[train], groups[test]).size == 0
723 split_dist = np.bincount(y[test]) / len(test)
724 assert_allclose(split_dist, expect_dist, atol=0.001)
725
726
727@pytest.mark.parametrize("cls_distr", [(0.4, 0.6), (0.3, 0.7), (0.2, 0.8), (0.8, 0.2)])
728@pytest.mark.parametrize("n_groups", [5, 30, 70])
729def test_stratified_group_kfold_against_group_kfold(cls_distr, n_groups):
730 # Check that given sufficient amount of samples StratifiedGroupKFold
731 # produces better stratified folds than regular GroupKFold
732 n_splits = 5
733 sgkf = StratifiedGroupKFold(n_splits=n_splits)
734 gkf = GroupKFold(n_splits=n_splits)
735 rng = np.random.RandomState(0)
736 n_points = 1000
737 y = rng.choice(2, size=n_points, p=cls_distr)
738 X = np.ones_like(y).reshape(-1, 1)
739 g = rng.choice(n_groups, n_points)
740 sgkf_folds = sgkf.split(X, y, groups=g)
741 gkf_folds = gkf.split(X, y, groups=g)
742 sgkf_entr = 0
743 gkf_entr = 0
744 for (sgkf_train, sgkf_test), (_, gkf_test) in zip(sgkf_folds, gkf_folds):
745 # check group constraint
746 assert np.intersect1d(g[sgkf_train], g[sgkf_test]).size == 0
747 sgkf_distr = np.bincount(y[sgkf_test]) / len(sgkf_test)
748 gkf_distr = np.bincount(y[gkf_test]) / len(gkf_test)
749 sgkf_entr += stats.entropy(sgkf_distr, qk=cls_distr)
750 gkf_entr += stats.entropy(gkf_distr, qk=cls_distr)
751 sgkf_entr /= n_splits
752 gkf_entr /= n_splits
753 assert sgkf_entr <= gkf_entr
754
755
756def test_shuffle_split():
757 ss1 = ShuffleSplit(test_size=0.2, random_state=0).split(X)
758 ss2 = ShuffleSplit(test_size=2, random_state=0).split(X)
759 ss3 = ShuffleSplit(test_size=np.int32(2), random_state=0).split(X)
760 ss4 = ShuffleSplit(test_size=2, random_state=0).split(X)
761 for t1, t2, t3, t4 in zip(ss1, ss2, ss3, ss4):
762 assert_array_equal(t1[0], t2[0])
763 assert_array_equal(t2[0], t3[0])
764 assert_array_equal(t3[0], t4[0])
765 assert_array_equal(t1[1], t2[1])
766 assert_array_equal(t2[1], t3[1])
767 assert_array_equal(t3[1], t4[1])
768
769
770@pytest.mark.parametrize("split_class", [ShuffleSplit, StratifiedShuffleSplit])
771@pytest.mark.parametrize(
772 "train_size, exp_train, exp_test", [(None, 9, 1), (8, 8, 2), (0.8, 8, 2)]
773)
774def test_shuffle_split_default_test_size(split_class, train_size, exp_train, exp_test):
775 # Check that the default value has the expected behavior, i.e. 0.1 if both
776 # unspecified or complement train_size unless both are specified.
777 X = np.ones(10)
778 y = np.ones(10)
779
780 X_train, X_test = next(split_class(train_size=train_size).split(X, y))
781
782 assert len(X_train) == exp_train
783 assert len(X_test) == exp_test
784
785
786@pytest.mark.parametrize(
787 "train_size, exp_train, exp_test", [(None, 8, 2), (7, 7, 3), (0.7, 7, 3)]
788)
789def test_group_shuffle_split_default_test_size(train_size, exp_train, exp_test):
790 # Check that the default value has the expected behavior, i.e. 0.2 if both
791 # unspecified or complement train_size unless both are specified.
792 X = np.ones(10)
793 y = np.ones(10)
794 groups = range(10)
795
796 X_train, X_test = next(GroupShuffleSplit(train_size=train_size).split(X, y, groups))
797
798 assert len(X_train) == exp_train
799 assert len(X_test) == exp_test
800
801
802def test_stratified_shuffle_split_init():
803 X = np.arange(7)
804 y = np.asarray([0, 1, 1, 1, 2, 2, 2])
805 # Check that error is raised if there is a class with only one sample
806 with pytest.raises(ValueError):
807 next(StratifiedShuffleSplit(3, test_size=0.2).split(X, y))
808
809 # Check that error is raised if the test set size is smaller than n_classes
810 with pytest.raises(ValueError):
811 next(StratifiedShuffleSplit(3, test_size=2).split(X, y))
812 # Check that error is raised if the train set size is smaller than
813 # n_classes
814 with pytest.raises(ValueError):
815 next(StratifiedShuffleSplit(3, test_size=3, train_size=2).split(X, y))
816
817 X = np.arange(9)
818 y = np.asarray([0, 0, 0, 1, 1, 1, 2, 2, 2])
819
820 # Train size or test size too small
821 with pytest.raises(ValueError):
822 next(StratifiedShuffleSplit(train_size=2).split(X, y))
823 with pytest.raises(ValueError):
824 next(StratifiedShuffleSplit(test_size=2).split(X, y))
825
826
827def test_stratified_shuffle_split_respects_test_size():
828 y = np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2])
829 test_size = 5
830 train_size = 10
831 sss = StratifiedShuffleSplit(
832 6, test_size=test_size, train_size=train_size, random_state=0
833 ).split(np.ones(len(y)), y)
834 for train, test in sss:
835 assert len(train) == train_size
836 assert len(test) == test_size
837
838
839def test_stratified_shuffle_split_iter():
840 ys = [
841 np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
842 np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
843 np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2] * 2),
844 np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
845 np.array([-1] * 800 + [1] * 50),
846 np.concatenate([[i] * (100 + i) for i in range(11)]),
847 [1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3],
848 ["1", "1", "1", "1", "2", "2", "2", "3", "3", "3", "3", "3"],
849 ]
850
851 for y in ys:
852 sss = StratifiedShuffleSplit(6, test_size=0.33, random_state=0).split(
853 np.ones(len(y)), y
854 )
855 y = np.asanyarray(y) # To make it indexable for y[train]
856 # this is how test-size is computed internally
857 # in _validate_shuffle_split
858 test_size = np.ceil(0.33 * len(y))
859 train_size = len(y) - test_size
860 for train, test in sss:
861 assert_array_equal(np.unique(y[train]), np.unique(y[test]))
862 # Checks if folds keep classes proportions
863 p_train = np.bincount(np.unique(y[train], return_inverse=True)[1]) / float(
864 len(y[train])
865 )
866 p_test = np.bincount(np.unique(y[test], return_inverse=True)[1]) / float(
867 len(y[test])
868 )
869 assert_array_almost_equal(p_train, p_test, 1)
870 assert len(train) + len(test) == y.size
871 assert len(train) == train_size
872 assert len(test) == test_size
873 assert_array_equal(np.intersect1d(train, test), [])
874
875
876def test_stratified_shuffle_split_even():
877 # Test the StratifiedShuffleSplit, indices are drawn with a
878 # equal chance
879 n_folds = 5
880 n_splits = 1000
881
882 def assert_counts_are_ok(idx_counts, p):
883 # Here we test that the distribution of the counts
884 # per index is close enough to a binomial
885 threshold = 0.05 / n_splits
886 bf = stats.binom(n_splits, p)
887 for count in idx_counts:
888 prob = bf.pmf(count)
889 assert prob > threshold, (
890 "An index is not drawn with chance corresponding to even draws"
891 )
892
893 for n_samples in (6, 22):
894 groups = np.array((n_samples // 2) * [0, 1])
895 splits = StratifiedShuffleSplit(
896 n_splits=n_splits, test_size=1.0 / n_folds, random_state=0
897 )
898
899 train_counts = [0] * n_samples
900 test_counts = [0] * n_samples
901 n_splits_actual = 0
902 for train, test in splits.split(X=np.ones(n_samples), y=groups):
903 n_splits_actual += 1
904 for counter, ids in [(train_counts, train), (test_counts, test)]:
905 for id in ids:
906 counter[id] += 1
907 assert n_splits_actual == n_splits
908
909 n_train, n_test = _validate_shuffle_split(
910 n_samples, test_size=1.0 / n_folds, train_size=1.0 - (1.0 / n_folds)
911 )
912
913 assert len(train) == n_train
914 assert len(test) == n_test
915 assert len(set(train).intersection(test)) == 0
916
917 group_counts = np.unique(groups)
918 assert splits.test_size == 1.0 / n_folds
919 assert n_train + n_test == len(groups)
920 assert len(group_counts) == 2
921 ex_test_p = float(n_test) / n_samples
922 ex_train_p = float(n_train) / n_samples
923
924 assert_counts_are_ok(train_counts, ex_train_p)
925 assert_counts_are_ok(test_counts, ex_test_p)
926
927
928def test_stratified_shuffle_split_overlap_train_test_bug():
929 # See https://github.com/scikit-learn/scikit-learn/issues/6121 for
930 # the original bug report
931 y = [0, 1, 2, 3] * 3 + [4, 5] * 5
932 X = np.ones_like(y)
933
934 sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=0)
935
936 train, test = next(sss.split(X=X, y=y))
937
938 # no overlap
939 assert_array_equal(np.intersect1d(train, test), [])
940
941 # complete partition
942 assert_array_equal(np.union1d(train, test), np.arange(len(y)))
943
944
945def test_stratified_shuffle_split_multilabel():
946 # fix for issue 9037
947 for y in [
948 np.array([[0, 1], [1, 0], [1, 0], [0, 1]]),
949 np.array([[0, 1], [1, 1], [1, 1], [0, 1]]),
950 ]:
951 X = np.ones_like(y)
952 sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=0)
953 train, test = next(sss.split(X=X, y=y))
954 y_train = y[train]
955 y_test = y[test]
956
957 # no overlap
958 assert_array_equal(np.intersect1d(train, test), [])
959
960 # complete partition
961 assert_array_equal(np.union1d(train, test), np.arange(len(y)))
962
963 # correct stratification of entire rows
964 # (by design, here y[:, 0] uniquely determines the entire row of y)
965 expected_ratio = np.mean(y[:, 0])
966 assert expected_ratio == np.mean(y_train[:, 0])
967 assert expected_ratio == np.mean(y_test[:, 0])
968
969
970def test_stratified_shuffle_split_multilabel_many_labels():
971 # fix in PR #9922: for multilabel data with > 1000 labels, str(row)
972 # truncates with an ellipsis for elements in positions 4 through
973 # len(row) - 4, so labels were not being correctly split using the powerset
974 # method for transforming a multilabel problem to a multiclass one; this
975 # test checks that this problem is fixed.
976 row_with_many_zeros = [1, 0, 1] + [0] * 1000 + [1, 0, 1]
977 row_with_many_ones = [1, 0, 1] + [1] * 1000 + [1, 0, 1]
978 y = np.array([row_with_many_zeros] * 10 + [row_with_many_ones] * 100)
979 X = np.ones_like(y)
980
981 sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=0)
982 train, test = next(sss.split(X=X, y=y))
983 y_train = y[train]
984 y_test = y[test]
985
986 # correct stratification of entire rows
987 # (by design, here y[:, 4] uniquely determines the entire row of y)
988 expected_ratio = np.mean(y[:, 4])
989 assert expected_ratio == np.mean(y_train[:, 4])
990 assert expected_ratio == np.mean(y_test[:, 4])
991
992
993def test_predefinedsplit_with_kfold_split():
994 # Check that PredefinedSplit can reproduce a split generated by Kfold.
995 folds = np.full(10, -1.0)
996 kf_train = []
997 kf_test = []
998 for i, (train_ind, test_ind) in enumerate(KFold(5, shuffle=True).split(X)):
999 kf_train.append(train_ind)
1000 kf_test.append(test_ind)
1001 folds[test_ind] = i
1002 ps = PredefinedSplit(folds)
1003 # n_splits is simply the no of unique folds
1004 assert len(np.unique(folds)) == ps.get_n_splits()
1005 ps_train, ps_test = zip(*ps.split())
1006 assert_array_equal(ps_train, kf_train)
1007 assert_array_equal(ps_test, kf_test)
1008
1009
1010def test_group_shuffle_split():
1011 for groups_i in test_groups:
1012 X = y = np.ones(len(groups_i))
1013 n_splits = 6
1014 test_size = 1.0 / 3
1015 slo = GroupShuffleSplit(n_splits, test_size=test_size, random_state=0)
1016
1017 # Make sure the repr works
1018 repr(slo)
1019
1020 # Test that the length is correct
1021 assert slo.get_n_splits(X, y, groups=groups_i) == n_splits
1022
1023 l_unique = np.unique(groups_i)
1024 l = np.asarray(groups_i)
1025
1026 for train, test in slo.split(X, y, groups=groups_i):
1027 # First test: no train group is in the test set and vice versa
1028 l_train_unique = np.unique(l[train])
1029 l_test_unique = np.unique(l[test])
1030 assert not np.any(np.isin(l[train], l_test_unique))
1031 assert not np.any(np.isin(l[test], l_train_unique))
1032
1033 # Second test: train and test add up to all the data
1034 assert l[train].size + l[test].size == l.size
1035
1036 # Third test: train and test are disjoint
1037 assert_array_equal(np.intersect1d(train, test), [])
1038
1039 # Fourth test:
1040 # unique train and test groups are correct, +- 1 for rounding error
1041 assert abs(len(l_test_unique) - round(test_size * len(l_unique))) <= 1
1042 assert (
1043 abs(len(l_train_unique) - round((1.0 - test_size) * len(l_unique))) <= 1
1044 )
1045
1046
1047def test_leave_one_p_group_out():
1048 logo = LeaveOneGroupOut()
1049 lpgo_1 = LeavePGroupsOut(n_groups=1)
1050 lpgo_2 = LeavePGroupsOut(n_groups=2)
1051
1052 # Make sure the repr works
1053 assert repr(logo) == "LeaveOneGroupOut()"
1054 assert repr(lpgo_1) == "LeavePGroupsOut(n_groups=1)"
1055 assert repr(lpgo_2) == "LeavePGroupsOut(n_groups=2)"
1056 assert repr(LeavePGroupsOut(n_groups=3)) == "LeavePGroupsOut(n_groups=3)"
1057
1058 for j, (cv, p_groups_out) in enumerate(((logo, 1), (lpgo_1, 1), (lpgo_2, 2))):
1059 for i, groups_i in enumerate(test_groups):
1060 n_groups = len(np.unique(groups_i))
1061 n_splits = n_groups if p_groups_out == 1 else n_groups * (n_groups - 1) / 2
1062 X = y = np.ones(len(groups_i))
1063
1064 # Test that the length is correct
1065 assert cv.get_n_splits(X, y, groups=groups_i) == n_splits
1066
1067 groups_arr = np.asarray(groups_i)
1068
1069 # Split using the original list / array / list of string groups_i
1070 for train, test in cv.split(X, y, groups=groups_i):
1071 # First test: no train group is in the test set and vice versa
1072 assert_array_equal(
1073 np.intersect1d(groups_arr[train], groups_arr[test]).tolist(), []
1074 )
1075
1076 # Second test: train and test add up to all the data
1077 assert len(train) + len(test) == len(groups_i)
1078
1079 # Third test:
1080 # The number of groups in test must be equal to p_groups_out
1081 assert np.unique(groups_arr[test]).shape[0], p_groups_out
1082
1083 # check get_n_splits() with dummy parameters
1084 assert logo.get_n_splits(None, None, ["a", "b", "c", "b", "c"]) == 3
1085 assert logo.get_n_splits(groups=[1.0, 1.1, 1.0, 1.2]) == 3
1086 assert lpgo_2.get_n_splits(None, None, np.arange(4)) == 6
1087 assert lpgo_1.get_n_splits(groups=np.arange(4)) == 4
1088
1089 # raise ValueError if a `groups` parameter is illegal
1090 with pytest.raises(ValueError):
1091 logo.get_n_splits(None, None, [0.0, np.nan, 0.0])
1092 with pytest.raises(ValueError):
1093 lpgo_2.get_n_splits(None, None, [0.0, np.inf, 0.0])
1094
1095 msg = "The 'groups' parameter should not be None."
1096 with pytest.raises(ValueError, match=msg):
1097 logo.get_n_splits(None, None, None)
1098 with pytest.raises(ValueError, match=msg):
1099 lpgo_1.get_n_splits(None, None, None)
1100
1101
1102def test_leave_group_out_changing_groups():
1103 # Check that LeaveOneGroupOut and LeavePGroupsOut work normally if
1104 # the groups variable is changed before calling split
1105 groups = np.array([0, 1, 2, 1, 1, 2, 0, 0])
1106 X = np.ones(len(groups))
1107 groups_changing = np.array(groups, copy=True)
1108 lolo = LeaveOneGroupOut().split(X, groups=groups)
1109 lolo_changing = LeaveOneGroupOut().split(X, groups=groups)
1110 lplo = LeavePGroupsOut(n_groups=2).split(X, groups=groups)
1111 lplo_changing = LeavePGroupsOut(n_groups=2).split(X, groups=groups)
1112 groups_changing[:] = 0
1113 for llo, llo_changing in [(lolo, lolo_changing), (lplo, lplo_changing)]:
1114 for (train, test), (train_chan, test_chan) in zip(llo, llo_changing):
1115 assert_array_equal(train, train_chan)
1116 assert_array_equal(test, test_chan)
1117
1118 # n_splits = no of 2 (p) group combinations of the unique groups = 3C2 = 3
1119 assert 3 == LeavePGroupsOut(n_groups=2).get_n_splits(X, y=X, groups=groups)
1120 # n_splits = no of unique groups (C(uniq_lbls, 1) = n_unique_groups)
1121 assert 3 == LeaveOneGroupOut().get_n_splits(X, y=X, groups=groups)
1122
1123
1124def test_leave_group_out_order_dependence():
1125 # Check that LeaveOneGroupOut orders the splits according to the index
1126 # of the group left out.
1127 groups = np.array([2, 2, 0, 0, 1, 1])
1128 X = np.ones(len(groups))
1129
1130 splits = iter(LeaveOneGroupOut().split(X, groups=groups))
1131
1132 expected_indices = [
1133 ([0, 1, 4, 5], [2, 3]),
1134 ([0, 1, 2, 3], [4, 5]),
1135 ([2, 3, 4, 5], [0, 1]),
1136 ]
1137
1138 for expected_train, expected_test in expected_indices:
1139 train, test = next(splits)
1140 assert_array_equal(train, expected_train)
1141 assert_array_equal(test, expected_test)
1142
1143
1144def test_leave_one_p_group_out_error_on_fewer_number_of_groups():
1145 X = y = groups = np.ones(0)
1146 msg = re.escape("Found array with 0 sample(s)")
1147 with pytest.raises(ValueError, match=msg):
1148 next(LeaveOneGroupOut().split(X, y, groups))
1149
1150 X = y = groups = np.ones(1)
1151 msg = re.escape(
1152 f"The groups parameter contains fewer than 2 unique groups ({groups})."
1153 " LeaveOneGroupOut expects at least 2."
1154 )
1155 with pytest.raises(ValueError, match=msg):
1156 next(LeaveOneGroupOut().split(X, y, groups))
1157
1158 X = y = groups = np.ones(1)
1159 msg = re.escape(
1160 "The groups parameter contains fewer than (or equal to) n_groups "
1161 f"(3) numbers of unique groups ({groups}). LeavePGroupsOut expects "
1162 "that at least n_groups + 1 (4) unique groups "
1163 "be present"
1164 )
1165 with pytest.raises(ValueError, match=msg):
1166 next(LeavePGroupsOut(n_groups=3).split(X, y, groups))
1167
1168 X = y = groups = np.arange(3)
1169 msg = re.escape(
1170 "The groups parameter contains fewer than (or equal to) n_groups "
1171 f"(3) numbers of unique groups ({groups}). LeavePGroupsOut expects "
1172 "that at least n_groups + 1 (4) unique groups "
1173 "be present"
1174 )
1175 with pytest.raises(ValueError, match=msg):
1176 next(LeavePGroupsOut(n_groups=3).split(X, y, groups))
1177
1178
1179def test_repeated_cv_value_errors():
1180 # n_repeats is not integer or <= 0
1181 for cv in (RepeatedKFold, RepeatedStratifiedKFold):
1182 with pytest.raises(ValueError):
1183 cv(n_repeats=0)
1184 with pytest.raises(ValueError):
1185 cv(n_repeats=1.5)
1186
1187
1188@pytest.mark.parametrize("RepeatedCV", [RepeatedKFold, RepeatedStratifiedKFold])
1189def test_repeated_cv_repr(RepeatedCV):
1190 n_splits, n_repeats = 2, 6
1191 repeated_cv = RepeatedCV(n_splits=n_splits, n_repeats=n_repeats)
1192 repeated_cv_repr = "{}(n_repeats=6, n_splits=2, random_state=None)".format(
1193 repeated_cv.__class__.__name__
1194 )
1195 assert repeated_cv_repr == repr(repeated_cv)
1196
1197
1198def test_repeated_kfold_determinstic_split():
1199 X = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
1200 random_state = 258173307