Aluode/PerceptionLabPortable
0
1"""
2The :mod:`sklearn.model_selection._split` module includes classes and
3functions to split the data based on a preset strategy.
4"""
5
6# Authors: The scikit-learn developers
7# SPDX-License-Identifier: BSD-3-Clause
8
9import numbers
10import warnings
11from abc import ABCMeta, abstractmethod
12from collections import defaultdict
13from collections.abc import Iterable
14from inspect import signature
15from itertools import chain, combinations
16from math import ceil, floor
17
18import numpy as np
19from scipy.special import comb
20
21from ..utils import (
22 _safe_indexing,
23 check_random_state,
24 indexable,
25 metadata_routing,
26)
27from ..utils._array_api import (
28 _convert_to_numpy,
29 ensure_common_namespace_device,
30 get_namespace,
31)
32from ..utils._param_validation import Interval, RealNotInt, validate_params
33from ..utils.extmath import _approximate_mode
34from ..utils.metadata_routing import _MetadataRequester
35from ..utils.multiclass import type_of_target
36from ..utils.validation import _num_samples, check_array, column_or_1d
37
38__all__ = [
39 "BaseCrossValidator",
40 "GroupKFold",
41 "GroupShuffleSplit",
42 "KFold",
43 "LeaveOneGroupOut",
44 "LeaveOneOut",
45 "LeavePGroupsOut",
46 "LeavePOut",
47 "PredefinedSplit",
48 "RepeatedKFold",
49 "RepeatedStratifiedKFold",
50 "ShuffleSplit",
51 "StratifiedGroupKFold",
52 "StratifiedKFold",
53 "StratifiedShuffleSplit",
54 "check_cv",
55 "train_test_split",
56]
57
58
59class _UnsupportedGroupCVMixin:
60 """Mixin for splitters that do not support Groups."""
61
62 def split(self, X, y=None, groups=None):
63 """Generate indices to split data into training and test set.
64
65 Parameters
66 ----------
67 X : array-like of shape (n_samples, n_features)
68 Training data, where `n_samples` is the number of samples
69 and `n_features` is the number of features.
70
71 y : array-like of shape (n_samples,)
72 The target variable for supervised learning problems.
73
74 groups : object
75 Always ignored, exists for compatibility.
76
77 Yields
78 ------
79 train : ndarray
80 The training set indices for that split.
81
82 test : ndarray
83 The testing set indices for that split.
84 """
85 if groups is not None:
86 warnings.warn(
87 f"The groups parameter is ignored by {self.__class__.__name__}",
88 UserWarning,
89 )
90 return super().split(X, y, groups=groups)
91
92
93class GroupsConsumerMixin(_MetadataRequester):
94 """A Mixin to ``groups`` by default.
95
96 This Mixin makes the object to request ``groups`` by default as ``True``.
97
98 .. versionadded:: 1.3
99 """
100
101 __metadata_request__split = {"groups": True}
102
103
104class BaseCrossValidator(_MetadataRequester, metaclass=ABCMeta):
105 """Base class for all cross-validators.
106
107 Implementations must define `_iter_test_masks` or `_iter_test_indices`.
108 """
109
110 # This indicates that by default CV splitters don't have a "groups" kwarg,
111 # unless indicated by inheriting from ``GroupsConsumerMixin``.
112 # This also prevents ``set_split_request`` to be generated for splitters
113 # which don't support ``groups``.
114 __metadata_request__split = {"groups": metadata_routing.UNUSED}
115
116 def split(self, X, y=None, groups=None):
117 """Generate indices to split data into training and test set.
118
119 Parameters
120 ----------
121 X : array-like of shape (n_samples, n_features)
122 Training data, where `n_samples` is the number of samples
123 and `n_features` is the number of features.
124
125 y : array-like of shape (n_samples,)
126 The target variable for supervised learning problems.
127
128 groups : array-like of shape (n_samples,), default=None
129 Group labels for the samples used while splitting the dataset into
130 train/test set.
131
132 Yields
133 ------
134 train : ndarray
135 The training set indices for that split.
136
137 test : ndarray
138 The testing set indices for that split.
139 """
140 X, y, groups = indexable(X, y, groups)
141 indices = np.arange(_num_samples(X))
142 for test_index in self._iter_test_masks(X, y, groups):
143 train_index = indices[np.logical_not(test_index)]
144 test_index = indices[test_index]
145 yield train_index, test_index
146
147 # Since subclasses must implement either _iter_test_masks or
148 # _iter_test_indices, neither can be abstract.
149 def _iter_test_masks(self, X=None, y=None, groups=None):
150 """Generates boolean masks corresponding to test sets.
151
152 By default, delegates to _iter_test_indices(X, y, groups)
153 """
154 for test_index in self._iter_test_indices(X, y, groups):
155 test_mask = np.zeros(_num_samples(X), dtype=bool)
156 test_mask[test_index] = True
157 yield test_mask
158
159 def _iter_test_indices(self, X=None, y=None, groups=None):
160 """Generates integer indices corresponding to test sets."""
161 raise NotImplementedError
162
163 @abstractmethod
164 def get_n_splits(self, X=None, y=None, groups=None):
165 """Returns the number of splitting iterations in the cross-validator."""
166
167 def __repr__(self):
168 return _build_repr(self)
169
170
171class LeaveOneOut(_UnsupportedGroupCVMixin, BaseCrossValidator):
172 """Leave-One-Out cross-validator.
173
174 Provides train/test indices to split data in train/test sets. Each
175 sample is used once as a test set (singleton) while the remaining
176 samples form the training set.
177
178 Note: ``LeaveOneOut()`` is equivalent to ``KFold(n_splits=n)`` and
179 ``LeavePOut(p=1)`` where ``n`` is the number of samples.
180
181 Due to the high number of test sets (which is the same as the
182 number of samples) this cross-validation method can be very costly.
183 For large datasets one should favor :class:`KFold`, :class:`ShuffleSplit`
184 or :class:`StratifiedKFold`.
185
186 Read more in the :ref:`User Guide <leave_one_out>`.
187
188 Examples
189 --------
190 >>> import numpy as np
191 >>> from sklearn.model_selection import LeaveOneOut
192 >>> X = np.array([[1, 2], [3, 4]])
193 >>> y = np.array([1, 2])
194 >>> loo = LeaveOneOut()
195 >>> loo.get_n_splits(X)
196 2
197 >>> print(loo)
198 LeaveOneOut()
199 >>> for i, (train_index, test_index) in enumerate(loo.split(X)):
200 ... print(f"Fold {i}:")
201 ... print(f" Train: index={train_index}")
202 ... print(f" Test: index={test_index}")
203 Fold 0:
204 Train: index=[1]
205 Test: index=[0]
206 Fold 1:
207 Train: index=[0]
208 Test: index=[1]
209
210 See Also
211 --------
212 LeaveOneGroupOut : For splitting the data according to explicit,
213 domain-specific stratification of the dataset.
214 GroupKFold : K-fold iterator variant with non-overlapping groups.
215 """
216
217 def _iter_test_indices(self, X, y=None, groups=None):
218 n_samples = _num_samples(X)
219 if n_samples <= 1:
220 raise ValueError(
221 "Cannot perform LeaveOneOut with n_samples={}.".format(n_samples)
222 )
223 return range(n_samples)
224
225 def get_n_splits(self, X, y=None, groups=None):
226 """Returns the number of splitting iterations in the cross-validator.
227
228 Parameters
229 ----------
230 X : array-like of shape (n_samples, n_features)
231 Training data, where `n_samples` is the number of samples
232 and `n_features` is the number of features.
233
234 y : object
235 Always ignored, exists for compatibility.
236
237 groups : object
238 Always ignored, exists for compatibility.
239
240 Returns
241 -------
242 n_splits : int
243 Returns the number of splitting iterations in the cross-validator.
244 """
245 if X is None:
246 raise ValueError("The 'X' parameter should not be None.")
247 return _num_samples(X)
248
249
250class LeavePOut(_UnsupportedGroupCVMixin, BaseCrossValidator):
251 """Leave-P-Out cross-validator.
252
253 Provides train/test indices to split data in train/test sets. This results
254 in testing on all distinct samples of size p, while the remaining n - p
255 samples form the training set in each iteration.
256
257 Note: ``LeavePOut(p)`` is NOT equivalent to
258 ``KFold(n_splits=n_samples // p)`` which creates non-overlapping test sets.
259
260 Due to the high number of iterations which grows combinatorically with the
261 number of samples this cross-validation method can be very costly. For
262 large datasets one should favor :class:`KFold`, :class:`StratifiedKFold`
263 or :class:`ShuffleSplit`.
264
265 Read more in the :ref:`User Guide <leave_p_out>`.
266
267 Parameters
268 ----------
269 p : int
270 Size of the test sets. Must be strictly less than the number of
271 samples.
272
273 Examples
274 --------
275 >>> import numpy as np
276 >>> from sklearn.model_selection import LeavePOut
277 >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
278 >>> y = np.array([1, 2, 3, 4])
279 >>> lpo = LeavePOut(2)
280 >>> lpo.get_n_splits(X)
281 6
282 >>> print(lpo)
283 LeavePOut(p=2)
284 >>> for i, (train_index, test_index) in enumerate(lpo.split(X)):
285 ... print(f"Fold {i}:")
286 ... print(f" Train: index={train_index}")
287 ... print(f" Test: index={test_index}")
288 Fold 0:
289 Train: index=[2 3]
290 Test: index=[0 1]
291 Fold 1:
292 Train: index=[1 3]
293 Test: index=[0 2]
294 Fold 2:
295 Train: index=[1 2]
296 Test: index=[0 3]
297 Fold 3:
298 Train: index=[0 3]
299 Test: index=[1 2]
300 Fold 4:
301 Train: index=[0 2]
302 Test: index=[1 3]
303 Fold 5:
304 Train: index=[0 1]
305 Test: index=[2 3]
306 """
307
308 def __init__(self, p):
309 self.p = p
310
311 def _iter_test_indices(self, X, y=None, groups=None):
312 n_samples = _num_samples(X)
313 if n_samples <= self.p:
314 raise ValueError(
315 "p={} must be strictly less than the number of samples={}".format(
316 self.p, n_samples
317 )
318 )
319 for combination in combinations(range(n_samples), self.p):
320 yield np.array(combination)
321
322 def get_n_splits(self, X, y=None, groups=None):
323 """Returns the number of splitting iterations in the cross-validator.
324
325 Parameters
326 ----------
327 X : array-like of shape (n_samples, n_features)
328 Training data, where `n_samples` is the number of samples
329 and `n_features` is the number of features.
330
331 y : object
332 Always ignored, exists for compatibility.
333
334 groups : object
335 Always ignored, exists for compatibility.
336 """
337 if X is None:
338 raise ValueError("The 'X' parameter should not be None.")
339 return int(comb(_num_samples(X), self.p, exact=True))
340
341
342class _BaseKFold(BaseCrossValidator, metaclass=ABCMeta):
343 """Base class for K-Fold cross-validators and TimeSeriesSplit."""
344
345 @abstractmethod
346 def __init__(self, n_splits, *, shuffle, random_state):
347 if not isinstance(n_splits, numbers.Integral):
348 raise ValueError(
349 "The number of folds must be of Integral type. "
350 "%s of type %s was passed." % (n_splits, type(n_splits))
351 )
352 n_splits = int(n_splits)
353
354 if n_splits <= 1:
355 raise ValueError(
356 "k-fold cross-validation requires at least one"
357 " train/test split by setting n_splits=2 or more,"
358 " got n_splits={0}.".format(n_splits)
359 )
360
361 if not isinstance(shuffle, bool):
362 raise TypeError("shuffle must be True or False; got {0}".format(shuffle))
363
364 if not shuffle and random_state is not None: # None is the default
365 raise ValueError(
366 (
367 "Setting a random_state has no effect since shuffle is "
368 "False. You should leave "
369 "random_state to its default (None), or set shuffle=True."
370 ),
371 )
372
373 self.n_splits = n_splits
374 self.shuffle = shuffle
375 self.random_state = random_state
376
377 def split(self, X, y=None, groups=None):
378 """Generate indices to split data into training and test set.
379
380 Parameters
381 ----------
382 X : array-like of shape (n_samples, n_features)
383 Training data, where `n_samples` is the number of samples
384 and `n_features` is the number of features.
385
386 y : array-like of shape (n_samples,), default=None
387 The target variable for supervised learning problems.
388
389 groups : array-like of shape (n_samples,), default=None
390 Group labels for the samples used while splitting the dataset into
391 train/test set.
392
393 Yields
394 ------
395 train : ndarray
396 The training set indices for that split.
397
398 test : ndarray
399 The testing set indices for that split.
400 """
401 X, y, groups = indexable(X, y, groups)
402 n_samples = _num_samples(X)
403 if self.n_splits > n_samples:
404 raise ValueError(
405 (
406 "Cannot have number of splits n_splits={0} greater"
407 " than the number of samples: n_samples={1}."
408 ).format(self.n_splits, n_samples)
409 )
410
411 for train, test in super().split(X, y, groups):
412 yield train, test
413
414 def get_n_splits(self, X=None, y=None, groups=None):
415 """Returns the number of splitting iterations in the cross-validator.
416
417 Parameters
418 ----------
419 X : object
420 Always ignored, exists for compatibility.
421
422 y : object
423 Always ignored, exists for compatibility.
424
425 groups : object
426 Always ignored, exists for compatibility.
427
428 Returns
429 -------
430 n_splits : int
431 Returns the number of splitting iterations in the cross-validator.
432 """
433 return self.n_splits
434
435
436class KFold(_UnsupportedGroupCVMixin, _BaseKFold):
437 """K-Fold cross-validator.
438
439 Provides train/test indices to split data in train/test sets. Split
440 dataset into k consecutive folds (without shuffling by default).
441
442 Each fold is then used once as a validation while the k - 1 remaining
443 folds form the training set.
444
445 Read more in the :ref:`User Guide <k_fold>`.
446
447 For visualisation of cross-validation behaviour and
448 comparison between common scikit-learn split methods
449 refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
450
451 Parameters
452 ----------
453 n_splits : int, default=5
454 Number of folds. Must be at least 2.
455
456 .. versionchanged:: 0.22
457 ``n_splits`` default value changed from 3 to 5.
458
459 shuffle : bool, default=False
460 Whether to shuffle the data before splitting into batches.
461 Note that the samples within each split will not be shuffled.
462
463 random_state : int, RandomState instance or None, default=None
464 When `shuffle` is True, `random_state` affects the ordering of the
465 indices, which controls the randomness of each fold. Otherwise, this
466 parameter has no effect.
467 Pass an int for reproducible output across multiple function calls.
468 See :term:`Glossary <random_state>`.
469
470 Examples
471 --------
472 >>> import numpy as np
473 >>> from sklearn.model_selection import KFold
474 >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
475 >>> y = np.array([1, 2, 3, 4])
476 >>> kf = KFold(n_splits=2)
477 >>> kf.get_n_splits(X)
478 2
479 >>> print(kf)
480 KFold(n_splits=2, random_state=None, shuffle=False)
481 >>> for i, (train_index, test_index) in enumerate(kf.split(X)):
482 ... print(f"Fold {i}:")
483 ... print(f" Train: index={train_index}")
484 ... print(f" Test: index={test_index}")
485 Fold 0:
486 Train: index=[2 3]
487 Test: index=[0 1]
488 Fold 1:
489 Train: index=[0 1]
490 Test: index=[2 3]
491
492 Notes
493 -----
494 The first ``n_samples % n_splits`` folds have size
495 ``n_samples // n_splits + 1``, other folds have size
496 ``n_samples // n_splits``, where ``n_samples`` is the number of samples.
497
498 Randomized CV splitters may return different results for each call of
499 split. You can make the results identical by setting `random_state`
500 to an integer.
501
502 See Also
503 --------
504 StratifiedKFold : Takes class information into account to avoid building
505 folds with imbalanced class distributions (for binary or multiclass
506 classification tasks).
507
508 GroupKFold : K-fold iterator variant with non-overlapping groups.
509
510 RepeatedKFold : Repeats K-Fold n times.
511 """
512
513 def __init__(self, n_splits=5, *, shuffle=False, random_state=None):
514 super().__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
515
516 def _iter_test_indices(self, X, y=None, groups=None):
517 n_samples = _num_samples(X)
518 indices = np.arange(n_samples)
519 if self.shuffle:
520 check_random_state(self.random_state).shuffle(indices)
521
522 n_splits = self.n_splits
523 fold_sizes = np.full(n_splits, n_samples // n_splits, dtype=int)
524 fold_sizes[: n_samples % n_splits] += 1
525 current = 0
526 for fold_size in fold_sizes:
527 start, stop = current, current + fold_size
528 yield indices[start:stop]
529 current = stop
530
531
532class GroupKFold(GroupsConsumerMixin, _BaseKFold):
533 """K-fold iterator variant with non-overlapping groups.
534
535 Each group will appear exactly once in the test set across all folds (the
536 number of distinct groups has to be at least equal to the number of folds).
537
538 The folds are approximately balanced in the sense that the number of
539 samples is approximately the same in each test fold when `shuffle` is True.
540
541 Read more in the :ref:`User Guide <group_k_fold>`.
542
543 For visualisation of cross-validation behaviour and
544 comparison between common scikit-learn split methods
545 refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
546
547 Parameters
548 ----------
549 n_splits : int, default=5
550 Number of folds. Must be at least 2.
551
552 .. versionchanged:: 0.22
553 ``n_splits`` default value changed from 3 to 5.
554
555 shuffle : bool, default=False
556 Whether to shuffle the groups before splitting into batches.
557 Note that the samples within each split will not be shuffled.
558
559 .. versionadded:: 1.6
560
561 random_state : int, RandomState instance or None, default=None
562 When `shuffle` is True, `random_state` affects the ordering of the
563 indices, which controls the randomness of each fold. Otherwise, this
564 parameter has no effect.
565 Pass an int for reproducible output across multiple function calls.
566 See :term:`Glossary <random_state>`.
567
568 .. versionadded:: 1.6
569
570 Notes
571 -----
572 Groups appear in an arbitrary order throughout the folds.
573
574 Examples
575 --------
576 >>> import numpy as np
577 >>> from sklearn.model_selection import GroupKFold
578 >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12]])
579 >>> y = np.array([1, 2, 3, 4, 5, 6])
580 >>> groups = np.array([0, 0, 2, 2, 3, 3])
581 >>> group_kfold = GroupKFold(n_splits=2)
582 >>> group_kfold.get_n_splits(X, y, groups)
583 2
584 >>> print(group_kfold)
585 GroupKFold(n_splits=2, random_state=None, shuffle=False)
586 >>> for i, (train_index, test_index) in enumerate(group_kfold.split(X, y, groups)):
587 ... print(f"Fold {i}:")
588 ... print(f" Train: index={train_index}, group={groups[train_index]}")
589 ... print(f" Test: index={test_index}, group={groups[test_index]}")
590 Fold 0:
591 Train: index=[2 3], group=[2 2]
592 Test: index=[0 1 4 5], group=[0 0 3 3]
593 Fold 1:
594 Train: index=[0 1 4 5], group=[0 0 3 3]
595 Test: index=[2 3], group=[2 2]
596
597 See Also
598 --------
599 LeaveOneGroupOut : For splitting the data according to explicit
600 domain-specific stratification of the dataset.
601
602 StratifiedKFold : Takes class information into account to avoid building
603 folds with imbalanced class proportions (for binary or multiclass
604 classification tasks).
605 """
606
607 def __init__(self, n_splits=5, *, shuffle=False, random_state=None):
608 super().__init__(n_splits, shuffle=shuffle, random_state=random_state)
609
610 def _iter_test_indices(self, X, y, groups):
611 if groups is None:
612 raise ValueError("The 'groups' parameter should not be None.")
613 groups = check_array(groups, input_name="groups", ensure_2d=False, dtype=None)
614
615 unique_groups, group_idx = np.unique(groups, return_inverse=True)
616 n_groups = len(unique_groups)
617
618 if self.n_splits > n_groups:
619 raise ValueError(
620 "Cannot have number of splits n_splits=%d greater"
621 " than the number of groups: %d." % (self.n_splits, n_groups)
622 )
623
624 if self.shuffle:
625 # Split and shuffle unique groups across n_splits
626 rng = check_random_state(self.random_state)
627 unique_groups = rng.permutation(unique_groups)
628 split_groups = np.array_split(unique_groups, self.n_splits)
629
630 for test_group_ids in split_groups:
631 test_mask = np.isin(groups, test_group_ids)
632 yield np.where(test_mask)[0]
633
634 else:
635 # Weight groups by their number of occurrences
636 n_samples_per_group = np.bincount(group_idx)
637
638 # Distribute the most frequent groups first
639 indices = np.argsort(n_samples_per_group)[::-1]
640 n_samples_per_group = n_samples_per_group[indices]
641
642 # Total weight of each fold
643 n_samples_per_fold = np.zeros(self.n_splits)
644
645 # Mapping from group index to fold index
646 group_to_fold = np.zeros(len(unique_groups))
647
648 # Distribute samples by adding the largest weight to the lightest fold
649 for group_index, weight in enumerate(n_samples_per_group):
650 lightest_fold = np.argmin(n_samples_per_fold)
651 n_samples_per_fold[lightest_fold] += weight
652 group_to_fold[indices[group_index]] = lightest_fold
653
654 indices = group_to_fold[group_idx]
655
656 for f in range(self.n_splits):
657 yield np.where(indices == f)[0]
658
659 def split(self, X, y=None, groups=None):
660 """Generate indices to split data into training and test set.
661
662 Parameters
663 ----------
664 X : array-like of shape (n_samples, n_features)
665 Training data, where `n_samples` is the number of samples
666 and `n_features` is the number of features.
667
668 y : array-like of shape (n_samples,), default=None
669 The target variable for supervised learning problems.
670
671 groups : array-like of shape (n_samples,)
672 Group labels for the samples used while splitting the dataset into
673 train/test set.
674
675 Yields
676 ------
677 train : ndarray
678 The training set indices for that split.
679
680 test : ndarray
681 The testing set indices for that split.
682 """
683 return super().split(X, y, groups)
684
685
686class StratifiedKFold(_BaseKFold):
687 """Class-wise stratified K-Fold cross-validator.
688
689 Provides train/test indices to split data in train/test sets.
690
691 This cross-validation object is a variation of KFold that returns
692 stratified folds. The folds are made by preserving the percentage of
693 samples for each class in `y` in a binary or multiclass classification
694 setting.
695
696 Read more in the :ref:`User Guide <stratified_k_fold>`.
697
698 For visualisation of cross-validation behaviour and
699 comparison between common scikit-learn split methods
700 refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
701
702 .. note::
703
704 Stratification on the class label solves an engineering problem rather
705 than a statistical one. See :ref:`stratification` for more details.
706
707 Parameters
708 ----------
709 n_splits : int, default=5
710 Number of folds. Must be at least 2.
711
712 .. versionchanged:: 0.22
713 ``n_splits`` default value changed from 3 to 5.
714
715 shuffle : bool, default=False
716 Whether to shuffle each class's samples before splitting into batches.
717 Note that the samples within each split will not be shuffled.
718
719 random_state : int, RandomState instance or None, default=None
720 When `shuffle` is True, `random_state` affects the ordering of the
721 indices, which controls the randomness of each fold for each class.
722 Otherwise, leave `random_state` as `None`.
723 Pass an int for reproducible output across multiple function calls.
724 See :term:`Glossary <random_state>`.
725
726 Examples
727 --------
728 >>> import numpy as np
729 >>> from sklearn.model_selection import StratifiedKFold
730 >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
731 >>> y = np.array([0, 0, 1, 1])
732 >>> skf = StratifiedKFold(n_splits=2)
733 >>> skf.get_n_splits(X, y)
734 2
735 >>> print(skf)
736 StratifiedKFold(n_splits=2, random_state=None, shuffle=False)
737 >>> for i, (train_index, test_index) in enumerate(skf.split(X, y)):
738 ... print(f"Fold {i}:")
739 ... print(f" Train: index={train_index}")
740 ... print(f" Test: index={test_index}")
741 Fold 0:
742 Train: index=[1 3]
743 Test: index=[0 2]
744 Fold 1:
745 Train: index=[0 2]
746 Test: index=[1 3]
747
748 Notes
749 -----
750 The implementation is designed to:
751
752 * Generate test sets such that all contain the same distribution of
753 classes, or as close as possible.
754 * Be invariant to class label: relabelling ``y = ["Happy", "Sad"]`` to
755 ``y = [1, 0]`` should not change the indices generated.
756 * Preserve order dependencies in the dataset ordering, when
757 ``shuffle=False``: all samples from class k in some test set were
758 contiguous in y, or separated in y by samples from classes other than k.
759 * Generate test sets where the smallest and largest differ by at most one
760 sample.
761
762 .. versionchanged:: 0.22
763 The previous implementation did not follow the last constraint.
764
765 See Also
766 --------
767 RepeatedStratifiedKFold : Repeats Stratified K-Fold n times.
768 """
769
770 def __init__(self, n_splits=5, *, shuffle=False, random_state=None):
771 super().__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
772
773 def _make_test_folds(self, X, y=None):
774 rng = check_random_state(self.random_state)
775 # XXX: as of now, cross-validation splitters only operate in NumPy-land
776 # without attempting to leverage array API namespace features. However
777 # they might be fed by array API inputs, e.g. in CV-enabled estimators so
778 # we need the following explicit conversion:
779 xp, is_array_api = get_namespace(y)
780 if is_array_api:
781 y = _convert_to_numpy(y, xp)
782 else:
783 y = np.asarray(y)
784 type_of_target_y = type_of_target(y)
785 allowed_target_types = ("binary", "multiclass")
786 if type_of_target_y not in allowed_target_types:
787 raise ValueError(
788 "Supported target types are: {}. Got {!r} instead.".format(
789 allowed_target_types, type_of_target_y
790 )
791 )
792
793 y = column_or_1d(y)
794
795 _, y_idx, y_inv = np.unique(y, return_index=True, return_inverse=True)
796 # y_inv encodes y according to lexicographic order. We invert y_idx to
797 # map the classes so that they are encoded by order of appearance:
798 # 0 represents the first label appearing in y, 1 the second, etc.
799 _, class_perm = np.unique(y_idx, return_inverse=True)
800 y_encoded = class_perm[y_inv]
801
802 n_classes = len(y_idx)
803 y_counts = np.bincount(y_encoded)
804 min_groups = np.min(y_counts)
805 if np.all(self.n_splits > y_counts):
806 raise ValueError(
807 "n_splits=%d cannot be greater than the"
808 " number of members in each class." % (self.n_splits)
809 )
810 if self.n_splits > min_groups:
811 warnings.warn(
812 "The least populated class in y has only %d"
813 " members, which is less than n_splits=%d."
814 % (min_groups, self.n_splits),
815 UserWarning,
816 )
817
818 # Determine the optimal number of samples from each class in each fold,
819 # using round robin over the sorted y. (This can be done direct from
820 # counts, but that code is unreadable.)
821 y_order = np.sort(y_encoded)
822 allocation = np.asarray(
823 [
824 np.bincount(y_order[i :: self.n_splits], minlength=n_classes)
825 for i in range(self.n_splits)
826 ]
827 )
828
829 # To maintain the data order dependencies as best as possible within
830 # the stratification constraint, we assign samples from each class in
831 # blocks (and then mess that up when shuffle=True).
832 test_folds = np.empty(len(y), dtype="i")
833 for k in range(n_classes):
834 # since the kth column of allocation stores the number of samples
835 # of class k in each test set, this generates blocks of fold
836 # indices corresponding to the allocation for class k.
837 folds_for_class = np.arange(self.n_splits).repeat(allocation[:, k])
838 if self.shuffle:
839 rng.shuffle(folds_for_class)
840 test_folds[y_encoded == k] = folds_for_class
841 return test_folds
842
843 def _iter_test_masks(self, X, y=None, groups=None):
844 test_folds = self._make_test_folds(X, y)
845 for i in range(self.n_splits):
846 yield test_folds == i
847
848 def split(self, X, y, groups=None):
849 """Generate indices to split data into training and test set.
850
851 Parameters
852 ----------
853 X : array-like of shape (n_samples, n_features)
854 Training data, where `n_samples` is the number of samples
855 and `n_features` is the number of features.
856
857 Note that providing ``y`` is sufficient to generate the splits and
858 hence ``np.zeros(n_samples)`` may be used as a placeholder for
859 ``X`` instead of actual training data.
860
861 y : array-like of shape (n_samples,)
862 The target variable for supervised learning problems.
863 Stratification is done based on the y labels.
864
865 groups : object
866 Always ignored, exists for compatibility.
867
868 Yields
869 ------
870 train : ndarray
871 The training set indices for that split.
872
873 test : ndarray
874 The testing set indices for that split.
875
876 Notes
877 -----
878 Randomized CV splitters may return different results for each call of
879 split. You can make the results identical by setting `random_state`
880 to an integer.
881 """
882 if groups is not None:
883 warnings.warn(
884 f"The groups parameter is ignored by {self.__class__.__name__}",
885 UserWarning,
886 )
887 y = check_array(y, input_name="y", ensure_2d=False, dtype=None)
888 return super().split(X, y, groups)
889
890
891class StratifiedGroupKFold(GroupsConsumerMixin, _BaseKFold):
892 """Class-wise stratified K-Fold iterator variant with non-overlapping groups.
893
894 This cross-validation object is a variation of StratifiedKFold attempts to
895 return stratified folds with non-overlapping groups. The folds are made by
896 preserving the percentage of samples for each class in `y` in a binary or
897 multiclass classification setting.
898
899 Each group will appear exactly once in the test set across all folds (the
900 number of distinct groups has to be at least equal to the number of folds).
901
902 The difference between :class:`GroupKFold`
903 and `StratifiedGroupKFold` is that
904 the former attempts to create balanced folds such that the number of
905 distinct groups is approximately the same in each fold, whereas
906 `StratifiedGroupKFold` attempts to create folds which preserve the
907 percentage of samples for each class as much as possible given the
908 constraint of non-overlapping groups between splits.
909
910 Read more in the :ref:`User Guide <stratified_group_k_fold>`.
911
912 For visualisation of cross-validation behaviour and
913 comparison between common scikit-learn split methods
914 refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
915
916 .. note::
917
918 Stratification on the class label solves an engineering problem rather
919 than a statistical one. See :ref:`stratification` for more details.
920
921 Parameters
922 ----------
923 n_splits : int, default=5
924 Number of folds. Must be at least 2.
925
926 shuffle : bool, default=False
927 Whether to shuffle each class's samples before splitting into batches.
928 Note that the samples within each split will not be shuffled.
929 This implementation can only shuffle groups that have approximately the
930 same y distribution, no global shuffle will be performed.
931
932 random_state : int or RandomState instance, default=None
933 When `shuffle` is True, `random_state` affects the ordering of the
934 indices, which controls the randomness of each fold for each class.
935 Otherwise, leave `random_state` as `None`.
936 Pass an int for reproducible output across multiple function calls.
937 See :term:`Glossary <random_state>`.
938
939 Examples
940 --------
941 >>> import numpy as np
942 >>> from sklearn.model_selection import StratifiedGroupKFold
943 >>> X = np.ones((17, 2))
944 >>> y = np.array([0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
945 >>> groups = np.array([1, 1, 2, 2, 3, 3, 3, 4, 5, 5, 5, 5, 6, 6, 7, 8, 8])
946 >>> sgkf = StratifiedGroupKFold(n_splits=3)
947 >>> sgkf.get_n_splits(X, y)
948 3
949 >>> print(sgkf)
950 StratifiedGroupKFold(n_splits=3, random_state=None, shuffle=False)
951 >>> for i, (train_index, test_index) in enumerate(sgkf.split(X, y, groups)):
952 ... print(f"Fold {i}:")
953 ... print(f" Train: index={train_index}")
954 ... print(f" group={groups[train_index]}")
955 ... print(f" Test: index={test_index}")
956 ... print(f" group={groups[test_index]}")
957 Fold 0:
958 Train: index=[ 0 1 2 3 7 8 9 10 11 15 16]
959 group=[1 1 2 2 4 5 5 5 5 8 8]
960 Test: index=[ 4 5 6 12 13 14]
961 group=[3 3 3 6 6 7]
962 Fold 1:
963 Train: index=[ 4 5 6 7 8 9 10 11 12 13 14]
964 group=[3 3 3 4 5 5 5 5 6 6 7]
965 Test: index=[ 0 1 2 3 15 16]
966 group=[1 1 2 2 8 8]
967 Fold 2:
968 Train: index=[ 0 1 2 3 4 5 6 12 13 14 15 16]
969 group=[1 1 2 2 3 3 3 6 6 7 8 8]
970 Test: index=[ 7 8 9 10 11]
971 group=[4 5 5 5 5]
972
973 Notes
974 -----
975 The implementation is designed to:
976
977 * Mimic the behavior of StratifiedKFold as much as possible for trivial
978 groups (e.g. when each group contains only one sample).
979 * Be invariant to class label: relabelling ``y = ["Happy", "Sad"]`` to
980 ``y = [1, 0]`` should not change the indices generated.
981 * Stratify based on samples as much as possible while keeping
982 non-overlapping groups constraint. That means that in some cases when
983 there is a small number of groups containing a large number of samples
984 the stratification will not be possible and the behavior will be close
985 to GroupKFold.
986
987 See also
988 --------
989 StratifiedKFold: Takes class information into account to build folds which
990 retain class distributions (for binary or multiclass classification
991 tasks).
992
993 GroupKFold: K-fold iterator variant with non-overlapping groups.
994 """
995
996 def __init__(self, n_splits=5, shuffle=False, random_state=None):
997 super().__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
998
999 def _iter_test_indices(self, X, y, groups):
1000 # Implementation is based on this kaggle kernel:
1001 # https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation
1002 # and is a subject to Apache 2.0 License. You may obtain a copy of the
1003 # License at http://www.apache.org/licenses/LICENSE-2.0
1004 # Changelist:
1005 # - Refactored function to a class following scikit-learn KFold
1006 # interface.
1007 # - Added heuristic for assigning group to the least populated fold in
1008 # cases when all other criteria are equal
1009 # - Swtch from using python ``Counter`` to ``np.unique`` to get class
1010 # distribution
1011 # - Added scikit-learn checks for input: checking that target is binary
1012 # or multiclass, checking passed random state, checking that number
1013 # of splits is less than number of members in each class, checking
1014 # that least populated class has more members than there are splits.
1015 rng = check_random_state(self.random_state)
1016 y = np.asarray(y)
1017 type_of_target_y = type_of_target(y)
1018 allowed_target_types = ("binary", "multiclass")
1019 if type_of_target_y not in allowed_target_types:
1020 raise ValueError(
1021 "Supported target types are: {}. Got {!r} instead.".format(
1022 allowed_target_types, type_of_target_y
1023 )
1024 )
1025
1026 y = column_or_1d(y)
1027 _, y_inv, y_cnt = np.unique(y, return_inverse=True, return_counts=True)
1028 if np.all(self.n_splits > y_cnt):
1029 raise ValueError(
1030 "n_splits=%d cannot be greater than the"
1031 " number of members in each class." % (self.n_splits)
1032 )
1033 n_smallest_class = np.min(y_cnt)
1034 if self.n_splits > n_smallest_class:
1035 warnings.warn(
1036 "The least populated class in y has only %d"
1037 " members, which is less than n_splits=%d."
1038 % (n_smallest_class, self.n_splits),
1039 UserWarning,
1040 )
1041 n_classes = len(y_cnt)
1042
1043 _, groups_inv, groups_cnt = np.unique(
1044 groups, return_inverse=True, return_counts=True
1045 )
1046 y_counts_per_group = np.zeros((len(groups_cnt), n_classes))
1047 for class_idx, group_idx in zip(y_inv, groups_inv):
1048 y_counts_per_group[group_idx, class_idx] += 1
1049
1050 y_counts_per_fold = np.zeros((self.n_splits, n_classes))
1051 groups_per_fold = defaultdict(set)
1052
1053 if self.shuffle:
1054 rng.shuffle(y_counts_per_group)
1055
1056 # Stable sort to keep shuffled order for groups with the same
1057 # class distribution variance
1058 sorted_groups_idx = np.argsort(
1059 -np.std(y_counts_per_group, axis=1), kind="mergesort"
1060 )
1061
1062 for group_idx in sorted_groups_idx:
1063 group_y_counts = y_counts_per_group[group_idx]
1064 best_fold = self._find_best_fold(
1065 y_counts_per_fold=y_counts_per_fold,
1066 y_cnt=y_cnt,
1067 group_y_counts=group_y_counts,
1068 )
1069 y_counts_per_fold[best_fold] += group_y_counts
1070 groups_per_fold[best_fold].add(group_idx)
1071
1072 for i in range(self.n_splits):
1073 test_indices = [
1074 idx
1075 for idx, group_idx in enumerate(groups_inv)
1076 if group_idx in groups_per_fold[i]
1077 ]
1078 yield test_indices
1079
1080 def _find_best_fold(self, y_counts_per_fold, y_cnt, group_y_counts):
1081 best_fold = None
1082 min_eval = np.inf
1083 min_samples_in_fold = np.inf
1084 for i in range(self.n_splits):
1085 y_counts_per_fold[i] += group_y_counts
1086 # Summarise the distribution over classes in each proposed fold
1087 std_per_class = np.std(y_counts_per_fold / y_cnt.reshape(1, -1), axis=0)
1088 y_counts_per_fold[i] -= group_y_counts
1089 fold_eval = np.mean(std_per_class)
1090 samples_in_fold = np.sum(y_counts_per_fold[i])
1091 is_current_fold_better = fold_eval < min_eval or (
1092 np.isclose(fold_eval, min_eval)
1093 and samples_in_fold < min_samples_in_fold
1094 )
1095 if is_current_fold_better:
1096 min_eval = fold_eval
1097 min_samples_in_fold = samples_in_fold
1098 best_fold = i
1099 return best_fold
1100
1101
1102class TimeSeriesSplit(_BaseKFold):
1103 """Time Series cross-validator.
1104
1105 Provides train/test indices to split time-ordered data, where other
1106 cross-validation methods are inappropriate, as they would lead to training
1107 on future data and evaluating on past data.
1108 To ensure comparable metrics across folds, samples must be equally spaced.
1109 Once this condition is met, each test set covers the same time duration,
1110 while the train set size accumulates data from previous splits.
1111
1112 This cross-validation object is a variation of :class:`KFold`.
1113 In the k-th split, it returns the first k folds as the train set and the
1114 (k+1)-th fold as the test set.
1115
1116 Note that, unlike standard cross-validation methods, successive
1117 training sets are supersets of those that come before them.
1118
1119 Read more in the :ref:`User Guide <time_series_split>`.
1120
1121 For visualisation of cross-validation behaviour and
1122 comparison between common scikit-learn split methods
1123 refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
1124
1125 .. versionadded:: 0.18
1126
1127 Parameters
1128 ----------
1129 n_splits : int, default=5
1130 Number of splits. Must be at least 2.
1131
1132 .. versionchanged:: 0.22
1133 ``n_splits`` default value changed from 3 to 5.
1134
1135 max_train_size : int, default=None
1136 Maximum size for a single training set.
1137
1138 test_size : int, default=None
1139 Used to limit the size of the test set. Defaults to
1140 ``n_samples // (n_splits + 1)``, which is the maximum allowed value
1141 with ``gap=0``.
1142
1143 .. versionadded:: 0.24
1144
1145 gap : int, default=0
1146 Number of samples to exclude from the end of each train set before
1147 the test set.
1148
1149 .. versionadded:: 0.24
1150
1151 Examples
1152 --------
1153 >>> import numpy as np
1154 >>> from sklearn.model_selection import TimeSeriesSplit
1155 >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4], [1, 2], [3, 4]])
1156 >>> y = np.array([1, 2, 3, 4, 5, 6])
1157 >>> tscv = TimeSeriesSplit()
1158 >>> print(tscv)
1159 TimeSeriesSplit(gap=0, max_train_size=None, n_splits=5, test_size=None)
1160 >>> for i, (train_index, test_index) in enumerate(tscv.split(X)):
1161 ... print(f"Fold {i}:")
1162 ... print(f" Train: index={train_index}")
1163 ... print(f" Test: index={test_index}")
1164 Fold 0:
1165 Train: index=[0]
1166 Test: index=[1]
1167 Fold 1:
1168 Train: index=[0 1]
1169 Test: index=[2]
1170 Fold 2:
1171 Train: index=[0 1 2]
1172 Test: index=[3]
1173 Fold 3:
1174 Train: index=[0 1 2 3]
1175 Test: index=[4]
1176 Fold 4:
1177 Train: index=[0 1 2 3 4]
1178 Test: index=[5]
1179 >>> # Fix test_size to 2 with 12 samples
1180 >>> X = np.random.randn(12, 2)
1181 >>> y = np.random.randint(0, 2, 12)
1182 >>> tscv = TimeSeriesSplit(n_splits=3, test_size=2)
1183 >>> for i, (train_index, test_index) in enumerate(tscv.split(X)):
1184 ... print(f"Fold {i}:")
1185 ... print(f" Train: index={train_index}")
1186 ... print(f" Test: index={test_index}")
1187 Fold 0:
1188 Train: index=[0 1 2 3 4 5]
1189 Test: index=[6 7]
1190 Fold 1:
1191 Train: index=[0 1 2 3 4 5 6 7]
1192 Test: index=[8 9]
1193 Fold 2:
1194 Train: index=[0 1 2 3 4 5 6 7 8 9]
1195 Test: index=[10 11]
1196 >>> # Add in a 2 period gap
1197 >>> tscv = TimeSeriesSplit(n_splits=3, test_size=2, gap=2)
1198 >>> for i, (train_index, test_index) in enumerate(tscv.split(X)):
1199 ... print(f"Fold {i}:")
1200 ... print(f" Train: index={train_index}")