Aluode/PerceptionLabPortable
0
1"""Tests for input validation functions"""
2
3import numbers
4import re
5import warnings
6from itertools import product
7from operator import itemgetter
8from tempfile import NamedTemporaryFile
9
10import numpy as np
11import pytest
12import scipy.sparse as sp
13from pytest import importorskip
14
15import sklearn
16from sklearn._config import config_context
17from sklearn._min_dependencies import dependent_packages
18from sklearn.base import BaseEstimator
19from sklearn.datasets import make_blobs
20from sklearn.ensemble import RandomForestRegressor
21from sklearn.exceptions import NotFittedError, PositiveSpectrumWarning
22from sklearn.linear_model import ARDRegression
23
24# TODO: add this estimator into the _mocking module in a further refactoring
25from sklearn.metrics.tests.test_score_objects import EstimatorWithFit
26from sklearn.neighbors import KNeighborsClassifier
27from sklearn.random_projection import _sparse_random_matrix
28from sklearn.svm import SVR
29from sklearn.utils import (
30 _safe_indexing,
31 as_float_array,
32 check_array,
33 check_symmetric,
34 check_X_y,
35 deprecated,
36)
37from sklearn.utils._array_api import (
38 _get_namespace_device_dtype_ids,
39 yield_namespace_device_dtype_combinations,
40)
41from sklearn.utils._mocking import (
42 MockDataFrame,
43 _MockEstimatorOnOffPrediction,
44)
45from sklearn.utils._testing import (
46 SkipTest,
47 TempMemmap,
48 _array_api_for_tests,
49 _convert_container,
50 assert_allclose,
51 assert_allclose_dense_sparse,
52 assert_array_equal,
53 create_memmap_backed_data,
54 skip_if_array_api_compat_not_configured,
55)
56from sklearn.utils.estimator_checks import _NotAnArray
57from sklearn.utils.fixes import (
58 COO_CONTAINERS,
59 CSC_CONTAINERS,
60 CSR_CONTAINERS,
61 DIA_CONTAINERS,
62 DOK_CONTAINERS,
63)
64from sklearn.utils.validation import (
65 FLOAT_DTYPES,
66 _allclose_dense_sparse,
67 _check_feature_names_in,
68 _check_method_params,
69 _check_psd_eigenvalues,
70 _check_response_method,
71 _check_sample_weight,
72 _check_y,
73 _deprecate_positional_args,
74 _estimator_has,
75 _get_feature_names,
76 _is_fitted,
77 _is_pandas_df,
78 _is_polars_df,
79 _num_features,
80 _num_samples,
81 _to_object_array,
82 assert_all_finite,
83 check_consistent_length,
84 check_is_fitted,
85 check_memory,
86 check_non_negative,
87 check_random_state,
88 check_scalar,
89 column_or_1d,
90 has_fit_parameter,
91 validate_data,
92)
93
94
95def test_make_rng():
96 # Check the check_random_state utility function behavior
97 assert check_random_state(None) is np.random.mtrand._rand
98 assert check_random_state(np.random) is np.random.mtrand._rand
99
100 rng_42 = np.random.RandomState(42)
101 assert check_random_state(42).randint(100) == rng_42.randint(100)
102
103 rng_42 = np.random.RandomState(42)
104 assert check_random_state(rng_42) is rng_42
105
106 rng_42 = np.random.RandomState(42)
107 assert check_random_state(43).randint(100) != rng_42.randint(100)
108
109 with pytest.raises(ValueError):
110 check_random_state("some invalid seed")
111
112
113def test_as_float_array():
114 # Test function for as_float_array
115 X = np.ones((3, 10), dtype=np.int32)
116 X = X + np.arange(10, dtype=np.int32)
117 X2 = as_float_array(X, copy=False)
118 assert X2.dtype == np.float32
119 # Another test
120 X = X.astype(np.int64)
121 X2 = as_float_array(X, copy=True)
122 # Checking that the array wasn't overwritten
123 assert as_float_array(X, copy=False) is not X
124 assert X2.dtype == np.float64
125 # Test int dtypes <= 32bit
126 tested_dtypes = [bool, np.int8, np.int16, np.int32, np.uint8, np.uint16, np.uint32]
127 for dtype in tested_dtypes:
128 X = X.astype(dtype)
129 X2 = as_float_array(X)
130 assert X2.dtype == np.float32
131
132 # Test object dtype
133 X = X.astype(object)
134 X2 = as_float_array(X, copy=True)
135 assert X2.dtype == np.float64
136
137 # Here, X is of the right type, it shouldn't be modified
138 X = np.ones((3, 2), dtype=np.float32)
139 assert as_float_array(X, copy=False) is X
140 # Test that if X is fortran ordered it stays
141 X = np.asfortranarray(X)
142 assert np.isfortran(as_float_array(X, copy=True))
143
144 # Test the copy parameter with some matrices
145 matrices = [
146 sp.csc_matrix(np.arange(5)).toarray(),
147 _sparse_random_matrix(10, 10, density=0.10).toarray(),
148 ]
149 for M in matrices:
150 N = as_float_array(M, copy=True)
151 N[0, 0] = np.nan
152 assert not np.isnan(M).any()
153
154
155@pytest.mark.parametrize(
156 "X", [np.random.random((10, 2)), sp.random(10, 2, format="csr")]
157)
158def test_as_float_array_nan(X):
159 X[5, 0] = np.nan
160 X[6, 1] = np.nan
161 X_converted = as_float_array(X, ensure_all_finite="allow-nan")
162 assert_allclose_dense_sparse(X_converted, X)
163
164
165def test_np_matrix():
166 # Confirm that input validation code does not return np.matrix
167 X = np.arange(12).reshape(3, 4)
168
169 assert not isinstance(as_float_array(X), np.matrix)
170 assert not isinstance(as_float_array(sp.csc_matrix(X)), np.matrix)
171
172
173def test_memmap():
174 # Confirm that input validation code doesn't copy memory mapped arrays
175
176 asflt = lambda x: as_float_array(x, copy=False)
177
178 with NamedTemporaryFile(prefix="sklearn-test") as tmp:
179 M = np.memmap(tmp, shape=(10, 10), dtype=np.float32)
180 M[:] = 0
181
182 for f in (check_array, np.asarray, asflt):
183 X = f(M)
184 X[:] = 1
185 assert_array_equal(X.ravel(), M.ravel())
186 X[:] = 0
187
188
189def test_ordering():
190 # Check that ordering is enforced correctly by validation utilities.
191 # We need to check each validation utility, because a 'copy' without
192 # 'order=K' will kill the ordering.
193 X = np.ones((10, 5))
194 for A in X, X.T:
195 for copy in (True, False):
196 B = check_array(A, order="C", copy=copy)
197 assert B.flags["C_CONTIGUOUS"]
198 B = check_array(A, order="F", copy=copy)
199 assert B.flags["F_CONTIGUOUS"]
200 if copy:
201 assert A is not B
202
203 X = sp.csr_matrix(X)
204 X.data = X.data[::-1]
205 assert not X.data.flags["C_CONTIGUOUS"]
206
207
208@pytest.mark.parametrize(
209 "value, ensure_all_finite",
210 [(np.inf, False), (np.nan, "allow-nan"), (np.nan, False)],
211)
212@pytest.mark.parametrize("retype", [np.asarray, sp.csr_matrix])
213def test_check_array_ensure_all_finite_valid(value, ensure_all_finite, retype):
214 X = retype(np.arange(4).reshape(2, 2).astype(float))
215 X[0, 0] = value
216 X_checked = check_array(X, ensure_all_finite=ensure_all_finite, accept_sparse=True)
217 assert_allclose_dense_sparse(X, X_checked)
218
219
220@pytest.mark.parametrize(
221 "value, input_name, ensure_all_finite, match_msg",
222 [
223 (np.inf, "", True, "Input contains infinity"),
224 (np.inf, "X", True, "Input X contains infinity"),
225 (np.inf, "sample_weight", True, "Input sample_weight contains infinity"),
226 (np.inf, "X", "allow-nan", "Input X contains infinity"),
227 (np.nan, "", True, "Input contains NaN"),
228 (np.nan, "X", True, "Input X contains NaN"),
229 (np.nan, "y", True, "Input y contains NaN"),
230 (
231 np.nan,
232 "",
233 "allow-inf",
234 "ensure_all_finite should be a bool or 'allow-nan'",
235 ),
236 (np.nan, "", 1, "Input contains NaN"),
237 ],
238)
239@pytest.mark.parametrize("retype", [np.asarray, sp.csr_matrix])
240def test_check_array_ensure_all_finite_invalid(
241 value, input_name, ensure_all_finite, match_msg, retype
242):
243 X = retype(np.arange(4).reshape(2, 2).astype(np.float64))
244 X[0, 0] = value
245 with pytest.raises(ValueError, match=match_msg):
246 check_array(
247 X,
248 input_name=input_name,
249 ensure_all_finite=ensure_all_finite,
250 accept_sparse=True,
251 )
252
253
254@pytest.mark.parametrize("input_name", ["X", "y", "sample_weight"])
255@pytest.mark.parametrize("retype", [np.asarray, sp.csr_matrix])
256def test_check_array_links_to_imputer_doc_only_for_X(input_name, retype):
257 data = retype(np.arange(4).reshape(2, 2).astype(np.float64))
258 data[0, 0] = np.nan
259 estimator = SVR()
260 extended_msg = (
261 f"\n{estimator.__class__.__name__} does not accept missing values"
262 " encoded as NaN natively. For supervised learning, you might want"
263 " to consider sklearn.ensemble.HistGradientBoostingClassifier and Regressor"
264 " which accept missing values encoded as NaNs natively."
265 " Alternatively, it is possible to preprocess the"
266 " data, for instance by using an imputer transformer in a pipeline"
267 " or drop samples with missing values. See"
268 " https://scikit-learn.org/stable/modules/impute.html"
269 " You can find a list of all estimators that handle NaN values"
270 " at the following page:"
271 " https://scikit-learn.org/stable/modules/impute.html"
272 "#estimators-that-handle-nan-values"
273 )
274
275 with pytest.raises(ValueError, match=f"Input {input_name} contains NaN") as ctx:
276 check_array(
277 data,
278 estimator=estimator,
279 input_name=input_name,
280 accept_sparse=True,
281 )
282
283 if input_name == "X":
284 assert extended_msg in ctx.value.args[0]
285 else:
286 assert extended_msg not in ctx.value.args[0]
287
288 if input_name == "X":
289 # Veriy that _validate_data is automatically called with the right argument
290 # to generate the same exception:
291 with pytest.raises(ValueError, match=f"Input {input_name} contains NaN") as ctx:
292 SVR().fit(data, np.ones(data.shape[0]))
293 assert extended_msg in ctx.value.args[0]
294
295
296def test_check_array_ensure_all_finite_object():
297 X = np.array([["a", "b", np.nan]], dtype=object).T
298
299 X_checked = check_array(X, dtype=None, ensure_all_finite="allow-nan")
300 assert X is X_checked
301
302 X_checked = check_array(X, dtype=None, ensure_all_finite=False)
303 assert X is X_checked
304
305 with pytest.raises(ValueError, match="Input contains NaN"):
306 check_array(X, dtype=None, ensure_all_finite=True)
307
308
309@pytest.mark.parametrize(
310 "X, err_msg",
311 [
312 (
313 np.array([[1, np.nan]]),
314 "Input contains NaN.",
315 ),
316 (
317 np.array([[1, np.nan]]),
318 "Input contains NaN.",
319 ),
320 (
321 np.array([[1, np.inf]]),
322 "Input contains infinity or a value too large for.*int",
323 ),
324 (np.array([[1, np.nan]], dtype=object), "cannot convert float NaN to integer"),
325 ],
326)
327@pytest.mark.parametrize("ensure_all_finite", [True, False])
328def test_check_array_ensure_all_finite_object_unsafe_casting(
329 X, err_msg, ensure_all_finite
330):
331 # casting a float array containing NaN or inf to int dtype should
332 # raise an error irrespective of the ensure_all_finite parameter.
333 with pytest.raises(ValueError, match=err_msg):
334 check_array(X, dtype=int, ensure_all_finite=ensure_all_finite)
335
336
337def test_check_array_series_err_msg():
338 """
339 Check that we raise a proper error message when passing a Series and we expect a
340 2-dimensional container.
341
342 Non-regression test for:
343 https://github.com/scikit-learn/scikit-learn/issues/27498
344 """
345 pd = pytest.importorskip("pandas")
346 ser = pd.Series([1, 2, 3])
347 msg = f"Expected a 2-dimensional container but got {type(ser)} instead."
348 with pytest.raises(ValueError, match=msg):
349 check_array(ser, ensure_2d=True)
350
351
352@pytest.mark.filterwarnings("ignore:Can't check dok sparse matrix for nan or inf")
353def test_check_array():
354 # accept_sparse == False
355 # raise error on sparse inputs
356 X = [[1, 2], [3, 4]]
357 X_csr = sp.csr_matrix(X)
358 with pytest.raises(TypeError):
359 check_array(X_csr)
360
361 # ensure_2d=False
362 X_array = check_array([0, 1, 2], ensure_2d=False)
363 assert X_array.ndim == 1
364 # ensure_2d=True with 1d array
365 with pytest.raises(ValueError, match="Expected 2D array, got 1D array instead"):
366 check_array([0, 1, 2], ensure_2d=True)
367
368 # ensure_2d=True with scalar array
369 with pytest.raises(ValueError, match="Expected 2D array, got scalar array instead"):
370 check_array(10, ensure_2d=True)
371
372 # ensure_2d=True with 1d sparse array
373 if hasattr(sp, "csr_array"):
374 sparse_row = next(iter(sp.csr_array(X)))
375 if sparse_row.ndim == 1:
376 # In scipy 1.14 and later, sparse row is 1D while it was 2D before.
377 with pytest.raises(ValueError, match="Expected 2D input, got"):
378 check_array(sparse_row, accept_sparse=True, ensure_2d=True)
379
380 # don't allow ndim > 3
381 X_ndim = np.arange(8).reshape(2, 2, 2)
382 with pytest.raises(ValueError):
383 check_array(X_ndim)
384 check_array(X_ndim, allow_nd=True) # doesn't raise
385
386 # dtype and order enforcement.
387 X_C = np.arange(4).reshape(2, 2).copy("C")
388 X_F = X_C.copy("F")
389 X_int = X_C.astype(int)
390 X_float = X_C.astype(float)
391 Xs = [X_C, X_F, X_int, X_float]
392 dtypes = [np.int32, int, float, np.float32, None, bool, object]
393 orders = ["C", "F", None]
394 copys = [True, False]
395
396 for X, dtype, order, copy in product(Xs, dtypes, orders, copys):
397 X_checked = check_array(X, dtype=dtype, order=order, copy=copy)
398 if dtype is not None:
399 assert X_checked.dtype == dtype
400 else:
401 assert X_checked.dtype == X.dtype
402 if order == "C":
403 assert X_checked.flags["C_CONTIGUOUS"]
404 assert not X_checked.flags["F_CONTIGUOUS"]
405 elif order == "F":
406 assert X_checked.flags["F_CONTIGUOUS"]
407 assert not X_checked.flags["C_CONTIGUOUS"]
408 if copy:
409 assert X is not X_checked
410 else:
411 # doesn't copy if it was already good
412 if (
413 X.dtype == X_checked.dtype
414 and X_checked.flags["C_CONTIGUOUS"] == X.flags["C_CONTIGUOUS"]
415 and X_checked.flags["F_CONTIGUOUS"] == X.flags["F_CONTIGUOUS"]
416 ):
417 assert X is X_checked
418
419 # allowed sparse != None
420
421 # try different type of sparse format
422 Xs = []
423 Xs.extend(
424 [
425 sparse_container(X_C)
426 for sparse_container in CSR_CONTAINERS
427 + CSC_CONTAINERS
428 + COO_CONTAINERS
429 + DOK_CONTAINERS
430 ]
431 )
432 Xs.extend([Xs[0].astype(np.int64), Xs[0].astype(np.float64)])
433
434 accept_sparses = [["csr", "coo"], ["coo", "dok"]]
435 # scipy sparse matrices do not support the object dtype so
436 # this dtype is skipped in this loop
437 non_object_dtypes = [dt for dt in dtypes if dt is not object]
438 for X, dtype, accept_sparse, copy in product(
439 Xs, non_object_dtypes, accept_sparses, copys
440 ):
441 X_checked = check_array(X, dtype=dtype, accept_sparse=accept_sparse, copy=copy)
442 if dtype is not None:
443 assert X_checked.dtype == dtype
444 else:
445 assert X_checked.dtype == X.dtype
446 if X.format in accept_sparse:
447 # no change if allowed
448 assert X.format == X_checked.format
449 else:
450 # got converted
451 assert X_checked.format == accept_sparse[0]
452 if copy:
453 assert X is not X_checked
454 else:
455 # doesn't copy if it was already good
456 if X.dtype == X_checked.dtype and X.format == X_checked.format:
457 assert X is X_checked
458
459 # other input formats
460 # convert lists to arrays
461 X_dense = check_array([[1, 2], [3, 4]])
462 assert isinstance(X_dense, np.ndarray)
463 # raise on too deep lists
464 with pytest.raises(ValueError):
465 check_array(X_ndim.tolist())
466 check_array(X_ndim.tolist(), allow_nd=True) # doesn't raise
467
468 # convert weird stuff to arrays
469 X_no_array = _NotAnArray(X_dense)
470 result = check_array(X_no_array)
471 assert isinstance(result, np.ndarray)
472
473 # check negative values when ensure_non_negative=True
474 X_neg = check_array([[1, 2], [-3, 4]])
475 err_msg = "Negative values in data passed to X in RandomForestRegressor"
476 with pytest.raises(ValueError, match=err_msg):
477 check_array(
478 X_neg,
479 ensure_non_negative=True,
480 input_name="X",
481 estimator=RandomForestRegressor(),
482 )
483
484
485@pytest.mark.parametrize(
486 "X",
487 [
488 [["1", "2"], ["3", "4"]],
489 np.array([["1", "2"], ["3", "4"]], dtype="U"),
490 np.array([["1", "2"], ["3", "4"]], dtype="S"),
491 [[b"1", b"2"], [b"3", b"4"]],
492 np.array([[b"1", b"2"], [b"3", b"4"]], dtype="V1"),
493 ],
494)
495def test_check_array_numeric_error(X):
496 """Test that check_array errors when it receives an array of bytes/string
497 while a numeric dtype is required."""
498 expected_msg = r"dtype='numeric' is not compatible with arrays of bytes/strings"
499 with pytest.raises(ValueError, match=expected_msg):
500 check_array(X, dtype="numeric")
501
502
503@pytest.mark.parametrize(
504 "pd_dtype", ["Int8", "Int16", "UInt8", "UInt16", "Float32", "Float64"]
505)
506@pytest.mark.parametrize(
507 "dtype, expected_dtype",
508 [
509 ([np.float32, np.float64], np.float32),
510 (np.float64, np.float64),
511 ("numeric", np.float64),
512 ],
513)
514def test_check_array_pandas_na_support(pd_dtype, dtype, expected_dtype):
515 # Test pandas numerical extension arrays with pd.NA
516 pd = pytest.importorskip("pandas")
517
518 if pd_dtype in {"Float32", "Float64"}:
519 # Extension dtypes with Floats was added in 1.2
520 pd = pytest.importorskip("pandas", minversion="1.2")
521
522 X_np = np.array(
523 [[1, 2, 3, np.nan, np.nan], [np.nan, np.nan, 8, 4, 6], [1, 2, 3, 4, 5]]
524 ).T
525
526 # Creates dataframe with numerical extension arrays with pd.NA
527 X = pd.DataFrame(X_np, dtype=pd_dtype, columns=["a", "b", "c"])
528 # column c has no nans
529 X["c"] = X["c"].astype("float")
530 X_checked = check_array(X, ensure_all_finite="allow-nan", dtype=dtype)
531 assert_allclose(X_checked, X_np)
532 assert X_checked.dtype == expected_dtype
533
534 X_checked = check_array(X, ensure_all_finite=False, dtype=dtype)
535 assert_allclose(X_checked, X_np)
536 assert X_checked.dtype == expected_dtype
537
538 msg = "Input contains NaN"
539 with pytest.raises(ValueError, match=msg):
540 check_array(X, ensure_all_finite=True)
541
542
543def test_check_array_panadas_na_support_series():
544 """Check check_array is correct with pd.NA in a series."""
545 pd = pytest.importorskip("pandas")
546
547 X_int64 = pd.Series([1, 2, pd.NA], dtype="Int64")
548
549 msg = "Input contains NaN"
550 with pytest.raises(ValueError, match=msg):
551 check_array(X_int64, ensure_all_finite=True, ensure_2d=False)
552
553 X_out = check_array(X_int64, ensure_all_finite=False, ensure_2d=False)
554 assert_allclose(X_out, [1, 2, np.nan])
555 assert X_out.dtype == np.float64
556
557 X_out = check_array(
558 X_int64, ensure_all_finite=False, ensure_2d=False, dtype=np.float32
559 )
560 assert_allclose(X_out, [1, 2, np.nan])
561 assert X_out.dtype == np.float32
562
563
564def test_check_array_pandas_dtype_casting():
565 # test that data-frames with homogeneous dtype are not upcast
566 pd = pytest.importorskip("pandas")
567 X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32)
568 X_df = pd.DataFrame(X)
569 assert check_array(X_df).dtype == np.float32
570 assert check_array(X_df, dtype=FLOAT_DTYPES).dtype == np.float32
571
572 X_df = X_df.astype({0: np.float16})
573 assert_array_equal(X_df.dtypes, (np.float16, np.float32, np.float32))
574 assert check_array(X_df).dtype == np.float32
575 assert check_array(X_df, dtype=FLOAT_DTYPES).dtype == np.float32
576
577 X_df = X_df.astype({0: np.int16})
578 # float16, int16, float32 casts to float32
579 assert check_array(X_df).dtype == np.float32
580 assert check_array(X_df, dtype=FLOAT_DTYPES).dtype == np.float32
581
582 X_df = X_df.astype({2: np.float16})
583 # float16, int16, float16 casts to float32
584 assert check_array(X_df).dtype == np.float32
585 assert check_array(X_df, dtype=FLOAT_DTYPES).dtype == np.float32
586
587 X_df = X_df.astype(np.int16)
588 assert check_array(X_df).dtype == np.int16
589 # we're not using upcasting rules for determining
590 # the target type yet, so we cast to the default of float64
591 assert check_array(X_df, dtype=FLOAT_DTYPES).dtype == np.float64
592
593 # check that we handle pandas dtypes in a semi-reasonable way
594 # this is actually tricky because we can't really know that this
595 # should be integer ahead of converting it.
596 cat_df = pd.DataFrame({"cat_col": pd.Categorical([1, 2, 3])})
597 assert check_array(cat_df).dtype == np.int64
598 assert check_array(cat_df, dtype=FLOAT_DTYPES).dtype == np.float64
599
600
601def test_check_array_on_mock_dataframe():
602 arr = np.array([[0.2, 0.7], [0.6, 0.5], [0.4, 0.1], [0.7, 0.2]])
603 mock_df = MockDataFrame(arr)
604 checked_arr = check_array(mock_df)
605 assert checked_arr.dtype == arr.dtype
606 checked_arr = check_array(mock_df, dtype=np.float32)
607 assert checked_arr.dtype == np.dtype(np.float32)
608
609
610def test_check_array_dtype_stability():
611 # test that lists with ints don't get converted to floats
612 X = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
613 assert check_array(X).dtype.kind == "i"
614 assert check_array(X, ensure_2d=False).dtype.kind == "i"
615
616
617def test_check_array_dtype_warning():
618 X_int_list = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
619 X_float32 = np.asarray(X_int_list, dtype=np.float32)
620 X_int64 = np.asarray(X_int_list, dtype=np.int64)
621 X_csr_float32 = sp.csr_matrix(X_float32)
622 X_csc_float32 = sp.csc_matrix(X_float32)
623 X_csc_int32 = sp.csc_matrix(X_int64, dtype=np.int32)
624 integer_data = [X_int64, X_csc_int32]
625 float32_data = [X_float32, X_csr_float32, X_csc_float32]
626 with warnings.catch_warnings():
627 warnings.simplefilter("error")
628
629 for X in integer_data:
630 X_checked = check_array(X, dtype=np.float64, accept_sparse=True)
631 assert X_checked.dtype == np.float64
632
633 for X in float32_data:
634 X_checked = check_array(
635 X, dtype=[np.float64, np.float32], accept_sparse=True
636 )
637 assert X_checked.dtype == np.float32
638 assert X_checked is X
639
640 X_checked = check_array(
641 X,
642 dtype=[np.float64, np.float32],
643 accept_sparse=["csr", "dok"],
644 copy=True,
645 )
646 assert X_checked.dtype == np.float32
647 assert X_checked is not X
648
649 X_checked = check_array(
650 X_csc_float32,
651 dtype=[np.float64, np.float32],
652 accept_sparse=["csr", "dok"],
653 copy=False,
654 )
655 assert X_checked.dtype == np.float32
656 assert X_checked is not X_csc_float32
657 assert X_checked.format == "csr"
658
659
660def test_check_array_accept_sparse_type_exception():
661 X = [[1, 2], [3, 4]]
662 X_csr = sp.csr_matrix(X)
663 invalid_type = SVR()
664
665 msg = (
666 "Sparse data was passed, but dense data is required. "
667 r"Use '.toarray\(\)' to convert to a dense numpy array."
668 )
669 with pytest.raises(TypeError, match=msg):
670 check_array(X_csr, accept_sparse=False)
671
672 msg = (
673 "Parameter 'accept_sparse' should be a string, "
674 "boolean or list of strings. You provided 'accept_sparse=.*'."
675 )
676 with pytest.raises(ValueError, match=msg):
677 check_array(X_csr, accept_sparse=invalid_type)
678
679 msg = (
680 "When providing 'accept_sparse' as a tuple or list, "
681 "it must contain at least one string value."
682 )
683 with pytest.raises(ValueError, match=msg):
684 check_array(X_csr, accept_sparse=[])
685 with pytest.raises(ValueError, match=msg):
686 check_array(X_csr, accept_sparse=())
687 with pytest.raises(TypeError, match="SVR"):
688 check_array(X_csr, accept_sparse=[invalid_type])
689
690
691def test_check_array_accept_sparse_no_exception():
692 X = [[1, 2], [3, 4]]
693 X_csr = sp.csr_matrix(X)
694
695 check_array(X_csr, accept_sparse=True)
696 check_array(X_csr, accept_sparse="csr")
697 check_array(X_csr, accept_sparse=["csr"])
698 check_array(X_csr, accept_sparse=("csr",))
699
700
701@pytest.fixture(params=["csr", "csc", "coo", "bsr"])
702def X_64bit(request):
703 X = sp.random(20, 10, format=request.param)
704
705 if request.param == "coo":
706 if hasattr(X, "coords"):
707 # for scipy >= 1.13 .coords is a new attribute and is a tuple. The
708 # .col and .row attributes do not seem to be able to change the
709 # dtype, for more details see https://github.com/scipy/scipy/pull/18530/
710 # and https://github.com/scipy/scipy/pull/20003 where .indices was
711 # renamed to .coords
712 X.coords = tuple(v.astype("int64") for v in X.coords)
713 else:
714 # scipy < 1.13
715 X.row = X.row.astype("int64")
716 X.col = X.col.astype("int64")
717 else:
718 X.indices = X.indices.astype("int64")
719 X.indptr = X.indptr.astype("int64")
720
721 yield X
722
723
724def test_check_array_accept_large_sparse_no_exception(X_64bit):
725 # When large sparse are allowed
726 check_array(X_64bit, accept_large_sparse=True, accept_sparse=True)
727
728
729def test_check_array_accept_large_sparse_raise_exception(X_64bit):
730 # When large sparse are not allowed
731 msg = (
732 "Only sparse matrices with 32-bit integer indices "
733 "are accepted. Got int64 indices. Please do report"
734 )
735 with pytest.raises(ValueError, match=msg):
736 check_array(X_64bit, accept_sparse=True, accept_large_sparse=False)
737
738
739def test_check_array_min_samples_and_features_messages():
740 # empty list is considered 2D by default:
741 msg = r"0 feature\(s\) \(shape=\(1, 0\)\) while a minimum of 1 is required."
742 with pytest.raises(ValueError, match=msg):
743 check_array([[]])
744
745 # If considered a 1D collection when ensure_2d=False, then the minimum
746 # number of samples will break:
747 msg = r"0 sample\(s\) \(shape=\(0,\)\) while a minimum of 1 is required."
748 with pytest.raises(ValueError, match=msg):
749 check_array([], ensure_2d=False)
750
751 # Invalid edge case when checking the default minimum sample of a scalar
752 msg = re.escape(
753 (
754 "Input should have at least 1 dimension i.e. satisfy "
755 "`len(x.shape) > 0`, got scalar `array(42)` instead."
756 )
757 )
758 with pytest.raises(TypeError, match=msg):
759 check_array(42, ensure_2d=False)
760
761 # Simulate a model that would need at least 2 samples to be well defined
762 X = np.ones((1, 10))
763 y = np.ones(1)
764 msg = r"1 sample\(s\) \(shape=\(1, 10\)\) while a minimum of 2 is required."
765 with pytest.raises(ValueError, match=msg):
766 check_X_y(X, y, ensure_min_samples=2)
767
768 # The same message is raised if the data has 2 dimensions even if this is
769 # not mandatory
770 with pytest.raises(ValueError, match=msg):
771 check_X_y(X, y, ensure_min_samples=2, ensure_2d=False)
772
773 # Simulate a model that would require at least 3 features (e.g. SelectKBest
774 # with k=3)
775 X = np.ones((10, 2))
776 y = np.ones(2)
777 msg = r"2 feature\(s\) \(shape=\(10, 2\)\) while a minimum of 3 is required."
778 with pytest.raises(ValueError, match=msg):
779 check_X_y(X, y, ensure_min_features=3)
780
781 # Only the feature check is enabled whenever the number of dimensions is 2
782 # even if allow_nd is enabled:
783 with pytest.raises(ValueError, match=msg):
784 check_X_y(X, y, ensure_min_features=3, allow_nd=True)
785
786 # Simulate a case where a pipeline stage as trimmed all the features of a
787 # 2D dataset.
788 X = np.empty(0).reshape(10, 0)
789 y = np.ones(10)
790 msg = r"0 feature\(s\) \(shape=\(10, 0\)\) while a minimum of 1 is required."
791 with pytest.raises(ValueError, match=msg):
792 check_X_y(X, y)
793
794 # nd-data is not checked for any minimum number of features by default:
795 X = np.ones((10, 0, 28, 28))
796 y = np.ones(10)
797 X_checked, y_checked = check_X_y(X, y, allow_nd=True)
798 assert_array_equal(X, X_checked)
799 assert_array_equal(y, y_checked)
800
801
802def test_check_array_complex_data_error():
803 X = np.array([[1 + 2j, 3 + 4j, 5 + 7j], [2 + 3j, 4 + 5j, 6 + 7j]])
804 with pytest.raises(ValueError, match="Complex data not supported"):
805 check_array(X)
806
807 # list of lists
808 X = [[1 + 2j, 3 + 4j, 5 + 7j], [2 + 3j, 4 + 5j, 6 + 7j]]
809 with pytest.raises(ValueError, match="Complex data not supported"):
810 check_array(X)
811
812 # tuple of tuples
813 X = ((1 + 2j, 3 + 4j, 5 + 7j), (2 + 3j, 4 + 5j, 6 + 7j))
814 with pytest.raises(ValueError, match="Complex data not supported"):
815 check_array(X)
816
817 # list of np arrays
818 X = [np.array([1 + 2j, 3 + 4j, 5 + 7j]), np.array([2 + 3j, 4 + 5j, 6 + 7j])]
819 with pytest.raises(ValueError, match="Complex data not supported"):
820 check_array(X)
821
822 # tuple of np arrays
823 X = (np.array([1 + 2j, 3 + 4j, 5 + 7j]), np.array([2 + 3j, 4 + 5j, 6 + 7j]))
824 with pytest.raises(ValueError, match="Complex data not supported"):
825 check_array(X)
826
827 # dataframe
828 X = MockDataFrame(np.array([[1 + 2j, 3 + 4j, 5 + 7j], [2 + 3j, 4 + 5j, 6 + 7j]]))
829 with pytest.raises(ValueError, match="Complex data not supported"):
830 check_array(X)
831
832 # sparse matrix
833 X = sp.coo_matrix([[0, 1 + 2j], [0, 0]])
834 with pytest.raises(ValueError, match="Complex data not supported"):
835 check_array(X)
836
837 # target variable does not always go through check_array but should
838 # never accept complex data either.
839 y = np.array([1 + 2j, 3 + 4j, 5 + 7j, 2 + 3j, 4 + 5j, 6 + 7j])
840 with pytest.raises(ValueError, match="Complex data not supported"):
841 _check_y(y)
842
843
844def test_has_fit_parameter():
845 assert not has_fit_parameter(KNeighborsClassifier, "sample_weight")
846 assert has_fit_parameter(RandomForestRegressor, "sample_weight")
847 assert has_fit_parameter(SVR, "sample_weight")
848 assert has_fit_parameter(SVR(), "sample_weight")
849
850 class TestClassWithDeprecatedFitMethod:
851 @deprecated("Deprecated for the purpose of testing has_fit_parameter")
852 def fit(self, X, y, sample_weight=None):
853 pass
854
855 assert has_fit_parameter(TestClassWithDeprecatedFitMethod, "sample_weight"), (
856 "has_fit_parameter fails for class with deprecated fit method."
857 )
858
859
860def test_check_symmetric():
861 arr_sym = np.array([[0, 1], [1, 2]])
862 arr_bad = np.ones(2)
863 arr_asym = np.array([[0, 2], [0, 2]])
864
865 test_arrays = {
866 "dense": arr_asym,
867 "dok": sp.dok_matrix(arr_asym),
868 "csr": sp.csr_matrix(arr_asym),
869 "csc": sp.csc_matrix(arr_asym),
870 "coo": sp.coo_matrix(arr_asym),
871 "lil": sp.lil_matrix(arr_asym),
872 "bsr": sp.bsr_matrix(arr_asym),
873 }
874
875 # check error for bad inputs
876 with pytest.raises(ValueError):
877 check_symmetric(arr_bad)
878
879 # check that asymmetric arrays are properly symmetrized
880 for arr_format, arr in test_arrays.items():
881 # Check for warnings and errors
882 with pytest.warns(UserWarning):
883 check_symmetric(arr)
884 with pytest.raises(ValueError):
885 check_symmetric(arr, raise_exception=True)
886
887 output = check_symmetric(arr, raise_warning=False)
888 if sp.issparse(output):
889 assert output.format == arr_format
890 assert_array_equal(output.toarray(), arr_sym)
891 else:
892 assert_array_equal(output, arr_sym)
893
894
895def test_check_is_fitted_with_is_fitted():
896 class Estimator(BaseEstimator):
897 def fit(self, **kwargs):
898 self._is_fitted = True
899 return self
900
901 def __sklearn_is_fitted__(self):
902 return hasattr(self, "_is_fitted") and self._is_fitted
903
904 with pytest.raises(NotFittedError):
905 check_is_fitted(Estimator())
906 check_is_fitted(Estimator().fit())
907
908
909def test_check_is_fitted_stateless():
910 """Check that check_is_fitted passes for stateless estimators."""
911
912 class StatelessEstimator(BaseEstimator):
913 def fit(self, **kwargs):
914 return self # pragma: no cover
915
916 def __sklearn_tags__(self):
917 tags = super().__sklearn_tags__()
918 tags.requires_fit = False
919 return tags
920
921 check_is_fitted(StatelessEstimator())
922
923
924def test_check_is_fitted():
925 # Check is TypeError raised when non estimator instance passed
926 with pytest.raises(TypeError):
927 check_is_fitted(ARDRegression)
928 with pytest.raises(TypeError):
929 check_is_fitted("SVR")
930
931 ard = ARDRegression()
932 svr = SVR()
933
934 try:
935 with pytest.raises(NotFittedError):
936 check_is_fitted(ard)
937 with pytest.raises(NotFittedError):
938 check_is_fitted(svr)
939 except ValueError:
940 assert False, "check_is_fitted failed with ValueError"
941
942 # NotFittedError is a subclass of both ValueError and AttributeError
943 msg = "Random message %(name)s, %(name)s"
944 match = "Random message ARDRegression, ARDRegression"
945 with pytest.raises(ValueError, match=match):
946 check_is_fitted(ard, msg=msg)
947
948 msg = "Another message %(name)s, %(name)s"
949 match = "Another message SVR, SVR"
950 with pytest.raises(AttributeError, match=match):
951 check_is_fitted(svr, msg=msg)
952
953 ard.fit(*make_blobs())
954 svr.fit(*make_blobs())
955
956 assert check_is_fitted(ard) is None
957 assert check_is_fitted(svr) is None
958
959
960def test_check_is_fitted_attributes():
961 class MyEstimator(BaseEstimator):
962 def fit(self, X, y):
963 return self
964
965 msg = "not fitted"
966 est = MyEstimator()
967
968 assert not _is_fitted(est, attributes=["a_", "b_"])
969 with pytest.raises(NotFittedError, match=msg):
970 check_is_fitted(est, attributes=["a_", "b_"])
971 assert not _is_fitted(est, attributes=["a_", "b_"], all_or_any=all)
972 with pytest.raises(NotFittedError, match=msg):
973 check_is_fitted(est, attributes=["a_", "b_"], all_or_any=all)
974 assert not _is_fitted(est, attributes=["a_", "b_"], all_or_any=any)
975 with pytest.raises(NotFittedError, match=msg):
976 check_is_fitted(est, attributes=["a_", "b_"], all_or_any=any)
977
978 est.a_ = "a"
979 assert not _is_fitted(est, attributes=["a_", "b_"])
980 with pytest.raises(NotFittedError, match=msg):
981 check_is_fitted(est, attributes=["a_", "b_"])
982 assert not _is_fitted(est, attributes=["a_", "b_"], all_or_any=all)
983 with pytest.raises(NotFittedError, match=msg):
984 check_is_fitted(est, attributes=["a_", "b_"], all_or_any=all)
985 assert _is_fitted(est, attributes=["a_", "b_"], all_or_any=any)
986 check_is_fitted(est, attributes=["a_", "b_"], all_or_any=any)
987
988 est.b_ = "b"
989 assert _is_fitted(est, attributes=["a_", "b_"])
990 check_is_fitted(est, attributes=["a_", "b_"])
991 assert _is_fitted(est, attributes=["a_", "b_"], all_or_any=all)
992 check_is_fitted(est, attributes=["a_", "b_"], all_or_any=all)
993 assert _is_fitted(est, attributes=["a_", "b_"], all_or_any=any)
994 check_is_fitted(est, attributes=["a_", "b_"], all_or_any=any)
995
996
997@pytest.mark.parametrize(
998 "wrap", [itemgetter(0), list, tuple], ids=["single", "list", "tuple"]
999)
1000def test_check_is_fitted_with_attributes(wrap):
1001 ard = ARDRegression()
1002 with pytest.raises(NotFittedError, match="is not fitted yet"):
1003 check_is_fitted(ard, wrap(["coef_"]))
1004
1005 ard.fit(*make_blobs())
1006
1007 # Does not raise
1008 check_is_fitted(ard, wrap(["coef_"]))
1009
1010 # Raises when using attribute that is not defined
1011 with pytest.raises(NotFittedError, match="is not fitted yet"):
1012 check_is_fitted(ard, wrap(["coef_bad_"]))
1013
1014
1015def test_check_consistent_length():
1016 """Test that `check_consistent_length` raises on inconsistent lengths and wrong
1017 input types trigger TypeErrors."""
1018 check_consistent_length([1], [2], [3], [4], [5])
1019 check_consistent_length([[1, 2], [[1, 2]]], [1, 2], ["a", "b"])
1020 check_consistent_length([1], (2,), np.array([3]), sp.csr_matrix((1, 2)))
1021 with pytest.raises(ValueError, match="inconsistent numbers of samples"):
1022 check_consistent_length([1, 2], [1])
1023 with pytest.raises(TypeError, match=r"got <\w+ 'int'>"):
1024 check_consistent_length([1, 2], 1)
1025 with pytest.raises(TypeError, match=r"got <\w+ 'object'>"):
1026 check_consistent_length([1, 2], object())
1027 with pytest.raises(TypeError):
1028 check_consistent_length([1, 2], np.array(1))
1029 # Despite ensembles having __len__ they must raise TypeError
1030 with pytest.raises(TypeError, match="Expected sequence or array-like"):
1031 check_consistent_length([1, 2], RandomForestRegressor())
1032 # XXX: We should have a test with a string, but what is correct behaviour?
1033
1034
1035@pytest.mark.parametrize(
1036 "array_namespace, device, _",
1037 yield_namespace_device_dtype_combinations(),
1038 ids=_get_namespace_device_dtype_ids,
1039)
1040def test_check_consistent_length_array_api(array_namespace, device, _):
1041 """Test that check_consistent_length works with different array types."""
1042 xp = _array_api_for_tests(array_namespace, device)
1043
1044 with config_context(array_api_dispatch=True):
1045 check_consistent_length(
1046 xp.asarray([1, 2, 3], device=device),
1047 xp.asarray([[1, 1], [2, 2], [3, 3]], device=device),
1048 [1, 2, 3],
1049 ["a", "b", "c"],
1050 np.asarray(("a", "b", "c"), dtype=object),
1051 sp.csr_array([[0, 1], [1, 0], [0, 0]]),
1052 )
1053
1054 with pytest.raises(ValueError, match="inconsistent numbers of samples"):
1055 check_consistent_length(
1056 xp.asarray([1, 2], device=device), xp.asarray([1], device=device)
1057 )
1058
1059
1060def test_check_dataframe_fit_attribute():
1061 # check pandas dataframe with 'fit' column does not raise error
1062 # https://github.com/scikit-learn/scikit-learn/issues/8415
1063 try:
1064 import pandas as pd
1065
1066 X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
1067 X_df = pd.DataFrame(X, columns=["a", "b", "fit"])
1068 check_consistent_length(X_df)
1069 except ImportError:
1070 raise SkipTest("Pandas not found")
1071
1072
1073def test_suppress_validation():
1074 X = np.array([0, np.inf])
1075 with pytest.raises(ValueError):
1076 assert_all_finite(X)
1077 sklearn.set_config(assume_finite=True)
1078 assert_all_finite(X)
1079 sklearn.set_config(assume_finite=False)
1080 with pytest.raises(ValueError):
1081 assert_all_finite(X)
1082
1083
1084def test_check_array_series():
1085 # regression test that check_array works on pandas Series
1086 pd = importorskip("pandas")
1087 res = check_array(pd.Series([1, 2, 3]), ensure_2d=False)
1088 assert_array_equal(res, np.array([1, 2, 3]))
1089
1090 # with categorical dtype (not a numpy dtype) (GH12699)
1091 s = pd.Series(["a", "b", "c"]).astype("category")
1092 res = check_array(s, dtype=None, ensure_2d=False)
1093 assert_array_equal(res, np.array(["a", "b", "c"], dtype=object))
1094
1095
1096@pytest.mark.parametrize(
1097 "dtype", ((np.float64, np.float32), np.float64, None, "numeric")
1098)
1099@pytest.mark.parametrize("bool_dtype", ("bool", "boolean"))
1100def test_check_dataframe_mixed_float_dtypes(dtype, bool_dtype):
1101 # pandas dataframe will coerce a boolean into a object, this is a mismatch
1102 # with np.result_type which will return a float
1103 # check_array needs to explicitly check for bool dtype in a dataframe for
1104 # this situation
1105 # https://github.com/scikit-learn/scikit-learn/issues/15787
1106
1107 pd = importorskip("pandas")
1108
1109 df = pd.DataFrame(
1110 {
1111 "int": [1, 2, 3],
1112 "float": [0, 0.1, 2.1],
1113 "bool": pd.Series([True, False, True], dtype=bool_dtype),
1114 },
1115 columns=["int", "float", "bool"],
1116 )
1117
1118 array = check_array(df, dtype=dtype)
1119 assert array.dtype == np.float64
1120 expected_array = np.array(
1121 [[1.0, 0.0, 1.0], [2.0, 0.1, 0.0], [3.0, 2.1, 1.0]], dtype=float
1122 )
1123 assert_allclose_dense_sparse(array, expected_array)
1124
1125
1126def test_check_dataframe_with_only_bool():
1127 """Check that dataframe with bool return a boolean arrays."""
1128 pd = importorskip("pandas")
1129 df = pd.DataFrame({"bool": [True, False, True]})
1130
1131 array = check_array(df, dtype=None)
1132 assert array.dtype == np.bool_
1133 assert_array_equal(array, [[True], [False], [True]])
1134
1135 # common dtype is int for bool + int
1136 df = pd.DataFrame(
1137 {"bool": [True, False, True], "int": [1, 2, 3]},
1138 columns=["bool", "int"],
1139 )
1140 array = check_array(df, dtype="numeric")
1141 assert array.dtype == np.int64
1142 assert_array_equal(array, [[1, 1], [0, 2], [1, 3]])
1143
1144
1145def test_check_dataframe_with_only_boolean():
1146 """Check that dataframe with boolean return a float array with dtype=None"""
1147 pd = importorskip("pandas")
1148 df = pd.DataFrame({"bool": pd.Series([True, False, True], dtype="boolean")})
1149
1150 array = check_array(df, dtype=None)
1151 assert array.dtype == np.float64
1152 assert_array_equal(array, [[True], [False], [True]])
1153
1154
1155class DummyMemory:
1156 def cache(self, func):
1157 return func
1158
1159
1160class WrongDummyMemory:
1161 pass
1162
1163
1164def test_check_memory(tmp_path):
1165 cache_directory = str(tmp_path / "cache_directory")
1166 memory = check_memory(cache_directory)
1167 assert memory.location == cache_directory
1168
1169 memory = check_memory(None)
1170 assert memory.location is None
1171
1172 dummy = DummyMemory()
1173 memory = check_memory(dummy)
1174 assert memory is dummy
1175
1176 msg = (
1177 "'memory' should be None, a string or have the same interface as"
1178 " joblib.Memory. Got memory='1' instead."
1179 )
1180 with pytest.raises(ValueError, match=msg):
1181 check_memory(1)
1182 dummy = WrongDummyMemory()
1183 msg = (
1184 "'memory' should be None, a string or have the same interface as"
1185 " joblib.Memory. Got memory='{}' instead.".format(dummy)
1186 )
1187 with pytest.raises(ValueError, match=msg):
1188 check_memory(dummy)
1189
1190
1191@pytest.mark.parametrize("copy", [True, False])
1192def test_check_array_memmap(copy):
1193 X = np.ones((4, 4))
1194 with TempMemmap(X, mmap_mode="r") as X_memmap:
1195 X_checked = check_array(X_memmap, copy=copy)
1196 assert np.may_share_memory(X_memmap, X_checked) == (not copy)
1197 assert X_checked.flags["WRITEABLE"] == copy
1198
1199
1200@pytest.mark.parametrize(