Aluode/PerceptionLabPortable
0
1import warnings
2
3import numpy as np
4import pytest
5
6import sklearn
7from sklearn.datasets import load_iris, make_blobs, make_circles
8from sklearn.decomposition import PCA, KernelPCA
9from sklearn.exceptions import NotFittedError
10from sklearn.linear_model import Perceptron
11from sklearn.metrics.pairwise import rbf_kernel
12from sklearn.model_selection import GridSearchCV
13from sklearn.pipeline import Pipeline
14from sklearn.preprocessing import StandardScaler
15from sklearn.utils._testing import (
16 assert_allclose,
17 assert_array_almost_equal,
18 assert_array_equal,
19)
20from sklearn.utils.fixes import CSR_CONTAINERS
21from sklearn.utils.validation import _check_psd_eigenvalues
22
23
24def test_kernel_pca(global_random_seed):
25 """Nominal test for all solvers and all known kernels + a custom one
26
27 It tests
28 - that fit_transform is equivalent to fit+transform
29 - that the shapes of transforms and inverse transforms are correct
30 """
31 rng = np.random.RandomState(global_random_seed)
32 X_fit = rng.random_sample((5, 4))
33 X_pred = rng.random_sample((2, 4))
34
35 def histogram(x, y, **kwargs):
36 # Histogram kernel implemented as a callable.
37 assert kwargs == {} # no kernel_params that we didn't ask for
38 return np.minimum(x, y).sum()
39
40 for eigen_solver in ("auto", "dense", "arpack", "randomized"):
41 for kernel in ("linear", "rbf", "poly", histogram):
42 # histogram kernel produces singular matrix inside linalg.solve
43 # XXX use a least-squares approximation?
44 inv = not callable(kernel)
45
46 # transform fit data
47 kpca = KernelPCA(
48 4, kernel=kernel, eigen_solver=eigen_solver, fit_inverse_transform=inv
49 )
50 X_fit_transformed = kpca.fit_transform(X_fit)
51 X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit)
52 assert_array_almost_equal(
53 np.abs(X_fit_transformed), np.abs(X_fit_transformed2)
54 )
55
56 # non-regression test: previously, gamma would be 0 by default,
57 # forcing all eigenvalues to 0 under the poly kernel
58 assert X_fit_transformed.size != 0
59
60 # transform new data
61 X_pred_transformed = kpca.transform(X_pred)
62 assert X_pred_transformed.shape[1] == X_fit_transformed.shape[1]
63
64 # inverse transform
65 if inv:
66 X_pred2 = kpca.inverse_transform(X_pred_transformed)
67 assert X_pred2.shape == X_pred.shape
68
69
70def test_kernel_pca_invalid_parameters():
71 """Check that kPCA raises an error if the parameters are invalid
72
73 Tests fitting inverse transform with a precomputed kernel raises a
74 ValueError.
75 """
76 estimator = KernelPCA(
77 n_components=10, fit_inverse_transform=True, kernel="precomputed"
78 )
79 err_ms = "Cannot fit_inverse_transform with a precomputed kernel"
80 with pytest.raises(ValueError, match=err_ms):
81 estimator.fit(np.random.randn(10, 10))
82
83
84def test_kernel_pca_consistent_transform(global_random_seed):
85 """Check robustness to mutations in the original training array
86
87 Test that after fitting a kPCA model, it stays independent of any
88 mutation of the values of the original data object by relying on an
89 internal copy.
90 """
91 # X_fit_ needs to retain the old, unmodified copy of X
92 state = np.random.RandomState(global_random_seed)
93 X = state.rand(10, 10)
94 kpca = KernelPCA(random_state=state).fit(X)
95 transformed1 = kpca.transform(X)
96
97 X_copy = X.copy()
98 X[:, 0] = 666
99 transformed2 = kpca.transform(X_copy)
100 assert_array_almost_equal(transformed1, transformed2)
101
102
103def test_kernel_pca_deterministic_output(global_random_seed):
104 """Test that Kernel PCA produces deterministic output
105
106 Tests that the same inputs and random state produce the same output.
107 """
108 rng = np.random.RandomState(global_random_seed)
109 X = rng.rand(10, 10)
110 eigen_solver = ("arpack", "dense")
111
112 for solver in eigen_solver:
113 transformed_X = np.zeros((20, 2))
114 for i in range(20):
115 kpca = KernelPCA(n_components=2, eigen_solver=solver, random_state=rng)
116 transformed_X[i, :] = kpca.fit_transform(X)[0]
117 assert_allclose(transformed_X, np.tile(transformed_X[0, :], 20).reshape(20, 2))
118
119
120@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
121def test_kernel_pca_sparse(csr_container, global_random_seed):
122 """Test that kPCA works on a sparse data input.
123
124 Same test as ``test_kernel_pca except inverse_transform`` since it's not
125 implemented for sparse matrices.
126 """
127 rng = np.random.RandomState(global_random_seed)
128 X_fit = csr_container(rng.random_sample((5, 4)))
129 X_pred = csr_container(rng.random_sample((2, 4)))
130
131 for eigen_solver in ("auto", "arpack", "randomized"):
132 for kernel in ("linear", "rbf", "poly"):
133 # transform fit data
134 kpca = KernelPCA(
135 4,
136 kernel=kernel,
137 eigen_solver=eigen_solver,
138 fit_inverse_transform=False,
139 random_state=0,
140 )
141 X_fit_transformed = kpca.fit_transform(X_fit)
142 X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit)
143 assert_array_almost_equal(
144 np.abs(X_fit_transformed), np.abs(X_fit_transformed2)
145 )
146
147 # transform new data
148 X_pred_transformed = kpca.transform(X_pred)
149 assert X_pred_transformed.shape[1] == X_fit_transformed.shape[1]
150
151 # inverse transform: not available for sparse matrices
152 # XXX: should we raise another exception type here? For instance:
153 # NotImplementedError.
154 with pytest.raises(NotFittedError):
155 kpca.inverse_transform(X_pred_transformed)
156
157
158@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"])
159@pytest.mark.parametrize("n_features", [4, 10])
160def test_kernel_pca_linear_kernel(solver, n_features, global_random_seed):
161 """Test that kPCA with linear kernel is equivalent to PCA for all solvers.
162
163 KernelPCA with linear kernel should produce the same output as PCA.
164 """
165 rng = np.random.RandomState(global_random_seed)
166 X_fit = rng.random_sample((5, n_features))
167 X_pred = rng.random_sample((2, n_features))
168
169 # for a linear kernel, kernel PCA should find the same projection as PCA
170 # modulo the sign (direction)
171 # fit only the first four components: fifth is near zero eigenvalue, so
172 # can be trimmed due to roundoff error
173 n_comps = 3 if solver == "arpack" else 4
174 assert_array_almost_equal(
175 np.abs(KernelPCA(n_comps, eigen_solver=solver).fit(X_fit).transform(X_pred)),
176 np.abs(
177 PCA(n_comps, svd_solver=solver if solver != "dense" else "full")
178 .fit(X_fit)
179 .transform(X_pred)
180 ),
181 )
182
183
184def test_kernel_pca_n_components():
185 """Test that `n_components` is correctly taken into account for projections
186
187 For all solvers this tests that the output has the correct shape depending
188 on the selected number of components.
189 """
190 rng = np.random.RandomState(0)
191 X_fit = rng.random_sample((5, 4))
192 X_pred = rng.random_sample((2, 4))
193
194 for eigen_solver in ("dense", "arpack", "randomized"):
195 for c in [1, 2, 4]:
196 kpca = KernelPCA(n_components=c, eigen_solver=eigen_solver)
197 shape = kpca.fit(X_fit).transform(X_pred).shape
198
199 assert shape == (2, c)
200
201
202def test_remove_zero_eig():
203 """Check that the ``remove_zero_eig`` parameter works correctly.
204
205 Tests that the null-space (Zero) eigenvalues are removed when
206 remove_zero_eig=True, whereas they are not by default.
207 """
208 X = np.array([[1 - 1e-30, 1], [1, 1], [1, 1 - 1e-20]])
209
210 # n_components=None (default) => remove_zero_eig is True
211 kpca = KernelPCA()
212 Xt = kpca.fit_transform(X)
213 assert Xt.shape == (3, 0)
214
215 kpca = KernelPCA(n_components=2)
216 Xt = kpca.fit_transform(X)
217 assert Xt.shape == (3, 2)
218
219 kpca = KernelPCA(n_components=2, remove_zero_eig=True)
220 Xt = kpca.fit_transform(X)
221 assert Xt.shape == (3, 0)
222
223
224def test_leave_zero_eig():
225 """Non-regression test for issue #12141 (PR #12143)
226
227 This test checks that fit().transform() returns the same result as
228 fit_transform() in case of non-removed zero eigenvalue.
229 """
230 X_fit = np.array([[1, 1], [0, 0]])
231
232 # Assert that even with all np warnings on, there is no div by zero warning
233 with warnings.catch_warnings():
234 # There might be warnings about the kernel being badly conditioned,
235 # but there should not be warnings about division by zero.
236 # (Numpy division by zero warning can have many message variants, but
237 # at least we know that it is a RuntimeWarning so lets check only this)
238 warnings.simplefilter("error", RuntimeWarning)
239 with np.errstate(all="warn"):
240 k = KernelPCA(n_components=2, remove_zero_eig=False, eigen_solver="dense")
241 # Fit, then transform
242 A = k.fit(X_fit).transform(X_fit)
243 # Do both at once
244 B = k.fit_transform(X_fit)
245 # Compare
246 assert_array_almost_equal(np.abs(A), np.abs(B))
247
248
249def test_kernel_pca_precomputed(global_random_seed):
250 """Test that kPCA works with a precomputed kernel, for all solvers"""
251 rng = np.random.RandomState(global_random_seed)
252 X_fit = rng.random_sample((5, 4))
253 X_pred = rng.random_sample((2, 4))
254
255 for eigen_solver in ("dense", "arpack", "randomized"):
256 X_kpca = (
257 KernelPCA(4, eigen_solver=eigen_solver, random_state=0)
258 .fit(X_fit)
259 .transform(X_pred)
260 )
261
262 X_kpca2 = (
263 KernelPCA(
264 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0
265 )
266 .fit(np.dot(X_fit, X_fit.T))
267 .transform(np.dot(X_pred, X_fit.T))
268 )
269
270 X_kpca_train = KernelPCA(
271 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0
272 ).fit_transform(np.dot(X_fit, X_fit.T))
273
274 X_kpca_train2 = (
275 KernelPCA(
276 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0
277 )
278 .fit(np.dot(X_fit, X_fit.T))
279 .transform(np.dot(X_fit, X_fit.T))
280 )
281
282 assert_array_almost_equal(np.abs(X_kpca), np.abs(X_kpca2))
283
284 assert_array_almost_equal(np.abs(X_kpca_train), np.abs(X_kpca_train2))
285
286
287@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"])
288def test_kernel_pca_precomputed_non_symmetric(solver):
289 """Check that the kernel centerer works.
290
291 Tests that a non symmetric precomputed kernel is actually accepted
292 because the kernel centerer does its job correctly.
293 """
294
295 # a non symmetric gram matrix
296 K = [[1, 2], [3, 40]]
297 kpca = KernelPCA(
298 kernel="precomputed", eigen_solver=solver, n_components=1, random_state=0
299 )
300 kpca.fit(K) # no error
301
302 # same test with centered kernel
303 Kc = [[9, -9], [-9, 9]]
304 kpca_c = KernelPCA(
305 kernel="precomputed", eigen_solver=solver, n_components=1, random_state=0
306 )
307 kpca_c.fit(Kc)
308
309 # comparison between the non-centered and centered versions
310 assert_array_equal(kpca.eigenvectors_, kpca_c.eigenvectors_)
311 assert_array_equal(kpca.eigenvalues_, kpca_c.eigenvalues_)
312
313
314def test_gridsearch_pipeline():
315 """Check that kPCA works as expected in a grid search pipeline
316
317 Test if we can do a grid-search to find parameters to separate
318 circles with a perceptron model.
319 """
320 X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0)
321 kpca = KernelPCA(kernel="rbf", n_components=2)
322 pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron(max_iter=5))])
323 param_grid = dict(kernel_pca__gamma=2.0 ** np.arange(-2, 2))
324 grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid)
325 grid_search.fit(X, y)
326 assert grid_search.best_score_ == 1
327
328
329def test_gridsearch_pipeline_precomputed():
330 """Check that kPCA works as expected in a grid search pipeline (2)
331
332 Test if we can do a grid-search to find parameters to separate
333 circles with a perceptron model. This test uses a precomputed kernel.
334 """
335 X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0)
336 kpca = KernelPCA(kernel="precomputed", n_components=2)
337 pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron(max_iter=5))])
338 param_grid = dict(Perceptron__max_iter=np.arange(1, 5))
339 grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid)
340 X_kernel = rbf_kernel(X, gamma=2.0)
341 grid_search.fit(X_kernel, y)
342 assert grid_search.best_score_ == 1
343
344
345def test_nested_circles():
346 """Check that kPCA projects in a space where nested circles are separable
347
348 Tests that 2D nested circles become separable with a perceptron when
349 projected in the first 2 kPCA using an RBF kernel, while raw samples
350 are not directly separable in the original space.
351 """
352 X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0)
353
354 # 2D nested circles are not linearly separable
355 train_score = Perceptron(max_iter=5).fit(X, y).score(X, y)
356 assert train_score < 0.8
357
358 # Project the circles data into the first 2 components of a RBF Kernel
359 # PCA model.
360 # Note that the gamma value is data dependent. If this test breaks
361 # and the gamma value has to be updated, the Kernel PCA example will
362 # have to be updated too.
363 kpca = KernelPCA(
364 kernel="rbf", n_components=2, fit_inverse_transform=True, gamma=2.0
365 )
366 X_kpca = kpca.fit_transform(X)
367
368 # The data is perfectly linearly separable in that space
369 train_score = Perceptron(max_iter=5).fit(X_kpca, y).score(X_kpca, y)
370 assert train_score == 1.0
371
372
373def test_kernel_conditioning():
374 """Check that ``_check_psd_eigenvalues`` is correctly called in kPCA
375
376 Non-regression test for issue #12140 (PR #12145).
377 """
378
379 # create a pathological X leading to small non-zero eigenvalue
380 X = [[5, 1], [5 + 1e-8, 1e-8], [5 + 1e-8, 0]]
381 kpca = KernelPCA(kernel="linear", n_components=2, fit_inverse_transform=True)
382 kpca.fit(X)
383
384 # check that the small non-zero eigenvalue was correctly set to zero
385 assert kpca.eigenvalues_.min() == 0
386 assert np.all(kpca.eigenvalues_ == _check_psd_eigenvalues(kpca.eigenvalues_))
387
388
389@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"])
390def test_precomputed_kernel_not_psd(solver):
391 """Check how KernelPCA works with non-PSD kernels depending on n_components
392
393 Tests for all methods what happens with a non PSD gram matrix (this
394 can happen in an isomap scenario, or with custom kernel functions, or
395 maybe with ill-posed datasets).
396
397 When ``n_component`` is large enough to capture a negative eigenvalue, an
398 error should be raised. Otherwise, KernelPCA should run without error
399 since the negative eigenvalues are not selected.
400 """
401
402 # a non PSD kernel with large eigenvalues, already centered
403 # it was captured from an isomap call and multiplied by 100 for compacity
404 K = [
405 [4.48, -1.0, 8.07, 2.33, 2.33, 2.33, -5.76, -12.78],
406 [-1.0, -6.48, 4.5, -1.24, -1.24, -1.24, -0.81, 7.49],
407 [8.07, 4.5, 15.48, 2.09, 2.09, 2.09, -11.1, -23.23],
408 [2.33, -1.24, 2.09, 4.0, -3.65, -3.65, 1.02, -0.9],
409 [2.33, -1.24, 2.09, -3.65, 4.0, -3.65, 1.02, -0.9],
410 [2.33, -1.24, 2.09, -3.65, -3.65, 4.0, 1.02, -0.9],
411 [-5.76, -0.81, -11.1, 1.02, 1.02, 1.02, 4.86, 9.75],
412 [-12.78, 7.49, -23.23, -0.9, -0.9, -0.9, 9.75, 21.46],
413 ]
414 # this gram matrix has 5 positive eigenvalues and 3 negative ones
415 # [ 52.72, 7.65, 7.65, 5.02, 0. , -0. , -6.13, -15.11]
416
417 # 1. ask for enough components to get a significant negative one
418 kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=7)
419 # make sure that the appropriate error is raised
420 with pytest.raises(ValueError, match="There are significant negative eigenvalues"):
421 kpca.fit(K)
422
423 # 2. ask for a small enough n_components to get only positive ones
424 kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=2)
425 if solver == "randomized":
426 # the randomized method is still inconsistent with the others on this
427 # since it selects the eigenvalues based on the largest 2 modules, not
428 # on the largest 2 values.
429 #
430 # At least we can ensure that we return an error instead of returning
431 # the wrong eigenvalues
432 with pytest.raises(
433 ValueError, match="There are significant negative eigenvalues"
434 ):
435 kpca.fit(K)
436 else:
437 # general case: make sure that it works
438 kpca.fit(K)
439
440
441@pytest.mark.parametrize("n_components", [4, 10, 20])
442def test_kernel_pca_solvers_equivalence(n_components):
443 """Check that 'dense' 'arpack' & 'randomized' solvers give similar results"""
444
445 # Generate random data
446 n_train, n_test = 1_000, 100
447 X, _ = make_circles(
448 n_samples=(n_train + n_test), factor=0.3, noise=0.05, random_state=0
449 )
450 X_fit, X_pred = X[:n_train, :], X[n_train:, :]
451
452 # reference (full)
453 ref_pred = (
454 KernelPCA(n_components, eigen_solver="dense", random_state=0)
455 .fit(X_fit)
456 .transform(X_pred)
457 )
458
459 # arpack
460 a_pred = (
461 KernelPCA(n_components, eigen_solver="arpack", random_state=0)
462 .fit(X_fit)
463 .transform(X_pred)
464 )
465 # check that the result is still correct despite the approx
466 assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred))
467
468 # randomized
469 r_pred = (
470 KernelPCA(n_components, eigen_solver="randomized", random_state=0)
471 .fit(X_fit)
472 .transform(X_pred)
473 )
474 # check that the result is still correct despite the approximation
475 assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred))
476
477
478def test_kernel_pca_inverse_transform_reconstruction():
479 """Test if the reconstruction is a good approximation.
480
481 Note that in general it is not possible to get an arbitrarily good
482 reconstruction because of kernel centering that does not
483 preserve all the information of the original data.
484 """
485 X, *_ = make_blobs(n_samples=100, n_features=4, random_state=0)
486
487 kpca = KernelPCA(
488 n_components=20, kernel="rbf", fit_inverse_transform=True, alpha=1e-3
489 )
490 X_trans = kpca.fit_transform(X)
491 X_reconst = kpca.inverse_transform(X_trans)
492 assert np.linalg.norm(X - X_reconst) / np.linalg.norm(X) < 1e-1
493
494
495def test_kernel_pca_raise_not_fitted_error():
496 X = np.random.randn(15).reshape(5, 3)
497 kpca = KernelPCA()
498 kpca.fit(X)
499 with pytest.raises(NotFittedError):
500 kpca.inverse_transform(X)
501
502
503def test_32_64_decomposition_shape():
504 """Test that the decomposition is similar for 32 and 64 bits data
505
506 Non regression test for
507 https://github.com/scikit-learn/scikit-learn/issues/18146
508 """
509 X, y = make_blobs(
510 n_samples=30, centers=[[0, 0, 0], [1, 1, 1]], random_state=0, cluster_std=0.1
511 )
512 X = StandardScaler().fit_transform(X)
513 X -= X.min()
514
515 # Compare the shapes (corresponds to the number of non-zero eigenvalues)
516 kpca = KernelPCA()
517 assert kpca.fit_transform(X).shape == kpca.fit_transform(X.astype(np.float32)).shape
518
519
520def test_kernel_pca_feature_names_out():
521 """Check feature names out for KernelPCA."""
522 X, *_ = make_blobs(n_samples=100, n_features=4, random_state=0)
523 kpca = KernelPCA(n_components=2).fit(X)
524
525 names = kpca.get_feature_names_out()
526 assert_array_equal([f"kernelpca{i}" for i in range(2)], names)
527
528
529def test_kernel_pca_inverse_correct_gamma(global_random_seed):
530 """Check that gamma is set correctly when not provided.
531
532 Non-regression test for #26280
533 """
534 rng = np.random.RandomState(global_random_seed)
535 X = rng.random_sample((5, 4))
536
537 kwargs = {
538 "n_components": 2,
539 "random_state": rng,
540 "fit_inverse_transform": True,
541 "kernel": "rbf",
542 }
543
544 expected_gamma = 1 / X.shape[1]
545 kpca1 = KernelPCA(gamma=None, **kwargs).fit(X)
546 kpca2 = KernelPCA(gamma=expected_gamma, **kwargs).fit(X)
547
548 assert kpca1.gamma_ == expected_gamma
549 assert kpca2.gamma_ == expected_gamma
550
551 X1_recon = kpca1.inverse_transform(kpca1.transform(X))
552 X2_recon = kpca2.inverse_transform(kpca1.transform(X))
553
554 assert_allclose(X1_recon, X2_recon)
555
556
557def test_kernel_pca_pandas_output():
558 """Check that KernelPCA works with pandas output when the solver is arpack.
559
560 Non-regression test for:
561 https://github.com/scikit-learn/scikit-learn/issues/27579
562 """
563 pytest.importorskip("pandas")
564 X, _ = load_iris(as_frame=True, return_X_y=True)
565 with sklearn.config_context(transform_output="pandas"):
566 KernelPCA(n_components=2, eigen_solver="arpack").fit_transform(X)
567