Aluode/PerceptionLabPortable
0
1import numpy as np
2import pytest
3import scipy.sparse as sp
4from numpy.random import RandomState
5from numpy.testing import assert_array_almost_equal, assert_array_equal
6from scipy import linalg
7
8from sklearn.datasets import make_classification
9from sklearn.utils._testing import assert_allclose
10from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS, LIL_CONTAINERS
11from sklearn.utils.sparsefuncs import (
12 _implicit_column_offset,
13 count_nonzero,
14 csc_median_axis_0,
15 incr_mean_variance_axis,
16 inplace_column_scale,
17 inplace_row_scale,
18 inplace_swap_column,
19 inplace_swap_row,
20 mean_variance_axis,
21 min_max_axis,
22)
23from sklearn.utils.sparsefuncs_fast import (
24 assign_rows_csr,
25 csr_row_norms,
26 inplace_csr_row_normalize_l1,
27 inplace_csr_row_normalize_l2,
28)
29
30
31@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
32@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
33@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
34def test_mean_variance_axis0(csc_container, csr_container, lil_container):
35 X, _ = make_classification(5, 4, random_state=0)
36 # Sparsify the array a little bit
37 X[0, 0] = 0
38 X[2, 1] = 0
39 X[4, 3] = 0
40 X_lil = lil_container(X)
41 X_lil[1, 0] = 0
42 X[1, 0] = 0
43
44 with pytest.raises(TypeError):
45 mean_variance_axis(X_lil, axis=0)
46
47 X_csr = csr_container(X_lil)
48 X_csc = csc_container(X_lil)
49
50 expected_dtypes = [
51 (np.float32, np.float32),
52 (np.float64, np.float64),
53 (np.int32, np.float64),
54 (np.int64, np.float64),
55 ]
56
57 for input_dtype, output_dtype in expected_dtypes:
58 X_test = X.astype(input_dtype)
59 for X_sparse in (X_csr, X_csc):
60 X_sparse = X_sparse.astype(input_dtype)
61 X_means, X_vars = mean_variance_axis(X_sparse, axis=0)
62 assert X_means.dtype == output_dtype
63 assert X_vars.dtype == output_dtype
64 assert_array_almost_equal(X_means, np.mean(X_test, axis=0))
65 assert_array_almost_equal(X_vars, np.var(X_test, axis=0))
66
67
68@pytest.mark.parametrize("dtype", [np.float32, np.float64])
69@pytest.mark.parametrize("sparse_constructor", CSC_CONTAINERS + CSR_CONTAINERS)
70def test_mean_variance_axis0_precision(dtype, sparse_constructor):
71 # Check that there's no big loss of precision when the real variance is
72 # exactly 0. (#19766)
73 rng = np.random.RandomState(0)
74 X = np.full(fill_value=100.0, shape=(1000, 1), dtype=dtype)
75 # Add some missing records which should be ignored:
76 missing_indices = rng.choice(np.arange(X.shape[0]), 10, replace=False)
77 X[missing_indices, 0] = np.nan
78 X = sparse_constructor(X)
79
80 # Random positive weights:
81 sample_weight = rng.rand(X.shape[0]).astype(dtype)
82
83 _, var = mean_variance_axis(X, weights=sample_weight, axis=0)
84
85 assert var < np.finfo(dtype).eps
86
87
88@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
89@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
90@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
91def test_mean_variance_axis1(csc_container, csr_container, lil_container):
92 X, _ = make_classification(5, 4, random_state=0)
93 # Sparsify the array a little bit
94 X[0, 0] = 0
95 X[2, 1] = 0
96 X[4, 3] = 0
97 X_lil = lil_container(X)
98 X_lil[1, 0] = 0
99 X[1, 0] = 0
100
101 with pytest.raises(TypeError):
102 mean_variance_axis(X_lil, axis=1)
103
104 X_csr = csr_container(X_lil)
105 X_csc = csc_container(X_lil)
106
107 expected_dtypes = [
108 (np.float32, np.float32),
109 (np.float64, np.float64),
110 (np.int32, np.float64),
111 (np.int64, np.float64),
112 ]
113
114 for input_dtype, output_dtype in expected_dtypes:
115 X_test = X.astype(input_dtype)
116 for X_sparse in (X_csr, X_csc):
117 X_sparse = X_sparse.astype(input_dtype)
118 X_means, X_vars = mean_variance_axis(X_sparse, axis=0)
119 assert X_means.dtype == output_dtype
120 assert X_vars.dtype == output_dtype
121 assert_array_almost_equal(X_means, np.mean(X_test, axis=0))
122 assert_array_almost_equal(X_vars, np.var(X_test, axis=0))
123
124
125@pytest.mark.parametrize(
126 ["Xw", "X", "weights"],
127 [
128 ([[0, 0, 1], [0, 2, 3]], [[0, 0, 1], [0, 2, 3]], [1, 1, 1]),
129 ([[0, 0, 1], [0, 1, 1]], [[0, 0, 0, 1], [0, 1, 1, 1]], [1, 2, 1]),
130 ([[0, 0, 1], [0, 1, 1]], [[0, 0, 1], [0, 1, 1]], None),
131 (
132 [[0, np.nan, 2], [0, np.nan, np.nan]],
133 [[0, np.nan, 2], [0, np.nan, np.nan]],
134 [1.0, 1.0, 1.0],
135 ),
136 (
137 [[0, 0], [1, np.nan], [2, 0], [0, 3], [np.nan, np.nan], [np.nan, 2]],
138 [
139 [0, 0, 0],
140 [1, 1, np.nan],
141 [2, 2, 0],
142 [0, 0, 3],
143 [np.nan, np.nan, np.nan],
144 [np.nan, np.nan, 2],
145 ],
146 [2.0, 1.0],
147 ),
148 (
149 [[1, 0, 1], [0, 3, 1]],
150 [[1, 0, 0, 0, 1], [0, 3, 3, 3, 1]],
151 np.array([1, 3, 1]),
152 ),
153 ],
154)
155@pytest.mark.parametrize("sparse_constructor", CSC_CONTAINERS + CSR_CONTAINERS)
156@pytest.mark.parametrize("dtype", [np.float32, np.float64])
157def test_incr_mean_variance_axis_weighted_axis1(
158 Xw, X, weights, sparse_constructor, dtype
159):
160 axis = 1
161 Xw_sparse = sparse_constructor(Xw).astype(dtype)
162 X_sparse = sparse_constructor(X).astype(dtype)
163
164 last_mean = np.zeros(np.shape(Xw)[0], dtype=dtype)
165 last_var = np.zeros_like(last_mean, dtype=dtype)
166 last_n = np.zeros_like(last_mean, dtype=np.int64)
167 means0, vars0, n_incr0 = incr_mean_variance_axis(
168 X=X_sparse,
169 axis=axis,
170 last_mean=last_mean,
171 last_var=last_var,
172 last_n=last_n,
173 weights=None,
174 )
175
176 means_w0, vars_w0, n_incr_w0 = incr_mean_variance_axis(
177 X=Xw_sparse,
178 axis=axis,
179 last_mean=last_mean,
180 last_var=last_var,
181 last_n=last_n,
182 weights=weights,
183 )
184
185 assert means_w0.dtype == dtype
186 assert vars_w0.dtype == dtype
187 assert n_incr_w0.dtype == dtype
188
189 means_simple, vars_simple = mean_variance_axis(X=X_sparse, axis=axis)
190
191 assert_array_almost_equal(means0, means_w0)
192 assert_array_almost_equal(means0, means_simple)
193 assert_array_almost_equal(vars0, vars_w0)
194 assert_array_almost_equal(vars0, vars_simple)
195 assert_array_almost_equal(n_incr0, n_incr_w0)
196
197 # check second round for incremental
198 means1, vars1, n_incr1 = incr_mean_variance_axis(
199 X=X_sparse,
200 axis=axis,
201 last_mean=means0,
202 last_var=vars0,
203 last_n=n_incr0,
204 weights=None,
205 )
206
207 means_w1, vars_w1, n_incr_w1 = incr_mean_variance_axis(
208 X=Xw_sparse,
209 axis=axis,
210 last_mean=means_w0,
211 last_var=vars_w0,
212 last_n=n_incr_w0,
213 weights=weights,
214 )
215
216 assert_array_almost_equal(means1, means_w1)
217 assert_array_almost_equal(vars1, vars_w1)
218 assert_array_almost_equal(n_incr1, n_incr_w1)
219
220 assert means_w1.dtype == dtype
221 assert vars_w1.dtype == dtype
222 assert n_incr_w1.dtype == dtype
223
224
225@pytest.mark.parametrize(
226 ["Xw", "X", "weights"],
227 [
228 ([[0, 0, 1], [0, 2, 3]], [[0, 0, 1], [0, 2, 3]], [1, 1]),
229 ([[0, 0, 1], [0, 1, 1]], [[0, 0, 1], [0, 1, 1], [0, 1, 1]], [1, 2]),
230 ([[0, 0, 1], [0, 1, 1]], [[0, 0, 1], [0, 1, 1]], None),
231 (
232 [[0, np.nan, 2], [0, np.nan, np.nan]],
233 [[0, np.nan, 2], [0, np.nan, np.nan]],
234 [1.0, 1.0],
235 ),
236 (
237 [[0, 0, 1, np.nan, 2, 0], [0, 3, np.nan, np.nan, np.nan, 2]],
238 [
239 [0, 0, 1, np.nan, 2, 0],
240 [0, 0, 1, np.nan, 2, 0],
241 [0, 3, np.nan, np.nan, np.nan, 2],
242 ],
243 [2.0, 1.0],
244 ),
245 (
246 [[1, 0, 1], [0, 0, 1]],
247 [[1, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1]],
248 np.array([1, 3]),
249 ),
250 ],
251)
252@pytest.mark.parametrize("sparse_constructor", CSC_CONTAINERS + CSR_CONTAINERS)
253@pytest.mark.parametrize("dtype", [np.float32, np.float64])
254def test_incr_mean_variance_axis_weighted_axis0(
255 Xw, X, weights, sparse_constructor, dtype
256):
257 axis = 0
258 Xw_sparse = sparse_constructor(Xw).astype(dtype)
259 X_sparse = sparse_constructor(X).astype(dtype)
260
261 last_mean = np.zeros(np.size(Xw, 1), dtype=dtype)
262 last_var = np.zeros_like(last_mean)
263 last_n = np.zeros_like(last_mean, dtype=np.int64)
264 means0, vars0, n_incr0 = incr_mean_variance_axis(
265 X=X_sparse,
266 axis=axis,
267 last_mean=last_mean,
268 last_var=last_var,
269 last_n=last_n,
270 weights=None,
271 )
272
273 means_w0, vars_w0, n_incr_w0 = incr_mean_variance_axis(
274 X=Xw_sparse,
275 axis=axis,
276 last_mean=last_mean,
277 last_var=last_var,
278 last_n=last_n,
279 weights=weights,
280 )
281
282 assert means_w0.dtype == dtype
283 assert vars_w0.dtype == dtype
284 assert n_incr_w0.dtype == dtype
285
286 means_simple, vars_simple = mean_variance_axis(X=X_sparse, axis=axis)
287
288 assert_array_almost_equal(means0, means_w0)
289 assert_array_almost_equal(means0, means_simple)
290 assert_array_almost_equal(vars0, vars_w0)
291 assert_array_almost_equal(vars0, vars_simple)
292 assert_array_almost_equal(n_incr0, n_incr_w0)
293
294 # check second round for incremental
295 means1, vars1, n_incr1 = incr_mean_variance_axis(
296 X=X_sparse,
297 axis=axis,
298 last_mean=means0,
299 last_var=vars0,
300 last_n=n_incr0,
301 weights=None,
302 )
303
304 means_w1, vars_w1, n_incr_w1 = incr_mean_variance_axis(
305 X=Xw_sparse,
306 axis=axis,
307 last_mean=means_w0,
308 last_var=vars_w0,
309 last_n=n_incr_w0,
310 weights=weights,
311 )
312
313 assert_array_almost_equal(means1, means_w1)
314 assert_array_almost_equal(vars1, vars_w1)
315 assert_array_almost_equal(n_incr1, n_incr_w1)
316
317 assert means_w1.dtype == dtype
318 assert vars_w1.dtype == dtype
319 assert n_incr_w1.dtype == dtype
320
321
322@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
323@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
324@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
325def test_incr_mean_variance_axis(csc_container, csr_container, lil_container):
326 for axis in [0, 1]:
327 rng = np.random.RandomState(0)
328 n_features = 50
329 n_samples = 10
330 if axis == 0:
331 data_chunks = [rng.randint(0, 2, size=n_features) for i in range(n_samples)]
332 else:
333 data_chunks = [rng.randint(0, 2, size=n_samples) for i in range(n_features)]
334
335 # default params for incr_mean_variance
336 last_mean = np.zeros(n_features) if axis == 0 else np.zeros(n_samples)
337 last_var = np.zeros_like(last_mean)
338 last_n = np.zeros_like(last_mean, dtype=np.int64)
339
340 # Test errors
341 X = np.array(data_chunks[0])
342 X = np.atleast_2d(X)
343 X = X.T if axis == 1 else X
344 X_lil = lil_container(X)
345 X_csr = csr_container(X_lil)
346
347 with pytest.raises(TypeError):
348 incr_mean_variance_axis(
349 X=axis, axis=last_mean, last_mean=last_var, last_var=last_n
350 )
351 with pytest.raises(TypeError):
352 incr_mean_variance_axis(
353 X_lil, axis=axis, last_mean=last_mean, last_var=last_var, last_n=last_n
354 )
355
356 # Test _incr_mean_and_var with a 1 row input
357 X_means, X_vars = mean_variance_axis(X_csr, axis)
358 X_means_incr, X_vars_incr, n_incr = incr_mean_variance_axis(
359 X_csr, axis=axis, last_mean=last_mean, last_var=last_var, last_n=last_n
360 )
361 assert_array_almost_equal(X_means, X_means_incr)
362 assert_array_almost_equal(X_vars, X_vars_incr)
363 # X.shape[axis] picks # samples
364 assert_array_equal(X.shape[axis], n_incr)
365
366 X_csc = csc_container(X_lil)
367 X_means, X_vars = mean_variance_axis(X_csc, axis)
368 assert_array_almost_equal(X_means, X_means_incr)
369 assert_array_almost_equal(X_vars, X_vars_incr)
370 assert_array_equal(X.shape[axis], n_incr)
371
372 # Test _incremental_mean_and_var with whole data
373 X = np.vstack(data_chunks)
374 X = X.T if axis == 1 else X
375 X_lil = lil_container(X)
376 X_csr = csr_container(X_lil)
377 X_csc = csc_container(X_lil)
378
379 expected_dtypes = [
380 (np.float32, np.float32),
381 (np.float64, np.float64),
382 (np.int32, np.float64),
383 (np.int64, np.float64),
384 ]
385
386 for input_dtype, output_dtype in expected_dtypes:
387 for X_sparse in (X_csr, X_csc):
388 X_sparse = X_sparse.astype(input_dtype)
389 last_mean = last_mean.astype(output_dtype)
390 last_var = last_var.astype(output_dtype)
391 X_means, X_vars = mean_variance_axis(X_sparse, axis)
392 X_means_incr, X_vars_incr, n_incr = incr_mean_variance_axis(
393 X_sparse,
394 axis=axis,
395 last_mean=last_mean,
396 last_var=last_var,
397 last_n=last_n,
398 )
399 assert X_means_incr.dtype == output_dtype
400 assert X_vars_incr.dtype == output_dtype
401 assert_array_almost_equal(X_means, X_means_incr)
402 assert_array_almost_equal(X_vars, X_vars_incr)
403 assert_array_equal(X.shape[axis], n_incr)
404
405
406@pytest.mark.parametrize("sparse_constructor", CSC_CONTAINERS + CSR_CONTAINERS)
407def test_incr_mean_variance_axis_dim_mismatch(sparse_constructor):
408 """Check that we raise proper error when axis=1 and the dimension mismatch.
409 Non-regression test for:
410 https://github.com/scikit-learn/scikit-learn/pull/18655
411 """
412 n_samples, n_features = 60, 4
413 rng = np.random.RandomState(42)
414 X = sparse_constructor(rng.rand(n_samples, n_features))
415
416 last_mean = np.zeros(n_features)
417 last_var = np.zeros_like(last_mean)
418 last_n = np.zeros(last_mean.shape, dtype=np.int64)
419
420 kwargs = dict(last_mean=last_mean, last_var=last_var, last_n=last_n)
421 mean0, var0, _ = incr_mean_variance_axis(X, axis=0, **kwargs)
422 assert_allclose(np.mean(X.toarray(), axis=0), mean0)
423 assert_allclose(np.var(X.toarray(), axis=0), var0)
424
425 # test ValueError if axis=1 and last_mean.size == n_features
426 with pytest.raises(ValueError):
427 incr_mean_variance_axis(X, axis=1, **kwargs)
428
429 # test inconsistent shapes of last_mean, last_var, last_n
430 kwargs = dict(last_mean=last_mean[:-1], last_var=last_var, last_n=last_n)
431 with pytest.raises(ValueError):
432 incr_mean_variance_axis(X, axis=0, **kwargs)
433
434
435@pytest.mark.parametrize(
436 "X1, X2",
437 [
438 (
439 sp.random(5, 2, density=0.8, format="csr", random_state=0),
440 sp.random(13, 2, density=0.8, format="csr", random_state=0),
441 ),
442 (
443 sp.random(5, 2, density=0.8, format="csr", random_state=0),
444 sp.hstack(
445 [
446 np.full((13, 1), fill_value=np.nan),
447 sp.random(13, 1, density=0.8, random_state=42),
448 ],
449 format="csr",
450 ),
451 ),
452 ],
453)
454@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
455def test_incr_mean_variance_axis_equivalence_mean_variance(X1, X2, csr_container):
456 # non-regression test for:
457 # https://github.com/scikit-learn/scikit-learn/issues/16448
458 # check that computing the incremental mean and variance is equivalent to
459 # computing the mean and variance on the stacked dataset.
460 X1 = csr_container(X1)
461 X2 = csr_container(X2)
462 axis = 0
463 last_mean, last_var = np.zeros(X1.shape[1]), np.zeros(X1.shape[1])
464 last_n = np.zeros(X1.shape[1], dtype=np.int64)
465 updated_mean, updated_var, updated_n = incr_mean_variance_axis(
466 X1, axis=axis, last_mean=last_mean, last_var=last_var, last_n=last_n
467 )
468 updated_mean, updated_var, updated_n = incr_mean_variance_axis(
469 X2, axis=axis, last_mean=updated_mean, last_var=updated_var, last_n=updated_n
470 )
471 X = sp.vstack([X1, X2])
472 assert_allclose(updated_mean, np.nanmean(X.toarray(), axis=axis))
473 assert_allclose(updated_var, np.nanvar(X.toarray(), axis=axis))
474 assert_allclose(updated_n, np.count_nonzero(~np.isnan(X.toarray()), axis=0))
475
476
477def test_incr_mean_variance_no_new_n():
478 # check the behaviour when we update the variance with an empty matrix
479 axis = 0
480 X1 = sp.random(5, 1, density=0.8, random_state=0).tocsr()
481 X2 = sp.random(0, 1, density=0.8, random_state=0).tocsr()
482 last_mean, last_var = np.zeros(X1.shape[1]), np.zeros(X1.shape[1])
483 last_n = np.zeros(X1.shape[1], dtype=np.int64)
484 last_mean, last_var, last_n = incr_mean_variance_axis(
485 X1, axis=axis, last_mean=last_mean, last_var=last_var, last_n=last_n
486 )
487 # update statistic with a column which should ignored
488 updated_mean, updated_var, updated_n = incr_mean_variance_axis(
489 X2, axis=axis, last_mean=last_mean, last_var=last_var, last_n=last_n
490 )
491 assert_allclose(updated_mean, last_mean)
492 assert_allclose(updated_var, last_var)
493 assert_allclose(updated_n, last_n)
494
495
496def test_incr_mean_variance_n_float():
497 # check the behaviour when last_n is just a number
498 axis = 0
499 X = sp.random(5, 2, density=0.8, random_state=0).tocsr()
500 last_mean, last_var = np.zeros(X.shape[1]), np.zeros(X.shape[1])
501 last_n = 0
502 _, _, new_n = incr_mean_variance_axis(
503 X, axis=axis, last_mean=last_mean, last_var=last_var, last_n=last_n
504 )
505 assert_allclose(new_n, np.full(X.shape[1], X.shape[0]))
506
507
508@pytest.mark.parametrize("axis", [0, 1])
509@pytest.mark.parametrize("sparse_constructor", CSC_CONTAINERS + CSR_CONTAINERS)
510def test_incr_mean_variance_axis_ignore_nan(axis, sparse_constructor):
511 old_means = np.array([535.0, 535.0, 535.0, 535.0])
512 old_variances = np.array([4225.0, 4225.0, 4225.0, 4225.0])
513 old_sample_count = np.array([2, 2, 2, 2], dtype=np.int64)
514
515 X = sparse_constructor(
516 np.array([[170, 170, 170, 170], [430, 430, 430, 430], [300, 300, 300, 300]])
517 )
518
519 X_nan = sparse_constructor(
520 np.array(
521 [
522 [170, np.nan, 170, 170],
523 [np.nan, 170, 430, 430],
524 [430, 430, np.nan, 300],
525 [300, 300, 300, np.nan],
526 ]
527 )
528 )
529
530 # we avoid creating specific data for axis 0 and 1: translating the data is
531 # enough.
532 if axis:
533 X = X.T
534 X_nan = X_nan.T
535
536 # take a copy of the old statistics since they are modified in place.
537 X_means, X_vars, X_sample_count = incr_mean_variance_axis(
538 X,
539 axis=axis,
540 last_mean=old_means.copy(),
541 last_var=old_variances.copy(),
542 last_n=old_sample_count.copy(),
543 )
544 X_nan_means, X_nan_vars, X_nan_sample_count = incr_mean_variance_axis(
545 X_nan,
546 axis=axis,
547 last_mean=old_means.copy(),
548 last_var=old_variances.copy(),
549 last_n=old_sample_count.copy(),
550 )
551
552 assert_allclose(X_nan_means, X_means)
553 assert_allclose(X_nan_vars, X_vars)
554 assert_allclose(X_nan_sample_count, X_sample_count)
555
556
557@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
558def test_mean_variance_illegal_axis(csr_container):
559 X, _ = make_classification(5, 4, random_state=0)
560 # Sparsify the array a little bit
561 X[0, 0] = 0
562 X[2, 1] = 0
563 X[4, 3] = 0
564 X_csr = csr_container(X)
565 with pytest.raises(ValueError):
566 mean_variance_axis(X_csr, axis=-3)
567 with pytest.raises(ValueError):
568 mean_variance_axis(X_csr, axis=2)
569 with pytest.raises(ValueError):
570 mean_variance_axis(X_csr, axis=-1)
571
572 with pytest.raises(ValueError):
573 incr_mean_variance_axis(
574 X_csr, axis=-3, last_mean=None, last_var=None, last_n=None
575 )
576
577 with pytest.raises(ValueError):
578 incr_mean_variance_axis(
579 X_csr, axis=2, last_mean=None, last_var=None, last_n=None
580 )
581
582 with pytest.raises(ValueError):
583 incr_mean_variance_axis(
584 X_csr, axis=-1, last_mean=None, last_var=None, last_n=None
585 )
586
587
588@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
589def test_densify_rows(csr_container):
590 for dtype in (np.float32, np.float64):
591 X = csr_container(
592 [[0, 3, 0], [2, 4, 0], [0, 0, 0], [9, 8, 7], [4, 0, 5]], dtype=dtype
593 )
594 X_rows = np.array([0, 2, 3], dtype=np.intp)
595 out = np.ones((6, X.shape[1]), dtype=dtype)
596 out_rows = np.array([1, 3, 4], dtype=np.intp)
597
598 expect = np.ones_like(out)
599 expect[out_rows] = X[X_rows, :].toarray()
600
601 assign_rows_csr(X, X_rows, out_rows, out)
602 assert_array_equal(out, expect)
603
604
605def test_inplace_column_scale():
606 rng = np.random.RandomState(0)
607 X = sp.random(100, 200, density=0.05)
608 Xr = X.tocsr()
609 Xc = X.tocsc()
610 XA = X.toarray()
611 scale = rng.rand(200)
612 XA *= scale
613
614 inplace_column_scale(Xc, scale)
615 inplace_column_scale(Xr, scale)
616 assert_array_almost_equal(Xr.toarray(), Xc.toarray())
617 assert_array_almost_equal(XA, Xc.toarray())
618 assert_array_almost_equal(XA, Xr.toarray())
619 with pytest.raises(TypeError):
620 inplace_column_scale(X.tolil(), scale)
621
622 X = X.astype(np.float32)
623 scale = scale.astype(np.float32)
624 Xr = X.tocsr()
625 Xc = X.tocsc()
626 XA = X.toarray()
627 XA *= scale
628 inplace_column_scale(Xc, scale)
629 inplace_column_scale(Xr, scale)
630 assert_array_almost_equal(Xr.toarray(), Xc.toarray())
631 assert_array_almost_equal(XA, Xc.toarray())
632 assert_array_almost_equal(XA, Xr.toarray())
633 with pytest.raises(TypeError):
634 inplace_column_scale(X.tolil(), scale)
635
636
637def test_inplace_row_scale():
638 rng = np.random.RandomState(0)
639 X = sp.random(100, 200, density=0.05)
640 Xr = X.tocsr()
641 Xc = X.tocsc()
642 XA = X.toarray()
643 scale = rng.rand(100)
644 XA *= scale.reshape(-1, 1)
645
646 inplace_row_scale(Xc, scale)
647 inplace_row_scale(Xr, scale)
648 assert_array_almost_equal(Xr.toarray(), Xc.toarray())
649 assert_array_almost_equal(XA, Xc.toarray())
650 assert_array_almost_equal(XA, Xr.toarray())
651 with pytest.raises(TypeError):
652 inplace_column_scale(X.tolil(), scale)
653
654 X = X.astype(np.float32)
655 scale = scale.astype(np.float32)
656 Xr = X.tocsr()
657 Xc = X.tocsc()
658 XA = X.toarray()
659 XA *= scale.reshape(-1, 1)
660 inplace_row_scale(Xc, scale)
661 inplace_row_scale(Xr, scale)
662 assert_array_almost_equal(Xr.toarray(), Xc.toarray())
663 assert_array_almost_equal(XA, Xc.toarray())
664 assert_array_almost_equal(XA, Xr.toarray())
665 with pytest.raises(TypeError):
666 inplace_column_scale(X.tolil(), scale)
667
668
669@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
670@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
671def test_inplace_swap_row(csc_container, csr_container):
672 X = np.array(
673 [[0, 3, 0], [2, 4, 0], [0, 0, 0], [9, 8, 7], [4, 0, 5]], dtype=np.float64
674 )
675 X_csr = csr_container(X)
676 X_csc = csc_container(X)
677
678 swap = linalg.get_blas_funcs(("swap",), (X,))
679 swap = swap[0]
680 X[0], X[-1] = swap(X[0], X[-1])
681 inplace_swap_row(X_csr, 0, -1)
682 inplace_swap_row(X_csc, 0, -1)
683 assert_array_equal(X_csr.toarray(), X_csc.toarray())
684 assert_array_equal(X, X_csc.toarray())
685 assert_array_equal(X, X_csr.toarray())
686
687 X[2], X[3] = swap(X[2], X[3])
688 inplace_swap_row(X_csr, 2, 3)
689 inplace_swap_row(X_csc, 2, 3)
690 assert_array_equal(X_csr.toarray(), X_csc.toarray())
691 assert_array_equal(X, X_csc.toarray())
692 assert_array_equal(X, X_csr.toarray())
693 with pytest.raises(TypeError):
694 inplace_swap_row(X_csr.tolil())
695
696 X = np.array(
697 [[0, 3, 0], [2, 4, 0], [0, 0, 0], [9, 8, 7], [4, 0, 5]], dtype=np.float32
698 )
699 X_csr = csr_container(X)
700 X_csc = csc_container(X)
701 swap = linalg.get_blas_funcs(("swap",), (X,))
702 swap = swap[0]
703 X[0], X[-1] = swap(X[0], X[-1])
704 inplace_swap_row(X_csr, 0, -1)
705 inplace_swap_row(X_csc, 0, -1)
706 assert_array_equal(X_csr.toarray(), X_csc.toarray())
707 assert_array_equal(X, X_csc.toarray())
708 assert_array_equal(X, X_csr.toarray())
709 X[2], X[3] = swap(X[2], X[3])
710 inplace_swap_row(X_csr, 2, 3)
711 inplace_swap_row(X_csc, 2, 3)
712 assert_array_equal(X_csr.toarray(), X_csc.toarray())
713 assert_array_equal(X, X_csc.toarray())
714 assert_array_equal(X, X_csr.toarray())
715 with pytest.raises(TypeError):
716 inplace_swap_row(X_csr.tolil())
717
718
719@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
720@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
721def test_inplace_swap_column(csc_container, csr_container):
722 X = np.array(
723 [[0, 3, 0], [2, 4, 0], [0, 0, 0], [9, 8, 7], [4, 0, 5]], dtype=np.float64
724 )
725 X_csr = csr_container(X)
726 X_csc = csc_container(X)
727
728 swap = linalg.get_blas_funcs(("swap",), (X,))
729 swap = swap[0]
730 X[:, 0], X[:, -1] = swap(X[:, 0], X[:, -1])
731 inplace_swap_column(X_csr, 0, -1)
732 inplace_swap_column(X_csc, 0, -1)
733 assert_array_equal(X_csr.toarray(), X_csc.toarray())
734 assert_array_equal(X, X_csc.toarray())
735 assert_array_equal(X, X_csr.toarray())
736
737 X[:, 0], X[:, 1] = swap(X[:, 0], X[:, 1])
738 inplace_swap_column(X_csr, 0, 1)
739 inplace_swap_column(X_csc, 0, 1)
740 assert_array_equal(X_csr.toarray(), X_csc.toarray())
741 assert_array_equal(X, X_csc.toarray())
742 assert_array_equal(X, X_csr.toarray())
743 with pytest.raises(TypeError):
744 inplace_swap_column(X_csr.tolil())
745
746 X = np.array(
747 [[0, 3, 0], [2, 4, 0], [0, 0, 0], [9, 8, 7], [4, 0, 5]], dtype=np.float32
748 )
749 X_csr = csr_container(X)
750 X_csc = csc_container(X)
751 swap = linalg.get_blas_funcs(("swap",), (X,))
752 swap = swap[0]
753 X[:, 0], X[:, -1] = swap(X[:, 0], X[:, -1])
754 inplace_swap_column(X_csr, 0, -1)
755 inplace_swap_column(X_csc, 0, -1)
756 assert_array_equal(X_csr.toarray(), X_csc.toarray())
757 assert_array_equal(X, X_csc.toarray())
758 assert_array_equal(X, X_csr.toarray())
759 X[:, 0], X[:, 1] = swap(X[:, 0], X[:, 1])
760 inplace_swap_column(X_csr, 0, 1)
761 inplace_swap_column(X_csc, 0, 1)
762 assert_array_equal(X_csr.toarray(), X_csc.toarray())
763 assert_array_equal(X, X_csc.toarray())
764 assert_array_equal(X, X_csr.toarray())
765 with pytest.raises(TypeError):
766 inplace_swap_column(X_csr.tolil())
767
768
769@pytest.mark.parametrize("dtype", [np.float32, np.float64])
770@pytest.mark.parametrize("axis", [0, 1, None])
771@pytest.mark.parametrize("sparse_format", CSC_CONTAINERS + CSR_CONTAINERS)
772@pytest.mark.parametrize(
773 "missing_values, min_func, max_func, ignore_nan",
774 [(0, np.min, np.max, False), (np.nan, np.nanmin, np.nanmax, True)],
775)
776@pytest.mark.parametrize("large_indices", [True, False])
777def test_min_max(
778 dtype,
779 axis,
780 sparse_format,
781 missing_values,
782 min_func,
783 max_func,
784 ignore_nan,
785 large_indices,
786):
787 X = np.array(
788 [
789 [0, 3, 0],
790 [2, -1, missing_values],
791 [0, 0, 0],
792 [9, missing_values, 7],
793 [4, 0, 5],
794 ],
795 dtype=dtype,
796 )
797 X_sparse = sparse_format(X)
798
799 if large_indices:
800 X_sparse.indices = X_sparse.indices.astype("int64")
801 X_sparse.indptr = X_sparse.indptr.astype("int64")
802
803 mins_sparse, maxs_sparse = min_max_axis(X_sparse, axis=axis, ignore_nan=ignore_nan)
804 assert_array_equal(mins_sparse, min_func(X, axis=axis))
805 assert_array_equal(maxs_sparse, max_func(X, axis=axis))
806
807
808@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
809@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
810def test_min_max_axis_errors(csc_container, csr_container):
811 X = np.array(
812 [[0, 3, 0], [2, -1, 0], [0, 0, 0], [9, 8, 7], [4, 0, 5]], dtype=np.float64
813 )
814 X_csr = csr_container(X)
815 X_csc = csc_container(X)
816 with pytest.raises(TypeError):
817 min_max_axis(X_csr.tolil(), axis=0)
818 with pytest.raises(ValueError):
819 min_max_axis(X_csr, axis=2)
820 with pytest.raises(ValueError):
821 min_max_axis(X_csc, axis=-3)
822
823
824@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
825@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
826def test_count_nonzero(csc_container, csr_container):
827 X = np.array(
828 [[0, 3, 0], [2, -1, 0], [0, 0, 0], [9, 8, 7], [4, 0, 5]], dtype=np.float64
829 )
830 X_csr = csr_container(X)
831 X_csc = csc_container(X)
832 X_nonzero = X != 0
833 sample_weight = [0.5, 0.2, 0.3, 0.1, 0.1]
834 X_nonzero_weighted = X_nonzero * np.array(sample_weight)[:, None]
835
836 for axis in [0, 1, -1, -2, None]:
837 assert_array_almost_equal(
838 count_nonzero(X_csr, axis=axis), X_nonzero.sum(axis=axis)
839 )
840 assert_array_almost_equal(
841 count_nonzero(X_csr, axis=axis, sample_weight=sample_weight),
842 X_nonzero_weighted.sum(axis=axis),
843 )
844
845 with pytest.raises(TypeError):
846 count_nonzero(X_csc)
847 with pytest.raises(ValueError):
848 count_nonzero(X_csr, axis=2)
849
850 assert count_nonzero(X_csr, axis=0).dtype == count_nonzero(X_csr, axis=1).dtype
851 assert (
852 count_nonzero(X_csr, axis=0, sample_weight=sample_weight).dtype
853 == count_nonzero(X_csr, axis=1, sample_weight=sample_weight).dtype
854 )
855
856 # Check dtypes with large sparse matrices too
857 # XXX: test fails on 32bit (Windows/Linux)
858 try:
859 X_csr.indices = X_csr.indices.astype(np.int64)
860 X_csr.indptr = X_csr.indptr.astype(np.int64)
861 assert count_nonzero(X_csr, axis=0).dtype == count_nonzero(X_csr, axis=1).dtype
862 assert (
863 count_nonzero(X_csr, axis=0, sample_weight=sample_weight).dtype
864 == count_nonzero(X_csr, axis=1, sample_weight=sample_weight).dtype
865 )
866 except TypeError as e:
867 assert "according to the rule 'safe'" in e.args[0] and np.intp().nbytes < 8, e
868
869
870@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
871@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
872def test_csc_row_median(csc_container, csr_container):
873 # Test csc_row_median actually calculates the median.
874
875 # Test that it gives the same output when X is dense.
876 rng = np.random.RandomState(0)
877 X = rng.rand(100, 50)
878 dense_median = np.median(X, axis=0)
879 csc = csc_container(X)
880 sparse_median = csc_median_axis_0(csc)
881 assert_array_equal(sparse_median, dense_median)
882
883 # Test that it gives the same output when X is sparse
884 X = rng.rand(51, 100)
885 X[X < 0.7] = 0.0
886 ind = rng.randint(0, 50, 10)
887 X[ind] = -X[ind]
888 csc = csc_container(X)
889 dense_median = np.median(X, axis=0)
890 sparse_median = csc_median_axis_0(csc)
891 assert_array_equal(sparse_median, dense_median)
892
893 # Test for toy data.
894 X = [[0, -2], [-1, -1], [1, 0], [2, 1]]
895 csc = csc_container(X)
896 assert_array_equal(csc_median_axis_0(csc), np.array([0.5, -0.5]))
897 X = [[0, -2], [-1, -5], [1, -3]]
898 csc = csc_container(X)
899 assert_array_equal(csc_median_axis_0(csc), np.array([0.0, -3]))
900
901 # Test that it raises an Error for non-csc matrices.
902 with pytest.raises(TypeError):
903 csc_median_axis_0(csr_container(X))
904
905
906@pytest.mark.parametrize(
907 "inplace_csr_row_normalize",
908 (inplace_csr_row_normalize_l1, inplace_csr_row_normalize_l2),
909)
910@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
911def test_inplace_normalize(csr_container, inplace_csr_row_normalize):
912 if csr_container is sp.csr_matrix:
913 ones = np.ones((10, 1))
914 else:
915 ones = np.ones(10)
916 rs = RandomState(10)
917
918 for dtype in (np.float64, np.float32):
919 X = rs.randn(10, 5).astype(dtype)
920 X_csr = csr_container(X)
921 for index_dtype in [np.int32, np.int64]:
922 # csr_matrix will use int32 indices by default,
923 # up-casting those to int64 when necessary
924 if index_dtype is np.int64:
925 X_csr.indptr = X_csr.indptr.astype(index_dtype)
926 X_csr.indices = X_csr.indices.astype(index_dtype)
927 assert X_csr.indices.dtype == index_dtype
928 assert X_csr.indptr.dtype == index_dtype
929 inplace_csr_row_normalize(X_csr)
930 assert X_csr.dtype == dtype
931 if inplace_csr_row_normalize is inplace_csr_row_normalize_l2:
932 X_csr.data **= 2
933 assert_array_almost_equal(np.abs(X_csr).sum(axis=1), ones)
934
935
936@pytest.mark.parametrize("dtype", [np.float32, np.float64])
937def test_csr_row_norms(dtype):
938 # checks that csr_row_norms returns the same output as
939 # scipy.sparse.linalg.norm, and that the dype is the same as X.dtype.
940 X = sp.random(100, 10, format="csr", dtype=dtype, random_state=42)
941
942 scipy_norms = sp.linalg.norm(X, axis=1) ** 2
943 norms = csr_row_norms(X)
944
945 assert norms.dtype == dtype
946 rtol = 1e-6 if dtype == np.float32 else 1e-7
947 assert_allclose(norms, scipy_norms, rtol=rtol)
948
949
950@pytest.fixture(scope="module", params=CSR_CONTAINERS + CSC_CONTAINERS)
951def centered_matrices(request):
952 """Returns equivalent tuple[sp.linalg.LinearOperator, np.ndarray]."""
953 sparse_container = request.param
954
955 random_state = np.random.default_rng(42)
956
957 X_sparse = sparse_container(
958 sp.random(500, 100, density=0.1, format="csr", random_state=random_state)
959 )
960 X_dense = X_sparse.toarray()
961 mu = np.asarray(X_sparse.mean(axis=0)).ravel()
962
963 X_sparse_centered = _implicit_column_offset(X_sparse, mu)
964 X_dense_centered = X_dense - mu
965
966 return X_sparse_centered, X_dense_centered
967
968
969def test_implicit_center_matmat(global_random_seed, centered_matrices):
970 X_sparse_centered, X_dense_centered = centered_matrices
971 rng = np.random.default_rng(global_random_seed)
972 Y = rng.standard_normal((X_dense_centered.shape[1], 50))
973 assert_allclose(X_dense_centered @ Y, X_sparse_centered.matmat(Y))
974 assert_allclose(X_dense_centered @ Y, X_sparse_centered @ Y)
975
976
977def test_implicit_center_matvec(global_random_seed, centered_matrices):
978 X_sparse_centered, X_dense_centered = centered_matrices
979 rng = np.random.default_rng(global_random_seed)
980 y = rng.standard_normal(X_dense_centered.shape[1])
981 assert_allclose(X_dense_centered @ y, X_sparse_centered.matvec(y))
982 assert_allclose(X_dense_centered @ y, X_sparse_centered @ y)
983
984
985def test_implicit_center_rmatmat(global_random_seed, centered_matrices):
986 X_sparse_centered, X_dense_centered = centered_matrices
987 rng = np.random.default_rng(global_random_seed)
988 Y = rng.standard_normal((X_dense_centered.shape[0], 50))
989 assert_allclose(X_dense_centered.T @ Y, X_sparse_centered.rmatmat(Y))
990 assert_allclose(X_dense_centered.T @ Y, X_sparse_centered.T @ Y)
991
992
993def test_implit_center_rmatvec(global_random_seed, centered_matrices):
994 X_sparse_centered, X_dense_centered = centered_matrices
995 rng = np.random.default_rng(global_random_seed)
996 y = rng.standard_normal(X_dense_centered.shape[0])
997 assert_allclose(X_dense_centered.T @ y, X_sparse_centered.rmatvec(y))
998 assert_allclose(X_dense_centered.T @ y, X_sparse_centered.T @ y)
999