Aluode/PerceptionLabPortable
0
1"""Compatibility fixes for older version of python, numpy and scipy
2
3If you add content to this file, please give the version of the package
4at which the fix is no longer needed.
5"""
6
7# Authors: The scikit-learn developers
8# SPDX-License-Identifier: BSD-3-Clause
9
10import platform
11import struct
12
13import numpy as np
14import scipy
15import scipy.sparse.linalg
16import scipy.stats
17from scipy import optimize
18
19try:
20 import pandas as pd
21except ImportError:
22 pd = None
23
24from ..externals._packaging.version import parse as parse_version
25from .parallel import _get_threadpool_controller
26
27_IS_32BIT = 8 * struct.calcsize("P") == 32
28_IS_WASM = platform.machine() in ["wasm32", "wasm64"]
29
30np_version = parse_version(np.__version__)
31np_base_version = parse_version(np_version.base_version)
32sp_version = parse_version(scipy.__version__)
33sp_base_version = parse_version(sp_version.base_version)
34
35# TODO: We can consider removing the containers and importing
36# directly from SciPy when sparse matrices will be deprecated.
37CSR_CONTAINERS = [scipy.sparse.csr_matrix, scipy.sparse.csr_array]
38CSC_CONTAINERS = [scipy.sparse.csc_matrix, scipy.sparse.csc_array]
39COO_CONTAINERS = [scipy.sparse.coo_matrix, scipy.sparse.coo_array]
40LIL_CONTAINERS = [scipy.sparse.lil_matrix, scipy.sparse.lil_array]
41DOK_CONTAINERS = [scipy.sparse.dok_matrix, scipy.sparse.dok_array]
42BSR_CONTAINERS = [scipy.sparse.bsr_matrix, scipy.sparse.bsr_array]
43DIA_CONTAINERS = [scipy.sparse.dia_matrix, scipy.sparse.dia_array]
44
45# Remove when minimum scipy version is 1.11.0
46try:
47 from scipy.sparse import sparray # noqa: F401
48
49 SPARRAY_PRESENT = True
50except ImportError:
51 SPARRAY_PRESENT = False
52
53
54def _object_dtype_isnan(X):
55 return X != X
56
57
58# TODO: Remove when SciPy 1.11 is the minimum supported version
59def _mode(a, axis=0):
60 if sp_version >= parse_version("1.9.0"):
61 mode = scipy.stats.mode(a, axis=axis, keepdims=True)
62 if sp_version >= parse_version("1.10.999"):
63 # scipy.stats.mode has changed returned array shape with axis=None
64 # and keepdims=True, see https://github.com/scipy/scipy/pull/17561
65 if axis is None:
66 mode = np.ravel(mode)
67 return mode
68 return scipy.stats.mode(a, axis=axis)
69
70
71# TODO: Remove when Scipy 1.12 is the minimum supported version
72if sp_base_version >= parse_version("1.12.0"):
73 _sparse_linalg_cg = scipy.sparse.linalg.cg
74else:
75
76 def _sparse_linalg_cg(A, b, **kwargs):
77 if "rtol" in kwargs:
78 kwargs["tol"] = kwargs.pop("rtol")
79 if "atol" not in kwargs:
80 kwargs["atol"] = "legacy"
81 return scipy.sparse.linalg.cg(A, b, **kwargs)
82
83
84# TODO : remove this when required minimum version of scipy >= 1.9.0
85def _yeojohnson_lambda(_neg_log_likelihood, x):
86 """Estimate the optimal Yeo-Johnson transformation parameter (lambda).
87
88 This function provides a compatibility workaround for versions of SciPy
89 older than 1.9.0, where `scipy.stats.yeojohnson` did not return
90 the estimated lambda directly.
91
92 Parameters
93 ----------
94 _neg_log_likelihood : callable
95 A function that computes the negative log-likelihood of the Yeo-Johnson
96 transformation for a given lambda. Used only for SciPy versions < 1.9.0.
97
98 x : array-like
99 Input data to estimate the Yeo-Johnson transformation parameter.
100
101 Returns
102 -------
103 lmbda : float
104 The estimated lambda parameter for the Yeo-Johnson transformation.
105 """
106 min_scipy_version = "1.9.0"
107
108 if sp_version < parse_version(min_scipy_version):
109 # choosing bracket -2, 2 like for boxcox
110 return optimize.brent(_neg_log_likelihood, brack=(-2, 2))
111
112 _, lmbda = scipy.stats.yeojohnson(x, lmbda=None)
113 return lmbda
114
115
116# TODO: Fuse the modern implementations of _sparse_min_max and _sparse_nan_min_max
117# into the public min_max_axis function when Scipy 1.11 is the minimum supported
118# version and delete the backport in the else branch below.
119if sp_base_version >= parse_version("1.11.0"):
120
121 def _sparse_min_max(X, axis):
122 the_min = X.min(axis=axis)
123 the_max = X.max(axis=axis)
124
125 if axis is not None:
126 the_min = the_min.toarray().ravel()
127 the_max = the_max.toarray().ravel()
128
129 return the_min, the_max
130
131 def _sparse_nan_min_max(X, axis):
132 the_min = X.nanmin(axis=axis)
133 the_max = X.nanmax(axis=axis)
134
135 if axis is not None:
136 the_min = the_min.toarray().ravel()
137 the_max = the_max.toarray().ravel()
138
139 return the_min, the_max
140
141else:
142 # This code is mostly taken from scipy 0.14 and extended to handle nans, see
143 # https://github.com/scikit-learn/scikit-learn/pull/11196
144 def _minor_reduce(X, ufunc):
145 major_index = np.flatnonzero(np.diff(X.indptr))
146
147 # reduceat tries casts X.indptr to intp, which errors
148 # if it is int64 on a 32 bit system.
149 # Reinitializing prevents this where possible, see #13737
150 X = type(X)((X.data, X.indices, X.indptr), shape=X.shape)
151 value = ufunc.reduceat(X.data, X.indptr[major_index])
152 return major_index, value
153
154 def _min_or_max_axis(X, axis, min_or_max):
155 N = X.shape[axis]
156 if N == 0:
157 raise ValueError("zero-size array to reduction operation")
158 M = X.shape[1 - axis]
159 mat = X.tocsc() if axis == 0 else X.tocsr()
160 mat.sum_duplicates()
161 major_index, value = _minor_reduce(mat, min_or_max)
162 not_full = np.diff(mat.indptr)[major_index] < N
163 value[not_full] = min_or_max(value[not_full], 0)
164 mask = value != 0
165 major_index = np.compress(mask, major_index)
166 value = np.compress(mask, value)
167
168 if axis == 0:
169 res = scipy.sparse.coo_matrix(
170 (value, (np.zeros(len(value)), major_index)),
171 dtype=X.dtype,
172 shape=(1, M),
173 )
174 else:
175 res = scipy.sparse.coo_matrix(
176 (value, (major_index, np.zeros(len(value)))),
177 dtype=X.dtype,
178 shape=(M, 1),
179 )
180 return res.toarray().ravel()
181
182 def _sparse_min_or_max(X, axis, min_or_max):
183 if axis is None:
184 if 0 in X.shape:
185 raise ValueError("zero-size array to reduction operation")
186 zero = X.dtype.type(0)
187 if X.nnz == 0:
188 return zero
189 m = min_or_max.reduce(X.data.ravel())
190 if X.nnz != np.prod(X.shape):
191 m = min_or_max(zero, m)
192 return m
193 if axis < 0:
194 axis += 2
195 if (axis == 0) or (axis == 1):
196 return _min_or_max_axis(X, axis, min_or_max)
197 else:
198 raise ValueError("invalid axis, use 0 for rows, or 1 for columns")
199
200 def _sparse_min_max(X, axis):
201 return (
202 _sparse_min_or_max(X, axis, np.minimum),
203 _sparse_min_or_max(X, axis, np.maximum),
204 )
205
206 def _sparse_nan_min_max(X, axis):
207 return (
208 _sparse_min_or_max(X, axis, np.fmin),
209 _sparse_min_or_max(X, axis, np.fmax),
210 )
211
212
213# For +1.25 NumPy versions exceptions and warnings are being moved
214# to a dedicated submodule.
215if np_version >= parse_version("1.25.0"):
216 from numpy.exceptions import ComplexWarning, VisibleDeprecationWarning
217else:
218 from numpy import ( # noqa: F401
219 ComplexWarning,
220 VisibleDeprecationWarning,
221 )
222
223
224# TODO: Adapt when Pandas > 2.2 is the minimum supported version
225def pd_fillna(pd, frame):
226 pd_version = parse_version(pd.__version__).base_version
227 if parse_version(pd_version) < parse_version("2.2"):
228 frame = frame.fillna(value=np.nan)
229 else:
230 infer_objects_kwargs = (
231 {} if parse_version(pd_version) >= parse_version("3") else {"copy": False}
232 )
233 with pd.option_context("future.no_silent_downcasting", True):
234 frame = frame.fillna(value=np.nan).infer_objects(**infer_objects_kwargs)
235 return frame
236
237
238# TODO: remove when SciPy 1.12 is the minimum supported version
239def _preserve_dia_indices_dtype(
240 sparse_container, original_container_format, requested_sparse_format
241):
242 """Preserve indices dtype for SciPy < 1.12 when converting from DIA to CSR/CSC.
243
244 For SciPy < 1.12, DIA arrays indices are upcasted to `np.int64` that is
245 inconsistent with DIA matrices. We downcast the indices dtype to `np.int32` to
246 be consistent with DIA matrices.
247
248 The converted indices arrays are affected back inplace to the sparse container.
249
250 Parameters
251 ----------
252 sparse_container : sparse container
253 Sparse container to be checked.
254 requested_sparse_format : str or bool
255 The type of format of `sparse_container`.
256
257 Notes
258 -----
259 See https://github.com/scipy/scipy/issues/19245 for more details.
260 """
261 if original_container_format == "dia_array" and requested_sparse_format in (
262 "csr",
263 "coo",
264 ):
265 if requested_sparse_format == "csr":
266 index_dtype = _smallest_admissible_index_dtype(
267 arrays=(sparse_container.indptr, sparse_container.indices),
268 maxval=max(sparse_container.nnz, sparse_container.shape[1]),
269 check_contents=True,
270 )
271 sparse_container.indices = sparse_container.indices.astype(
272 index_dtype, copy=False
273 )
274 sparse_container.indptr = sparse_container.indptr.astype(
275 index_dtype, copy=False
276 )
277 else: # requested_sparse_format == "coo"
278 index_dtype = _smallest_admissible_index_dtype(
279 maxval=max(sparse_container.shape)
280 )
281 sparse_container.row = sparse_container.row.astype(index_dtype, copy=False)
282 sparse_container.col = sparse_container.col.astype(index_dtype, copy=False)
283
284
285# TODO: remove when SciPy 1.12 is the minimum supported version
286def _smallest_admissible_index_dtype(arrays=(), maxval=None, check_contents=False):
287 """Based on input (integer) arrays `a`, determine a suitable index data
288 type that can hold the data in the arrays.
289
290 This function returns `np.int64` if it either required by `maxval` or based on the
291 largest precision of the dtype of the arrays passed as argument, or by their
292 contents (when `check_contents is True`). If none of the condition requires
293 `np.int64` then this function returns `np.int32`.
294
295 Parameters
296 ----------
297 arrays : ndarray or tuple of ndarrays, default=()
298 Input arrays whose types/contents to check.
299
300 maxval : float, default=None
301 Maximum value needed.
302
303 check_contents : bool, default=False
304 Whether to check the values in the arrays and not just their types.
305 By default, check only the types.
306
307 Returns
308 -------
309 dtype : {np.int32, np.int64}
310 Suitable index data type (int32 or int64).
311 """
312
313 int32min = np.int32(np.iinfo(np.int32).min)
314 int32max = np.int32(np.iinfo(np.int32).max)
315
316 if maxval is not None:
317 if maxval > np.iinfo(np.int64).max:
318 raise ValueError(
319 f"maxval={maxval} is to large to be represented as np.int64."
320 )
321 if maxval > int32max:
322 return np.int64
323
324 if isinstance(arrays, np.ndarray):
325 arrays = (arrays,)
326
327 for arr in arrays:
328 if not isinstance(arr, np.ndarray):
329 raise TypeError(
330 f"Arrays should be of type np.ndarray, got {type(arr)} instead."
331 )
332 if not np.issubdtype(arr.dtype, np.integer):
333 raise ValueError(
334 f"Array dtype {arr.dtype} is not supported for index dtype. We expect "
335 "integral values."
336 )
337 if not np.can_cast(arr.dtype, np.int32):
338 if not check_contents:
339 # when `check_contents` is False, we stay on the safe side and return
340 # np.int64.
341 return np.int64
342 if arr.size == 0:
343 # a bigger type not needed yet, let's look at the next array
344 continue
345 else:
346 maxval = arr.max()
347 minval = arr.min()
348 if minval < int32min or maxval > int32max:
349 # a big index type is actually needed
350 return np.int64
351
352 return np.int32
353
354
355# TODO: Remove when Scipy 1.12 is the minimum supported version
356if sp_version < parse_version("1.12"):
357 from ..externals._scipy.sparse.csgraph import laplacian
358else:
359 from scipy.sparse.csgraph import (
360 laplacian, # noqa: F401 # pragma: no cover
361 )
362
363
364# TODO: Remove when Python min version >= 3.12.
365def tarfile_extractall(tarfile, path):
366 try:
367 # Use filter="data" to prevent the most dangerous security issues.
368 # For more details, see
369 # https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall
370 tarfile.extractall(path, filter="data")
371 except TypeError:
372 tarfile.extractall(path)
373
374
375def _in_unstable_openblas_configuration():
376 """Return True if in an unstable configuration for OpenBLAS"""
377
378 # Import libraries which might load OpenBLAS.
379 import numpy # noqa: F401
380 import scipy # noqa: F401
381
382 modules_info = _get_threadpool_controller().info()
383
384 open_blas_used = any(info["internal_api"] == "openblas" for info in modules_info)
385 if not open_blas_used:
386 return False
387
388 # OpenBLAS 0.3.16 fixed instability for arm64, see:
389 # https://github.com/xianyi/OpenBLAS/blob/1b6db3dbba672b4f8af935bd43a1ff6cff4d20b7/Changelog.txt#L56-L58
390 openblas_arm64_stable_version = parse_version("0.3.16")
391 for info in modules_info:
392 if info["internal_api"] != "openblas":
393 continue
394 openblas_version = info.get("version")
395 openblas_architecture = info.get("architecture")
396 if openblas_version is None or openblas_architecture is None:
397 # Cannot be sure that OpenBLAS is good enough. Assume unstable:
398 return True # pragma: no cover
399 if (
400 openblas_architecture == "neoversen1"
401 and parse_version(openblas_version) < openblas_arm64_stable_version
402 ):
403 # See discussions in https://github.com/numpy/numpy/issues/19411
404 return True # pragma: no cover
405 return False
406
407
408# TODO: Remove when Scipy 1.15 is the minimum supported version. In scipy 1.15,
409# the internal info details (via 'iprint' and 'disp' options) were dropped,
410# following the LBFGS rewrite from Fortran to C, see
411# https://github.com/scipy/scipy/issues/23186#issuecomment-2987801035. For
412# scipy 1.15, 'iprint' and 'disp' have no effect and for scipy >= 1.16 a
413# DeprecationWarning is emitted.
414def _get_additional_lbfgs_options_dict(key, value):
415 return {} if sp_version >= parse_version("1.15") else {key: value}
416
417
418# TODO(pyarrow): Remove when minimum pyarrow version is 17.0.0
419PYARROW_VERSION_BELOW_17 = False
420try:
421 import pyarrow
422
423 pyarrow_version = parse_version(pyarrow.__version__)
424 if pyarrow_version < parse_version("17.0.0"):
425 PYARROW_VERSION_BELOW_17 = True
426except ModuleNotFoundError: # pragma: no cover
427 pass
428