Aluode/PerceptionLabPortable

1"""Kernel ridge regression."""
2
3# Authors: The scikit-learn developers
4# SPDX-License-Identifier: BSD-3-Clause
5
6from numbers import Real
7
8import numpy as np
9
10from .base import BaseEstimator, MultiOutputMixin, RegressorMixin, _fit_context
11from .linear_model._ridge import _solve_cholesky_kernel
12from .metrics.pairwise import PAIRWISE_KERNEL_FUNCTIONS, pairwise_kernels
13from .utils._param_validation import Interval, StrOptions
14from .utils.validation import _check_sample_weight, check_is_fitted, validate_data
15
16
17class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator):
18    """Kernel ridge regression.
19
20    Kernel ridge regression (KRR) combines ridge regression (linear least
21    squares with l2-norm regularization) with the kernel trick. It thus
22    learns a linear function in the space induced by the respective kernel and
23    the data. For non-linear kernels, this corresponds to a non-linear
24    function in the original space.
25
26    The form of the model learned by KRR is identical to support vector
27    regression (SVR). However, different loss functions are used: KRR uses
28    squared error loss while support vector regression uses epsilon-insensitive
29    loss, both combined with l2 regularization. In contrast to SVR, fitting a
30    KRR model can be done in closed-form and is typically faster for
31    medium-sized datasets. On the other hand, the learned model is non-sparse
32    and thus slower than SVR, which learns a sparse model for epsilon > 0, at
33    prediction-time.
34
35    This estimator has built-in support for multi-variate regression
36    (i.e., when y is a 2d-array of shape [n_samples, n_targets]).
37
38    Read more in the :ref:`User Guide <kernel_ridge>`.
39
40    Parameters
41    ----------
42    alpha : float or array-like of shape (n_targets,), default=1.0
43        Regularization strength; must be a positive float. Regularization
44        improves the conditioning of the problem and reduces the variance of
45        the estimates. Larger values specify stronger regularization.
46        Alpha corresponds to ``1 / (2C)`` in other linear models such as
47        :class:`~sklearn.linear_model.LogisticRegression` or
48        :class:`~sklearn.svm.LinearSVC`. If an array is passed, penalties are
49        assumed to be specific to the targets. Hence they must correspond in
50        number. See :ref:`ridge_regression` for formula.
51
52    kernel : str or callable, default="linear"
53        Kernel mapping used internally. This parameter is directly passed to
54        :class:`~sklearn.metrics.pairwise.pairwise_kernels`.
55        If `kernel` is a string, it must be one of the metrics
56        in `pairwise.PAIRWISE_KERNEL_FUNCTIONS` or "precomputed".
57        If `kernel` is "precomputed", X is assumed to be a kernel matrix.
58        Alternatively, if `kernel` is a callable function, it is called on
59        each pair of instances (rows) and the resulting value recorded. The
60        callable should take two rows from X as input and return the
61        corresponding kernel value as a single number. This means that
62        callables from :mod:`sklearn.metrics.pairwise` are not allowed, as
63        they operate on matrices, not single samples. Use the string
64        identifying the kernel instead.
65
66    gamma : float, default=None
67        Gamma parameter for the RBF, laplacian, polynomial, exponential chi2
68        and sigmoid kernels. Interpretation of the default value is left to
69        the kernel; see the documentation for sklearn.metrics.pairwise.
70        Ignored by other kernels.
71
72    degree : float, default=3
73        Degree of the polynomial kernel. Ignored by other kernels.
74
75    coef0 : float, default=1
76        Zero coefficient for polynomial and sigmoid kernels.
77        Ignored by other kernels.
78
79    kernel_params : dict, default=None
80        Additional parameters (keyword arguments) for kernel function passed
81        as callable object.
82
83    Attributes
84    ----------
85    dual_coef_ : ndarray of shape (n_samples,) or (n_samples, n_targets)
86        Representation of weight vector(s) in kernel space
87
88    X_fit_ : {ndarray, sparse matrix} of shape (n_samples, n_features)
89        Training data, which is also required for prediction. If
90        kernel == "precomputed" this is instead the precomputed
91        training matrix, of shape (n_samples, n_samples).
92
93    n_features_in_ : int
94        Number of features seen during :term:`fit`.
95
96        .. versionadded:: 0.24
97
98    feature_names_in_ : ndarray of shape (`n_features_in_`,)
99        Names of features seen during :term:`fit`. Defined only when `X`
100        has feature names that are all strings.
101
102        .. versionadded:: 1.0
103
104    See Also
105    --------
106    sklearn.gaussian_process.GaussianProcessRegressor : Gaussian
107        Process regressor providing automatic kernel hyperparameters
108        tuning and predictions uncertainty.
109    sklearn.linear_model.Ridge : Linear ridge regression.
110    sklearn.linear_model.RidgeCV : Ridge regression with built-in
111        cross-validation.
112    sklearn.svm.SVR : Support Vector Regression accepting a large variety
113        of kernels.
114
115    References
116    ----------
117    * Kevin P. Murphy
118      "Machine Learning: A Probabilistic Perspective", The MIT Press
119      chapter 14.4.3, pp. 492-493
120
121    Examples
122    --------
123    >>> from sklearn.kernel_ridge import KernelRidge
124    >>> import numpy as np
125    >>> n_samples, n_features = 10, 5
126    >>> rng = np.random.RandomState(0)
127    >>> y = rng.randn(n_samples)
128    >>> X = rng.randn(n_samples, n_features)
129    >>> krr = KernelRidge(alpha=1.0)
130    >>> krr.fit(X, y)
131    KernelRidge(alpha=1.0)
132    """
133
134    _parameter_constraints: dict = {
135        "alpha": [Interval(Real, 0, None, closed="left"), "array-like"],
136        "kernel": [
137            StrOptions(set(PAIRWISE_KERNEL_FUNCTIONS.keys()) | {"precomputed"}),
138            callable,
139        ],
140        "gamma": [Interval(Real, 0, None, closed="left"), None],
141        "degree": [Interval(Real, 0, None, closed="left")],
142        "coef0": [Interval(Real, None, None, closed="neither")],
143        "kernel_params": [dict, None],
144    }
145
146    def __init__(
147        self,
148        alpha=1,
149        *,
150        kernel="linear",
151        gamma=None,
152        degree=3,
153        coef0=1,
154        kernel_params=None,
155    ):
156        self.alpha = alpha
157        self.kernel = kernel
158        self.gamma = gamma
159        self.degree = degree
160        self.coef0 = coef0
161        self.kernel_params = kernel_params
162
163    def _get_kernel(self, X, Y=None):
164        if callable(self.kernel):
165            params = self.kernel_params or {}
166        else:
167            params = {"gamma": self.gamma, "degree": self.degree, "coef0": self.coef0}
168        return pairwise_kernels(X, Y, metric=self.kernel, filter_params=True, **params)
169
170    def __sklearn_tags__(self):
171        tags = super().__sklearn_tags__()
172        tags.input_tags.sparse = True
173        tags.input_tags.pairwise = self.kernel == "precomputed"
174        return tags
175
176    @_fit_context(prefer_skip_nested_validation=True)
177    def fit(self, X, y, sample_weight=None):
178        """Fit Kernel Ridge regression model.
179
180        Parameters
181        ----------
182        X : {array-like, sparse matrix} of shape (n_samples, n_features)
183            Training data. If kernel == "precomputed" this is instead
184            a precomputed kernel matrix, of shape (n_samples, n_samples).
185
186        y : array-like of shape (n_samples,) or (n_samples, n_targets)
187            Target values.
188
189        sample_weight : float or array-like of shape (n_samples,), default=None
190            Individual weights for each sample, ignored if None is passed.
191
192        Returns
193        -------
194        self : object
195            Returns the instance itself.
196        """
197        # Convert data
198        X, y = validate_data(
199            self, X, y, accept_sparse=("csr", "csc"), multi_output=True, y_numeric=True
200        )
201        if sample_weight is not None and not isinstance(sample_weight, float):
202            sample_weight = _check_sample_weight(sample_weight, X)
203
204        K = self._get_kernel(X)
205        alpha = np.atleast_1d(self.alpha)
206
207        ravel = False
208        if len(y.shape) == 1:
209            y = y.reshape(-1, 1)
210            ravel = True
211
212        copy = self.kernel == "precomputed"
213        self.dual_coef_ = _solve_cholesky_kernel(K, y, alpha, sample_weight, copy)
214        if ravel:
215            self.dual_coef_ = self.dual_coef_.ravel()
216
217        self.X_fit_ = X
218
219        return self
220
221    def predict(self, X):
222        """Predict using the kernel ridge model.
223
224        Parameters
225        ----------
226        X : {array-like, sparse matrix} of shape (n_samples, n_features)
227            Samples. If kernel == "precomputed" this is instead a
228            precomputed kernel matrix, shape = [n_samples,
229            n_samples_fitted], where n_samples_fitted is the number of
230            samples used in the fitting for this estimator.
231
232        Returns
233        -------
234        C : ndarray of shape (n_samples,) or (n_samples, n_targets)
235            Returns predicted values.
236        """
237        check_is_fitted(self)
238        X = validate_data(self, X, accept_sparse=("csr", "csc"), reset=False)
239        K = self._get_kernel(X, self.X_fit_)
240        return np.dot(K, self.dual_coef_)
241