Aluode/PerceptionLabPortable
0
1"""Naive Bayes algorithms.
2
3These are supervised learning methods based on applying Bayes' theorem with strong
4(naive) feature independence assumptions.
5"""
6
7# Authors: The scikit-learn developers
8# SPDX-License-Identifier: BSD-3-Clause
9
10import warnings
11from abc import ABCMeta, abstractmethod
12from numbers import Integral, Real
13
14import numpy as np
15from scipy.special import logsumexp
16
17from .base import (
18 BaseEstimator,
19 ClassifierMixin,
20 _fit_context,
21)
22from .preprocessing import LabelBinarizer, binarize, label_binarize
23from .utils._param_validation import Interval
24from .utils.extmath import safe_sparse_dot
25from .utils.multiclass import _check_partial_fit_first_call
26from .utils.validation import (
27 _check_n_features,
28 _check_sample_weight,
29 check_is_fitted,
30 check_non_negative,
31 validate_data,
32)
33
34__all__ = [
35 "BernoulliNB",
36 "CategoricalNB",
37 "ComplementNB",
38 "GaussianNB",
39 "MultinomialNB",
40]
41
42
43class _BaseNB(ClassifierMixin, BaseEstimator, metaclass=ABCMeta):
44 """Abstract base class for naive Bayes estimators"""
45
46 @abstractmethod
47 def _joint_log_likelihood(self, X):
48 """Compute the unnormalized posterior log probability of X
49
50 I.e. ``log P(c) + log P(x|c)`` for all rows x of X, as an array-like of
51 shape (n_samples, n_classes).
52
53 Public methods predict, predict_proba, predict_log_proba, and
54 predict_joint_log_proba pass the input through _check_X before handing it
55 over to _joint_log_likelihood. The term "joint log likelihood" is used
56 interchangibly with "joint log probability".
57 """
58
59 @abstractmethod
60 def _check_X(self, X):
61 """To be overridden in subclasses with the actual checks.
62
63 Only used in predict* methods.
64 """
65
66 def predict_joint_log_proba(self, X):
67 """Return joint log probability estimates for the test vector X.
68
69 For each row x of X and class y, the joint log probability is given by
70 ``log P(x, y) = log P(y) + log P(x|y),``
71 where ``log P(y)`` is the class prior probability and ``log P(x|y)`` is
72 the class-conditional probability.
73
74 Parameters
75 ----------
76 X : array-like of shape (n_samples, n_features)
77 The input samples.
78
79 Returns
80 -------
81 C : ndarray of shape (n_samples, n_classes)
82 Returns the joint log-probability of the samples for each class in
83 the model. The columns correspond to the classes in sorted
84 order, as they appear in the attribute :term:`classes_`.
85 """
86 check_is_fitted(self)
87 X = self._check_X(X)
88 return self._joint_log_likelihood(X)
89
90 def predict(self, X):
91 """
92 Perform classification on an array of test vectors X.
93
94 Parameters
95 ----------
96 X : array-like of shape (n_samples, n_features)
97 The input samples.
98
99 Returns
100 -------
101 C : ndarray of shape (n_samples,)
102 Predicted target values for X.
103 """
104 check_is_fitted(self)
105 X = self._check_X(X)
106 jll = self._joint_log_likelihood(X)
107 return self.classes_[np.argmax(jll, axis=1)]
108
109 def predict_log_proba(self, X):
110 """
111 Return log-probability estimates for the test vector X.
112
113 Parameters
114 ----------
115 X : array-like of shape (n_samples, n_features)
116 The input samples.
117
118 Returns
119 -------
120 C : array-like of shape (n_samples, n_classes)
121 Returns the log-probability of the samples for each class in
122 the model. The columns correspond to the classes in sorted
123 order, as they appear in the attribute :term:`classes_`.
124 """
125 check_is_fitted(self)
126 X = self._check_X(X)
127 jll = self._joint_log_likelihood(X)
128 # normalize by P(x) = P(f_1, ..., f_n)
129 log_prob_x = logsumexp(jll, axis=1)
130 return jll - np.atleast_2d(log_prob_x).T
131
132 def predict_proba(self, X):
133 """
134 Return probability estimates for the test vector X.
135
136 Parameters
137 ----------
138 X : array-like of shape (n_samples, n_features)
139 The input samples.
140
141 Returns
142 -------
143 C : array-like of shape (n_samples, n_classes)
144 Returns the probability of the samples for each class in
145 the model. The columns correspond to the classes in sorted
146 order, as they appear in the attribute :term:`classes_`.
147 """
148 return np.exp(self.predict_log_proba(X))
149
150
151class GaussianNB(_BaseNB):
152 """
153 Gaussian Naive Bayes (GaussianNB).
154
155 Can perform online updates to model parameters via :meth:`partial_fit`.
156 For details on algorithm used to update feature means and variance online,
157 see `Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque
158 <http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf>`_.
159
160 Read more in the :ref:`User Guide <gaussian_naive_bayes>`.
161
162 Parameters
163 ----------
164 priors : array-like of shape (n_classes,), default=None
165 Prior probabilities of the classes. If specified, the priors are not
166 adjusted according to the data.
167
168 var_smoothing : float, default=1e-9
169 Portion of the largest variance of all features that is added to
170 variances for calculation stability.
171
172 .. versionadded:: 0.20
173
174 Attributes
175 ----------
176 class_count_ : ndarray of shape (n_classes,)
177 number of training samples observed in each class.
178
179 class_prior_ : ndarray of shape (n_classes,)
180 probability of each class.
181
182 classes_ : ndarray of shape (n_classes,)
183 class labels known to the classifier.
184
185 epsilon_ : float
186 absolute additive value to variances.
187
188 n_features_in_ : int
189 Number of features seen during :term:`fit`.
190
191 .. versionadded:: 0.24
192
193 feature_names_in_ : ndarray of shape (`n_features_in_`,)
194 Names of features seen during :term:`fit`. Defined only when `X`
195 has feature names that are all strings.
196
197 .. versionadded:: 1.0
198
199 var_ : ndarray of shape (n_classes, n_features)
200 Variance of each feature per class.
201
202 .. versionadded:: 1.0
203
204 theta_ : ndarray of shape (n_classes, n_features)
205 mean of each feature per class.
206
207 See Also
208 --------
209 BernoulliNB : Naive Bayes classifier for multivariate Bernoulli models.
210 CategoricalNB : Naive Bayes classifier for categorical features.
211 ComplementNB : Complement Naive Bayes classifier.
212 MultinomialNB : Naive Bayes classifier for multinomial models.
213
214 Examples
215 --------
216 >>> import numpy as np
217 >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
218 >>> Y = np.array([1, 1, 1, 2, 2, 2])
219 >>> from sklearn.naive_bayes import GaussianNB
220 >>> clf = GaussianNB()
221 >>> clf.fit(X, Y)
222 GaussianNB()
223 >>> print(clf.predict([[-0.8, -1]]))
224 [1]
225 >>> clf_pf = GaussianNB()
226 >>> clf_pf.partial_fit(X, Y, np.unique(Y))
227 GaussianNB()
228 >>> print(clf_pf.predict([[-0.8, -1]]))
229 [1]
230 """
231
232 _parameter_constraints: dict = {
233 "priors": ["array-like", None],
234 "var_smoothing": [Interval(Real, 0, None, closed="left")],
235 }
236
237 def __init__(self, *, priors=None, var_smoothing=1e-9):
238 self.priors = priors
239 self.var_smoothing = var_smoothing
240
241 @_fit_context(prefer_skip_nested_validation=True)
242 def fit(self, X, y, sample_weight=None):
243 """Fit Gaussian Naive Bayes according to X, y.
244
245 Parameters
246 ----------
247 X : array-like of shape (n_samples, n_features)
248 Training vectors, where `n_samples` is the number of samples
249 and `n_features` is the number of features.
250
251 y : array-like of shape (n_samples,)
252 Target values.
253
254 sample_weight : array-like of shape (n_samples,), default=None
255 Weights applied to individual samples (1. for unweighted).
256
257 .. versionadded:: 0.17
258 Gaussian Naive Bayes supports fitting with *sample_weight*.
259
260 Returns
261 -------
262 self : object
263 Returns the instance itself.
264 """
265 y = validate_data(self, y=y)
266 return self._partial_fit(
267 X, y, np.unique(y), _refit=True, sample_weight=sample_weight
268 )
269
270 def _check_X(self, X):
271 """Validate X, used only in predict* methods."""
272 return validate_data(self, X, reset=False)
273
274 @staticmethod
275 def _update_mean_variance(n_past, mu, var, X, sample_weight=None):
276 """Compute online update of Gaussian mean and variance.
277
278 Given starting sample count, mean, and variance, a new set of
279 points X, and optionally sample weights, return the updated mean and
280 variance. (NB - each dimension (column) in X is treated as independent
281 -- you get variance, not covariance).
282
283 Can take scalar mean and variance, or vector mean and variance to
284 simultaneously update a number of independent Gaussians.
285
286 See Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque:
287
288 http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf
289
290 Parameters
291 ----------
292 n_past : int
293 Number of samples represented in old mean and variance. If sample
294 weights were given, this should contain the sum of sample
295 weights represented in old mean and variance.
296
297 mu : array-like of shape (number of Gaussians,)
298 Means for Gaussians in original set.
299
300 var : array-like of shape (number of Gaussians,)
301 Variances for Gaussians in original set.
302
303 sample_weight : array-like of shape (n_samples,), default=None
304 Weights applied to individual samples (1. for unweighted).
305
306 Returns
307 -------
308 total_mu : array-like of shape (number of Gaussians,)
309 Updated mean for each Gaussian over the combined set.
310
311 total_var : array-like of shape (number of Gaussians,)
312 Updated variance for each Gaussian over the combined set.
313 """
314 if X.shape[0] == 0:
315 return mu, var
316
317 # Compute (potentially weighted) mean and variance of new datapoints
318 if sample_weight is not None:
319 n_new = float(sample_weight.sum())
320 if np.isclose(n_new, 0.0):
321 return mu, var
322 new_mu = np.average(X, axis=0, weights=sample_weight)
323 new_var = np.average((X - new_mu) ** 2, axis=0, weights=sample_weight)
324 else:
325 n_new = X.shape[0]
326 new_var = np.var(X, axis=0)
327 new_mu = np.mean(X, axis=0)
328
329 if n_past == 0:
330 return new_mu, new_var
331
332 n_total = float(n_past + n_new)
333
334 # Combine mean of old and new data, taking into consideration
335 # (weighted) number of observations
336 total_mu = (n_new * new_mu + n_past * mu) / n_total
337
338 # Combine variance of old and new data, taking into consideration
339 # (weighted) number of observations. This is achieved by combining
340 # the sum-of-squared-differences (ssd)
341 old_ssd = n_past * var
342 new_ssd = n_new * new_var
343 total_ssd = old_ssd + new_ssd + (n_new * n_past / n_total) * (mu - new_mu) ** 2
344 total_var = total_ssd / n_total
345
346 return total_mu, total_var
347
348 @_fit_context(prefer_skip_nested_validation=True)
349 def partial_fit(self, X, y, classes=None, sample_weight=None):
350 """Incremental fit on a batch of samples.
351
352 This method is expected to be called several times consecutively
353 on different chunks of a dataset so as to implement out-of-core
354 or online learning.
355
356 This is especially useful when the whole dataset is too big to fit in
357 memory at once.
358
359 This method has some performance and numerical stability overhead,
360 hence it is better to call partial_fit on chunks of data that are
361 as large as possible (as long as fitting in the memory budget) to
362 hide the overhead.
363
364 Parameters
365 ----------
366 X : array-like of shape (n_samples, n_features)
367 Training vectors, where `n_samples` is the number of samples and
368 `n_features` is the number of features.
369
370 y : array-like of shape (n_samples,)
371 Target values.
372
373 classes : array-like of shape (n_classes,), default=None
374 List of all the classes that can possibly appear in the y vector.
375
376 Must be provided at the first call to partial_fit, can be omitted
377 in subsequent calls.
378
379 sample_weight : array-like of shape (n_samples,), default=None
380 Weights applied to individual samples (1. for unweighted).
381
382 .. versionadded:: 0.17
383
384 Returns
385 -------
386 self : object
387 Returns the instance itself.
388 """
389 return self._partial_fit(
390 X, y, classes, _refit=False, sample_weight=sample_weight
391 )
392
393 def _partial_fit(self, X, y, classes=None, _refit=False, sample_weight=None):
394 """Actual implementation of Gaussian NB fitting.
395
396 Parameters
397 ----------
398 X : array-like of shape (n_samples, n_features)
399 Training vectors, where `n_samples` is the number of samples and
400 `n_features` is the number of features.
401
402 y : array-like of shape (n_samples,)
403 Target values.
404
405 classes : array-like of shape (n_classes,), default=None
406 List of all the classes that can possibly appear in the y vector.
407
408 Must be provided at the first call to partial_fit, can be omitted
409 in subsequent calls.
410
411 _refit : bool, default=False
412 If true, act as though this were the first time we called
413 _partial_fit (ie, throw away any past fitting and start over).
414
415 sample_weight : array-like of shape (n_samples,), default=None
416 Weights applied to individual samples (1. for unweighted).
417
418 Returns
419 -------
420 self : object
421 """
422 if _refit:
423 self.classes_ = None
424
425 first_call = _check_partial_fit_first_call(self, classes)
426 X, y = validate_data(self, X, y, reset=first_call)
427 if sample_weight is not None:
428 sample_weight = _check_sample_weight(sample_weight, X)
429
430 # If the ratio of data variance between dimensions is too small, it
431 # will cause numerical errors. To address this, we artificially
432 # boost the variance by epsilon, a small fraction of the standard
433 # deviation of the largest dimension.
434 self.epsilon_ = self.var_smoothing * np.var(X, axis=0).max()
435
436 if first_call:
437 # This is the first call to partial_fit:
438 # initialize various cumulative counters
439 n_features = X.shape[1]
440 n_classes = len(self.classes_)
441 self.theta_ = np.zeros((n_classes, n_features))
442 self.var_ = np.zeros((n_classes, n_features))
443
444 self.class_count_ = np.zeros(n_classes, dtype=np.float64)
445
446 # Initialise the class prior
447 # Take into account the priors
448 if self.priors is not None:
449 priors = np.asarray(self.priors)
450 # Check that the provided prior matches the number of classes
451 if len(priors) != n_classes:
452 raise ValueError("Number of priors must match number of classes.")
453 # Check that the sum is 1
454 if not np.isclose(priors.sum(), 1.0):
455 raise ValueError("The sum of the priors should be 1.")
456 # Check that the priors are non-negative
457 if (priors < 0).any():
458 raise ValueError("Priors must be non-negative.")
459 self.class_prior_ = priors
460 else:
461 # Initialize the priors to zeros for each class
462 self.class_prior_ = np.zeros(len(self.classes_), dtype=np.float64)
463 else:
464 if X.shape[1] != self.theta_.shape[1]:
465 msg = "Number of features %d does not match previous data %d."
466 raise ValueError(msg % (X.shape[1], self.theta_.shape[1]))
467 # Put epsilon back in each time
468 self.var_[:, :] -= self.epsilon_
469
470 classes = self.classes_
471
472 unique_y = np.unique(y)
473 unique_y_in_classes = np.isin(unique_y, classes)
474
475 if not np.all(unique_y_in_classes):
476 raise ValueError(
477 "The target label(s) %s in y do not exist in the initial classes %s"
478 % (unique_y[~unique_y_in_classes], classes)
479 )
480
481 for y_i in unique_y:
482 i = classes.searchsorted(y_i)
483 X_i = X[y == y_i, :]
484
485 if sample_weight is not None:
486 sw_i = sample_weight[y == y_i]
487 N_i = sw_i.sum()
488 else:
489 sw_i = None
490 N_i = X_i.shape[0]
491
492 new_theta, new_sigma = self._update_mean_variance(
493 self.class_count_[i], self.theta_[i, :], self.var_[i, :], X_i, sw_i
494 )
495
496 self.theta_[i, :] = new_theta
497 self.var_[i, :] = new_sigma
498 self.class_count_[i] += N_i
499
500 self.var_[:, :] += self.epsilon_
501
502 # Update if only no priors is provided
503 if self.priors is None:
504 # Empirical prior, with sample_weight taken into account
505 self.class_prior_ = self.class_count_ / self.class_count_.sum()
506
507 return self
508
509 def _joint_log_likelihood(self, X):
510 joint_log_likelihood = []
511 for i in range(np.size(self.classes_)):
512 jointi = np.log(self.class_prior_[i])
513 n_ij = -0.5 * np.sum(np.log(2.0 * np.pi * self.var_[i, :]))
514 n_ij -= 0.5 * np.sum(((X - self.theta_[i, :]) ** 2) / (self.var_[i, :]), 1)
515 joint_log_likelihood.append(jointi + n_ij)
516
517 joint_log_likelihood = np.array(joint_log_likelihood).T
518 return joint_log_likelihood
519
520
521class _BaseDiscreteNB(_BaseNB):
522 """Abstract base class for naive Bayes on discrete/categorical data
523
524 Any estimator based on this class should provide:
525
526 __init__
527 _joint_log_likelihood(X) as per _BaseNB
528 _update_feature_log_prob(alpha)
529 _count(X, Y)
530 """
531
532 _parameter_constraints: dict = {
533 "alpha": [Interval(Real, 0, None, closed="left"), "array-like"],
534 "fit_prior": ["boolean"],
535 "class_prior": ["array-like", None],
536 "force_alpha": ["boolean"],
537 }
538
539 def __init__(self, alpha=1.0, fit_prior=True, class_prior=None, force_alpha=True):
540 self.alpha = alpha
541 self.fit_prior = fit_prior
542 self.class_prior = class_prior
543 self.force_alpha = force_alpha
544
545 @abstractmethod
546 def _count(self, X, Y):
547 """Update counts that are used to calculate probabilities.
548
549 The counts make up a sufficient statistic extracted from the data.
550 Accordingly, this method is called each time `fit` or `partial_fit`
551 update the model. `class_count_` and `feature_count_` must be updated
552 here along with any model specific counts.
553
554 Parameters
555 ----------
556 X : {ndarray, sparse matrix} of shape (n_samples, n_features)
557 The input samples.
558 Y : ndarray of shape (n_samples, n_classes)
559 Binarized class labels.
560 """
561
562 @abstractmethod
563 def _update_feature_log_prob(self, alpha):
564 """Update feature log probabilities based on counts.
565
566 This method is called each time `fit` or `partial_fit` update the
567 model.
568
569 Parameters
570 ----------
571 alpha : float
572 smoothing parameter. See :meth:`_check_alpha`.
573 """
574
575 def _check_X(self, X):
576 """Validate X, used only in predict* methods."""
577 return validate_data(self, X, accept_sparse="csr", reset=False)
578
579 def _check_X_y(self, X, y, reset=True):
580 """Validate X and y in fit methods."""
581 return validate_data(self, X, y, accept_sparse="csr", reset=reset)
582
583 def _update_class_log_prior(self, class_prior=None):
584 """Update class log priors.
585
586 The class log priors are based on `class_prior`, class count or the
587 number of classes. This method is called each time `fit` or
588 `partial_fit` update the model.
589 """
590 n_classes = len(self.classes_)
591 if class_prior is not None:
592 if len(class_prior) != n_classes:
593 raise ValueError("Number of priors must match number of classes.")
594 self.class_log_prior_ = np.log(class_prior)
595 elif self.fit_prior:
596 with warnings.catch_warnings():
597 # silence the warning when count is 0 because class was not yet
598 # observed
599 warnings.simplefilter("ignore", RuntimeWarning)
600 log_class_count = np.log(self.class_count_)
601
602 # empirical prior, with sample_weight taken into account
603 self.class_log_prior_ = log_class_count - np.log(self.class_count_.sum())
604 else:
605 self.class_log_prior_ = np.full(n_classes, -np.log(n_classes))
606
607 def _check_alpha(self):
608 alpha = (
609 np.asarray(self.alpha) if not isinstance(self.alpha, Real) else self.alpha
610 )
611 alpha_min = np.min(alpha)
612 if isinstance(alpha, np.ndarray):
613 if not alpha.shape[0] == self.n_features_in_:
614 raise ValueError(
615 "When alpha is an array, it should contains `n_features`. "
616 f"Got {alpha.shape[0]} elements instead of {self.n_features_in_}."
617 )
618 # check that all alpha are positive
619 if alpha_min < 0:
620 raise ValueError("All values in alpha must be greater than 0.")
621 alpha_lower_bound = 1e-10
622 if alpha_min < alpha_lower_bound and not self.force_alpha:
623 warnings.warn(
624 "alpha too small will result in numeric errors, setting alpha ="
625 f" {alpha_lower_bound:.1e}. Use `force_alpha=True` to keep alpha"
626 " unchanged."
627 )
628 return np.maximum(alpha, alpha_lower_bound)
629 return alpha
630
631 @_fit_context(prefer_skip_nested_validation=True)
632 def partial_fit(self, X, y, classes=None, sample_weight=None):
633 """Incremental fit on a batch of samples.
634
635 This method is expected to be called several times consecutively
636 on different chunks of a dataset so as to implement out-of-core
637 or online learning.
638
639 This is especially useful when the whole dataset is too big to fit in
640 memory at once.
641
642 This method has some performance overhead hence it is better to call
643 partial_fit on chunks of data that are as large as possible
644 (as long as fitting in the memory budget) to hide the overhead.
645
646 Parameters
647 ----------
648 X : {array-like, sparse matrix} of shape (n_samples, n_features)
649 Training vectors, where `n_samples` is the number of samples and
650 `n_features` is the number of features.
651
652 y : array-like of shape (n_samples,)
653 Target values.
654
655 classes : array-like of shape (n_classes,), default=None
656 List of all the classes that can possibly appear in the y vector.
657
658 Must be provided at the first call to partial_fit, can be omitted
659 in subsequent calls.
660
661 sample_weight : array-like of shape (n_samples,), default=None
662 Weights applied to individual samples (1. for unweighted).
663
664 Returns
665 -------
666 self : object
667 Returns the instance itself.
668 """
669 first_call = not hasattr(self, "classes_")
670
671 X, y = self._check_X_y(X, y, reset=first_call)
672 _, n_features = X.shape
673
674 if _check_partial_fit_first_call(self, classes):
675 # This is the first call to partial_fit:
676 # initialize various cumulative counters
677 n_classes = len(classes)
678 self._init_counters(n_classes, n_features)
679
680 Y = label_binarize(y, classes=self.classes_)
681 if Y.shape[1] == 1:
682 if len(self.classes_) == 2:
683 Y = np.concatenate((1 - Y, Y), axis=1)
684 else: # degenerate case: just one class
685 Y = np.ones_like(Y)
686
687 if X.shape[0] != Y.shape[0]:
688 msg = "X.shape[0]=%d and y.shape[0]=%d are incompatible."
689 raise ValueError(msg % (X.shape[0], y.shape[0]))
690
691 # label_binarize() returns arrays with dtype=np.int64.
692 # We convert it to np.float64 to support sample_weight consistently
693 Y = Y.astype(np.float64, copy=False)
694 if sample_weight is not None:
695 sample_weight = _check_sample_weight(sample_weight, X)
696 sample_weight = np.atleast_2d(sample_weight)
697 Y *= sample_weight.T
698
699 class_prior = self.class_prior
700
701 # Count raw events from data before updating the class log prior
702 # and feature log probas
703 self._count(X, Y)
704
705 # XXX: OPTIM: we could introduce a public finalization method to
706 # be called by the user explicitly just once after several consecutive
707 # calls to partial_fit and prior any call to predict[_[log_]proba]
708 # to avoid computing the smooth log probas at each call to partial fit
709 alpha = self._check_alpha()
710 self._update_feature_log_prob(alpha)
711 self._update_class_log_prior(class_prior=class_prior)
712 return self
713
714 @_fit_context(prefer_skip_nested_validation=True)
715 def fit(self, X, y, sample_weight=None):
716 """Fit Naive Bayes classifier according to X, y.
717
718 Parameters
719 ----------
720 X : {array-like, sparse matrix} of shape (n_samples, n_features)
721 Training vectors, where `n_samples` is the number of samples and
722 `n_features` is the number of features.
723
724 y : array-like of shape (n_samples,)
725 Target values.
726
727 sample_weight : array-like of shape (n_samples,), default=None
728 Weights applied to individual samples (1. for unweighted).
729
730 Returns
731 -------
732 self : object
733 Returns the instance itself.
734 """
735 X, y = self._check_X_y(X, y)
736 _, n_features = X.shape
737
738 labelbin = LabelBinarizer()
739 Y = labelbin.fit_transform(y)
740 self.classes_ = labelbin.classes_
741 if Y.shape[1] == 1:
742 if len(self.classes_) == 2:
743 Y = np.concatenate((1 - Y, Y), axis=1)
744 else: # degenerate case: just one class
745 Y = np.ones_like(Y)
746
747 # LabelBinarizer().fit_transform() returns arrays with dtype=np.int64.
748 # We convert it to np.float64 to support sample_weight consistently;
749 # this means we also don't have to cast X to floating point
750 if sample_weight is not None:
751 Y = Y.astype(np.float64, copy=False)
752 sample_weight = _check_sample_weight(sample_weight, X)
753 sample_weight = np.atleast_2d(sample_weight)
754 Y *= sample_weight.T
755
756 class_prior = self.class_prior
757
758 # Count raw events from data before updating the class log prior
759 # and feature log probas
760 n_classes = Y.shape[1]
761 self._init_counters(n_classes, n_features)
762 self._count(X, Y)
763 alpha = self._check_alpha()
764 self._update_feature_log_prob(alpha)
765 self._update_class_log_prior(class_prior=class_prior)
766 return self
767
768 def _init_counters(self, n_classes, n_features):
769 self.class_count_ = np.zeros(n_classes, dtype=np.float64)
770 self.feature_count_ = np.zeros((n_classes, n_features), dtype=np.float64)
771
772 def __sklearn_tags__(self):
773 tags = super().__sklearn_tags__()
774 tags.input_tags.sparse = True
775 tags.classifier_tags.poor_score = True
776 return tags
777
778
779class MultinomialNB(_BaseDiscreteNB):
780 """
781 Naive Bayes classifier for multinomial models.
782
783 The multinomial Naive Bayes classifier is suitable for classification with
784 discrete features (e.g., word counts for text classification). The
785 multinomial distribution normally requires integer feature counts. However,
786 in practice, fractional counts such as tf-idf may also work.
787
788 Read more in the :ref:`User Guide <multinomial_naive_bayes>`.
789
790 Parameters
791 ----------
792 alpha : float or array-like of shape (n_features,), default=1.0
793 Additive (Laplace/Lidstone) smoothing parameter
794 (set alpha=0 and force_alpha=True, for no smoothing).
795
796 force_alpha : bool, default=True
797 If False and alpha is less than 1e-10, it will set alpha to
798 1e-10. If True, alpha will remain unchanged. This may cause
799 numerical errors if alpha is too close to 0.
800
801 .. versionadded:: 1.2
802 .. versionchanged:: 1.4
803 The default value of `force_alpha` changed to `True`.
804
805 fit_prior : bool, default=True
806 Whether to learn class prior probabilities or not.
807 If false, a uniform prior will be used.
808
809 class_prior : array-like of shape (n_classes,), default=None
810 Prior probabilities of the classes. If specified, the priors are not
811 adjusted according to the data.
812
813 Attributes
814 ----------
815 class_count_ : ndarray of shape (n_classes,)
816 Number of samples encountered for each class during fitting. This
817 value is weighted by the sample weight when provided.
818
819 class_log_prior_ : ndarray of shape (n_classes,)
820 Smoothed empirical log probability for each class.
821
822 classes_ : ndarray of shape (n_classes,)
823 Class labels known to the classifier
824
825 feature_count_ : ndarray of shape (n_classes, n_features)
826 Number of samples encountered for each (class, feature)
827 during fitting. This value is weighted by the sample weight when
828 provided.
829
830 feature_log_prob_ : ndarray of shape (n_classes, n_features)
831 Empirical log probability of features
832 given a class, ``P(x_i|y)``.
833
834 n_features_in_ : int
835 Number of features seen during :term:`fit`.
836
837 .. versionadded:: 0.24
838
839 feature_names_in_ : ndarray of shape (`n_features_in_`,)
840 Names of features seen during :term:`fit`. Defined only when `X`
841 has feature names that are all strings.
842
843 .. versionadded:: 1.0
844
845 See Also
846 --------
847 BernoulliNB : Naive Bayes classifier for multivariate Bernoulli models.
848 CategoricalNB : Naive Bayes classifier for categorical features.
849 ComplementNB : Complement Naive Bayes classifier.
850 GaussianNB : Gaussian Naive Bayes.
851
852 References
853 ----------
854 C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
855 Information Retrieval. Cambridge University Press, pp. 234-265.
856 https://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html
857
858 Examples
859 --------
860 >>> import numpy as np
861 >>> rng = np.random.RandomState(1)
862 >>> X = rng.randint(5, size=(6, 100))
863 >>> y = np.array([1, 2, 3, 4, 5, 6])
864 >>> from sklearn.naive_bayes import MultinomialNB
865 >>> clf = MultinomialNB()
866 >>> clf.fit(X, y)
867 MultinomialNB()
868 >>> print(clf.predict(X[2:3]))
869 [3]
870 """
871
872 def __init__(
873 self, *, alpha=1.0, force_alpha=True, fit_prior=True, class_prior=None
874 ):
875 super().__init__(
876 alpha=alpha,
877 fit_prior=fit_prior,
878 class_prior=class_prior,
879 force_alpha=force_alpha,
880 )
881
882 def __sklearn_tags__(self):
883 tags = super().__sklearn_tags__()
884 tags.input_tags.positive_only = True
885 return tags
886
887 def _count(self, X, Y):
888 """Count and smooth feature occurrences."""
889 check_non_negative(X, "MultinomialNB (input X)")
890 self.feature_count_ += safe_sparse_dot(Y.T, X)
891 self.class_count_ += Y.sum(axis=0)
892
893 def _update_feature_log_prob(self, alpha):
894 """Apply smoothing to raw counts and recompute log probabilities"""
895 smoothed_fc = self.feature_count_ + alpha
896 smoothed_cc = smoothed_fc.sum(axis=1)
897
898 self.feature_log_prob_ = np.log(smoothed_fc) - np.log(
899 smoothed_cc.reshape(-1, 1)
900 )
901
902 def _joint_log_likelihood(self, X):
903 """Calculate the posterior log probability of the samples X"""
904 return safe_sparse_dot(X, self.feature_log_prob_.T) + self.class_log_prior_
905
906
907class ComplementNB(_BaseDiscreteNB):
908 """The Complement Naive Bayes classifier described in Rennie et al. (2003).
909
910 The Complement Naive Bayes classifier was designed to correct the "severe
911 assumptions" made by the standard Multinomial Naive Bayes classifier. It is
912 particularly suited for imbalanced data sets.
913
914 Read more in the :ref:`User Guide <complement_naive_bayes>`.
915
916 .. versionadded:: 0.20
917
918 Parameters
919 ----------
920 alpha : float or array-like of shape (n_features,), default=1.0
921 Additive (Laplace/Lidstone) smoothing parameter
922 (set alpha=0 and force_alpha=True, for no smoothing).
923
924 force_alpha : bool, default=True
925 If False and alpha is less than 1e-10, it will set alpha to
926 1e-10. If True, alpha will remain unchanged. This may cause
927 numerical errors if alpha is too close to 0.
928
929 .. versionadded:: 1.2
930 .. versionchanged:: 1.4
931 The default value of `force_alpha` changed to `True`.
932
933 fit_prior : bool, default=True
934 Only used in edge case with a single class in the training set.
935
936 class_prior : array-like of shape (n_classes,), default=None
937 Prior probabilities of the classes. Not used.
938
939 norm : bool, default=False
940 Whether or not a second normalization of the weights is performed. The
941 default behavior mirrors the implementations found in Mahout and Weka,
942 which do not follow the full algorithm described in Table 9 of the
943 paper.
944
945 Attributes
946 ----------
947 class_count_ : ndarray of shape (n_classes,)
948 Number of samples encountered for each class during fitting. This
949 value is weighted by the sample weight when provided.
950
951 class_log_prior_ : ndarray of shape (n_classes,)
952 Smoothed empirical log probability for each class. Only used in edge
953 case with a single class in the training set.
954
955 classes_ : ndarray of shape (n_classes,)
956 Class labels known to the classifier
957
958 feature_all_ : ndarray of shape (n_features,)
959 Number of samples encountered for each feature during fitting. This
960 value is weighted by the sample weight when provided.
961
962 feature_count_ : ndarray of shape (n_classes, n_features)
963 Number of samples encountered for each (class, feature) during fitting.
964 This value is weighted by the sample weight when provided.
965
966 feature_log_prob_ : ndarray of shape (n_classes, n_features)
967 Empirical weights for class complements.
968
969 n_features_in_ : int
970 Number of features seen during :term:`fit`.
971
972 .. versionadded:: 0.24
973
974 feature_names_in_ : ndarray of shape (`n_features_in_`,)
975 Names of features seen during :term:`fit`. Defined only when `X`
976 has feature names that are all strings.
977
978 .. versionadded:: 1.0
979
980 See Also
981 --------
982 BernoulliNB : Naive Bayes classifier for multivariate Bernoulli models.
983 CategoricalNB : Naive Bayes classifier for categorical features.
984 GaussianNB : Gaussian Naive Bayes.
985 MultinomialNB : Naive Bayes classifier for multinomial models.
986
987 References
988 ----------
989 Rennie, J. D., Shih, L., Teevan, J., & Karger, D. R. (2003).
990 Tackling the poor assumptions of naive bayes text classifiers. In ICML
991 (Vol. 3, pp. 616-623).
992 https://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf
993
994 Examples
995 --------
996 >>> import numpy as np
997 >>> rng = np.random.RandomState(1)
998 >>> X = rng.randint(5, size=(6, 100))
999 >>> y = np.array([1, 2, 3, 4, 5, 6])
1000 >>> from sklearn.naive_bayes import ComplementNB
1001 >>> clf = ComplementNB()
1002 >>> clf.fit(X, y)
1003 ComplementNB()
1004 >>> print(clf.predict(X[2:3]))
1005 [3]
1006 """
1007
1008 _parameter_constraints: dict = {
1009 **_BaseDiscreteNB._parameter_constraints,
1010 "norm": ["boolean"],
1011 }
1012
1013 def __init__(
1014 self,
1015 *,
1016 alpha=1.0,
1017 force_alpha=True,
1018 fit_prior=True,
1019 class_prior=None,
1020 norm=False,
1021 ):
1022 super().__init__(
1023 alpha=alpha,
1024 force_alpha=force_alpha,
1025 fit_prior=fit_prior,
1026 class_prior=class_prior,
1027 )
1028 self.norm = norm
1029
1030 def __sklearn_tags__(self):
1031 tags = super().__sklearn_tags__()
1032 tags.input_tags.positive_only = True
1033 return tags
1034
1035 def _count(self, X, Y):
1036 """Count feature occurrences."""
1037 check_non_negative(X, "ComplementNB (input X)")
1038 self.feature_count_ += safe_sparse_dot(Y.T, X)
1039 self.class_count_ += Y.sum(axis=0)
1040 self.feature_all_ = self.feature_count_.sum(axis=0)
1041
1042 def _update_feature_log_prob(self, alpha):
1043 """Apply smoothing to raw counts and compute the weights."""
1044 comp_count = self.feature_all_ + alpha - self.feature_count_
1045 logged = np.log(comp_count / comp_count.sum(axis=1, keepdims=True))
1046 # _BaseNB.predict uses argmax, but ComplementNB operates with argmin.
1047 if self.norm:
1048 summed = logged.sum(axis=1, keepdims=True)
1049 feature_log_prob = logged / summed
1050 else:
1051 feature_log_prob = -logged
1052 self.feature_log_prob_ = feature_log_prob
1053
1054 def _joint_log_likelihood(self, X):
1055 """Calculate the class scores for the samples in X."""
1056 jll = safe_sparse_dot(X, self.feature_log_prob_.T)
1057 if len(self.classes_) == 1:
1058 jll += self.class_log_prior_
1059 return jll
1060
1061
1062class BernoulliNB(_BaseDiscreteNB):
1063 """Naive Bayes classifier for multivariate Bernoulli models.
1064
1065 Like MultinomialNB, this classifier is suitable for discrete data. The
1066 difference is that while MultinomialNB works with occurrence counts,
1067 BernoulliNB is designed for binary/boolean features.
1068
1069 Read more in the :ref:`User Guide <bernoulli_naive_bayes>`.
1070
1071 Parameters
1072 ----------
1073 alpha : float or array-like of shape (n_features,), default=1.0
1074 Additive (Laplace/Lidstone) smoothing parameter
1075 (set alpha=0 and force_alpha=True, for no smoothing).
1076
1077 force_alpha : bool, default=True
1078 If False and alpha is less than 1e-10, it will set alpha to
1079 1e-10. If True, alpha will remain unchanged. This may cause
1080 numerical errors if alpha is too close to 0.
1081
1082 .. versionadded:: 1.2
1083 .. versionchanged:: 1.4
1084 The default value of `force_alpha` changed to `True`.
1085
1086 binarize : float or None, default=0.0
1087 Threshold for binarizing (mapping to booleans) of sample features.
1088 If None, input is presumed to already consist of binary vectors.
1089
1090 fit_prior : bool, default=True
1091 Whether to learn class prior probabilities or not.
1092 If false, a uniform prior will be used.
1093
1094 class_prior : array-like of shape (n_classes,), default=None
1095 Prior probabilities of the classes. If specified, the priors are not
1096 adjusted according to the data.
1097
1098 Attributes
1099 ----------
1100 class_count_ : ndarray of shape (n_classes,)
1101 Number of samples encountered for each class during fitting. This
1102 value is weighted by the sample weight when provided.
1103
1104 class_log_prior_ : ndarray of shape (n_classes,)
1105 Log probability of each class (smoothed).
1106
1107 classes_ : ndarray of shape (n_classes,)
1108 Class labels known to the classifier
1109
1110 feature_count_ : ndarray of shape (n_classes, n_features)
1111 Number of samples encountered for each (class, feature)
1112 during fitting. This value is weighted by the sample weight when
1113 provided.
1114
1115 feature_log_prob_ : ndarray of shape (n_classes, n_features)
1116 Empirical log probability of features given a class, P(x_i|y).
1117
1118 n_features_in_ : int
1119 Number of features seen during :term:`fit`.
1120
1121 .. versionadded:: 0.24
1122
1123 feature_names_in_ : ndarray of shape (`n_features_in_`,)
1124 Names of features seen during :term:`fit`. Defined only when `X`
1125 has feature names that are all strings.
1126
1127 .. versionadded:: 1.0
1128
1129 See Also
1130 --------
1131 CategoricalNB : Naive Bayes classifier for categorical features.
1132 ComplementNB : The Complement Naive Bayes classifier
1133 described in Rennie et al. (2003).
1134 GaussianNB : Gaussian Naive Bayes (GaussianNB).
1135 MultinomialNB : Naive Bayes classifier for multinomial models.
1136
1137 References
1138 ----------
1139 C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
1140 Information Retrieval. Cambridge University Press, pp. 234-265.
1141 https://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html
1142
1143 A. McCallum and K. Nigam (1998). A comparison of event models for naive
1144 Bayes text classification. Proc. AAAI/ICML-98 Workshop on Learning for
1145 Text Categorization, pp. 41-48.
1146
1147 V. Metsis, I. Androutsopoulos and G. Paliouras (2006). Spam filtering with
1148 naive Bayes -- Which naive Bayes? 3rd Conf. on Email and Anti-Spam (CEAS).
1149
1150 Examples
1151 --------
1152 >>> import numpy as np
1153 >>> rng = np.random.RandomState(1)
1154 >>> X = rng.randint(5, size=(6, 100))
1155 >>> Y = np.array([1, 2, 3, 4, 4, 5])
1156 >>> from sklearn.naive_bayes import BernoulliNB
1157 >>> clf = BernoulliNB()
1158 >>> clf.fit(X, Y)
1159 BernoulliNB()
1160 >>> print(clf.predict(X[2:3]))
1161 [3]
1162 """
1163
1164 _parameter_constraints: dict = {
1165 **_BaseDiscreteNB._parameter_constraints,
1166 "binarize": [None, Interval(Real, 0, None, closed="left")],
1167 }
1168
1169 def __init__(
1170 self,
1171 *,
1172 alpha=1.0,
1173 force_alpha=True,
1174 binarize=0.0,
1175 fit_prior=True,
1176 class_prior=None,
1177 ):
1178 super().__init__(
1179 alpha=alpha,
1180 fit_prior=fit_prior,
1181 class_prior=class_prior,
1182 force_alpha=force_alpha,
1183 )
1184 self.binarize = binarize
1185
1186 def _check_X(self, X):
1187 """Validate X, used only in predict* methods."""
1188 X = super()._check_X(X)
1189 if self.binarize is not None:
1190 X = binarize(X, threshold=self.binarize)
1191 return X
1192
1193 def _check_X_y(self, X, y, reset=True):
1194 X, y = super()._check_X_y(X, y, reset=reset)
1195 if self.binarize is not None:
1196 X = binarize(X, threshold=self.binarize)
1197 return X, y
1198
1199 def _count(self, X, Y):
1200 """Count and smooth feature occurrences."""