Aluode/PerceptionLabPortable
0
1from re import escape
2
3import numpy as np
4import pytest
5import scipy.sparse as sp
6from numpy.testing import assert_allclose
7
8from sklearn import datasets, svm
9from sklearn.base import BaseEstimator, ClassifierMixin
10from sklearn.datasets import load_breast_cancer
11from sklearn.exceptions import NotFittedError
12from sklearn.impute import SimpleImputer
13from sklearn.linear_model import (
14 ElasticNet,
15 Lasso,
16 LinearRegression,
17 LogisticRegression,
18 Perceptron,
19 Ridge,
20 SGDClassifier,
21)
22from sklearn.metrics import precision_score, recall_score
23from sklearn.model_selection import GridSearchCV, cross_val_score
24from sklearn.multiclass import (
25 OneVsOneClassifier,
26 OneVsRestClassifier,
27 OutputCodeClassifier,
28)
29from sklearn.naive_bayes import MultinomialNB
30from sklearn.neighbors import KNeighborsClassifier
31from sklearn.pipeline import Pipeline, make_pipeline
32from sklearn.svm import SVC, LinearSVC
33from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
34from sklearn.utils import (
35 check_array,
36 shuffle,
37)
38from sklearn.utils._mocking import CheckingClassifier
39from sklearn.utils._testing import assert_almost_equal, assert_array_equal
40from sklearn.utils.fixes import (
41 COO_CONTAINERS,
42 CSC_CONTAINERS,
43 CSR_CONTAINERS,
44 DOK_CONTAINERS,
45 LIL_CONTAINERS,
46)
47from sklearn.utils.multiclass import check_classification_targets, type_of_target
48
49iris = datasets.load_iris()
50rng = np.random.RandomState(0)
51perm = rng.permutation(iris.target.size)
52iris.data = iris.data[perm]
53iris.target = iris.target[perm]
54n_classes = 3
55
56
57def test_ovr_exceptions():
58 ovr = OneVsRestClassifier(LinearSVC(random_state=0))
59
60 # test predicting without fitting
61 with pytest.raises(NotFittedError):
62 ovr.predict([])
63
64 # Fail on multioutput data
65 msg = "Multioutput target data is not supported with label binarization"
66 with pytest.raises(ValueError, match=msg):
67 X = np.array([[1, 0], [0, 1]])
68 y = np.array([[1, 2], [3, 1]])
69 OneVsRestClassifier(MultinomialNB()).fit(X, y)
70
71 with pytest.raises(ValueError, match=msg):
72 X = np.array([[1, 0], [0, 1]])
73 y = np.array([[1.5, 2.4], [3.1, 0.8]])
74 OneVsRestClassifier(MultinomialNB()).fit(X, y)
75
76
77def test_check_classification_targets():
78 # Test that check_classification_target return correct type. #5782
79 y = np.array([0.0, 1.1, 2.0, 3.0])
80 msg = type_of_target(y)
81 with pytest.raises(ValueError, match=msg):
82 check_classification_targets(y)
83
84
85def test_ovr_fit_predict():
86 # A classifier which implements decision_function.
87 ovr = OneVsRestClassifier(LinearSVC(random_state=0))
88 pred = ovr.fit(iris.data, iris.target).predict(iris.data)
89 assert len(ovr.estimators_) == n_classes
90
91 clf = LinearSVC(random_state=0)
92 pred2 = clf.fit(iris.data, iris.target).predict(iris.data)
93 assert np.mean(iris.target == pred) == np.mean(iris.target == pred2)
94
95 # A classifier which implements predict_proba.
96 ovr = OneVsRestClassifier(MultinomialNB())
97 pred = ovr.fit(iris.data, iris.target).predict(iris.data)
98 assert np.mean(iris.target == pred) > 0.65
99
100
101def test_ovr_partial_fit():
102 # Test if partial_fit is working as intended
103 X, y = shuffle(iris.data, iris.target, random_state=0)
104 ovr = OneVsRestClassifier(MultinomialNB())
105 ovr.partial_fit(X[:100], y[:100], np.unique(y))
106 ovr.partial_fit(X[100:], y[100:])
107 pred = ovr.predict(X)
108 ovr2 = OneVsRestClassifier(MultinomialNB())
109 pred2 = ovr2.fit(X, y).predict(X)
110
111 assert_almost_equal(pred, pred2)
112 assert len(ovr.estimators_) == len(np.unique(y))
113 assert np.mean(y == pred) > 0.65
114
115 # Test when mini batches doesn't have all classes
116 # with SGDClassifier
117 X = np.abs(np.random.randn(14, 2))
118 y = [1, 1, 1, 1, 2, 3, 3, 0, 0, 2, 3, 1, 2, 3]
119
120 ovr = OneVsRestClassifier(
121 SGDClassifier(max_iter=1, tol=None, shuffle=False, random_state=0)
122 )
123 ovr.partial_fit(X[:7], y[:7], np.unique(y))
124 ovr.partial_fit(X[7:], y[7:])
125 pred = ovr.predict(X)
126 ovr1 = OneVsRestClassifier(
127 SGDClassifier(max_iter=1, tol=None, shuffle=False, random_state=0)
128 )
129 pred1 = ovr1.fit(X, y).predict(X)
130 assert np.mean(pred == y) == np.mean(pred1 == y)
131
132 # test partial_fit only exists if estimator has it:
133 ovr = OneVsRestClassifier(SVC())
134 assert not hasattr(ovr, "partial_fit")
135
136
137def test_ovr_partial_fit_exceptions():
138 ovr = OneVsRestClassifier(MultinomialNB())
139 X = np.abs(np.random.randn(14, 2))
140 y = [1, 1, 1, 1, 2, 3, 3, 0, 0, 2, 3, 1, 2, 3]
141 ovr.partial_fit(X[:7], y[:7], np.unique(y))
142 # If a new class that was not in the first call of partial fit is seen
143 # it should raise ValueError
144 y1 = [5] + y[7:-1]
145 msg = r"Mini-batch contains \[.+\] while classes must be subset of \[.+\]"
146 with pytest.raises(ValueError, match=msg):
147 ovr.partial_fit(X=X[7:], y=y1)
148
149
150def test_ovr_ovo_regressor():
151 # test that ovr and ovo work on regressors which don't have a decision_
152 # function
153 ovr = OneVsRestClassifier(DecisionTreeRegressor())
154 pred = ovr.fit(iris.data, iris.target).predict(iris.data)
155 assert len(ovr.estimators_) == n_classes
156 assert_array_equal(np.unique(pred), [0, 1, 2])
157 # we are doing something sensible
158 assert np.mean(pred == iris.target) > 0.9
159
160 ovr = OneVsOneClassifier(DecisionTreeRegressor())
161 pred = ovr.fit(iris.data, iris.target).predict(iris.data)
162 assert len(ovr.estimators_) == n_classes * (n_classes - 1) / 2
163 assert_array_equal(np.unique(pred), [0, 1, 2])
164 # we are doing something sensible
165 assert np.mean(pred == iris.target) > 0.9
166
167
168@pytest.mark.parametrize(
169 "sparse_container",
170 CSR_CONTAINERS + CSC_CONTAINERS + COO_CONTAINERS + DOK_CONTAINERS + LIL_CONTAINERS,
171)
172def test_ovr_fit_predict_sparse(sparse_container):
173 base_clf = MultinomialNB(alpha=1)
174
175 X, Y = datasets.make_multilabel_classification(
176 n_samples=100,
177 n_features=20,
178 n_classes=5,
179 n_labels=3,
180 length=50,
181 allow_unlabeled=True,
182 random_state=0,
183 )
184
185 X_train, Y_train = X[:80], Y[:80]
186 X_test = X[80:]
187
188 clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
189 Y_pred = clf.predict(X_test)
190
191 clf_sprs = OneVsRestClassifier(base_clf).fit(X_train, sparse_container(Y_train))
192 Y_pred_sprs = clf_sprs.predict(X_test)
193
194 assert clf.multilabel_
195 assert sp.issparse(Y_pred_sprs)
196 assert_array_equal(Y_pred_sprs.toarray(), Y_pred)
197
198 # Test predict_proba
199 Y_proba = clf_sprs.predict_proba(X_test)
200
201 # predict assigns a label if the probability that the
202 # sample has the label is greater than 0.5.
203 pred = Y_proba > 0.5
204 assert_array_equal(pred, Y_pred_sprs.toarray())
205
206 # Test decision_function
207 clf = svm.SVC()
208 clf_sprs = OneVsRestClassifier(clf).fit(X_train, sparse_container(Y_train))
209 dec_pred = (clf_sprs.decision_function(X_test) > 0).astype(int)
210 assert_array_equal(dec_pred, clf_sprs.predict(X_test).toarray())
211
212
213def test_ovr_always_present():
214 # Test that ovr works with classes that are always present or absent.
215 # Note: tests is the case where _ConstantPredictor is utilised
216 X = np.ones((10, 2))
217 X[:5, :] = 0
218
219 # Build an indicator matrix where two features are always on.
220 # As list of lists, it would be: [[int(i >= 5), 2, 3] for i in range(10)]
221 y = np.zeros((10, 3))
222 y[5:, 0] = 1
223 y[:, 1] = 1
224 y[:, 2] = 1
225
226 ovr = OneVsRestClassifier(LogisticRegression())
227 msg = r"Label .+ is present in all training examples"
228 with pytest.warns(UserWarning, match=msg):
229 ovr.fit(X, y)
230 y_pred = ovr.predict(X)
231 assert_array_equal(np.array(y_pred), np.array(y))
232 y_pred = ovr.decision_function(X)
233 assert np.unique(y_pred[:, -2:]) == 1
234 y_pred = ovr.predict_proba(X)
235 assert_array_equal(y_pred[:, -1], np.ones(X.shape[0]))
236
237 # y has a constantly absent label
238 y = np.zeros((10, 2))
239 y[5:, 0] = 1 # variable label
240 ovr = OneVsRestClassifier(LogisticRegression())
241
242 msg = r"Label not 1 is present in all training examples"
243 with pytest.warns(UserWarning, match=msg):
244 ovr.fit(X, y)
245 y_pred = ovr.predict_proba(X)
246 assert_array_equal(y_pred[:, -1], np.zeros(X.shape[0]))
247
248
249def test_ovr_multiclass():
250 # Toy dataset where features correspond directly to labels.
251 X = np.array([[0, 0, 5], [0, 5, 0], [3, 0, 0], [0, 0, 6], [6, 0, 0]])
252 y = ["eggs", "spam", "ham", "eggs", "ham"]
253 Y = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0]])
254
255 classes = set("ham eggs spam".split())
256
257 for base_clf in (
258 MultinomialNB(),
259 LinearSVC(random_state=0),
260 LinearRegression(),
261 Ridge(),
262 ElasticNet(),
263 ):
264 clf = OneVsRestClassifier(base_clf).fit(X, y)
265 assert set(clf.classes_) == classes
266 y_pred = clf.predict(np.array([[0, 0, 4]]))[0]
267 assert_array_equal(y_pred, ["eggs"])
268
269 # test input as label indicator matrix
270 clf = OneVsRestClassifier(base_clf).fit(X, Y)
271 y_pred = clf.predict([[0, 0, 4]])[0]
272 assert_array_equal(y_pred, [0, 0, 1])
273
274
275def test_ovr_binary():
276 # Toy dataset where features correspond directly to labels.
277 X = np.array([[0, 0, 5], [0, 5, 0], [3, 0, 0], [0, 0, 6], [6, 0, 0]])
278 y = ["eggs", "spam", "spam", "eggs", "spam"]
279 Y = np.array([[0, 1, 1, 0, 1]]).T
280
281 classes = set("eggs spam".split())
282
283 def conduct_test(base_clf, test_predict_proba=False):
284 clf = OneVsRestClassifier(base_clf).fit(X, y)
285 assert set(clf.classes_) == classes
286 y_pred = clf.predict(np.array([[0, 0, 4]]))[0]
287 assert_array_equal(y_pred, ["eggs"])
288 if hasattr(base_clf, "decision_function"):
289 dec = clf.decision_function(X)
290 assert dec.shape == (5,)
291
292 if test_predict_proba:
293 X_test = np.array([[0, 0, 4]])
294 probabilities = clf.predict_proba(X_test)
295 assert 2 == len(probabilities[0])
296 assert clf.classes_[np.argmax(probabilities, axis=1)] == clf.predict(X_test)
297
298 # test input as label indicator matrix
299 clf = OneVsRestClassifier(base_clf).fit(X, Y)
300 y_pred = clf.predict([[3, 0, 0]])[0]
301 assert y_pred == 1
302
303 for base_clf in (
304 LinearSVC(random_state=0),
305 LinearRegression(),
306 Ridge(),
307 ElasticNet(),
308 ):
309 conduct_test(base_clf)
310
311 for base_clf in (MultinomialNB(), SVC(probability=True), LogisticRegression()):
312 conduct_test(base_clf, test_predict_proba=True)
313
314
315def test_ovr_multilabel():
316 # Toy dataset where features correspond directly to labels.
317 X = np.array([[0, 4, 5], [0, 5, 0], [3, 3, 3], [4, 0, 6], [6, 0, 0]])
318 y = np.array([[0, 1, 1], [0, 1, 0], [1, 1, 1], [1, 0, 1], [1, 0, 0]])
319
320 for base_clf in (
321 MultinomialNB(),
322 LinearSVC(random_state=0),
323 LinearRegression(),
324 Ridge(),
325 ElasticNet(),
326 Lasso(alpha=0.5),
327 ):
328 clf = OneVsRestClassifier(base_clf).fit(X, y)
329 y_pred = clf.predict([[0, 4, 4]])[0]
330 assert_array_equal(y_pred, [0, 1, 1])
331 assert clf.multilabel_
332
333
334def test_ovr_fit_predict_svc():
335 ovr = OneVsRestClassifier(svm.SVC())
336 ovr.fit(iris.data, iris.target)
337 assert len(ovr.estimators_) == 3
338 assert ovr.score(iris.data, iris.target) > 0.9
339
340
341def test_ovr_multilabel_dataset():
342 base_clf = MultinomialNB(alpha=1)
343 for au, prec, recall in zip((True, False), (0.51, 0.66), (0.51, 0.80)):
344 X, Y = datasets.make_multilabel_classification(
345 n_samples=100,
346 n_features=20,
347 n_classes=5,
348 n_labels=2,
349 length=50,
350 allow_unlabeled=au,
351 random_state=0,
352 )
353 X_train, Y_train = X[:80], Y[:80]
354 X_test, Y_test = X[80:], Y[80:]
355 clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
356 Y_pred = clf.predict(X_test)
357
358 assert clf.multilabel_
359 assert_almost_equal(
360 precision_score(Y_test, Y_pred, average="micro"), prec, decimal=2
361 )
362 assert_almost_equal(
363 recall_score(Y_test, Y_pred, average="micro"), recall, decimal=2
364 )
365
366
367def test_ovr_multilabel_predict_proba():
368 base_clf = MultinomialNB(alpha=1)
369 for au in (False, True):
370 X, Y = datasets.make_multilabel_classification(
371 n_samples=100,
372 n_features=20,
373 n_classes=5,
374 n_labels=3,
375 length=50,
376 allow_unlabeled=au,
377 random_state=0,
378 )
379 X_train, Y_train = X[:80], Y[:80]
380 X_test = X[80:]
381 clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
382
383 # Decision function only estimator.
384 decision_only = OneVsRestClassifier(svm.SVR()).fit(X_train, Y_train)
385 assert not hasattr(decision_only, "predict_proba")
386
387 # Estimator with predict_proba disabled, depending on parameters.
388 decision_only = OneVsRestClassifier(svm.SVC(probability=False))
389 assert not hasattr(decision_only, "predict_proba")
390 decision_only.fit(X_train, Y_train)
391 assert not hasattr(decision_only, "predict_proba")
392 assert hasattr(decision_only, "decision_function")
393
394 # Estimator which can get predict_proba enabled after fitting
395 gs = GridSearchCV(
396 svm.SVC(probability=False), param_grid={"probability": [True]}
397 )
398 proba_after_fit = OneVsRestClassifier(gs)
399 assert not hasattr(proba_after_fit, "predict_proba")
400 proba_after_fit.fit(X_train, Y_train)
401 assert hasattr(proba_after_fit, "predict_proba")
402
403 Y_pred = clf.predict(X_test)
404 Y_proba = clf.predict_proba(X_test)
405
406 # predict assigns a label if the probability that the
407 # sample has the label is greater than 0.5.
408 pred = Y_proba > 0.5
409 assert_array_equal(pred, Y_pred)
410
411
412def test_ovr_single_label_predict_proba():
413 base_clf = MultinomialNB(alpha=1)
414 X, Y = iris.data, iris.target
415 X_train, Y_train = X[:80], Y[:80]
416 X_test = X[80:]
417 clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
418
419 # Decision function only estimator.
420 decision_only = OneVsRestClassifier(svm.SVR()).fit(X_train, Y_train)
421 assert not hasattr(decision_only, "predict_proba")
422
423 Y_pred = clf.predict(X_test)
424 Y_proba = clf.predict_proba(X_test)
425
426 assert_almost_equal(Y_proba.sum(axis=1), 1.0)
427 # predict assigns a label if the probability that the
428 # sample has the label with the greatest predictive probability.
429 pred = Y_proba.argmax(axis=1)
430 assert not (pred - Y_pred).any()
431
432
433def test_ovr_single_label_predict_proba_zero():
434 """Check that predic_proba returns all zeros when the base estimator
435 never predicts the positive class.
436 """
437
438 class NaiveBinaryClassifier(BaseEstimator, ClassifierMixin):
439 def fit(self, X, y):
440 self.classes_ = np.unique(y)
441 return self
442
443 def predict_proba(self, X):
444 proba = np.ones((len(X), 2))
445 # Probability of being the positive class is always 0
446 proba[:, 1] = 0
447 return proba
448
449 base_clf = NaiveBinaryClassifier()
450 X, y = iris.data, iris.target # Three-class problem with 150 samples
451
452 clf = OneVsRestClassifier(base_clf).fit(X, y)
453 y_proba = clf.predict_proba(X)
454
455 assert_allclose(y_proba, 0.0)
456
457
458def test_ovr_multilabel_decision_function():
459 X, Y = datasets.make_multilabel_classification(
460 n_samples=100,
461 n_features=20,
462 n_classes=5,
463 n_labels=3,
464 length=50,
465 allow_unlabeled=True,
466 random_state=0,
467 )
468 X_train, Y_train = X[:80], Y[:80]
469 X_test = X[80:]
470 clf = OneVsRestClassifier(svm.SVC()).fit(X_train, Y_train)
471 assert_array_equal(
472 (clf.decision_function(X_test) > 0).astype(int), clf.predict(X_test)
473 )
474
475
476def test_ovr_single_label_decision_function():
477 X, Y = datasets.make_classification(n_samples=100, n_features=20, random_state=0)
478 X_train, Y_train = X[:80], Y[:80]
479 X_test = X[80:]
480 clf = OneVsRestClassifier(svm.SVC()).fit(X_train, Y_train)
481 assert_array_equal(clf.decision_function(X_test).ravel() > 0, clf.predict(X_test))
482
483
484def test_ovr_gridsearch():
485 ovr = OneVsRestClassifier(LinearSVC(random_state=0))
486 Cs = [0.1, 0.5, 0.8]
487 cv = GridSearchCV(ovr, {"estimator__C": Cs})
488 cv.fit(iris.data, iris.target)
489 best_C = cv.best_estimator_.estimators_[0].C
490 assert best_C in Cs
491
492
493def test_ovr_pipeline():
494 # Test with pipeline of length one
495 # This test is needed because the multiclass estimators may fail to detect
496 # the presence of predict_proba or decision_function.
497 clf = Pipeline([("tree", DecisionTreeClassifier())])
498 ovr_pipe = OneVsRestClassifier(clf)
499 ovr_pipe.fit(iris.data, iris.target)
500 ovr = OneVsRestClassifier(DecisionTreeClassifier())
501 ovr.fit(iris.data, iris.target)
502 assert_array_equal(ovr.predict(iris.data), ovr_pipe.predict(iris.data))
503
504
505def test_ovo_exceptions():
506 ovo = OneVsOneClassifier(LinearSVC(random_state=0))
507 with pytest.raises(NotFittedError):
508 ovo.predict([])
509
510
511def test_ovo_fit_on_list():
512 # Test that OneVsOne fitting works with a list of targets and yields the
513 # same output as predict from an array
514 ovo = OneVsOneClassifier(LinearSVC(random_state=0))
515 prediction_from_array = ovo.fit(iris.data, iris.target).predict(iris.data)
516 iris_data_list = [list(a) for a in iris.data]
517 prediction_from_list = ovo.fit(iris_data_list, list(iris.target)).predict(
518 iris_data_list
519 )
520 assert_array_equal(prediction_from_array, prediction_from_list)
521
522
523def test_ovo_fit_predict():
524 # A classifier which implements decision_function.
525 ovo = OneVsOneClassifier(LinearSVC(random_state=0))
526 ovo.fit(iris.data, iris.target).predict(iris.data)
527 assert len(ovo.estimators_) == n_classes * (n_classes - 1) / 2
528
529 # A classifier which implements predict_proba.
530 ovo = OneVsOneClassifier(MultinomialNB())
531 ovo.fit(iris.data, iris.target).predict(iris.data)
532 assert len(ovo.estimators_) == n_classes * (n_classes - 1) / 2
533
534
535def test_ovo_partial_fit_predict():
536 temp = datasets.load_iris()
537 X, y = temp.data, temp.target
538 ovo1 = OneVsOneClassifier(MultinomialNB())
539 ovo1.partial_fit(X[:100], y[:100], np.unique(y))
540 ovo1.partial_fit(X[100:], y[100:])
541 pred1 = ovo1.predict(X)
542
543 ovo2 = OneVsOneClassifier(MultinomialNB())
544 ovo2.fit(X, y)
545 pred2 = ovo2.predict(X)
546 assert len(ovo1.estimators_) == n_classes * (n_classes - 1) / 2
547 assert np.mean(y == pred1) > 0.65
548 assert_almost_equal(pred1, pred2)
549
550 # Test when mini-batches have binary target classes
551 ovo1 = OneVsOneClassifier(MultinomialNB())
552 ovo1.partial_fit(X[:60], y[:60], np.unique(y))
553 ovo1.partial_fit(X[60:], y[60:])
554 pred1 = ovo1.predict(X)
555 ovo2 = OneVsOneClassifier(MultinomialNB())
556 pred2 = ovo2.fit(X, y).predict(X)
557
558 assert_almost_equal(pred1, pred2)
559 assert len(ovo1.estimators_) == len(np.unique(y))
560 assert np.mean(y == pred1) > 0.65
561
562 ovo = OneVsOneClassifier(MultinomialNB())
563 X = np.random.rand(14, 2)
564 y = [1, 1, 2, 3, 3, 0, 0, 4, 4, 4, 4, 4, 2, 2]
565 ovo.partial_fit(X[:7], y[:7], [0, 1, 2, 3, 4])
566 ovo.partial_fit(X[7:], y[7:])
567 pred = ovo.predict(X)
568 ovo2 = OneVsOneClassifier(MultinomialNB())
569 pred2 = ovo2.fit(X, y).predict(X)
570 assert_almost_equal(pred, pred2)
571
572 # raises error when mini-batch does not have classes from all_classes
573 ovo = OneVsOneClassifier(MultinomialNB())
574 error_y = [0, 1, 2, 3, 4, 5, 2]
575 message_re = escape(
576 "Mini-batch contains {0} while it must be subset of {1}".format(
577 np.unique(error_y), np.unique(y)
578 )
579 )
580 with pytest.raises(ValueError, match=message_re):
581 ovo.partial_fit(X[:7], error_y, np.unique(y))
582
583 # test partial_fit only exists if estimator has it:
584 ovr = OneVsOneClassifier(SVC())
585 assert not hasattr(ovr, "partial_fit")
586
587
588def test_ovo_decision_function():
589 n_samples = iris.data.shape[0]
590
591 ovo_clf = OneVsOneClassifier(LinearSVC(random_state=0))
592 # first binary
593 ovo_clf.fit(iris.data, iris.target == 0)
594 decisions = ovo_clf.decision_function(iris.data)
595 assert decisions.shape == (n_samples,)
596
597 # then multi-class
598 ovo_clf.fit(iris.data, iris.target)
599 decisions = ovo_clf.decision_function(iris.data)
600
601 assert decisions.shape == (n_samples, n_classes)
602 assert_array_equal(decisions.argmax(axis=1), ovo_clf.predict(iris.data))
603
604 # Compute the votes
605 votes = np.zeros((n_samples, n_classes))
606
607 k = 0
608 for i in range(n_classes):
609 for j in range(i + 1, n_classes):
610 pred = ovo_clf.estimators_[k].predict(iris.data)
611 votes[pred == 0, i] += 1
612 votes[pred == 1, j] += 1
613 k += 1
614
615 # Extract votes and verify
616 assert_array_equal(votes, np.round(decisions))
617
618 for class_idx in range(n_classes):
619 # For each sample and each class, there only 3 possible vote levels
620 # because they are only 3 distinct class pairs thus 3 distinct
621 # binary classifiers.
622 # Therefore, sorting predictions based on votes would yield
623 # mostly tied predictions:
624 assert set(votes[:, class_idx]).issubset(set([0.0, 1.0, 2.0]))
625
626 # The OVO decision function on the other hand is able to resolve
627 # most of the ties on this data as it combines both the vote counts
628 # and the aggregated confidence levels of the binary classifiers
629 # to compute the aggregate decision function. The iris dataset
630 # has 150 samples with a couple of duplicates. The OvO decisions
631 # can resolve most of the ties:
632 assert len(np.unique(decisions[:, class_idx])) > 146
633
634
635def test_ovo_gridsearch():
636 ovo = OneVsOneClassifier(LinearSVC(random_state=0))
637 Cs = [0.1, 0.5, 0.8]
638 cv = GridSearchCV(ovo, {"estimator__C": Cs})
639 cv.fit(iris.data, iris.target)
640 best_C = cv.best_estimator_.estimators_[0].C
641 assert best_C in Cs
642
643
644def test_ovo_ties():
645 # Test that ties are broken using the decision function,
646 # not defaulting to the smallest label
647 X = np.array([[1, 2], [2, 1], [-2, 1], [-2, -1]])
648 y = np.array([2, 0, 1, 2])
649 multi_clf = OneVsOneClassifier(Perceptron(shuffle=False, max_iter=4, tol=None))
650 ovo_prediction = multi_clf.fit(X, y).predict(X)
651 ovo_decision = multi_clf.decision_function(X)
652
653 # Classifiers are in order 0-1, 0-2, 1-2
654 # Use decision_function to compute the votes and the normalized
655 # sum_of_confidences, which is used to disambiguate when there is a tie in
656 # votes.
657 votes = np.round(ovo_decision)
658 normalized_confidences = ovo_decision - votes
659
660 # For the first point, there is one vote per class
661 assert_array_equal(votes[0, :], 1)
662 # For the rest, there is no tie and the prediction is the argmax
663 assert_array_equal(np.argmax(votes[1:], axis=1), ovo_prediction[1:])
664 # For the tie, the prediction is the class with the highest score
665 assert ovo_prediction[0] == normalized_confidences[0].argmax()
666
667
668def test_ovo_ties2():
669 # test that ties can not only be won by the first two labels
670 X = np.array([[1, 2], [2, 1], [-2, 1], [-2, -1]])
671 y_ref = np.array([2, 0, 1, 2])
672
673 # cycle through labels so that each label wins once
674 for i in range(3):
675 y = (y_ref + i) % 3
676 multi_clf = OneVsOneClassifier(Perceptron(shuffle=False, max_iter=4, tol=None))
677 ovo_prediction = multi_clf.fit(X, y).predict(X)
678 assert ovo_prediction[0] == i % 3
679
680
681def test_ovo_string_y():
682 # Test that the OvO doesn't mess up the encoding of string labels
683 X = np.eye(4)
684 y = np.array(["a", "b", "c", "d"])
685
686 ovo = OneVsOneClassifier(LinearSVC())
687 ovo.fit(X, y)
688 assert_array_equal(y, ovo.predict(X))
689
690
691def test_ovo_one_class():
692 # Test error for OvO with one class
693 X = np.eye(4)
694 y = np.array(["a"] * 4)
695
696 ovo = OneVsOneClassifier(LinearSVC())
697 msg = "when only one class"
698 with pytest.raises(ValueError, match=msg):
699 ovo.fit(X, y)
700
701
702def test_ovo_float_y():
703 # Test that the OvO errors on float targets
704 X = iris.data
705 y = iris.data[:, 0]
706
707 ovo = OneVsOneClassifier(LinearSVC())
708 msg = "Unknown label type"
709 with pytest.raises(ValueError, match=msg):
710 ovo.fit(X, y)
711
712
713def test_ecoc_exceptions():
714 ecoc = OutputCodeClassifier(LinearSVC(random_state=0))
715 with pytest.raises(NotFittedError):
716 ecoc.predict([])
717
718
719def test_ecoc_fit_predict():
720 # A classifier which implements decision_function.
721 ecoc = OutputCodeClassifier(LinearSVC(random_state=0), code_size=2, random_state=0)
722 ecoc.fit(iris.data, iris.target).predict(iris.data)
723 assert len(ecoc.estimators_) == n_classes * 2
724
725 # A classifier which implements predict_proba.
726 ecoc = OutputCodeClassifier(MultinomialNB(), code_size=2, random_state=0)
727 ecoc.fit(iris.data, iris.target).predict(iris.data)
728 assert len(ecoc.estimators_) == n_classes * 2
729
730
731def test_ecoc_gridsearch():
732 ecoc = OutputCodeClassifier(LinearSVC(random_state=0), random_state=0)
733 Cs = [0.1, 0.5, 0.8]
734 cv = GridSearchCV(ecoc, {"estimator__C": Cs})
735 cv.fit(iris.data, iris.target)
736 best_C = cv.best_estimator_.estimators_[0].C
737 assert best_C in Cs
738
739
740def test_ecoc_float_y():
741 # Test that the OCC errors on float targets
742 X = iris.data
743 y = iris.data[:, 0]
744
745 ovo = OutputCodeClassifier(LinearSVC())
746 msg = "Unknown label type"
747 with pytest.raises(ValueError, match=msg):
748 ovo.fit(X, y)
749
750
751@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
752def test_ecoc_delegate_sparse_base_estimator(csc_container):
753 # Non-regression test for
754 # https://github.com/scikit-learn/scikit-learn/issues/17218
755 X, y = iris.data, iris.target
756 X_sp = csc_container(X)
757
758 # create an estimator that does not support sparse input
759 base_estimator = CheckingClassifier(
760 check_X=check_array,
761 check_X_params={"ensure_2d": True, "accept_sparse": False},
762 )
763 ecoc = OutputCodeClassifier(base_estimator, random_state=0)
764
765 with pytest.raises(TypeError, match="Sparse data was passed"):
766 ecoc.fit(X_sp, y)
767
768 ecoc.fit(X, y)
769 with pytest.raises(TypeError, match="Sparse data was passed"):
770 ecoc.predict(X_sp)
771
772 # smoke test to check when sparse input should be supported
773 ecoc = OutputCodeClassifier(LinearSVC(random_state=0))
774 ecoc.fit(X_sp, y).predict(X_sp)
775 assert len(ecoc.estimators_) == 4
776
777
778def test_pairwise_indices():
779 clf_precomputed = svm.SVC(kernel="precomputed")
780 X, y = iris.data, iris.target
781
782 ovr_false = OneVsOneClassifier(clf_precomputed)
783 linear_kernel = np.dot(X, X.T)
784 ovr_false.fit(linear_kernel, y)
785
786 n_estimators = len(ovr_false.estimators_)
787 precomputed_indices = ovr_false.pairwise_indices_
788
789 for idx in precomputed_indices:
790 assert (
791 idx.shape[0] * n_estimators / (n_estimators - 1) == linear_kernel.shape[0]
792 )
793
794
795def test_pairwise_n_features_in():
796 """Check the n_features_in_ attributes of the meta and base estimators
797
798 When the training data is a regular design matrix, everything is intuitive.
799 However, when the training data is a precomputed kernel matrix, the
800 multiclass strategy can resample the kernel matrix of the underlying base
801 estimator both row-wise and column-wise and this has a non-trivial impact
802 on the expected value for the n_features_in_ of both the meta and the base
803 estimators.
804 """
805 X, y = iris.data, iris.target
806
807 # Remove the last sample to make the classes not exactly balanced and make
808 # the test more interesting.
809 assert y[-1] == 0
810 X = X[:-1]
811 y = y[:-1]
812
813 # Fitting directly on the design matrix:
814 assert X.shape == (149, 4)
815
816 clf_notprecomputed = svm.SVC(kernel="linear").fit(X, y)
817 assert clf_notprecomputed.n_features_in_ == 4
818
819 ovr_notprecomputed = OneVsRestClassifier(clf_notprecomputed).fit(X, y)
820 assert ovr_notprecomputed.n_features_in_ == 4
821 for est in ovr_notprecomputed.estimators_:
822 assert est.n_features_in_ == 4
823
824 ovo_notprecomputed = OneVsOneClassifier(clf_notprecomputed).fit(X, y)
825 assert ovo_notprecomputed.n_features_in_ == 4
826 assert ovo_notprecomputed.n_classes_ == 3
827 assert len(ovo_notprecomputed.estimators_) == 3
828 for est in ovo_notprecomputed.estimators_:
829 assert est.n_features_in_ == 4
830
831 # When working with precomputed kernels we have one "feature" per training
832 # sample:
833 K = X @ X.T
834 assert K.shape == (149, 149)
835
836 clf_precomputed = svm.SVC(kernel="precomputed").fit(K, y)
837 assert clf_precomputed.n_features_in_ == 149
838
839 ovr_precomputed = OneVsRestClassifier(clf_precomputed).fit(K, y)
840 assert ovr_precomputed.n_features_in_ == 149
841 assert ovr_precomputed.n_classes_ == 3
842 assert len(ovr_precomputed.estimators_) == 3
843 for est in ovr_precomputed.estimators_:
844 assert est.n_features_in_ == 149
845
846 # This becomes really interesting with OvO and precomputed kernel together:
847 # internally, OvO will drop the samples of the classes not part of the pair
848 # of classes under consideration for a given binary classifier. Since we
849 # use a precomputed kernel, it will also drop the matching columns of the
850 # kernel matrix, and therefore we have fewer "features" as result.
851 #
852 # Since class 0 has 49 samples, and class 1 and 2 have 50 samples each, a
853 # single OvO binary classifier works with a sub-kernel matrix of shape
854 # either (99, 99) or (100, 100).
855 ovo_precomputed = OneVsOneClassifier(clf_precomputed).fit(K, y)
856 assert ovo_precomputed.n_features_in_ == 149
857 assert ovr_precomputed.n_classes_ == 3
858 assert len(ovr_precomputed.estimators_) == 3
859 assert ovo_precomputed.estimators_[0].n_features_in_ == 99 # class 0 vs class 1
860 assert ovo_precomputed.estimators_[1].n_features_in_ == 99 # class 0 vs class 2
861 assert ovo_precomputed.estimators_[2].n_features_in_ == 100 # class 1 vs class 2
862
863
864@pytest.mark.parametrize(
865 "MultiClassClassifier", [OneVsRestClassifier, OneVsOneClassifier]
866)
867def test_pairwise_tag(MultiClassClassifier):
868 clf_precomputed = svm.SVC(kernel="precomputed")
869 clf_notprecomputed = svm.SVC()
870
871 ovr_false = MultiClassClassifier(clf_notprecomputed)
872 assert not ovr_false.__sklearn_tags__().input_tags.pairwise
873
874 ovr_true = MultiClassClassifier(clf_precomputed)
875 assert ovr_true.__sklearn_tags__().input_tags.pairwise
876
877
878@pytest.mark.parametrize(
879 "MultiClassClassifier", [OneVsRestClassifier, OneVsOneClassifier]
880)
881def test_pairwise_cross_val_score(MultiClassClassifier):
882 clf_precomputed = svm.SVC(kernel="precomputed")
883 clf_notprecomputed = svm.SVC(kernel="linear")
884
885 X, y = iris.data, iris.target
886
887 multiclass_clf_notprecomputed = MultiClassClassifier(clf_notprecomputed)
888 multiclass_clf_precomputed = MultiClassClassifier(clf_precomputed)
889
890 linear_kernel = np.dot(X, X.T)
891 score_not_precomputed = cross_val_score(
892 multiclass_clf_notprecomputed, X, y, error_score="raise"
893 )
894 score_precomputed = cross_val_score(
895 multiclass_clf_precomputed, linear_kernel, y, error_score="raise"
896 )
897 assert_array_equal(score_precomputed, score_not_precomputed)
898
899
900@pytest.mark.parametrize(
901 "MultiClassClassifier", [OneVsRestClassifier, OneVsOneClassifier]
902)
903# FIXME: we should move this test in `estimator_checks` once we are able
904# to construct meta-estimator instances
905def test_support_missing_values(MultiClassClassifier):
906 # smoke test to check that pipeline OvR and OvO classifiers are letting
907 # the validation of missing values to
908 # the underlying pipeline or classifiers
909 rng = np.random.RandomState(42)
910 X, y = iris.data, iris.target
911 X = np.copy(X) # Copy to avoid that the original data is modified
912 mask = rng.choice([1, 0], X.shape, p=[0.1, 0.9]).astype(bool)
913 X[mask] = np.nan
914 lr = make_pipeline(SimpleImputer(), LogisticRegression(random_state=rng))
915
916 MultiClassClassifier(lr).fit(X, y).score(X, y)
917
918
919@pytest.mark.parametrize("make_y", [np.ones, np.zeros])
920def test_constant_int_target(make_y):
921 """Check that constant y target does not raise.
922
923 Non-regression test for #21869
924 """
925 X = np.ones((10, 2))
926 y = make_y((10, 1), dtype=np.int32)
927 ovr = OneVsRestClassifier(LogisticRegression())
928
929 ovr.fit(X, y)
930 y_pred = ovr.predict_proba(X)
931 expected = np.zeros((X.shape[0], 2))
932 expected[:, 0] = 1
933 assert_allclose(y_pred, expected)
934
935
936def test_ovo_consistent_binary_classification():
937 """Check that ovo is consistent with binary classifier.
938
939 Non-regression test for #13617.
940 """
941 X, y = load_breast_cancer(return_X_y=True)
942
943 clf = KNeighborsClassifier(n_neighbors=8, weights="distance")
944 ovo = OneVsOneClassifier(clf)
945
946 clf.fit(X, y)
947 ovo.fit(X, y)
948
949 assert_array_equal(clf.predict(X), ovo.predict(X))
950
951
952def test_multiclass_estimator_attribute_error():
953 """Check that we raise the proper AttributeError when the final estimator
954 does not implement the `partial_fit` method, which is decorated with
955 `available_if`.
956
957 Non-regression test for:
958 https://github.com/scikit-learn/scikit-learn/issues/28108
959 """
960 iris = datasets.load_iris()
961
962 # LogisticRegression does not implement 'partial_fit' and should raise an
963 # AttributeError
964 clf = OneVsRestClassifier(estimator=LogisticRegression(random_state=42))
965
966 outer_msg = "This 'OneVsRestClassifier' has no attribute 'partial_fit'"
967 inner_msg = "'LogisticRegression' object has no attribute 'partial_fit'"
968 with pytest.raises(AttributeError, match=outer_msg) as exec_info:
969 clf.partial_fit(iris.data, iris.target)
970 assert isinstance(exec_info.value.__cause__, AttributeError)
971 assert inner_msg in str(exec_info.value.__cause__)
972