Aluode/PerceptionLabPortable
0
1# Authors: The scikit-learn developers
2# SPDX-License-Identifier: BSD-3-Clause
3
4from array import array
5from collections.abc import Iterable, Mapping
6from numbers import Number
7from operator import itemgetter
8
9import numpy as np
10import scipy.sparse as sp
11
12from sklearn.utils import metadata_routing
13
14from ..base import BaseEstimator, TransformerMixin, _fit_context
15from ..utils import check_array
16from ..utils.validation import check_is_fitted
17
18
19class DictVectorizer(TransformerMixin, BaseEstimator):
20 """Transforms lists of feature-value mappings to vectors.
21
22 This transformer turns lists of mappings (dict-like objects) of feature
23 names to feature values into Numpy arrays or scipy.sparse matrices for use
24 with scikit-learn estimators.
25
26 When feature values are strings, this transformer will do a binary one-hot
27 (aka one-of-K) coding: one boolean-valued feature is constructed for each
28 of the possible string values that the feature can take on. For instance,
29 a feature "f" that can take on the values "ham" and "spam" will become two
30 features in the output, one signifying "f=ham", the other "f=spam".
31
32 If a feature value is a sequence or set of strings, this transformer
33 will iterate over the values and will count the occurrences of each string
34 value.
35
36 However, note that this transformer will only do a binary one-hot encoding
37 when feature values are of type string. If categorical features are
38 represented as numeric values such as int or iterables of strings, the
39 DictVectorizer can be followed by
40 :class:`~sklearn.preprocessing.OneHotEncoder` to complete
41 binary one-hot encoding.
42
43 Features that do not occur in a sample (mapping) will have a zero value
44 in the resulting array/matrix.
45
46 For an efficiency comparison of the different feature extractors, see
47 :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py`.
48
49 Read more in the :ref:`User Guide <dict_feature_extraction>`.
50
51 Parameters
52 ----------
53 dtype : dtype, default=np.float64
54 The type of feature values. Passed to Numpy array/scipy.sparse matrix
55 constructors as the dtype argument.
56 separator : str, default="="
57 Separator string used when constructing new features for one-hot
58 coding.
59 sparse : bool, default=True
60 Whether transform should produce scipy.sparse matrices.
61 sort : bool, default=True
62 Whether ``feature_names_`` and ``vocabulary_`` should be
63 sorted when fitting.
64
65 Attributes
66 ----------
67 vocabulary_ : dict
68 A dictionary mapping feature names to feature indices.
69
70 feature_names_ : list
71 A list of length n_features containing the feature names (e.g., "f=ham"
72 and "f=spam").
73
74 See Also
75 --------
76 FeatureHasher : Performs vectorization using only a hash function.
77 sklearn.preprocessing.OrdinalEncoder : Handles nominal/categorical
78 features encoded as columns of arbitrary data types.
79
80 Examples
81 --------
82 >>> from sklearn.feature_extraction import DictVectorizer
83 >>> v = DictVectorizer(sparse=False)
84 >>> D = [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]
85 >>> X = v.fit_transform(D)
86 >>> X
87 array([[2., 0., 1.],
88 [0., 1., 3.]])
89 >>> v.inverse_transform(X) == [{'bar': 2.0, 'foo': 1.0},
90 ... {'baz': 1.0, 'foo': 3.0}]
91 True
92 >>> v.transform({'foo': 4, 'unseen_feature': 3})
93 array([[0., 0., 4.]])
94 """
95
96 # This isn't something that people should be routing / using in a pipeline.
97 __metadata_request__inverse_transform = {"dict_type": metadata_routing.UNUSED}
98
99 _parameter_constraints: dict = {
100 "dtype": "no_validation", # validation delegated to numpy,
101 "separator": [str],
102 "sparse": ["boolean"],
103 "sort": ["boolean"],
104 }
105
106 def __init__(self, *, dtype=np.float64, separator="=", sparse=True, sort=True):
107 self.dtype = dtype
108 self.separator = separator
109 self.sparse = sparse
110 self.sort = sort
111
112 def _add_iterable_element(
113 self,
114 f,
115 v,
116 feature_names,
117 vocab,
118 *,
119 fitting=True,
120 transforming=False,
121 indices=None,
122 values=None,
123 ):
124 """Add feature names for iterable of strings"""
125 for vv in v:
126 if isinstance(vv, str):
127 feature_name = "%s%s%s" % (f, self.separator, vv)
128 vv = 1
129 else:
130 raise TypeError(
131 f"Unsupported type {type(vv)} in iterable "
132 "value. Only iterables of string are "
133 "supported."
134 )
135 if fitting and feature_name not in vocab:
136 vocab[feature_name] = len(feature_names)
137 feature_names.append(feature_name)
138
139 if transforming and feature_name in vocab:
140 indices.append(vocab[feature_name])
141 values.append(self.dtype(vv))
142
143 @_fit_context(prefer_skip_nested_validation=True)
144 def fit(self, X, y=None):
145 """Learn a list of feature name -> indices mappings.
146
147 Parameters
148 ----------
149 X : Mapping or iterable over Mappings
150 Dict(s) or Mapping(s) from feature names (arbitrary Python
151 objects) to feature values (strings or convertible to dtype).
152
153 .. versionchanged:: 0.24
154 Accepts multiple string values for one categorical feature.
155
156 y : (ignored)
157 Ignored parameter.
158
159 Returns
160 -------
161 self : object
162 DictVectorizer class instance.
163 """
164 feature_names = []
165 vocab = {}
166
167 for x in X:
168 for f, v in x.items():
169 if isinstance(v, str):
170 feature_name = "%s%s%s" % (f, self.separator, v)
171 elif isinstance(v, Number) or (v is None):
172 feature_name = f
173 elif isinstance(v, Mapping):
174 raise TypeError(
175 f"Unsupported value type {type(v)} "
176 f"for {f}: {v}.\n"
177 "Mapping objects are not supported."
178 )
179 elif isinstance(v, Iterable):
180 feature_name = None
181 self._add_iterable_element(f, v, feature_names, vocab)
182
183 if feature_name is not None:
184 if feature_name not in vocab:
185 vocab[feature_name] = len(feature_names)
186 feature_names.append(feature_name)
187
188 if self.sort:
189 feature_names.sort()
190 vocab = {f: i for i, f in enumerate(feature_names)}
191
192 self.feature_names_ = feature_names
193 self.vocabulary_ = vocab
194
195 return self
196
197 def _transform(self, X, fitting):
198 # Sanity check: Python's array has no way of explicitly requesting the
199 # signed 32-bit integers that scipy.sparse needs, so we use the next
200 # best thing: typecode "i" (int). However, if that gives larger or
201 # smaller integers than 32-bit ones, np.frombuffer screws up.
202 assert array("i").itemsize == 4, (
203 "sizeof(int) != 4 on your platform; please report this at"
204 " https://github.com/scikit-learn/scikit-learn/issues and"
205 " include the output from platform.platform() in your bug report"
206 )
207
208 dtype = self.dtype
209 if fitting:
210 feature_names = []
211 vocab = {}
212 else:
213 feature_names = self.feature_names_
214 vocab = self.vocabulary_
215
216 transforming = True
217
218 # Process everything as sparse regardless of setting
219 X = [X] if isinstance(X, Mapping) else X
220
221 indices = array("i")
222 indptr = [0]
223 # XXX we could change values to an array.array as well, but it
224 # would require (heuristic) conversion of dtype to typecode...
225 values = []
226
227 # collect all the possible feature names and build sparse matrix at
228 # same time
229 for x in X:
230 for f, v in x.items():
231 if isinstance(v, str):
232 feature_name = "%s%s%s" % (f, self.separator, v)
233 v = 1
234 elif isinstance(v, Number) or (v is None):
235 feature_name = f
236 elif not isinstance(v, Mapping) and isinstance(v, Iterable):
237 feature_name = None
238 self._add_iterable_element(
239 f,
240 v,
241 feature_names,
242 vocab,
243 fitting=fitting,
244 transforming=transforming,
245 indices=indices,
246 values=values,
247 )
248 else:
249 raise TypeError(
250 f"Unsupported value Type {type(v)} "
251 f"for {f}: {v}.\n"
252 f"{type(v)} objects are not supported."
253 )
254
255 if feature_name is not None:
256 if fitting and feature_name not in vocab:
257 vocab[feature_name] = len(feature_names)
258 feature_names.append(feature_name)
259
260 if feature_name in vocab:
261 indices.append(vocab[feature_name])
262 values.append(self.dtype(v))
263
264 indptr.append(len(indices))
265
266 if len(indptr) == 1:
267 raise ValueError("Sample sequence X is empty.")
268
269 indices = np.frombuffer(indices, dtype=np.intc)
270 shape = (len(indptr) - 1, len(vocab))
271
272 result_matrix = sp.csr_matrix(
273 (values, indices, indptr), shape=shape, dtype=dtype
274 )
275
276 # Sort everything if asked
277 if fitting and self.sort:
278 feature_names.sort()
279 map_index = np.empty(len(feature_names), dtype=np.int32)
280 for new_val, f in enumerate(feature_names):
281 map_index[new_val] = vocab[f]
282 vocab[f] = new_val
283 result_matrix = result_matrix[:, map_index]
284
285 if self.sparse:
286 result_matrix.sort_indices()
287 else:
288 result_matrix = result_matrix.toarray()
289
290 if fitting:
291 self.feature_names_ = feature_names
292 self.vocabulary_ = vocab
293
294 return result_matrix
295
296 @_fit_context(prefer_skip_nested_validation=True)
297 def fit_transform(self, X, y=None):
298 """Learn a list of feature name -> indices mappings and transform X.
299
300 Like fit(X) followed by transform(X), but does not require
301 materializing X in memory.
302
303 Parameters
304 ----------
305 X : Mapping or iterable over Mappings
306 Dict(s) or Mapping(s) from feature names (arbitrary Python
307 objects) to feature values (strings or convertible to dtype).
308
309 .. versionchanged:: 0.24
310 Accepts multiple string values for one categorical feature.
311
312 y : (ignored)
313 Ignored parameter.
314
315 Returns
316 -------
317 Xa : {array, sparse matrix}
318 Feature vectors; always 2-d.
319 """
320 return self._transform(X, fitting=True)
321
322 def inverse_transform(self, X, dict_type=dict):
323 """Transform array or sparse matrix X back to feature mappings.
324
325 X must have been produced by this DictVectorizer's transform or
326 fit_transform method; it may only have passed through transformers
327 that preserve the number of features and their order.
328
329 In the case of one-hot/one-of-K coding, the constructed feature
330 names and values are returned rather than the original ones.
331
332 Parameters
333 ----------
334 X : {array-like, sparse matrix} of shape (n_samples, n_features)
335 Sample matrix.
336 dict_type : type, default=dict
337 Constructor for feature mappings. Must conform to the
338 collections.Mapping API.
339
340 Returns
341 -------
342 X_original : list of dict_type objects of shape (n_samples,)
343 Feature mappings for the samples in X.
344 """
345 check_is_fitted(self, "feature_names_")
346
347 # COO matrix is not subscriptable
348 X = check_array(X, accept_sparse=["csr", "csc"])
349 n_samples = X.shape[0]
350
351 names = self.feature_names_
352 dicts = [dict_type() for _ in range(n_samples)]
353
354 if sp.issparse(X):
355 for i, j in zip(*X.nonzero()):
356 dicts[i][names[j]] = X[i, j]
357 else:
358 for i, d in enumerate(dicts):
359 for j, v in enumerate(X[i, :]):
360 if v != 0:
361 d[names[j]] = X[i, j]
362
363 return dicts
364
365 def transform(self, X):
366 """Transform feature->value dicts to array or sparse matrix.
367
368 Named features not encountered during fit or fit_transform will be
369 silently ignored.
370
371 Parameters
372 ----------
373 X : Mapping or iterable over Mappings of shape (n_samples,)
374 Dict(s) or Mapping(s) from feature names (arbitrary Python
375 objects) to feature values (strings or convertible to dtype).
376
377 Returns
378 -------
379 Xa : {array, sparse matrix}
380 Feature vectors; always 2-d.
381 """
382 check_is_fitted(self, ["feature_names_", "vocabulary_"])
383 return self._transform(X, fitting=False)
384
385 def get_feature_names_out(self, input_features=None):
386 """Get output feature names for transformation.
387
388 Parameters
389 ----------
390 input_features : array-like of str or None, default=None
391 Not used, present here for API consistency by convention.
392
393 Returns
394 -------
395 feature_names_out : ndarray of str objects
396 Transformed feature names.
397 """
398 check_is_fitted(self, "feature_names_")
399 if any(not isinstance(name, str) for name in self.feature_names_):
400 feature_names = [str(name) for name in self.feature_names_]
401 else:
402 feature_names = self.feature_names_
403 return np.asarray(feature_names, dtype=object)
404
405 def restrict(self, support, indices=False):
406 """Restrict the features to those in support using feature selection.
407
408 This function modifies the estimator in-place.
409
410 Parameters
411 ----------
412 support : array-like
413 Boolean mask or list of indices (as returned by the get_support
414 member of feature selectors).
415 indices : bool, default=False
416 Whether support is a list of indices.
417
418 Returns
419 -------
420 self : object
421 DictVectorizer class instance.
422
423 Examples
424 --------
425 >>> from sklearn.feature_extraction import DictVectorizer
426 >>> from sklearn.feature_selection import SelectKBest, chi2
427 >>> v = DictVectorizer()
428 >>> D = [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]
429 >>> X = v.fit_transform(D)
430 >>> support = SelectKBest(chi2, k=2).fit(X, [0, 1])
431 >>> v.get_feature_names_out()
432 array(['bar', 'baz', 'foo'], ...)
433 >>> v.restrict(support.get_support())
434 DictVectorizer()
435 >>> v.get_feature_names_out()
436 array(['bar', 'foo'], ...)
437 """
438 check_is_fitted(self, "feature_names_")
439
440 if not indices:
441 support = np.where(support)[0]
442
443 names = self.feature_names_
444 new_vocab = {}
445 for i in support:
446 new_vocab[names[i]] = len(new_vocab)
447
448 self.vocabulary_ = new_vocab
449 self.feature_names_ = [
450 f for f, i in sorted(new_vocab.items(), key=itemgetter(1))
451 ]
452
453 return self
454
455 def __sklearn_tags__(self):
456 tags = super().__sklearn_tags__()
457 tags.input_tags.dict = True
458 tags.input_tags.two_d_array = False
459 return tags
460