Aluode/PerceptionLabPortable
0
1# Copyright 2022 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""15Doc utilities: Utilities related to documentation16"""17 18import functools19import inspect20import re21import textwrap22import types23from collections import OrderedDict24 25 26def get_docstring_indentation_level(func):27 """Return the indentation level of the start of the docstring of a class or function (or method)."""28 # We assume classes are always defined in the global scope29 if inspect.isclass(func):30 return 431 source = inspect.getsource(func)32 first_line = source.splitlines()[0]33 function_def_level = len(first_line) - len(first_line.lstrip())34 return 4 + function_def_level35 36 37def add_start_docstrings(*docstr):38 def docstring_decorator(fn):39 fn.__doc__ = "".join(docstr) + (fn.__doc__ if fn.__doc__ is not None else "")40 return fn41 42 return docstring_decorator43 44 45def add_start_docstrings_to_model_forward(*docstr):46 def docstring_decorator(fn):47 class_name = f"[`{fn.__qualname__.split('.')[0]}`]"48 intro = rf""" The {class_name} forward method, overrides the `__call__` special method.49 50 <Tip>51 52 Although the recipe for forward pass needs to be defined within this function, one should call the [`Module`]53 instance afterwards instead of this since the former takes care of running the pre and post processing steps while54 the latter silently ignores them.55 56 </Tip>57"""58 59 correct_indentation = get_docstring_indentation_level(fn)60 current_doc = fn.__doc__ if fn.__doc__ is not None else ""61 try:62 first_non_empty = next(line for line in current_doc.splitlines() if line.strip() != "")63 doc_indentation = len(first_non_empty) - len(first_non_empty.lstrip())64 except StopIteration:65 doc_indentation = correct_indentation66 67 docs = docstr68 # In this case, the correct indentation level (class method, 2 Python levels) was respected, and we should69 # correctly reindent everything. Otherwise, the doc uses a single indentation level70 if doc_indentation == 4 + correct_indentation:71 docs = [textwrap.indent(textwrap.dedent(doc), " " * correct_indentation) for doc in docstr]72 intro = textwrap.indent(textwrap.dedent(intro), " " * correct_indentation)73 74 docstring = "".join(docs) + current_doc75 fn.__doc__ = intro + docstring76 return fn77 78 return docstring_decorator79 80 81def add_end_docstrings(*docstr):82 def docstring_decorator(fn):83 fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "".join(docstr)84 return fn85 86 return docstring_decorator87 88 89PT_RETURN_INTRODUCTION = r"""90 Returns:91 [`{full_output_type}`] or `tuple(torch.FloatTensor)`: A [`{full_output_type}`] or a tuple of92 `torch.FloatTensor` (if `return_dict=False` is passed or when `config.return_dict=False`) comprising various93 elements depending on the configuration ([`{config_class}`]) and inputs.94 95"""96 97 98TF_RETURN_INTRODUCTION = r"""99 Returns:100 [`{full_output_type}`] or `tuple(tf.Tensor)`: A [`{full_output_type}`] or a tuple of `tf.Tensor` (if101 `return_dict=False` is passed or when `config.return_dict=False`) comprising various elements depending on the102 configuration ([`{config_class}`]) and inputs.103 104"""105 106 107def _get_indent(t):108 """Returns the indentation in the first line of t"""109 search = re.search(r"^(\s*)\S", t)110 return "" if search is None else search.groups()[0]111 112 113def _convert_output_args_doc(output_args_doc):114 """Convert output_args_doc to display properly."""115 # Split output_arg_doc in blocks argument/description116 indent = _get_indent(output_args_doc)117 blocks = []118 current_block = ""119 for line in output_args_doc.split("\n"):120 # If the indent is the same as the beginning, the line is the name of new arg.121 if _get_indent(line) == indent:122 if len(current_block) > 0:123 blocks.append(current_block[:-1])124 current_block = f"{line}\n"125 else:126 # Otherwise it's part of the description of the current arg.127 # We need to remove 2 spaces to the indentation.128 current_block += f"{line[2:]}\n"129 blocks.append(current_block[:-1])130 131 # Format each block for proper rendering132 for i in range(len(blocks)):133 blocks[i] = re.sub(r"^(\s+)(\S+)(\s+)", r"\1- **\2**\3", blocks[i])134 blocks[i] = re.sub(r":\s*\n\s*(\S)", r" -- \1", blocks[i])135 136 return "\n".join(blocks)137 138 139def _prepare_output_docstrings(output_type, config_class, min_indent=None, add_intro=True):140 """141 Prepares the return part of the docstring using `output_type`.142 """143 output_docstring = output_type.__doc__144 params_docstring = None145 if output_docstring is not None:146 # Remove the head of the docstring to keep the list of args only147 lines = output_docstring.split("\n")148 i = 0149 while i < len(lines) and re.search(r"^\s*(Args|Parameters):\s*$", lines[i]) is None:150 i += 1151 if i < len(lines):152 params_docstring = "\n".join(lines[(i + 1) :])153 params_docstring = _convert_output_args_doc(params_docstring)154 elif add_intro:155 raise ValueError(156 f"No `Args` or `Parameters` section is found in the docstring of `{output_type.__name__}`. Make sure it has "157 "docstring and contain either `Args` or `Parameters`."158 )159 160 # Add the return introduction161 if add_intro:162 full_output_type = f"{output_type.__module__}.{output_type.__name__}"163 intro = TF_RETURN_INTRODUCTION if output_type.__name__.startswith("TF") else PT_RETURN_INTRODUCTION164 intro = intro.format(full_output_type=full_output_type, config_class=config_class)165 else:166 full_output_type = str(output_type)167 intro = f"\nReturns:\n `{full_output_type}`"168 if params_docstring is not None:169 intro += ":\n"170 171 result = intro172 if params_docstring is not None:173 result += params_docstring174 175 # Apply minimum indent if necessary176 if min_indent is not None:177 lines = result.split("\n")178 # Find the indent of the first nonempty line179 i = 0180 while len(lines[i]) == 0:181 i += 1182 indent = len(_get_indent(lines[i]))183 # If too small, add indentation to all nonempty lines184 if indent < min_indent:185 to_add = " " * (min_indent - indent)186 lines = [(f"{to_add}{line}" if len(line) > 0 else line) for line in lines]187 result = "\n".join(lines)188 189 return result190 191 192FAKE_MODEL_DISCLAIMER = """193 <Tip warning={true}>194 195 This example uses a random model as the real ones are all very big. To get proper results, you should use196 {real_checkpoint} instead of {fake_checkpoint}. If you get out-of-memory when loading that checkpoint, you can try197 adding `device_map="auto"` in the `from_pretrained` call.198 199 </Tip>200"""201 202 203PT_TOKEN_CLASSIFICATION_SAMPLE = r"""204 Example:205 206 ```python207 >>> from transformers import AutoTokenizer, {model_class}208 >>> import torch209 210 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")211 >>> model = {model_class}.from_pretrained("{checkpoint}")212 213 >>> inputs = tokenizer(214 ... "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="pt"215 ... )216 217 >>> with torch.no_grad():218 ... logits = model(**inputs).logits219 220 >>> predicted_token_class_ids = logits.argmax(-1)221 222 >>> # Note that tokens are classified rather then input words which means that223 >>> # there might be more predicted token classes than words.224 >>> # Multiple token classes might account for the same word225 >>> predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]]226 >>> predicted_tokens_classes227 {expected_output}228 229 >>> labels = predicted_token_class_ids230 >>> loss = model(**inputs, labels=labels).loss231 >>> round(loss.item(), 2)232 {expected_loss}233 ```234"""235 236PT_QUESTION_ANSWERING_SAMPLE = r"""237 Example:238 239 ```python240 >>> from transformers import AutoTokenizer, {model_class}241 >>> import torch242 243 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")244 >>> model = {model_class}.from_pretrained("{checkpoint}")245 246 >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"247 248 >>> inputs = tokenizer(question, text, return_tensors="pt")249 >>> with torch.no_grad():250 ... outputs = model(**inputs)251 252 >>> answer_start_index = outputs.start_logits.argmax()253 >>> answer_end_index = outputs.end_logits.argmax()254 255 >>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1]256 >>> tokenizer.decode(predict_answer_tokens, skip_special_tokens=True)257 {expected_output}258 259 >>> # target is "nice puppet"260 >>> target_start_index = torch.tensor([{qa_target_start_index}])261 >>> target_end_index = torch.tensor([{qa_target_end_index}])262 263 >>> outputs = model(**inputs, start_positions=target_start_index, end_positions=target_end_index)264 >>> loss = outputs.loss265 >>> round(loss.item(), 2)266 {expected_loss}267 ```268"""269 270PT_SEQUENCE_CLASSIFICATION_SAMPLE = r"""271 Example of single-label classification:272 273 ```python274 >>> import torch275 >>> from transformers import AutoTokenizer, {model_class}276 277 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")278 >>> model = {model_class}.from_pretrained("{checkpoint}")279 280 >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")281 282 >>> with torch.no_grad():283 ... logits = model(**inputs).logits284 285 >>> predicted_class_id = logits.argmax().item()286 >>> model.config.id2label[predicted_class_id]287 {expected_output}288 289 >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)`290 >>> num_labels = len(model.config.id2label)291 >>> model = {model_class}.from_pretrained("{checkpoint}", num_labels=num_labels)292 293 >>> labels = torch.tensor([1])294 >>> loss = model(**inputs, labels=labels).loss295 >>> round(loss.item(), 2)296 {expected_loss}297 ```298 299 Example of multi-label classification:300 301 ```python302 >>> import torch303 >>> from transformers import AutoTokenizer, {model_class}304 305 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")306 >>> model = {model_class}.from_pretrained("{checkpoint}", problem_type="multi_label_classification")307 308 >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")309 310 >>> with torch.no_grad():311 ... logits = model(**inputs).logits312 313 >>> predicted_class_ids = torch.arange(0, logits.shape[-1])[torch.sigmoid(logits).squeeze(dim=0) > 0.5]314 315 >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)`316 >>> num_labels = len(model.config.id2label)317 >>> model = {model_class}.from_pretrained(318 ... "{checkpoint}", num_labels=num_labels, problem_type="multi_label_classification"319 ... )320 321 >>> labels = torch.sum(322 ... torch.nn.functional.one_hot(predicted_class_ids[None, :].clone(), num_classes=num_labels), dim=1323 ... ).to(torch.float)324 >>> loss = model(**inputs, labels=labels).loss325 ```326"""327 328PT_MASKED_LM_SAMPLE = r"""329 Example:330 331 ```python332 >>> from transformers import AutoTokenizer, {model_class}333 >>> import torch334 335 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")336 >>> model = {model_class}.from_pretrained("{checkpoint}")337 338 >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="pt")339 340 >>> with torch.no_grad():341 ... logits = model(**inputs).logits342 343 >>> # retrieve index of {mask}344 >>> mask_token_index = (inputs.input_ids == tokenizer.mask_token_id)[0].nonzero(as_tuple=True)[0]345 346 >>> predicted_token_id = logits[0, mask_token_index].argmax(axis=-1)347 >>> tokenizer.decode(predicted_token_id)348 {expected_output}349 350 >>> labels = tokenizer("The capital of France is Paris.", return_tensors="pt")["input_ids"]351 >>> # mask labels of non-{mask} tokens352 >>> labels = torch.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100)353 354 >>> outputs = model(**inputs, labels=labels)355 >>> round(outputs.loss.item(), 2)356 {expected_loss}357 ```358"""359 360PT_BASE_MODEL_SAMPLE = r"""361 Example:362 363 ```python364 >>> from transformers import AutoTokenizer, {model_class}365 >>> import torch366 367 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")368 >>> model = {model_class}.from_pretrained("{checkpoint}")369 370 >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")371 >>> outputs = model(**inputs)372 373 >>> last_hidden_states = outputs.last_hidden_state374 ```375"""376 377PT_MULTIPLE_CHOICE_SAMPLE = r"""378 Example:379 380 ```python381 >>> from transformers import AutoTokenizer, {model_class}382 >>> import torch383 384 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")385 >>> model = {model_class}.from_pretrained("{checkpoint}")386 387 >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."388 >>> choice0 = "It is eaten with a fork and a knife."389 >>> choice1 = "It is eaten while held in the hand."390 >>> labels = torch.tensor(0).unsqueeze(0) # choice0 is correct (according to Wikipedia ;)), batch size 1391 392 >>> encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors="pt", padding=True)393 >>> outputs = model(**{{k: v.unsqueeze(0) for k, v in encoding.items()}}, labels=labels) # batch size is 1394 395 >>> # the linear classifier still needs to be trained396 >>> loss = outputs.loss397 >>> logits = outputs.logits398 ```399"""400 401PT_CAUSAL_LM_SAMPLE = r"""402 Example:403 404 ```python405 >>> import torch406 >>> from transformers import AutoTokenizer, {model_class}407 408 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")409 >>> model = {model_class}.from_pretrained("{checkpoint}")410 411 >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")412 >>> outputs = model(**inputs, labels=inputs["input_ids"])413 >>> loss = outputs.loss414 >>> logits = outputs.logits415 ```416"""417 418PT_SPEECH_BASE_MODEL_SAMPLE = r"""419 Example:420 421 ```python422 >>> from transformers import AutoProcessor, {model_class}423 >>> import torch424 >>> from datasets import load_dataset425 426 >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")427 >>> dataset = dataset.sort("id")428 >>> sampling_rate = dataset.features["audio"].sampling_rate429 430 >>> processor = AutoProcessor.from_pretrained("{checkpoint}")431 >>> model = {model_class}.from_pretrained("{checkpoint}")432 433 >>> # audio file is decoded on the fly434 >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt")435 >>> with torch.no_grad():436 ... outputs = model(**inputs)437 438 >>> last_hidden_states = outputs.last_hidden_state439 >>> list(last_hidden_states.shape)440 {expected_output}441 ```442"""443 444PT_SPEECH_CTC_SAMPLE = r"""445 Example:446 447 ```python448 >>> from transformers import AutoProcessor, {model_class}449 >>> from datasets import load_dataset450 >>> import torch451 452 >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")453 >>> dataset = dataset.sort("id")454 >>> sampling_rate = dataset.features["audio"].sampling_rate455 456 >>> processor = AutoProcessor.from_pretrained("{checkpoint}")457 >>> model = {model_class}.from_pretrained("{checkpoint}")458 459 >>> # audio file is decoded on the fly460 >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt")461 >>> with torch.no_grad():462 ... logits = model(**inputs).logits463 >>> predicted_ids = torch.argmax(logits, dim=-1)464 465 >>> # transcribe speech466 >>> transcription = processor.batch_decode(predicted_ids)467 >>> transcription[0]468 {expected_output}469 470 >>> inputs["labels"] = processor(text=dataset[0]["text"], return_tensors="pt").input_ids471 472 >>> # compute loss473 >>> loss = model(**inputs).loss474 >>> round(loss.item(), 2)475 {expected_loss}476 ```477"""478 479PT_SPEECH_SEQ_CLASS_SAMPLE = r"""480 Example:481 482 ```python483 >>> from transformers import AutoFeatureExtractor, {model_class}484 >>> from datasets import load_dataset485 >>> import torch486 487 >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")488 >>> dataset = dataset.sort("id")489 >>> sampling_rate = dataset.features["audio"].sampling_rate490 491 >>> feature_extractor = AutoFeatureExtractor.from_pretrained("{checkpoint}")492 >>> model = {model_class}.from_pretrained("{checkpoint}")493 494 >>> # audio file is decoded on the fly495 >>> inputs = feature_extractor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt")496 497 >>> with torch.no_grad():498 ... logits = model(**inputs).logits499 500 >>> predicted_class_ids = torch.argmax(logits, dim=-1).item()501 >>> predicted_label = model.config.id2label[predicted_class_ids]502 >>> predicted_label503 {expected_output}504 505 >>> # compute loss - target_label is e.g. "down"506 >>> target_label = model.config.id2label[0]507 >>> inputs["labels"] = torch.tensor([model.config.label2id[target_label]])508 >>> loss = model(**inputs).loss509 >>> round(loss.item(), 2)510 {expected_loss}511 ```512"""513 514 515PT_SPEECH_FRAME_CLASS_SAMPLE = r"""516 Example:517 518 ```python519 >>> from transformers import AutoFeatureExtractor, {model_class}520 >>> from datasets import load_dataset521 >>> import torch522 523 >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")524 >>> dataset = dataset.sort("id")525 >>> sampling_rate = dataset.features["audio"].sampling_rate526 527 >>> feature_extractor = AutoFeatureExtractor.from_pretrained("{checkpoint}")528 >>> model = {model_class}.from_pretrained("{checkpoint}")529 530 >>> # audio file is decoded on the fly531 >>> inputs = feature_extractor(dataset[0]["audio"]["array"], return_tensors="pt", sampling_rate=sampling_rate)532 >>> with torch.no_grad():533 ... logits = model(**inputs).logits534 535 >>> probabilities = torch.sigmoid(logits[0])536 >>> # labels is a one-hot array of shape (num_frames, num_speakers)537 >>> labels = (probabilities > 0.5).long()538 >>> labels[0].tolist()539 {expected_output}540 ```541"""542 543 544PT_SPEECH_XVECTOR_SAMPLE = r"""545 Example:546 547 ```python548 >>> from transformers import AutoFeatureExtractor, {model_class}549 >>> from datasets import load_dataset550 >>> import torch551 552 >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")553 >>> dataset = dataset.sort("id")554 >>> sampling_rate = dataset.features["audio"].sampling_rate555 556 >>> feature_extractor = AutoFeatureExtractor.from_pretrained("{checkpoint}")557 >>> model = {model_class}.from_pretrained("{checkpoint}")558 559 >>> # audio file is decoded on the fly560 >>> inputs = feature_extractor(561 ... [d["array"] for d in dataset[:2]["audio"]], sampling_rate=sampling_rate, return_tensors="pt", padding=True562 ... )563 >>> with torch.no_grad():564 ... embeddings = model(**inputs).embeddings565 566 >>> embeddings = torch.nn.functional.normalize(embeddings, dim=-1).cpu()567 568 >>> # the resulting embeddings can be used for cosine similarity-based retrieval569 >>> cosine_sim = torch.nn.CosineSimilarity(dim=-1)570 >>> similarity = cosine_sim(embeddings[0], embeddings[1])571 >>> threshold = 0.7 # the optimal threshold is dataset-dependent572 >>> if similarity < threshold:573 ... print("Speakers are not the same!")574 >>> round(similarity.item(), 2)575 {expected_output}576 ```577"""578 579PT_VISION_BASE_MODEL_SAMPLE = r"""580 Example:581 582 ```python583 >>> from transformers import AutoImageProcessor, {model_class}584 >>> import torch585 >>> from datasets import load_dataset586 587 >>> dataset = load_dataset("huggingface/cats-image")588 >>> image = dataset["test"]["image"][0]589 590 >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}")591 >>> model = {model_class}.from_pretrained("{checkpoint}")592 593 >>> inputs = image_processor(image, return_tensors="pt")594 595 >>> with torch.no_grad():596 ... outputs = model(**inputs)597 598 >>> last_hidden_states = outputs.last_hidden_state599 >>> list(last_hidden_states.shape)600 {expected_output}601 ```602"""603 604PT_VISION_SEQ_CLASS_SAMPLE = r"""605 Example:606 607 ```python608 >>> from transformers import AutoImageProcessor, {model_class}609 >>> import torch610 >>> from datasets import load_dataset611 612 >>> dataset = load_dataset("huggingface/cats-image")613 >>> image = dataset["test"]["image"][0]614 615 >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}")616 >>> model = {model_class}.from_pretrained("{checkpoint}")617 618 >>> inputs = image_processor(image, return_tensors="pt")619 620 >>> with torch.no_grad():621 ... logits = model(**inputs).logits622 623 >>> # model predicts one of the 1000 ImageNet classes624 >>> predicted_label = logits.argmax(-1).item()625 >>> print(model.config.id2label[predicted_label])626 {expected_output}627 ```628"""629 630 631PT_SAMPLE_DOCSTRINGS = {632 "SequenceClassification": PT_SEQUENCE_CLASSIFICATION_SAMPLE,633 "QuestionAnswering": PT_QUESTION_ANSWERING_SAMPLE,634 "TokenClassification": PT_TOKEN_CLASSIFICATION_SAMPLE,635 "MultipleChoice": PT_MULTIPLE_CHOICE_SAMPLE,636 "MaskedLM": PT_MASKED_LM_SAMPLE,637 "LMHead": PT_CAUSAL_LM_SAMPLE,638 "BaseModel": PT_BASE_MODEL_SAMPLE,639 "SpeechBaseModel": PT_SPEECH_BASE_MODEL_SAMPLE,640 "CTC": PT_SPEECH_CTC_SAMPLE,641 "AudioClassification": PT_SPEECH_SEQ_CLASS_SAMPLE,642 "AudioFrameClassification": PT_SPEECH_FRAME_CLASS_SAMPLE,643 "AudioXVector": PT_SPEECH_XVECTOR_SAMPLE,644 "VisionBaseModel": PT_VISION_BASE_MODEL_SAMPLE,645 "ImageClassification": PT_VISION_SEQ_CLASS_SAMPLE,646}647 648 649TEXT_TO_AUDIO_SPECTROGRAM_SAMPLE = r"""650 Example:651 652 ```python653 >>> from transformers import AutoProcessor, {model_class}, SpeechT5HifiGan654 655 >>> model = {model_class}.from_pretrained("{checkpoint}")656 657 >>> processor = AutoProcessor.from_pretrained("{checkpoint}")658 >>> vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")659 >>> inputs = processor(text="Hello, my dog is cute", return_tensors="pt")660 661 >>> # generate speech662 >>> speech = model.generate(inputs["input_ids"], speaker_embeddings=speaker_embeddings, vocoder=vocoder)663 ```664"""665 666 667TEXT_TO_AUDIO_WAVEFORM_SAMPLE = r"""668 Example:669 670 ```python671 >>> from transformers import AutoProcessor, {model_class}672 673 >>> model = {model_class}.from_pretrained("{checkpoint}")674 675 >>> processor = AutoProcessor.from_pretrained("{checkpoint}")676 >>> inputs = processor(text="Hello, my dog is cute", return_tensors="pt")677 678 >>> # generate speech679 >>> speech = model(inputs["input_ids"])680 ```681"""682 683 684AUDIO_FRAME_CLASSIFICATION_SAMPLE = PT_SPEECH_FRAME_CLASS_SAMPLE685 686 687AUDIO_XVECTOR_SAMPLE = PT_SPEECH_XVECTOR_SAMPLE688 689 690IMAGE_TO_TEXT_SAMPLE = r"""691 Example:692 693 ```python694 >>> from PIL import Image695 >>> import requests696 >>> from transformers import AutoProcessor, {model_class}697 698 >>> processor = AutoProcessor.from_pretrained("{checkpoint}")699 >>> model = {model_class}.from_pretrained("{checkpoint}")700 701 >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"702 >>> image = Image.open(requests.get(url, stream=True).raw)703 704 >>> inputs = processor(images=image, return_tensors="pt")705 706 >>> outputs = model(**inputs)707 ```708"""709 710 711DEPTH_ESTIMATION_SAMPLE = r"""712 Example:713 714 ```python715 >>> from transformers import AutoImageProcessor, {model_class}716 >>> import torch717 >>> from PIL import Image718 >>> import requests719 720 >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"721 >>> image = Image.open(requests.get(url, stream=True).raw)722 723 >>> processor = AutoImageProcessor.from_pretrained("{checkpoint}")724 >>> model = {model_class}.from_pretrained("{checkpoint}")725 726 >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu")727 >>> model.to(device)728 729 >>> # prepare image for the model730 >>> inputs = processor(images=image, return_tensors="pt").to(device)731 732 >>> with torch.no_grad():733 ... outputs = model(**inputs)734 735 >>> # interpolate to original size736 >>> post_processed_output = processor.post_process_depth_estimation(737 ... outputs, [(image.height, image.width)],738 ... )739 >>> predicted_depth = post_processed_output[0]["predicted_depth"]740 ```741"""742 743 744VIDEO_CLASSIFICATION_SAMPLE = r"""745 Example:746 747 ```python748 ```749"""750 751 752ZERO_SHOT_OBJECT_DETECTION_SAMPLE = r"""753 Example:754 755 ```python756 ```757"""758 759 760IMAGE_TO_IMAGE_SAMPLE = r"""761 Example:762 763 ```python764 ```765"""766 767 768IMAGE_FEATURE_EXTRACTION_SAMPLE = r"""769 Example:770 771 ```python772 ```773"""774 775 776DOCUMENT_QUESTION_ANSWERING_SAMPLE = r"""777 Example:778 779 ```python780 ```781"""782 783 784NEXT_SENTENCE_PREDICTION_SAMPLE = r"""785 Example:786 787 ```python788 ```789"""790 791 792MULTIPLE_CHOICE_SAMPLE = PT_MULTIPLE_CHOICE_SAMPLE793 794 795PRETRAINING_SAMPLE = r"""796 Example:797 798 ```python799 ```800"""801MASK_GENERATION_SAMPLE = r"""802 Example:803 804 ```python805 ```806"""807 808 809VISUAL_QUESTION_ANSWERING_SAMPLE = r"""810 Example:811 812 ```python813 ```814"""815 816 817TEXT_GENERATION_SAMPLE = r"""818 Example:819 820 ```python821 ```822"""823 824 825IMAGE_CLASSIFICATION_SAMPLE = PT_VISION_SEQ_CLASS_SAMPLE826 827 828IMAGE_SEGMENTATION_SAMPLE = r"""829 Example:830 831 ```python832 ```833"""834 835 836FILL_MASK_SAMPLE = r"""837 Example:838 839 ```python840 ```841"""842 843 844OBJECT_DETECTION_SAMPLE = r"""845 Example:846 847 ```python848 ```849"""850 851 852QUESTION_ANSWERING_SAMPLE = PT_QUESTION_ANSWERING_SAMPLE853 854 855TEXT2TEXT_GENERATION_SAMPLE = r"""856 Example:857 858 ```python859 ```860"""861 862 863TEXT_CLASSIFICATION_SAMPLE = PT_SEQUENCE_CLASSIFICATION_SAMPLE864 865 866TABLE_QUESTION_ANSWERING_SAMPLE = r"""867 Example:868 869 ```python870 ```871"""872 873 874TOKEN_CLASSIFICATION_SAMPLE = PT_TOKEN_CLASSIFICATION_SAMPLE875 876 877AUDIO_CLASSIFICATION_SAMPLE = PT_SPEECH_SEQ_CLASS_SAMPLE878 879 880AUTOMATIC_SPEECH_RECOGNITION_SAMPLE = PT_SPEECH_CTC_SAMPLE881 882 883ZERO_SHOT_IMAGE_CLASSIFICATION_SAMPLE = r"""884 Example:885 886 ```python887 ```888"""889 890 891IMAGE_TEXT_TO_TEXT_GENERATION_SAMPLE = r"""892 Example:893 894 ```python895 >>> from PIL import Image896 >>> import requests897 >>> from transformers import AutoProcessor, {model_class}898 899 >>> model = {model_class}.from_pretrained("{checkpoint}")900 >>> processor = AutoProcessor.from_pretrained("{checkpoint}")901 902 >>> messages = [903 ... {{904 ... "role": "user", "content": [905 ... {{"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"}},906 ... {{"type": "text", "text": "Where is the cat standing?"}},907 ... ]908 ... }},909 ... ]910 911 >>> inputs = processor.apply_chat_template(912 ... messages,913 ... tokenize=True,914 ... return_dict=True,915 ... return_tensors="pt",916 ... add_generation_prompt=True917 ... )918 >>> # Generate919 >>> generate_ids = model.generate(**inputs)920 >>> processor.batch_decode(generate_ids, skip_special_tokens=True)[0]921 ```922"""923 924 925PIPELINE_TASKS_TO_SAMPLE_DOCSTRINGS = OrderedDict(926 [927 ("text-to-audio-spectrogram", TEXT_TO_AUDIO_SPECTROGRAM_SAMPLE),928 ("text-to-audio-waveform", TEXT_TO_AUDIO_WAVEFORM_SAMPLE),929 ("automatic-speech-recognition", AUTOMATIC_SPEECH_RECOGNITION_SAMPLE),930 ("audio-frame-classification", AUDIO_FRAME_CLASSIFICATION_SAMPLE),931 ("audio-classification", AUDIO_CLASSIFICATION_SAMPLE),932 ("audio-xvector", AUDIO_XVECTOR_SAMPLE),933 ("image-text-to-text", IMAGE_TEXT_TO_TEXT_GENERATION_SAMPLE),934 ("image-to-text", IMAGE_TO_TEXT_SAMPLE),935 ("visual-question-answering", VISUAL_QUESTION_ANSWERING_SAMPLE),936 ("depth-estimation", DEPTH_ESTIMATION_SAMPLE),937 ("video-classification", VIDEO_CLASSIFICATION_SAMPLE),938 ("zero-shot-image-classification", ZERO_SHOT_IMAGE_CLASSIFICATION_SAMPLE),939 ("image-classification", IMAGE_CLASSIFICATION_SAMPLE),940 ("zero-shot-object-detection", ZERO_SHOT_OBJECT_DETECTION_SAMPLE),941 ("object-detection", OBJECT_DETECTION_SAMPLE),942 ("image-segmentation", IMAGE_SEGMENTATION_SAMPLE),943 ("image-to-image", IMAGE_TO_IMAGE_SAMPLE),944 ("image-feature-extraction", IMAGE_FEATURE_EXTRACTION_SAMPLE),945 ("text-generation", TEXT_GENERATION_SAMPLE),946 ("table-question-answering", TABLE_QUESTION_ANSWERING_SAMPLE),947 ("document-question-answering", DOCUMENT_QUESTION_ANSWERING_SAMPLE),948 ("question-answering", QUESTION_ANSWERING_SAMPLE),949 ("text2text-generation", TEXT2TEXT_GENERATION_SAMPLE),950 ("next-sentence-prediction", NEXT_SENTENCE_PREDICTION_SAMPLE),951 ("multiple-choice", MULTIPLE_CHOICE_SAMPLE),952 ("text-classification", TEXT_CLASSIFICATION_SAMPLE),953 ("token-classification", TOKEN_CLASSIFICATION_SAMPLE),954 ("fill-mask", FILL_MASK_SAMPLE),955 ("mask-generation", MASK_GENERATION_SAMPLE),956 ("pretraining", PRETRAINING_SAMPLE),957 ]958)959 960# Ordered dict to look for more specialized model mappings first961# before falling back to the more generic ones.962MODELS_TO_PIPELINE = OrderedDict(963 [964 # Audio965 ("MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES", "text-to-audio-spectrogram"),966 ("MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES", "text-to-audio-waveform"),967 ("MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "automatic-speech-recognition"),968 ("MODEL_FOR_CTC_MAPPING_NAMES", "automatic-speech-recognition"),969 ("MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES", "audio-frame-classification"),970 ("MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "audio-classification"),971 ("MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "audio-xvector"),972 # Vision973 ("MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES", "image-text-to-text"),974 ("MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES", "image-to-text"),975 ("MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES", "visual-question-answering"),976 ("MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "depth-estimation"),977 ("MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "video-classification"),978 ("MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES", "zero-shot-image-classification"),979 ("MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "image-classification"),980 ("MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES", "zero-shot-object-detection"),981 ("MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "object-detection"),982 ("MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "image-segmentation"),983 ("MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES", "image-to-image"),984 ("MODEL_FOR_IMAGE_MAPPING_NAMES", "image-feature-extraction"),985 # Text/tokens986 ("MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "text-generation"),987 ("MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES", "table-question-answering"),988 ("MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES", "document-question-answering"),989 ("MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "question-answering"),990 ("MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "text2text-generation"),991 ("MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES", "next-sentence-prediction"),992 ("MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "multiple-choice"),993 ("MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "text-classification"),994 ("MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "token-classification"),995 ("MODEL_FOR_MASKED_LM_MAPPING_NAMES", "fill-mask"),996 ("MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "mask-generation"),997 ("MODEL_FOR_PRETRAINING_MAPPING_NAMES", "pretraining"),998 ]999)1000 1001 1002TF_TOKEN_CLASSIFICATION_SAMPLE = r"""1003 Example:1004 1005 ```python1006 >>> from transformers import AutoTokenizer, {model_class}1007 >>> import tensorflow as tf1008 1009 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")1010 >>> model = {model_class}.from_pretrained("{checkpoint}")1011 1012 >>> inputs = tokenizer(1013 ... "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="tf"1014 ... )1015 1016 >>> logits = model(**inputs).logits1017 >>> predicted_token_class_ids = tf.math.argmax(logits, axis=-1)1018 1019 >>> # Note that tokens are classified rather then input words which means that1020 >>> # there might be more predicted token classes than words.1021 >>> # Multiple token classes might account for the same word1022 >>> predicted_tokens_classes = [model.config.id2label[t] for t in predicted_token_class_ids[0].numpy().tolist()]1023 >>> predicted_tokens_classes1024 {expected_output}1025 ```1026 1027 ```python1028 >>> labels = predicted_token_class_ids1029 >>> loss = tf.math.reduce_mean(model(**inputs, labels=labels).loss)1030 >>> round(float(loss), 2)1031 {expected_loss}1032 ```1033"""1034 1035TF_QUESTION_ANSWERING_SAMPLE = r"""1036 Example:1037 1038 ```python1039 >>> from transformers import AutoTokenizer, {model_class}1040 >>> import tensorflow as tf1041 1042 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")1043 >>> model = {model_class}.from_pretrained("{checkpoint}")1044 1045 >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"1046 1047 >>> inputs = tokenizer(question, text, return_tensors="tf")1048 >>> outputs = model(**inputs)1049 1050 >>> answer_start_index = int(tf.math.argmax(outputs.start_logits, axis=-1)[0])1051 >>> answer_end_index = int(tf.math.argmax(outputs.end_logits, axis=-1)[0])1052 1053 >>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1]1054 >>> tokenizer.decode(predict_answer_tokens)1055 {expected_output}1056 ```1057 1058 ```python1059 >>> # target is "nice puppet"1060 >>> target_start_index = tf.constant([{qa_target_start_index}])1061 >>> target_end_index = tf.constant([{qa_target_end_index}])1062 1063 >>> outputs = model(**inputs, start_positions=target_start_index, end_positions=target_end_index)1064 >>> loss = tf.math.reduce_mean(outputs.loss)1065 >>> round(float(loss), 2)1066 {expected_loss}1067 ```1068"""1069 1070TF_SEQUENCE_CLASSIFICATION_SAMPLE = r"""1071 Example:1072 1073 ```python1074 >>> from transformers import AutoTokenizer, {model_class}1075 >>> import tensorflow as tf1076 1077 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")1078 >>> model = {model_class}.from_pretrained("{checkpoint}")1079 1080 >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")1081 1082 >>> logits = model(**inputs).logits1083 1084 >>> predicted_class_id = int(tf.math.argmax(logits, axis=-1)[0])1085 >>> model.config.id2label[predicted_class_id]1086 {expected_output}1087 ```1088 1089 ```python1090 >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)`1091 >>> num_labels = len(model.config.id2label)1092 >>> model = {model_class}.from_pretrained("{checkpoint}", num_labels=num_labels)1093 1094 >>> labels = tf.constant(1)1095 >>> loss = model(**inputs, labels=labels).loss1096 >>> round(float(loss), 2)1097 {expected_loss}1098 ```1099"""1100 1101TF_MASKED_LM_SAMPLE = r"""1102 Example:1103 1104 ```python1105 >>> from transformers import AutoTokenizer, {model_class}1106 >>> import tensorflow as tf1107 1108 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")1109 >>> model = {model_class}.from_pretrained("{checkpoint}")1110 1111 >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="tf")1112 >>> logits = model(**inputs).logits1113 1114 >>> # retrieve index of {mask}1115 >>> mask_token_index = tf.where((inputs.input_ids == tokenizer.mask_token_id)[0])1116 >>> selected_logits = tf.gather_nd(logits[0], indices=mask_token_index)1117 1118 >>> predicted_token_id = tf.math.argmax(selected_logits, axis=-1)1119 >>> tokenizer.decode(predicted_token_id)1120 {expected_output}1121 ```1122 1123 ```python1124 >>> labels = tokenizer("The capital of France is Paris.", return_tensors="tf")["input_ids"]1125 >>> # mask labels of non-{mask} tokens1126 >>> labels = tf.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100)1127 1128 >>> outputs = model(**inputs, labels=labels)1129 >>> round(float(outputs.loss), 2)1130 {expected_loss}1131 ```1132"""1133 1134TF_BASE_MODEL_SAMPLE = r"""1135 Example:1136 1137 ```python1138 >>> from transformers import AutoTokenizer, {model_class}1139 >>> import tensorflow as tf1140 1141 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")1142 >>> model = {model_class}.from_pretrained("{checkpoint}")1143 1144 >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")1145 >>> outputs = model(inputs)1146 1147 >>> last_hidden_states = outputs.last_hidden_state1148 ```1149"""1150 1151TF_MULTIPLE_CHOICE_SAMPLE = r"""1152 Example:1153 1154 ```python1155 >>> from transformers import AutoTokenizer, {model_class}1156 >>> import tensorflow as tf1157 1158 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")1159 >>> model = {model_class}.from_pretrained("{checkpoint}")1160 1161 >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."1162 >>> choice0 = "It is eaten with a fork and a knife."1163 >>> choice1 = "It is eaten while held in the hand."1164 1165 >>> encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors="tf", padding=True)1166 >>> inputs = {{k: tf.expand_dims(v, 0) for k, v in encoding.items()}}1167 >>> outputs = model(inputs) # batch size is 11168 1169 >>> # the linear classifier still needs to be trained1170 >>> logits = outputs.logits1171 ```1172"""1173 1174TF_CAUSAL_LM_SAMPLE = r"""1175 Example:1176 1177 ```python1178 >>> from transformers import AutoTokenizer, {model_class}1179 >>> import tensorflow as tf1180 1181 >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}")1182 >>> model = {model_class}.from_pretrained("{checkpoint}")1183 1184 >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")1185 >>> outputs = model(inputs)1186 >>> logits = outputs.logits1187 ```1188"""1189 1190TF_SPEECH_BASE_MODEL_SAMPLE = r"""1191 Example:1192 1193 ```python1194 >>> from transformers import AutoProcessor, {model_class}1195 >>> from datasets import load_dataset1196 1197 >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")1198 >>> dataset = dataset.sort("id")1199 >>> sampling_rate = dataset.features["audio"].sampling_rate1200