chendl/compositional_test
1
1# Copyright 2020 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import unittest16 17from transformers import (18 MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,19 TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,20 Pipeline,21 ZeroShotClassificationPipeline,22 pipeline,23)24from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow25 26from .test_pipelines_common import ANY27 28 29@is_pipeline_test30class ZeroShotClassificationPipelineTests(unittest.TestCase):31 model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING32 tf_model_mapping = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING33 34 def get_test_pipeline(self, model, tokenizer, processor):35 classifier = ZeroShotClassificationPipeline(36 model=model, tokenizer=tokenizer, candidate_labels=["polics", "health"]37 )38 return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]39 40 def run_pipeline_test(self, classifier, _):41 outputs = classifier("Who are you voting for in 2020?", candidate_labels="politics")42 self.assertEqual(outputs, {"sequence": ANY(str), "labels": [ANY(str)], "scores": [ANY(float)]})43 44 # No kwarg45 outputs = classifier("Who are you voting for in 2020?", ["politics"])46 self.assertEqual(outputs, {"sequence": ANY(str), "labels": [ANY(str)], "scores": [ANY(float)]})47 48 outputs = classifier("Who are you voting for in 2020?", candidate_labels=["politics"])49 self.assertEqual(outputs, {"sequence": ANY(str), "labels": [ANY(str)], "scores": [ANY(float)]})50 51 outputs = classifier("Who are you voting for in 2020?", candidate_labels="politics, public health")52 self.assertEqual(53 outputs, {"sequence": ANY(str), "labels": [ANY(str), ANY(str)], "scores": [ANY(float), ANY(float)]}54 )55 self.assertAlmostEqual(sum(nested_simplify(outputs["scores"])), 1.0)56 57 outputs = classifier("Who are you voting for in 2020?", candidate_labels=["politics", "public health"])58 self.assertEqual(59 outputs, {"sequence": ANY(str), "labels": [ANY(str), ANY(str)], "scores": [ANY(float), ANY(float)]}60 )61 self.assertAlmostEqual(sum(nested_simplify(outputs["scores"])), 1.0)62 63 outputs = classifier(64 "Who are you voting for in 2020?", candidate_labels="politics", hypothesis_template="This text is about {}"65 )66 self.assertEqual(outputs, {"sequence": ANY(str), "labels": [ANY(str)], "scores": [ANY(float)]})67 68 # https://github.com/huggingface/transformers/issues/1384669 outputs = classifier(["I am happy"], ["positive", "negative"])70 self.assertEqual(71 outputs,72 [73 {"sequence": ANY(str), "labels": [ANY(str), ANY(str)], "scores": [ANY(float), ANY(float)]}74 for i in range(1)75 ],76 )77 outputs = classifier(["I am happy", "I am sad"], ["positive", "negative"])78 self.assertEqual(79 outputs,80 [81 {"sequence": ANY(str), "labels": [ANY(str), ANY(str)], "scores": [ANY(float), ANY(float)]}82 for i in range(2)83 ],84 )85 86 with self.assertRaises(ValueError):87 classifier("", candidate_labels="politics")88 89 with self.assertRaises(TypeError):90 classifier(None, candidate_labels="politics")91 92 with self.assertRaises(ValueError):93 classifier("Who are you voting for in 2020?", candidate_labels="")94 95 with self.assertRaises(TypeError):96 classifier("Who are you voting for in 2020?", candidate_labels=None)97 98 with self.assertRaises(ValueError):99 classifier(100 "Who are you voting for in 2020?",101 candidate_labels="politics",102 hypothesis_template="Not formatting template",103 )104 105 with self.assertRaises(AttributeError):106 classifier(107 "Who are you voting for in 2020?",108 candidate_labels="politics",109 hypothesis_template=None,110 )111 112 self.run_entailment_id(classifier)113 114 def run_entailment_id(self, zero_shot_classifier: Pipeline):115 config = zero_shot_classifier.model.config116 original_label2id = config.label2id117 original_entailment = zero_shot_classifier.entailment_id118 119 config.label2id = {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}120 self.assertEqual(zero_shot_classifier.entailment_id, -1)121 122 config.label2id = {"entailment": 0, "neutral": 1, "contradiction": 2}123 self.assertEqual(zero_shot_classifier.entailment_id, 0)124 125 config.label2id = {"ENTAIL": 0, "NON-ENTAIL": 1}126 self.assertEqual(zero_shot_classifier.entailment_id, 0)127 128 config.label2id = {"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}129 self.assertEqual(zero_shot_classifier.entailment_id, 2)130 131 zero_shot_classifier.model.config.label2id = original_label2id132 self.assertEqual(original_entailment, zero_shot_classifier.entailment_id)133 134 @require_torch135 def test_truncation(self):136 zero_shot_classifier = pipeline(137 "zero-shot-classification",138 model="sshleifer/tiny-distilbert-base-cased-distilled-squad",139 framework="pt",140 )141 # There was a regression in 4.10 for this142 # Adding a test so we don't make the mistake again.143 # https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499144 zero_shot_classifier(145 "Who are you voting for in 2020?" * 100, candidate_labels=["politics", "public health", "science"]146 )147 148 @require_torch149 def test_small_model_pt(self):150 zero_shot_classifier = pipeline(151 "zero-shot-classification",152 model="sshleifer/tiny-distilbert-base-cased-distilled-squad",153 framework="pt",154 )155 outputs = zero_shot_classifier(156 "Who are you voting for in 2020?", candidate_labels=["politics", "public health", "science"]157 )158 159 self.assertEqual(160 nested_simplify(outputs),161 {162 "sequence": "Who are you voting for in 2020?",163 "labels": ["science", "public health", "politics"],164 "scores": [0.333, 0.333, 0.333],165 },166 )167 168 @require_tf169 def test_small_model_tf(self):170 zero_shot_classifier = pipeline(171 "zero-shot-classification",172 model="sshleifer/tiny-distilbert-base-cased-distilled-squad",173 framework="tf",174 )175 outputs = zero_shot_classifier(176 "Who are you voting for in 2020?", candidate_labels=["politics", "public health", "science"]177 )178 179 self.assertEqual(180 nested_simplify(outputs),181 {182 "sequence": "Who are you voting for in 2020?",183 "labels": ["science", "public health", "politics"],184 "scores": [0.333, 0.333, 0.333],185 },186 )187 188 @slow189 @require_torch190 def test_large_model_pt(self):191 zero_shot_classifier = pipeline("zero-shot-classification", model="roberta-large-mnli", framework="pt")192 outputs = zero_shot_classifier(193 "Who are you voting for in 2020?", candidate_labels=["politics", "public health", "science"]194 )195 196 self.assertEqual(197 nested_simplify(outputs),198 {199 "sequence": "Who are you voting for in 2020?",200 "labels": ["politics", "public health", "science"],201 "scores": [0.976, 0.015, 0.009],202 },203 )204 outputs = zero_shot_classifier(205 "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"206 " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"207 " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"208 " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"209 " machine translation tasks show these models to be superior in quality while being more parallelizable"210 " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"211 " English-to-German translation task, improving over the existing best results, including ensembles by"212 " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"213 " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"214 " fraction of the training costs of the best models from the literature. We show that the Transformer"215 " generalizes well to other tasks by applying it successfully to English constituency parsing both with"216 " large and limited training data.",217 candidate_labels=["machine learning", "statistics", "translation", "vision"],218 multi_label=True,219 )220 self.assertEqual(221 nested_simplify(outputs),222 {223 "sequence": (224 "The dominant sequence transduction models are based on complex recurrent or convolutional neural"225 " networks in an encoder-decoder configuration. The best performing models also connect the"226 " encoder and decoder through an attention mechanism. We propose a new simple network"227 " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"228 " and convolutions entirely. Experiments on two machine translation tasks show these models to be"229 " superior in quality while being more parallelizable and requiring significantly less time to"230 " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"231 " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"232 " English-to-French translation task, our model establishes a new single-model state-of-the-art"233 " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"234 " costs of the best models from the literature. We show that the Transformer generalizes well to"235 " other tasks by applying it successfully to English constituency parsing both with large and"236 " limited training data."237 ),238 "labels": ["translation", "machine learning", "vision", "statistics"],239 "scores": [0.817, 0.713, 0.018, 0.018],240 },241 )242 243 @slow244 @require_tf245 def test_large_model_tf(self):246 zero_shot_classifier = pipeline("zero-shot-classification", model="roberta-large-mnli", framework="tf")247 outputs = zero_shot_classifier(248 "Who are you voting for in 2020?", candidate_labels=["politics", "public health", "science"]249 )250 251 self.assertEqual(252 nested_simplify(outputs),253 {254 "sequence": "Who are you voting for in 2020?",255 "labels": ["politics", "public health", "science"],256 "scores": [0.976, 0.015, 0.009],257 },258 )259 outputs = zero_shot_classifier(260 "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"261 " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"262 " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"263 " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"264 " machine translation tasks show these models to be superior in quality while being more parallelizable"265 " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"266 " English-to-German translation task, improving over the existing best results, including ensembles by"267 " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"268 " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"269 " fraction of the training costs of the best models from the literature. We show that the Transformer"270 " generalizes well to other tasks by applying it successfully to English constituency parsing both with"271 " large and limited training data.",272 candidate_labels=["machine learning", "statistics", "translation", "vision"],273 multi_label=True,274 )275 self.assertEqual(276 nested_simplify(outputs),277 {278 "sequence": (279 "The dominant sequence transduction models are based on complex recurrent or convolutional neural"280 " networks in an encoder-decoder configuration. The best performing models also connect the"281 " encoder and decoder through an attention mechanism. We propose a new simple network"282 " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"283 " and convolutions entirely. Experiments on two machine translation tasks show these models to be"284 " superior in quality while being more parallelizable and requiring significantly less time to"285 " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"286 " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"287 " English-to-French translation task, our model establishes a new single-model state-of-the-art"288 " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"289 " costs of the best models from the literature. We show that the Transformer generalizes well to"290 " other tasks by applying it successfully to English constituency parsing both with large and"291 " limited training data."292 ),293 "labels": ["translation", "machine learning", "vision", "statistics"],294 "scores": [0.817, 0.713, 0.018, 0.018],295 },296 )297