chendl/compositional_test
1
1# Copyright 2021 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import unittest16 17from transformers import (18 MODEL_FOR_OBJECT_DETECTION_MAPPING,19 AutoFeatureExtractor,20 AutoModelForObjectDetection,21 ObjectDetectionPipeline,22 is_vision_available,23 pipeline,24)25from transformers.testing_utils import (26 is_pipeline_test,27 nested_simplify,28 require_pytesseract,29 require_tf,30 require_timm,31 require_torch,32 require_vision,33 slow,34)35 36from .test_pipelines_common import ANY37 38 39if is_vision_available():40 from PIL import Image41else:42 43 class Image:44 @staticmethod45 def open(*args, **kwargs):46 pass47 48 49@is_pipeline_test50@require_vision51@require_timm52@require_torch53class ObjectDetectionPipelineTests(unittest.TestCase):54 model_mapping = MODEL_FOR_OBJECT_DETECTION_MAPPING55 56 def get_test_pipeline(self, model, tokenizer, processor):57 object_detector = ObjectDetectionPipeline(model=model, image_processor=processor)58 return object_detector, ["./tests/fixtures/tests_samples/COCO/000000039769.png"]59 60 def run_pipeline_test(self, object_detector, examples):61 outputs = object_detector("./tests/fixtures/tests_samples/COCO/000000039769.png", threshold=0.0)62 63 self.assertGreater(len(outputs), 0)64 for detected_object in outputs:65 self.assertEqual(66 detected_object,67 {68 "score": ANY(float),69 "label": ANY(str),70 "box": {"xmin": ANY(int), "ymin": ANY(int), "xmax": ANY(int), "ymax": ANY(int)},71 },72 )73 74 import datasets75 76 dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", "image", split="test")77 78 batch = [79 Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),80 "http://images.cocodataset.org/val2017/000000039769.jpg",81 # RGBA82 dataset[0]["file"],83 # LA84 dataset[1]["file"],85 # L86 dataset[2]["file"],87 ]88 batch_outputs = object_detector(batch, threshold=0.0)89 90 self.assertEqual(len(batch), len(batch_outputs))91 for outputs in batch_outputs:92 self.assertGreater(len(outputs), 0)93 for detected_object in outputs:94 self.assertEqual(95 detected_object,96 {97 "score": ANY(float),98 "label": ANY(str),99 "box": {"xmin": ANY(int), "ymin": ANY(int), "xmax": ANY(int), "ymax": ANY(int)},100 },101 )102 103 @require_tf104 @unittest.skip("Object detection not implemented in TF")105 def test_small_model_tf(self):106 pass107 108 @require_torch109 def test_small_model_pt(self):110 model_id = "hf-internal-testing/tiny-detr-mobilenetsv3"111 112 model = AutoModelForObjectDetection.from_pretrained(model_id)113 feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)114 object_detector = ObjectDetectionPipeline(model=model, feature_extractor=feature_extractor)115 116 outputs = object_detector("http://images.cocodataset.org/val2017/000000039769.jpg", threshold=0.0)117 118 self.assertEqual(119 nested_simplify(outputs, decimals=4),120 [121 {"score": 0.3376, "label": "LABEL_0", "box": {"xmin": 159, "ymin": 120, "xmax": 480, "ymax": 359}},122 {"score": 0.3376, "label": "LABEL_0", "box": {"xmin": 159, "ymin": 120, "xmax": 480, "ymax": 359}},123 ],124 )125 126 outputs = object_detector(127 [128 "http://images.cocodataset.org/val2017/000000039769.jpg",129 "http://images.cocodataset.org/val2017/000000039769.jpg",130 ],131 threshold=0.0,132 )133 134 self.assertEqual(135 nested_simplify(outputs, decimals=4),136 [137 [138 {"score": 0.3376, "label": "LABEL_0", "box": {"xmin": 159, "ymin": 120, "xmax": 480, "ymax": 359}},139 {"score": 0.3376, "label": "LABEL_0", "box": {"xmin": 159, "ymin": 120, "xmax": 480, "ymax": 359}},140 ],141 [142 {"score": 0.3376, "label": "LABEL_0", "box": {"xmin": 159, "ymin": 120, "xmax": 480, "ymax": 359}},143 {"score": 0.3376, "label": "LABEL_0", "box": {"xmin": 159, "ymin": 120, "xmax": 480, "ymax": 359}},144 ],145 ],146 )147 148 @require_torch149 @slow150 def test_large_model_pt(self):151 model_id = "facebook/detr-resnet-50"152 153 model = AutoModelForObjectDetection.from_pretrained(model_id)154 feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)155 object_detector = ObjectDetectionPipeline(model=model, feature_extractor=feature_extractor)156 157 outputs = object_detector("http://images.cocodataset.org/val2017/000000039769.jpg")158 self.assertEqual(159 nested_simplify(outputs, decimals=4),160 [161 {"score": 0.9982, "label": "remote", "box": {"xmin": 40, "ymin": 70, "xmax": 175, "ymax": 117}},162 {"score": 0.9960, "label": "remote", "box": {"xmin": 333, "ymin": 72, "xmax": 368, "ymax": 187}},163 {"score": 0.9955, "label": "couch", "box": {"xmin": 0, "ymin": 1, "xmax": 639, "ymax": 473}},164 {"score": 0.9988, "label": "cat", "box": {"xmin": 13, "ymin": 52, "xmax": 314, "ymax": 470}},165 {"score": 0.9987, "label": "cat", "box": {"xmin": 345, "ymin": 23, "xmax": 640, "ymax": 368}},166 ],167 )168 169 outputs = object_detector(170 [171 "http://images.cocodataset.org/val2017/000000039769.jpg",172 "http://images.cocodataset.org/val2017/000000039769.jpg",173 ]174 )175 self.assertEqual(176 nested_simplify(outputs, decimals=4),177 [178 [179 {"score": 0.9982, "label": "remote", "box": {"xmin": 40, "ymin": 70, "xmax": 175, "ymax": 117}},180 {"score": 0.9960, "label": "remote", "box": {"xmin": 333, "ymin": 72, "xmax": 368, "ymax": 187}},181 {"score": 0.9955, "label": "couch", "box": {"xmin": 0, "ymin": 1, "xmax": 639, "ymax": 473}},182 {"score": 0.9988, "label": "cat", "box": {"xmin": 13, "ymin": 52, "xmax": 314, "ymax": 470}},183 {"score": 0.9987, "label": "cat", "box": {"xmin": 345, "ymin": 23, "xmax": 640, "ymax": 368}},184 ],185 [186 {"score": 0.9982, "label": "remote", "box": {"xmin": 40, "ymin": 70, "xmax": 175, "ymax": 117}},187 {"score": 0.9960, "label": "remote", "box": {"xmin": 333, "ymin": 72, "xmax": 368, "ymax": 187}},188 {"score": 0.9955, "label": "couch", "box": {"xmin": 0, "ymin": 1, "xmax": 639, "ymax": 473}},189 {"score": 0.9988, "label": "cat", "box": {"xmin": 13, "ymin": 52, "xmax": 314, "ymax": 470}},190 {"score": 0.9987, "label": "cat", "box": {"xmin": 345, "ymin": 23, "xmax": 640, "ymax": 368}},191 ],192 ],193 )194 195 @require_torch196 @slow197 def test_integration_torch_object_detection(self):198 model_id = "facebook/detr-resnet-50"199 200 object_detector = pipeline("object-detection", model=model_id)201 202 outputs = object_detector("http://images.cocodataset.org/val2017/000000039769.jpg")203 self.assertEqual(204 nested_simplify(outputs, decimals=4),205 [206 {"score": 0.9982, "label": "remote", "box": {"xmin": 40, "ymin": 70, "xmax": 175, "ymax": 117}},207 {"score": 0.9960, "label": "remote", "box": {"xmin": 333, "ymin": 72, "xmax": 368, "ymax": 187}},208 {"score": 0.9955, "label": "couch", "box": {"xmin": 0, "ymin": 1, "xmax": 639, "ymax": 473}},209 {"score": 0.9988, "label": "cat", "box": {"xmin": 13, "ymin": 52, "xmax": 314, "ymax": 470}},210 {"score": 0.9987, "label": "cat", "box": {"xmin": 345, "ymin": 23, "xmax": 640, "ymax": 368}},211 ],212 )213 214 outputs = object_detector(215 [216 "http://images.cocodataset.org/val2017/000000039769.jpg",217 "http://images.cocodataset.org/val2017/000000039769.jpg",218 ]219 )220 self.assertEqual(221 nested_simplify(outputs, decimals=4),222 [223 [224 {"score": 0.9982, "label": "remote", "box": {"xmin": 40, "ymin": 70, "xmax": 175, "ymax": 117}},225 {"score": 0.9960, "label": "remote", "box": {"xmin": 333, "ymin": 72, "xmax": 368, "ymax": 187}},226 {"score": 0.9955, "label": "couch", "box": {"xmin": 0, "ymin": 1, "xmax": 639, "ymax": 473}},227 {"score": 0.9988, "label": "cat", "box": {"xmin": 13, "ymin": 52, "xmax": 314, "ymax": 470}},228 {"score": 0.9987, "label": "cat", "box": {"xmin": 345, "ymin": 23, "xmax": 640, "ymax": 368}},229 ],230 [231 {"score": 0.9982, "label": "remote", "box": {"xmin": 40, "ymin": 70, "xmax": 175, "ymax": 117}},232 {"score": 0.9960, "label": "remote", "box": {"xmin": 333, "ymin": 72, "xmax": 368, "ymax": 187}},233 {"score": 0.9955, "label": "couch", "box": {"xmin": 0, "ymin": 1, "xmax": 639, "ymax": 473}},234 {"score": 0.9988, "label": "cat", "box": {"xmin": 13, "ymin": 52, "xmax": 314, "ymax": 470}},235 {"score": 0.9987, "label": "cat", "box": {"xmin": 345, "ymin": 23, "xmax": 640, "ymax": 368}},236 ],237 ],238 )239 240 @require_torch241 @slow242 def test_threshold(self):243 threshold = 0.9985244 model_id = "facebook/detr-resnet-50"245 246 object_detector = pipeline("object-detection", model=model_id)247 248 outputs = object_detector("http://images.cocodataset.org/val2017/000000039769.jpg", threshold=threshold)249 self.assertEqual(250 nested_simplify(outputs, decimals=4),251 [252 {"score": 0.9988, "label": "cat", "box": {"xmin": 13, "ymin": 52, "xmax": 314, "ymax": 470}},253 {"score": 0.9987, "label": "cat", "box": {"xmin": 345, "ymin": 23, "xmax": 640, "ymax": 368}},254 ],255 )256 257 @require_torch258 @require_pytesseract259 @slow260 def test_layoutlm(self):261 model_id = "Narsil/layoutlmv3-finetuned-funsd"262 threshold = 0.9993263 264 object_detector = pipeline("object-detection", model=model_id, threshold=threshold)265 266 outputs = object_detector(267 "https://huggingface.co/spaces/impira/docquery/resolve/2359223c1837a7587402bda0f2643382a6eefeab/invoice.png"268 )269 self.assertEqual(270 nested_simplify(outputs, decimals=4),271 [272 {"score": 0.9993, "label": "I-ANSWER", "box": {"xmin": 294, "ymin": 254, "xmax": 343, "ymax": 264}},273 {"score": 0.9993, "label": "I-ANSWER", "box": {"xmin": 294, "ymin": 254, "xmax": 343, "ymax": 264}},274 ],275 )276