chendl/compositional_test

1# Copyright 2021 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import unittest16 17from transformers import (18    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,19    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,20    PreTrainedTokenizer,21    is_vision_available,22)23from transformers.pipelines import ImageClassificationPipeline, pipeline24from transformers.testing_utils import (25    is_pipeline_test,26    nested_simplify,27    require_tf,28    require_torch,29    require_torch_or_tf,30    require_vision,31    slow,32)33 34from .test_pipelines_common import ANY35 36 37if is_vision_available():38    from PIL import Image39else:40 41    class Image:42        @staticmethod43        def open(*args, **kwargs):44            pass45 46 47@is_pipeline_test48@require_torch_or_tf49@require_vision50class ImageClassificationPipelineTests(unittest.TestCase):51    model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING52    tf_model_mapping = TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING53 54    def get_test_pipeline(self, model, tokenizer, processor):55        image_classifier = ImageClassificationPipeline(model=model, image_processor=processor, top_k=2)56        examples = [57            Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),58            "http://images.cocodataset.org/val2017/000000039769.jpg",59        ]60        return image_classifier, examples61 62    def run_pipeline_test(self, image_classifier, examples):63        outputs = image_classifier("./tests/fixtures/tests_samples/COCO/000000039769.png")64 65        self.assertEqual(66            outputs,67            [68                {"score": ANY(float), "label": ANY(str)},69                {"score": ANY(float), "label": ANY(str)},70            ],71        )72 73        import datasets74 75        dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", "image", split="test")76 77        # Accepts URL + PIL.Image + lists78        outputs = image_classifier(79            [80                Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),81                "http://images.cocodataset.org/val2017/000000039769.jpg",82                # RGBA83                dataset[0]["file"],84                # LA85                dataset[1]["file"],86                # L87                dataset[2]["file"],88            ]89        )90        self.assertEqual(91            outputs,92            [93                [94                    {"score": ANY(float), "label": ANY(str)},95                    {"score": ANY(float), "label": ANY(str)},96                ],97                [98                    {"score": ANY(float), "label": ANY(str)},99                    {"score": ANY(float), "label": ANY(str)},100                ],101                [102                    {"score": ANY(float), "label": ANY(str)},103                    {"score": ANY(float), "label": ANY(str)},104                ],105                [106                    {"score": ANY(float), "label": ANY(str)},107                    {"score": ANY(float), "label": ANY(str)},108                ],109                [110                    {"score": ANY(float), "label": ANY(str)},111                    {"score": ANY(float), "label": ANY(str)},112                ],113            ],114        )115 116    @require_torch117    def test_small_model_pt(self):118        small_model = "hf-internal-testing/tiny-random-vit"119        image_classifier = pipeline("image-classification", model=small_model)120 121        outputs = image_classifier("http://images.cocodataset.org/val2017/000000039769.jpg")122        self.assertEqual(123            nested_simplify(outputs, decimals=4),124            [{"label": "LABEL_1", "score": 0.574}, {"label": "LABEL_0", "score": 0.426}],125        )126 127        outputs = image_classifier(128            [129                "http://images.cocodataset.org/val2017/000000039769.jpg",130                "http://images.cocodataset.org/val2017/000000039769.jpg",131            ],132            top_k=2,133        )134        self.assertEqual(135            nested_simplify(outputs, decimals=4),136            [137                [{"label": "LABEL_1", "score": 0.574}, {"label": "LABEL_0", "score": 0.426}],138                [{"label": "LABEL_1", "score": 0.574}, {"label": "LABEL_0", "score": 0.426}],139            ],140        )141 142    @require_tf143    def test_small_model_tf(self):144        small_model = "hf-internal-testing/tiny-random-vit"145        image_classifier = pipeline("image-classification", model=small_model, framework="tf")146 147        outputs = image_classifier("http://images.cocodataset.org/val2017/000000039769.jpg")148        self.assertEqual(149            nested_simplify(outputs, decimals=4),150            [{"label": "LABEL_1", "score": 0.574}, {"label": "LABEL_0", "score": 0.426}],151        )152 153        outputs = image_classifier(154            [155                "http://images.cocodataset.org/val2017/000000039769.jpg",156                "http://images.cocodataset.org/val2017/000000039769.jpg",157            ],158            top_k=2,159        )160        self.assertEqual(161            nested_simplify(outputs, decimals=4),162            [163                [{"label": "LABEL_1", "score": 0.574}, {"label": "LABEL_0", "score": 0.426}],164                [{"label": "LABEL_1", "score": 0.574}, {"label": "LABEL_0", "score": 0.426}],165            ],166        )167 168    def test_custom_tokenizer(self):169        tokenizer = PreTrainedTokenizer()170 171        # Assert that the pipeline can be initialized with a feature extractor that is not in any mapping172        image_classifier = pipeline(173            "image-classification", model="hf-internal-testing/tiny-random-vit", tokenizer=tokenizer174        )175 176        self.assertIs(image_classifier.tokenizer, tokenizer)177 178    @slow179    @require_torch180    def test_perceiver(self):181        # Perceiver is not tested by `run_pipeline_test` properly.182        # That is because the type of feature_extractor and model preprocessor need to be kept183        # in sync, which is not the case in the current design184        image_classifier = pipeline("image-classification", model="deepmind/vision-perceiver-conv")185        outputs = image_classifier("http://images.cocodataset.org/val2017/000000039769.jpg")186        self.assertEqual(187            nested_simplify(outputs, decimals=4),188            [189                {"score": 0.4385, "label": "tabby, tabby cat"},190                {"score": 0.321, "label": "tiger cat"},191                {"score": 0.0502, "label": "Egyptian cat"},192                {"score": 0.0137, "label": "crib, cot"},193                {"score": 0.007, "label": "radiator"},194            ],195        )196 197        image_classifier = pipeline("image-classification", model="deepmind/vision-perceiver-fourier")198        outputs = image_classifier("http://images.cocodataset.org/val2017/000000039769.jpg")199        self.assertEqual(200            nested_simplify(outputs, decimals=4),201            [202                {"score": 0.5658, "label": "tabby, tabby cat"},203                {"score": 0.1309, "label": "tiger cat"},204                {"score": 0.0722, "label": "Egyptian cat"},205                {"score": 0.0707, "label": "remote control, remote"},206                {"score": 0.0082, "label": "computer keyboard, keypad"},207            ],208        )209 210        image_classifier = pipeline("image-classification", model="deepmind/vision-perceiver-learned")211        outputs = image_classifier("http://images.cocodataset.org/val2017/000000039769.jpg")212        self.assertEqual(213            nested_simplify(outputs, decimals=4),214            [215                {"score": 0.3022, "label": "tabby, tabby cat"},216                {"score": 0.2362, "label": "Egyptian cat"},217                {"score": 0.1856, "label": "tiger cat"},218                {"score": 0.0324, "label": "remote control, remote"},219                {"score": 0.0096, "label": "quilt, comforter, comfort, puff"},220            ],221        )222