karolmajek/maxdeeplab

1# coding=utf-82# Copyright 2021 The Deeplab2 Authors.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16"""Test for max_deeplab.py."""17import numpy as np18import tensorflow as tf19 20from deeplab2.model.post_processor import max_deeplab21 22 23class PostProcessingTest(tf.test.TestCase):24 25  def test_filter_by_count(self):26    input_index_map = tf.convert_to_tensor(27        [[[1, 1, 1, 1],28          [1, 2, 2, 1],29          [3, 3, 3, 3],30          [4, 5, 5, 5]],31         [[4, 5, 5, 5],32          [3, 3, 3, 3],33          [1, 2, 2, 1],34          [1, 1, 1, 1]]], dtype=tf.float32)35    area_limit = 336    filtered_index_map, mask = max_deeplab._filter_by_count(37        input_index_map, area_limit)38 39    expected_filtered_index_map = tf.convert_to_tensor(40        [[[1, 1, 1, 1],41          [1, 0, 0, 1],42          [3, 3, 3, 3],43          [0, 5, 5, 5]],44         [[0, 5, 5, 5],45          [3, 3, 3, 3],46          [1, 0, 0, 1],47          [1, 1, 1, 1]]], dtype=tf.float32)48    np.testing.assert_equal(filtered_index_map.numpy(),49                            expected_filtered_index_map.numpy())50    expected_mask = tf.convert_to_tensor(51        [[[1, 1, 1, 1],52          [1, 0, 0, 1],53          [1, 1, 1, 1],54          [0, 1, 1, 1]],55         [[0, 1, 1, 1],56          [1, 1, 1, 1],57          [1, 0, 0, 1],58          [1, 1, 1, 1]]], dtype=tf.float32)59    np.testing.assert_equal(mask.numpy(), expected_mask.numpy())60 61  def test_get_mask_id_and_semantic_maps(self):62    height = 2163    width = 2164    num_mask_slots = 565    num_thing_stuff_classes = 1966    thing_class_ids = list(range(11, 19))67    stuff_class_ids = list(range(0, 11))68    pixel_space_mask_logits = tf.random.uniform(69        (height, width, num_mask_slots), minval=-10, maxval=10)70    # Class scores are normalized beforehand (softmax-ed beforehand).71    transformer_class_probs = tf.random.uniform(72        (num_mask_slots, num_thing_stuff_classes + 1), minval=0, maxval=1)73    input_shape = [41, 41]74    pixel_confidence_threshold = 0.475    transformer_class_confidence_threshold = 0.776    pieces = 277 78    mask_id_map, semantic_map, thing_mask, stuff_mask = (79        max_deeplab._get_mask_id_and_semantic_maps(80            thing_class_ids, stuff_class_ids, pixel_space_mask_logits,81            transformer_class_probs, input_shape, pixel_confidence_threshold,82            transformer_class_confidence_threshold, pieces)83        )84    self.assertListEqual(mask_id_map.get_shape().as_list(), input_shape)85    self.assertListEqual(semantic_map.get_shape().as_list(), input_shape)86    self.assertListEqual(thing_mask.get_shape().as_list(), input_shape)87    self.assertListEqual(stuff_mask.get_shape().as_list(), input_shape)88 89  def test_merge_mask_id_and_semantic_maps(self):90    mask_id_maps = tf.convert_to_tensor(91        [[[1, 1, 1, 1],92          [1, 2, 2, 1],93          [3, 3, 4, 4],94          [5, 5, 6, 6]]], dtype=tf.int32)95    semantic_maps = tf.convert_to_tensor(96        [[[0, 0, 0, 0],97          [0, 1, 1, 0],98          [2, 2, 2, 2],99          [2, 2, 3, 3]]], dtype=tf.int32)100    thing_masks = tf.convert_to_tensor(101        [[[0, 0, 0, 0],102          [0, 0, 0, 0],103          [1, 1, 1, 1],104          [1, 0, 1, 1]]], dtype=tf.float32)  # thing_class_ids = [2, 3]105    stuff_masks = tf.convert_to_tensor(106        [[[1, 1, 1, 0],107          [1, 1, 1, 1],108          [0, 0, 0, 0],109          [0, 0, 0, 0]]], dtype=tf.float32)  # stuff_class_ids = [0, 1]110 111    batch_size = 3112    mask_id_maps = tf.repeat(mask_id_maps, repeats=batch_size, axis=0)113    semantic_maps = tf.repeat(semantic_maps, repeats=batch_size, axis=0)114    thing_masks = tf.repeat(thing_masks, repeats=batch_size, axis=0)115    stuff_masks = tf.repeat(stuff_masks, repeats=batch_size, axis=0)116 117    label_divisor = 100118    stuff_area_limit = 3119    void_label = 255120    thing_area_limit = 2121    # The expected_panoptic_prediction is computed as follows.122    # All un-certain regions will be labeled as `void_label * label_divisor`.123    # For `thing` segmentation, instance 3, 4, and 6 are kept, but instance 5124    # is re-labeled as `void_label * label_divisor` since its area had been125    # reduced by `confident_regions` and is then filtered by thing_area_limit.126    # For `stuff` segmentation, class-0 region is kept, while class-1 region127    # is re-labeled as `void_label * label_divisor` since its area is smaller128    # than stuff_area_limit.129    expected_panoptic_prediction = tf.convert_to_tensor(130        [[[0, 0, 0, void_label * label_divisor],131          [0, void_label * label_divisor, void_label * label_divisor, 0],132          [2 * label_divisor + 3, 2 * label_divisor + 3, 2 * label_divisor + 4,133           2 * label_divisor + 4],134          [void_label * label_divisor, void_label * label_divisor,135           3 * label_divisor + 6, 3 * label_divisor + 6]]],136        dtype=tf.int32)137    expected_panoptic_prediction = tf.repeat(138        expected_panoptic_prediction, repeats=batch_size, axis=0)139    panoptic_prediction = (140        max_deeplab._merge_mask_id_and_semantic_maps(141            mask_id_maps, semantic_maps, thing_masks, stuff_masks, void_label,142            label_divisor, thing_area_limit, stuff_area_limit))143 144    np.testing.assert_equal(expected_panoptic_prediction.numpy(),145                            panoptic_prediction.numpy())146 147  def test_get_panoptic_predictions(self):148    batch = 1149    height = 5150    width = 5151    num_thing_stuff_classes = 2152    thing_class_ids = list(range(1, num_thing_stuff_classes + 1))  # [1, 2]153    label_divisor = 10154    stuff_area_limit = 3155    void_label = 0  # `class-0` is `void`156 157    o, x = 10, -10158    pixel_space_mask_logits = tf.convert_to_tensor(159        [[[[o, o, o, o, o],  # instance-1 mask160           [o, x, x, o, o],161           [x, x, x, x, x],162           [x, x, x, x, x],163           [x, x, x, x, x]],164 165          [[x, x, x, x, x],  # instance-2 mask166           [x, o, o, x, x],167           [x, o, o, x, x],168           [x, o, o, x, x],169           [x, x, x, x, x]],170 171          [[x, x, x, x, x],  # instance-3 mask172           [x, x, x, x, x],173           [o, x, x, o, o],174           [o, x, x, o, o],175           [o, o, o, o, o]]]],176        dtype=tf.float32)177    pixel_space_mask_logits = tf.transpose(pixel_space_mask_logits,178                                           perm=[0, 2, 3, 1])  # b, h, w, c179    # class scores are 0-1 normalized beforehand.180    # 3-rd column (class-2) represents `void` class scores.181    transformer_class_logits = tf.convert_to_tensor(182        [[183            [o, x, x],  # instance-1 -- class-0184            [o, x, x],  # instance-2 -- class-0185            [x, o, x],  # instance-3 -- class-1186        ]], dtype=tf.float32)187 188    input_shape = [5, 5]189    pixel_confidence_threshold = 0.4190    transformer_class_confidence_threshold = 0.7191    thing_area_limit = 3192    pieces = 1  # No piece-wise operation used.193 194    panoptic_maps, mask_id_maps, semantic_maps = (195        max_deeplab._get_panoptic_predictions(196            pixel_space_mask_logits, transformer_class_logits, thing_class_ids,197            void_label, label_divisor, thing_area_limit, stuff_area_limit,198            input_shape, pixel_confidence_threshold,199            transformer_class_confidence_threshold, pieces)200        )201    self.assertSequenceEqual(panoptic_maps.shape, (batch, height, width))202    self.assertSequenceEqual(semantic_maps.shape, (batch, height, width))203    self.assertSequenceEqual(mask_id_maps.shape, (batch, height, width))204    expected_panoptic_maps = [[  # label_divisor = 10205        [11, 11, 11, 11, 11],  # 11: semantic_id=1, instance_id=1206        [11, 12, 12, 11, 11],  # 12: semantic_id=1, instance_id=2207        [23, 12, 12, 23, 23],  # 23: semantic_id=2, instance_id=3208        [23, 12, 12, 23, 23],209        [23, 23, 23, 23, 23],210    ]]211    np.testing.assert_array_equal(panoptic_maps, expected_panoptic_maps)212    expected_mask_id_maps = [[213        [1, 1, 1, 1, 1],214        [1, 2, 2, 1, 1],215        [3, 2, 2, 3, 3],216        [3, 2, 2, 3, 3],217        [3, 3, 3, 3, 3],218    ]]219    np.testing.assert_array_equal(mask_id_maps, expected_mask_id_maps)220    expected_semantic_maps = [[221        [1, 1, 1, 1, 1],222        [1, 1, 1, 1, 1],223        [2, 1, 1, 2, 2],224        [2, 1, 1, 2, 2],225        [2, 2, 2, 2, 2],226    ]]227    np.testing.assert_array_equal(semantic_maps, expected_semantic_maps)228 229 230if __name__ == '__main__':231  tf.test.main()232