Aluode/PerceptionLabPortable

1# Copyright 2020 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15from math import ceil16 17 18def assert_device_map(device_map, num_blocks):19    blocks = list(range(0, num_blocks))20 21    device_map_blocks = [item for sublist in list(device_map.values()) for item in sublist]22 23    # Duplicate check24    duplicate_blocks = []25    for i in device_map_blocks:26        if device_map_blocks.count(i) > 1 and i not in duplicate_blocks:27            duplicate_blocks.append(i)28    # Missing blocks29    missing_blocks = [i for i in blocks if i not in device_map_blocks]30    extra_blocks = [i for i in device_map_blocks if i not in blocks]31 32    if len(duplicate_blocks) != 0:33        raise ValueError(34            "Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device."35            " These attention blocks were specified more than once: " + str(duplicate_blocks)36        )37    if len(missing_blocks) != 0:38        raise ValueError(39            "There are attention blocks for this model that are not specified in the device_map. Add these attention "40            "blocks to a device on the device_map: " + str(missing_blocks)41        )42    if len(extra_blocks) != 0:43        raise ValueError(44            "The device_map contains more attention blocks than this model has. Remove these from the device_map:"45            + str(extra_blocks)46        )47 48 49def get_device_map(n_layers, devices):50    """Returns a dictionary of layers distributed evenly across all devices."""51    layers = list(range(n_layers))52    n_blocks = int(ceil(n_layers / len(devices)))53    layers_list = [layers[i : i + n_blocks] for i in range(0, n_layers, n_blocks)]54 55    return dict(zip(devices, layers_list))56