diffusion-reasoning/gdsd_countdown_dream

1# coding=utf-82# Copyright 2024 The Dream team, HKUNLP Group and the HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Dream model configuration"""16 17from transformers.configuration_utils import PretrainedConfig18from transformers.modeling_rope_utils import rope_config_validation19from transformers.utils import logging20 21 22logger = logging.get_logger(__name__)23 24 25class DreamConfig(PretrainedConfig):26    model_type = "Dream"27    keys_to_ignore_at_inference = ["past_key_values"]28 29    def __init__(30        self,31        vocab_size=151936,32        hidden_size=4096,33        intermediate_size=22016,34        num_hidden_layers=32,35        num_attention_heads=32,36        num_key_value_heads=32,37        hidden_act="silu",38        max_position_embeddings=32768,39        initializer_range=0.02,40        rms_norm_eps=1e-6,41        use_cache=False,  # cache not used in diffusion42        tie_word_embeddings=False,43        rope_theta=10000.0,44        rope_scaling=None,45        use_sliding_window=False,46        sliding_window=4096,47        max_window_layers=28,48        attention_dropout=0.0,49        mask_token_id=151666,50        pad_token_id=151643,51        **kwargs,52    ):53        self.vocab_size = vocab_size54        self.max_position_embeddings = max_position_embeddings55        self.hidden_size = hidden_size56        self.intermediate_size = intermediate_size57        self.num_hidden_layers = num_hidden_layers58        self.num_attention_heads = num_attention_heads59        self.use_sliding_window = use_sliding_window60        self.sliding_window = sliding_window if use_sliding_window else None61        self.max_window_layers = max_window_layers62 63        # for backward compatibility64        if num_key_value_heads is None:65            num_key_value_heads = num_attention_heads66 67        self.num_key_value_heads = num_key_value_heads68        self.hidden_act = hidden_act69        self.initializer_range = initializer_range70        self.rms_norm_eps = rms_norm_eps71        self.use_cache = use_cache72        self.rope_theta = rope_theta73        self.rope_scaling = rope_scaling74        self.attention_dropout = attention_dropout75        # Validate the correctness of rotary position embeddings parameters76        # BC: if there is a 'type' field, move it to 'rope_type'.77        if self.rope_scaling is not None and "type" in self.rope_scaling:78            self.rope_scaling["rope_type"] = self.rope_scaling["type"]79        rope_config_validation(self)80        81        super().__init__(82            tie_word_embeddings=tie_word_embeddings,83            **kwargs,84        )85        self.mask_token_id = mask_token_id86        self.pad_token_id = pad_token_id87