fred-dev/comfy_ui_ali

1import re2import torch3import logging4 5# conversion code from https://github.com/huggingface/diffusers/blob/main/scripts/convert_diffusers_to_original_stable_diffusion.py6 7# ================#8# VAE Conversion #9# ================#10 11vae_conversion_map = [12    # (stable-diffusion, HF Diffusers)13    ("nin_shortcut", "conv_shortcut"),14    ("norm_out", "conv_norm_out"),15    ("mid.attn_1.", "mid_block.attentions.0."),16]17 18for i in range(4):19    # down_blocks have two resnets20    for j in range(2):21        hf_down_prefix = f"encoder.down_blocks.{i}.resnets.{j}."22        sd_down_prefix = f"encoder.down.{i}.block.{j}."23        vae_conversion_map.append((sd_down_prefix, hf_down_prefix))24 25    if i < 3:26        hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0."27        sd_downsample_prefix = f"down.{i}.downsample."28        vae_conversion_map.append((sd_downsample_prefix, hf_downsample_prefix))29 30        hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."31        sd_upsample_prefix = f"up.{3 - i}.upsample."32        vae_conversion_map.append((sd_upsample_prefix, hf_upsample_prefix))33 34    # up_blocks have three resnets35    # also, up blocks in hf are numbered in reverse from sd36    for j in range(3):37        hf_up_prefix = f"decoder.up_blocks.{i}.resnets.{j}."38        sd_up_prefix = f"decoder.up.{3 - i}.block.{j}."39        vae_conversion_map.append((sd_up_prefix, hf_up_prefix))40 41# this part accounts for mid blocks in both the encoder and the decoder42for i in range(2):43    hf_mid_res_prefix = f"mid_block.resnets.{i}."44    sd_mid_res_prefix = f"mid.block_{i + 1}."45    vae_conversion_map.append((sd_mid_res_prefix, hf_mid_res_prefix))46 47vae_conversion_map_attn = [48    # (stable-diffusion, HF Diffusers)49    ("norm.", "group_norm."),50    ("q.", "query."),51    ("k.", "key."),52    ("v.", "value."),53    ("q.", "to_q."),54    ("k.", "to_k."),55    ("v.", "to_v."),56    ("proj_out.", "to_out.0."),57    ("proj_out.", "proj_attn."),58]59 60 61def reshape_weight_for_sd(w, conv3d=False):62    # convert HF linear weights to SD conv2d weights63    if conv3d:64        return w.reshape(*w.shape, 1, 1, 1)65    else:66        return w.reshape(*w.shape, 1, 1)67 68 69def convert_vae_state_dict(vae_state_dict):70    mapping = {k: k for k in vae_state_dict.keys()}71    conv3d = False72    for k, v in mapping.items():73        for sd_part, hf_part in vae_conversion_map:74            v = v.replace(hf_part, sd_part)75        if v.endswith(".conv.weight"):76            if not conv3d and vae_state_dict[k].ndim == 5:77                conv3d = True78        mapping[k] = v79    for k, v in mapping.items():80        if "attentions" in k:81            for sd_part, hf_part in vae_conversion_map_attn:82                v = v.replace(hf_part, sd_part)83            mapping[k] = v84    new_state_dict = {v: vae_state_dict[k] for k, v in mapping.items()}85    weights_to_convert = ["q", "k", "v", "proj_out"]86    for k, v in new_state_dict.items():87        for weight_name in weights_to_convert:88            if f"mid.attn_1.{weight_name}.weight" in k:89                logging.debug(f"Reshaping {k} for SD format")90                new_state_dict[k] = reshape_weight_for_sd(v, conv3d=conv3d)91    return new_state_dict92 93 94# =========================#95# Text Encoder Conversion #96# =========================#97 98 99textenc_conversion_lst = [100    # (stable-diffusion, HF Diffusers)101    ("resblocks.", "text_model.encoder.layers."),102    ("ln_1", "layer_norm1"),103    ("ln_2", "layer_norm2"),104    (".c_fc.", ".fc1."),105    (".c_proj.", ".fc2."),106    (".attn", ".self_attn"),107    ("ln_final.", "transformer.text_model.final_layer_norm."),108    ("token_embedding.weight", "transformer.text_model.embeddings.token_embedding.weight"),109    ("positional_embedding", "transformer.text_model.embeddings.position_embedding.weight"),110]111protected = {re.escape(x[1]): x[0] for x in textenc_conversion_lst}112textenc_pattern = re.compile("|".join(protected.keys()))113 114# Ordering is from https://github.com/pytorch/pytorch/blob/master/test/cpp/api/modules.cpp115code2idx = {"q": 0, "k": 1, "v": 2}116 117 118# This function exists because at the time of writing torch.cat can't do fp8 with cuda119def cat_tensors(tensors):120    x = 0121    for t in tensors:122        x += t.shape[0]123 124    shape = [x] + list(tensors[0].shape)[1:]125    out = torch.empty(shape, device=tensors[0].device, dtype=tensors[0].dtype)126 127    x = 0128    for t in tensors:129        out[x:x + t.shape[0]] = t130        x += t.shape[0]131 132    return out133 134 135def convert_text_enc_state_dict_v20(text_enc_dict, prefix=""):136    new_state_dict = {}137    capture_qkv_weight = {}138    capture_qkv_bias = {}139    for k, v in text_enc_dict.items():140        if not k.startswith(prefix):141            continue142        if (143                k.endswith(".self_attn.q_proj.weight")144                or k.endswith(".self_attn.k_proj.weight")145                or k.endswith(".self_attn.v_proj.weight")146        ):147            k_pre = k[: -len(".q_proj.weight")]148            k_code = k[-len("q_proj.weight")]149            if k_pre not in capture_qkv_weight:150                capture_qkv_weight[k_pre] = [None, None, None]151            capture_qkv_weight[k_pre][code2idx[k_code]] = v152            continue153 154        if (155                k.endswith(".self_attn.q_proj.bias")156                or k.endswith(".self_attn.k_proj.bias")157                or k.endswith(".self_attn.v_proj.bias")158        ):159            k_pre = k[: -len(".q_proj.bias")]160            k_code = k[-len("q_proj.bias")]161            if k_pre not in capture_qkv_bias:162                capture_qkv_bias[k_pre] = [None, None, None]163            capture_qkv_bias[k_pre][code2idx[k_code]] = v164            continue165 166        text_proj = "transformer.text_projection.weight"167        if k.endswith(text_proj):168            new_state_dict[k.replace(text_proj, "text_projection")] = v.transpose(0, 1).contiguous()169        else:170            relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k)171            new_state_dict[relabelled_key] = v172 173    for k_pre, tensors in capture_qkv_weight.items():174        if None in tensors:175            raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing")176        relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre)177        new_state_dict[relabelled_key + ".in_proj_weight"] = cat_tensors(tensors)178 179    for k_pre, tensors in capture_qkv_bias.items():180        if None in tensors:181            raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing")182        relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre)183        new_state_dict[relabelled_key + ".in_proj_bias"] = cat_tensors(tensors)184 185    return new_state_dict186 187 188def convert_text_enc_state_dict(text_enc_dict):189    return text_enc_dict190