mrcuddle/llama3.2-11B-Vision_instruct-Coder

1---2language:3  - en4  - de5  - fr6  - it7  - pt8  - hi9  - es10  - th11library_name: transformers12pipeline_tag: image-text-to-text13tags:14  - meta15  - pytorch16  - llama17  - llama-318  - vision19base_model:20- meta-llama/Llama-3.2-11B-Vision-Instruct21- rombodawg/Llama-3-8B-Instruct-Coder22---23# Llama-3-8B-Instruct-Coder + Llama3.2Vision Adapter24 25This model was created using the script below. It is compatible with:26 27* Llama 3.1 8B & 70B28  29Respectively30 31* Llama Vision 3.2 11B & 90B32  33## Merge Script34 35```python36from transformers import MllamaForConditionalGeneration, MllamaProcessor, AutoModelForCausalLM37 38# NOTE: You need sufficient DRAM to load both models at once (otherwise, need to process layer by layer which is not shown here)39 40multimodal_model_path = "meta-llama/Llama-3.2-11B-Vision-Instruct"  # Original Llama vision model (11B or 90B)41text_model_path = "rombodawg/Llama-3-8B-Instruct-Coder"  # Model to be merged (8B or 70B)42save_path = "models/merged_model"43 44multimodal_model = MllamaForConditionalGeneration.from_pretrained(multimodal_model_path, device_map="cpu", torch_dtype=torch.bfloat16)45multimodal_processor = MllamaProcessor.from_pretrained(multimodal_model_path)46text_model = AutoModelForCausalLM.from_pretrained(text_model_path, device_map="cpu", torch_dtype=torch.bfloat16)47 48state_dict_multimodal = multimodal_model.state_dict()49state_dict_text = text_model.state_dict()50 51num_decoder_layers_text = text_model.config.num_hidden_layers52num_decoder_layers_vision = multimodal_model.config.text_config.num_hidden_layers53 54# Find the list of inserted layers in multimodal Llama55inserted_layers = set()56for key_multimodal in state_dict_multimodal.keys():57    if "language_model" in key_multimodal and "cross_attn" in key_multimodal and ".layers." in key_multimodal:58        layer_num_multimodal = int(key_multimodal.split(".layers.")[1].split(".")[0]) if ".layers." in key_multimodal else None59        if layer_num_multimodal is not None: inserted_layers.add(layer_num_multimodal)60# Here are the hard-coded list of layers added:61# inserted_layers = {3, 8, 13, 18, 23, 28, 33, 38, 43, 48, 53, 58, 63, 68, 73, 78, 83, 88, 93, 98} $ For 90B62inserted_layers = {3, 8, 13, 18, 23, 28, 33, 38}  # For 11B63 64assert len(inserted_layers) == num_decoder_layers_vision - num_decoder_layers_text, "# of added layers do not match"65 66# Build decoder layer map from multimodal layer# to text layer#, skipping layers listed in inserted_layers67layer_map = dict()68layer_num_multimodal = 069for layer_num_text in range(num_decoder_layers_text):70    while layer_num_multimodal in inserted_layers: layer_num_multimodal += 1  # Increment to skip mismatched layers71    layer_map[layer_num_multimodal] = layer_num_text72    layer_num_multimodal += 173 74for key_multimodal in state_dict_multimodal.keys():75    if "language_model" not in key_multimodal: continue  # A multi-modal param76    if "cross_attn" in key_multimodal: continue  # A multi-modal param77    key_text = key_multimodal.replace("language_model.", "")78    if "embed_tokens.weight" in key_multimodal:  # Handle embed tokens separately79        assert key_text in state_dict_text, f"Key not found: {key_text}"80        extra_tokens = state_dict_multimodal[key_multimodal].shape[0] - state_dict_text[key_text].shape[0]81        state_dict_multimodal[key_multimodal][:state_dict_text[key_text].shape[0], :].copy_(state_dict_text[key_text])82        print(f"Replaced {key_multimodal} with {key_text} (preserving last {extra_tokens} tokens)")83        continue84    if "lm_head" in key_multimodal or "model.norm.weight" in key_multimodal:  # Handle other non-decoder layers separately85        assert key_text in state_dict_text, f"Key not found: {key_text}"86        state_dict_multimodal[key_multimodal].copy_(state_dict_text[key_text])87        print(f"Replaced {key_multimodal} with {key_text}")88        continue89    layer_num_multimodal = int(key_multimodal.split(".layers.")[1].split(".")[0]) if ".layers." in key_multimodal else None90    assert layer_num_multimodal is not None, f"Unknown non-decoder key encountered: {key_multimodal}"91    if layer_num_multimodal in inserted_layers: continue  # Skip mismatched layers92    assert layer_num_multimodal in layer_map, f"Layer not found in layer_map: {layer_num_multimodal}"93    layer_num_text = layer_map[layer_num_multimodal]94    key_text = key_text.replace(f".layers.{layer_num_multimodal}.", f".layers.{layer_num_text}.")95    assert key_text in state_dict_text, f"Key not found: {key_text}"96    state_dict_multimodal[key_multimodal].copy_(state_dict_text[key_text])97    print(f"Replaced {key_multimodal} with {key_text}")98 99print("Merged model successfully. Saving...")100# Apply the changes101multimodal_model.load_state_dict(state_dict_multimodal)102 103# Create save_path if it does not exist104os.makedirs(save_path, exist_ok=True)105multimodal_model.save_pretrained(save_path, safe_serialization=True, max_shard_size="8192MB")106multimodal_processor.save_pretrained(save_path)107print(f"Model saved to {save_path}")108```109 110## Model Inference:111 112```python113import requests114import torch115from PIL import Image116from transformers import MllamaForConditionalGeneration, AutoProcessor117 118model_id = "rombodawg/Llama-3-8B-Instruct-Coder"119 120model = MllamaForConditionalGeneration.from_pretrained(121    model_id,122    torch_dtype=torch.bfloat16,123    device_map="auto",124)125processor = AutoProcessor.from_pretrained(model_id)126```127 128## License129 130This project is licensed under the MIT License.