RicardoEstep/RPBizkit-v5-12B-Lorablated

models:
  - model: DreadPoor/Krix-12B-Model_Stock
  - model: ArliAI/Mistral-Nemo-12B-ArliAI-RPMax-v1.2
  - model: DavidAU/MN-GRAND-Gutenberg-Lyra4-Lyra-12B-DARKNESS
  - model: TheDrummer/UnslopNemo-12B-v4.1
  - model: inflatebot/MN-12B-Mag-Mell-R1
  - model: yamatazen/FusionEngine-12B
  - model: ChaoticNeutrals/Nera_Noctis-12B
  - model: HumanLLMs/Human-Like-Mistral-Nemo-Instruct-2407
  - model: LatitudeGames/Wayfarer-12B
  - model: allura-org/MN-12b-RP-Ink
  - model: rAIfle/Questionable-MN-bf16
  - model: SicariusSicariiStuff/Impish_Bloodmoon_12B
  - model: ReadyArt/Forgotten-Safeword-12B-v4.0
  - model: allura-org/Bigger-Body-12b
  - model: romaingrx/red-teamer-mistral-nemo    # I had to manually download this model.
  - model: Delta-Vector/Ohashi-NeMo-12B
  - model: Elizezen/Himeyuri-v0.1-12B
  - model: shisa-ai/shisa-v2-mistral-nemo-12b
merge_method: karcher
base_model: ReadyArt/Forgotten-Safeword-12B-v4.0
parameters:
  normalize: true
  karcher_steps: 128   # Ultra-Extremely Clean, Stable, High‑quality Merge.
  tolerance: 1e-8    # Extremely smoothness and Minimal artifacts.
dtype: bfloat16
tokenizer:
  source: base

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel

# -------- Configuration --------
base_model_path = "./output"
tokenizer_path = "./output"
lora_path = "nbeerbower/Mistral-Nemo-12B-abliterated-LORA"
output_path = "./RPBizkit-v5-12B-Lorablated"

# Hybrid scaling (recommended starting values)
ATTENTION_SCALE = 0.7   # Strong (but not complete) overwrite on attention.
MLP_SCALE = 0.3         # Light influence on MLP for stability.

# --------------------------

print("Loading base model...")
model = AutoModelForCausalLM.from_pretrained(
    base_model_path,
    torch_dtype=torch.bfloat16,
    low_cpu_mem_usage=True,
    trust_remote_code=True
)

# --- Fix Embeddeds ---
expected_vocab_size = 131072
current_vocab_size = model.get_input_embeddings().weight.shape[0]
if current_vocab_size != expected_vocab_size:
    print(f"Resizing embeddings from {current_vocab_size} to {expected_vocab_size}...")
    model.resize_token_embeddings(expected_vocab_size)

# --- Apply LoRA ---
print("Applying LoRA...")
model = PeftModel.from_pretrained(
    model,
    lora_path,
    adapter_name="default",
    is_trainable=False
)

# --- HYBRID SCALING ---
print("Applying hybrid scaling...")
adapter_name = "default"
for name, module in model.named_modules():
    if hasattr(module, "scaling"):
        # Strong behavioral overwrite on attention
        if any(x in name for x in ["q_proj", "k_proj", "v_proj", "o_proj"]):
            module.scaling = {adapter_name: ATTENTION_SCALE}
        # Light influence on MLP
        elif any(x in name for x in ["up_proj", "down_proj", "gate_proj"]):
            module.scaling = {adapter_name: MLP_SCALE}

# --- Merging the LoRA ---
print("Merging LoRA into base weights...")
model = model.merge_and_unload(progressbar=True)

# --- Adding Tokenizer ---
print("Loading tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True)

# --- Save Final Model ---
print("Saving final hybrid-merged model...")
model.save_pretrained(output_path, safe_serialization=True)
tokenizer.save_pretrained(output_path)

print("Hybrid merge complete!")