Modelpublic

zerofata/G4-MeroMero-v2-31B-GGUF

sourceHugging Faceapache-2.0updated 2mo agoView on Hugging Face

40likes10kdownloads

Model Card

font-family: var(--sans); color: var(--text); max-width: 900px; margin: 0 auto; padding: 0 0 60px; line-height: 1.7; font-size: 1rem; background: radial-gradient(ellipse at 50% 0%, rgba(91,157,255,0.08) 0%, transparent 50%), radial-gradient(ellipse at 50% 100%, rgba(239,93,118,0.05) 0%, transparent 50%), var(--bg); }

/ ── Profile Card ── / .gs-profile { border-bottom: none; position: relative; background: var(--surface); margin-bottom: 0; } .gs-profile-art { position: relative; } .gs-profile-art img { display: block; width: 100%; height: 380px; object-fit: cover; margin-top: 0px; } .gs-ident { position: absolute; bottom: 0; left: 0; right: 0; padding: 120px 44px 28px; background: linear-gradient( to top, rgba(11,12,16,0.88) 0%, rgba(11,12,16,0.5) 50%, transparent 100% ); } .gs-profile-info { padding: 20px 44px 36px; display: flex; flex-direction: column; gap: 20px; } .gs-profile-label { display: flex; align-items: baseline; gap: 10px; font-family: var(--mono); letter-spacing: 0.14em; text-transform: uppercase; } .gs-profile-label .gs-snum { font-size: 0.62rem; font-weight: 700; color: var(--crimson); opacity: 1; position: static; transform: none; } .gs-profile-label .gs-stitle { font-size: 0.62rem; color: var(--dim); font-weight: 700; letter-spacing: 0.14em; } .gs-profile-label .gs-stitle::before { content: none; } .gs-name { font-family: var(--sans); font-size: 3.2rem; font-weight: 900; color: #ffffff; letter-spacing: 0.06em; line-height: 1; margin: 0 0 10px; text-shadow: 0 2px 12px rgba(0,0,0,0.3); overflow-wrap: break-word; } .gs-base { font-family: var(--mono); font-size: 0.68rem; color: var(--crimson); letter-spacing: 0.14em; text-transform: uppercase; display: block; } .gs-profile-bio p { margin: 0 0 14px; font-size: 0.95rem; } .gs-profile-bio p:last-child { margin-bottom: 0; }

/ ── Sections ── / .gs-section { padding: 0; } .gs-shead { position: relative; display: flex; align-items: center; gap: 14px; padding: 16px 44px 14px; margin-bottom: 28px; border-top: 2px solid; border-image: linear-gradient(90deg, var(--crimson), var(--azure)) 1; } .gs-snum { font-family: var(--mono); font-size: 2.2rem; font-weight: 900; color: var(--crimson); letter-spacing: 0.06em; opacity: 0.12; position: absolute; right: 44px; top: 50%; transform: translateY(-50%); line-height: 1; } .gs-stitle { font-size: 1.05rem; font-weight: 700; letter-spacing: 0.1em; text-transform: uppercase; color: var(--bright); } .gs-stitle::before { content: '\2726'; color: var(--crimson); font-size: 0.8em; margin-right: 8px; } .gs-sbody { padding: 0 44px 44px; } .gs-sbody p { margin: 0 0 14px; font-size: 0.95rem; } .gs-sbody p:last-child { margin-bottom: 0; }

/ ── Data panels ── / .gs-stack { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } .gs-stack .gs-panel:nth-child(3) { grid-column: 1 / -1; } .gs-panel { border: 1px solid var(--edge); border-left: 3px solid var(--crimson); position: relative; background: var(--surface); box-shadow: 0 2px 12px rgba(0,0,0,0.45); } .gs-panel::before { content: ''; position: absolute; top: -1px; right: -1px; width: 10px; height: 10px; border-top: 1px solid var(--crimson); border-right: 1px solid var(--crimson); opacity: 0.4; } .gs-panel::after { content: ''; position: absolute; bottom: -1px; right: -1px; width: 10px; height: 10px; border-bottom: 1px solid var(--azure); border-right: 1px solid var(--azure); opacity: 0.3; } .gs-panel-head { font-family: var(--mono); font-size: 0.68rem; font-weight: 700; letter-spacing: 0.14em; text-transform: uppercase; color: var(--dim); padding: 10px 16px; border-bottom: 1px solid var(--edge); } .gs-panel-head::after { content: ' \2726'; color: var(--crimson); opacity: 0.5; } .gs-row { display: grid; grid-template-columns: 10ch 1fr; align-items: baseline; column-gap: 4px; padding: 9px 16px; border-bottom: 1px solid var(--edge); font-size: 0.9rem; } .gs-row:last-child { border-bottom: none; } .gs-key { font-family: var(--mono); font-size: 0.9rem; color: var(--dim); } .gs-key::after { content: ':'; } .gs-val { color: var(--bright); font-size: 0.9rem; } .gs-row .gs-val:only-child { grid-column: 1 / -1; }

/ ── Quantizations (compact) ── / .gs-section--compact .gs-shead { border-top: 1px solid var(--edge); border-image-source: none; padding: 12px 44px 10px; margin-bottom: 18px; } .gs-section--compact .gs-snum { opacity: 0.08; } .gs-section--compact .gs-stitle::before { content: '\2726'; } .gs-section--compact .gs-sbody { padding: 0 44px 32px; } .gs-qrow { display: flex; gap: 12px; flex-wrap: wrap; justify-content: center; } .gs-qpanel { background: var(--surface); border: 1px solid var(--edge); border-left: 3px solid var(--crimson); display: flex; align-items: center; gap: 16px; padding: 12px 24px; border-radius: 4px; position: relative; box-shadow: 0 2px 12px rgba(0,0,0,0.45); } .gs-qpanel::before { content: ''; position: absolute; top: -1px; right: -1px; width: 10px; height: 10px; border-top: 1px solid var(--crimson); border-right: 1px solid var(--crimson); opacity: 0.4; } .gs-qpanel::after { content: ''; position: absolute; bottom: -1px; right: -1px; width: 10px; height: 10px; border-bottom: 1px solid var(--azure); border-right: 1px solid var(--azure); opacity: 0.3; } .gs-qtype { font-family: var(--mono); font-size: 0.58rem; font-weight: 700; letter-spacing: 0.18em; text-transform: uppercase; color: var(--crimson); flex-shrink: 0; } .gs-qsep { width: 1px; height: 16px; background: var(--rule); flex-shrink: 0; } .gs-qpanel a { color: var(--bright); text-decoration: none; font-size: 0.9rem; border-bottom: 1px solid var(--rule); } .gs-qpanel a:hover { color: var(--crimson); border-bottom-color: var(--crimson); }

/ ── Journal (Creation Process) ── / .gs-section--journal .gs-sbody { margin: 0 44px; padding: 24px 32px 32px; background: var(--surface); border: 1px solid var(--edge); border-left: 4px solid var(--azure); position: relative; margin-bottom: 0; } .gs-section--journal .gs-sbody::before { content: ''; position: absolute; top: -1px; right: -1px; width: 12px; height: 12px; border-top: 1px solid var(--azure); border-right: 1px solid var(--azure); opacity: 0.3; } .gs-section--journal .gs-sbody::after { content: ''; position: absolute; bottom: -1px; left: -1px; width: 12px; height: 12px; border-bottom: 1px solid var(--crimson); border-left: 1px solid var(--crimson); opacity: 0.3; } .gs-section--journal .gs-sbody p:first-child { font-style: italic; color: var(--bright); }

/ ── Links ── / .gs a { color: var(--bright); text-decoration: none; border-bottom: 1px solid var(--rule); } .gs a:hover { color: var(--crimson); border-bottom-color: var(--crimson); }

/ ── Dropdown ── / .gs details { border: 1px solid var(--edge); border-left: 3px solid var(--crimson); margin-top: 24px; position: relative; background: var(--surface); box-shadow: 0 2px 12px rgba(0,0,0,0.45); } .gs details::before { content: ''; position: absolute; top: -1px; right: -1px; width: 10px; height: 10px; border-top: 1px solid var(--crimson); border-right: 1px solid var(--crimson); opacity: 0.4; } .gs details::after { content: ''; position: absolute; bottom: -1px; right: -1px; width: 10px; height: 10px; border-bottom: 1px solid var(--azure); border-right: 1px solid var(--azure); opacity: 0.3; } .gs summary { list-style: none; padding: 11px 16px; cursor: pointer; font-family: var(--mono); font-size: 0.72rem; font-weight: 700; letter-spacing: 0.12em; text-transform: uppercase; color: var(--dim); user-select: none; display: flex; align-items: center; gap: 10px; } .gs summary::-webkit-details-marker { display: none; } .gs summary::before { content: '+'; color: var(--crimson); font-size: 1rem; line-height: 1; flex-shrink: 0; } .gs details[open] summary::before { content: '−'; } .gs summary:hover { color: var(--bright); } .gs-detail-body { padding: 22px 18px; border-top: 1px solid var(--edge); } .gs-detail-body p { margin: 0 0 16px; font-size: 0.9rem; } .gs-cfg-title { font-family: var(--mono); font-size: 0.72rem; font-weight: 700; letter-spacing: 0.1em; text-transform: uppercase; color: var(--dim); margin: 0 0 8px; }

/ ── Eval table ── / .gs-etable { width: 100%; border-collapse: collapse; background: var(--surface); border: 1px solid var(--edge); border-left: 3px solid var(--crimson); box-shadow: 0 2px 12px rgba(0,0,0,0.45); } .gs-etable th { font-family: var(--mono); font-size: 0.68rem; font-weight: 700; letter-spacing: 0.14em; text-transform: uppercase; color: var(--dim); text-align: left; padding: 10px 16px; border-bottom: 1px solid var(--edge); } .gs-etable th.gs-ecol { color: var(--crimson); } .gs-etable td { padding: 9px 16px; border-bottom: 1px solid var(--edge); font-size: 0.9rem; color: var(--bright); } .gs-etable tr:last-child td { border-bottom: none; } .gs-etable td:first-child { font-family: var(--mono); font-size: 0.85rem; color: var(--dim); } .gs-etable td:nth-child(n+2) { color: var(--text); } .gs-etable td.gs-best { font-weight: 700; color: var(--bright); } .gs-etable tr.gs-egroup td { font-family: var(--mono); font-size: 0.62rem; font-weight: 700; letter-spacing: 0.16em; text-transform: uppercase; color: var(--crimson); background: var(--az-glow); padding: 7px 16px; }

/ ── Code ── / .gs pre { background: #12141b; border: 1px solid var(--edge); border-left: 2px solid var(--azure); padding: 16px 18px; overflow-x: auto; font-family: var(--mono); font-size: 0.76rem; line-height: 1.6; color: var(--text); margin: 0 0 22px; } .gs pre:last-child { margin-bottom: 0; } .gs pre code { background: none; color: inherit; padding: 0; } .gs code { font-family: var(--mono); font-size: 0.875em; color: var(--crimson); background: var(--az-glow); padding: 2px 5px; } </style> <html lang="en"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>Stardom</title> <link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;900&family=JetBrains+Mono:wght@400;700&display=swap" rel="stylesheet"> </head> <body> <div class="gs">

<div class="gs-profile"> <div class="gs-profile-art"> <img src="https://cdn-uploads.huggingface.co/production/uploads/65b19c6c638328850e12d38c/7dR919fTUTuxEHhvvRki8.png" alt="image"> <div class="gs-ident"> <h1 class="gs-name">Mero Mero v2</h1> <span class="gs-base">Gemma4 31B</span> </div> </div> </div>

<div class="gs-section"> <div class="gs-shead"> <span class="gs-snum">01</span> <span class="gs-stitle">Overview</span> </div> <div class="gs-sbody"> <p></p> <p>A finetune of Gemma 4 31B designed for creative tasks, particularly narrative RP. Intended to be a more creative version of <a href="https://huggingface.co/zerofata/G4-MeroMero-31B">G4-MeroMero-31B</a>.</p> <p>This model is the result of a lot of experimentation and learning. Trying to make Gemma 4 more creative without destroying the intelligence is... difficult. To put it mildly.</p> <p style="margin-top:16px">Heavily inspired by a few research papers, <a href="https://arxiv.org/abs/2604.03136">StoryScope: Investigating idiosyncrasies in AI fiction </a> and particularly <a href="https://arxiv.org/abs/2605.26492">Elias in the Lighthouse, Again?</a>. Measuring these narrative tics and attractors against simple prompts seems to be a good way to target the model's slop and kick start giving Gemma 4 some diversity: anything that repeatedly occurs across generations of such a generic prompt is something the model is overusing.</p> <p>Compared to the original, swipes are notably more diverse and feel less like Gemma. RP slop is measurably lower (at least for the type of slop I measure). IFEval / GSM8K / MMLU-Pro are the same as stock with no obvious degradation. The only intelligence drop I've really noticed so far is when you get a swipe that goes a bit hot.</p> <p>Supports both thinking and non thinking. Reasoning averages longer than stock Gemma 4, but shorter than MeroMero v1.</p> </div> </div>

<div class="gs-section"> <div class="gs-shead"> <span class="gs-snum">02</span> <span class="gs-stitle">SillyTavern Settings</span> </div> <div class="gs-sbody"> <div class="gs-stack"> <div class="gs-panel"> <div class="gs-panel-head">Suggested Roleplay Format</div> <div class="gs-row"><span class="gs-key">Actions</span><span class="gs-val">In plaintext</span></div> <div class="gs-row"><span class="gs-key">Dialogue</span><span class="gs-val">"In quotes"</span></div> <div class="gs-row"><span class="gs-key">Thoughts</span><span class="gs-val">In asterisks</span></div> </div> <div class="gs-panel"> <div class="gs-panel-head">Recommended Samplers</div> <div class="gs-row"><span class="gs-key">Temp</span><span class="gs-val">0.8 - 1.0</span></div> <div class="gs-row"><span class="gs-key">MinP</span><span class="gs-val">0.05</span></div> <div class="gs-row"></span><span class="gs-val"></span></div> </div> <div class="gs-panel"> <div class="gs-panel-head">Instruct</div> <div class="gs-row"><span class="gs-val"><a href="https://huggingface.co/zerofata/G4-MeroMero-v2-31B/raw/main/Gemma4-Think.json">Gemma 4 - Think</a></span></div> <div class="gs-row"><span class="gs-val"><a href="https://huggingface.co/zerofata/G4-MeroMero-v2-31B/raw/main/Gemma4-NoThink.json">Gemma 4 - NoThink</a></span></div> </div> </div> </div> </div>

<div class="gs-section gs-section--compact"> <div class="gs-shead"> <span class="gs-snum">03</span> <span class="gs-stitle">Quantizations</span> </div> <div class="gs-sbody"> <div class="gs-qrow"> <div class="gs-qpanel"> <span class="gs-qtype">GGUF</span> <div class="gs-qsep"></div> <a href="https://huggingface.co/zerofata/G4-MeroMero-v2-31B-GGUF">iMatrix</a> </div> </div> </div> </div>

<div class="gs-section"> <div class="gs-shead"> <span class="gs-snum">04</span> <span class="gs-stitle">Evaluation</span> </div> <div class="gs-sbody"> <table class="gs-etable"> <tr><th></th><th class="gs-ecol">Mero Mero v2</th><th>Mero Mero v1</th><th>Stock Gemma 4</th></tr> <tr class="gs-egroup"><td colspan="4">Swipe diversity — given an RP conversation, generate 8 swipes and evaluate how varied the beats in those swipes are, GLM-judged with a rubric.</td></tr> <tr><td>Thinking off</td><td class="gs-best">0.72</td><td>0.57</td><td>0.43</td></tr> <tr><td>Thinking on</td><td class="gs-best">0.62</td><td>0.49</td><td>0.32</td></tr> <tr class="gs-egroup"><td colspan="4">Slop & attractors — lower is better</td></tr> <tr><td>Slop per 1k words, RP replies</td><td class="gs-best">15.5</td><td>18.0</td><td>18.5</td></tr> <tr><td>Slop per 1k words, stories</td><td class="gs-best">7.4</td><td>8.3</td><td>8.8</td></tr> <tr><td>Bare-prompt stories hitting an attractor</td><td class="gs-best">66%</td><td>98%</td><td>99%</td></tr> <tr class="gs-egroup"><td colspan="4">Top attractor markers — each model's six most frequent, stories containing each of 144</td></tr> <tr><td>#1</td><td>Tuesday · 28</td><td>Elias · 96</td><td>Elias · 102</td></tr> <tr><td>#2</td><td>Arthur · 20</td><td>Tuesday · 81</td><td>Tuesday · 90</td></tr> <tr><td>#3</td><td>Elias · 19</td><td>Clara · 57</td><td>Clara · 80</td></tr> <tr><td>#4</td><td>Leo · 16</td><td>Oakhaven · 46</td><td>Oakhaven · 60</td></tr> <tr><td>#5</td><td>Elara · 14</td><td>Arthur · 21</td><td>Thorne · 23</td></tr> <tr><td>#6</td><td>Clara · 14</td><td>Leo · 20</td><td>Arthur · 16</td></tr> <tr class="gs-egroup"><td colspan="4">Thinking length — words per think block, RP replies; shorter is better</td></tr> <tr><td>Mean / median</td><td>341 / 305</td><td>382 / 342</td><td class="gs-best">263 / 253</td></tr> <tr class="gs-egroup"><td colspan="4">General benchmarks — thinking off; IFEval & GSM8K full, MMLU-Pro 40q per category</td></tr> <tr><td>IFEval</td><td class="gs-best">90.2</td><td>89.8</td><td>89.8</td></tr> <tr><td>GSM8K</td><td class="gs-best">97.0</td><td>96.1</td><td>96.7</td></tr> <tr><td>MMLU-Pro</td><td class="gs-best">85.5</td><td>85.4</td><td>84.6</td></tr> </table> </div> </div>

<div class="gs-section gs-section--journal"> <div class="gs-shead"> <span class="gs-snum">05</span> <span class="gs-stitle">Creation Process</span> </div> <div class="gs-sbody"> <p>Creation Process: SFT > Merge > GRPO > GRPO > on-policy SFT</p> <p><strong>Stage 1 — Diversity SFT.</strong> Stock Gemma 4 collapses hard on underspecified creative prompts ("Write a story." basically always gives clockmaker or memory related stories in a shop with Elias). Trained on ~4,000 short stories curated against the storyscope narrative prompts and found attractors. The dataset is a mix of human stories and synthetic stories from a set of frontier models, with diverse generation prompts swapped out for generic ones and filtered for quality. I also included some of the usual creative instruct and roleplay data. The model came out alright. Creative, but notably worse at instruction following with degraded intelligence. SLERP-merged back into the original instruct at t=0.5, which basically reverted it to stock Gemma 4 with slightly improved prose and creativity (similar to MeroMero).</p> <p><strong>Stage 2 — Creative GRPO (with think disabled).</strong> TRL GRPO (via Axolotl), 8 rollouts per prompt on the same bare prompts. Reward stack: LLM-judge diversity and coherence rewards, an attractor-marker penalty seeded from stock and then updated with whatever started appearing as new attractors during training, narrative-rate penalties and deterministic degeneracy guards (checking for non-Latin characters, joined words etc). 300 steps.</p> <p><strong>Stage 3 — RP logic GRPO (with think enabled).</strong> 100 further steps on multi-turn roleplay contexts: a thinking check to ensure it always parsed correctly, a logic-defect judge (DeepSeek-V4 Flash with a rubric), per-context attractor lists mined from k=8 baselines of the stage 2 model, and the same degeneracy checks as stage 2.</p> <p><strong>Stage 4 — On-policy multi-outcome RP SFT.</strong> ~3,300 samples the model wrote itself on roleplay contexts, filtered to keep only varied continuations. The model wasn't able to produce diverse results purely on-policy, so a natural response was generated first, then each sample went through a GLM-5.2 critique pipeline: error detection, plus steering ideas injected as OOC commands for re-generation (DeepSeek-V4-Pro / GLM-5.2 alternating). Everything was then filtered for errors, slop and any degeneracy as usual. Roughly 60% of samples include thinking. Last turn only.</p> <p>Trained using Axolotl.</p> <details> <summary>Stage 1 — Diversity SFT Config (Axolotl)</summary> <div class="gs-detail-body"> <pre><code>basemodel: google/gemma-4-31B-it   plugins: - axolotl.integrations.cutcrossentropy.CutCrossEntropyPlugin - axolotl.integrations.liger.LigerPlugin ligerlayernorm: true ligerrope: true ligerrmsnorm: true ligergluactivation: true ligerrmsnormgated: true strict: false cutcrossentropy: true   datasets: - path: ./data/diversitysftmasked.jsonl valsetsize: 0 outputdir: ./G4-31B-SFT-v10-2   sequencelen: 8192 padtosequencelen: true samplepacking: true   adapter: lora lorar: 64 loraalpha: 64 peftuserslora: true loradropout: 0.0 freezemmmodules: true loratargetmodules: 'model.languagemodel.layers.[\d]+.(checkpointwrappedmodule.)?(mlp|selfattn).(up|down|gate|q|k|v|o)proj'   gradientaccumulationsteps: 1 microbatchsize: 4 numepochs: 2 optimizer: adamwtorchfused lrscheduler: cosine learningrate: 1e-5 maxgradnorm: 1.0 warmupratio: 0.1 weightdecay: 0.05 savesperepoch: 2   bf16: auto tf32: true   # FA2 not supported sdpattention: true flashattention: false   fsdpconfig: fsdpversion: 2 offloadparams: false cpuramefficientloading: false autowrappolicy: TRANSFORMERBASEDWRAP transformerlayerclstowrap: Gemma4TextDecoderLayer statedicttype: FULLSTATEDICT shardingstrategy: FULLSHARD reshardafterforward: true activationcheckpointing: true</code></pre> </div> </details> <details> <summary>Stage 1 — Mergekit Config</summary> <div class="gs-detail-body"> <pre><code>mergemethod: slerp basemodel: google/gemma-4-31B-it models: - model: google/gemma-4-31B-it - model: ApocalypseParty/G4-31B-SFT-v10-2 parameters: t: 0.5 dtype: bfloat16</code></pre> </div> </details> <details> <summary>Stage 2 — Creative GRPO Config (Axolotl)</summary> <div class="gs-detail-body"> <pre><code>basemodel: /workspace/models/configCA # stage 1 output   rl: grpo   trl: rewardfuncs: - rewardsg4.rewardjudgediversity - rewardsg4.rewardjudgecoherence - rewardsg4.rewardattractor - rewardsg4.rewardnarrative - rewardsg4.rewardsane rewardweights: [3.0, 3.0, 0.75, 1.0, 1.0] beta: 0.02 numgenerations: 8 maxcompletionlength: 1600 temperature: 1.0 usevllm: true scalerewards: true losstype: grpo epsilon: 0.2 generationkwargs: stoptokenids: [1, 106, 50] topk: 64 topp: 0.95   datasets: - path: /workspace/data/sfttrainfinal.jsonl type: ebftchat.transform   sequencelen: 2048 microbatchsize: 2 gradientaccumulationsteps: 4 maxsteps: 200   learningrate: 4.0e-6 optimizer: adamwtorchfused lrscheduler: cosine warmupsteps: 10 weightdecay: 0.01   adapter: lora lorar: 64 loraalpha: 64 peftuserslora: true loradropout: 0.0 freezemmmodules: true loratargetmodules: 'model.languagemodel.layers.[\d]+.(checkpointwrappedmodule.)?(mlp|selfattn).(up|down|gate|q|k|v|o)proj'   maxgradnorm: 1.0 bf16: auto tf32: true sdpattention: true flashattention: false gradientcheckpointing: true   # shipped weights use checkpoint-100 of this run</code></pre> </div> </details> <details> <summary>Stage 3 — RP Logic GRPO Config (Axolotl)</summary> <div class="gs-detail-body"> <pre><code>basemodel: /workspace/models/r4b100 # stage 2 output   rl: grpo   trl: rewardfuncs: - rewardsrp.rewardthinking # format gate on the think block - rewardsrp.rewardlogic # constraint-grounded defect judge - rewardsrp.rewardattractor # frozen per-context lists from stock k=8 - rewardsrp.rewardsane # deterministic glitch guards rewardweights: [2.0, 3.0, 1.0, 1.0] beta: 0.02 numgenerations: 8 maxcompletionlength: 2560 temperature: 1.0 usevllm: true scalerewards: true losstype: grpo epsilon: 0.2 generationkwargs: stoptokenids: [1, 106, 50] topk: 64 topp: 0.95   datasets: - path: /workspace/rp/rp3train.jsonl type: ebftchat.transform   sequencelen: 8192 microbatchsize: 1 gradientaccumulationsteps: 8 maxsteps: 100   learningrate: 3.0e-6 optimizer: adamwtorchfused lrscheduler: cosine warmupsteps: 10 weightdecay: 0.01   adapter: lora lorar: 64 loraalpha: 64 peftuserslora: true loradropout: 0.0 freezemmmodules: true loratargetmodules: 'model.languagemodel.layers.[\d]+.(checkpointwrappedmodule.)?(mlp|selfattn).(up|down|gate|q|k|v|o)proj'   maxgradnorm: 1.0 bf16: auto tf32: true sdpattention: true flashattention: false gradientcheckpointing: true</code></pre> </div> </details> <details> <summary>Stage 4 — On-Policy Multi-Outcome SFT Config (Axolotl)</summary> <div class="gs-detail-body"> <pre><code>basemodel: ApocalypseParty/G4-31B-r4b100-GRPO-rp100 # stage 3 output   plugins: - axolotl.integrations.cutcrossentropy.CutCrossEntropyPlugin - axolotl.integrations.liger.LigerPlugin ligerlayernorm: true ligerrope: true ligerrmsnorm: true ligergluactivation: true ligerrmsnormgated: true strict: false cutcrossentropy: true   datasets: - path: ./data/g4onpolicyrpmasked.jsonl valsetsize: 0 outputdir: ./G4-31B-r4b100-GRPO-rp100-sft   sequencelen: 8192 padtosequencelen: true samplepacking: true   adapter: lora lorar: 64 loraalpha: 64 peftuserslora: false loradropout: 0.0 freezemmmodules: true loratargetmodules: 'model.languagemodel.layers.[\d]+.(checkpointwrappedmodule.)?(mlp|selfattn).(up|down|gate|q|k|v|o)proj'   gradientaccumulationsteps: 2 microbatchsize: 1 numepochs: 1 optimizer: adamwtorchfused lrscheduler: cosine learningrate: 4e-5 maxgradnorm: 1.0 warmupratio: 0.1 weightdecay: 0.05 savesperepoch: 2   bf16: auto tf32: true   # FA2 not supported sdpattention: true flashattention: false   fsdpconfig: fsdpversion: 2 offloadparams: false cpuramefficientloading: false autowrappolicy: TRANSFORMERBASEDWRAP transformerlayerclstowrap: Gemma4TextDecoderLayer statedicttype: FULLSTATEDICT shardingstrategy: FULLSHARD reshardafterforward: true activationcheckpointing: true</code></pre> </div> </details> </div> </div>

</div> </body> </html>