woodfireind/H3-ScriptGen
024
1#!/usr/bin/env python32"""Build chat SFT JSONL from scriptlib (+ optional TVTropes) for H3 script-LoRA.3 4Reads numbered screenplays under ../scriptlib/*.txt, splits them into short5scene-ish chunks, and emits two kinds of training rows:6 71. **format** — classic slugline/action/dialogue → MiniMax FL2VA scene block82. **premise** — short premise derived from the chunk → full H3 scene beat9 10Optionally samples TVTropes titles/tropes as extra premise seeds (no script body).11 12Output: train_dataset.full.jsonl (append or overwrite).13 14Example:15 python build_sft_from_scriptlib.py --max-scripts 102 --chunks-per-script 416"""17 18from __future__ import annotations19 20import argparse21import json22import random23import re24from pathlib import Path25 26ROOT = Path(__file__).resolve().parent27SCRIPTLIB = ROOT.parent / "scriptlib"28TROPES_TV = ROOT.parent / "TVTropesData" / "tv_tropes.csv"29TROPES_MASTER = ROOT.parent / "TVTropesData" / "tropes.csv"30OUT_DEFAULT = ROOT / "train_dataset.full.jsonl"31 32H3_SYSTEM = (33 "You write ONE MiniMax-H3 FL2VA scene beat for Backlot. "34 "Output fields: ACTION, SHOT, STORYBOARD_PROMPT, H3_MODE, H3_VIDEO_PROMPT "35 "(alignment line + integrated_multimodal_description + overall_soundscape + "36 "non_diegetic_music), LORA, AUDIO, DURATION 5. Prefer a single continuous shot "37 "from Picture 1 to Picture 2. Dialogue only inside <d>[Language] ...</d>."38)39 40SLUGLINE_RE = re.compile(41 r"^\s*(INT\.|EXT\.|INT/EXT\.|I/E\.|EST\.)\s+.+$",42 re.I | re.M,43)44CHAR_RE = re.compile(r"^\s{10,}([A-Z][A-Z0-9 .'\-]{1,40})\s*(\(.*\))?\s*$")45 46 47def split_scenes(text: str, max_chars: int = 1800) -> list[str]:48 """Split screenplay into chunks on sluglines, then size-cap."""49 text = text.replace("\r\n", "\n").replace("\r", "\n")50 # Prefer slugline splits51 parts: list[str] = []52 matches = list(SLUGLINE_RE.finditer(text))53 if matches:54 for i, m in enumerate(matches):55 start = m.start()56 end = matches[i + 1].start() if i + 1 < len(matches) else len(text)57 chunk = text[start:end].strip()58 if len(chunk) > 80:59 parts.append(chunk)60 else:61 # Seinfeld-style parenthetical locations: (Comedy club)62 blocks = re.split(r"\n\s*\([^)\n]{3,80}\)\s*\n", text)63 parts = [b.strip() for b in blocks if len(b.strip()) > 120]64 65 # Size-cap / merge66 out: list[str] = []67 for p in parts:68 if len(p) <= max_chars:69 out.append(p)70 else:71 # take head of long scene (opening beat)72 out.append(p[:max_chars].rsplit("\n", 1)[0])73 return out74 75 76def extract_title(text: str) -> str:77 for line in text.splitlines()[:25]:78 s = line.strip()79 if len(s) > 2 and s.isupper() and not s.startswith("WRITTEN"):80 return s.title()81 return "Untitled"82 83 84def extract_dialogue_pairs(chunk: str, limit: int = 2) -> list[tuple[str, str]]:85 lines = chunk.splitlines()86 pairs: list[tuple[str, str]] = []87 i = 088 while i < len(lines) and len(pairs) < limit:89 m = CHAR_RE.match(lines[i])90 if not m:91 i += 192 continue93 name = m.group(1).strip().title()94 i += 195 dial: list[str] = []96 while i < len(lines):97 L = lines[i]98 if CHAR_RE.match(L) or SLUGLINE_RE.match(L):99 break100 if L.strip().startswith("(") and L.strip().endswith(")"):101 i += 1102 continue103 if L.strip():104 dial.append(L.strip())105 elif dial:106 break107 i += 1108 if dial:109 pairs.append((name, " ".join(dial)))110 return pairs111 112 113def extract_action_lines(chunk: str, max_sents: int = 2) -> str:114 acts: list[str] = []115 for line in chunk.splitlines():116 s = line.strip()117 if not s or CHAR_RE.match(line) or SLUGLINE_RE.match(line):118 continue119 if s.startswith("(") and s.endswith(")"):120 continue121 # skip all-caps character names122 if s.isupper() and len(s) < 40:123 continue124 if len(s) > 20:125 acts.append(s)126 if len(acts) >= max_sents:127 break128 return " ".join(acts) if acts else "The scene plays out continuously."129 130 131def slugline_from_chunk(chunk: str) -> str:132 m = SLUGLINE_RE.search(chunk)133 if m:134 return m.group(0).strip().upper()135 # parenthetical location136 m2 = re.search(r"\(([^)\n]{3,60})\)", chunk)137 if m2:138 return f"INT. {m2.group(1).upper()}"139 return "INT. LOCATION - DAY"140 141 142def chunk_to_h3_assistant(chunk: str, title: str) -> str:143 slug = slugline_from_chunk(chunk)144 action = extract_action_lines(chunk)145 dialogue = extract_dialogue_pairs(chunk, limit=1)146 # Compress action for ~5s beat147 action_short = action148 if len(action_short) > 280:149 action_short = action_short[:280].rsplit(" ", 1)[0] + "."150 151 dial_line = ""152 multimodal_extra = ""153 if dialogue:154 name, line = dialogue[0]155 # keep dialogue short156 if len(line) > 160:157 line = line[:160].rsplit(" ", 1)[0] + "..."158 dial_line = f"DIALOGUE — {name}: {line}\n"159 multimodal_extra = (160 f" {name} (S1) says: <d>[English] {line}</d>"161 )162 163 storyboard = (164 f"cinematic still from {title}: {action_short[:200]}, "165 f"detailed environment, live-action, film lighting"166 )167 168 body = (169 f"[Shot 1] Live-action, cinematic, medium shot establishing the scene. "170 f"{action_short} The camera pushes in with small amplitude at slow speed."171 f"{multimodal_extra} "172 f"The framing begins on the composition of Picture 1 and continuously "173 f"evolves until it settles into the composition of Picture 2."174 )175 176 return (177 f"## SCENE 1 — {slug}\n"178 f"ACTION: {action_short}\n"179 f"{dial_line}"180 f"SHOT: medium shot, push in with small amplitude at slow speed\n"181 f"STORYBOARD_PROMPT: {storyboard}\n"182 f"H3_MODE: FL2VA\n"183 f"H3_VIDEO_PROMPT:\n"184 f"How the reference pictures align with the target video — Picture 1 "185 f"(from Shot 1) aligns with the 0.00-second mark of the target video; "186 f"Picture 2 (from Shot 1) aligns with the 5.00-second mark of the target video.\n\n"187 f"integrated_multimodal_description: {body}\n\n"188 f"overall_soundscape: Room tone and soft environmental ambience matching the location.\n\n"189 f"non_diegetic_music: N/A\n"190 f"LORA: none\n"191 f"AUDIO: none\n"192 f"DURATION: 5"193 )194 195 196def row(messages: list[dict]) -> str:197 return json.dumps({"messages": messages}, ensure_ascii=False)198 199 200def build_from_scripts(201 script_dir: Path,202 *,203 max_scripts: int,204 chunks_per_script: int,205 rng: random.Random,206) -> list[str]:207 files = sorted(script_dir.glob("*.txt"), key=lambda p: int(p.stem) if p.stem.isdigit() else p.stem)208 if max_scripts > 0:209 files = files[:max_scripts]210 rows: list[str] = []211 for path in files:212 try:213 text = path.read_text(encoding="utf-8", errors="replace")214 except OSError:215 continue216 title = extract_title(text)217 chunks = split_scenes(text)218 if not chunks:219 continue220 rng.shuffle(chunks)221 for chunk in chunks[:chunks_per_script]:222 assistant = chunk_to_h3_assistant(chunk, title)223 # Format transfer: classic excerpt → H3 beat224 rows.append(225 row(226 [227 {"role": "system", "content": H3_SYSTEM},228 {229 "role": "user",230 "content": (231 f"Rewrite this screenplay beat as a single ~5s MiniMax-H3 "232 f"FL2VA scene for Backlot (first storyboard panel → last panel).\n\n"233 f"SOURCE TITLE: {title}\n\n"234 f"SCREENPLAY EXCERPT:\n{chunk[:1600]}"235 ),236 },237 {"role": "assistant", "content": assistant},238 ]239 )240 )241 # Premise → scene (trope-style)242 premise = extract_action_lines(chunk, max_sents=1)243 rows.append(244 row(245 [246 {"role": "system", "content": H3_SYSTEM},247 {248 "role": "user",249 "content": (250 f"Premise: {premise}\n"251 f"Setting: derived from {title}\n"252 f"Tone: cinematic\n"253 f"Write SCENE 1 now (H3 FL2VA, DURATION 5)."254 ),255 },256 {"role": "assistant", "content": assistant},257 ]258 )259 )260 return rows261 262 263def build_from_tropes(path: Path, *, n: int, rng: random.Random) -> list[str]:264 if not path.exists() or n <= 0:265 return []266 # Stream a sample of lines (file is large)267 import csv268 269 rows_out: list[str] = []270 with path.open(encoding="utf-8", errors="replace", newline="") as f:271 reader = csv.DictReader(f)272 # sample reservoir273 reservoir: list[dict] = []274 for i, rec in enumerate(reader):275 if i < 5000:276 reservoir.append(rec)277 else:278 j = rng.randint(0, i)279 if j < 5000:280 reservoir[j] = rec281 if i > 200_000: # don't scan entire multi-hundred-MB file282 break283 rng.shuffle(reservoir)284 for rec in reservoir[:n]:285 title = (rec.get("Title") or rec.get("title") or "Untitled").strip()286 trope = (rec.get("Trope") or rec.get("trope") or "PlotTwist").strip()287 example = (rec.get("Example") or rec.get("Description") or "").strip()288 if len(example) > 400:289 example = example[:400] + "..."290 premise = f"A scene in {title} illustrating the trope '{trope}'. {example}"291 # Lightweight target (model will learn shape from script rows primarily)292 assistant = (293 f"## SCENE 1 — INT. SETTING - DAY\n"294 f"ACTION: Characters enact a brief beat embodying {trope}.\n"295 f"SHOT: medium shot, static shot\n"296 f"STORYBOARD_PROMPT: cinematic still for {title}, {trope}, live-action\n"297 f"H3_MODE: FL2VA\n"298 f"H3_VIDEO_PROMPT:\n"299 f"How the reference pictures align with the target video — Picture 1 "300 f"(from Shot 1) aligns with the 0.00-second mark of the target video; "301 f"Picture 2 (from Shot 1) aligns with the 5.00-second mark of the target video.\n\n"302 f"integrated_multimodal_description: [Shot 1] Live-action, cinematic, a medium "303 f"shot introduces the situation for {trope}. The camera holds a static shot as "304 f"the beat resolves into the final composition of Picture 2.\n\n"305 f"overall_soundscape: Soft room tone.\n\n"306 f"non_diegetic_music: N/A\n"307 f"LORA: none\n"308 f"AUDIO: none\n"309 f"DURATION: 5"310 )311 rows_out.append(312 row(313 [314 {"role": "system", "content": H3_SYSTEM},315 {316 "role": "user",317 "content": f"Premise: {premise}\nWrite SCENE 1 now (H3 FL2VA, DURATION 5).",318 },319 {"role": "assistant", "content": assistant},320 ]321 )322 )323 return rows_out324 325 326def main() -> None:327 ap = argparse.ArgumentParser()328 ap.add_argument("--scriptlib", type=Path, default=SCRIPTLIB)329 ap.add_argument("--out", type=Path, default=OUT_DEFAULT)330 ap.add_argument("--max-scripts", type=int, default=0, help="0 = all")331 ap.add_argument("--chunks-per-script", type=int, default=4)332 ap.add_argument("--tropes", type=int, default=80, help="extra TVTropes premise rows")333 ap.add_argument("--seed", type=int, default=42)334 ap.add_argument("--include-seed", action="store_true", help="prepend train_dataset.jsonl")335 args = ap.parse_args()336 337 rng = random.Random(args.seed)338 if not args.scriptlib.is_dir():339 raise SystemExit(f"scriptlib not found: {args.scriptlib}")340 341 rows = build_from_scripts(342 args.scriptlib,343 max_scripts=args.max_scripts,344 chunks_per_script=args.chunks_per_script,345 rng=rng,346 )347 rows += build_from_tropes(TROPES_TV, n=args.tropes, rng=rng)348 349 seed_path = ROOT / "train_dataset.jsonl"350 if args.include_seed and seed_path.exists():351 seed_rows = [ln for ln in seed_path.read_text().splitlines() if ln.strip()]352 rows = seed_rows + rows353 354 rng.shuffle(rows)355 args.out.parent.mkdir(parents=True, exist_ok=True)356 args.out.write_text("\n".join(rows) + "\n", encoding="utf-8")357 print(f"wrote {len(rows)} rows → {args.out}")358 359 360if __name__ == "__main__":361 main()362