Spatial9/GravityLLM
0
1from __future__ import annotations2 3import base644import json5import os6import tempfile7from pathlib import Path8from typing import Any, Dict9 10import gradio as gr11from huggingface_hub import InferenceClient12from huggingface_hub.errors import HfHubHTTPError, InferenceTimeoutError13 14from utils.scene_tools import (15 SCHEMA,16 extract_first_json_block,17 heuristic_scene,18 parse_json_text,19 plot_scene,20 scene_markdown,21 scene_table,22 validate_scene,23)24 25ROOT = Path(__file__).resolve().parent26ASSETS = ROOT / "assets"27EXAMPLES = ROOT / "examples"28 29APP_TITLE = "GravityLLM"30DEFAULT_MODEL_ID = os.getenv("GRAVITYLLM_MODEL_ID", "your-namespace/GravityLLM-AutoPosition")31DEFAULT_BACKEND = os.getenv("GRAVITYLLM_BACKEND", "hybrid")32HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")33 34SYSTEM_PREFIX = (35 "You are GravityLLM (Spatial9 AutoPosition SLM). "36 "Generate ONLY valid JSON matching the Spatial9Scene schema. "37 "No markdown. No explanation. No code fences.\n\n"38)39 40EXAMPLE_FILES = {41 "Club Drop": EXAMPLES / "club_drop.json",42 "Cinematic Break": EXAMPLES / "cinematic_break.json",43 "Podcast Voice": EXAMPLES / "podcast_voice.json",44}45 46 47def logo_data_uri() -> str:48 logo_bytes = (ASSETS / "spatial9_logo.png").read_bytes()49 return "data:image/png;base64," + base64.b64encode(logo_bytes).decode("utf-8")50 51 52def load_example(name: str) -> str:53 path = EXAMPLE_FILES.get(name, next(iter(EXAMPLE_FILES.values())))54 return path.read_text(encoding="utf-8")55 56 57def build_prompt(payload: Dict[str, Any]) -> str:58 return SYSTEM_PREFIX + "INPUT:\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\nOUTPUT:\n"59 60 61def remote_generate(62 payload: Dict[str, Any],63 model_id: str,64 temperature: float,65 top_p: float,66 max_new_tokens: int,67 use_grammar: bool,68) -> tuple[Dict[str, Any], str]:69 prompt = build_prompt(payload)70 client = InferenceClient(model=model_id, token=HF_TOKEN)71 72 call_kwargs = dict(73 prompt=prompt,74 model=model_id,75 max_new_tokens=max_new_tokens,76 temperature=temperature,77 top_p=top_p,78 repetition_penalty=1.05,79 return_full_text=False,80 )81 if use_grammar:82 call_kwargs["grammar"] = {"type": "json", "value": SCHEMA}83 84 try:85 response = client.text_generation(**call_kwargs)86 scene = parse_json_text(response)87 return scene, f"remote-model ({model_id})"88 except Exception as first_error:89 if use_grammar:90 try:91 call_kwargs.pop("grammar", None)92 response = client.text_generation(**call_kwargs)93 scene = parse_json_text(response)94 return scene, f"remote-model ({model_id}, grammar-fallback)"95 except Exception as second_error:96 raise RuntimeError(f"{type(first_error).__name__}: {first_error}\n\nFallback: {type(second_error).__name__}: {second_error}") from second_error97 raise98 99 100def write_download_file(scene: Dict[str, Any]) -> str:101 fd, path = tempfile.mkstemp(prefix="gravityllm_scene_", suffix=".json")102 os.close(fd)103 Path(path).write_text(json.dumps(scene, ensure_ascii=False, indent=2), encoding="utf-8")104 return path105 106 107def generate_scene(108 payload_text: str,109 model_id: str,110 backend: str,111 temperature: float,112 top_p: float,113 max_new_tokens: int,114 use_grammar: bool,115):116 try:117 payload = parse_json_text(payload_text)118 except Exception as exc:119 msg = f"### Invalid input JSON\n\n- {type(exc).__name__}: {exc}"120 return "", msg, None, [], None, "Fix the input JSON and try again."121 122 backend = backend or DEFAULT_BACKEND123 model_id = model_id.strip() or DEFAULT_MODEL_ID124 125 scene = None126 backend_used = "rules-engine demo"127 status = "Scene generated."128 129 if backend in {"remote-model", "hybrid"}:130 try:131 scene, backend_used = remote_generate(payload, model_id, temperature, top_p, max_new_tokens, use_grammar)132 status = f"Generated from remote model: {model_id}"133 except (InferenceTimeoutError, HfHubHTTPError, RuntimeError, ValueError, json.JSONDecodeError) as exc:134 if backend == "remote-model":135 msg = f"### Remote generation failed\n\n- {type(exc).__name__}: {exc}"136 return "", msg, None, [], None, "Remote inference failed."137 scene = heuristic_scene(payload)138 backend_used = "rules-engine demo (remote fallback)"139 status = f"Remote generation failed; heuristic fallback used. Details: {type(exc).__name__}: {exc}"140 141 if scene is None:142 scene = heuristic_scene(payload)143 144 valid, errors = validate_scene(scene)145 download_path = write_download_file(scene)146 figure = plot_scene(scene)147 table = scene_table(scene)148 summary = scene_markdown(scene, valid, errors, backend_used)149 return json.dumps(scene, ensure_ascii=False, indent=2), summary, figure, table, download_path, status150 151 152def validate_only(scene_text: str):153 try:154 scene = parse_json_text(scene_text)155 except Exception as exc:156 return f"### Invalid scene JSON\n\n- {type(exc).__name__}: {exc}", None, []157 valid, errors = validate_scene(scene)158 summary = scene_markdown(scene, valid, errors, "manual validation")159 return summary, plot_scene(scene), scene_table(scene)160 161 162def build_payload(target_format, style, section, bpm, energy, max_objects):163 payload = {164 "target_format": target_format,165 "max_objects": int(max_objects),166 "style": style,167 "section": section,168 "global": {"bpm": int(bpm), "energy": float(energy)},169 "stems": [170 {"id": "lead", "class": "lead_vocal", "lufs": -17.0, "transient": 0.25, "band_energy": {"low": 0.08, "mid": 0.67, "high": 0.25}, "leadness": 0.96},171 {"id": "kick", "class": "kick", "lufs": -10.6, "transient": 0.96, "band_energy": {"low": 0.82, "mid": 0.12, "high": 0.06}, "leadness": 0.22},172 {"id": "bass", "class": "bass", "lufs": -12.5, "transient": 0.58, "band_energy": {"low": 0.86, "mid": 0.10, "high": 0.04}, "leadness": 0.30},173 {"id": "pad", "class": "pad", "lufs": -21.5, "transient": 0.05, "band_energy": {"low": 0.20, "mid": 0.50, "high": 0.30}, "leadness": 0.08},174 {"id": "fx", "class": "fx", "lufs": -24.0, "transient": 0.22, "band_energy": {"low": 0.10, "mid": 0.24, "high": 0.66}, "leadness": 0.04},175 ],176 "rules": [177 {"type": "anchor", "track_class": "lead_vocal", "az_deg": 0, "el_deg": 10, "dist_m": 1.6},178 {"type": "mono_low_end", "hz_below": 120},179 {"type": "width_pref", "track_class": "pad", "min_width": 0.75},180 ],181 }182 return json.dumps(payload, ensure_ascii=False, indent=2)183 184 185hero_html = f"""186<div class="hero-wrap">187 <div class="hero-left">188 <div class="hero-logo-card">189 <img class="hero-logo" src="{logo_data_uri()}" alt="Spatial9 logo"/>190 </div>191 </div>192 <div class="hero-right">193 <div class="eyebrow">SPATIAL9 • HUGGING FACE SPACE</div>194 <h1>GravityLLM Studio</h1>195 <p class="hero-copy">196 Constraint-conditioned immersive scene generation with schema-guided JSON output,197 remote Hugging Face inference, heuristic fallback, and a live spatial preview.198 </p>199 <div class="hero-chips">200 <span>IAMF Ready</span>201 <span>Schema Validated</span>202 <span>Spatial Preview</span>203 <span>Branded Demo</span>204 </div>205 </div>206</div>207"""208 209css = """210:root {211 --g-bg: #f6f9ff;212 --g-panel: rgba(255,255,255,0.86);213 --g-panel-strong: rgba(255,255,255,0.96);214 --g-line: #dbe7f6;215 --g-ink: #15233d;216 --g-sub: #5f728f;217 --g-accent: #1f6fe5;218 --g-accent-2: #0f9bb9;219}220.gradio-container {221 background:222 radial-gradient(circle at top left, rgba(55,120,246,0.10), transparent 32%),223 radial-gradient(circle at bottom right, rgba(15,155,185,0.10), transparent 28%),224 var(--g-bg);225}226.hero-wrap {227 display: grid;228 grid-template-columns: 280px 1fr;229 gap: 28px;230 padding: 22px 8px 12px 8px;231 align-items: center;232}233.hero-logo-card {234 background: linear-gradient(180deg, rgba(255,255,255,0.96), rgba(247,250,255,0.90));235 border: 1px solid var(--g-line);236 box-shadow: 0 18px 42px rgba(31,58,114,0.08);237 border-radius: 28px;238 padding: 24px;239 display: flex;240 justify-content: center;241 align-items: center;242 min-height: 170px;243}244.hero-logo {245 width: 100%;246 max-width: 220px;247 object-fit: contain;248}249.hero-right h1 {250 font-size: 2.5rem;251 margin: 0;252 color: var(--g-ink);253}254.hero-copy {255 color: var(--g-sub);256 font-size: 1.06rem;257 line-height: 1.6;258 max-width: 780px;259}260.eyebrow {261 color: var(--g-accent);262 font-size: 0.92rem;263 letter-spacing: 0.14em;264 font-weight: 700;265 margin-bottom: 8px;266}267.hero-chips {268 display: flex;269 flex-wrap: wrap;270 gap: 10px;271 margin-top: 14px;272}273.hero-chips span {274 background: rgba(239,246,255,0.96);275 border: 1px solid #cfe0fb;276 color: #28558f;277 border-radius: 999px;278 padding: 8px 12px;279 font-size: 0.9rem;280 font-weight: 600;281}282.card-note {283 color: var(--g-sub);284}285.block-panel {286 background: var(--g-panel);287 border: 1px solid var(--g-line);288 border-radius: 22px;289 padding: 10px;290}291footer {visibility: hidden;}292@media (max-width: 900px) {293 .hero-wrap {grid-template-columns: 1fr;}294}295"""296 297with gr.Blocks(298 title=f"{APP_TITLE} Studio",299 fill_width=True,300 css=css,301 theme=gr.themes.Soft(302 primary_hue="blue",303 secondary_hue="cyan",304 neutral_hue="slate",305 radius_size="lg",306 ),307) as demo:308 gr.HTML(hero_html)309 310 with gr.Tabs():311 with gr.Tab("GravityLLM Studio"):312 with gr.Row():313 with gr.Column(scale=11):314 example_name = gr.Dropdown(315 choices=list(EXAMPLE_FILES.keys()),316 value="Club Drop",317 label="Example payload",318 )319 load_btn = gr.Button("Load Example", variant="secondary")320 payload_box = gr.Code(321 value=load_example("Club Drop"),322 language="json",323 label="Constraint + stem feature payload",324 lines=26,325 )326 327 with gr.Column(scale=6):328 model_id = gr.Textbox(329 value=DEFAULT_MODEL_ID,330 label="Model repo or endpoint",331 info="Set your Hugging Face model repo id or inference endpoint URL.",332 )333 backend = gr.Dropdown(334 choices=["hybrid", "remote-model", "rules-engine demo"],335 value=DEFAULT_BACKEND if DEFAULT_BACKEND in {"hybrid", "remote-model", "rules-engine demo"} else "hybrid",336 label="Backend",337 )338 temperature = gr.Slider(0.0, 1.2, value=0.2, step=0.05, label="Temperature")339 top_p = gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="Top-p")340 max_new_tokens = gr.Slider(128, 1400, value=900, step=16, label="Max new tokens")341 use_grammar = gr.Checkbox(342 value=True,343 label="Use JSON schema grammar when remote backend supports it",344 )345 run_btn = gr.Button("Generate Spatial Scene", variant="primary")346 status = gr.Textbox(label="Status", interactive=False)347 348 with gr.Row():349 with gr.Column(scale=9):350 output_box = gr.Code(language="json", label="Generated Spatial9Scene JSON", lines=26)351 download = gr.File(label="Download scene JSON")352 with gr.Column(scale=7):353 summary = gr.Markdown("### Ready\n\nLoad an example or paste your own payload.")354 plot = gr.Plot(label="Spatial scene preview")355 356 object_table = gr.Dataframe(357 headers=["id", "class", "az_deg", "el_deg", "dist_m", "width", "gain_db"],358 datatype=["str", "str", "number", "number", "number", "number", "number"],359 row_count=(0, "dynamic"),360 col_count=(7, "fixed"),361 label="Object inspector",362 )363 364 with gr.Tab("Prompt Builder"):365 gr.Markdown("Build a starter payload, then send it to GravityLLM Studio.")366 with gr.Row():367 target_format = gr.Dropdown(["iamf", "binaural", "5.1.4", "7.1.4"], value="iamf", label="Target format")368 style = gr.Dropdown(["club", "cinematic", "podcast", "live", "intimate"], value="club", label="Style")369 section = gr.Dropdown(["intro", "verse", "break", "drop", "full"], value="drop", label="Section")370 with gr.Row():371 bpm = gr.Slider(0, 200, value=128, step=1, label="BPM")372 energy = gr.Slider(0.0, 1.0, value=0.92, step=0.01, label="Energy")373 max_objects_builder = gr.Slider(1, 32, value=10, step=1, label="Max objects")374 build_btn = gr.Button("Build Payload", variant="primary")375 builder_output = gr.Code(language="json", label="Starter payload", lines=24)376 send_to_studio_btn = gr.Button("Send to Studio", variant="secondary")377 378 with gr.Tab("Validate Existing Scene"):379 scene_input = gr.Code(language="json", label="Paste a Spatial9Scene JSON", lines=24)380 validate_btn = gr.Button("Validate Scene", variant="primary")381 validate_summary = gr.Markdown()382 validate_plot = gr.Plot()383 validate_table = gr.Dataframe(384 headers=["id", "class", "az_deg", "el_deg", "dist_m", "width", "gain_db"],385 datatype=["str", "str", "number", "number", "number", "number", "number"],386 row_count=(0, "dynamic"),387 col_count=(7, "fixed"),388 label="Validated object inspector",389 )390 391 with gr.Tab("About"):392 gr.Image(value=str(ASSETS / "gravityllm_space_banner.png"), label="GravityLLM banner", show_download_button=False, show_fullscreen_button=False)393 gr.Markdown(394 """395### What this Space does396 397- Turns **constraints + stem descriptors** into **Spatial9Scene JSON**398- Can call a remote Hugging Face model repo through `InferenceClient`399- Falls back to a deterministic **rules engine** so the demo stays usable400- Validates outputs against the included JSON schema401- Renders a spatial top-down preview of object positions402 403### Environment variables404 405- `GRAVITYLLM_MODEL_ID` — model repo id or endpoint URL406- `HF_TOKEN` — required if the model is gated or private407- `GRAVITYLLM_BACKEND` — optional default: `hybrid`, `remote-model`, or `rules-engine demo`408 409### Recommended setup410 4111. Upload your GravityLLM model repo.4122. Train and push weights.4133. Upload this Space repo.4144. Set `GRAVITYLLM_MODEL_ID` in the Space settings.415 """416 )417 418 load_btn.click(fn=load_example, inputs=example_name, outputs=payload_box)419 build_btn.click(420 fn=build_payload,421 inputs=[target_format, style, section, bpm, energy, max_objects_builder],422 outputs=builder_output,423 )424 send_to_studio_btn.click(fn=lambda x: x, inputs=builder_output, outputs=payload_box)425 run_btn.click(426 fn=generate_scene,427 inputs=[payload_box, model_id, backend, temperature, top_p, max_new_tokens, use_grammar],428 outputs=[output_box, summary, plot, object_table, download, status],429 )430 validate_btn.click(431 fn=validate_only,432 inputs=scene_input,433 outputs=[validate_summary, validate_plot, validate_table],434 )435 436if __name__ == "__main__":437 demo.launch()438