luuuulinnnn/scene-extrapolation-benchmark
Scene-Extrapolation Benchmark Suite A unified, honest evaluation of generative novel-view synthesis (NVS) in the regimes where per-scene 3D-Gaussian-Splatting reconstruction fails: Extrapolative — held-out, unobserved views (not interpolation between dense captures). Dynamic — moving foreground content. Long-horizon — chained / loop-closing camera trajectories where drift accumulates. Memory / revisit — recall when the camera returns to a previously-observed pose (do methods… See the full description on the dataset page: https://huggingface.co/datasets/luuuulinnnn/scene-extrapolation-benchmark.
0815
1{2 "axes": [3 {4 "name": "Static",5 "span": 16 },7 {8 "name": "Dynamic",9 "span": 410 },11 {12 "name": "Long-horizon",13 "span": 214 }15 ],16 "columns": [17 {18 "id": "eval50",19 "label": "DL3DV static (eval50)",20 "benchdir": "eval50",21 "total": 50,22 "axis": "Static"23 },24 {25 "id": "dyn_nvidia",26 "label": "NVIDIA (secondary)",27 "benchdir": "dyn_nvidia",28 "total": 24,29 "axis": "Dynamic"30 },31 {32 "id": "dyn_kubric4d",33 "label": "Kubric-4D (GCD)",34 "benchdir": "dyn_kubric4d",35 "total": 103,36 "axis": "Dynamic"37 },38 {39 "id": "dyn_spatialvid",40 "label": "SpatialVID",41 "benchdir": "dyn_spatialvid",42 "total": 30,43 "axis": "Dynamic"44 },45 {46 "id": "lh_dl3dv",47 "label": "LH-DL3DV drift/loop",48 "benchdir": "lh_dl3dv",49 "total": 42,50 "axis": "Long-horizon"51 },52 {53 "id": "worldscore",54 "label": "WorldScore",55 "benchdir": "worldscore",56 "total": 8,57 "axis": "Long-horizon"58 }59 ],60 "groups": [61 {62 "name": "Ours (warp+inpaint VACE)",63 "methods": [64 {65 "id": "vace14b",66 "label": "VACE-14B (ours)"67 },68 {69 "id": "vace14b_ft_lh",70 "label": "VACE-14B ft (long-horizon)"71 },72 {73 "id": "vace14b_ft_dyn",74 "label": "VACE-14B ft (dynamic)"75 },76 {77 "id": "vace14b_ft_mixed",78 "label": "VACE-14B ft (mixed)"79 },80 {81 "id": "i2v_base",82 "label": "I2V base"83 },84 {85 "id": "i2v_ft",86 "label": "I2V ft"87 }88 ]89 },90 {91 "name": "Diffusion baselines",92 "methods": [93 {94 "id": "gen3c",95 "label": "GEN3C"96 },97 {98 "id": "voyager",99 "label": "Voyager"100 },101 {102 "id": "flexworld",103 "label": "FlexWorld"104 },105 {106 "id": "viewcrafter",107 "label": "ViewCrafter"108 },109 {110 "id": "framecrafter",111 "label": "FrameCrafter"112 },113 {114 "id": "nvs_solver",115 "label": "NVS-Solver"116 },117 {118 "id": "viewextrapolator",119 "label": "ViewExtrapolator"120 },121 {122 "id": "trajcrafter",123 "label": "TrajCrafter"124 }125 ]126 },127 {128 "name": "Per-scene 3DGS",129 "methods": [130 {131 "id": "shape_of_motion",132 "label": "Shape-of-Motion (4D-GS)"133 }134 ]135 },136 {137 "name": "Long-horizon SOTA (world models)",138 "methods": [139 {140 "id": "lyra2",141 "label": "Lyra-2"142 },143 {144 "id": "matrix3d",145 "label": "Matrix3D"146 },147 {148 "id": "spmem",149 "label": "SpatialMem"150 },151 {152 "id": "vmem",153 "label": "VMem"154 },155 {156 "id": "matrixgame3",157 "label": "MatrixGame-3"158 }159 ]160 }161 ],162 "highlight": "vace14b",163 "matrix": {164 "vace14b": {165 "eval50": {166 "status": "scored",167 "text": "\u2713 scored (n=50)",168 "n": 50169 },170 "dyn_nvidia": {171 "status": "scored",172 "text": "\u2713 scored (n=18)",173 "n": 18174 },175 "dyn_kubric4d": {176 "status": "scored",177 "text": "\u2713 scored (n=91)",178 "n": 91179 },180 "dyn_spatialvid": {181 "status": "scored",182 "text": "\u2713 scored (n=30)",183 "n": 30184 },185 "lh_dl3dv": {186 "status": "scored",187 "text": "\u2713 scored (n=42)",188 "n": 42189 },190 "worldscore": {191 "status": "scored",192 "text": "\u2713 scored (n=8)",193 "n": 8194 }195 },196 "vace14b_ft_lh": {197 "eval50": {198 "status": "pending",199 "text": "\u2014 pending",200 "reason": "",201 "n": null202 },203 "dyn_nvidia": {204 "status": "pending",205 "text": "\u2014 pending",206 "reason": "",207 "n": null208 },209 "dyn_kubric4d": {210 "status": "pending",211 "text": "\u2014 pending",212 "reason": "",213 "n": null214 },215 "dyn_spatialvid": {216 "status": "pending",217 "text": "\u2014 pending",218 "reason": "",219 "n": null220 },221 "lh_dl3dv": {222 "status": "running",223 "text": "\ud83d\udd04 running (gen: 3/42, ft@4000)",224 "n": null,225 "gen": 3226 },227 "worldscore": {228 "status": "pending",229 "text": "\u2014 pending",230 "reason": "",231 "n": null232 }233 },234 "vace14b_ft_dyn": {235 "eval50": {236 "status": "pending",237 "text": "\u2014 pending",238 "reason": "",239 "n": null240 },241 "dyn_nvidia": {242 "status": "pending",243 "text": "\u2014 pending",244 "reason": "",245 "n": null246 },247 "dyn_kubric4d": {248 "status": "pending",249 "text": "\u2014 pending",250 "reason": "",251 "n": null252 },253 "dyn_spatialvid": {254 "status": "pending",255 "text": "\u2014 pending",256 "reason": "",257 "n": null258 },259 "lh_dl3dv": {260 "status": "pending",261 "text": "\u2014 pending",262 "reason": "",263 "n": null264 },265 "worldscore": {266 "status": "pending",267 "text": "\u2014 pending",268 "reason": "",269 "n": null270 }271 },272 "vace14b_ft_mixed": {273 "eval50": {274 "status": "pending",275 "text": "\u2014 pending",276 "reason": "",277 "n": null278 },279 "dyn_nvidia": {280 "status": "pending",281 "text": "\u2014 pending",282 "reason": "",283 "n": null284 },285 "dyn_kubric4d": {286 "status": "pending",287 "text": "\u2014 pending",288 "reason": "",289 "n": null290 },291 "dyn_spatialvid": {292 "status": "pending",293 "text": "\u2014 pending",294 "reason": "",295 "n": null296 },297 "lh_dl3dv": {298 "status": "pending",299 "text": "\u2014 pending",300 "reason": "",301 "n": null302 },303 "worldscore": {304 "status": "pending",305 "text": "\u2014 pending",306 "reason": "",307 "n": null308 }309 },310 "i2v_base": {311 "eval50": {312 "status": "scored",313 "text": "\u2713 scored (n=34)",314 "n": 34315 },316 "dyn_nvidia": {317 "status": "scored",318 "text": "\u2713 scored (n=21)",319 "n": 21320 },321 "dyn_kubric4d": {322 "status": "scored",323 "text": "\u2713 scored (n=90)",324 "n": 90325 },326 "dyn_spatialvid": {327 "status": "scored",328 "text": "\u2713 scored (n=29)",329 "n": 29330 },331 "lh_dl3dv": {332 "status": "running",333 "text": "\ud83d\udd04 running (gen: 19/42)",334 "n": null,335 "gen": 19336 },337 "worldscore": {338 "status": "pending",339 "text": "\u2014 pending",340 "reason": "",341 "n": null342 }343 },344 "i2v_ft": {345 "eval50": {346 "status": "pending",347 "text": "\u2014 pending",348 "reason": "",349 "n": null350 },351 "dyn_nvidia": {352 "status": "pending",353 "text": "\u2014 pending",354 "reason": "",355 "n": null356 },357 "dyn_kubric4d": {358 "status": "pending",359 "text": "\u2014 pending",360 "reason": "",361 "n": null362 },363 "dyn_spatialvid": {364 "status": "pending",365 "text": "\u2014 pending",366 "reason": "",367 "n": null368 },369 "lh_dl3dv": {370 "status": "pending",371 "text": "\u2014 pending",372 "reason": "",373 "n": null374 },375 "worldscore": {376 "status": "pending",377 "text": "\u2014 pending",378 "reason": "",379 "n": null380 }381 },382 "gen3c": {383 "eval50": {384 "status": "scored",385 "text": "\u2713 scored (n=50)",386 "n": 50387 },388 "dyn_nvidia": {389 "status": "scored",390 "text": "\u2713 scored (n=21)",391 "n": 21392 },393 "dyn_kubric4d": {394 "status": "scored",395 "text": "\u2713 scored (n=100)",396 "n": 100397 },398 "dyn_spatialvid": {399 "status": "scored",400 "text": "\u2713 scored (n=20)",401 "n": 20402 },403 "lh_dl3dv": {404 "status": "scored",405 "text": "\u2713 scored (n=42)",406 "n": 42407 },408 "worldscore": {409 "status": "scored",410 "text": "\u2713 scored (n=8)",411 "n": 8412 }413 },414 "voyager": {415 "eval50": {416 "status": "scored",417 "text": "\u2713 scored (n=49)",418 "n": 49419 },420 "dyn_nvidia": {421 "status": "scored",422 "text": "\u2713 scored (n=18)",423 "n": 18424 },425 "dyn_kubric4d": {426 "status": "scored",427 "text": "\u2713 scored (n=5)",428 "n": 5429 },430 "dyn_spatialvid": {431 "status": "scored",432 "text": "\u2713 scored (n=30)",433 "n": 30434 },435 "lh_dl3dv": {436 "status": "scored",437 "text": "\u2713 scored (n=33)",438 "n": 33439 },440 "worldscore": {441 "status": "scored",442 "text": "\u2713 scored (n=8)",443 "n": 8444 }445 },446 "flexworld": {447 "eval50": {448 "status": "scored",449 "text": "\u2713 scored (n=50)",450 "n": 50451 },452 "dyn_nvidia": {453 "status": "scored",454 "text": "\u2713 scored (n=18)",455 "n": 18456 },457 "dyn_kubric4d": {458 "status": "scored",459 "text": "\u2713 scored (n=100)",460 "n": 100461 },462 "dyn_spatialvid": {463 "status": "pending",464 "text": "\u2014 pending",465 "reason": "",466 "n": null467 },468 "lh_dl3dv": {469 "status": "scored",470 "text": "\u2713 scored (n=42)",471 "n": 42472 },473 "worldscore": {474 "status": "scored",475 "text": "\u2713 scored (n=8)",476 "n": 8477 }478 },479 "viewcrafter": {480 "eval50": {481 "status": "scored",482 "text": "\u2713 scored (n=36)",483 "n": 36484 },485 "dyn_nvidia": {486 "status": "scored",487 "text": "\u2713 scored (n=21)",488 "n": 21489 },490 "dyn_kubric4d": {491 "status": "scored",492 "text": "\u2713 scored (n=100)",493 "n": 100494 },495 "dyn_spatialvid": {496 "status": "scored",497 "text": "\u2713 scored (n=29)",498 "n": 29499 },500 "lh_dl3dv": {501 "status": "scored",502 "text": "\u2713 scored (n=42)",503 "n": 42504 },505 "worldscore": {506 "status": "scored",507 "text": "\u2713 scored (n=8)",508 "n": 8509 }510 },511 "framecrafter": {512 "eval50": {513 "status": "scored",514 "text": "\u2713 scored (n=50)",515 "n": 50516 },517 "dyn_nvidia": {518 "status": "scored",519 "text": "\u2713 scored (n=21)",520 "n": 21521 },522 "dyn_kubric4d": {523 "status": "scored",524 "text": "\u2713 scored (n=100)",525 "n": 100526 },527 "dyn_spatialvid": {528 "status": "scored",529 "text": "\u2713 scored (n=27)",530 "n": 27531 },532 "lh_dl3dv": {533 "status": "scored",534 "text": "\u2713 scored (n=41)",535 "n": 41536 },537 "worldscore": {538 "status": "scored",539 "text": "\u2713 scored (n=8)",540 "n": 8541 }542 },543 "nvs_solver": {544 "eval50": {545 "status": "scored",546 "text": "\u2713 scored (n=45)",547 "n": 45548 },549 "dyn_nvidia": {550 "status": "scored",551 "text": "\u2713 scored (n=18)",552 "n": 18553 },554 "dyn_kubric4d": {555 "status": "scored",556 "text": "\u2713 scored (n=100)",557 "n": 100558 },559 "dyn_spatialvid": {560 "status": "scored",561 "text": "\u2713 scored (n=30)",562 "n": 30563 },564 "lh_dl3dv": {565 "status": "scored",566 "text": "\u2713 scored (n=42)",567 "n": 42568 },569 "worldscore": {570 "status": "scored",571 "text": "\u2713 scored (n=8)",572 "n": 8573 }574 },575 "viewextrapolator": {576 "eval50": {577 "status": "scored",578 "text": "\u2713 scored (n=50)",579 "n": 50580 },581 "dyn_nvidia": {582 "status": "scored",583 "text": "\u2713 scored (n=18)",584 "n": 18585 },586 "dyn_kubric4d": {587 "status": "pending",588 "text": "\u2014 pending",589 "reason": "",590 "n": null591 },592 "dyn_spatialvid": {593 "status": "scored",594 "text": "\u2713 scored (n=30)",595 "n": 30596 },597 "lh_dl3dv": {598 "status": "scored",599 "text": "\u2713 scored (n=32)",600 "n": 32601 },602 "worldscore": {603 "status": "scored",604 "text": "\u2713 scored (n=8)",605 "n": 8606 }607 },608 "trajcrafter": {609 "eval50": {610 "status": "scored",611 "text": "\u2713 scored (n=48)",612 "n": 48613 },614 "dyn_nvidia": {615 "status": "scored",616 "text": "\u2713 scored (n=14)",617 "n": 14618 },619 "dyn_kubric4d": {620 "status": "scored",621 "text": "\u2713 scored (n=67)",622 "n": 67623 },624 "dyn_spatialvid": {625 "status": "scored",626 "text": "\u2713 scored (n=24)",627 "n": 24628 },629 "lh_dl3dv": {630 "status": "scored",631 "text": "\u2713 scored (n=42)",632 "n": 42633 },634 "worldscore": {635 "status": "scored",636 "text": "\u2713 scored (n=8)",637 "n": 8638 }639 },640 "shape_of_motion": {641 "eval50": {642 "status": "na",643 "text": "\u23ed N/A",644 "reason": "per-scene 4D-GS, not applicable",645 "n": null646 },647 "dyn_nvidia": {648 "status": "scored",649 "text": "\u2713 scored (n=21)",650 "n": 21651 },652 "dyn_kubric4d": {653 "status": "scored",654 "text": "\u2713 scored (n=15)",655 "n": 15656 },657 "dyn_spatialvid": {658 "status": "running",659 "text": "\ud83d\udd04 running (gen: 30/30)",660 "n": null,661 "gen": 30662 },663 "lh_dl3dv": {664 "status": "running",665 "text": "\ud83d\udd04 running (gen: 42/42)",666 "n": null,667 "gen": 42668 },669 "worldscore": {670 "status": "na",671 "text": "\u23ed N/A",672 "reason": "per-scene 4D-GS, not applicable",673 "n": null674 }675 },676 "lyra2": {677 "eval50": {678 "status": "na",679 "text": "\u23ed N/A",680 "reason": "LH world-model, eval on LH only",681 "n": null682 },683 "dyn_nvidia": {684 "status": "na",685 "text": "\u23ed N/A",686 "reason": "LH world-model, eval on LH only",687 "n": null688 },689 "dyn_kubric4d": {690 "status": "na",691 "text": "\u23ed N/A",692 "reason": "LH world-model, eval on LH only",693 "n": null694 },695 "dyn_spatialvid": {696 "status": "na",697 "text": "\u23ed N/A",698 "reason": "LH world-model, eval on LH only",699 "n": null700 },701 "lh_dl3dv": {702 "status": "running",703 "text": "\ud83d\udd04 running (gen: 24/42)",704 "n": null,705 "gen": 24706 },707 "worldscore": {708 "status": "pending",709 "text": "\u2014 pending",710 "reason": "",711 "n": null712 }713 },714 "matrix3d": {715 "eval50": {716 "status": "na",717 "text": "\u23ed N/A",718 "reason": "LH world-model, eval on LH only",719 "n": null720 },721 "dyn_nvidia": {722 "status": "na",723 "text": "\u23ed N/A",724 "reason": "LH world-model, eval on LH only",725 "n": null726 },727 "dyn_kubric4d": {728 "status": "na",729 "text": "\u23ed N/A",730 "reason": "LH world-model, eval on LH only",731 "n": null732 },733 "dyn_spatialvid": {734 "status": "na",735 "text": "\u23ed N/A",736 "reason": "LH world-model, eval on LH only",737 "n": null738 },739 "lh_dl3dv": {740 "status": "pending",741 "text": "\u2014 pending (queued)",742 "reason": "queued",743 "n": null744 },745 "worldscore": {746 "status": "pending",747 "text": "\u2014 pending",748 "reason": "",749 "n": null750 }751 },752 "spmem": {753 "eval50": {754 "status": "na",755 "text": "\u23ed N/A",756 "reason": "LH world-model, eval on LH only",757 "n": null758 },759 "dyn_nvidia": {760 "status": "na",761 "text": "\u23ed N/A",762 "reason": "LH world-model, eval on LH only",763 "n": null764 },765 "dyn_kubric4d": {766 "status": "na",767 "text": "\u23ed N/A",768 "reason": "LH world-model, eval on LH only",769 "n": null770 },771 "dyn_spatialvid": {772 "status": "na",773 "text": "\u23ed N/A",774 "reason": "LH world-model, eval on LH only",775 "n": null776 },777 "lh_dl3dv": {778 "status": "pending",779 "text": "\u2014 pending (spmem running, no outputs yet)",780 "reason": "spmem running, no outputs yet",781 "n": null782 },783 "worldscore": {784 "status": "pending",785 "text": "\u2014 pending",786 "reason": "",787 "n": null788 }789 },790 "vmem": {791 "eval50": {792 "status": "na",793 "text": "\u23ed N/A",794 "reason": "LH world-model, eval on LH only",795 "n": null796 },797 "dyn_nvidia": {798 "status": "na",799 "text": "\u23ed N/A",800 "reason": "LH world-model, eval on LH only",801 "n": null802 },803 "dyn_kubric4d": {804 "status": "na",805 "text": "\u23ed N/A",806 "reason": "LH world-model, eval on LH only",807 "n": null808 },809 "dyn_spatialvid": {810 "status": "na",811 "text": "\u23ed N/A",812 "reason": "LH world-model, eval on LH only",813 "n": null814 },815 "lh_dl3dv": {816 "status": "pending",817 "text": "\u2014 pending (gated)",818 "reason": "gated",819 "n": null820 },821 "worldscore": {822 "status": "pending",823 "text": "\u2014 pending",824 "reason": "",825 "n": null826 }827 },828 "matrixgame3": {829 "eval50": {830 "status": "na",831 "text": "\u23ed N/A",832 "reason": "LH world-model, eval on LH only",833 "n": null834 },835 "dyn_nvidia": {836 "status": "na",837 "text": "\u23ed N/A",838 "reason": "LH world-model, eval on LH only",839 "n": null840 },841 "dyn_kubric4d": {842 "status": "na",843 "text": "\u23ed N/A",844 "reason": "LH world-model, eval on LH only",845 "n": null846 },847 "dyn_spatialvid": {848 "status": "na",849 "text": "\u23ed N/A",850 "reason": "LH world-model, eval on LH only",851 "n": null852 },853 "lh_dl3dv": {854 "status": "na",855 "text": "\u23ed N/A",856 "reason": "run broken",857 "n": null858 },859 "worldscore": {860 "status": "pending",861 "text": "\u2014 pending",862 "reason": "",863 "n": null864 }865 }866 },867 "summary": {868 "n_methods": 20,869 "n_benchmarks": 6,870 "cells_total": 120,871 "n_cells": 120,872 "scored": 58,873 "running": 5,874 "na": 23,875 "pending": 34,876 "pending_plus_na": 57877 }878}