CyberSys/fruit-fly-simulation
0
1import * as THREE from 'three';2import { OrbitControls } from 'three/addons/controls/OrbitControls.js';3import { parseBinarySTL, transformVertices } from './body/stl.js';4import { Gait } from './gait.js';5import { requestBytes } from './data-loader.js';6import { restPose } from './controller.js';7import { EffectComposer } from 'three/addons/postprocessing/EffectComposer.js';8import { RenderPass } from 'three/addons/postprocessing/RenderPass.js';9import { MacroOutputPass } from './macro-output.js';10import { MacroDOFPass, attachDepthBuffers } from './macro-dof.js';11import {12 createFlyMaterial,13 cloneFlyMaterial,14 createBristleMaterial,15 addFlyBristles,16} from './fly-appearance.js';17import {18 CAMERA_FOV,19 CAMERA_OFFSETS,20 cameraFit,21 cameraPullback,22 cameraTarget,23} from './camera-config.js';24const clamp = THREE.MathUtils.clamp;25const POSE_FIELDS = Object.keys(restPose()).filter((key) => key !== 'behavior');26/** Weld duplicate vertices before computing smooth normals. */27function geometryFor(raw, scale, mirror) {28 const positions = transformVertices(raw, scale, mirror);29 if (mirror)30 for (let i = 0; i < positions.length; i += 9)31 for (let k = 0; k < 3; k++) {32 const t = positions[i + 3 + k];33 positions[i + 3 + k] = positions[i + 6 + k];34 positions[i + 6 + k] = t;35 }36 const unique = [],37 indices = [],38 lookup = new Map();39 for (let i = 0; i < positions.length; i += 3) {40 const key = [positions[i], positions[i + 1], positions[i + 2]]41 .map((v) => Math.round(v * 1e5))42 .join(',');43 let ix = lookup.get(key);44 if (ix === undefined) {45 ix = unique.length / 3;46 lookup.set(key, ix);47 unique.push(positions[i], positions[i + 1], positions[i + 2]);48 }49 indices.push(ix);50 }51 const geometry = new THREE.BufferGeometry();52 geometry.setAttribute('position', new THREE.Float32BufferAttribute(unique, 3));53 geometry.setIndex(indices);54 geometry.computeVertexNormals();55 return geometry;56}57export class FlyScene {58 constructor(canvas) {59 this.canvas = canvas;60 this.scene = new THREE.Scene();61 this.scene.background = null;62 this.renderer = new THREE.WebGLRenderer({63 canvas,64 antialias: true,65 alpha: false,66 powerPreference: 'high-performance',67 });68 this.renderer.setClearColor(0x000000, 0);69 this.renderer.setPixelRatio(Math.min(1.5, devicePixelRatio || 1));70 this.renderer.outputColorSpace = THREE.SRGBColorSpace;71 this.renderer.toneMapping = THREE.ACESFilmicToneMapping;72 this.renderer.toneMappingExposure = 1.05;73 this.renderer.shadowMap.enabled = true;74 this.renderer.shadowMap.type = THREE.PCFSoftShadowMap;75 this.camera = new THREE.PerspectiveCamera(CAMERA_FOV, 1, 0.03, 100);76 this.camera.up.set(0, 0, 1);77 this.controls = new OrbitControls(this.camera, canvas);78 this.controls.enableDamping = true;79 this.controls.minDistance = 3.5;80 this.controls.maxDistance = 18;81 this.controls.maxPolarAngle = Math.PI * 0.49;82 this.controls.target.set(-0.4, 0, 0.68);83 this.controls.enablePan = false;84 this.orbitStart = () => {85 this.trackHeading = false;86 this.onCameraChange?.('free');87 };88 this.controls.addEventListener('start', this.orbitStart);89 this.composer = new EffectComposer(this.renderer);90 attachDepthBuffers(this.composer, this.renderer);91 this.beautyPass = new RenderPass(this.scene, this.camera);92 this.dof = new MacroDOFPass(this.camera);93 this.outputPass = new MacroOutputPass();94 this.composer.addPass(this.beautyPass);95 this.composer.addPass(this.dof);96 this.composer.addPass(this.outputPass);97 this.depthOfField = true;98 this.focusPoint = new THREE.Vector3();99 this.scene.add(new THREE.HemisphereLight(0xdce8ff, 0x4c3928, 1.6));100 this.key = new THREE.DirectionalLight(0xffe6c6, 4.2);101 this.key.position.set(2, -4, 7);102 this.key.castShadow = true;103 this.key.shadow.mapSize.set(2048, 2048);104 Object.assign(this.key.shadow.camera, {105 left: -5,106 right: 5,107 top: 5,108 bottom: -5,109 near: 0.1,110 far: 20,111 });112 this.key.shadow.bias = -0.00015;113 this.key.shadow.normalBias = 0.012;114 this.scene.add(this.key, this.key.target);115 const rim = new THREE.DirectionalLight(0xc9ddff, 2.6);116 rim.position.set(-3, 4, 4);117 this.scene.add(rim);118 const fill = new THREE.DirectionalLight(0xe0d4cf, 1.1);119 fill.position.set(4, 3, 2);120 this.scene.add(fill);121 // Seeded, seamless fine grain and short fibres on an authored 2 mm ground tile.122 const canvasTex = document.createElement('canvas');123 canvasTex.width = canvasTex.height = 1024;124 const context = canvasTex.getContext('2d'),125 img = context.createImageData(1024, 1024);126 let seed = 47;127 const random = () => {128 seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;129 return seed / 4294967296;130 };131 for (let i = 0; i < img.data.length; i += 4) {132 const c = 100 + random() * 33;133 img.data[i] = c;134 img.data[i + 1] = c;135 img.data[i + 2] = c;136 img.data[i + 3] = 255;137 }138 context.putImageData(img, 0, 0);139 context.lineWidth = 0.7;140 for (let i = 0; i < 6500; i++) {141 const x = random() * 1024,142 y = random() * 1024,143 angle = random() * Math.PI * 2,144 length = 4 + random() * 13,145 dx = Math.cos(angle) * length,146 dy = Math.sin(angle) * length;147 context.strokeStyle = `rgba(${i % 2 ? '190,190,190' : '45,45,45'},.36)`;148 for (const ox of [-1024, 0, 1024])149 for (const oy of [-1024, 0, 1024]) {150 context.beginPath();151 context.moveTo(x + ox, y + oy);152 context.quadraticCurveTo(153 x + ox + dx * 0.4 - dy * 0.3,154 y + oy + dy * 0.4 + dx * 0.3,155 x + ox + dx,156 y + oy + dy,157 );158 context.stroke();159 }160 }161 this.texture = new THREE.CanvasTexture(canvasTex);162 this.texture.wrapS = this.texture.wrapT = THREE.RepeatWrapping;163 this.texture.repeat.set(80, 80);164 this.texture.anisotropy = Math.min(8, this.renderer.capabilities.getMaxAnisotropy());165 const floorMaterial = new THREE.MeshStandardMaterial({166 color: 0x202832,167 roughness: 0.96,168 bumpMap: this.texture,169 bumpScale: 0.005,170 transparent: true,171 });172 // Fade the distant ground into the display-space backdrop, preserving contact173 // texture and shadows near the specimen. vViewPosition.z is positive depth.174 this.floorFadeStart = { value: 9 };175 floorMaterial.onBeforeCompile = (shader) => {176 shader.uniforms.floorFadeStart = this.floorFadeStart;177 shader.fragmentShader =178 'uniform float floorFadeStart;\n' +179 shader.fragmentShader.replace(180 '#include <opaque_fragment>',181 'diffuseColor.a *= 1.0 - smoothstep(floorFadeStart, floorFadeStart + 15.0, vViewPosition.z);\n#include <opaque_fragment>',182 );183 };184 floorMaterial.customProgramCacheKey = () => 'macro-ground-distance-fade-v1';185 this.floor = new THREE.Mesh(new THREE.PlaneGeometry(160, 160), floorMaterial);186 this.floor.receiveShadow = true;187 this.floor.renderOrder = -1;188 this.floor.position.z = -0.015;189 this.scene.add(this.floor);190 this.fly = new THREE.Group();191 this.fly.rotation.order = 'ZYX';192 this.scene.add(this.fly);193 this.meshes = [];194 this.wingEchoes = [];195 this.materials = new Map();196 this.bristleMaterial = createBristleMaterial();197 this.pose = restPose();198 this.previous = { ...this.pose };199 this.next = { ...this.pose };200 this.arrival = performance.now();201 this.interval = 100;202 this.follow = true;203 this.path = new Float32Array(9000);204 this.pathCount = 0;205 const pg = new THREE.BufferGeometry();206 pg.setAttribute('position', new THREE.BufferAttribute(this.path, 3));207 pg.setDrawRange(0, 0);208 this.trail = new THREE.Line(209 pg,210 new THREE.LineBasicMaterial({ color: 0x7ee2d5, transparent: true, opacity: 0.22 }),211 );212 this.scene.add(this.trail);213 this.observer = new ResizeObserver(() => this.resize());214 this.observer.observe(canvas);215 this.setCamera('side');216 this.render = this.render.bind(this);217 this.frame = requestAnimationFrame(this.render);218 }219 async load() {220 this.model = JSON.parse(221 new TextDecoder().decode(await requestBytes('./body/assets/model.json')),222 );223 this.gait = new Gait(this.model);224 const files = [...new Set(Object.values(this.model.meshes).map((m) => m.file))],225 raw = new Map();226 // Four asset downloads at a time, avoiding a burst of 39 simultaneous requests.227 let cursor = 0;228 await Promise.all(229 Array.from({ length: 4 }, async () => {230 while (cursor < files.length) {231 const name = files[cursor++];232 raw.set(name, parseBinarySTL(await requestBytes('./body/assets/meshes/' + name)));233 }234 }),235 );236 if (this.disposed) return;237 for (const [name, m] of Object.entries(this.model.meshes)) {238 const wing = name.includes('wing'),239 eye = name.endsWith('eye');240 let key = wing241 ? 'wing'242 : eye243 ? 'eye'244 : name.includes('abdomen')245 ? 'abdomen'246 : name.includes('tarsus') || name.includes('arista')247 ? 'dark'248 : 'cuticle';249 if (!this.materials.has(key)) this.materials.set(key, createFlyMaterial(key));250 const mesh = new THREE.Mesh(251 geometryFor(raw.get(m.file), this.model.meshScale, m.mirror),252 this.materials.get(key),253 );254 mesh.matrixAutoUpdate = false;255 mesh.castShadow = !wing;256 mesh.receiveShadow = !wing;257 this.fly.add(mesh);258 this.meshes.push({ name, mesh });259 addFlyBristles(mesh, name, this.bristleMaterial);260 if (name === 'c_head') {261 mesh.geometry.computeBoundingSphere();262 this.focusMesh = mesh;263 }264 if (wing) {265 for (const phaseOffset of [(-Math.PI * 2) / 3, (Math.PI * 2) / 3]) {266 const material = cloneFlyMaterial(mesh.material);267 material.opacity = 0;268 const echo = new THREE.Mesh(mesh.geometry, material);269 echo.matrixAutoUpdate = false;270 echo.visible = false;271 this.fly.add(echo);272 this.wingEchoes.push({ name, mesh: echo, phaseOffset });273 }274 }275 }276 // RAF may have evaluated the resting gait while meshes were downloading.277 this.lastGaitTime = undefined;278 this.applyPose(this.pose);279 }280 setCamera(view) {281 const damping = this.controls.enableDamping;282 this.controls.enableDamping = false;283 this.controls.update();284 this.controls.enableDamping = damping;285 this.cameraView = view;286 this.trackHeading = view === 'follow';287 this.cameraYaw = this.pose.yaw;288 this.fitScale = cameraFit(view, this.camera.aspect);289 this.pullback = cameraPullback(this.pose);290 this.controls.target.fromArray(cameraTarget(this.pose));291 this.camera.up.set(0, 0, 1);292 this.camera.position293 .copy(this.controls.target)294 .add(295 new THREE.Vector3(...CAMERA_OFFSETS[view])296 .applyAxisAngle(new THREE.Vector3(0, 0, 1), view === 'follow' ? this.pose.yaw : 0)297 .multiplyScalar(this.fitScale * this.pullback),298 );299 this.controls.update();300 this.onCameraChange?.(view);301 }302 setDepthOfField(enabled) {303 this.depthOfField = enabled;304 this.dof.enabled = enabled;305 }306 resize() {307 const r = this.canvas.getBoundingClientRect();308 if (!r.width || !r.height) return;309 const dpr = Math.min(1.5, devicePixelRatio || 1, Math.sqrt(1200000 / (r.width * r.height)));310 this.renderer.setPixelRatio(dpr);311 this.renderer.setSize(r.width, r.height, false);312 this.composer.setPixelRatio(dpr);313 this.composer.setSize(r.width, r.height);314 this.camera.aspect = r.width / r.height;315 const surface = this.canvas.closest('.fly-surface'),316 top =317 surface.querySelector('.fly-head').getBoundingClientRect().height +318 surface.querySelector('.response-bar').getBoundingClientRect().height,319 bottom = surface.querySelector('.specimen-line').getBoundingClientRect().height;320 this.camera.setViewOffset(r.width, r.height, 0, -(top - bottom) / 2, r.width, r.height);321 this.camera.updateProjectionMatrix();322 const fit = cameraFit(this.cameraView, this.camera.aspect);323 this.controls.maxDistance = 24 * fit;324 this.camera.position325 .sub(this.controls.target)326 .multiplyScalar(fit / this.fitScale)327 .add(this.controls.target);328 this.fitScale = fit;329 this.controls.update();330 }331 update(pose) {332 const now = performance.now();333 this.previous = { ...this.pose };334 this.next = { ...pose };335 this.interval = clamp(now - this.arrival, 10, 1000);336 this.arrival = now;337 }338 applyPose(p) {339 const target = new THREE.Vector3(...cameraTarget(p)),340 delta = target.sub(new THREE.Vector3(...cameraTarget(this.pose)));341 if (this.follow) {342 this.camera.position.add(delta);343 this.controls.target.add(delta);344 }345 if (this.trackHeading) {346 const dt = Math.max(0, p.time - this.pose.time),347 turn = (p.yaw - this.cameraYaw) * (1 - Math.exp(-dt / 0.16));348 this.camera.position349 .sub(this.controls.target)350 .applyAxisAngle(new THREE.Vector3(0, 0, 1), turn)351 .add(this.controls.target);352 this.cameraYaw += turn;353 }354 const pullback = cameraPullback(p);355 this.camera.position356 .sub(this.controls.target)357 .multiplyScalar(pullback / this.pullback)358 .add(this.controls.target);359 this.pullback = pullback;360 this.pose = { ...p };361 this.fly.position.set(p.z, p.x, p.y);362 this.fly.rotation.set(p.bank || 0, p.pitch || 0, p.yaw, 'ZYX');363 if (this.gait && this.lastGaitTime !== p.time) {364 this.lastGaitTime = p.time;365 const transforms = this.gait.update(p);366 for (const { name, mesh } of this.meshes) {367 mesh.matrix.set(...transforms[name]);368 mesh.matrixWorldNeedsUpdate = true;369 }370 }371 // Wing exposures use interpolated neural time, so pause freezes them.372 // Only wing transforms are repeated; the six-leg IK is solved once per pose.373 if (this.gait) {374 const blur = clamp(((p.wingOpen || 0) - 0.7) / 0.3, 0, 1);375 for (const phaseOffset of [(-Math.PI * 2) / 3, (Math.PI * 2) / 3]) {376 const echoes = this.wingEchoes.filter((e) => e.phaseOffset === phaseOffset);377 const transforms = blur > 0 ? this.gait.wingTransforms(p, phaseOffset) : null;378 for (const { name, mesh } of echoes) {379 mesh.visible = blur > 0;380 if (transforms) {381 mesh.material.opacity = 0.065 * blur;382 mesh.matrix.set(...transforms[name]);383 mesh.matrixWorldNeedsUpdate = true;384 }385 }386 }387 }388 this.floor.position.x = p.z;389 this.floor.position.y = p.x;390 this.texture.offset.set(p.z / 2, p.x / 2);391 this.key.position.set(p.z + 2, p.x - 4, 7);392 this.key.target.position.set(p.z, p.x, 0);393 }394 render(now) {395 if (this.disposed) return;396 this.frame = requestAnimationFrame(this.render);397 if (document.hidden) return;398 const t = clamp((now - this.arrival) / this.interval, 0, 1),399 p = {};400 for (const key of POSE_FIELDS)401 p[key] = this.previous[key] + (this.next[key] - this.previous[key]) * t;402 this.applyPose(p);403 if (Math.hypot(p.x - (this.lastPathX ?? 0), p.z - (this.lastPathZ ?? 0)) > 0.08) {404 this.lastPathX = p.x;405 this.lastPathZ = p.z;406 if (this.pathCount === 3000) {407 this.path.copyWithin(0, 3);408 this.pathCount--;409 }410 this.path.set([p.z, p.x, p.y + 0.005], this.pathCount++ * 3);411 this.trail.geometry.attributes.position.needsUpdate = true;412 this.trail.geometry.setDrawRange(0, this.pathCount);413 this.trail.geometry.computeBoundingSphere();414 }415 this.controls.update();416 this.floorFadeStart.value = Math.max(417 9,418 this.camera.position.distanceTo(this.controls.target) + 2,419 );420 const origin = this.controls.target.clone(),421 right = new THREE.Vector3(1, 0, 0).applyQuaternion(this.camera.quaternion).add(origin);422 const a = origin.project(this.camera),423 b = right.project(this.camera);424 const pixels = (Math.abs(b.x - a.x) * this.canvas.clientWidth) / 2,425 unit = pixels > 180 ? 0.5 : 1;426 document.getElementById('scale-bar').style.width = pixels * unit + 'px';427 document.getElementById('scale-value').textContent = unit + ' mm';428 if (this.depthOfField) {429 this.fly.updateMatrixWorld(true);430 this.camera.updateMatrixWorld();431 if (this.focusMesh)432 this.focusPoint433 .copy(this.focusMesh.geometry.boundingSphere.center)434 .applyMatrix4(this.focusMesh.matrixWorld);435 else this.focusPoint.copy(this.controls.target);436 this.focusPoint.applyMatrix4(this.camera.matrixWorldInverse);437 this.dof.uniforms.focus.value = Math.max(this.camera.near, -this.focusPoint.z);438 }439 this.composer.render();440 }441 reset() {442 const zero = restPose();443 this.gait = new Gait(this.model);444 this.lastGaitTime = undefined;445 this.applyPose(zero);446 if (this.trackHeading) this.setCamera('follow');447 this.previous = { ...zero };448 this.next = { ...zero };449 this.pathCount = 0;450 this.lastPathX = this.lastPathZ = 0;451 this.trail.geometry.setDrawRange(0, 0);452 this.arrival = performance.now();453 }454 dispose() {455 this.disposed = true;456 cancelAnimationFrame(this.frame);457 this.observer.disconnect();458 this.controls.removeEventListener('start', this.orbitStart);459 this.controls.dispose();460 const geometries = new Set(),461 materials = new Set();462 this.scene.traverse((o) => {463 if (o.geometry) geometries.add(o.geometry);464 if (o.material) materials.add(o.material);465 });466 materials.add(this.bristleMaterial);467 for (const g of geometries) g.dispose();468 for (const m of materials) m.dispose();469 this.texture.dispose();470 this.key.shadow.dispose();471 for (const pass of this.composer.passes) pass.dispose();472 this.composer.dispose();473 this.renderer.dispose();474 }475}476