import * as THREE from 'three'; import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm'; import { createVRMAnimationClip, VRMAnimationLoaderPlugin, VRMLookAtQuaternionProxy } from '@pixiv/three-vrm-animation'; import { VRM_PATH, WS_URL } from './config.js'; import { loadVRM } from './vrmLoader.js'; import { AudioManager } from './audioManager.js'; import { AnimationManager } from './animationManager.js'; import { loadMixamoAnimation } from './loadMixamoAnimation.js'; import { showSubtitleStreaming } from './subtitles.js'; import { initWebcam, takePictureAndUpload } from './webcam.js'; import { connectWS, initUI } from "./ui.js"; import { initVRMClickDetector } from './vrmClickDetector.js'; import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js'; import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js'; import { BokehPass } from 'three/examples/jsm/postprocessing/BokehPass.js'; // app.js β€” add near top with your other imports (no extra imports needed) // Better trimAnimationClip: construct new tracks (compatible with more three.js builds) function trimAnimationClip(clip, startTime, endTime) { // startTime/endTime in seconds (endTime is exclusive) startTime = Math.max(0, startTime || 0); const fullDuration = clip.duration; endTime = (typeof endTime === 'number' && endTime >= 0) ? Math.min(fullDuration, endTime) : fullDuration; if (endTime <= startTime) { console.warn('trimAnimationClip: invalid range', { startTime, endTime, duration: fullDuration }); return null; } const newTracks = []; for (const track of clip.tracks) { // Use the typed arrays directly for speed/compatibility const times = track.times; // Float32Array or Array const values = track.values; // Float32Array or Array const stride = track.getValueSize(); const keptTimes = []; const keptValues = []; // iterate keyframes and pick those within range // times.length should equal values.length / stride for (let i = 0; i < times.length; i++) { const t = times[i]; if (t >= startTime && t <= endTime) { keptTimes.push(t - startTime); // shift so new clip starts at 0 const baseIndex = i * stride; for (let s = 0; s < stride; s++) { keptValues.push(values[baseIndex + s]); } } } if (keptTimes.length > 0) { // Create typed arrays expected by KeyframeTrack constructors const TimesArray = Float32Array; const ValuesArray = Float32Array; const newTimes = new TimesArray(keptTimes); const newValues = new ValuesArray(keptValues); // Build a new track using the same constructor (works for Number/Vector/Quaternion) // Signature: new TrackConstructor(name, times, values, interpolation?) let NewTrack; try { NewTrack = new track.constructor(track.name, newTimes, newValues, track.getInterpolation ? track.getInterpolation() : undefined); } catch (err) { // Fallback: try without interpolation argument NewTrack = new track.constructor(track.name, newTimes, newValues); } newTracks.push(NewTrack); } } const newDuration = endTime - startTime; const newName = `${clip.name || 'vrma_clip'}_trimmed_${startTime.toFixed(3)}-${endTime.toFixed(3)}`; return new THREE.AnimationClip(newName, newDuration, newTracks); } // Handle messages from server function handleServerMessage(msg) { //console.log("πŸ“© Received from server:", msg); if (msg.type === "start_animation") { // Play animation on VRM } else if (msg.type === "take_picture") { // Trigger snapshot from camera } } // Connect WebSocket + UI connectWS(handleServerMessage); initUI(); // Your existing VRM rendering logic continues here.. // Initialize webcam once on startup if needed await initWebcam(); // VRM LOADERS // Global variables let currentMixer = null; let vrm = null; let renderer = null; let scene = null; let camera = null; let controls = null; let animationMgr = null; // ADD THIS const clock = new THREE.Clock(); let currentVrm = null; let currentAction = null; (async () => { // β€” Renderer / Scene / Camera β€” // TRANSPARENT BACKGROUND ALPHA renderer = new THREE.WebGLRenderer({ alpha: true }); renderer.setSize(window.innerWidth, window.innerHeight); renderer.setPixelRatio(window.devicePixelRatio); renderer.shadowMap.enabled = true; renderer.shadowMap.type = THREE.PCFSoftShadowMap; document.body.appendChild(renderer.domElement); // SCENE scene = new THREE.Scene(); // load in room const glbloader = new GLTFLoader(); glbloader.load('./backgrounds/glb/RikoRoomFixed.glb', function(gltf) { const model = gltf.scene; model.scale.set(1, 1, 1); // Start with no scale reduction model.position.set(-3, 0, -0.7); // x,z model.rotation.set(0,2.4,0); // use the second number to rorate rooms model.traverse((child) => { if (child.isMesh) { child.geometry.computeBoundingBox(); child.geometry.computeBoundingSphere(); child.material.depthWrite = true; child.material.depthTest = true; child.material.polygonOffset = true; child.material.polygonOffsetFactor = -1; child.material.polygonOffsetUnits = -1; child.updateMatrix(); child.updateMatrixWorld(true); child.geometry.computeVertexNormals(); }}); scene.add(model); }); // CAMERA camera = new THREE.PerspectiveCamera(30, window.innerWidth/window.innerHeight, 0.1, 20); camera.position.set(0,1,0.9); controls = new OrbitControls(camera, renderer.domElement); controls.target.set(0,1.1,0); controls.update(); // helpers // const gridHelper = new THREE.GridHelper( 10, 10 ); // scene.add( gridHelper ); // const axesHelper = new THREE.AxesHelper( 5 ); // scene.add( axesHelper ); // β€” Light β€” // === Adjustable variables === const SOFTBOX_INTENSITY = 3.2; // how bright the softbox is const SOFTBOX_DISTANCE = 5; // how far the light reaches const SOFTBOX_ANGLE = Math.PI / 5; // spotlight cone angle (narrower = more focused) const SOFTBOX_SOFTNESS = 0.6; // penumbra (0 = hard edge, 1 = very soft) const AMBIENT_INTENSITY = 1.0; // keep low so background stays dark const BACKGROUND_COLOR = 0x111111; // === Scene background === scene.background = new THREE.Color(BACKGROUND_COLOR); // === Ambient light (very low just to avoid pitch black shadows) === const ambientLight = new THREE.AmbientLight(0xffffff, AMBIENT_INTENSITY); scene.add(ambientLight); // === Softbox (spotlight approximation) === const softboxLight = new THREE.SpotLight( 0xffffff, // color SOFTBOX_INTENSITY, // intensity SOFTBOX_DISTANCE, // distance light reaches SOFTBOX_ANGLE, // cone angle SOFTBOX_SOFTNESS, // penumbra (softness) 2 // decay (higher = faster falloff) ); // Position light in front of model, slightly above softboxLight.position.set(1, 1.5, 1); // Aim light at model’s head softboxLight.target.position.set(0, 1.3, 0); // Add to scene scene.add(softboxLight); scene.add(softboxLight.target); // === Optional: visual helper === // Uncomment to see spotlight cone // const softboxHelper = new THREE.SpotLightHelper(softboxLight); // scene.add(softboxHelper); // === Optional: visible bulb marker === const lightPoint = new THREE.Mesh( new THREE.SphereGeometry(0.05), new THREE.MeshBasicMaterial({ color: 0xff0000 }) ); lightPoint.position.copy(softboxLight.position); scene.add(lightPoint); // FOCUS ADJUSTMENT const composer = new EffectComposer(renderer); composer.addPass(new RenderPass(scene, camera)); const bokehPass = new BokehPass(scene, camera, { focus: 0.9, // distance from camera where it's sharp aperture: 0.003, // smaller = stronger blur (try 0.0005 to 0.001) maxblur: 0, // max blur radius }); composer.addPass(bokehPass); // DEV TOOL FOR FOCUS VISUAL const focusPoint = new THREE.Mesh( new THREE.SphereGeometry(0.02), new THREE.MeshBasicMaterial({ color: 0xff0000 }) ); focusPoint.position.set(0, 1, 0.9); // match focus distance scene.add(focusPoint); // β€” Load VRM β€” const vrmData = await loadVRM(VRM_PATH, scene); vrm = vrmData.vrm; const loader = vrmData.loader; currentMixer = new THREE.AnimationMixer(vrm.scene); // Assuming `vrm` is already loaded PRINT ALL BONES // vrm.scene.traverse((object) => { // if (object.isBone) { // console.log(object.name); // } // }); // Init click detection initVRMClickDetector(renderer, camera, vrm, (region, partName) => { // This is where you can send it to LLM or server console.log(`πŸ“© Sending region click to server: ${region}`); // Example: ws.send(JSON.stringify({ type: "region_click", region })); }); // β€” Managers β€” const audioMgr = new AudioManager(vrm); animationMgr = new AnimationManager(vrm, audioMgr, renderer, scene, camera, controls); // Start the clock and animation loop clock.start(); // SINGLE ANIMATION LOOP THAT HANDLES EVERYTHING function animate() { requestAnimationFrame(animate); const deltaTime = clock.getDelta(); // Update current VRMA mixer if it exists if (currentMixer) { currentMixer.update(deltaTime); } // Update the animation manager (idle animations) if (animationMgr) { animationMgr.update(deltaTime); } // Update VRM and render vrm.update(deltaTime); // FOR MORE FLAT LOOK USE // renderer.render(scene, camera); composer.render(scene, camera); controls.update(); } // Start the animation loop once animate(); // β€” WebSocket Listener β€” const ws = new WebSocket(WS_URL); ws.onopen = () => { // console.log('WebSocket connected'); // document.getElementById('status').textContent = 'WS connected'; }; ws.onerror = err => console.error('WS error', err); ws.onmessage = async ({ data }) => { let msg; try { msg = JSON.parse(data); console.log(msg) } catch { return; } if (msg.type === 'start_animation') { const { audio_path, audio_text, audio_duraction, expression = 'neutral' } = msg; audioMgr.setExpression(expression); try { await audioMgr.setupAudio(audio_path); await audioMgr.analyzeAudio(); // SUBTITLE SECTION !!!! UNCOMMENT FOR IN BROWSER MODE, COMMENT OUT FOR OBS MODE // stream by word by default, you can stream by letter if you want by adding , "letter" // showSubtitleStreaming(audio_text, audio_duraction, "letter"); // 4 seconds total // play audio animationMgr.play(); } catch (e) { console.error('Failed to play animation:', e); } } if (msg.type === 'start_vrma') { const { animation_url, play_once = false, crop_start = 0.0, // seconds to crop off the beginning crop_end = 0.0 // seconds to crop off the end } = msg; try { // Load VRMA as before const gltfVrma = await loader.loadAsync(animation_url); const vrmAnimation = gltfVrma.userData.vrmAnimations[0]; let clip = createVRMAnimationClip(vrmAnimation, vrm); // If cropping is requested, compute new start/end times and trim // crop_start/crop_end are amounts in seconds to remove const startTime = Math.max(0, parseFloat(crop_start) || 0); const endTime = Math.max(0, clip.duration - (parseFloat(crop_end) || 0)); if (startTime > 0 || (parseFloat(crop_end) || 0) > 0) { const trimmed = trimAnimationClip(clip, startTime, endTime); if (trimmed) { clip = trimmed; } else { console.warn('VRMA trim returned null β€” using original clip'); } } // Notify animation manager animationMgr.setVRMAPlaying(true); // Stop previous action gracefully if (currentAction && currentAction.isRunning()) { currentAction.crossFadeTo(currentAction, 0.2, false); // small fade-out (safe guard) } // Use existing mixer (currentMixer created at VRM load) if (!currentMixer) { currentMixer = new THREE.AnimationMixer(vrm.scene); } // Create action const newAction = currentMixer.clipAction(clip); // If play_once requested, set LoopOnce and clamp at end if (play_once) { newAction.setLoop(THREE.LoopOnce, 0); newAction.clampWhenFinished = true; newAction.enabled = true; } else { // default behavior: LoopRepeat (three's default), no clamp newAction.setLoop(THREE.LoopRepeat, Infinity); newAction.clampWhenFinished = false; } // reset and play (with small crossfade from currentAction if present) newAction.reset(); newAction.play(); if (currentAction && currentAction !== newAction) { currentAction.crossFadeTo(newAction, 0.5, false); } // remove previous 'finished' listener if any (to avoid duplicates) if (currentMixer._vrmaFinishedListener) { try { currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener); } catch (e) {} currentMixer._vrmaFinishedListener = null; } // Listen to mixer finished events (fires when *an* action finishes). const onFinished = (e) => { // only react if the finished action is the one we just played if (e.action === newAction) { animationMgr.setVRMAPlaying(false); // if we played once, stop the action and optionally reset mixer state if (play_once) { try { newAction.stop(); } catch (err) {} } // (optional) null out currentMixer if you want to discard it // currentMixer = null; // remove this listener if (currentMixer && currentMixer._vrmaFinishedListener) { currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener); currentMixer._vrmaFinishedListener = null; } } }; // attach listener and store it on mixer for cleanup later currentMixer._vrmaFinishedListener = onFinished; currentMixer.addEventListener('finished', onFinished); currentAction = newAction; } catch (err) { console.error("Failed to load VRMA animation:", err); } } if (msg.type === 'start_mixamo') { const { animation_url } = msg; try { console.log("THIS IS A MIXAMO ANIMATION") console.log(animation_url) currentVrm = vrm // Load animation const clip = await loadMixamoAnimation( animation_url, currentVrm ); // animation blending const newAction = currentMixer.clipAction( clip ); newAction.reset().play(); if ( currentAction && currentAction !== newAction ) { currentAction.crossFadeTo( newAction, 0.5, false ); } currentAction = newAction; } catch (err) { console.error("Failed to load MIXAMO animation:", err); } } // take picture if (msg.type === 'take_picture') { console.log("πŸ“Έ Received command to take picture from server."); await takePictureAndUpload(); } }; // β€” Handle resize β€” window.addEventListener('resize', () => { camera.aspect = window.innerWidth/window.innerHeight; camera.updateProjectionMatrix(); renderer.setSize(window.innerWidth, window.innerHeight); }); })();