import * as THREE from 'three'; import { VRM_PATH, WS_URL } from './config.js'; import { createScene } from './scene/sceneSetup.js'; import { loadSceneObjects, describeObjects } from './scene/objectLoader.js'; import { SCENE_OBJECTS } from './scene/objectsConfig.js'; import { loadRoom } from './scene/roomLoader.js'; import { ROOM_CONFIG } from './scene/roomConfig.js'; import { createLocationRegistry } from './scene/locationRegistry.js'; import { NAMED_LOCATIONS, DEBUG_MARKERS } from './scene/locationsConfig.js'; import { loadVRM } from './core/vrmLoader.js'; import { AudioManager } from './core/audioManager.js'; import { PlaybackController } from './core/playbackController.js'; import { AnimationManager } from './animation/animationManager.js'; import { MovementController } from './animation/movementController.js'; import { showSubtitleStreaming, setSubtitlesEnabled, getSubtitlesEnabled } from './overlay/subtitles.js'; import { initWebcam, takePictureAndUpload } from './interaction/webcam.js'; import { connectWS, initUI } from './ui.js'; import { initVRMClickDetector } from './interaction/vrmClickDetector.js'; import { isVRSupported } from './vr/vrManager.js'; // --- WebSocket + UI (early init, no dependencies) --- function handleServerMessage(msg) { console.log("📩 Received from server:", msg); } connectWS(handleServerMessage); initUI(); // --- Subtitle toggle --- const subtitleBtn = document.getElementById('toggle-subtitles-button'); if (subtitleBtn) { subtitleBtn.addEventListener('click', () => { const nowEnabled = !getSubtitlesEnabled(); setSubtitlesEnabled(nowEnabled); subtitleBtn.classList.toggle('off', !nowEnabled); }); } // Initialize webcam await initWebcam(); // --- App State --- let vrm = null; let renderer = null; let scene = null; let camera = null; let controls = null; let audioMgr = null; let animationMgr = null; let movementController = null; let playbackController = null; let vrManager = null; let vrPositionTracker = null; const clock = new THREE.Clock(); (async () => { // --- Check VR support --- const vrSupported = await isVRSupported(); if (vrSupported) console.log('WebXR VR supported - VR mode available'); // --- Scene Setup --- // Adjust these options to change the scene configuration. // See scene/sceneSetup.js DEFAULTS for all available options. ({ renderer, scene, camera, controls } = createScene({ // background: 0x000000, // Uncomment for black background // gridHelper: 50, // Uncomment for grid // axesHelper: 5, // Uncomment for axes })); // --- Room (environment GLB) --- await loadRoom(scene, ROOM_CONFIG); // --- Scene Objects (GLB props) --- const sceneObjects = await loadSceneObjects(scene, SCENE_OBJECTS); window.sceneObjects = sceneObjects; window.describeSceneObjects = () => describeObjects(sceneObjects); // --- Named locations (waypoints + scene objects merged) --- const locations = createLocationRegistry({ scene, namedLocations: NAMED_LOCATIONS, sceneObjects, debugMarkers: DEBUG_MARKERS, }); window.locations = locations; console.log('📍 Locations registered:', locations.describe()); // --- Load VRM --- const vrmData = await loadVRM(VRM_PATH, scene); vrm = vrmData.vrm; // --- Click Detection --- initVRMClickDetector(renderer, camera, vrm, (region, partName) => { console.log(`📩 Region clicked: ${region}`); }); // --- Managers --- audioMgr = new AudioManager(vrm); animationMgr = new AnimationManager(vrm, audioMgr, renderer, scene, camera, controls); animationMgr.setState('idle'); // --- Movement Controller --- movementController = new MovementController(vrm, scene, animationMgr); await movementController.loadIdleAnimation('./animations/mixamo/Idle.fbx'); await movementController.loadWalkAnimation('./animations/mixamo/Walking_inplace.fbx'); await movementController.loadAdditiveAnimation('./animations/mixamo/Waving.fbx', 'wave'); // --- VR Setup (conditional) --- if (vrSupported) { const { VRManager } = await import('./vr/vrManager.js'); const { VRPositionTracker } = await import('./vr/vrPositionTracker.js'); vrManager = new VRManager(renderer, scene, camera, controls); vrManager.init(); // Enable proximity-based touch interaction on the VRM in VR vrManager.initTouchDetection(vrm, { onTouch: (region, boneName, handIndex) => { console.log(`VR touch: ${region} (hand ${handIndex})`); }, }); vrPositionTracker = new VRPositionTracker(camera, vrManager.getDolly()); window.addEventListener('vr-session-start', () => vrPositionTracker.enable()); window.addEventListener('vr-session-end', () => vrPositionTracker.disable()); window.addEventListener('vr-unlock-audio', async () => { try { await playbackController?.unlockOnce(); } catch (e) {} }); window.vrManager = vrManager; window.vrPositionTracker = vrPositionTracker; } // --- Playback Controller (mobile audio unlock) --- playbackController = new PlaybackController(audioMgr); playbackController.initPersistent(); window.playbackController = playbackController; window.audioMgr = audioMgr; window.animationMgr = animationMgr; // Gesture-based audio unlock (mobile browsers require user gesture) const onFirstGesture = async () => { try { await playbackController.unlockOnce(); } catch (e) {} window.removeEventListener('pointerdown', onFirstGesture, { capture: true }); window.removeEventListener('touchend', onFirstGesture, { capture: true }); window.removeEventListener('keydown', onFirstGesture, { capture: true }); }; window.addEventListener('pointerdown', onFirstGesture, { capture: true }); window.addEventListener('touchend', onFirstGesture, { capture: true }); window.addEventListener('keydown', onFirstGesture, { capture: true }); // --- Animation Loop --- clock.start(); function animate() { const deltaTime = clock.getDelta(); if (animationMgr) animationMgr.update(deltaTime); if (movementController) movementController.update(deltaTime); if (vrManager) vrManager.update(); if (vrPositionTracker) vrPositionTracker.update(); vrm.update(deltaTime); renderer.render(scene, camera); if (!vrManager || !vrManager.isInVRSession) { controls.update(); } } if (vrSupported) { renderer.setAnimationLoop(animate); } else { (function loop() { requestAnimationFrame(loop); animate(); })(); } // --- WebSocket Message Handler --- const ws = new WebSocket(WS_URL); ws.onopen = () => console.log('✅ WebSocket connected'); ws.onerror = err => console.error('WS error', err); ws.onmessage = async ({ data }) => { let msg; try { msg = JSON.parse(data); console.log('📨 Message received:', msg); } catch { return; } // --- Movement commands --- if (msg.type === 'walk_to') { const { x, y, z, speed } = msg; if (speed) movementController.setSpeed(speed); movementController.walkTo(x, y, z); } if (msg.type === 'stop_movement') { movementController.stop(); } if (msg.type === 'teleport_to') { const { x, y, z } = msg; movementController.teleportTo(x, y, z); } if (msg.type === 'set_speed') { movementController.setSpeed(msg.speed); } if (msg.type === 'load_walk_animation') { await movementController.loadWalkAnimation(msg.url); } if (msg.type === 'load_idle_animation') { await movementController.loadIdleAnimation(msg.url); } // --- Additive blending --- if (msg.type === 'set_additive_weight') { const { anim_name, weight, duration } = msg; movementController.setAdditiveWeight(anim_name, weight, duration || 0.25); } if (msg.type === 'play_additive_once') { const { anim_name, fade_in, fade_out } = msg; movementController.playAdditiveOnce(anim_name, fade_in || 0.25, fade_out || 0.25); } if (msg.type === 'load_additive_animation') { const { url, name } = msg; await movementController.loadAdditiveAnimation(url, name); } if (msg.type === 'load_and_play_additive') { const { url, name, weight = 1.0, play_once = false, fade_in = 0.25, fade_out = 0.25 } = msg; try { const loaded = await movementController.loadAdditiveAnimation(url, name); if (loaded) { if (play_once) { movementController.playAdditiveOnce(name, fade_in, fade_out); } else { movementController.setAdditiveWeight(name, weight, fade_in); } } } catch (err) { console.error(`Failed to load and play additive animation ${name}:`, err); } } // --- State control --- if (msg.type === 'set_state') { const { state } = msg; if (animationMgr && ['idle', 'listening', 'thinking', 'talking'].includes(state)) { animationMgr.setState(state); } } if (msg.type === 'set_movement_lock_duration') { const { duration } = msg; if (animationMgr && typeof duration === 'number') { animationMgr.setMovementLockDuration(duration); } } // --- Audio playback --- if (msg.type === 'start_animation') { const { audio_path, expression = 'neutral', audio_text, audio_duraction } = msg; audioMgr.setExpression(expression); // Show subtitles while this chunk plays if (audio_text && audio_duraction) { showSubtitleStreaming(audio_text, audio_duraction); } try { try { await playbackController.unlockOnce(); } catch (e) {} const ok = await playbackController.playAudioUrl(audio_path); if (!ok) console.warn('Playback failed (animation will still run)'); animationMgr.play(); } catch (e) { console.error('Failed to start audio/animation:', e); } } // --- VRMA animation --- if (msg.type === 'start_vrma') { const { animation_url, play_once = false, crop_start = 0.0, crop_end = 0.0, lock_position = false, track_position = true, } = msg; await movementController.playVRMA(animation_url, { play_once, crop_start, crop_end, lock_position, track_position, }); } // --- Mixamo animation --- if (msg.type === 'start_mixamo') { const { animation_url, play_once = false, lock_position = false, track_position = true } = msg; await movementController.playMixamo(animation_url, { play_once, lock_position, track_position, }); } // --- Webcam --- if (msg.type === 'take_picture') { await takePictureAndUpload(); } }; })();