319 lines
10 KiB
JavaScript
319 lines
10 KiB
JavaScript
import * as THREE from 'three';
|
|
|
|
import { VRM_PATH, WS_URL } from './config.js';
|
|
import { createScene } from './scene/sceneSetup.js';
|
|
import { loadSceneObjects, describeObjects } from './scene/objectLoader.js';
|
|
import { SCENE_OBJECTS } from './scene/objectsConfig.js';
|
|
import { loadRoom } from './scene/roomLoader.js';
|
|
import { ROOM_CONFIG } from './scene/roomConfig.js';
|
|
import { createLocationRegistry } from './scene/locationRegistry.js';
|
|
import { NAMED_LOCATIONS, DEBUG_MARKERS } from './scene/locationsConfig.js';
|
|
import { loadVRM } from './core/vrmLoader.js';
|
|
import { AudioManager } from './core/audioManager.js';
|
|
import { PlaybackController } from './core/playbackController.js';
|
|
import { AnimationManager } from './animation/animationManager.js';
|
|
import { MovementController } from './animation/movementController.js';
|
|
import { showSubtitleStreaming, setSubtitlesEnabled, getSubtitlesEnabled } from './overlay/subtitles.js';
|
|
import { initWebcam, takePictureAndUpload } from './interaction/webcam.js';
|
|
import { connectWS, initUI } from './ui.js';
|
|
import { initVRMClickDetector } from './interaction/vrmClickDetector.js';
|
|
import { isVRSupported } from './vr/vrManager.js';
|
|
|
|
// --- WebSocket + UI (early init, no dependencies) ---
|
|
function handleServerMessage(msg) {
|
|
console.log("📩 Received from server:", msg);
|
|
}
|
|
connectWS(handleServerMessage);
|
|
initUI();
|
|
|
|
// --- Subtitle toggle ---
|
|
const subtitleBtn = document.getElementById('toggle-subtitles-button');
|
|
if (subtitleBtn) {
|
|
subtitleBtn.addEventListener('click', () => {
|
|
const nowEnabled = !getSubtitlesEnabled();
|
|
setSubtitlesEnabled(nowEnabled);
|
|
subtitleBtn.classList.toggle('off', !nowEnabled);
|
|
});
|
|
}
|
|
|
|
// Initialize webcam
|
|
await initWebcam();
|
|
|
|
// --- App State ---
|
|
let vrm = null;
|
|
let renderer = null;
|
|
let scene = null;
|
|
let camera = null;
|
|
let controls = null;
|
|
let audioMgr = null;
|
|
let animationMgr = null;
|
|
let movementController = null;
|
|
let playbackController = null;
|
|
let vrManager = null;
|
|
let vrPositionTracker = null;
|
|
const clock = new THREE.Clock();
|
|
|
|
(async () => {
|
|
// --- Check VR support ---
|
|
const vrSupported = await isVRSupported();
|
|
if (vrSupported) console.log('WebXR VR supported - VR mode available');
|
|
|
|
// --- Scene Setup ---
|
|
// Adjust these options to change the scene configuration.
|
|
// See scene/sceneSetup.js DEFAULTS for all available options.
|
|
({ renderer, scene, camera, controls } = createScene({
|
|
// background: 0x000000, // Uncomment for black background
|
|
// gridHelper: 50, // Uncomment for grid
|
|
// axesHelper: 5, // Uncomment for axes
|
|
}));
|
|
|
|
// --- Room (environment GLB) ---
|
|
await loadRoom(scene, ROOM_CONFIG);
|
|
|
|
// --- Scene Objects (GLB props) ---
|
|
const sceneObjects = await loadSceneObjects(scene, SCENE_OBJECTS);
|
|
window.sceneObjects = sceneObjects;
|
|
window.describeSceneObjects = () => describeObjects(sceneObjects);
|
|
|
|
// --- Named locations (waypoints + scene objects merged) ---
|
|
const locations = createLocationRegistry({
|
|
scene,
|
|
namedLocations: NAMED_LOCATIONS,
|
|
sceneObjects,
|
|
debugMarkers: DEBUG_MARKERS,
|
|
});
|
|
window.locations = locations;
|
|
console.log('📍 Locations registered:', locations.describe());
|
|
|
|
// --- Load VRM ---
|
|
const vrmData = await loadVRM(VRM_PATH, scene);
|
|
vrm = vrmData.vrm;
|
|
|
|
// --- Click Detection ---
|
|
initVRMClickDetector(renderer, camera, vrm, (region, partName) => {
|
|
console.log(`📩 Region clicked: ${region}`);
|
|
});
|
|
|
|
// --- Managers ---
|
|
audioMgr = new AudioManager(vrm);
|
|
animationMgr = new AnimationManager(vrm, audioMgr, renderer, scene, camera, controls);
|
|
animationMgr.setState('idle');
|
|
|
|
// --- Movement Controller ---
|
|
movementController = new MovementController(vrm, scene, animationMgr);
|
|
await movementController.loadIdleAnimation('./animations/mixamo/Idle.fbx');
|
|
await movementController.loadWalkAnimation('./animations/mixamo/Walking_inplace.fbx');
|
|
await movementController.loadAdditiveAnimation('./animations/mixamo/Waving.fbx', 'wave');
|
|
|
|
// --- VR Setup (conditional) ---
|
|
if (vrSupported) {
|
|
const { VRManager } = await import('./vr/vrManager.js');
|
|
const { VRPositionTracker } = await import('./vr/vrPositionTracker.js');
|
|
|
|
vrManager = new VRManager(renderer, scene, camera, controls);
|
|
vrManager.init();
|
|
|
|
// Enable proximity-based touch interaction on the VRM in VR
|
|
vrManager.initTouchDetection(vrm, {
|
|
onTouch: (region, boneName, handIndex) => {
|
|
console.log(`VR touch: ${region} (hand ${handIndex})`);
|
|
},
|
|
});
|
|
|
|
vrPositionTracker = new VRPositionTracker(camera, vrManager.getDolly());
|
|
|
|
window.addEventListener('vr-session-start', () => vrPositionTracker.enable());
|
|
window.addEventListener('vr-session-end', () => vrPositionTracker.disable());
|
|
window.addEventListener('vr-unlock-audio', async () => {
|
|
try { await playbackController?.unlockOnce(); } catch (e) {}
|
|
});
|
|
|
|
window.vrManager = vrManager;
|
|
window.vrPositionTracker = vrPositionTracker;
|
|
}
|
|
|
|
// --- Playback Controller (mobile audio unlock) ---
|
|
playbackController = new PlaybackController(audioMgr);
|
|
playbackController.initPersistent();
|
|
|
|
window.playbackController = playbackController;
|
|
window.audioMgr = audioMgr;
|
|
window.animationMgr = animationMgr;
|
|
|
|
// Gesture-based audio unlock (mobile browsers require user gesture)
|
|
const onFirstGesture = async () => {
|
|
try { await playbackController.unlockOnce(); } catch (e) {}
|
|
window.removeEventListener('pointerdown', onFirstGesture, { capture: true });
|
|
window.removeEventListener('touchend', onFirstGesture, { capture: true });
|
|
window.removeEventListener('keydown', onFirstGesture, { capture: true });
|
|
};
|
|
window.addEventListener('pointerdown', onFirstGesture, { capture: true });
|
|
window.addEventListener('touchend', onFirstGesture, { capture: true });
|
|
window.addEventListener('keydown', onFirstGesture, { capture: true });
|
|
|
|
// --- Animation Loop ---
|
|
clock.start();
|
|
|
|
function animate() {
|
|
const deltaTime = clock.getDelta();
|
|
|
|
if (animationMgr) animationMgr.update(deltaTime);
|
|
if (movementController) movementController.update(deltaTime);
|
|
if (vrManager) vrManager.update();
|
|
if (vrPositionTracker) vrPositionTracker.update();
|
|
|
|
vrm.update(deltaTime);
|
|
renderer.render(scene, camera);
|
|
|
|
if (!vrManager || !vrManager.isInVRSession) {
|
|
controls.update();
|
|
}
|
|
}
|
|
|
|
if (vrSupported) {
|
|
renderer.setAnimationLoop(animate);
|
|
} else {
|
|
(function loop() {
|
|
requestAnimationFrame(loop);
|
|
animate();
|
|
})();
|
|
}
|
|
|
|
// --- WebSocket Message Handler ---
|
|
const ws = new WebSocket(WS_URL);
|
|
ws.onopen = () => console.log('✅ WebSocket connected');
|
|
ws.onerror = err => console.error('WS error', err);
|
|
|
|
ws.onmessage = async ({ data }) => {
|
|
let msg;
|
|
try {
|
|
msg = JSON.parse(data);
|
|
console.log('📨 Message received:', msg);
|
|
} catch { return; }
|
|
|
|
// --- Movement commands ---
|
|
if (msg.type === 'walk_to') {
|
|
const { x, y, z, speed } = msg;
|
|
if (speed) movementController.setSpeed(speed);
|
|
movementController.walkTo(x, y, z);
|
|
}
|
|
|
|
if (msg.type === 'stop_movement') {
|
|
movementController.stop();
|
|
}
|
|
|
|
if (msg.type === 'teleport_to') {
|
|
const { x, y, z } = msg;
|
|
movementController.teleportTo(x, y, z);
|
|
}
|
|
|
|
if (msg.type === 'set_speed') {
|
|
movementController.setSpeed(msg.speed);
|
|
}
|
|
|
|
if (msg.type === 'load_walk_animation') {
|
|
await movementController.loadWalkAnimation(msg.url);
|
|
}
|
|
|
|
if (msg.type === 'load_idle_animation') {
|
|
await movementController.loadIdleAnimation(msg.url);
|
|
}
|
|
|
|
// --- Additive blending ---
|
|
if (msg.type === 'set_additive_weight') {
|
|
const { anim_name, weight, duration } = msg;
|
|
movementController.setAdditiveWeight(anim_name, weight, duration || 0.25);
|
|
}
|
|
|
|
if (msg.type === 'play_additive_once') {
|
|
const { anim_name, fade_in, fade_out } = msg;
|
|
movementController.playAdditiveOnce(anim_name, fade_in || 0.25, fade_out || 0.25);
|
|
}
|
|
|
|
if (msg.type === 'load_additive_animation') {
|
|
const { url, name } = msg;
|
|
await movementController.loadAdditiveAnimation(url, name);
|
|
}
|
|
|
|
if (msg.type === 'load_and_play_additive') {
|
|
const { url, name, weight = 1.0, play_once = false, fade_in = 0.25, fade_out = 0.25 } = msg;
|
|
try {
|
|
const loaded = await movementController.loadAdditiveAnimation(url, name);
|
|
if (loaded) {
|
|
if (play_once) {
|
|
movementController.playAdditiveOnce(name, fade_in, fade_out);
|
|
} else {
|
|
movementController.setAdditiveWeight(name, weight, fade_in);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error(`Failed to load and play additive animation ${name}:`, err);
|
|
}
|
|
}
|
|
|
|
// --- State control ---
|
|
if (msg.type === 'set_state') {
|
|
const { state } = msg;
|
|
if (animationMgr && ['idle', 'listening', 'thinking', 'talking'].includes(state)) {
|
|
animationMgr.setState(state);
|
|
}
|
|
}
|
|
|
|
if (msg.type === 'set_movement_lock_duration') {
|
|
const { duration } = msg;
|
|
if (animationMgr && typeof duration === 'number') {
|
|
animationMgr.setMovementLockDuration(duration);
|
|
}
|
|
}
|
|
|
|
// --- Audio playback ---
|
|
if (msg.type === 'start_animation') {
|
|
const { audio_path, expression = 'neutral', audio_text, audio_duraction } = msg;
|
|
audioMgr.setExpression(expression);
|
|
|
|
// Show subtitles while this chunk plays
|
|
if (audio_text && audio_duraction) {
|
|
showSubtitleStreaming(audio_text, audio_duraction);
|
|
}
|
|
|
|
try {
|
|
try { await playbackController.unlockOnce(); } catch (e) {}
|
|
const ok = await playbackController.playAudioUrl(audio_path);
|
|
if (!ok) console.warn('Playback failed (animation will still run)');
|
|
animationMgr.play();
|
|
} catch (e) {
|
|
console.error('Failed to start audio/animation:', e);
|
|
}
|
|
}
|
|
|
|
// --- VRMA animation ---
|
|
if (msg.type === 'start_vrma') {
|
|
const {
|
|
animation_url,
|
|
play_once = false,
|
|
crop_start = 0.0,
|
|
crop_end = 0.0,
|
|
lock_position = false,
|
|
track_position = true,
|
|
} = msg;
|
|
await movementController.playVRMA(animation_url, {
|
|
play_once, crop_start, crop_end, lock_position, track_position,
|
|
});
|
|
}
|
|
|
|
// --- Mixamo animation ---
|
|
if (msg.type === 'start_mixamo') {
|
|
const { animation_url, play_once = false, lock_position = false, track_position = true } = msg;
|
|
await movementController.playMixamo(animation_url, {
|
|
play_once, lock_position, track_position,
|
|
});
|
|
}
|
|
|
|
// --- Webcam ---
|
|
if (msg.type === 'take_picture') {
|
|
await takePictureAndUpload();
|
|
}
|
|
};
|
|
|
|
})();
|