429 lines
13 KiB
JavaScript
429 lines
13 KiB
JavaScript
import * as THREE from 'three';
|
|
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
|
|
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
|
|
import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
|
|
import { createVRMAnimationClip, VRMAnimationLoaderPlugin, VRMLookAtQuaternionProxy } from '@pixiv/three-vrm-animation';
|
|
|
|
import { VRM_PATH, WS_URL } from './config.js';
|
|
import { loadVRM } from './vrmLoader.js';
|
|
import { AudioManager } from './audioManager.js';
|
|
import { AnimationManager } from './animationManager.js';
|
|
import { loadMixamoAnimation } from './loadMixamoAnimation.js';
|
|
import { showSubtitleStreaming } from './subtitles.js';
|
|
import { initWebcam, takePictureAndUpload } from './webcam.js';
|
|
import { connectWS, initUI } from "./ui.js";
|
|
import { initVRMClickDetector } from './vrmClickDetector.js';
|
|
|
|
|
|
// app.js — add near top with your other imports (no extra imports needed)
|
|
|
|
// Better trimAnimationClip: construct new tracks (compatible with more three.js builds)
|
|
function trimAnimationClip(clip, startTime, endTime) {
|
|
// startTime/endTime in seconds (endTime is exclusive)
|
|
startTime = Math.max(0, startTime || 0);
|
|
const fullDuration = clip.duration;
|
|
endTime = (typeof endTime === 'number' && endTime >= 0) ? Math.min(fullDuration, endTime) : fullDuration;
|
|
|
|
if (endTime <= startTime) {
|
|
console.warn('trimAnimationClip: invalid range', { startTime, endTime, duration: fullDuration });
|
|
return null;
|
|
}
|
|
|
|
const newTracks = [];
|
|
|
|
for (const track of clip.tracks) {
|
|
// Use the typed arrays directly for speed/compatibility
|
|
const times = track.times; // Float32Array or Array
|
|
const values = track.values; // Float32Array or Array
|
|
const stride = track.getValueSize();
|
|
|
|
const keptTimes = [];
|
|
const keptValues = [];
|
|
|
|
// iterate keyframes and pick those within range
|
|
// times.length should equal values.length / stride
|
|
for (let i = 0; i < times.length; i++) {
|
|
const t = times[i];
|
|
if (t >= startTime && t <= endTime) {
|
|
keptTimes.push(t - startTime); // shift so new clip starts at 0
|
|
const baseIndex = i * stride;
|
|
for (let s = 0; s < stride; s++) {
|
|
keptValues.push(values[baseIndex + s]);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (keptTimes.length > 0) {
|
|
// Create typed arrays expected by KeyframeTrack constructors
|
|
const TimesArray = Float32Array;
|
|
const ValuesArray = Float32Array;
|
|
|
|
const newTimes = new TimesArray(keptTimes);
|
|
const newValues = new ValuesArray(keptValues);
|
|
|
|
// Build a new track using the same constructor (works for Number/Vector/Quaternion)
|
|
// Signature: new TrackConstructor(name, times, values, interpolation?)
|
|
let NewTrack;
|
|
try {
|
|
NewTrack = new track.constructor(track.name, newTimes, newValues, track.getInterpolation ? track.getInterpolation() : undefined);
|
|
} catch (err) {
|
|
// Fallback: try without interpolation argument
|
|
NewTrack = new track.constructor(track.name, newTimes, newValues);
|
|
}
|
|
|
|
newTracks.push(NewTrack);
|
|
}
|
|
}
|
|
|
|
const newDuration = endTime - startTime;
|
|
const newName = `${clip.name || 'vrma_clip'}_trimmed_${startTime.toFixed(3)}-${endTime.toFixed(3)}`;
|
|
return new THREE.AnimationClip(newName, newDuration, newTracks);
|
|
}
|
|
|
|
|
|
|
|
|
|
// Handle messages from server
|
|
function handleServerMessage(msg) {
|
|
//console.log("📩 Received from server:", msg);
|
|
|
|
if (msg.type === "start_animation") {
|
|
// Play animation on VRM
|
|
} else if (msg.type === "take_picture") {
|
|
// Trigger snapshot from camera
|
|
}
|
|
}
|
|
|
|
// Connect WebSocket + UI
|
|
connectWS(handleServerMessage);
|
|
initUI();
|
|
|
|
// Your existing VRM rendering logic continues here...
|
|
|
|
|
|
|
|
|
|
// Initialize webcam once on startup if needed
|
|
await initWebcam();
|
|
|
|
// VRM LOADERS
|
|
|
|
// Global variables
|
|
let currentMixer = null;
|
|
let vrm = null;
|
|
let renderer = null;
|
|
let scene = null;
|
|
let camera = null;
|
|
let controls = null;
|
|
let animationMgr = null; // ADD THIS
|
|
const clock = new THREE.Clock();
|
|
let currentVrm = null;
|
|
let currentAction = null;
|
|
|
|
(async () => {
|
|
// — Renderer / Scene / Camera —
|
|
|
|
// TRANSPARENT BACKGROUND ALPHA
|
|
renderer = new THREE.WebGLRenderer({ alpha: true });
|
|
renderer.setSize(window.innerWidth, window.innerHeight);
|
|
renderer.setPixelRatio(window.devicePixelRatio);
|
|
renderer.shadowMap.enabled = true;
|
|
renderer.shadowMap.type = THREE.PCFSoftShadowMap;
|
|
document.body.appendChild(renderer.domElement);
|
|
|
|
// SCENE
|
|
scene = new THREE.Scene();
|
|
|
|
|
|
|
|
// CAMERA
|
|
camera = new THREE.PerspectiveCamera(30, window.innerWidth/window.innerHeight, 0.1, 20);
|
|
camera.position.set(0,1,0.9);
|
|
|
|
controls = new OrbitControls(camera, renderer.domElement);
|
|
controls.target.set(0,1.1,0);
|
|
controls.update();
|
|
|
|
// helpers
|
|
// const gridHelper = new THREE.GridHelper( 10, 10 );
|
|
// scene.add( gridHelper );
|
|
|
|
// const axesHelper = new THREE.AxesHelper( 5 );
|
|
// scene.add( axesHelper );
|
|
|
|
// — Light —
|
|
const dirLight = new THREE.DirectionalLight(0xffffff, 1);
|
|
dirLight.position.set(3,15,-5);
|
|
scene.add(dirLight);
|
|
scene.add(new THREE.AmbientLight(0xffffff, 2.1));
|
|
|
|
// — Load VRM —
|
|
const vrmData = await loadVRM(VRM_PATH, scene);
|
|
vrm = vrmData.vrm;
|
|
const loader = vrmData.loader;
|
|
currentMixer = new THREE.AnimationMixer(vrm.scene);
|
|
|
|
// Assuming `vrm` is already loaded PRINT ALL BONES
|
|
// vrm.scene.traverse((object) => {
|
|
// if (object.isBone) {
|
|
// console.log(object.name);
|
|
// }
|
|
// });
|
|
|
|
|
|
|
|
// Init click detection
|
|
initVRMClickDetector(renderer, camera, vrm, (region, partName) => {
|
|
// This is where you can send it to LLM or server
|
|
console.log(`📩 Sending region click to server: ${region}`);
|
|
// Example: ws.send(JSON.stringify({ type: "region_click", region }));
|
|
});
|
|
|
|
// — Managers —
|
|
const audioMgr = new AudioManager(vrm);
|
|
animationMgr = new AnimationManager(vrm, audioMgr, renderer, scene, camera, controls);
|
|
|
|
|
|
// Unlock AudioContext on first user interaction
|
|
function unlockAudio() {
|
|
if (audioMgr.audioContext && audioMgr.audioContext.state === "suspended") {
|
|
audioMgr.audioContext.resume().then(() => {
|
|
console.log("🔓 AudioContext unlocked");
|
|
// also unlock <audio> element if needed
|
|
if (audioMgr.audioElement) {
|
|
audioMgr.audioElement.play().then(() => {
|
|
audioMgr.audioElement.pause(); // immediately pause so it's silent
|
|
audioMgr.audioElement.currentTime = 0;
|
|
}).catch(err => {
|
|
console.warn("Audio element unlock failed:", err);
|
|
});
|
|
}
|
|
});
|
|
// remove all listeners so this runs only once
|
|
window.removeEventListener("touchend", unlockAudio);
|
|
window.removeEventListener("click", unlockAudio);
|
|
}
|
|
}
|
|
|
|
// Attach to ANY gesture
|
|
window.addEventListener("touchend", unlockAudio, { once: true });
|
|
window.addEventListener("click", unlockAudio, { once: true });
|
|
|
|
|
|
|
|
// Start the clock and animation loop
|
|
clock.start();
|
|
|
|
// SINGLE ANIMATION LOOP THAT HANDLES EVERYTHING
|
|
function animate() {
|
|
requestAnimationFrame(animate);
|
|
const deltaTime = clock.getDelta();
|
|
|
|
// Update current VRMA mixer if it exists
|
|
if (currentMixer) {
|
|
currentMixer.update(deltaTime);
|
|
}
|
|
|
|
// Update the animation manager (idle animations)
|
|
if (animationMgr) {
|
|
animationMgr.update(deltaTime);
|
|
}
|
|
|
|
// Update VRM and render
|
|
vrm.update(deltaTime);
|
|
renderer.render(scene, camera);
|
|
controls.update();
|
|
}
|
|
|
|
// Start the animation loop once
|
|
animate();
|
|
|
|
// — WebSocket Listener —
|
|
const ws = new WebSocket(WS_URL);
|
|
ws.onopen = () => {
|
|
// console.log('WebSocket connected');
|
|
// document.getElementById('status').textContent = 'WS connected';
|
|
};
|
|
ws.onerror = err => console.error('WS error', err);
|
|
ws.onmessage = async ({ data }) => {
|
|
let msg;
|
|
try {
|
|
msg = JSON.parse(data);
|
|
console.log(msg)
|
|
} catch {
|
|
return;
|
|
}
|
|
|
|
if (msg.type === 'start_animation') {
|
|
const { audio_path, audio_text, audio_duraction, expression = 'neutral' } = msg;
|
|
audioMgr.setExpression(expression);
|
|
try {
|
|
await audioMgr.setupAudio(audio_path);
|
|
await audioMgr.analyzeAudio();
|
|
// SUBTITLE SECTION !!!! UNCOMMENT FOR IN BROWSER MODE, COMMENT OUT FOR OBS MODE
|
|
// stream by word by default, you can stream by letter if you want by adding , "letter"
|
|
// showSubtitleStreaming(audio_text, audio_duraction, "letter"); // 4 seconds total
|
|
// play audio
|
|
|
|
// ✅ Explicitly play audio for mobile
|
|
try {
|
|
await audioMgr.audioElement.play();
|
|
} catch(err) {
|
|
console.warn("Audio play prevented by mobile browser:", err);
|
|
}
|
|
|
|
animationMgr.play();
|
|
} catch (e) {
|
|
console.error('Failed to play animation:', e);
|
|
}
|
|
}
|
|
|
|
if (msg.type === 'start_vrma') {
|
|
const {
|
|
animation_url,
|
|
play_once = false,
|
|
crop_start = 0.0, // seconds to crop off the beginning
|
|
crop_end = 0.0 // seconds to crop off the end
|
|
} = msg;
|
|
|
|
try {
|
|
// Load VRMA as before
|
|
const gltfVrma = await loader.loadAsync(animation_url);
|
|
const vrmAnimation = gltfVrma.userData.vrmAnimations[0];
|
|
let clip = createVRMAnimationClip(vrmAnimation, vrm);
|
|
|
|
// If cropping is requested, compute new start/end times and trim
|
|
// crop_start/crop_end are amounts in seconds to remove
|
|
const startTime = Math.max(0, parseFloat(crop_start) || 0);
|
|
const endTime = Math.max(0, clip.duration - (parseFloat(crop_end) || 0));
|
|
if (startTime > 0 || (parseFloat(crop_end) || 0) > 0) {
|
|
const trimmed = trimAnimationClip(clip, startTime, endTime);
|
|
if (trimmed) {
|
|
clip = trimmed;
|
|
} else {
|
|
console.warn('VRMA trim returned null — using original clip');
|
|
}
|
|
}
|
|
|
|
// Notify animation manager
|
|
animationMgr.setVRMAPlaying(true);
|
|
|
|
// Stop previous action gracefully
|
|
if (currentAction && currentAction.isRunning()) {
|
|
currentAction.crossFadeTo(currentAction, 0.2, false); // small fade-out (safe guard)
|
|
}
|
|
|
|
// Use existing mixer (currentMixer created at VRM load)
|
|
if (!currentMixer) {
|
|
currentMixer = new THREE.AnimationMixer(vrm.scene);
|
|
}
|
|
|
|
// Create action
|
|
const newAction = currentMixer.clipAction(clip);
|
|
|
|
// If play_once requested, set LoopOnce and clamp at end
|
|
if (play_once) {
|
|
newAction.setLoop(THREE.LoopOnce, 0);
|
|
newAction.clampWhenFinished = true;
|
|
newAction.enabled = true;
|
|
} else {
|
|
// default behavior: LoopRepeat (three's default), no clamp
|
|
newAction.setLoop(THREE.LoopRepeat, Infinity);
|
|
newAction.clampWhenFinished = false;
|
|
}
|
|
|
|
// reset and play (with small crossfade from currentAction if present)
|
|
newAction.reset();
|
|
newAction.play();
|
|
|
|
if (currentAction && currentAction !== newAction) {
|
|
currentAction.crossFadeTo(newAction, 0.5, false);
|
|
}
|
|
|
|
// remove previous 'finished' listener if any (to avoid duplicates)
|
|
if (currentMixer._vrmaFinishedListener) {
|
|
try { currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener); } catch (e) {}
|
|
currentMixer._vrmaFinishedListener = null;
|
|
}
|
|
|
|
// Listen to mixer finished events (fires when *an* action finishes).
|
|
const onFinished = (e) => {
|
|
// only react if the finished action is the one we just played
|
|
if (e.action === newAction) {
|
|
animationMgr.setVRMAPlaying(false);
|
|
|
|
// if we played once, stop the action and optionally reset mixer state
|
|
if (play_once) {
|
|
try { newAction.stop(); } catch (err) {}
|
|
}
|
|
|
|
// (optional) null out currentMixer if you want to discard it
|
|
// currentMixer = null;
|
|
|
|
// remove this listener
|
|
if (currentMixer && currentMixer._vrmaFinishedListener) {
|
|
currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener);
|
|
currentMixer._vrmaFinishedListener = null;
|
|
}
|
|
}
|
|
};
|
|
|
|
// attach listener and store it on mixer for cleanup later
|
|
currentMixer._vrmaFinishedListener = onFinished;
|
|
currentMixer.addEventListener('finished', onFinished);
|
|
|
|
currentAction = newAction;
|
|
|
|
} catch (err) {
|
|
console.error("Failed to load VRMA animation:", err);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if (msg.type === 'start_mixamo') {
|
|
const { animation_url } = msg;
|
|
try {
|
|
console.log("THIS IS A MIXAMO ANIMATION")
|
|
console.log(animation_url)
|
|
currentVrm = vrm
|
|
|
|
// Load animation
|
|
const clip = await loadMixamoAnimation( animation_url, currentVrm );
|
|
|
|
// animation blending
|
|
const newAction = currentMixer.clipAction( clip );
|
|
newAction.reset().play();
|
|
|
|
if ( currentAction && currentAction !== newAction ) {
|
|
currentAction.crossFadeTo( newAction, 0.5, false );
|
|
}
|
|
|
|
currentAction = newAction;
|
|
}
|
|
|
|
|
|
catch (err) {
|
|
console.error("Failed to load MIXAMO animation:", err);
|
|
}
|
|
}
|
|
|
|
// take picture
|
|
if (msg.type === 'take_picture') {
|
|
console.log("📸 Received command to take picture from server.");
|
|
await takePictureAndUpload();
|
|
|
|
}
|
|
};
|
|
|
|
|
|
|
|
|
|
// — Handle resize —
|
|
window.addEventListener('resize', () => {
|
|
camera.aspect = window.innerWidth/window.innerHeight;
|
|
camera.updateProjectionMatrix();
|
|
renderer.setSize(window.innerWidth, window.innerHeight);
|
|
});
|
|
|
|
})(); |