490 lines
15 KiB
JavaScript
490 lines
15 KiB
JavaScript
import * as THREE from 'three';
|
||
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
|
||
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
|
||
import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
|
||
import { createVRMAnimationClip, VRMAnimationLoaderPlugin, VRMLookAtQuaternionProxy } from '@pixiv/three-vrm-animation';
|
||
|
||
import { VRM_PATH, WS_URL } from './config.js';
|
||
import { loadVRM } from './vrmLoader.js';
|
||
import { AudioManager } from './audioManager.js';
|
||
import { AnimationManager } from './animationManager.js';
|
||
import { loadMixamoAnimation } from './loadMixamoAnimation.js';
|
||
import { showSubtitleStreaming } from './subtitles.js';
|
||
import { initWebcam, takePictureAndUpload } from './webcam.js';
|
||
import { connectWS, initUI } from "./ui.js";
|
||
import { initVRMClickDetector } from './vrmClickDetector.js';
|
||
|
||
import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js';
|
||
import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js';
|
||
import { BokehPass } from 'three/examples/jsm/postprocessing/BokehPass.js';
|
||
|
||
|
||
// app.js — add near top with your other imports (no extra imports needed)
|
||
|
||
// Better trimAnimationClip: construct new tracks (compatible with more three.js builds)
|
||
function trimAnimationClip(clip, startTime, endTime) {
|
||
// startTime/endTime in seconds (endTime is exclusive)
|
||
startTime = Math.max(0, startTime || 0);
|
||
const fullDuration = clip.duration;
|
||
endTime = (typeof endTime === 'number' && endTime >= 0) ? Math.min(fullDuration, endTime) : fullDuration;
|
||
|
||
if (endTime <= startTime) {
|
||
console.warn('trimAnimationClip: invalid range', { startTime, endTime, duration: fullDuration });
|
||
return null;
|
||
}
|
||
|
||
const newTracks = [];
|
||
|
||
for (const track of clip.tracks) {
|
||
// Use the typed arrays directly for speed/compatibility
|
||
const times = track.times; // Float32Array or Array
|
||
const values = track.values; // Float32Array or Array
|
||
const stride = track.getValueSize();
|
||
|
||
const keptTimes = [];
|
||
const keptValues = [];
|
||
|
||
// iterate keyframes and pick those within range
|
||
// times.length should equal values.length / stride
|
||
for (let i = 0; i < times.length; i++) {
|
||
const t = times[i];
|
||
if (t >= startTime && t <= endTime) {
|
||
keptTimes.push(t - startTime); // shift so new clip starts at 0
|
||
const baseIndex = i * stride;
|
||
for (let s = 0; s < stride; s++) {
|
||
keptValues.push(values[baseIndex + s]);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (keptTimes.length > 0) {
|
||
// Create typed arrays expected by KeyframeTrack constructors
|
||
const TimesArray = Float32Array;
|
||
const ValuesArray = Float32Array;
|
||
|
||
const newTimes = new TimesArray(keptTimes);
|
||
const newValues = new ValuesArray(keptValues);
|
||
|
||
// Build a new track using the same constructor (works for Number/Vector/Quaternion)
|
||
// Signature: new TrackConstructor(name, times, values, interpolation?)
|
||
let NewTrack;
|
||
try {
|
||
NewTrack = new track.constructor(track.name, newTimes, newValues, track.getInterpolation ? track.getInterpolation() : undefined);
|
||
} catch (err) {
|
||
// Fallback: try without interpolation argument
|
||
NewTrack = new track.constructor(track.name, newTimes, newValues);
|
||
}
|
||
|
||
newTracks.push(NewTrack);
|
||
}
|
||
}
|
||
|
||
const newDuration = endTime - startTime;
|
||
const newName = `${clip.name || 'vrma_clip'}_trimmed_${startTime.toFixed(3)}-${endTime.toFixed(3)}`;
|
||
return new THREE.AnimationClip(newName, newDuration, newTracks);
|
||
}
|
||
|
||
|
||
|
||
|
||
// Handle messages from server
|
||
function handleServerMessage(msg) {
|
||
//console.log("📩 Received from server:", msg);
|
||
|
||
if (msg.type === "start_animation") {
|
||
// Play animation on VRM
|
||
} else if (msg.type === "take_picture") {
|
||
// Trigger snapshot from camera
|
||
}
|
||
}
|
||
|
||
// Connect WebSocket + UI
|
||
connectWS(handleServerMessage);
|
||
initUI();
|
||
|
||
// Your existing VRM rendering logic continues here..
|
||
|
||
|
||
|
||
|
||
// Initialize webcam once on startup if needed
|
||
await initWebcam();
|
||
|
||
// VRM LOADERS
|
||
|
||
// Global variables
|
||
let currentMixer = null;
|
||
let vrm = null;
|
||
let renderer = null;
|
||
let scene = null;
|
||
let camera = null;
|
||
let controls = null;
|
||
let animationMgr = null; // ADD THIS
|
||
const clock = new THREE.Clock();
|
||
let currentVrm = null;
|
||
let currentAction = null;
|
||
|
||
(async () => {
|
||
// — Renderer / Scene / Camera —
|
||
|
||
// TRANSPARENT BACKGROUND ALPHA
|
||
renderer = new THREE.WebGLRenderer({ alpha: true });
|
||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||
renderer.setPixelRatio(window.devicePixelRatio);
|
||
renderer.shadowMap.enabled = true;
|
||
renderer.shadowMap.type = THREE.PCFSoftShadowMap;
|
||
document.body.appendChild(renderer.domElement);
|
||
|
||
// SCENE
|
||
scene = new THREE.Scene();
|
||
// load in room
|
||
const glbloader = new GLTFLoader();
|
||
|
||
glbloader.load('./backgrounds/glb/RikoRoomFixed.glb', function(gltf) {
|
||
const model = gltf.scene;
|
||
model.scale.set(1, 1, 1); // Start with no scale reduction
|
||
model.position.set(-3, 0, -0.7); // x,z
|
||
model.rotation.set(0,2.4,0); // use the second number to rorate rooms
|
||
model.traverse((child) => {
|
||
if (child.isMesh) {
|
||
child.geometry.computeBoundingBox();
|
||
child.geometry.computeBoundingSphere();
|
||
|
||
child.material.depthWrite = true;
|
||
child.material.depthTest = true;
|
||
child.material.polygonOffset = true;
|
||
child.material.polygonOffsetFactor = -1;
|
||
child.material.polygonOffsetUnits = -1;
|
||
|
||
child.updateMatrix();
|
||
child.updateMatrixWorld(true);
|
||
|
||
child.geometry.computeVertexNormals();
|
||
}});
|
||
scene.add(model);
|
||
});
|
||
|
||
|
||
// CAMERA
|
||
camera = new THREE.PerspectiveCamera(30, window.innerWidth/window.innerHeight, 0.1, 20);
|
||
camera.position.set(0,1,0.9);
|
||
|
||
controls = new OrbitControls(camera, renderer.domElement);
|
||
controls.target.set(0,1.1,0);
|
||
controls.update();
|
||
|
||
// helpers
|
||
// const gridHelper = new THREE.GridHelper( 10, 10 );
|
||
// scene.add( gridHelper );
|
||
|
||
// const axesHelper = new THREE.AxesHelper( 5 );
|
||
// scene.add( axesHelper );
|
||
|
||
// — Light —
|
||
|
||
// === Adjustable variables ===
|
||
const SOFTBOX_INTENSITY = 3.2; // how bright the softbox is
|
||
const SOFTBOX_DISTANCE = 5; // how far the light reaches
|
||
const SOFTBOX_ANGLE = Math.PI / 5; // spotlight cone angle (narrower = more focused)
|
||
const SOFTBOX_SOFTNESS = 0.6; // penumbra (0 = hard edge, 1 = very soft)
|
||
const AMBIENT_INTENSITY = 1.0; // keep low so background stays dark
|
||
const BACKGROUND_COLOR = 0x111111;
|
||
|
||
// === Scene background ===
|
||
scene.background = new THREE.Color(BACKGROUND_COLOR);
|
||
|
||
// === Ambient light (very low just to avoid pitch black shadows) ===
|
||
const ambientLight = new THREE.AmbientLight(0xffffff, AMBIENT_INTENSITY);
|
||
scene.add(ambientLight);
|
||
|
||
// === Softbox (spotlight approximation) ===
|
||
const softboxLight = new THREE.SpotLight(
|
||
0xffffff, // color
|
||
SOFTBOX_INTENSITY, // intensity
|
||
SOFTBOX_DISTANCE, // distance light reaches
|
||
SOFTBOX_ANGLE, // cone angle
|
||
SOFTBOX_SOFTNESS, // penumbra (softness)
|
||
2 // decay (higher = faster falloff)
|
||
);
|
||
|
||
// Position light in front of model, slightly above
|
||
softboxLight.position.set(1, 1.5, 1);
|
||
|
||
// Aim light at model’s head
|
||
softboxLight.target.position.set(0, 1.3, 0);
|
||
|
||
// Add to scene
|
||
scene.add(softboxLight);
|
||
scene.add(softboxLight.target);
|
||
|
||
// === Optional: visual helper ===
|
||
// Uncomment to see spotlight cone
|
||
// const softboxHelper = new THREE.SpotLightHelper(softboxLight);
|
||
// scene.add(softboxHelper);
|
||
|
||
|
||
// === Optional: visible bulb marker ===
|
||
const lightPoint = new THREE.Mesh(
|
||
new THREE.SphereGeometry(0.05),
|
||
new THREE.MeshBasicMaterial({ color: 0xff0000 })
|
||
);
|
||
lightPoint.position.copy(softboxLight.position);
|
||
scene.add(lightPoint);
|
||
|
||
|
||
// FOCUS ADJUSTMENT
|
||
const composer = new EffectComposer(renderer);
|
||
composer.addPass(new RenderPass(scene, camera));
|
||
|
||
const bokehPass = new BokehPass(scene, camera, {
|
||
focus: 0.9, // distance from camera where it's sharp
|
||
aperture: 0.003, // smaller = stronger blur (try 0.0005 to 0.001)
|
||
maxblur: 0, // max blur radius
|
||
});
|
||
composer.addPass(bokehPass);
|
||
|
||
|
||
// DEV TOOL FOR FOCUS VISUAL
|
||
const focusPoint = new THREE.Mesh(
|
||
new THREE.SphereGeometry(0.02),
|
||
new THREE.MeshBasicMaterial({ color: 0xff0000 })
|
||
);
|
||
focusPoint.position.set(0, 1, 0.9); // match focus distance
|
||
scene.add(focusPoint);
|
||
|
||
// — Load VRM —
|
||
const vrmData = await loadVRM(VRM_PATH, scene);
|
||
vrm = vrmData.vrm;
|
||
const loader = vrmData.loader;
|
||
currentMixer = new THREE.AnimationMixer(vrm.scene);
|
||
|
||
// Assuming `vrm` is already loaded PRINT ALL BONES
|
||
// vrm.scene.traverse((object) => {
|
||
// if (object.isBone) {
|
||
// console.log(object.name);
|
||
// }
|
||
// });
|
||
|
||
|
||
|
||
// Init click detection
|
||
initVRMClickDetector(renderer, camera, vrm, (region, partName) => {
|
||
// This is where you can send it to LLM or server
|
||
console.log(`📩 Sending region click to server: ${region}`);
|
||
// Example: ws.send(JSON.stringify({ type: "region_click", region }));
|
||
});
|
||
|
||
// — Managers —
|
||
const audioMgr = new AudioManager(vrm);
|
||
animationMgr = new AnimationManager(vrm, audioMgr, renderer, scene, camera, controls);
|
||
|
||
// Start the clock and animation loop
|
||
clock.start();
|
||
|
||
// SINGLE ANIMATION LOOP THAT HANDLES EVERYTHING
|
||
function animate() {
|
||
requestAnimationFrame(animate);
|
||
const deltaTime = clock.getDelta();
|
||
|
||
// Update current VRMA mixer if it exists
|
||
if (currentMixer) {
|
||
currentMixer.update(deltaTime);
|
||
}
|
||
|
||
// Update the animation manager (idle animations)
|
||
if (animationMgr) {
|
||
animationMgr.update(deltaTime);
|
||
}
|
||
|
||
// Update VRM and render
|
||
vrm.update(deltaTime);
|
||
// FOR MORE FLAT LOOK USE
|
||
// renderer.render(scene, camera);
|
||
composer.render(scene, camera);
|
||
controls.update();
|
||
}
|
||
|
||
// Start the animation loop once
|
||
animate();
|
||
|
||
// — WebSocket Listener —
|
||
const ws = new WebSocket(WS_URL);
|
||
ws.onopen = () => {
|
||
// console.log('WebSocket connected');
|
||
// document.getElementById('status').textContent = 'WS connected';
|
||
};
|
||
ws.onerror = err => console.error('WS error', err);
|
||
ws.onmessage = async ({ data }) => {
|
||
let msg;
|
||
try {
|
||
msg = JSON.parse(data);
|
||
console.log(msg)
|
||
} catch {
|
||
return;
|
||
}
|
||
|
||
if (msg.type === 'start_animation') {
|
||
const { audio_path, audio_text, audio_duraction, expression = 'neutral' } = msg;
|
||
audioMgr.setExpression(expression);
|
||
try {
|
||
await audioMgr.setupAudio(audio_path);
|
||
await audioMgr.analyzeAudio();
|
||
// SUBTITLE SECTION !!!! UNCOMMENT FOR IN BROWSER MODE, COMMENT OUT FOR OBS MODE
|
||
// stream by word by default, you can stream by letter if you want by adding , "letter"
|
||
// showSubtitleStreaming(audio_text, audio_duraction, "letter"); // 4 seconds total
|
||
// play audio
|
||
animationMgr.play();
|
||
} catch (e) {
|
||
console.error('Failed to play animation:', e);
|
||
}
|
||
}
|
||
|
||
if (msg.type === 'start_vrma') {
|
||
const {
|
||
animation_url,
|
||
play_once = false,
|
||
crop_start = 0.0, // seconds to crop off the beginning
|
||
crop_end = 0.0 // seconds to crop off the end
|
||
} = msg;
|
||
|
||
try {
|
||
// Load VRMA as before
|
||
const gltfVrma = await loader.loadAsync(animation_url);
|
||
const vrmAnimation = gltfVrma.userData.vrmAnimations[0];
|
||
let clip = createVRMAnimationClip(vrmAnimation, vrm);
|
||
|
||
// If cropping is requested, compute new start/end times and trim
|
||
// crop_start/crop_end are amounts in seconds to remove
|
||
const startTime = Math.max(0, parseFloat(crop_start) || 0);
|
||
const endTime = Math.max(0, clip.duration - (parseFloat(crop_end) || 0));
|
||
if (startTime > 0 || (parseFloat(crop_end) || 0) > 0) {
|
||
const trimmed = trimAnimationClip(clip, startTime, endTime);
|
||
if (trimmed) {
|
||
clip = trimmed;
|
||
} else {
|
||
console.warn('VRMA trim returned null — using original clip');
|
||
}
|
||
}
|
||
|
||
// Notify animation manager
|
||
animationMgr.setVRMAPlaying(true);
|
||
|
||
// Stop previous action gracefully
|
||
if (currentAction && currentAction.isRunning()) {
|
||
currentAction.crossFadeTo(currentAction, 0.2, false); // small fade-out (safe guard)
|
||
}
|
||
|
||
// Use existing mixer (currentMixer created at VRM load)
|
||
if (!currentMixer) {
|
||
currentMixer = new THREE.AnimationMixer(vrm.scene);
|
||
}
|
||
|
||
// Create action
|
||
const newAction = currentMixer.clipAction(clip);
|
||
|
||
// If play_once requested, set LoopOnce and clamp at end
|
||
if (play_once) {
|
||
newAction.setLoop(THREE.LoopOnce, 0);
|
||
newAction.clampWhenFinished = true;
|
||
newAction.enabled = true;
|
||
} else {
|
||
// default behavior: LoopRepeat (three's default), no clamp
|
||
newAction.setLoop(THREE.LoopRepeat, Infinity);
|
||
newAction.clampWhenFinished = false;
|
||
}
|
||
|
||
// reset and play (with small crossfade from currentAction if present)
|
||
newAction.reset();
|
||
newAction.play();
|
||
|
||
if (currentAction && currentAction !== newAction) {
|
||
currentAction.crossFadeTo(newAction, 0.5, false);
|
||
}
|
||
|
||
// remove previous 'finished' listener if any (to avoid duplicates)
|
||
if (currentMixer._vrmaFinishedListener) {
|
||
try { currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener); } catch (e) {}
|
||
currentMixer._vrmaFinishedListener = null;
|
||
}
|
||
|
||
// Listen to mixer finished events (fires when *an* action finishes).
|
||
const onFinished = (e) => {
|
||
// only react if the finished action is the one we just played
|
||
if (e.action === newAction) {
|
||
animationMgr.setVRMAPlaying(false);
|
||
|
||
// if we played once, stop the action and optionally reset mixer state
|
||
if (play_once) {
|
||
try { newAction.stop(); } catch (err) {}
|
||
}
|
||
|
||
// (optional) null out currentMixer if you want to discard it
|
||
// currentMixer = null;
|
||
|
||
// remove this listener
|
||
if (currentMixer && currentMixer._vrmaFinishedListener) {
|
||
currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener);
|
||
currentMixer._vrmaFinishedListener = null;
|
||
}
|
||
}
|
||
};
|
||
|
||
// attach listener and store it on mixer for cleanup later
|
||
currentMixer._vrmaFinishedListener = onFinished;
|
||
currentMixer.addEventListener('finished', onFinished);
|
||
|
||
currentAction = newAction;
|
||
|
||
} catch (err) {
|
||
console.error("Failed to load VRMA animation:", err);
|
||
}
|
||
}
|
||
|
||
|
||
|
||
if (msg.type === 'start_mixamo') {
|
||
const { animation_url } = msg;
|
||
try {
|
||
console.log("THIS IS A MIXAMO ANIMATION")
|
||
console.log(animation_url)
|
||
currentVrm = vrm
|
||
|
||
// Load animation
|
||
const clip = await loadMixamoAnimation( animation_url, currentVrm );
|
||
|
||
// animation blending
|
||
const newAction = currentMixer.clipAction( clip );
|
||
newAction.reset().play();
|
||
|
||
if ( currentAction && currentAction !== newAction ) {
|
||
currentAction.crossFadeTo( newAction, 0.5, false );
|
||
}
|
||
|
||
currentAction = newAction;
|
||
}
|
||
|
||
|
||
catch (err) {
|
||
console.error("Failed to load MIXAMO animation:", err);
|
||
}
|
||
}
|
||
|
||
// take picture
|
||
if (msg.type === 'take_picture') {
|
||
console.log("📸 Received command to take picture from server.");
|
||
await takePictureAndUpload();
|
||
|
||
}
|
||
};
|
||
|
||
|
||
|
||
|
||
// — Handle resize —
|
||
window.addEventListener('resize', () => {
|
||
camera.aspect = window.innerWidth/window.innerHeight;
|
||
camera.updateProjectionMatrix();
|
||
renderer.setSize(window.innerWidth, window.innerHeight);
|
||
});
|
||
|
||
})(); |