Ai_Assistant/client/_archive/app-RIKOROOM.js

490 lines
15 KiB
JavaScript
Raw Normal View History

2026-05-24 13:31:30 +02:00
import * as THREE from 'three';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
import { createVRMAnimationClip, VRMAnimationLoaderPlugin, VRMLookAtQuaternionProxy } from '@pixiv/three-vrm-animation';
import { VRM_PATH, WS_URL } from './config.js';
import { loadVRM } from './vrmLoader.js';
import { AudioManager } from './audioManager.js';
import { AnimationManager } from './animationManager.js';
import { loadMixamoAnimation } from './loadMixamoAnimation.js';
import { showSubtitleStreaming } from './subtitles.js';
import { initWebcam, takePictureAndUpload } from './webcam.js';
import { connectWS, initUI } from "./ui.js";
import { initVRMClickDetector } from './vrmClickDetector.js';
import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js';
import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js';
import { BokehPass } from 'three/examples/jsm/postprocessing/BokehPass.js';
// app.js — add near top with your other imports (no extra imports needed)
// Better trimAnimationClip: construct new tracks (compatible with more three.js builds)
function trimAnimationClip(clip, startTime, endTime) {
// startTime/endTime in seconds (endTime is exclusive)
startTime = Math.max(0, startTime || 0);
const fullDuration = clip.duration;
endTime = (typeof endTime === 'number' && endTime >= 0) ? Math.min(fullDuration, endTime) : fullDuration;
if (endTime <= startTime) {
console.warn('trimAnimationClip: invalid range', { startTime, endTime, duration: fullDuration });
return null;
}
const newTracks = [];
for (const track of clip.tracks) {
// Use the typed arrays directly for speed/compatibility
const times = track.times; // Float32Array or Array
const values = track.values; // Float32Array or Array
const stride = track.getValueSize();
const keptTimes = [];
const keptValues = [];
// iterate keyframes and pick those within range
// times.length should equal values.length / stride
for (let i = 0; i < times.length; i++) {
const t = times[i];
if (t >= startTime && t <= endTime) {
keptTimes.push(t - startTime); // shift so new clip starts at 0
const baseIndex = i * stride;
for (let s = 0; s < stride; s++) {
keptValues.push(values[baseIndex + s]);
}
}
}
if (keptTimes.length > 0) {
// Create typed arrays expected by KeyframeTrack constructors
const TimesArray = Float32Array;
const ValuesArray = Float32Array;
const newTimes = new TimesArray(keptTimes);
const newValues = new ValuesArray(keptValues);
// Build a new track using the same constructor (works for Number/Vector/Quaternion)
// Signature: new TrackConstructor(name, times, values, interpolation?)
let NewTrack;
try {
NewTrack = new track.constructor(track.name, newTimes, newValues, track.getInterpolation ? track.getInterpolation() : undefined);
} catch (err) {
// Fallback: try without interpolation argument
NewTrack = new track.constructor(track.name, newTimes, newValues);
}
newTracks.push(NewTrack);
}
}
const newDuration = endTime - startTime;
const newName = `${clip.name || 'vrma_clip'}_trimmed_${startTime.toFixed(3)}-${endTime.toFixed(3)}`;
return new THREE.AnimationClip(newName, newDuration, newTracks);
}
// Handle messages from server
function handleServerMessage(msg) {
//console.log("📩 Received from server:", msg);
if (msg.type === "start_animation") {
// Play animation on VRM
} else if (msg.type === "take_picture") {
// Trigger snapshot from camera
}
}
// Connect WebSocket + UI
connectWS(handleServerMessage);
initUI();
// Your existing VRM rendering logic continues here..
// Initialize webcam once on startup if needed
await initWebcam();
// VRM LOADERS
// Global variables
let currentMixer = null;
let vrm = null;
let renderer = null;
let scene = null;
let camera = null;
let controls = null;
let animationMgr = null; // ADD THIS
const clock = new THREE.Clock();
let currentVrm = null;
let currentAction = null;
(async () => {
// — Renderer / Scene / Camera —
// TRANSPARENT BACKGROUND ALPHA
renderer = new THREE.WebGLRenderer({ alpha: true });
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.setPixelRatio(window.devicePixelRatio);
renderer.shadowMap.enabled = true;
renderer.shadowMap.type = THREE.PCFSoftShadowMap;
document.body.appendChild(renderer.domElement);
// SCENE
scene = new THREE.Scene();
// load in room
const glbloader = new GLTFLoader();
glbloader.load('./backgrounds/glb/RikoRoomFixed.glb', function(gltf) {
const model = gltf.scene;
model.scale.set(1, 1, 1); // Start with no scale reduction
model.position.set(-3, 0, -0.7); // x,z
model.rotation.set(0,2.4,0); // use the second number to rorate rooms
model.traverse((child) => {
if (child.isMesh) {
child.geometry.computeBoundingBox();
child.geometry.computeBoundingSphere();
child.material.depthWrite = true;
child.material.depthTest = true;
child.material.polygonOffset = true;
child.material.polygonOffsetFactor = -1;
child.material.polygonOffsetUnits = -1;
child.updateMatrix();
child.updateMatrixWorld(true);
child.geometry.computeVertexNormals();
}});
scene.add(model);
});
// CAMERA
camera = new THREE.PerspectiveCamera(30, window.innerWidth/window.innerHeight, 0.1, 20);
camera.position.set(0,1,0.9);
controls = new OrbitControls(camera, renderer.domElement);
controls.target.set(0,1.1,0);
controls.update();
// helpers
// const gridHelper = new THREE.GridHelper( 10, 10 );
// scene.add( gridHelper );
// const axesHelper = new THREE.AxesHelper( 5 );
// scene.add( axesHelper );
// — Light —
// === Adjustable variables ===
const SOFTBOX_INTENSITY = 3.2; // how bright the softbox is
const SOFTBOX_DISTANCE = 5; // how far the light reaches
const SOFTBOX_ANGLE = Math.PI / 5; // spotlight cone angle (narrower = more focused)
const SOFTBOX_SOFTNESS = 0.6; // penumbra (0 = hard edge, 1 = very soft)
const AMBIENT_INTENSITY = 1.0; // keep low so background stays dark
const BACKGROUND_COLOR = 0x111111;
// === Scene background ===
scene.background = new THREE.Color(BACKGROUND_COLOR);
// === Ambient light (very low just to avoid pitch black shadows) ===
const ambientLight = new THREE.AmbientLight(0xffffff, AMBIENT_INTENSITY);
scene.add(ambientLight);
// === Softbox (spotlight approximation) ===
const softboxLight = new THREE.SpotLight(
0xffffff, // color
SOFTBOX_INTENSITY, // intensity
SOFTBOX_DISTANCE, // distance light reaches
SOFTBOX_ANGLE, // cone angle
SOFTBOX_SOFTNESS, // penumbra (softness)
2 // decay (higher = faster falloff)
);
// Position light in front of model, slightly above
softboxLight.position.set(1, 1.5, 1);
// Aim light at models head
softboxLight.target.position.set(0, 1.3, 0);
// Add to scene
scene.add(softboxLight);
scene.add(softboxLight.target);
// === Optional: visual helper ===
// Uncomment to see spotlight cone
// const softboxHelper = new THREE.SpotLightHelper(softboxLight);
// scene.add(softboxHelper);
// === Optional: visible bulb marker ===
const lightPoint = new THREE.Mesh(
new THREE.SphereGeometry(0.05),
new THREE.MeshBasicMaterial({ color: 0xff0000 })
);
lightPoint.position.copy(softboxLight.position);
scene.add(lightPoint);
// FOCUS ADJUSTMENT
const composer = new EffectComposer(renderer);
composer.addPass(new RenderPass(scene, camera));
const bokehPass = new BokehPass(scene, camera, {
focus: 0.9, // distance from camera where it's sharp
aperture: 0.003, // smaller = stronger blur (try 0.0005 to 0.001)
maxblur: 0, // max blur radius
});
composer.addPass(bokehPass);
// DEV TOOL FOR FOCUS VISUAL
const focusPoint = new THREE.Mesh(
new THREE.SphereGeometry(0.02),
new THREE.MeshBasicMaterial({ color: 0xff0000 })
);
focusPoint.position.set(0, 1, 0.9); // match focus distance
scene.add(focusPoint);
// — Load VRM —
const vrmData = await loadVRM(VRM_PATH, scene);
vrm = vrmData.vrm;
const loader = vrmData.loader;
currentMixer = new THREE.AnimationMixer(vrm.scene);
// Assuming `vrm` is already loaded PRINT ALL BONES
// vrm.scene.traverse((object) => {
// if (object.isBone) {
// console.log(object.name);
// }
// });
// Init click detection
initVRMClickDetector(renderer, camera, vrm, (region, partName) => {
// This is where you can send it to LLM or server
console.log(`📩 Sending region click to server: ${region}`);
// Example: ws.send(JSON.stringify({ type: "region_click", region }));
});
// — Managers —
const audioMgr = new AudioManager(vrm);
animationMgr = new AnimationManager(vrm, audioMgr, renderer, scene, camera, controls);
// Start the clock and animation loop
clock.start();
// SINGLE ANIMATION LOOP THAT HANDLES EVERYTHING
function animate() {
requestAnimationFrame(animate);
const deltaTime = clock.getDelta();
// Update current VRMA mixer if it exists
if (currentMixer) {
currentMixer.update(deltaTime);
}
// Update the animation manager (idle animations)
if (animationMgr) {
animationMgr.update(deltaTime);
}
// Update VRM and render
vrm.update(deltaTime);
// FOR MORE FLAT LOOK USE
// renderer.render(scene, camera);
composer.render(scene, camera);
controls.update();
}
// Start the animation loop once
animate();
// — WebSocket Listener —
const ws = new WebSocket(WS_URL);
ws.onopen = () => {
// console.log('WebSocket connected');
// document.getElementById('status').textContent = 'WS connected';
};
ws.onerror = err => console.error('WS error', err);
ws.onmessage = async ({ data }) => {
let msg;
try {
msg = JSON.parse(data);
console.log(msg)
} catch {
return;
}
if (msg.type === 'start_animation') {
const { audio_path, audio_text, audio_duraction, expression = 'neutral' } = msg;
audioMgr.setExpression(expression);
try {
await audioMgr.setupAudio(audio_path);
await audioMgr.analyzeAudio();
// SUBTITLE SECTION !!!! UNCOMMENT FOR IN BROWSER MODE, COMMENT OUT FOR OBS MODE
// stream by word by default, you can stream by letter if you want by adding , "letter"
// showSubtitleStreaming(audio_text, audio_duraction, "letter"); // 4 seconds total
// play audio
animationMgr.play();
} catch (e) {
console.error('Failed to play animation:', e);
}
}
if (msg.type === 'start_vrma') {
const {
animation_url,
play_once = false,
crop_start = 0.0, // seconds to crop off the beginning
crop_end = 0.0 // seconds to crop off the end
} = msg;
try {
// Load VRMA as before
const gltfVrma = await loader.loadAsync(animation_url);
const vrmAnimation = gltfVrma.userData.vrmAnimations[0];
let clip = createVRMAnimationClip(vrmAnimation, vrm);
// If cropping is requested, compute new start/end times and trim
// crop_start/crop_end are amounts in seconds to remove
const startTime = Math.max(0, parseFloat(crop_start) || 0);
const endTime = Math.max(0, clip.duration - (parseFloat(crop_end) || 0));
if (startTime > 0 || (parseFloat(crop_end) || 0) > 0) {
const trimmed = trimAnimationClip(clip, startTime, endTime);
if (trimmed) {
clip = trimmed;
} else {
console.warn('VRMA trim returned null — using original clip');
}
}
// Notify animation manager
animationMgr.setVRMAPlaying(true);
// Stop previous action gracefully
if (currentAction && currentAction.isRunning()) {
currentAction.crossFadeTo(currentAction, 0.2, false); // small fade-out (safe guard)
}
// Use existing mixer (currentMixer created at VRM load)
if (!currentMixer) {
currentMixer = new THREE.AnimationMixer(vrm.scene);
}
// Create action
const newAction = currentMixer.clipAction(clip);
// If play_once requested, set LoopOnce and clamp at end
if (play_once) {
newAction.setLoop(THREE.LoopOnce, 0);
newAction.clampWhenFinished = true;
newAction.enabled = true;
} else {
// default behavior: LoopRepeat (three's default), no clamp
newAction.setLoop(THREE.LoopRepeat, Infinity);
newAction.clampWhenFinished = false;
}
// reset and play (with small crossfade from currentAction if present)
newAction.reset();
newAction.play();
if (currentAction && currentAction !== newAction) {
currentAction.crossFadeTo(newAction, 0.5, false);
}
// remove previous 'finished' listener if any (to avoid duplicates)
if (currentMixer._vrmaFinishedListener) {
try { currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener); } catch (e) {}
currentMixer._vrmaFinishedListener = null;
}
// Listen to mixer finished events (fires when *an* action finishes).
const onFinished = (e) => {
// only react if the finished action is the one we just played
if (e.action === newAction) {
animationMgr.setVRMAPlaying(false);
// if we played once, stop the action and optionally reset mixer state
if (play_once) {
try { newAction.stop(); } catch (err) {}
}
// (optional) null out currentMixer if you want to discard it
// currentMixer = null;
// remove this listener
if (currentMixer && currentMixer._vrmaFinishedListener) {
currentMixer.removeEventListener('finished', currentMixer._vrmaFinishedListener);
currentMixer._vrmaFinishedListener = null;
}
}
};
// attach listener and store it on mixer for cleanup later
currentMixer._vrmaFinishedListener = onFinished;
currentMixer.addEventListener('finished', onFinished);
currentAction = newAction;
} catch (err) {
console.error("Failed to load VRMA animation:", err);
}
}
if (msg.type === 'start_mixamo') {
const { animation_url } = msg;
try {
console.log("THIS IS A MIXAMO ANIMATION")
console.log(animation_url)
currentVrm = vrm
// Load animation
const clip = await loadMixamoAnimation( animation_url, currentVrm );
// animation blending
const newAction = currentMixer.clipAction( clip );
newAction.reset().play();
if ( currentAction && currentAction !== newAction ) {
currentAction.crossFadeTo( newAction, 0.5, false );
}
currentAction = newAction;
}
catch (err) {
console.error("Failed to load MIXAMO animation:", err);
}
}
// take picture
if (msg.type === 'take_picture') {
console.log("📸 Received command to take picture from server.");
await takePictureAndUpload();
}
};
// — Handle resize —
window.addEventListener('resize', () => {
camera.aspect = window.innerWidth/window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
});
})();