Ai_Assistant/_Backup/client/app-room.js

import * as THREE from 'three';
import { OrbitControls }   from 'three/addons/controls/OrbitControls.js';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
import { createVRMAnimationClip, VRMAnimationLoaderPlugin, VRMLookAtQuaternionProxy } from '@pixiv/three-vrm-animation';

import { VRM_PATH, WS_URL }       from './config.js';
import { loadVRM }                from './vrmLoader.js';
import { AudioManager }           from './audioManager.js';
import { AnimationManager }       from './animationManager.js';
import { loadMixamoAnimation } from './loadMixamoAnimation.js';
import { showSubtitleStreaming } from './subtitles.js';

// Global variables
let currentMixer = null;
let vrm = null;
let renderer = null;
let scene = null;
let camera = null;
let controls = null;
let animationMgr = null; // ADD THIS
const clock = new THREE.Clock();
let currentVrm = null;
let currentAction = null;

(async () => {
  // — Renderer / Scene / Camera —

  // TRANSPARENT BACKGROUND ALPHA 
  renderer = new THREE.WebGLRenderer({ alpha: true });
  renderer.setSize(window.innerWidth, window.innerHeight);
  renderer.setPixelRatio(window.devicePixelRatio);
  renderer.shadowMap.enabled = true;
  renderer.shadowMap.type = THREE.PCFSoftShadowMap;
  document.body.appendChild(renderer.domElement);

  // SCENE
  scene = new THREE.Scene();


      // load in room 
  const glbloader = new GLTFLoader();

    glbloader.load('./backgrounds/glb/anime_class_room.glb', function(gltf) {
    const model = gltf.scene;
    model.scale.set(0.45, 0.45, 0.45); // Start with no scale reduction
    model.position.set(-2.7, 0, -0.7); // x,z
    model.rotation.set(0,2.3,0); // use the second number to rorate rooms 
    model.traverse((child) => {
      if (child.isMesh) {
        child.geometry.computeBoundingBox();
        child.geometry.computeBoundingSphere();
        
        child.material.depthWrite = true;
        child.material.depthTest = true;
        child.material.polygonOffset = true;
        child.material.polygonOffsetFactor = -1;
        child.material.polygonOffsetUnits = -1;

        child.updateMatrix();
        child.updateMatrixWorld(true);
        
        child.geometry.computeVertexNormals();
      }});    
    scene.add(model);
  });

  // CAMERA
  camera = new THREE.PerspectiveCamera(30, window.innerWidth/window.innerHeight, 0.1, 20);
  camera.position.set(0,1,0.9);

  controls = new OrbitControls(camera, renderer.domElement);
  controls.target.set(0,1.1,0);
  controls.update();

  // helpers
  // const gridHelper = new THREE.GridHelper( 10, 10 );
  // scene.add( gridHelper );

  // const axesHelper = new THREE.AxesHelper( 5 );
  // scene.add( axesHelper );

  // — Light —
  const dirLight = new THREE.DirectionalLight(0xffffff, 1);
  dirLight.position.set(3,15,-5);
  scene.add(dirLight);
  scene.add(new THREE.AmbientLight(0xffffff, 2.1));

  // — Load VRM —
  const vrmData = await loadVRM(VRM_PATH, scene);
  vrm = vrmData.vrm;
  const loader = vrmData.loader;
  currentMixer = new THREE.AnimationMixer(vrm.scene);

  // — Managers —
  const audioMgr = new AudioManager(vrm);
  animationMgr = new AnimationManager(vrm, audioMgr, renderer, scene, camera, controls);

  // Start the clock and animation loop
  clock.start();
  
  // SINGLE ANIMATION LOOP THAT HANDLES EVERYTHING
  function animate() {
    requestAnimationFrame(animate);
    const deltaTime = clock.getDelta();
    
    // Update current VRMA mixer if it exists
    if (currentMixer) {
      currentMixer.update(deltaTime);
    }
    
    // Update the animation manager (idle animations)
    if (animationMgr) {
      animationMgr.update(deltaTime);
    }
    
    // Update VRM and render
    vrm.update(deltaTime);
    renderer.render(scene, camera);
    controls.update();
  }
  
  // Start the animation loop once
  animate();

  // — WebSocket Listener —
  const ws = new WebSocket(WS_URL);
  ws.onopen = () => {
    // console.log('WebSocket connected');
    // document.getElementById('status').textContent = 'WS connected';
  };
  ws.onerror = err => console.error('WS error', err);
  ws.onmessage = async ({ data }) => {
    let msg;
    try {
      msg = JSON.parse(data);
      console.log(msg)
    } catch {
      return;
    }
    
    if (msg.type === 'start_animation') {
      const { audio_path, audio_text, audio_duraction, expression = 'neutral' } = msg;
      audioMgr.setExpression(expression);
      try {
        await audioMgr.setupAudio(audio_path);
        await audioMgr.analyzeAudio();
        // SUBTITLE SECTION !!!! UNCOMMENT FOR IN BROWSER MODE, COMMENT OUT FOR OBS MODE
        // stream by word by default, you can stream by letter if you want by adding , "letter"
        // showSubtitleStreaming(audio_text, audio_duraction, "letter"); // 4 seconds total
        // play audio 
        animationMgr.play();
      } catch (e) {
        console.error('Failed to play animation:', e);
      }
    }

    if (msg.type === 'start_vrma') {
      const { animation_url } = msg;
      try {
        // Load VRMA
        const gltfVrma = await loader.loadAsync(animation_url);
        const vrmAnimation = gltfVrma.userData.vrmAnimations[0];

        // Create animation clip
        const clip = createVRMAnimationClip(vrmAnimation, vrm);
        
        // Stop previous mixer if it exists
        // if (currentMixer) {
        //   currentMixer.stopAllAction();
        // }
        
        // Tell AnimationManager that VRMA is playing
        animationMgr.setVRMAPlaying(true);
        
        // Create new mixer and assign to global variable

        // animation blending
        const newAction = currentMixer.clipAction( clip );
        // newAction.setLoop(THREE.LoopOnce, 1);
        newAction.reset().play();


        if ( currentAction && currentAction !== newAction ) {
        currentAction.crossFadeTo( newAction, 0.5, false );
      }

        currentAction = newAction;
        
        // Listen for when the animation finishes
        action.addEventListener('finished', () => {
          animationMgr.setVRMAPlaying(false);
          currentMixer = null;
        });
        
      } catch (err) {
        console.error("Failed to load VRMA animation:", err);
      }
    }


    if (msg.type === 'start_mixamo') {
      const { animation_url } = msg;
      try {
        console.log("THIS IS A MIXAMO ANIMATION")
        console.log(animation_url)
        currentVrm = vrm

        // Load animation
		    const clip = await loadMixamoAnimation( animation_url, currentVrm );

        // animation blending
        const newAction = currentMixer.clipAction( clip );
        newAction.reset().play();

        if ( currentAction && currentAction !== newAction ) {
        currentAction.crossFadeTo( newAction, 0.5, false );
      }

      currentAction = newAction;
	} 

        
      catch (err) {
        console.error("Failed to load MIXAMO animation:", err);
      }
    }
  };

  // — Handle resize —
  window.addEventListener('resize', () => {
    camera.aspect = window.innerWidth/window.innerHeight;
    camera.updateProjectionMatrix();
    renderer.setSize(window.innerWidth, window.innerHeight);
  });

})();