// app.js (rewritten) - persistent audio + single-gesture unlock + VRM integration import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"; import { createVRMAnimationClip, VRMAnimationLoaderPlugin, VRMLookAtQuaternionProxy, } from "@pixiv/three-vrm-animation"; import { VRM_PATH, WS_URL } from "./config.js"; import { loadVRM } from "./vrmLoader.js"; import { AudioManager } from "./audioManager.js"; import { AnimationManager } from "./animationManager.js"; import { loadMixamoAnimation } from "./loadMixamoAnimation.js"; import { showSubtitleStreaming } from "./subtitles.js"; import { initWebcam, takePictureAndUpload } from "./webcam.js"; import { connectWS, initUI } from "./ui.js"; import { initVRMClickDetector } from "./vrmClickDetector.js"; // ---- small helper ------------------------------------------------ function ensureAbsoluteUrl(url) { try { new URL(url); return url; } catch (e) { if (!url) return url; if (url.startsWith("/")) return `${location.origin}${url}`; return `${location.origin}/${url}`; } } // ---- trimAnimationClip (same as yours, kept) --------------------- function trimAnimationClip(clip, startTime, endTime) { startTime = Math.max(0, startTime || 0); const fullDuration = clip.duration; endTime = typeof endTime === "number" && endTime >= 0 ? Math.min(fullDuration, endTime) : fullDuration; if (endTime <= startTime) { console.warn("trimAnimationClip: invalid range", { startTime, endTime, duration: fullDuration, }); return null; } const newTracks = []; for (const track of clip.tracks) { const times = track.times; const values = track.values; const stride = track.getValueSize(); const keptTimes = []; const keptValues = []; for (let i = 0; i < times.length; i++) { const t = times[i]; if (t >= startTime && t <= endTime) { keptTimes.push(t - startTime); const baseIndex = i * stride; for (let s = 0; s < stride; s++) keptValues.push(values[baseIndex + s]); } } if (keptTimes.length > 0) { const TimesArray = Float32Array; const ValuesArray = Float32Array; const newTimes = new TimesArray(keptTimes); const newValues = new ValuesArray(keptValues); let NewTrack; try { NewTrack = new track.constructor( track.name, newTimes, newValues, track.getInterpolation ? track.getInterpolation() : undefined, ); } catch (err) { NewTrack = new track.constructor( track.name, newTimes, newValues, ); } newTracks.push(NewTrack); } } const newDuration = endTime - startTime; const newName = `${clip.name || "vrma_clip"}_trimmed_${startTime.toFixed(3)}-${endTime.toFixed(3)}`; return new THREE.AnimationClip(newName, newDuration, newTracks); } // ---- connect UI early to avoid race issues ------------------------ function handleServerMessage(msg) { /* kept for compatibility; actual handling below */ } connectWS(handleServerMessage); initUI(); // ---- webcam init (await) ----------------------------------------- await initWebcam(); // ---- globals ----------------------------------------------------- let currentMixer = null; let vrm = null; let renderer = null; let scene = null; let camera = null; let controls = null; let animationMgr = null; const clock = new THREE.Clock(); let currentVrm = null; let currentAction = null; // ---- PlaybackController (single-file, persistent) ---------------- class PlaybackController { constructor(audioMgr) { this.audioMgr = audioMgr; this._inited = false; this._unlocked = false; this.el = null; // persistent audio element this._analyserAttached = false; } initPersistent() { if (this._inited) return; this._inited = true; // create or reuse audio element and attach to audioMgr if (!this.audioMgr.audioElement) { const a = document.createElement("audio"); a.crossOrigin = "anonymous"; a.preload = "auto"; a.playsInline = true; a.setAttribute("playsinline", ""); a.setAttribute("webkit-playsinline", ""); a.style.display = "none"; document.body.appendChild(a); this.audioMgr.audioElement = a; } this.el = this.audioMgr.audioElement; // try to create audioContext & analyser if audioMgr doesn't already have them try { if (!this.audioMgr.audioContext) { const AC = window.AudioContext || window.webkitAudioContext; if (AC) { this.audioMgr.audioContext = new AC(); // keep it suspended until user gesture } } // if we have context but not analyser, create one and hook element if (this.audioMgr.audioContext && !this.audioMgr.analyser) { try { const src = this.audioMgr.audioContext.createMediaElementSource( this.el, ); const analyser = this.audioMgr.audioContext.createAnalyser(); analyser.fftSize = 2048; src.connect(analyser); analyser.connect(this.audioMgr.audioContext.destination); this.audioMgr.analyser = analyser; this.audioMgr.timeDomainData = new Uint8Array( analyser.fftSize, ); this.audioMgr.freqData = new Uint8Array( analyser.frequencyBinCount, ); this._analyserAttached = true; } catch (e) { // some browsers won't allow createMediaElementSource until context resumed/gesture; we'll retry later console.warn("Could not attach analyser now:", e); } } // visibility resume helper document.addEventListener("visibilitychange", async () => { if ( document.visibilityState === "visible" && this.audioMgr.audioContext && this.audioMgr.audioContext.state === "suspended" ) { try { await this.audioMgr.audioContext.resume(); console.log( "🔁 resumed audio context on visibilitychange", ); } catch (e) {} } }); } catch (e) { console.warn("PlaybackController init error:", e); } } // Unlock ONCE (call from user gesture). Returns true on success. async unlockOnce() { // If already unlocked, return true if (this._unlocked) return true; this.initPersistent(); // 1) Try to resume / create AudioContext let ctx = null; try { if (!this.audioMgr.audioContext) { const AC = window.AudioContext || window.webkitAudioContext; if (AC) { this.audioMgr.audioContext = new AC(); console.log("🔧 created AudioContext (unlock)"); } } ctx = this.audioMgr.audioContext; if (ctx && ctx.state === "suspended") { try { await ctx.resume(); } catch (e) { console.warn("resume() failed:", e); } } } catch (e) { console.warn("AudioContext creation/resume failed:", e); } // 2) Prefer AudioContext silent buffer (works on many browsers) try { if (ctx && ctx.state === "running") { const sampleRate = ctx.sampleRate || 44100; const length = Math.max(1, Math.floor(sampleRate * 0.01)); // tiny const buffer = ctx.createBuffer(1, length, sampleRate); const src = ctx.createBufferSource(); src.buffer = buffer; src.connect(ctx.destination); src.start(0); await new Promise((res) => setTimeout(res, 40)); try { src.stop(); } catch (e) {} this._unlocked = true; // ensure analyser is attached now if not already this._tryAttachAnalyser(); console.log("🔓 unlocked via AudioContext silent buffer"); return true; } } catch (e) { console.warn("silent buffer unlock failed:", e); } // 3) Fallback: muted play/pause on persistent element (wait for playing) try { const el = this.el; if (!el) throw new Error("no audio element"); const hadSrc = !!el.src; if (!hadSrc) { // tiny silent data URI to allow play event el.src = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEAESsAACJWAAACABAAZGF0YQAAAAA="; } el.muted = true; el.playsInline = true; el.setAttribute("playsinline", ""); el.setAttribute("webkit-playsinline", ""); // create promise for 'playing' or timeout const p = new Promise((resolve, reject) => { let done = false; const onPlaying = () => { if (!done) { done = true; cleanup(); resolve(true); } }; const onError = () => { if (!done) { done = true; cleanup(); reject(new Error("audio error")); } }; const timeoutId = setTimeout(() => { if (!done) { done = true; cleanup(); reject(new Error("timeout")); } }, 1200); function cleanup() { el.removeEventListener("playing", onPlaying); el.removeEventListener("error", onError); clearTimeout(timeoutId); } el.addEventListener("playing", onPlaying); el.addEventListener("error", onError); // try play (may throw) try { const prom = el.play(); if (prom && prom.catch) prom.catch(() => {}); // swallow immediate rejection, rely on events } catch (err) { cleanup(); reject(err); } }); await p; // pause but keep src for future playback try { el.pause(); el.currentTime = 0; } catch (e) {} el.muted = false; this._unlocked = true; this._tryAttachAnalyser(); console.log("🔓 unlocked via muted audio element fallback"); return true; } catch (e) { console.warn("muted element fallback failed:", e); } console.warn("unlockOnce: could not unlock audio on this gesture"); return false; } _tryAttachAnalyser() { try { if ( this.audioMgr.audioContext && this.el && !this.audioMgr.analyser ) { try { const src = this.audioMgr.audioContext.createMediaElementSource( this.el, ); const analyser = this.audioMgr.audioContext.createAnalyser(); analyser.fftSize = 2048; src.connect(analyser); analyser.connect(this.audioMgr.audioContext.destination); this.audioMgr.analyser = analyser; this.audioMgr.timeDomainData = new Uint8Array( analyser.fftSize, ); this.audioMgr.freqData = new Uint8Array( analyser.frequencyBinCount, ); this._analyserAttached = true; console.log("🎛️ analyser attached to persistent element"); } catch (e) { // might fail in some browsers until after resume; ignore console.warn( "attachAnalyser failed (will retry later):", e, ); } } } catch (e) { console.warn("error in _tryAttachAnalyser:", e); } } // play url (reuses same element) async playAudioUrl(url) { if (!url) return false; this.initPersistent(); // Ensure unlocked before playing if (!this._unlocked) { console.warn( "playAudioUrl: audio not unlocked yet. Call unlockOnce via a user gesture (tap/keydown).", ); // we'll still try to auto-unlock (may fail) try { await this.unlockOnce(); } catch (e) {} } const abs = ensureAbsoluteUrl(url); const el = this.el; // stop any current playback and reset try { el.pause(); el.currentTime = 0; } catch (e) {} // set src only if changed to avoid reload thrash if (!el.src || el.src !== abs) { el.src = abs; try { el.load(); } catch (e) {} } // ensure AudioContext running try { if ( this.audioMgr.audioContext && this.audioMgr.audioContext.state === "suspended" ) { await this.audioMgr.audioContext.resume(); } } catch (e) { /* ignore */ } // play try { await el.play(); // ensure analyser available for lip sync (try attach if missing) this._tryAttachAnalyser(); // if audioMgr has analyzeAudio method, call it (no harm) if (typeof this.audioMgr.analyzeAudio === "function") { try { await this.audioMgr.analyzeAudio(); } catch (e) {} } console.log("▶️ play started", abs); return true; } catch (err) { console.warn( "play() blocked, attempting muted-first fallback:", err, ); } // muted-first fallback try { el.muted = true; await el.play(); await new Promise((r) => setTimeout(r, 80)); el.muted = false; this._tryAttachAnalyser(); if (typeof this.audioMgr.analyzeAudio === "function") try { await this.audioMgr.analyzeAudio(); } catch (e) {} console.log("▶️ play started via muted-first path", abs); return true; } catch (err) { console.warn("muted-first failed:", err); } // transient fallback (rare) try { const tmp = new Audio(abs); tmp.playsInline = true; tmp.crossOrigin = "anonymous"; tmp.setAttribute("playsinline", ""); tmp.setAttribute("webkit-playsinline", ""); document.body.appendChild(tmp); await tmp.play(); el.src = abs; try { el.load(); } catch (e) {} tmp.pause(); tmp.remove(); this._tryAttachAnalyser(); if (typeof this.audioMgr.analyzeAudio === "function") try { await this.audioMgr.analyzeAudio(); } catch (e) {} console.log("▶️ transient played", abs); return true; } catch (err) { console.warn("transient fallback failed:", err); } console.error("All playback strategies failed for", abs); return false; } } // ---- main IIFE -------------------------------------------------- (async () => { // renderer, camera, scene renderer = new THREE.WebGLRenderer({ alpha: true }); renderer.setSize(window.innerWidth, window.innerHeight); renderer.setPixelRatio(window.devicePixelRatio); renderer.shadowMap.enabled = true; renderer.shadowMap.type = THREE.PCFSoftShadowMap; document.body.appendChild(renderer.domElement); scene = new THREE.Scene(); camera = new THREE.PerspectiveCamera( 30, window.innerWidth / window.innerHeight, 0.1, 20, ); camera.position.set(0, 1, 0.9); controls = new OrbitControls(camera, renderer.domElement); controls.target.set(0, 1.1, 0); controls.update(); const dirLight = new THREE.DirectionalLight(0xffffff, 1); dirLight.position.set(3, 15, -5); scene.add(dirLight); scene.add(new THREE.AmbientLight(0xffffff, 2.1)); // load VRM const vrmData = await loadVRM(VRM_PATH, scene); vrm = vrmData.vrm; const loader = vrmData.loader; currentMixer = new THREE.AnimationMixer(vrm.scene); // init click detector (make clicks unlock audio too) initVRMClickDetector(renderer, camera, vrm, (region, partName) => { try { if (window._playback) window._playback.unlockOnce(); } catch (e) {} console.log("📩 VRM clicked region:", region, partName); }); // create managers const audioMgr = new AudioManager(vrm); animationMgr = new AnimationManager( vrm, audioMgr, renderer, scene, camera, controls, ); // expose for debugging window.audioMgr = audioMgr; window.animationMgr = animationMgr; // create playback controller and attach a one-time unlock listener (user gesture) window._playback = new PlaybackController(audioMgr); const onFirstGesture = async () => { try { await window._playback.unlockOnce(); } catch (e) { console.warn("unlockOnce direct error:", e); } // remove listeners after first unlock window.removeEventListener("pointerdown", onFirstGesture, { capture: true, }); window.removeEventListener("touchend", onFirstGesture, { capture: true, }); window.removeEventListener("keydown", onFirstGesture, { capture: true, }); }; window.addEventListener("pointerdown", onFirstGesture, { capture: true }); window.addEventListener("touchend", onFirstGesture, { capture: true }); window.addEventListener("keydown", onFirstGesture, { capture: true }); // start animation loop clock.start(); function animate() { requestAnimationFrame(animate); const delta = clock.getDelta(); if (currentMixer) currentMixer.update(delta); if (animationMgr) animationMgr.update(delta); if (vrm) vrm.update(delta); renderer.render(scene, camera); controls.update(); } animate(); // websocket handling const ws = new WebSocket(WS_URL); ws.onopen = () => { console.log("WS connected"); }; ws.onerror = (err) => console.error("WS error", err); ws.onmessage = async ({ data }) => { let msg; try { msg = JSON.parse(data); } catch { return; } console.log("WS msg:", msg); if (msg.type === "start_animation") { const { audio_path, audio_text = "", audio_duraction = 0, expression = "neutral", } = msg; audioMgr.setExpression(expression); try { // ensure unlocked (best-effort) and play using single persistent element try { await window._playback.unlockOnce(); } catch (e) { console.warn("unlockOnce thrown:", e); } const ok = await window._playback.playAudioUrl(audio_path); if (!ok) console.warn("playback failed (animation will still run)"); animationMgr.play(); // optional subtitles (in-browser) // showSubtitleStreaming(audio_text, audio_duraction, 'letter'); } catch (e) { console.error("Failed to start audio/animation:", e); } } if (msg.type === "start_vrma") { const { animation_url, play_once = false, crop_start = 0.0, crop_end = 0.0, } = msg; try { const gltfVrma = await loader.loadAsync(animation_url); const vrmAnimation = gltfVrma.userData.vrmAnimations[0]; let clip = createVRMAnimationClip(vrmAnimation, vrm); const startTime = Math.max(0, parseFloat(crop_start) || 0); const endTime = Math.max( 0, clip.duration - (parseFloat(crop_end) || 0), ); if (startTime > 0 || (parseFloat(crop_end) || 0) > 0) { const trimmed = trimAnimationClip(clip, startTime, endTime); if (trimmed) clip = trimmed; else console.warn("VRMA trim returned null"); } animationMgr.setVRMAPlaying(true); if (!currentMixer) currentMixer = new THREE.AnimationMixer(vrm.scene); const newAction = currentMixer.clipAction(clip); if (play_once) { newAction.setLoop(THREE.LoopOnce, 0); newAction.clampWhenFinished = true; newAction.enabled = true; } else { newAction.setLoop(THREE.LoopRepeat, Infinity); newAction.clampWhenFinished = false; } newAction.reset(); newAction.play(); if (currentAction && currentAction !== newAction) currentAction.crossFadeTo(newAction, 0.5, false); if (currentMixer._vrmaFinishedListener) { try { currentMixer.removeEventListener( "finished", currentMixer._vrmaFinishedListener, ); } catch (e) {} currentMixer._vrmaFinishedListener = null; } const onFinished = (e) => { if (e.action === newAction) { animationMgr.setVRMAPlaying(false); if (play_once) try { newAction.stop(); } catch (e) {} if ( currentMixer && currentMixer._vrmaFinishedListener ) { currentMixer.removeEventListener( "finished", currentMixer._vrmaFinishedListener, ); currentMixer._vrmaFinishedListener = null; } } }; currentMixer._vrmaFinishedListener = onFinished; currentMixer.addEventListener("finished", onFinished); currentAction = newAction; } catch (err) { console.error("Failed to load VRMA animation:", err); } } if (msg.type === "start_mixamo") { const { animation_url } = msg; try { currentVrm = vrm; const clip = await loadMixamoAnimation( animation_url, currentVrm, ); const newAction = currentMixer.clipAction(clip); newAction.reset().play(); if (currentAction && currentAction !== newAction) currentAction.crossFadeTo(newAction, 0.5, false); currentAction = newAction; } catch (err) { console.error("Failed to load MIXAMO animation:", err); } } if (msg.type === "take_picture") { console.log("📸 Received command to take picture from server."); await takePictureAndUpload(); } }; // resize handler window.addEventListener("resize", () => { camera.aspect = window.innerWidth / window.innerHeight; camera.updateProjectionMatrix(); renderer.setSize(window.innerWidth, window.innerHeight); }); })(); // end IIFE