-
+
+
+
+
+
+ {{ faceDetector.currentTiltAngle() }}°
-
{{ audioEngine.sensitivity() }}%
diff --git a/src/app/pages/player/player.page.scss b/src/app/pages/player/player.page.scss
index 14ea070..5f4755c 100644
--- a/src/app/pages/player/player.page.scss
+++ b/src/app/pages/player/player.page.scss
@@ -455,6 +455,113 @@ ion-content.full-screen-content {
}
}
+.voice-threshold-overlay {
+ position: fixed;
+ left: 15px;
+ top: 50%;
+ transform: translateY(-50%);
+ z-index: 1000;
+ pointer-events: auto;
+
+ @media (orientation: landscape) {
+ left: 15px;
+ }
+
+ .slider-card {
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ padding: 8px 6px 12px 6px;
+ background: rgba(18, 18, 18, 0.85) !important;
+ backdrop-filter: blur(25px);
+ border-radius: 24px;
+ border: 1px solid rgba(46, 213, 115, 0.4);
+ box-shadow: 0 8px 32px rgba(0, 0, 0, 0.6);
+ gap: 8px;
+ animation: slideInLeft 0.3s cubic-bezier(0.175, 0.885, 0.32, 1.275);
+ }
+
+ .close-slider-btn {
+ --padding-start: 0;
+ --padding-end: 0;
+ margin: 0;
+ height: 36px;
+ width: 36px;
+ --color: var(--ion-color-secondary);
+
+ ion-icon {
+ font-size: 1.5rem;
+ }
+ }
+
+ .slider-wrapper {
+ height: 180px;
+ width: 40px;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ position: relative;
+ padding: 10px 0;
+ }
+
+ .custom-vertical-slider {
+ position: relative;
+ width: 8px;
+ height: 100%;
+ background: rgba(255, 255, 255, 0.1);
+ border-radius: 4px;
+
+ .slider-track {
+ position: absolute;
+ width: 100%;
+ height: 100%;
+ }
+
+ .slider-fill {
+ position: absolute;
+ bottom: 0;
+ width: 100%;
+ background: linear-gradient(to top, #2ed573, #7bed9f);
+ border-radius: 4px;
+ box-shadow: 0 0 10px rgba(46, 213, 115, 0.3);
+ transition: height 0.05s linear;
+ }
+
+ .slider-knob {
+ position: absolute;
+ left: 50%;
+ transform: translate(-50%, 50%);
+ width: 28px;
+ height: 28px;
+ background: #2ed573;
+ border-radius: 50%;
+ box-shadow: 0 2px 10px rgba(0, 0, 0, 0.5), 0 0 15px rgba(46, 213, 115, 0.5);
+ border: 2px solid #fff;
+ transition: bottom 0.05s linear;
+ }
+ }
+
+ .sensitivity-label {
+ font-size: 0.8rem;
+ font-weight: 800;
+ color: #2ed573;
+ min-width: 40px;
+ text-align: center;
+ letter-spacing: 0.5px;
+ }
+}
+
+@keyframes slideInLeft {
+ from {
+ opacity: 0;
+ transform: translateX(-20px);
+ }
+ to {
+ opacity: 1;
+ transform: translateX(0);
+ }
+}
+
.floating-autoscroll-bar {
position: fixed;
bottom: 120px;
@@ -612,3 +719,54 @@ ion-content.full-screen-content {
transform: translate(-50%, 0);
}
}
+
+.camera-preview-floating {
+ position: fixed;
+ bottom: 60px;
+ right: 15px;
+ width: 90px;
+ height: 120px;
+ border-radius: 12px;
+ overflow: hidden;
+ border: 1px solid rgba(var(--ion-color-secondary-rgb), 0.4);
+ box-shadow: 0 4px 20px rgba(0, 0, 0, 0.5);
+ z-index: 1000;
+ display: flex;
+ flex-direction: column;
+ background: #000;
+
+ @media (orientation: landscape) {
+ bottom: 15px;
+ right: 80px;
+ }
+
+ video {
+ width: 100%;
+ height: 100%;
+ object-fit: cover;
+ transform: scaleX(-1); // Mirror camera preview
+ }
+
+ .camera-preview-overlay {
+ position: absolute;
+ bottom: 0;
+ left: 0;
+ right: 0;
+ background: rgba(0, 0, 0, 0.6);
+ padding: 2px 0;
+ display: flex;
+ justify-content: center;
+ align-items: center;
+
+ .angle-indicator {
+ font-size: 0.75rem;
+ font-weight: 700;
+ color: #fff;
+
+ &.tilted {
+ color: #2ed573;
+ text-shadow: 0 0 5px #2ed573;
+ }
+ }
+ }
+}
diff --git a/src/app/pages/player/player.page.ts b/src/app/pages/player/player.page.ts
index 5590cae..9f88b98 100644
--- a/src/app/pages/player/player.page.ts
+++ b/src/app/pages/player/player.page.ts
@@ -3,7 +3,6 @@ import { DomSanitizer, SafeResourceUrl } from '@angular/platform-browser';
import { ActivatedRoute, Router } from '@angular/router';
import { AlertController, ToastController, GestureController } from '@ionic/angular';
import { CantiService, Canto } from '../../services/canti.service';
-import { AudioEngineService } from '../../services/audio-engine.service';
import { LyricsParserService, ParsedSection } from '../../services/lyrics-parser.service';
import { ThemeService } from '../../services/theme.service';
import { SettingsService } from '../../services/settings.service';
@@ -13,6 +12,7 @@ import { YoutubePlayerService } from '../../services/youtube-player.service';
import { MyCantiService } from '../../services/my-canti.service';
import { ComunitaService } from '../../services/comunita.service';
import { StatsService } from '../../services/stats.service';
+import { FaceDetectorService } from '../../services/face-detector.service';
@Component({
selector: 'app-player',
@@ -26,7 +26,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
/** true = show chords (accordi mode), false = text only */
public showChords = signal
(false);
- public showSensitivitySlider = signal(false);
/** Autoscroll standard */
public isAutoscrolling = signal(false);
@@ -98,7 +97,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
private lastAdvanceTimestamp: number = 0; // For safety cooldown
private readonly ADVANCE_COOLDOWN = 1500; // 1.5 seconds min
- public wordsSpokenInCurrentPage: number = 0;
public lastScrollBlock = signal<'start' | 'center'>('start');
private lastProcessedTranscript: string = '';
private initialStartTime: number = 0;
@@ -106,7 +104,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
private route = inject(ActivatedRoute);
public router = inject(Router);
public cantiService = inject(CantiService);
- public audioEngine = inject(AudioEngineService);
private lyricsParser = inject(LyricsParserService);
private alertCtrl = inject(AlertController);
private toastCtrl = inject(ToastController);
@@ -120,6 +117,9 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
private gestureCtrl = inject(GestureController);
private el = inject(ElementRef);
private sanitizer = inject(DomSanitizer);
+ public faceDetector = inject(FaceDetectorService);
+
+ public enableCameraNavigation = signal(false);
private songStartTime: number = 0;
@@ -135,23 +135,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
this.showChords.set(this.settingsService.showChordsDefault());
}, { allowSignalWrites: true });
- // Avanzamento guidato dai picchi acustici con conteggio parole della riga in corso
- effect(() => {
- const count = this.audioEngine.linesDetected();
- if (count > 0 && this.audioEngine.isListening()) {
- this.handleAcousticPeak();
- }
- });
-
- // Avanzamento in tempo reale tramite riconoscimento vocale di prossimità (driver primario veloce)
- effect(() => {
- const transcript = this.audioEngine.backgroundTranscript();
- const isSpeechActive = this.audioEngine.speechRecognitionActive();
- if (transcript && this.audioEngine.isListening() && isSpeechActive) {
- this.checkWordAdvancement();
- }
- });
-
// Auto-scroll logic: keep active line in view
effect(() => {
const idx = this.currentLineIndex();
@@ -275,26 +258,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
}
});
gestureX.enable();
-
- const gestureY = this.gestureCtrl.create({
- el: this.el.nativeElement,
- direction: 'y',
- gestureName: 'swipe-song-y',
- canStart: (ev) => {
- const target = ev.event.target as HTMLElement;
- return !target.closest('ion-footer') && !target.closest('ion-header');
- },
- onEnd: (ev) => {
- if (Math.abs(ev.deltaY) > 50) {
- if (ev.deltaY > 0) {
- this.prev();
- } else {
- this.next();
- }
- }
- }
- });
- gestureY.enable();
}
@@ -361,25 +324,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
this.initialPinchDistance = null;
}
- private lastWheelTime = 0;
- private readonly WHEEL_COOLDOWN = 400; // ms to avoid too rapid advancement
-
- onWheel(event: WheelEvent) {
- event.preventDefault();
- const now = Date.now();
- if (now - this.lastWheelTime < this.WHEEL_COOLDOWN) {
- return;
- }
- if (Math.abs(event.deltaY) > 5) {
- if (event.deltaY > 0) {
- this.next();
- } else {
- this.prev();
- }
- this.lastWheelTime = now;
- }
- }
-
private getDistance(t1: Touch, t2: Touch): number {
return Math.sqrt(Math.pow(t1.clientX - t2.clientX, 2) + Math.pow(t1.clientY - t2.clientY, 2));
}
@@ -434,40 +378,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
}
}
- toggleListening() {
- if (this.audioEngine.isListening()) {
- this.audioEngine.stopListening();
- this.showSensitivitySlider.set(false);
- } else {
- this.audioEngine.startListening();
- this.showSensitivitySlider.set(true);
- }
- }
-
- toggleSensitivitySlider(event: Event) {
- event.stopPropagation();
- this.showSensitivitySlider.update(v => !v);
- }
-
- onSensitivityChange(event: any) {
- this.audioEngine.sensitivity.set(event.detail.value);
- }
-
- handleSensitivityTouch(event: TouchEvent) {
- event.preventDefault();
- const touch = event.touches[0];
- const target = event.currentTarget as HTMLElement;
- const rect = target.getBoundingClientRect();
-
- // Calculate percentage based on Y position (bottom is 50%, top is 100%)
- const rawPercentage = 100 - ((touch.clientY - rect.top) / rect.height * 100);
- // Map 0-100 raw to 50-100 range
- let percentage = 50 + (rawPercentage * 0.5);
- percentage = Math.max(50, Math.min(100, Math.round(percentage)));
-
- this.audioEngine.sensitivity.set(percentage);
- }
-
toggleAudio() {
if (this.youtubePlayerService.currentCantoId() === this.canto()?.id) {
this.youtubePlayerService.togglePlayPause();
@@ -492,10 +402,8 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
restart() {
this.currentLineIndex.set(0);
- this.audioEngine.resetWordCount();
this.youtubePlayerService.seekTo(0);
this.channel.postMessage({ type: 'SYNC_INDEX', index: 0 });
- this.wordsSpokenInCurrentPage = 0;
this.lastScrollBlock.set('center');
}
@@ -549,26 +457,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
}
}
- private getLineWords(lineIndex: number): string[] {
- const allLines = this.getAllLines();
- const lineText = allLines[lineIndex]?.text || '';
- return lineText
- .toLowerCase()
- .split(/\s+/)
- .map((w: string) => w.replace(/[^a-zA-Z0-9àèìòùáéíóú]/g, ''))
- .filter((w: string) => w.length >= 3);
- }
-
- public getPageWords(startIndex: number, pageSize: number): string[] {
- const allWords: string[] = [];
- const totalLines = this.getTotalLines();
- const end = Math.min(totalLines, startIndex + pageSize);
- for (let i = startIndex; i < end; i++) {
- allWords.push(...this.getLineWords(i));
- }
- return allWords;
- }
-
public getVisibleLineIndices(): number[] {
try {
const container = document.querySelector('.lyrics-container') as HTMLElement;
@@ -607,127 +495,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
}
}
- public getPageTargetWordCount(): number {
- const visibleIndices = this.getVisibleLineIndices();
- let totalRawWords = 0;
- const currentIdx = this.currentLineIndex();
-
- if (visibleIndices.length > 0) {
- if (currentIdx === 0) {
- // Sulla prima pagina, contiamo solo la prima metà delle righe visibili
- // per far avanzare l'applicazione non appena si arriva a metà della prima schermata
- const halfLength = Math.max(1, Math.ceil(visibleIndices.length / 2));
- for (let i = 0; i < halfLength; i++) {
- const idx = visibleIndices[i];
- totalRawWords += this.getLineRawWordCount(idx);
- }
- } else {
- // Dalla seconda pagina in poi, contiamo solo le righe attive e future (escludendo il testo già cantato in alto)
- for (const idx of visibleIndices) {
- if (idx >= currentIdx) {
- totalRawWords += this.getLineRawWordCount(idx);
- }
- }
- }
- } else {
- // Fallback: calcolo teorico basato sulla pagina corrente
- const pageSize = this.calculatePageStepSize();
- const totalLines = this.getTotalLines();
-
- let end: number;
- if (currentIdx === 0) {
- // Fallback prima pagina: metà delle righe della pagina teorica
- const halfPageSize = Math.max(1, Math.ceil(pageSize / 2));
- end = Math.min(totalLines, currentIdx + halfPageSize);
- } else {
- end = Math.min(totalLines, currentIdx + pageSize);
- }
-
- for (let i = currentIdx; i < end; i++) {
- totalRawWords += this.getLineRawWordCount(i);
- }
- }
-
- // Soglia al 70% delle parole contate nella schermata visibile
- return Math.max(1, Math.ceil(totalRawWords * 0.70));
- }
-
- private checkWordAdvancement() {
- const now = Date.now();
-
- // Cooldown per prevenire avanzamenti multipli troppo veloci
- if (now - this.lastAdvanceTimestamp < this.ADVANCE_COOLDOWN) {
- return;
- }
-
- const currentIdx = this.currentLineIndex();
- let nextPageIdx: number;
- const totalLines = this.getTotalLines();
-
- if (currentIdx === 0) {
- // Sulla prima pagina, la "pagina successiva" è in realtà la seconda metà della prima schermata
- const visibleIndices = this.getVisibleLineIndices();
- if (visibleIndices.length > 0) {
- const halfLength = Math.max(1, Math.ceil(visibleIndices.length / 2));
- nextPageIdx = visibleIndices[halfLength] !== undefined ? visibleIndices[halfLength] : halfLength;
- } else {
- const pageSize = this.calculatePageStepSize();
- const halfPageSize = Math.max(1, Math.ceil(pageSize / 2));
- nextPageIdx = currentIdx + halfPageSize;
- }
- } else {
- nextPageIdx = this.calculateNextPageStartIndex();
- }
-
- if (nextPageIdx >= totalLines) return;
-
- const pageSize = this.calculatePageStepSize();
-
- // Raccoglie le parole del blocco successivo (metà pagina se siamo all'inizio, altrimenti intera pagina)
- const nextPageWords = this.getPageWords(nextPageIdx, currentIdx === 0 ? Math.max(1, Math.ceil(pageSize / 2)) : pageSize);
- if (nextPageWords.length === 0) return;
-
- // Parole rilevate di recente (con tolleranza di ritardo della Speech API)
- const recentWords = this.audioEngine.getRecentWordsSince(this.lastAdvanceTimestamp);
-
- // Controlla se l'utente ha cantato parole della pagina successiva -> Avanzamento istantaneo immediato
- const matchNext = nextPageWords.some(word => recentWords.includes(word));
- if (matchNext) {
- console.log(`[SmartKaraoke] Parola chiave del blocco successivo rilevata! Avanzamento immediato a: ${nextPageIdx}`);
- this.lastScrollBlock.set('center');
- this.currentLineIndex.set(nextPageIdx);
- this.channel.postMessage({ type: 'SYNC_INDEX', index: nextPageIdx });
- this.lastAdvanceTimestamp = now;
- this.audioEngine.resetWordCount();
- this.wordsSpokenInCurrentPage = 0;
- }
- }
-
- public getLineRawWordCount(lineIndex: number): number {
- const allLines = this.getAllLines();
- const lineText = allLines[lineIndex]?.text || '';
- const words = lineText.trim().split(/\s+/).filter((w: string) => w.length > 0);
- return words.length;
- }
-
- public getLineTargetWordCount(lineIndex: number): number {
- const rawCount = this.getLineRawWordCount(lineIndex);
- // Soglia proporzionale tollerante (65% del conteggio parole effettivo) per compensare legature melodiche
- return Math.max(1, Math.ceil(rawCount * 0.65));
- }
-
- private handleAcousticPeak() {
- const targetWordCount = this.getPageTargetWordCount();
-
- this.wordsSpokenInCurrentPage++;
- console.log(`[PeakAdvance] Peak detected. Spoken words on page: ${this.wordsSpokenInCurrentPage}/${targetWordCount}`);
-
- if (this.wordsSpokenInCurrentPage >= targetWordCount) {
- console.log(`[PeakAdvance] Target page word count reached (${targetWordCount}). Advancing page.`);
- this.next(true);
- }
- }
-
private calculateNextPageStartIndex(): number {
try {
const container = document.querySelector('.lyrics-container') as HTMLElement;
@@ -781,8 +548,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
next(isAutomatic: boolean = false) {
this.lastAdvanceTimestamp = Date.now();
- this.audioEngine.resetWordCount();
- this.wordsSpokenInCurrentPage = 0;
const totalLines = this.getTotalLines();
const prevIdx = this.currentLineIndex();
@@ -825,8 +590,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
prev() {
this.lastAdvanceTimestamp = Date.now();
- this.audioEngine.resetWordCount();
- this.wordsSpokenInCurrentPage = 0;
const prevIdx = this.currentLineIndex();
if (prevIdx > 0) {
@@ -1064,10 +827,46 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
}
}
+ toggleCameraNavigation() {
+ if (this.enableCameraNavigation()) {
+ this.stopCameraNavigation();
+ } else {
+ this.startCameraNavigation();
+ }
+ }
+
+ startCameraNavigation() {
+ this.enableCameraNavigation.set(true);
+
+ setTimeout(async () => {
+ const videoEl = document.querySelector('#face-preview-video') as HTMLVideoElement;
+ if (videoEl) {
+ try {
+ await this.faceDetector.start(videoEl, (direction) => {
+ console.log(`[PlayerPage] Head tilt trigger received: ${direction}`);
+ if (direction === 'next') {
+ this.next(false);
+ } else {
+ this.prev();
+ }
+ });
+ } catch (e) {
+ this.enableCameraNavigation.set(false);
+ alert('Impossibile accedere alla fotocamera. Assicurati di aver concesso i permessi e di usare HTTPS.');
+ }
+ }
+ }, 300);
+ }
+
+ stopCameraNavigation() {
+ this.enableCameraNavigation.set(false);
+ this.faceDetector.stop();
+ }
+
ngOnDestroy() {
this.stopAutoscroll();
this.logPreviousSongTime();
- this.audioEngine.stopListening();
+ this.stopCameraNavigation();
this.channel.close();
}
}
diff --git a/src/app/services/audio-engine.service.ts b/src/app/services/audio-engine.service.ts
index 3914943..aa7c4b6 100644
--- a/src/app/services/audio-engine.service.ts
+++ b/src/app/services/audio-engine.service.ts
@@ -13,6 +13,16 @@ export class AudioEngineService {
public sensitivity = signal(75); // Default sensitivity (50-100)
public speechRecognitionActive = signal(false);
+ // Voice/Guitar detection signals
+ public voiceDetected = signal(false);
+ public guitarDetected = signal(false);
+
+ /** Soglia del punteggio voce: valori più bassi = più sensibile alla voce (range 0.05 – 0.50) */
+ public voiceThreshold = signal(0.15);
+
+ /** Debug: punteggio voce corrente (0-1) per feedback visivo */
+ public voiceScore = signal(0);
+
// Real-time background continuous transcript and word buffer for reliability checks
public backgroundTranscript = signal('');
private backgroundRecognition: any = null;
@@ -23,9 +33,6 @@ export class AudioEngineService {
}
private audioContext: AudioContext | null = null;
- private highpassFilter: BiquadFilterNode | null = null;
- private lowpassFilter: BiquadFilterNode | null = null;
- private vocalBandpass: BiquadFilterNode | null = null;
private analyser: AnalyserNode | null = null;
private stream: MediaStream | null = null;
private animationFrame: number | null = null;
@@ -35,6 +42,7 @@ export class AudioEngineService {
private isSpeaking: boolean = false;
private lastSilenceTime: number = Date.now();
private lastWordTime: number = 0;
+ private speakingStartTime: number = 0;
// Constants for tuning - Optimized for close proximity (singer/guitarist)
private readonly SILENCE_GAP = 100; // ms
@@ -44,111 +52,57 @@ export class AudioEngineService {
private lineStarted: boolean = false;
private noiseFloor: number = 30; // Adaptive noise floor starting point
+ // --- Advanced Voice Detection State ---
+
+ // Smoothing buffer (16 frames ~260ms a 60fps)
+ private readonly SMOOTHING_FRAMES = 16;
+ private voiceScoreBuffer: number[] = [];
+
+ // Spectral flux: previous frame spectrum for change detection
+ private previousSpectrum: Float32Array | null = null;
+
+ // Hysteresis: once voice is detected, it stays on for at least this many frames
+ private readonly VOICE_HOLD_FRAMES = 12; // ~200ms
+ private voiceHoldCounter: number = 0;
+
+ // Sub-band definitions (Hz) - 8 fine-grained bands
+ private readonly BANDS = [
+ { name: 'sub_bass', start: 80, end: 150 }, // Fondamentali basse chitarra (Mi2=82, La2=110)
+ { name: 'bass', start: 150, end: 300 }, // Fondamentali voce maschile/femminile + chitarra
+ { name: 'low_mid', start: 300, end: 600 }, // Armoniche chitarra dominanti
+ { name: 'mid', start: 600, end: 1000 }, // Zona transizione
+ { name: 'formant_f1', start: 1000, end: 1500 }, // Formante F1 voce (vocali aperte)
+ { name: 'formant_f2', start: 1500, end: 2500 }, // Formante F2 voce (KEY: quasi assente in chitarra)
+ { name: 'formant_f3', start: 2500, end: 3500 }, // Formante F3 voce (sibilanti morbide)
+ { name: 'sibilants', start: 3500, end: 6000 }, // Consonanti sibilanti (s, t, f, sh) - SOLO voce
+ ];
+
constructor() {
this.initSpeechRecognition();
- this.initBackgroundRecognition();
}
private initBackgroundRecognition() {
- const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
- if (SpeechRecognition) {
- this.backgroundRecognition = new SpeechRecognition();
- this.backgroundRecognition.continuous = true;
- this.backgroundRecognition.interimResults = true;
- this.backgroundRecognition.lang = 'it-IT';
-
- this.backgroundRecognition.onstart = () => {
- console.log('Background speech recognition started successfully');
- this.speechRecognitionActive.set(true);
- };
-
- this.backgroundRecognition.onresult = (event: any) => {
- let interimTranscript = '';
- let finalTranscript = '';
- for (let i = event.resultIndex; i < event.results.length; ++i) {
- if (event.results[i].isFinal) {
- finalTranscript += event.results[i][0].transcript;
- } else {
- interimTranscript += event.results[i][0].transcript;
- }
- }
- const fullTranscript = (finalTranscript + ' ' + interimTranscript).trim().toLowerCase();
- if (fullTranscript) {
- this.backgroundTranscript.set(fullTranscript);
- this.updateRecentWords(fullTranscript);
- }
- };
-
- this.backgroundRecognition.onerror = (err: any) => {
- console.error('Background recognition error:', err);
- this.speechRecognitionActive.set(false);
- if (this.isListening() && err.error !== 'aborted' && err.error !== 'not-allowed') {
- setTimeout(() => this.startBackgroundRecognition(), 1000);
- }
- };
-
- this.backgroundRecognition.onend = () => {
- console.log('Background speech recognition ended');
- if (this.isListening()) {
- this.startBackgroundRecognition();
- } else {
- this.speechRecognitionActive.set(false);
- }
- };
- }
+ // Disattivato per utilizzare esclusivamente l'analizzatore FFT acustico locale
}
private startBackgroundRecognition() {
- if (!this.backgroundRecognition) return;
- try {
- this.backgroundRecognition.start();
- } catch (e) {
- // Recognition already started or running
- }
+ // Disattivato
}
private stopBackgroundRecognition() {
- if (!this.backgroundRecognition) return;
- try {
- this.backgroundRecognition.stop();
- } catch (e) {
- // Recognition already stopped
- }
+ // Disattivato
}
private updateRecentWords(transcript: string) {
- const words = transcript.split(/\s+/);
- const now = Date.now();
- words.forEach(w => {
- // Strip special characters and keep alphanumeric Italian letters
- const cleaned = w.replace(/[^a-zA-Z0-9àèìòùáéíóú]/g, '').toLowerCase();
- if (cleaned.length >= 3) {
- // Avoid adding duplicate words within the last 3 seconds
- const existing = this.recentWordsBuffer.find(item => item.word === cleaned && now - item.timestamp < 3000);
- if (!existing) {
- this.recentWordsBuffer.push({ word: cleaned, timestamp: now });
- }
- }
- });
-
- // Keep only words from the last 10 seconds to make it highly responsive
- this.recentWordsBuffer = this.recentWordsBuffer.filter(item => now - item.timestamp < 10000);
+ // Disattivato
}
public getRecentWords(): string[] {
- const now = Date.now();
- this.recentWordsBuffer = this.recentWordsBuffer.filter(item => now - item.timestamp < 10000);
- return this.recentWordsBuffer.map(item => item.word);
+ return [];
}
public getRecentWordsSince(timestamp: number): string[] {
- const now = Date.now();
- // Consente una sovrapposizione di 1.5 secondi per coprire la latenza di SpeechRecognition
- const cutoff = Math.max(timestamp - 1500, now - 10000);
- this.recentWordsBuffer = this.recentWordsBuffer.filter(item => now - item.timestamp < 10000);
- return this.recentWordsBuffer
- .filter(item => item.timestamp > cutoff)
- .map(item => item.word);
+ return [];
}
public clearRecentWords() {
@@ -202,6 +156,20 @@ export class AudioEngineService {
async startListening() {
if (this.isListening()) return;
+ this.isListening.set(true);
+ this.voiceDetected.set(false);
+ this.guitarDetected.set(false);
+ this.voiceScoreBuffer = [];
+ this.previousSpectrum = null;
+ this.voiceHoldCounter = 0;
+ this.clearRecentWords();
+
+ console.log('[AudioEngine] Starting local FFT Audio Analyser');
+ await this.startOfflineAnalyser();
+ }
+
+ public async startOfflineAnalyser() {
+ if (this.analyser) return; // Già attivo
try {
this.stream = await navigator.mediaDevices.getUserMedia({
audio: {
@@ -213,126 +181,421 @@ export class AudioEngineService {
this.audioContext = new AudioContext();
const source = this.audioContext.createMediaStreamSource(this.stream);
- // Catena di filtri in cascata per isolamento vocale
- this.highpassFilter = this.audioContext.createBiquadFilter();
- this.highpassFilter.type = 'highpass';
- this.highpassFilter.frequency.value = 150; // Taglia i bassi e rombi di fondo
-
- this.lowpassFilter = this.audioContext.createBiquadFilter();
- this.lowpassFilter.type = 'lowpass';
- this.lowpassFilter.frequency.value = 2000; // Taglia sibili e armonici acuti di chitarre
-
- this.vocalBandpass = this.audioContext.createBiquadFilter();
- this.vocalBandpass.type = 'bandpass';
- this.vocalBandpass.frequency.value = 800; // Centro della gamma vocale primaria
- this.vocalBandpass.Q.value = 1.2; // Filtro stretto focalizzato sulla voce
-
+ // FFT a 4096 punti per risoluzione ~10.7 Hz/bin (a 44100 Hz)
this.analyser = this.audioContext.createAnalyser();
- this.analyser.fftSize = 512;
+ this.analyser.fftSize = 4096;
+ this.analyser.smoothingTimeConstant = 0.4; // Smoothing moderato per stabilità spettrale
- // Connessione in cascata: source -> highpass -> lowpass -> bandpass -> analyser
- source.connect(this.highpassFilter);
- this.highpassFilter.connect(this.lowpassFilter);
- this.lowpassFilter.connect(this.vocalBandpass);
- this.vocalBandpass.connect(this.analyser);
-
- this.isListening.set(true);
- this.clearRecentWords();
- this.startBackgroundRecognition();
+ source.connect(this.analyser);
this.processAudio();
} catch (err) {
- console.error('Error accessing microphone', err);
+ console.error('[AudioEngine] Error accessing microphone for offline analyser:', err);
+ // Se fallisce anche il mic locale, proviamo a ripristinare lo stato
+ this.stopListening();
alert('Errore microfono: assicurati di usare HTTPS e di aver dato i permessi.');
}
}
stopListening() {
if (this.animationFrame) cancelAnimationFrame(this.animationFrame);
+
+ // Rilascia le risorse del microfono locale
this.stream?.getTracks().forEach(track => track.stop());
this.audioContext?.close();
+ this.stream = null;
+ this.audioContext = null;
+ this.analyser = null;
+
this.isListening.set(false);
this.energyLevel.set(0);
+ this.voiceDetected.set(false);
+ this.guitarDetected.set(false);
+ this.voiceScore.set(0);
+ this.voiceScoreBuffer = [];
+ this.previousSpectrum = null;
+ this.voiceHoldCounter = 0;
this.stopBackgroundRecognition();
this.clearRecentWords();
- this.highpassFilter = null;
- this.lowpassFilter = null;
- this.vocalBandpass = null;
this.speechRecognitionActive.set(false);
}
+ // =====================================================================
+ // ANALISI SPETTRALE AVANZATA MULTI-FEATURE
+ // =====================================================================
+
+ /**
+ * Calcola l'energia media in una sotto-banda dello spettro.
+ */
+ private getBandEnergy(dataArray: Uint8Array, binWidth: number, startHz: number, endHz: number): number {
+ const startBin = Math.max(0, Math.floor(startHz / binWidth));
+ const endBin = Math.min(dataArray.length - 1, Math.floor(endHz / binWidth));
+ if (endBin <= startBin) return 0;
+ let sum = 0;
+ for (let i = startBin; i <= endBin; i++) {
+ sum += dataArray[i];
+ }
+ return sum / (endBin - startBin + 1);
+ }
+
+ /**
+ * FEATURE 1: Rapporto formanti vocali F2+F3 vs bande strumentali (low+mid)
+ *
+ * La voce umana ha formanti F2 (1500-2500 Hz) e F3 (2500-3500 Hz) molto prominenti.
+ * La chitarra acustica ha pochissima energia in queste bande.
+ * Rapporto alto = voce, basso = strumento.
+ */
+ private calcFormantRatio(bandEnergies: number[]): number {
+ const instrumentEnergy = bandEnergies[0] + bandEnergies[1] + bandEnergies[2] + bandEnergies[3]; // 80-1000 Hz
+ const formantEnergy = bandEnergies[5] + bandEnergies[6]; // F2 (1500-2500) + F3 (2500-3500)
+ return formantEnergy / (instrumentEnergy + 0.001);
+ }
+
+ /**
+ * FEATURE 2: Energia sibilanti (3500-6000 Hz)
+ *
+ * Solo la voce umana produce consonanti sibilanti come "s", "t", "f", "sh", "z"
+ * che hanno energia significativa sopra 3500 Hz. Gli strumenti acustici (chitarra,
+ * pianoforte, etc.) hanno energia trascurabile in questa banda.
+ *
+ * Ritorna un valore normalizzato 0-1.
+ */
+ private calcSibilantScore(bandEnergies: number[], totalEnergy: number): number {
+ if (totalEnergy < 1) return 0;
+ const sibilantEnergy = bandEnergies[7]; // 3500-6000 Hz
+ // Normalizza: una sibilante tipica ha 15-40% dell'energia totale
+ return Math.min(1, sibilantEnergy / (totalEnergy * 0.15 + 0.001));
+ }
+
+ /**
+ * FEATURE 3: Spectral Flatness (entropia di Wiener)
+ *
+ * Misura quanto lo spettro è "piatto" (rumoroso) vs "piccato" (tonale).
+ * - Chitarra: spettro molto tonale con picchi armonici netti → flatness BASSA
+ * - Voce (consonanti): spettro più rumoroso → flatness ALTA
+ * - Voce (vocali): moderatamente tonale ma con formanti larghe → flatness MEDIA
+ *
+ * Formula: media_geometrica / media_aritmetica (0=tono puro, 1=rumore bianco)
+ */
+ private calcSpectralFlatness(dataArray: Uint8Array, binWidth: number): number {
+ // Calcola nella regione 300-6000 Hz (dove la discriminazione è più utile)
+ const startBin = Math.max(1, Math.floor(300 / binWidth));
+ const endBin = Math.min(dataArray.length - 1, Math.floor(6000 / binWidth));
+ const n = endBin - startBin + 1;
+ if (n <= 0) return 0;
+
+ let logSum = 0;
+ let arithmeticSum = 0;
+ let zeroCount = 0;
+
+ for (let i = startBin; i <= endBin; i++) {
+ const val = Math.max(dataArray[i], 0.001); // Avoid log(0)
+ logSum += Math.log(val);
+ arithmeticSum += val;
+ if (dataArray[i] === 0) zeroCount++;
+ }
+
+ if (zeroCount > n * 0.5) return 0; // Troppi zeri = silenzio
+
+ const geometricMean = Math.exp(logSum / n);
+ const arithmeticMean = arithmeticSum / n;
+
+ if (arithmeticMean < 0.001) return 0;
+ return Math.min(1, geometricMean / arithmeticMean);
+ }
+
+ /**
+ * FEATURE 4: Spectral Flux (velocità di cambiamento spettrale)
+ *
+ * Misura quanto rapidamente cambia la forma dello spettro tra frame successivi.
+ * - Voce: alta flux (alternanza vocali/consonanti, prosodia)
+ * - Chitarra: bassa flux (note sostenute, spettro stabile)
+ *
+ * Ritorna un valore normalizzato 0-1.
+ */
+ private calcSpectralFlux(dataArray: Uint8Array, binWidth: number): number {
+ const startBin = Math.max(0, Math.floor(200 / binWidth));
+ const endBin = Math.min(dataArray.length - 1, Math.floor(5000 / binWidth));
+
+ if (!this.previousSpectrum) {
+ this.previousSpectrum = new Float32Array(dataArray.length);
+ for (let i = 0; i < dataArray.length; i++) {
+ this.previousSpectrum[i] = dataArray[i];
+ }
+ return 0;
+ }
+
+ let flux = 0;
+ let maxFlux = 0;
+ for (let i = startBin; i <= endBin; i++) {
+ const diff = dataArray[i] - this.previousSpectrum[i];
+ // Solo variazioni positive (onset) per evitare il decadimento naturale
+ if (diff > 0) {
+ flux += diff * diff;
+ }
+ maxFlux += 255 * 255; // Massimo teorico
+ }
+
+ // Aggiorna il buffer dello spettro precedente
+ for (let i = 0; i < dataArray.length; i++) {
+ this.previousSpectrum[i] = dataArray[i];
+ }
+
+ if (maxFlux === 0) return 0;
+ // Normalizza e scala logaritmicamente per sensibilità
+ const normalizedFlux = flux / maxFlux;
+ return Math.min(1, Math.sqrt(normalizedFlux) * 10); // Amplifica le piccole variazioni
+ }
+
+ /**
+ * FEATURE 5: Formant Peak Detection
+ *
+ * Cerca picchi spettrali caratteristici nella regione delle formanti vocali (800-3500 Hz).
+ * La voce umana produce 2-4 picchi prominenti (formanti F1-F4).
+ * La chitarra ha uno spettro armonico regolare senza picchi formantici.
+ *
+ * Ritorna il numero di picchi formantici rilevati (0-4), normalizzato 0-1.
+ */
+ private calcFormantPeaks(dataArray: Uint8Array, binWidth: number): number {
+ const startBin = Math.max(0, Math.floor(800 / binWidth));
+ const endBin = Math.min(dataArray.length - 1, Math.floor(3500 / binWidth));
+
+ // Calcola la media locale per determinare la soglia di prominenza
+ let sum = 0;
+ for (let i = startBin; i <= endBin; i++) {
+ sum += dataArray[i];
+ }
+ const avgEnergy = sum / Math.max(1, endBin - startBin + 1);
+
+ if (avgEnergy < 5) return 0; // Silenzio
+
+ // Cerca picchi: un bin è un picco se è maggiore dei 5 bin a sinistra e 5 a destra
+ // e supera la media di almeno 30%
+ const peakThreshold = avgEnergy * 1.3;
+ const windowSize = Math.max(3, Math.floor(100 / binWidth)); // ~100 Hz di finestra
+ let peakCount = 0;
+ let lastPeakBin = -windowSize * 2; // Evita di contare picchi troppo vicini
+
+ for (let i = startBin + windowSize; i <= endBin - windowSize; i++) {
+ if (dataArray[i] < peakThreshold) continue;
+
+ let isPeak = true;
+ for (let j = 1; j <= windowSize; j++) {
+ if (dataArray[i] < dataArray[i - j] || dataArray[i] < dataArray[i + j]) {
+ isPeak = false;
+ break;
+ }
+ }
+
+ if (isPeak && (i - lastPeakBin) > windowSize) {
+ peakCount++;
+ lastPeakBin = i;
+ if (peakCount >= 4) break; // Max 4 formanti
+ }
+ }
+
+ return Math.min(1, peakCount / 3); // 3 formanti = punteggio pieno
+ }
+
+ /**
+ * FEATURE 6: Rapporto energia alta/bassa (spectral tilt)
+ *
+ * La voce umana (soprattutto da vicino al microfono) ha uno spettro più "piatto"
+ * con energia significativa anche nelle alte frequenze.
+ * La chitarra ha una forte caduta sopra 1-2 kHz.
+ */
+ private calcSpectralTilt(bandEnergies: number[]): number {
+ const lowEnergy = bandEnergies[0] + bandEnergies[1] + bandEnergies[2]; // 80-600 Hz
+ const highEnergy = bandEnergies[5] + bandEnergies[6] + bandEnergies[7]; // 1500-6000 Hz
+ if (lowEnergy < 0.001) return 0;
+ // Un rapporto alto indica più energia nelle alte frequenze (voce)
+ return Math.min(1, highEnergy / (lowEnergy + 0.001));
+ }
+
+ /**
+ * Calcola il punteggio voce combinato da tutte le feature.
+ *
+ * Pesi delle feature (calibrati per massimizzare la discriminazione):
+ * - Formant ratio: 25% (molto discriminante)
+ * - Sibilant score: 20% (quasi esclusivo della voce)
+ * - Spectral flatness: 15% (voce più rumorosa di chitarra)
+ * - Spectral flux: 15% (voce cambia più rapidamente)
+ * - Formant peaks: 15% (struttura formanti unica della voce)
+ * - Spectral tilt: 10% (distribuzione energia)
+ */
+ private calcVoiceScore(
+ dataArray: Uint8Array,
+ bandEnergies: number[],
+ totalEnergy: number,
+ binWidth: number
+ ): number {
+ const formantRatio = this.calcFormantRatio(bandEnergies);
+ const sibilantScore = this.calcSibilantScore(bandEnergies, totalEnergy);
+ const flatness = this.calcSpectralFlatness(dataArray, binWidth);
+ const flux = this.calcSpectralFlux(dataArray, binWidth);
+ const formantPeaks = this.calcFormantPeaks(dataArray, binWidth);
+ const spectralTilt = this.calcSpectralTilt(bandEnergies);
+
+ // Normalizza formantRatio: tipicamente 0.05-0.5 per voce, 0-0.05 per chitarra
+ const normalizedFormantRatio = Math.min(1, formantRatio / 0.4);
+
+ // Punteggio pesato
+ const score =
+ normalizedFormantRatio * 0.25 +
+ sibilantScore * 0.20 +
+ flatness * 0.15 +
+ flux * 0.15 +
+ formantPeaks * 0.15 +
+ spectralTilt * 0.10;
+
+ return Math.min(1, Math.max(0, score));
+ }
+
+ // =====================================================================
+ // MAIN AUDIO PROCESSING LOOP
+ // =====================================================================
+
private processAudio() {
if (!this.analyser) return;
- const bufferLength = this.analyser.frequencyBinCount;
+ const bufferLength = this.analyser.frequencyBinCount; // 2048 con fftSize=4096
const dataArray = new Uint8Array(bufferLength);
-
- // Con la catena di filtri in cascata, l'energia vocale è centrata tra 200 Hz e 2000 Hz.
const sampleRate = this.audioContext ? this.audioContext.sampleRate : 44100;
- const binWidth = sampleRate / 512;
- const startBin = Math.max(0, Math.floor(200 / binWidth));
- const endBin = Math.min(bufferLength - 1, Math.floor(2000 / binWidth));
+ const binWidth = sampleRate / 4096; // ~10.7 Hz/bin
+
+ // Energia minima per considerare che c'è suono (evita falsi positivi nel silenzio)
+ const MIN_ENERGY = 5;
const analyze = () => {
if (!this.analyser) return;
this.analyser.getByteFrequencyData(dataArray);
- // Calcola l'energia media ESCLUSIVAMENTE nella gamma di frequenza della voce
- let sum = 0;
- for (let i = startBin; i <= endBin; i++) {
- sum += dataArray[i];
- }
- const avgEnergy = sum / (endBin - startBin + 1);
- this.energyLevel.set(avgEnergy);
+ // Calcola energia in ciascuna delle 8 sotto-bande
+ const bandEnergies: number[] = this.BANDS.map(band =>
+ this.getBandEnergy(dataArray, binWidth, band.start, band.end)
+ );
+
+ // Energia totale media
+ const totalEnergy = bandEnergies.reduce((a, b) => a + b, 0) / bandEnergies.length;
+
+ // Feedback visuale (barra energia) — usa la media delle bande vocali per coerenza
+ const vocalDisplayEnergy = (bandEnergies[4] + bandEnergies[5] + bandEnergies[6]) / 3;
+ this.energyLevel.set(vocalDisplayEnergy);
const now = Date.now();
+ const hasSound = totalEnergy > MIN_ENERGY;
+
+ if (hasSound) {
+ // Calcola il punteggio voce multi-feature
+ const rawScore = this.calcVoiceScore(dataArray, bandEnergies, totalEnergy, binWidth);
+
+ // Aggiungi al buffer di smoothing
+ this.voiceScoreBuffer.push(rawScore);
+ if (this.voiceScoreBuffer.length > this.SMOOTHING_FRAMES) {
+ this.voiceScoreBuffer.shift();
+ }
+
+ // Media pesata: i frame più recenti contano di più
+ let weightedSum = 0;
+ let weightTotal = 0;
+ for (let i = 0; i < this.voiceScoreBuffer.length; i++) {
+ const weight = (i + 1); // Peso crescente
+ weightedSum += this.voiceScoreBuffer[i] * weight;
+ weightTotal += weight;
+ }
+ const smoothedScore = weightedSum / weightTotal;
+
+ this.voiceScore.set(smoothedScore);
+
+ // Determinazione con isteresi
+ const threshold = this.voiceThreshold();
+
+ if (smoothedScore > threshold) {
+ // Voce rilevata
+ this.voiceDetected.set(true);
+ this.guitarDetected.set(false);
+ this.voiceHoldCounter = this.VOICE_HOLD_FRAMES;
+ } else if (this.voiceHoldCounter > 0) {
+ // Isteresi: mantieni lo stato "voce" per evitare oscillazioni
+ this.voiceHoldCounter--;
+ // Lo stato resta quello precedente (voce)
+ } else {
+ // Strumento/rumore ambientale
+ this.voiceDetected.set(false);
+ this.guitarDetected.set(true);
+ }
+ } else {
+ // Silenzio
+ this.voiceDetected.set(false);
+ this.guitarDetected.set(false);
+ this.voiceScore.set(0);
+ this.voiceHoldCounter = 0;
+ }
+
+ // --- Logica conteggio parole (solo se voce rilevata) ---
// Tracciamento adattivo del rumore di fondo
- if (avgEnergy < this.noiseFloor) {
- this.noiseFloor = this.noiseFloor * 0.95 + avgEnergy * 0.05;
+ if (totalEnergy < this.noiseFloor) {
+ this.noiseFloor = this.noiseFloor * 0.95 + totalEnergy * 0.05;
} else {
- this.noiseFloor = this.noiseFloor * 0.998 + avgEnergy * 0.002;
+ this.noiseFloor = this.noiseFloor * 0.998 + totalEnergy * 0.002;
}
// Soglia di sensibilità definita dall'utente
const userThreshold = 220 - (this.sensitivity() * 2.0);
- // La soglia effettiva si adatta al rumore di fondo per evitare falsi avanzamenti
const currentThreshold = Math.max(userThreshold, this.noiseFloor + 12);
- if (avgEnergy > currentThreshold) {
- if (avgEnergy > this.peakEnergy) {
- this.peakEnergy = avgEnergy;
+ // *** CONTEGGIO PAROLE SOLO QUANDO LA VOCE UMANA È DOMINANTE ***
+ const isVoice = this.voiceDetected();
+
+ if (isVoice && totalEnergy > currentThreshold) {
+ if (totalEnergy > this.peakEnergy) {
+ this.peakEnergy = totalEnergy;
}
if (!this.isSpeaking) {
this.isSpeaking = true;
- this.peakEnergy = avgEnergy;
+ this.speakingStartTime = now;
+ this.peakEnergy = totalEnergy;
this.lastSilenceTime = now;
}
- // Rilevamento della caduta di energia relativa per identificare stacchi tra sillabe/parole (legato acustico)
- const dropRatio = (this.peakEnergy - avgEnergy) / this.peakEnergy;
+ // Rilevamento della caduta di energia relativa per stacchi sillabici
+ const dropRatio = (this.peakEnergy - totalEnergy) / this.peakEnergy;
if (this.isSpeaking && dropRatio > 0.20 && this.peakEnergy > currentThreshold * 1.1) {
- if (now - this.lastWordTime > this.COOLDOWN) {
+ const speakDuration = now - this.speakingStartTime;
+ if (now - this.lastWordTime > this.COOLDOWN && speakDuration > 120) {
this.linesDetected.update(v => v + 1);
this.lastWordTime = now;
- this.peakEnergy = avgEnergy; // Reset peak
+ this.peakEnergy = totalEnergy;
this.isSpeaking = false;
- console.log('Line advanced - relative drop detected (highly sensitive)', { dropRatio, avgEnergy, peak: this.peakEnergy });
+ console.log('[VoiceDetect] Word counted - voice energy drop', {
+ score: this.voiceScore().toFixed(3),
+ dropRatio: dropRatio.toFixed(2),
+ duration: speakDuration
+ });
}
}
this.lastSilenceTime = now;
} else {
- // Sotto soglia (silenzio)
+ // Sotto soglia, chitarra, o silenzio
if (this.isSpeaking && (now - this.lastSilenceTime > 180)) {
- if (now - this.lastWordTime > this.COOLDOWN) {
+ // Calcola la durata reale del segnale vocale escludendo la finestra di silenzio (180ms)
+ const actualSoundDuration = this.lastSilenceTime - this.speakingStartTime;
+ if (isVoice && now - this.lastWordTime > this.COOLDOWN && actualSoundDuration > 100) {
this.linesDetected.update(v => v + 1);
this.lastWordTime = now;
- console.log('Line advanced - silence detected');
+ console.log('[VoiceDetect] Word counted - voice silence gap', { duration: actualSoundDuration });
}
this.isSpeaking = false;
this.peakEnergy = 0;
}
+
+ // Se la chitarra è dominante, resetta lo stato di parlato
+ if (!isVoice && this.isSpeaking) {
+ this.isSpeaking = false;
+ this.peakEnergy = 0;
+ }
}
this.animationFrame = requestAnimationFrame(analyze);
diff --git a/src/app/services/face-detector.service.ts b/src/app/services/face-detector.service.ts
new file mode 100644
index 0000000..bf25374
--- /dev/null
+++ b/src/app/services/face-detector.service.ts
@@ -0,0 +1,215 @@
+import { Injectable, signal } from '@angular/core';
+
+@Injectable({
+ providedIn: 'root'
+})
+export class FaceDetectorService {
+ public isCameraActive = signal(false);
+ public currentTiltAngle = signal(0);
+ public isTilted = signal(false);
+
+ private stream: MediaStream | null = null;
+ private camera: any = null;
+ private faceMesh: any = null;
+ private onTiltCallback: ((direction: 'next' | 'prev') => void) | null = null;
+
+ // Gesture state machine
+ private tiltStartTime: number = 0;
+ private inCooldown: boolean = false;
+ private readonly TILT_THRESHOLD = 15; // Degrees to trigger next/prev page
+ private readonly TILT_HOLD_MS = 300; // How long to hold the tilt
+ private readonly RETURN_THRESHOLD = 6; // Degrees to reset cooldown
+
+ constructor() {}
+
+ /**
+ * Loads MediaPipe scripts dynamically if not already loaded.
+ */
+ private loadScripts(): Promise {
+ return new Promise((resolve, reject) => {
+ if ((window as any).FaceMesh && (window as any).Camera) {
+ resolve();
+ return;
+ }
+
+ const cameraScript = document.createElement('script');
+ cameraScript.src = 'https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js';
+ cameraScript.crossOrigin = 'anonymous';
+
+ const faceMeshScript = document.createElement('script');
+ faceMeshScript.src = 'https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/face_mesh.js';
+ faceMeshScript.crossOrigin = 'anonymous';
+
+ cameraScript.onload = () => {
+ document.head.appendChild(faceMeshScript);
+ };
+
+ faceMeshScript.onload = () => {
+ resolve();
+ };
+
+ cameraScript.onerror = (err) => reject(err);
+ faceMeshScript.onerror = (err) => reject(err);
+
+ document.head.appendChild(cameraScript);
+ });
+ }
+
+ /**
+ * Starts camera capture and face mesh tracking.
+ */
+ async start(videoElement: HTMLVideoElement, onTilt: (direction: 'next' | 'prev') => void): Promise {
+ if (this.isCameraActive()) return;
+ this.onTiltCallback = onTilt;
+
+ try {
+ await this.loadScripts();
+
+ // Request camera permissions and stream
+ this.stream = await navigator.mediaDevices.getUserMedia({
+ video: {
+ width: { ideal: 320 },
+ height: { ideal: 240 },
+ facingMode: 'user'
+ },
+ audio: false
+ });
+
+ videoElement.srcObject = this.stream;
+ videoElement.setAttribute('playsinline', 'true');
+ videoElement.muted = true;
+ videoElement.play();
+
+ const FaceMeshLib = (window as any).FaceMesh;
+ const CameraLib = (window as any).Camera;
+
+ if (!FaceMeshLib || !CameraLib) {
+ throw new Error('MediaPipe libraries failed to initialize.');
+ }
+
+ this.faceMesh = new FaceMeshLib({
+ locateFile: (file: string) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`
+ });
+
+ this.faceMesh.setOptions({
+ maxNumFaces: 1,
+ refineLandmarks: false,
+ minDetectionConfidence: 0.6,
+ minTrackingConfidence: 0.6
+ });
+
+ this.faceMesh.onResults((results: any) => {
+ this.processLandmarks(results);
+ });
+
+ this.camera = new CameraLib(videoElement, {
+ onFrame: async () => {
+ if (this.isCameraActive() && this.faceMesh) {
+ await this.faceMesh.send({ image: videoElement });
+ }
+ },
+ width: 320,
+ height: 240
+ });
+
+ this.isCameraActive.set(true);
+ await this.camera.start();
+ console.log('[FaceDetector] Face tracking started successfully.');
+ } catch (err) {
+ console.error('[FaceDetector] Failed to start face tracking:', err);
+ this.stop();
+ throw err;
+ }
+ }
+
+ /**
+ * Processes landmarks to calculate head tilt angle.
+ */
+ private processLandmarks(results: any) {
+ if (!results.multiFaceLandmarks || results.multiFaceLandmarks.length === 0) {
+ this.currentTiltAngle.set(0);
+ this.isTilted.set(false);
+ return;
+ }
+
+ const landmarks = results.multiFaceLandmarks[0];
+
+ // Left eye corner (landmark 33) and right eye corner (landmark 263)
+ const leftEye = landmarks[33];
+ const rightEye = landmarks[263];
+
+ if (!leftEye || !rightEye) return;
+
+ // Calculate angle in degrees
+ const dy = rightEye.y - leftEye.y;
+ const dx = rightEye.x - leftEye.x;
+
+ // Normalize angle (roll)
+ let angle = Math.atan2(dy, dx) * (180 / Math.PI);
+
+ // Smooth angle updates
+ this.currentTiltAngle.set(Math.round(angle));
+
+ const absAngle = Math.abs(angle);
+
+ if (absAngle > this.TILT_THRESHOLD) {
+ this.isTilted.set(true);
+
+ if (!this.inCooldown) {
+ if (this.tiltStartTime === 0) {
+ this.tiltStartTime = Date.now();
+ } else if (Date.now() - this.tiltStartTime > this.TILT_HOLD_MS) {
+ // Trigger the tilt gesture!
+ const direction = angle > 0 ? 'prev' : 'next';
+ console.log(`[FaceDetector] Head tilt gesture detected! Angle: ${angle.toFixed(1)}°, Direction: ${direction}`);
+ if (this.onTiltCallback) {
+ this.onTiltCallback(direction);
+ }
+ this.inCooldown = true;
+ this.tiltStartTime = 0;
+ }
+ }
+ } else {
+ this.isTilted.set(false);
+ this.tiltStartTime = 0;
+
+ // Reset cooldown when the head returns near the center
+ if (absAngle < this.RETURN_THRESHOLD) {
+ this.inCooldown = false;
+ }
+ }
+ }
+
+ /**
+ * Stops camera capture and releases face mesh resources.
+ */
+ stop() {
+ this.isCameraActive.set(false);
+ this.currentTiltAngle.set(0);
+ this.isTilted.set(false);
+ this.inCooldown = false;
+ this.tiltStartTime = 0;
+
+ if (this.camera) {
+ try {
+ this.camera.stop();
+ } catch (e) {}
+ this.camera = null;
+ }
+
+ if (this.stream) {
+ this.stream.getTracks().forEach(track => track.stop());
+ this.stream = null;
+ }
+
+ if (this.faceMesh) {
+ try {
+ this.faceMesh.close();
+ } catch (e) {}
+ this.faceMesh = null;
+ }
+
+ this.onTiltCallback = null;
+ console.log('[FaceDetector] Face tracking stopped.');
+ }
+}