Avanzamento con riconoscimento facciale

This commit is contained in:
David Frassi
2026-06-05 16:00:00 +02:00
parent 92e955915e
commit e7e43468b3
5 changed files with 832 additions and 421 deletions
+10 -34
View File
@@ -45,8 +45,7 @@
[class.full-screen-container]="settingsService.fullscreenMode()" [class.full-screen-container]="settingsService.fullscreenMode()"
(touchstart)="onTouchStart($event)" (touchstart)="onTouchStart($event)"
(touchmove)="onTouchMove($event)" (touchmove)="onTouchMove($event)"
(touchend)="onTouchEnd()" (touchend)="onTouchEnd()">
(wheel)="onWheel($event)">
<!-- Landscape Side Controls (Scrollable) --> <!-- Landscape Side Controls (Scrollable) -->
@@ -137,20 +136,6 @@
</div> </div>
</ion-toolbar> </ion-toolbar>
<!-- Visualizzatore Vocale & Log del Riconoscimento in Tempo Reale -->
<div class="transcript-area" *ngIf="settingsService.enableAcousticAutoscroll() && audioEngine.isListening()" style="background: rgba(0, 0, 0, 0.55); backdrop-filter: blur(16px); -webkit-backdrop-filter: blur(16px); padding: 10px 14px; border-top: 1px solid rgba(255,255,255,0.08); display: flex; flex-direction: column; gap: 8px; align-items: center; justify-content: center; position: relative;">
<div class="energy-bar-container" style="width: 100%; height: 5px; background: rgba(255,255,255,0.12); border-radius: 3px; overflow: hidden;">
<div class="energy-bar" [style.width.%]="math.min(100, audioEngine.energyLevel() * 3)" style="height: 100%; background: linear-gradient(90deg, var(--ion-color-secondary) 0%, #20e9ff 100%); border-radius: 3px; transition: width 0.08s ease-out;"></div>
</div>
<div class="speech-log-row" style="display: flex; align-items: center; justify-content: center; width: 100%;">
<div class="acoustic-counter-badge" style="display: inline-flex; align-items: center; gap: 6px; background: rgba(32, 233, 255, 0.12); border: 1px solid rgba(32, 233, 255, 0.25); color: #20e9ff; padding: 4px 10px; border-radius: 20px; font-size: 0.75rem; font-weight: 700;">
<span style="width: 6px; height: 6px; background: #20e9ff; border-radius: 50%; display: inline-block; box-shadow: 0 0 8px #20e9ff; animation: pulse 1.5s infinite;"></span>
<span>Parole Rilevate: {{ wordsSpokenInCurrentPage }} / {{ getPageTargetWordCount() }}</span>
</div>
</div>
</div>
<ion-toolbar class="bg-gradient slim-toolbar"> <ion-toolbar class="bg-gradient slim-toolbar">
<div class="slim-controls"> <div class="slim-controls">
<!-- Autoscroll Standard in Portrait Footer --> <!-- Autoscroll Standard in Portrait Footer -->
@@ -167,9 +152,9 @@
</ion-button> </ion-button>
</div> </div>
<!-- Karaoke Toggle in Portrait --> <!-- Camera Head Gestures Toggle in Portrait -->
<ion-button *ngIf="settingsService.enableAcousticAutoscroll()" fill="clear" size="small" (click)="toggleListening()" [color]="audioEngine.isListening() ? 'danger' : 'secondary'" class="mic-btn-portrait"> <ion-button fill="clear" size="small" (click)="toggleCameraNavigation()" [color]="enableCameraNavigation() ? 'success' : 'secondary'">
<ion-icon slot="icon-only" [name]="audioEngine.isListening() ? 'mic' : 'mic-off'"></ion-icon> <ion-icon slot="icon-only" [name]="enableCameraNavigation() ? 'videocam' : 'videocam-off-outline'"></ion-icon>
</ion-button> </ion-button>
@@ -220,21 +205,12 @@
</ion-toolbar> </ion-toolbar>
</ion-footer> </ion-footer>
<!-- Vertical Sensitivity Slider Overlay --> <!-- Floating Camera Preview for Head Navigation -->
<div class="mic-sensitivity-overlay" *ngIf="settingsService.enableAcousticAutoscroll() && showSensitivitySlider() && audioEngine.isListening()"> <div class="camera-preview-floating glass" *ngIf="enableCameraNavigation()">
<div class="slider-card glass"> <video id="face-preview-video" muted playsinline autoplay></video>
<ion-button fill="clear" color="secondary" (click)="toggleSensitivitySlider($event)" class="close-slider-btn"> <div class="camera-preview-overlay">
<ion-icon name="close-outline"></ion-icon> <div class="angle-indicator" [class.tilted]="faceDetector.isTilted()">
</ion-button> {{ faceDetector.currentTiltAngle() }}°
<div class="slider-wrapper"
(touchstart)="handleSensitivityTouch($event)"
(touchmove)="handleSensitivityTouch($event)">
<div class="custom-vertical-slider">
<div class="slider-track"></div>
<div class="slider-fill" [style.height.%]="(audioEngine.sensitivity() - 50) * 2"></div>
<div class="slider-knob" [style.bottom.%]="(audioEngine.sensitivity() - 50) * 2"></div>
</div>
</div> </div>
<span class="sensitivity-label outfit-font">{{ audioEngine.sensitivity() }}%</span>
</div> </div>
</div> </div>
+158
View File
@@ -455,6 +455,113 @@ ion-content.full-screen-content {
} }
} }
.voice-threshold-overlay {
position: fixed;
left: 15px;
top: 50%;
transform: translateY(-50%);
z-index: 1000;
pointer-events: auto;
@media (orientation: landscape) {
left: 15px;
}
.slider-card {
display: flex;
flex-direction: column;
align-items: center;
padding: 8px 6px 12px 6px;
background: rgba(18, 18, 18, 0.85) !important;
backdrop-filter: blur(25px);
border-radius: 24px;
border: 1px solid rgba(46, 213, 115, 0.4);
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.6);
gap: 8px;
animation: slideInLeft 0.3s cubic-bezier(0.175, 0.885, 0.32, 1.275);
}
.close-slider-btn {
--padding-start: 0;
--padding-end: 0;
margin: 0;
height: 36px;
width: 36px;
--color: var(--ion-color-secondary);
ion-icon {
font-size: 1.5rem;
}
}
.slider-wrapper {
height: 180px;
width: 40px;
display: flex;
align-items: center;
justify-content: center;
position: relative;
padding: 10px 0;
}
.custom-vertical-slider {
position: relative;
width: 8px;
height: 100%;
background: rgba(255, 255, 255, 0.1);
border-radius: 4px;
.slider-track {
position: absolute;
width: 100%;
height: 100%;
}
.slider-fill {
position: absolute;
bottom: 0;
width: 100%;
background: linear-gradient(to top, #2ed573, #7bed9f);
border-radius: 4px;
box-shadow: 0 0 10px rgba(46, 213, 115, 0.3);
transition: height 0.05s linear;
}
.slider-knob {
position: absolute;
left: 50%;
transform: translate(-50%, 50%);
width: 28px;
height: 28px;
background: #2ed573;
border-radius: 50%;
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.5), 0 0 15px rgba(46, 213, 115, 0.5);
border: 2px solid #fff;
transition: bottom 0.05s linear;
}
}
.sensitivity-label {
font-size: 0.8rem;
font-weight: 800;
color: #2ed573;
min-width: 40px;
text-align: center;
letter-spacing: 0.5px;
}
}
@keyframes slideInLeft {
from {
opacity: 0;
transform: translateX(-20px);
}
to {
opacity: 1;
transform: translateX(0);
}
}
.floating-autoscroll-bar { .floating-autoscroll-bar {
position: fixed; position: fixed;
bottom: 120px; bottom: 120px;
@@ -612,3 +719,54 @@ ion-content.full-screen-content {
transform: translate(-50%, 0); transform: translate(-50%, 0);
} }
} }
.camera-preview-floating {
position: fixed;
bottom: 60px;
right: 15px;
width: 90px;
height: 120px;
border-radius: 12px;
overflow: hidden;
border: 1px solid rgba(var(--ion-color-secondary-rgb), 0.4);
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.5);
z-index: 1000;
display: flex;
flex-direction: column;
background: #000;
@media (orientation: landscape) {
bottom: 15px;
right: 80px;
}
video {
width: 100%;
height: 100%;
object-fit: cover;
transform: scaleX(-1); // Mirror camera preview
}
.camera-preview-overlay {
position: absolute;
bottom: 0;
left: 0;
right: 0;
background: rgba(0, 0, 0, 0.6);
padding: 2px 0;
display: flex;
justify-content: center;
align-items: center;
.angle-indicator {
font-size: 0.75rem;
font-weight: 700;
color: #fff;
&.tilted {
color: #2ed573;
text-shadow: 0 0 5px #2ed573;
}
}
}
}
+41 -242
View File
@@ -3,7 +3,6 @@ import { DomSanitizer, SafeResourceUrl } from '@angular/platform-browser';
import { ActivatedRoute, Router } from '@angular/router'; import { ActivatedRoute, Router } from '@angular/router';
import { AlertController, ToastController, GestureController } from '@ionic/angular'; import { AlertController, ToastController, GestureController } from '@ionic/angular';
import { CantiService, Canto } from '../../services/canti.service'; import { CantiService, Canto } from '../../services/canti.service';
import { AudioEngineService } from '../../services/audio-engine.service';
import { LyricsParserService, ParsedSection } from '../../services/lyrics-parser.service'; import { LyricsParserService, ParsedSection } from '../../services/lyrics-parser.service';
import { ThemeService } from '../../services/theme.service'; import { ThemeService } from '../../services/theme.service';
import { SettingsService } from '../../services/settings.service'; import { SettingsService } from '../../services/settings.service';
@@ -13,6 +12,7 @@ import { YoutubePlayerService } from '../../services/youtube-player.service';
import { MyCantiService } from '../../services/my-canti.service'; import { MyCantiService } from '../../services/my-canti.service';
import { ComunitaService } from '../../services/comunita.service'; import { ComunitaService } from '../../services/comunita.service';
import { StatsService } from '../../services/stats.service'; import { StatsService } from '../../services/stats.service';
import { FaceDetectorService } from '../../services/face-detector.service';
@Component({ @Component({
selector: 'app-player', selector: 'app-player',
@@ -26,7 +26,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
/** true = show chords (accordi mode), false = text only */ /** true = show chords (accordi mode), false = text only */
public showChords = signal<boolean>(false); public showChords = signal<boolean>(false);
public showSensitivitySlider = signal<boolean>(false);
/** Autoscroll standard */ /** Autoscroll standard */
public isAutoscrolling = signal<boolean>(false); public isAutoscrolling = signal<boolean>(false);
@@ -98,7 +97,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
private lastAdvanceTimestamp: number = 0; // For safety cooldown private lastAdvanceTimestamp: number = 0; // For safety cooldown
private readonly ADVANCE_COOLDOWN = 1500; // 1.5 seconds min private readonly ADVANCE_COOLDOWN = 1500; // 1.5 seconds min
public wordsSpokenInCurrentPage: number = 0;
public lastScrollBlock = signal<'start' | 'center'>('start'); public lastScrollBlock = signal<'start' | 'center'>('start');
private lastProcessedTranscript: string = ''; private lastProcessedTranscript: string = '';
private initialStartTime: number = 0; private initialStartTime: number = 0;
@@ -106,7 +104,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
private route = inject(ActivatedRoute); private route = inject(ActivatedRoute);
public router = inject(Router); public router = inject(Router);
public cantiService = inject(CantiService); public cantiService = inject(CantiService);
public audioEngine = inject(AudioEngineService);
private lyricsParser = inject(LyricsParserService); private lyricsParser = inject(LyricsParserService);
private alertCtrl = inject(AlertController); private alertCtrl = inject(AlertController);
private toastCtrl = inject(ToastController); private toastCtrl = inject(ToastController);
@@ -120,6 +117,9 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
private gestureCtrl = inject(GestureController); private gestureCtrl = inject(GestureController);
private el = inject(ElementRef); private el = inject(ElementRef);
private sanitizer = inject(DomSanitizer); private sanitizer = inject(DomSanitizer);
public faceDetector = inject(FaceDetectorService);
public enableCameraNavigation = signal<boolean>(false);
private songStartTime: number = 0; private songStartTime: number = 0;
@@ -135,23 +135,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
this.showChords.set(this.settingsService.showChordsDefault()); this.showChords.set(this.settingsService.showChordsDefault());
}, { allowSignalWrites: true }); }, { allowSignalWrites: true });
// Avanzamento guidato dai picchi acustici con conteggio parole della riga in corso
effect(() => {
const count = this.audioEngine.linesDetected();
if (count > 0 && this.audioEngine.isListening()) {
this.handleAcousticPeak();
}
});
// Avanzamento in tempo reale tramite riconoscimento vocale di prossimità (driver primario veloce)
effect(() => {
const transcript = this.audioEngine.backgroundTranscript();
const isSpeechActive = this.audioEngine.speechRecognitionActive();
if (transcript && this.audioEngine.isListening() && isSpeechActive) {
this.checkWordAdvancement();
}
});
// Auto-scroll logic: keep active line in view // Auto-scroll logic: keep active line in view
effect(() => { effect(() => {
const idx = this.currentLineIndex(); const idx = this.currentLineIndex();
@@ -275,26 +258,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
} }
}); });
gestureX.enable(); gestureX.enable();
const gestureY = this.gestureCtrl.create({
el: this.el.nativeElement,
direction: 'y',
gestureName: 'swipe-song-y',
canStart: (ev) => {
const target = ev.event.target as HTMLElement;
return !target.closest('ion-footer') && !target.closest('ion-header');
},
onEnd: (ev) => {
if (Math.abs(ev.deltaY) > 50) {
if (ev.deltaY > 0) {
this.prev();
} else {
this.next();
}
}
}
});
gestureY.enable();
} }
@@ -361,25 +324,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
this.initialPinchDistance = null; this.initialPinchDistance = null;
} }
private lastWheelTime = 0;
private readonly WHEEL_COOLDOWN = 400; // ms to avoid too rapid advancement
onWheel(event: WheelEvent) {
event.preventDefault();
const now = Date.now();
if (now - this.lastWheelTime < this.WHEEL_COOLDOWN) {
return;
}
if (Math.abs(event.deltaY) > 5) {
if (event.deltaY > 0) {
this.next();
} else {
this.prev();
}
this.lastWheelTime = now;
}
}
private getDistance(t1: Touch, t2: Touch): number { private getDistance(t1: Touch, t2: Touch): number {
return Math.sqrt(Math.pow(t1.clientX - t2.clientX, 2) + Math.pow(t1.clientY - t2.clientY, 2)); return Math.sqrt(Math.pow(t1.clientX - t2.clientX, 2) + Math.pow(t1.clientY - t2.clientY, 2));
} }
@@ -434,40 +378,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
} }
} }
toggleListening() {
if (this.audioEngine.isListening()) {
this.audioEngine.stopListening();
this.showSensitivitySlider.set(false);
} else {
this.audioEngine.startListening();
this.showSensitivitySlider.set(true);
}
}
toggleSensitivitySlider(event: Event) {
event.stopPropagation();
this.showSensitivitySlider.update(v => !v);
}
onSensitivityChange(event: any) {
this.audioEngine.sensitivity.set(event.detail.value);
}
handleSensitivityTouch(event: TouchEvent) {
event.preventDefault();
const touch = event.touches[0];
const target = event.currentTarget as HTMLElement;
const rect = target.getBoundingClientRect();
// Calculate percentage based on Y position (bottom is 50%, top is 100%)
const rawPercentage = 100 - ((touch.clientY - rect.top) / rect.height * 100);
// Map 0-100 raw to 50-100 range
let percentage = 50 + (rawPercentage * 0.5);
percentage = Math.max(50, Math.min(100, Math.round(percentage)));
this.audioEngine.sensitivity.set(percentage);
}
toggleAudio() { toggleAudio() {
if (this.youtubePlayerService.currentCantoId() === this.canto()?.id) { if (this.youtubePlayerService.currentCantoId() === this.canto()?.id) {
this.youtubePlayerService.togglePlayPause(); this.youtubePlayerService.togglePlayPause();
@@ -492,10 +402,8 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
restart() { restart() {
this.currentLineIndex.set(0); this.currentLineIndex.set(0);
this.audioEngine.resetWordCount();
this.youtubePlayerService.seekTo(0); this.youtubePlayerService.seekTo(0);
this.channel.postMessage({ type: 'SYNC_INDEX', index: 0 }); this.channel.postMessage({ type: 'SYNC_INDEX', index: 0 });
this.wordsSpokenInCurrentPage = 0;
this.lastScrollBlock.set('center'); this.lastScrollBlock.set('center');
} }
@@ -549,26 +457,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
} }
} }
private getLineWords(lineIndex: number): string[] {
const allLines = this.getAllLines();
const lineText = allLines[lineIndex]?.text || '';
return lineText
.toLowerCase()
.split(/\s+/)
.map((w: string) => w.replace(/[^a-zA-Z0-9àèìòùáéíóú]/g, ''))
.filter((w: string) => w.length >= 3);
}
public getPageWords(startIndex: number, pageSize: number): string[] {
const allWords: string[] = [];
const totalLines = this.getTotalLines();
const end = Math.min(totalLines, startIndex + pageSize);
for (let i = startIndex; i < end; i++) {
allWords.push(...this.getLineWords(i));
}
return allWords;
}
public getVisibleLineIndices(): number[] { public getVisibleLineIndices(): number[] {
try { try {
const container = document.querySelector('.lyrics-container') as HTMLElement; const container = document.querySelector('.lyrics-container') as HTMLElement;
@@ -607,127 +495,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
} }
} }
public getPageTargetWordCount(): number {
const visibleIndices = this.getVisibleLineIndices();
let totalRawWords = 0;
const currentIdx = this.currentLineIndex();
if (visibleIndices.length > 0) {
if (currentIdx === 0) {
// Sulla prima pagina, contiamo solo la prima metà delle righe visibili
// per far avanzare l'applicazione non appena si arriva a metà della prima schermata
const halfLength = Math.max(1, Math.ceil(visibleIndices.length / 2));
for (let i = 0; i < halfLength; i++) {
const idx = visibleIndices[i];
totalRawWords += this.getLineRawWordCount(idx);
}
} else {
// Dalla seconda pagina in poi, contiamo solo le righe attive e future (escludendo il testo già cantato in alto)
for (const idx of visibleIndices) {
if (idx >= currentIdx) {
totalRawWords += this.getLineRawWordCount(idx);
}
}
}
} else {
// Fallback: calcolo teorico basato sulla pagina corrente
const pageSize = this.calculatePageStepSize();
const totalLines = this.getTotalLines();
let end: number;
if (currentIdx === 0) {
// Fallback prima pagina: metà delle righe della pagina teorica
const halfPageSize = Math.max(1, Math.ceil(pageSize / 2));
end = Math.min(totalLines, currentIdx + halfPageSize);
} else {
end = Math.min(totalLines, currentIdx + pageSize);
}
for (let i = currentIdx; i < end; i++) {
totalRawWords += this.getLineRawWordCount(i);
}
}
// Soglia al 70% delle parole contate nella schermata visibile
return Math.max(1, Math.ceil(totalRawWords * 0.70));
}
private checkWordAdvancement() {
const now = Date.now();
// Cooldown per prevenire avanzamenti multipli troppo veloci
if (now - this.lastAdvanceTimestamp < this.ADVANCE_COOLDOWN) {
return;
}
const currentIdx = this.currentLineIndex();
let nextPageIdx: number;
const totalLines = this.getTotalLines();
if (currentIdx === 0) {
// Sulla prima pagina, la "pagina successiva" è in realtà la seconda metà della prima schermata
const visibleIndices = this.getVisibleLineIndices();
if (visibleIndices.length > 0) {
const halfLength = Math.max(1, Math.ceil(visibleIndices.length / 2));
nextPageIdx = visibleIndices[halfLength] !== undefined ? visibleIndices[halfLength] : halfLength;
} else {
const pageSize = this.calculatePageStepSize();
const halfPageSize = Math.max(1, Math.ceil(pageSize / 2));
nextPageIdx = currentIdx + halfPageSize;
}
} else {
nextPageIdx = this.calculateNextPageStartIndex();
}
if (nextPageIdx >= totalLines) return;
const pageSize = this.calculatePageStepSize();
// Raccoglie le parole del blocco successivo (metà pagina se siamo all'inizio, altrimenti intera pagina)
const nextPageWords = this.getPageWords(nextPageIdx, currentIdx === 0 ? Math.max(1, Math.ceil(pageSize / 2)) : pageSize);
if (nextPageWords.length === 0) return;
// Parole rilevate di recente (con tolleranza di ritardo della Speech API)
const recentWords = this.audioEngine.getRecentWordsSince(this.lastAdvanceTimestamp);
// Controlla se l'utente ha cantato parole della pagina successiva -> Avanzamento istantaneo immediato
const matchNext = nextPageWords.some(word => recentWords.includes(word));
if (matchNext) {
console.log(`[SmartKaraoke] Parola chiave del blocco successivo rilevata! Avanzamento immediato a: ${nextPageIdx}`);
this.lastScrollBlock.set('center');
this.currentLineIndex.set(nextPageIdx);
this.channel.postMessage({ type: 'SYNC_INDEX', index: nextPageIdx });
this.lastAdvanceTimestamp = now;
this.audioEngine.resetWordCount();
this.wordsSpokenInCurrentPage = 0;
}
}
public getLineRawWordCount(lineIndex: number): number {
const allLines = this.getAllLines();
const lineText = allLines[lineIndex]?.text || '';
const words = lineText.trim().split(/\s+/).filter((w: string) => w.length > 0);
return words.length;
}
public getLineTargetWordCount(lineIndex: number): number {
const rawCount = this.getLineRawWordCount(lineIndex);
// Soglia proporzionale tollerante (65% del conteggio parole effettivo) per compensare legature melodiche
return Math.max(1, Math.ceil(rawCount * 0.65));
}
private handleAcousticPeak() {
const targetWordCount = this.getPageTargetWordCount();
this.wordsSpokenInCurrentPage++;
console.log(`[PeakAdvance] Peak detected. Spoken words on page: ${this.wordsSpokenInCurrentPage}/${targetWordCount}`);
if (this.wordsSpokenInCurrentPage >= targetWordCount) {
console.log(`[PeakAdvance] Target page word count reached (${targetWordCount}). Advancing page.`);
this.next(true);
}
}
private calculateNextPageStartIndex(): number { private calculateNextPageStartIndex(): number {
try { try {
const container = document.querySelector('.lyrics-container') as HTMLElement; const container = document.querySelector('.lyrics-container') as HTMLElement;
@@ -781,8 +548,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
next(isAutomatic: boolean = false) { next(isAutomatic: boolean = false) {
this.lastAdvanceTimestamp = Date.now(); this.lastAdvanceTimestamp = Date.now();
this.audioEngine.resetWordCount();
this.wordsSpokenInCurrentPage = 0;
const totalLines = this.getTotalLines(); const totalLines = this.getTotalLines();
const prevIdx = this.currentLineIndex(); const prevIdx = this.currentLineIndex();
@@ -825,8 +590,6 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
prev() { prev() {
this.lastAdvanceTimestamp = Date.now(); this.lastAdvanceTimestamp = Date.now();
this.audioEngine.resetWordCount();
this.wordsSpokenInCurrentPage = 0;
const prevIdx = this.currentLineIndex(); const prevIdx = this.currentLineIndex();
if (prevIdx > 0) { if (prevIdx > 0) {
@@ -1064,10 +827,46 @@ export class PlayerPage implements OnInit, AfterViewInit, OnDestroy {
} }
} }
toggleCameraNavigation() {
if (this.enableCameraNavigation()) {
this.stopCameraNavigation();
} else {
this.startCameraNavigation();
}
}
startCameraNavigation() {
this.enableCameraNavigation.set(true);
setTimeout(async () => {
const videoEl = document.querySelector('#face-preview-video') as HTMLVideoElement;
if (videoEl) {
try {
await this.faceDetector.start(videoEl, (direction) => {
console.log(`[PlayerPage] Head tilt trigger received: ${direction}`);
if (direction === 'next') {
this.next(false);
} else {
this.prev();
}
});
} catch (e) {
this.enableCameraNavigation.set(false);
alert('Impossibile accedere alla fotocamera. Assicurati di aver concesso i permessi e di usare HTTPS.');
}
}
}, 300);
}
stopCameraNavigation() {
this.enableCameraNavigation.set(false);
this.faceDetector.stop();
}
ngOnDestroy() { ngOnDestroy() {
this.stopAutoscroll(); this.stopAutoscroll();
this.logPreviousSongTime(); this.logPreviousSongTime();
this.audioEngine.stopListening(); this.stopCameraNavigation();
this.channel.close(); this.channel.close();
} }
} }
+408 -145
View File
@@ -13,6 +13,16 @@ export class AudioEngineService {
public sensitivity = signal<number>(75); // Default sensitivity (50-100) public sensitivity = signal<number>(75); // Default sensitivity (50-100)
public speechRecognitionActive = signal<boolean>(false); public speechRecognitionActive = signal<boolean>(false);
// Voice/Guitar detection signals
public voiceDetected = signal<boolean>(false);
public guitarDetected = signal<boolean>(false);
/** Soglia del punteggio voce: valori più bassi = più sensibile alla voce (range 0.05 0.50) */
public voiceThreshold = signal<number>(0.15);
/** Debug: punteggio voce corrente (0-1) per feedback visivo */
public voiceScore = signal<number>(0);
// Real-time background continuous transcript and word buffer for reliability checks // Real-time background continuous transcript and word buffer for reliability checks
public backgroundTranscript = signal<string>(''); public backgroundTranscript = signal<string>('');
private backgroundRecognition: any = null; private backgroundRecognition: any = null;
@@ -23,9 +33,6 @@ export class AudioEngineService {
} }
private audioContext: AudioContext | null = null; private audioContext: AudioContext | null = null;
private highpassFilter: BiquadFilterNode | null = null;
private lowpassFilter: BiquadFilterNode | null = null;
private vocalBandpass: BiquadFilterNode | null = null;
private analyser: AnalyserNode | null = null; private analyser: AnalyserNode | null = null;
private stream: MediaStream | null = null; private stream: MediaStream | null = null;
private animationFrame: number | null = null; private animationFrame: number | null = null;
@@ -35,6 +42,7 @@ export class AudioEngineService {
private isSpeaking: boolean = false; private isSpeaking: boolean = false;
private lastSilenceTime: number = Date.now(); private lastSilenceTime: number = Date.now();
private lastWordTime: number = 0; private lastWordTime: number = 0;
private speakingStartTime: number = 0;
// Constants for tuning - Optimized for close proximity (singer/guitarist) // Constants for tuning - Optimized for close proximity (singer/guitarist)
private readonly SILENCE_GAP = 100; // ms private readonly SILENCE_GAP = 100; // ms
@@ -44,111 +52,57 @@ export class AudioEngineService {
private lineStarted: boolean = false; private lineStarted: boolean = false;
private noiseFloor: number = 30; // Adaptive noise floor starting point private noiseFloor: number = 30; // Adaptive noise floor starting point
// --- Advanced Voice Detection State ---
// Smoothing buffer (16 frames ~260ms a 60fps)
private readonly SMOOTHING_FRAMES = 16;
private voiceScoreBuffer: number[] = [];
// Spectral flux: previous frame spectrum for change detection
private previousSpectrum: Float32Array | null = null;
// Hysteresis: once voice is detected, it stays on for at least this many frames
private readonly VOICE_HOLD_FRAMES = 12; // ~200ms
private voiceHoldCounter: number = 0;
// Sub-band definitions (Hz) - 8 fine-grained bands
private readonly BANDS = [
{ name: 'sub_bass', start: 80, end: 150 }, // Fondamentali basse chitarra (Mi2=82, La2=110)
{ name: 'bass', start: 150, end: 300 }, // Fondamentali voce maschile/femminile + chitarra
{ name: 'low_mid', start: 300, end: 600 }, // Armoniche chitarra dominanti
{ name: 'mid', start: 600, end: 1000 }, // Zona transizione
{ name: 'formant_f1', start: 1000, end: 1500 }, // Formante F1 voce (vocali aperte)
{ name: 'formant_f2', start: 1500, end: 2500 }, // Formante F2 voce (KEY: quasi assente in chitarra)
{ name: 'formant_f3', start: 2500, end: 3500 }, // Formante F3 voce (sibilanti morbide)
{ name: 'sibilants', start: 3500, end: 6000 }, // Consonanti sibilanti (s, t, f, sh) - SOLO voce
];
constructor() { constructor() {
this.initSpeechRecognition(); this.initSpeechRecognition();
this.initBackgroundRecognition();
} }
private initBackgroundRecognition() { private initBackgroundRecognition() {
const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition; // Disattivato per utilizzare esclusivamente l'analizzatore FFT acustico locale
if (SpeechRecognition) {
this.backgroundRecognition = new SpeechRecognition();
this.backgroundRecognition.continuous = true;
this.backgroundRecognition.interimResults = true;
this.backgroundRecognition.lang = 'it-IT';
this.backgroundRecognition.onstart = () => {
console.log('Background speech recognition started successfully');
this.speechRecognitionActive.set(true);
};
this.backgroundRecognition.onresult = (event: any) => {
let interimTranscript = '';
let finalTranscript = '';
for (let i = event.resultIndex; i < event.results.length; ++i) {
if (event.results[i].isFinal) {
finalTranscript += event.results[i][0].transcript;
} else {
interimTranscript += event.results[i][0].transcript;
}
}
const fullTranscript = (finalTranscript + ' ' + interimTranscript).trim().toLowerCase();
if (fullTranscript) {
this.backgroundTranscript.set(fullTranscript);
this.updateRecentWords(fullTranscript);
}
};
this.backgroundRecognition.onerror = (err: any) => {
console.error('Background recognition error:', err);
this.speechRecognitionActive.set(false);
if (this.isListening() && err.error !== 'aborted' && err.error !== 'not-allowed') {
setTimeout(() => this.startBackgroundRecognition(), 1000);
}
};
this.backgroundRecognition.onend = () => {
console.log('Background speech recognition ended');
if (this.isListening()) {
this.startBackgroundRecognition();
} else {
this.speechRecognitionActive.set(false);
}
};
}
} }
private startBackgroundRecognition() { private startBackgroundRecognition() {
if (!this.backgroundRecognition) return; // Disattivato
try {
this.backgroundRecognition.start();
} catch (e) {
// Recognition already started or running
}
} }
private stopBackgroundRecognition() { private stopBackgroundRecognition() {
if (!this.backgroundRecognition) return; // Disattivato
try {
this.backgroundRecognition.stop();
} catch (e) {
// Recognition already stopped
}
} }
private updateRecentWords(transcript: string) { private updateRecentWords(transcript: string) {
const words = transcript.split(/\s+/); // Disattivato
const now = Date.now();
words.forEach(w => {
// Strip special characters and keep alphanumeric Italian letters
const cleaned = w.replace(/[^a-zA-Z0-9àèìòùáéíóú]/g, '').toLowerCase();
if (cleaned.length >= 3) {
// Avoid adding duplicate words within the last 3 seconds
const existing = this.recentWordsBuffer.find(item => item.word === cleaned && now - item.timestamp < 3000);
if (!existing) {
this.recentWordsBuffer.push({ word: cleaned, timestamp: now });
}
}
});
// Keep only words from the last 10 seconds to make it highly responsive
this.recentWordsBuffer = this.recentWordsBuffer.filter(item => now - item.timestamp < 10000);
} }
public getRecentWords(): string[] { public getRecentWords(): string[] {
const now = Date.now(); return [];
this.recentWordsBuffer = this.recentWordsBuffer.filter(item => now - item.timestamp < 10000);
return this.recentWordsBuffer.map(item => item.word);
} }
public getRecentWordsSince(timestamp: number): string[] { public getRecentWordsSince(timestamp: number): string[] {
const now = Date.now(); return [];
// Consente una sovrapposizione di 1.5 secondi per coprire la latenza di SpeechRecognition
const cutoff = Math.max(timestamp - 1500, now - 10000);
this.recentWordsBuffer = this.recentWordsBuffer.filter(item => now - item.timestamp < 10000);
return this.recentWordsBuffer
.filter(item => item.timestamp > cutoff)
.map(item => item.word);
} }
public clearRecentWords() { public clearRecentWords() {
@@ -202,6 +156,20 @@ export class AudioEngineService {
async startListening() { async startListening() {
if (this.isListening()) return; if (this.isListening()) return;
this.isListening.set(true);
this.voiceDetected.set(false);
this.guitarDetected.set(false);
this.voiceScoreBuffer = [];
this.previousSpectrum = null;
this.voiceHoldCounter = 0;
this.clearRecentWords();
console.log('[AudioEngine] Starting local FFT Audio Analyser');
await this.startOfflineAnalyser();
}
public async startOfflineAnalyser() {
if (this.analyser) return; // Già attivo
try { try {
this.stream = await navigator.mediaDevices.getUserMedia({ this.stream = await navigator.mediaDevices.getUserMedia({
audio: { audio: {
@@ -213,126 +181,421 @@ export class AudioEngineService {
this.audioContext = new AudioContext(); this.audioContext = new AudioContext();
const source = this.audioContext.createMediaStreamSource(this.stream); const source = this.audioContext.createMediaStreamSource(this.stream);
// Catena di filtri in cascata per isolamento vocale // FFT a 4096 punti per risoluzione ~10.7 Hz/bin (a 44100 Hz)
this.highpassFilter = this.audioContext.createBiquadFilter();
this.highpassFilter.type = 'highpass';
this.highpassFilter.frequency.value = 150; // Taglia i bassi e rombi di fondo
this.lowpassFilter = this.audioContext.createBiquadFilter();
this.lowpassFilter.type = 'lowpass';
this.lowpassFilter.frequency.value = 2000; // Taglia sibili e armonici acuti di chitarre
this.vocalBandpass = this.audioContext.createBiquadFilter();
this.vocalBandpass.type = 'bandpass';
this.vocalBandpass.frequency.value = 800; // Centro della gamma vocale primaria
this.vocalBandpass.Q.value = 1.2; // Filtro stretto focalizzato sulla voce
this.analyser = this.audioContext.createAnalyser(); this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 512; this.analyser.fftSize = 4096;
this.analyser.smoothingTimeConstant = 0.4; // Smoothing moderato per stabilità spettrale
// Connessione in cascata: source -> highpass -> lowpass -> bandpass -> analyser source.connect(this.analyser);
source.connect(this.highpassFilter);
this.highpassFilter.connect(this.lowpassFilter);
this.lowpassFilter.connect(this.vocalBandpass);
this.vocalBandpass.connect(this.analyser);
this.isListening.set(true);
this.clearRecentWords();
this.startBackgroundRecognition();
this.processAudio(); this.processAudio();
} catch (err) { } catch (err) {
console.error('Error accessing microphone', err); console.error('[AudioEngine] Error accessing microphone for offline analyser:', err);
// Se fallisce anche il mic locale, proviamo a ripristinare lo stato
this.stopListening();
alert('Errore microfono: assicurati di usare HTTPS e di aver dato i permessi.'); alert('Errore microfono: assicurati di usare HTTPS e di aver dato i permessi.');
} }
} }
stopListening() { stopListening() {
if (this.animationFrame) cancelAnimationFrame(this.animationFrame); if (this.animationFrame) cancelAnimationFrame(this.animationFrame);
// Rilascia le risorse del microfono locale
this.stream?.getTracks().forEach(track => track.stop()); this.stream?.getTracks().forEach(track => track.stop());
this.audioContext?.close(); this.audioContext?.close();
this.stream = null;
this.audioContext = null;
this.analyser = null;
this.isListening.set(false); this.isListening.set(false);
this.energyLevel.set(0); this.energyLevel.set(0);
this.voiceDetected.set(false);
this.guitarDetected.set(false);
this.voiceScore.set(0);
this.voiceScoreBuffer = [];
this.previousSpectrum = null;
this.voiceHoldCounter = 0;
this.stopBackgroundRecognition(); this.stopBackgroundRecognition();
this.clearRecentWords(); this.clearRecentWords();
this.highpassFilter = null;
this.lowpassFilter = null;
this.vocalBandpass = null;
this.speechRecognitionActive.set(false); this.speechRecognitionActive.set(false);
} }
// =====================================================================
// ANALISI SPETTRALE AVANZATA MULTI-FEATURE
// =====================================================================
/**
* Calcola l'energia media in una sotto-banda dello spettro.
*/
private getBandEnergy(dataArray: Uint8Array, binWidth: number, startHz: number, endHz: number): number {
const startBin = Math.max(0, Math.floor(startHz / binWidth));
const endBin = Math.min(dataArray.length - 1, Math.floor(endHz / binWidth));
if (endBin <= startBin) return 0;
let sum = 0;
for (let i = startBin; i <= endBin; i++) {
sum += dataArray[i];
}
return sum / (endBin - startBin + 1);
}
/**
* FEATURE 1: Rapporto formanti vocali F2+F3 vs bande strumentali (low+mid)
*
* La voce umana ha formanti F2 (1500-2500 Hz) e F3 (2500-3500 Hz) molto prominenti.
* La chitarra acustica ha pochissima energia in queste bande.
* Rapporto alto = voce, basso = strumento.
*/
private calcFormantRatio(bandEnergies: number[]): number {
const instrumentEnergy = bandEnergies[0] + bandEnergies[1] + bandEnergies[2] + bandEnergies[3]; // 80-1000 Hz
const formantEnergy = bandEnergies[5] + bandEnergies[6]; // F2 (1500-2500) + F3 (2500-3500)
return formantEnergy / (instrumentEnergy + 0.001);
}
/**
* FEATURE 2: Energia sibilanti (3500-6000 Hz)
*
* Solo la voce umana produce consonanti sibilanti come "s", "t", "f", "sh", "z"
* che hanno energia significativa sopra 3500 Hz. Gli strumenti acustici (chitarra,
* pianoforte, etc.) hanno energia trascurabile in questa banda.
*
* Ritorna un valore normalizzato 0-1.
*/
private calcSibilantScore(bandEnergies: number[], totalEnergy: number): number {
if (totalEnergy < 1) return 0;
const sibilantEnergy = bandEnergies[7]; // 3500-6000 Hz
// Normalizza: una sibilante tipica ha 15-40% dell'energia totale
return Math.min(1, sibilantEnergy / (totalEnergy * 0.15 + 0.001));
}
/**
* FEATURE 3: Spectral Flatness (entropia di Wiener)
*
* Misura quanto lo spettro è "piatto" (rumoroso) vs "piccato" (tonale).
* - Chitarra: spettro molto tonale con picchi armonici netti → flatness BASSA
* - Voce (consonanti): spettro più rumoroso → flatness ALTA
* - Voce (vocali): moderatamente tonale ma con formanti larghe → flatness MEDIA
*
* Formula: media_geometrica / media_aritmetica (0=tono puro, 1=rumore bianco)
*/
private calcSpectralFlatness(dataArray: Uint8Array, binWidth: number): number {
// Calcola nella regione 300-6000 Hz (dove la discriminazione è più utile)
const startBin = Math.max(1, Math.floor(300 / binWidth));
const endBin = Math.min(dataArray.length - 1, Math.floor(6000 / binWidth));
const n = endBin - startBin + 1;
if (n <= 0) return 0;
let logSum = 0;
let arithmeticSum = 0;
let zeroCount = 0;
for (let i = startBin; i <= endBin; i++) {
const val = Math.max(dataArray[i], 0.001); // Avoid log(0)
logSum += Math.log(val);
arithmeticSum += val;
if (dataArray[i] === 0) zeroCount++;
}
if (zeroCount > n * 0.5) return 0; // Troppi zeri = silenzio
const geometricMean = Math.exp(logSum / n);
const arithmeticMean = arithmeticSum / n;
if (arithmeticMean < 0.001) return 0;
return Math.min(1, geometricMean / arithmeticMean);
}
/**
* FEATURE 4: Spectral Flux (velocità di cambiamento spettrale)
*
* Misura quanto rapidamente cambia la forma dello spettro tra frame successivi.
* - Voce: alta flux (alternanza vocali/consonanti, prosodia)
* - Chitarra: bassa flux (note sostenute, spettro stabile)
*
* Ritorna un valore normalizzato 0-1.
*/
private calcSpectralFlux(dataArray: Uint8Array, binWidth: number): number {
const startBin = Math.max(0, Math.floor(200 / binWidth));
const endBin = Math.min(dataArray.length - 1, Math.floor(5000 / binWidth));
if (!this.previousSpectrum) {
this.previousSpectrum = new Float32Array(dataArray.length);
for (let i = 0; i < dataArray.length; i++) {
this.previousSpectrum[i] = dataArray[i];
}
return 0;
}
let flux = 0;
let maxFlux = 0;
for (let i = startBin; i <= endBin; i++) {
const diff = dataArray[i] - this.previousSpectrum[i];
// Solo variazioni positive (onset) per evitare il decadimento naturale
if (diff > 0) {
flux += diff * diff;
}
maxFlux += 255 * 255; // Massimo teorico
}
// Aggiorna il buffer dello spettro precedente
for (let i = 0; i < dataArray.length; i++) {
this.previousSpectrum[i] = dataArray[i];
}
if (maxFlux === 0) return 0;
// Normalizza e scala logaritmicamente per sensibilità
const normalizedFlux = flux / maxFlux;
return Math.min(1, Math.sqrt(normalizedFlux) * 10); // Amplifica le piccole variazioni
}
/**
* FEATURE 5: Formant Peak Detection
*
* Cerca picchi spettrali caratteristici nella regione delle formanti vocali (800-3500 Hz).
* La voce umana produce 2-4 picchi prominenti (formanti F1-F4).
* La chitarra ha uno spettro armonico regolare senza picchi formantici.
*
* Ritorna il numero di picchi formantici rilevati (0-4), normalizzato 0-1.
*/
private calcFormantPeaks(dataArray: Uint8Array, binWidth: number): number {
const startBin = Math.max(0, Math.floor(800 / binWidth));
const endBin = Math.min(dataArray.length - 1, Math.floor(3500 / binWidth));
// Calcola la media locale per determinare la soglia di prominenza
let sum = 0;
for (let i = startBin; i <= endBin; i++) {
sum += dataArray[i];
}
const avgEnergy = sum / Math.max(1, endBin - startBin + 1);
if (avgEnergy < 5) return 0; // Silenzio
// Cerca picchi: un bin è un picco se è maggiore dei 5 bin a sinistra e 5 a destra
// e supera la media di almeno 30%
const peakThreshold = avgEnergy * 1.3;
const windowSize = Math.max(3, Math.floor(100 / binWidth)); // ~100 Hz di finestra
let peakCount = 0;
let lastPeakBin = -windowSize * 2; // Evita di contare picchi troppo vicini
for (let i = startBin + windowSize; i <= endBin - windowSize; i++) {
if (dataArray[i] < peakThreshold) continue;
let isPeak = true;
for (let j = 1; j <= windowSize; j++) {
if (dataArray[i] < dataArray[i - j] || dataArray[i] < dataArray[i + j]) {
isPeak = false;
break;
}
}
if (isPeak && (i - lastPeakBin) > windowSize) {
peakCount++;
lastPeakBin = i;
if (peakCount >= 4) break; // Max 4 formanti
}
}
return Math.min(1, peakCount / 3); // 3 formanti = punteggio pieno
}
/**
* FEATURE 6: Rapporto energia alta/bassa (spectral tilt)
*
* La voce umana (soprattutto da vicino al microfono) ha uno spettro più "piatto"
* con energia significativa anche nelle alte frequenze.
* La chitarra ha una forte caduta sopra 1-2 kHz.
*/
private calcSpectralTilt(bandEnergies: number[]): number {
const lowEnergy = bandEnergies[0] + bandEnergies[1] + bandEnergies[2]; // 80-600 Hz
const highEnergy = bandEnergies[5] + bandEnergies[6] + bandEnergies[7]; // 1500-6000 Hz
if (lowEnergy < 0.001) return 0;
// Un rapporto alto indica più energia nelle alte frequenze (voce)
return Math.min(1, highEnergy / (lowEnergy + 0.001));
}
/**
* Calcola il punteggio voce combinato da tutte le feature.
*
* Pesi delle feature (calibrati per massimizzare la discriminazione):
* - Formant ratio: 25% (molto discriminante)
* - Sibilant score: 20% (quasi esclusivo della voce)
* - Spectral flatness: 15% (voce più rumorosa di chitarra)
* - Spectral flux: 15% (voce cambia più rapidamente)
* - Formant peaks: 15% (struttura formanti unica della voce)
* - Spectral tilt: 10% (distribuzione energia)
*/
private calcVoiceScore(
dataArray: Uint8Array,
bandEnergies: number[],
totalEnergy: number,
binWidth: number
): number {
const formantRatio = this.calcFormantRatio(bandEnergies);
const sibilantScore = this.calcSibilantScore(bandEnergies, totalEnergy);
const flatness = this.calcSpectralFlatness(dataArray, binWidth);
const flux = this.calcSpectralFlux(dataArray, binWidth);
const formantPeaks = this.calcFormantPeaks(dataArray, binWidth);
const spectralTilt = this.calcSpectralTilt(bandEnergies);
// Normalizza formantRatio: tipicamente 0.05-0.5 per voce, 0-0.05 per chitarra
const normalizedFormantRatio = Math.min(1, formantRatio / 0.4);
// Punteggio pesato
const score =
normalizedFormantRatio * 0.25 +
sibilantScore * 0.20 +
flatness * 0.15 +
flux * 0.15 +
formantPeaks * 0.15 +
spectralTilt * 0.10;
return Math.min(1, Math.max(0, score));
}
// =====================================================================
// MAIN AUDIO PROCESSING LOOP
// =====================================================================
private processAudio() { private processAudio() {
if (!this.analyser) return; if (!this.analyser) return;
const bufferLength = this.analyser.frequencyBinCount; const bufferLength = this.analyser.frequencyBinCount; // 2048 con fftSize=4096
const dataArray = new Uint8Array(bufferLength); const dataArray = new Uint8Array(bufferLength);
// Con la catena di filtri in cascata, l'energia vocale è centrata tra 200 Hz e 2000 Hz.
const sampleRate = this.audioContext ? this.audioContext.sampleRate : 44100; const sampleRate = this.audioContext ? this.audioContext.sampleRate : 44100;
const binWidth = sampleRate / 512; const binWidth = sampleRate / 4096; // ~10.7 Hz/bin
const startBin = Math.max(0, Math.floor(200 / binWidth));
const endBin = Math.min(bufferLength - 1, Math.floor(2000 / binWidth)); // Energia minima per considerare che c'è suono (evita falsi positivi nel silenzio)
const MIN_ENERGY = 5;
const analyze = () => { const analyze = () => {
if (!this.analyser) return; if (!this.analyser) return;
this.analyser.getByteFrequencyData(dataArray); this.analyser.getByteFrequencyData(dataArray);
// Calcola l'energia media ESCLUSIVAMENTE nella gamma di frequenza della voce // Calcola energia in ciascuna delle 8 sotto-bande
let sum = 0; const bandEnergies: number[] = this.BANDS.map(band =>
for (let i = startBin; i <= endBin; i++) { this.getBandEnergy(dataArray, binWidth, band.start, band.end)
sum += dataArray[i]; );
}
const avgEnergy = sum / (endBin - startBin + 1); // Energia totale media
this.energyLevel.set(avgEnergy); const totalEnergy = bandEnergies.reduce((a, b) => a + b, 0) / bandEnergies.length;
// Feedback visuale (barra energia) — usa la media delle bande vocali per coerenza
const vocalDisplayEnergy = (bandEnergies[4] + bandEnergies[5] + bandEnergies[6]) / 3;
this.energyLevel.set(vocalDisplayEnergy);
const now = Date.now(); const now = Date.now();
const hasSound = totalEnergy > MIN_ENERGY;
if (hasSound) {
// Calcola il punteggio voce multi-feature
const rawScore = this.calcVoiceScore(dataArray, bandEnergies, totalEnergy, binWidth);
// Aggiungi al buffer di smoothing
this.voiceScoreBuffer.push(rawScore);
if (this.voiceScoreBuffer.length > this.SMOOTHING_FRAMES) {
this.voiceScoreBuffer.shift();
}
// Media pesata: i frame più recenti contano di più
let weightedSum = 0;
let weightTotal = 0;
for (let i = 0; i < this.voiceScoreBuffer.length; i++) {
const weight = (i + 1); // Peso crescente
weightedSum += this.voiceScoreBuffer[i] * weight;
weightTotal += weight;
}
const smoothedScore = weightedSum / weightTotal;
this.voiceScore.set(smoothedScore);
// Determinazione con isteresi
const threshold = this.voiceThreshold();
if (smoothedScore > threshold) {
// Voce rilevata
this.voiceDetected.set(true);
this.guitarDetected.set(false);
this.voiceHoldCounter = this.VOICE_HOLD_FRAMES;
} else if (this.voiceHoldCounter > 0) {
// Isteresi: mantieni lo stato "voce" per evitare oscillazioni
this.voiceHoldCounter--;
// Lo stato resta quello precedente (voce)
} else {
// Strumento/rumore ambientale
this.voiceDetected.set(false);
this.guitarDetected.set(true);
}
} else {
// Silenzio
this.voiceDetected.set(false);
this.guitarDetected.set(false);
this.voiceScore.set(0);
this.voiceHoldCounter = 0;
}
// --- Logica conteggio parole (solo se voce rilevata) ---
// Tracciamento adattivo del rumore di fondo // Tracciamento adattivo del rumore di fondo
if (avgEnergy < this.noiseFloor) { if (totalEnergy < this.noiseFloor) {
this.noiseFloor = this.noiseFloor * 0.95 + avgEnergy * 0.05; this.noiseFloor = this.noiseFloor * 0.95 + totalEnergy * 0.05;
} else { } else {
this.noiseFloor = this.noiseFloor * 0.998 + avgEnergy * 0.002; this.noiseFloor = this.noiseFloor * 0.998 + totalEnergy * 0.002;
} }
// Soglia di sensibilità definita dall'utente // Soglia di sensibilità definita dall'utente
const userThreshold = 220 - (this.sensitivity() * 2.0); const userThreshold = 220 - (this.sensitivity() * 2.0);
// La soglia effettiva si adatta al rumore di fondo per evitare falsi avanzamenti
const currentThreshold = Math.max(userThreshold, this.noiseFloor + 12); const currentThreshold = Math.max(userThreshold, this.noiseFloor + 12);
if (avgEnergy > currentThreshold) { // *** CONTEGGIO PAROLE SOLO QUANDO LA VOCE UMANA È DOMINANTE ***
if (avgEnergy > this.peakEnergy) { const isVoice = this.voiceDetected();
this.peakEnergy = avgEnergy;
if (isVoice && totalEnergy > currentThreshold) {
if (totalEnergy > this.peakEnergy) {
this.peakEnergy = totalEnergy;
} }
if (!this.isSpeaking) { if (!this.isSpeaking) {
this.isSpeaking = true; this.isSpeaking = true;
this.peakEnergy = avgEnergy; this.speakingStartTime = now;
this.peakEnergy = totalEnergy;
this.lastSilenceTime = now; this.lastSilenceTime = now;
} }
// Rilevamento della caduta di energia relativa per identificare stacchi tra sillabe/parole (legato acustico) // Rilevamento della caduta di energia relativa per stacchi sillabici
const dropRatio = (this.peakEnergy - avgEnergy) / this.peakEnergy; const dropRatio = (this.peakEnergy - totalEnergy) / this.peakEnergy;
if (this.isSpeaking && dropRatio > 0.20 && this.peakEnergy > currentThreshold * 1.1) { if (this.isSpeaking && dropRatio > 0.20 && this.peakEnergy > currentThreshold * 1.1) {
if (now - this.lastWordTime > this.COOLDOWN) { const speakDuration = now - this.speakingStartTime;
if (now - this.lastWordTime > this.COOLDOWN && speakDuration > 120) {
this.linesDetected.update(v => v + 1); this.linesDetected.update(v => v + 1);
this.lastWordTime = now; this.lastWordTime = now;
this.peakEnergy = avgEnergy; // Reset peak this.peakEnergy = totalEnergy;
this.isSpeaking = false; this.isSpeaking = false;
console.log('Line advanced - relative drop detected (highly sensitive)', { dropRatio, avgEnergy, peak: this.peakEnergy }); console.log('[VoiceDetect] Word counted - voice energy drop', {
score: this.voiceScore().toFixed(3),
dropRatio: dropRatio.toFixed(2),
duration: speakDuration
});
} }
} }
this.lastSilenceTime = now; this.lastSilenceTime = now;
} else { } else {
// Sotto soglia (silenzio) // Sotto soglia, chitarra, o silenzio
if (this.isSpeaking && (now - this.lastSilenceTime > 180)) { if (this.isSpeaking && (now - this.lastSilenceTime > 180)) {
if (now - this.lastWordTime > this.COOLDOWN) { // Calcola la durata reale del segnale vocale escludendo la finestra di silenzio (180ms)
const actualSoundDuration = this.lastSilenceTime - this.speakingStartTime;
if (isVoice && now - this.lastWordTime > this.COOLDOWN && actualSoundDuration > 100) {
this.linesDetected.update(v => v + 1); this.linesDetected.update(v => v + 1);
this.lastWordTime = now; this.lastWordTime = now;
console.log('Line advanced - silence detected'); console.log('[VoiceDetect] Word counted - voice silence gap', { duration: actualSoundDuration });
} }
this.isSpeaking = false; this.isSpeaking = false;
this.peakEnergy = 0; this.peakEnergy = 0;
} }
// Se la chitarra è dominante, resetta lo stato di parlato
if (!isVoice && this.isSpeaking) {
this.isSpeaking = false;
this.peakEnergy = 0;
}
} }
this.animationFrame = requestAnimationFrame(analyze); this.animationFrame = requestAnimationFrame(analyze);
+215
View File
@@ -0,0 +1,215 @@
import { Injectable, signal } from '@angular/core';
@Injectable({
providedIn: 'root'
})
export class FaceDetectorService {
public isCameraActive = signal<boolean>(false);
public currentTiltAngle = signal<number>(0);
public isTilted = signal<boolean>(false);
private stream: MediaStream | null = null;
private camera: any = null;
private faceMesh: any = null;
private onTiltCallback: ((direction: 'next' | 'prev') => void) | null = null;
// Gesture state machine
private tiltStartTime: number = 0;
private inCooldown: boolean = false;
private readonly TILT_THRESHOLD = 15; // Degrees to trigger next/prev page
private readonly TILT_HOLD_MS = 300; // How long to hold the tilt
private readonly RETURN_THRESHOLD = 6; // Degrees to reset cooldown
constructor() {}
/**
* Loads MediaPipe scripts dynamically if not already loaded.
*/
private loadScripts(): Promise<void> {
return new Promise((resolve, reject) => {
if ((window as any).FaceMesh && (window as any).Camera) {
resolve();
return;
}
const cameraScript = document.createElement('script');
cameraScript.src = 'https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js';
cameraScript.crossOrigin = 'anonymous';
const faceMeshScript = document.createElement('script');
faceMeshScript.src = 'https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/face_mesh.js';
faceMeshScript.crossOrigin = 'anonymous';
cameraScript.onload = () => {
document.head.appendChild(faceMeshScript);
};
faceMeshScript.onload = () => {
resolve();
};
cameraScript.onerror = (err) => reject(err);
faceMeshScript.onerror = (err) => reject(err);
document.head.appendChild(cameraScript);
});
}
/**
* Starts camera capture and face mesh tracking.
*/
async start(videoElement: HTMLVideoElement, onTilt: (direction: 'next' | 'prev') => void): Promise<void> {
if (this.isCameraActive()) return;
this.onTiltCallback = onTilt;
try {
await this.loadScripts();
// Request camera permissions and stream
this.stream = await navigator.mediaDevices.getUserMedia({
video: {
width: { ideal: 320 },
height: { ideal: 240 },
facingMode: 'user'
},
audio: false
});
videoElement.srcObject = this.stream;
videoElement.setAttribute('playsinline', 'true');
videoElement.muted = true;
videoElement.play();
const FaceMeshLib = (window as any).FaceMesh;
const CameraLib = (window as any).Camera;
if (!FaceMeshLib || !CameraLib) {
throw new Error('MediaPipe libraries failed to initialize.');
}
this.faceMesh = new FaceMeshLib({
locateFile: (file: string) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`
});
this.faceMesh.setOptions({
maxNumFaces: 1,
refineLandmarks: false,
minDetectionConfidence: 0.6,
minTrackingConfidence: 0.6
});
this.faceMesh.onResults((results: any) => {
this.processLandmarks(results);
});
this.camera = new CameraLib(videoElement, {
onFrame: async () => {
if (this.isCameraActive() && this.faceMesh) {
await this.faceMesh.send({ image: videoElement });
}
},
width: 320,
height: 240
});
this.isCameraActive.set(true);
await this.camera.start();
console.log('[FaceDetector] Face tracking started successfully.');
} catch (err) {
console.error('[FaceDetector] Failed to start face tracking:', err);
this.stop();
throw err;
}
}
/**
* Processes landmarks to calculate head tilt angle.
*/
private processLandmarks(results: any) {
if (!results.multiFaceLandmarks || results.multiFaceLandmarks.length === 0) {
this.currentTiltAngle.set(0);
this.isTilted.set(false);
return;
}
const landmarks = results.multiFaceLandmarks[0];
// Left eye corner (landmark 33) and right eye corner (landmark 263)
const leftEye = landmarks[33];
const rightEye = landmarks[263];
if (!leftEye || !rightEye) return;
// Calculate angle in degrees
const dy = rightEye.y - leftEye.y;
const dx = rightEye.x - leftEye.x;
// Normalize angle (roll)
let angle = Math.atan2(dy, dx) * (180 / Math.PI);
// Smooth angle updates
this.currentTiltAngle.set(Math.round(angle));
const absAngle = Math.abs(angle);
if (absAngle > this.TILT_THRESHOLD) {
this.isTilted.set(true);
if (!this.inCooldown) {
if (this.tiltStartTime === 0) {
this.tiltStartTime = Date.now();
} else if (Date.now() - this.tiltStartTime > this.TILT_HOLD_MS) {
// Trigger the tilt gesture!
const direction = angle > 0 ? 'prev' : 'next';
console.log(`[FaceDetector] Head tilt gesture detected! Angle: ${angle.toFixed(1)}°, Direction: ${direction}`);
if (this.onTiltCallback) {
this.onTiltCallback(direction);
}
this.inCooldown = true;
this.tiltStartTime = 0;
}
}
} else {
this.isTilted.set(false);
this.tiltStartTime = 0;
// Reset cooldown when the head returns near the center
if (absAngle < this.RETURN_THRESHOLD) {
this.inCooldown = false;
}
}
}
/**
* Stops camera capture and releases face mesh resources.
*/
stop() {
this.isCameraActive.set(false);
this.currentTiltAngle.set(0);
this.isTilted.set(false);
this.inCooldown = false;
this.tiltStartTime = 0;
if (this.camera) {
try {
this.camera.stop();
} catch (e) {}
this.camera = null;
}
if (this.stream) {
this.stream.getTracks().forEach(track => track.stop());
this.stream = null;
}
if (this.faceMesh) {
try {
this.faceMesh.close();
} catch (e) {}
this.faceMesh = null;
}
this.onTiltCallback = null;
console.log('[FaceDetector] Face tracking stopped.');
}
}