<?phpnamespace App\Media;use App\Model\Lyrics;/** * Lyrics made from the song itself: Whisper listens to it and writes each sung line with its time, then Claude Code * (when it is on the computer: `claude -p`) puts the words right, line for line (the times stay Whisper's), and the * result is a .LYR of 2001 in var/lyr. Everything runs in the background, one step after the other. * * Whisper: whisper.cpp (whisper-cli) of the computer, else faster-whisper (the same model on CTranslate2, int8, every * core: a song in a few seconds; openai-whisper took half an hour for one minute on one core), installed by WinZik the * first time in its own Python (venv in its data folder, no PyTorch), else openai-whisper of the computer. Windows * without Python: whisper.cpp's release + its "base" model. ffmpeg (Tools) makes the 16 kHz sound Whisper reads. * * States: idle | installing | converting | listening | correcting | done | error */final class Transcriber{ public const CPP_ZIP = 'https://github.com/ggml-org/whisper.cpp/releases/latest/download/whisper-bin-x64.zip'; public const MODEL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin'; public string $state = 'idle'; /** whisper: Whisper (+ Claude puts its words right); internet: Claude searches the web for the real lyrics (aligned on Whisper's times when it is there) */ public string $mode = 'whisper'; /** @var array{artist: string, title: string, duration: float} what is known of the song (for the search) */ private array $hint = ['artist' => '', 'title' => '', 'duration' => 0.0]; public string $error = ''; public string $song = ''; /** the .LYR written (done) */ public string $result = ''; public bool $corrected = false; private ?Process $job = null; private ?FileDownload $download = null; /** @var list<array{0: list<string>, 1: string}> the install steps left: [command, what it does] */ private array $steps = []; /** @var list<array{0: float, 1: string}> */ private array $lines = []; private string $work; /** @var (\Closure(string): string)|null where the .LYR of a song goes (WinZik::lyricsCachePath()); null: var/lyr/<name>.lyr */ public ?\Closure $lyricsPath = null; /** @param (\Closure(list<string>): Process)|null $runner */ public function __construct(private Tools $tools, private string $dataDir, private string $lyrDir, private ?\Closure $runner = null, public bool $useClaude = true, private ?\JumPHP\Http\Http $http = null) { $this->work = $dataDir . '/tmp/whisper'; } private function run(array $command): Process { return $this->runner !== null ? ($this->runner)($command) : Process::start($command, $this->work . '/logs'); } public function busy(): bool { return !in_array($this->state, ['idle', 'done', 'error'], true); } /** whisper.cpp: [binary, model] when both are there. @return array{0: string, 1: string}|null */ private function cpp(): ?array { $bin = Tools::onPath('whisper-cli') ?? Tools::onPath('whisper-cpp'); if ($bin === null) { foreach ((array)glob($this->dataDir . '/bin/whisper/{,*/,*/*/}' . Tools::exe('whisper-cli'), GLOB_BRACE) as $f) { $bin = (string)$f; } } $model = null; foreach ([$this->dataDir . '/models/ggml-base.bin', (getenv('HOME') ?: '') . '/.cache/whisper.cpp/ggml-base.bin'] as $m) { if (is_file($m)) { $model = $m; break; } } return $bin !== null ? [$bin, $model ?? ''] : null; } /** openai-whisper of the computer. */ private function openai(): ?string { return Tools::onPath('whisper'); } /** The Python of WinZik's faster-whisper (var/whisper), when it is installed. */ private function faster(): ?string { $python = $this->dataDir . (PHP_OS_FAMILY === 'Windows' ? '/whisper/Scripts/python.exe' : '/whisper/bin/python'); $mark = $this->dataDir . '/whisper/faster-whisper.ok'; return is_file($python) && is_file($mark) ? $python : null; } /** What WinZik's Python runs: the 16 kHz WAV read as it is (no audio library to break), segments as JSON. */ private const SCRIPT = <<<'PY'import json, os, sys, waveimport numpy as npfrom faster_whisper import WhisperModelwith wave.open(sys.argv[1], 'rb') as w: audio = np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16).astype(np.float32) / 32768.0model = WhisperModel(sys.argv[3], device='cpu', compute_type='int8', cpu_threads=os.cpu_count() or 4)# no voice filter: sung voices over music are not "speech" for it, everything was left outsegments, info = model.transcribe(audio, vad_filter=False, beam_size=5, condition_on_previous_text=False)out = [{'start': round(s.start, 2), 'text': s.text} for s in segments]with open(sys.argv[2], 'w', encoding='utf-8') as f: json.dump({'language': info.language, 'segments': out}, f, ensure_ascii=False)PY; public static function claude(): ?string { return Tools::onPath('claude'); } /** Whisper can listen now (nothing to install). */ public function ready(): bool { $cpp = $this->cpp(); return ($cpp !== null && $cpp[1] !== '') || $this->faster() !== null || $this->openai() !== null; } /** * Lyrics for this song: Whisper (installed first when needed), then Claude, then the .LYR; or ($mode internet) * Claude's search on the web, aligned on Whisper when it is installed. * @param array{artist?: string, title?: string, duration?: float} $hint */ public function start(string $song, string $mode = 'whisper', array $hint = []): bool { if ($this->busy() || !is_file($song)) { return false; } $this->mode = $mode === 'internet' ? 'internet' : 'whisper'; $this->hint = ['artist' => (string)($hint['artist'] ?? ''), 'title' => (string)($hint['title'] ?? pathinfo($song, PATHINFO_FILENAME)), 'duration' => (float)($hint['duration'] ?? 0)]; $this->song = $song; $this->error = $this->result = ''; $this->corrected = false; $this->lines = []; @mkdir($this->work, 0777, true); if ($this->mode === 'internet') { if (self::claude() === null) { $this->fail('Claude Code n\'est pas installé sur cet ordinateur (claude)'); return false; } $this->askInternet(); // Claude finds where they are, WinZik takes them there return true; } if ($this->tools->program('ffmpeg') === null) { $this->fail('ffmpeg pas encore prêt'); return false; } if ($this->ready()) { $this->convert(); } else { $this->install(); } return true; } private function install(): void { $this->state = 'installing'; $cpp = $this->cpp(); if ($cpp !== null && $cpp[1] === '') { // whisper.cpp is there, not its model $this->download = new FileDownload(self::MODEL, $this->dataDir . '/models/ggml-base.bin'); return; } $python = PHP_OS_FAMILY === 'Windows' ? (Tools::onPath('py') ?? Tools::onPath('python')) : Tools::onPath('python3'); if (PHP_OS_FAMILY === 'Windows' && $python === null) { $this->download = new FileDownload(self::CPP_ZIP, $this->work . '/whisper-bin-x64.zip'); return; } if ($python === null) { $this->fail('Whisper absent : installez python3 (WinZik y installe faster-whisper) ou whisper.cpp (whisper-cli)'); return; } $venv = $this->dataDir . '/whisper'; $pip = PHP_OS_FAMILY === 'Windows' ? "$venv/Scripts/pip.exe" : "$venv/bin/pip"; $this->steps = [ [[$python, '-m', 'venv', $venv], 'Python de Whisper'], [[$pip, 'install', '--quiet', 'faster-whisper'], 'faster-whisper'], ]; $this->nextStep(); } private function nextStep(): void { $step = array_shift($this->steps); if ($step === null && is_dir($this->dataDir . '/whisper') && !is_file($this->dataDir . '/whisper/faster-whisper.ok')) { @touch($this->dataDir . '/whisper/faster-whisper.ok'); } if ($step === null) { $this->ready() ? $this->convert() : $this->fail("Whisper n'a pas pu être installé"); return; } $this->error = $step[1]; $this->job = $this->run($step[0]); } /** ffmpeg: the song as 16 kHz mono WAV, what Whisper reads. */ private function convert(): void { $this->state = 'converting'; $this->job = $this->run([(string)$this->tools->program('ffmpeg'), '-y', '-v', 'error', '-i', $this->song, '-vn', '-ac', '1', '-ar', '16000', '-c:a', 'pcm_s16le', $this->work . '/song.wav']); } private function listen(): void { $this->state = 'listening'; @unlink($this->work . '/song.json'); $cpp = $this->cpp(); if ($cpp !== null && $cpp[1] !== '') { $this->job = $this->run([$cpp[0], '-m', $cpp[1], '-f', $this->work . '/song.wav', '-l', 'auto', '-oj', '-of', $this->work . '/song', '-t', (string)max(2, min(8, (int)(getenv('NUMBER_OF_PROCESSORS') ?: 4)))]); return; } $faster = $this->faster(); if ($faster !== null) { file_put_contents($this->dataDir . '/whisper/winzik-whisper.py', self::SCRIPT); $this->job = $this->run([$faster, $this->dataDir . '/whisper/winzik-whisper.py', $this->work . '/song.wav', $this->work . '/song.json', 'base']); return; } $this->job = $this->run([(string)$this->openai(), $this->work . '/song.wav', '--model', 'base', '--output_format', 'json', '--output_dir', $this->work, '--verbose', 'False', '--fp16', 'False']); } /** Quitting: whisper / ffmpeg / the install ended now (nothing left running alone); again: nothing. */ public function stop(): void { if ($this->job !== null) { $this->job->stop(); $this->job->cleanup(); $this->job = null; } $this->download = null; $this->steps = []; if ($this->busy()) { $this->state = 'idle'; } } /** Each frame. */ public function poll(): void { if ($this->download !== null) { if ($this->download->poll()) { $d = $this->download; $this->download = null; if (!$d->ok()) { $this->fail('téléchargement de Whisper : ' . $d->error); } elseif (str_ends_with($d->target, '.zip')) { // whisper.cpp for Windows: unpacked, then its model @mkdir($this->dataDir . '/bin/whisper', 0777, true); $this->job = $this->run(['tar', '-xf', $d->target, '-C', $this->dataDir . '/bin/whisper']); $this->steps = []; } else { $this->ready() ? $this->convert() : $this->install(); } } return; } if ($this->job === null) { return; } $this->job->poll(); if (!$this->job->done()) { return; } $job = $this->job; $this->job = null; $ok = $job->ok(); $out = $job->output(); $why = $job->error(); $job->cleanup(); match ($this->state) { 'installing' => $ok ? ($this->cpp() !== null && $this->cpp()[1] === '' ? $this->install() : $this->nextStep()) : $this->fail('installation de Whisper : ' . ($why ?: $this->error)), 'converting' => $ok ? $this->listen() : $this->fail('ffmpeg : ' . $why), 'listening' => $ok ? $this->heard() : $this->fail('Whisper : ' . ($why ?: 'échec')), 'correcting' => $this->finish($ok ? self::correctedLines($out, count($this->lines)) : null), 'searching' => $ok ? $this->fetchSources(self::sources($out)) : $this->fail('Claude : ' . ($why ?: 'échec')), default => null, }; } /** * Claude Code searches the web (WebSearch, WebFetch) for WHERE the synchronised lyrics of the song are: LRCLIB entries, * .lrc files. It does not write them (they belong to their authors: it would not anyway): WinZik takes them there. */ private function askInternet(): void { $this->state = 'searching'; $who = trim($this->hint['artist'] . ' - ' . $this->hint['title'], ' -'); $length = $this->hint['duration'] > 0 ? sprintf(' (durée %d:%02d)', intdiv((int)$this->hint['duration'], 60), (int)$this->hint['duration'] % 60) : ''; $prompt = "Trouve sur Internet où sont publiées les paroles synchronisées (format LRC, avec les temps) de la chanson « $who »$length. " . "Cherche d'abord sur lrclib.net (son API publique : https://lrclib.net/api/search?q=... ; essaie plusieurs orthographes de l'artiste et du titre), " . "puis ailleurs (fichiers .lrc). Ne recopie pas les paroles : réponds UNIQUEMENT avec où les prendre, une source par ligne, la meilleure d'abord, " . "au format LRCLIB:<identifiant numérique> ou URL:<adresse directe d'un fichier .lrc>. Si tu ne trouves rien, réponds seulement : INTROUVABLE"; // only these two tools, no MCP server: the pages it reads (and the song's tags in the prompt) could ask it for more $this->job = $this->run([(string)self::claude(), '-p', $prompt, '--tools', 'WebSearch,WebFetch', '--allowedTools', 'WebSearch,WebFetch', '--strict-mcp-config']); } /** @return list<string> the sources Claude gave: "lrclib:123", "url:https://..." */ public static function sources(string $answer): array { $out = []; foreach (preg_split('/\r?\n/', $answer) ?: [] as $row) { if (preg_match('/LRCLIB\s*:\s*(\d+)/i', $row, $m)) { $out[] = 'lrclib:' . $m[1]; } elseif (preg_match('#URL\s*:\s*(https?://\S+)#i', $row, $m) && self::publicUrl($m[1])) { $out[] = 'url:' . $m[1]; } } return array_values(array_unique($out)); } /** * An address WinZik may fetch for Claude: https, on a named server of the Internet. Claude's answer comes from pages * of the web (they can tell it what to answer): never this computer or its local network (a router, a local service). */ public static function publicUrl(string $url): bool { $u = parse_url($url); if (!is_array($u) || strtolower((string)($u['scheme'] ?? '')) !== 'https' || !isset($u['host']) || isset($u['user']) || isset($u['pass'])) { return false; } $host = strtolower(trim((string)$u['host'], '[]')); if ($host === '' || $host === 'localhost' || str_ends_with($host, '.localhost') || str_ends_with($host, '.local') || !str_contains($host, '.')) { return false; } if (filter_var($host, FILTER_VALIDATE_IP) !== false) { return filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE) !== false; } // a name, not an address written another way ("0x7f.1", "2130706433": this computer for the resolver) return (bool)preg_match('/^[a-z0-9.-]+$/', $host) && !preg_match('/^(0x[0-9a-f]*|[0-9]+)(\.(0x[0-9a-f]*|[0-9]+))*\.?$/', $host); } /** The sources one after the other until one gives synchronised lyrics. @param list<string> $sources */ private function fetchSources(array $sources): void { $source = array_shift($sources); if ($source === null || $this->http === null) { $this->fail($this->http === null ? 'pas de réseau ici' : "Claude n'a pas trouvé de paroles synchronisées pour ce morceau"); return; } $this->state = 'downloading'; $url = str_starts_with($source, 'lrclib:') ? 'https://lrclib.net/api/get/' . substr($source, 7) : substr($source, 4); $this->http->request('GET', $url, null, ['User-Agent' => 'WinZik (https://jumphp.net/winzik)'], 20)->then(function (\JumPHP\Http\HttpResponse $r) use ($sources, $source): void { $lrc = $r->body; if (str_starts_with($source, 'lrclib:')) { $data = $r->ok() ? $r->json() : null; $lrc = is_array($data) ? (string)($data['syncedLyrics'] ?? '') : ''; } $lyrics = $r->ok() ? Lrclib::fromLrc($lrc) : null; if ($lyrics === null) { $this->fetchSources($sources); return; } $this->lines = $lyrics->lines; $this->corrected = true; $this->finish(null); }, fn () => $this->fetchSources($sources)); } /** "12.40|a line" (or "[00:12.40] a line") as the lines of a .LYR; [] when it found nothing. @return list<array{0: float, 1: string}> */ public static function timedLines(string $answer): array { $lines = []; foreach (preg_split('/\r?\n/', trim($answer)) ?: [] as $row) { if (preg_match('/^\s*(\d+(?:[.,]\d+)?)\s*\|\s*(.+)$/u', $row, $m)) { $lines[] = [(float)str_replace(',', '.', $m[1]), trim($m[2])]; } elseif (preg_match('/^\s*\[(\d+):(\d+(?:[.:]\d+)?)\]\s*(.+)$/u', $row, $m)) { $lines[] = [(int)$m[1] * 60 + (float)str_replace(':', '.', $m[2]), trim($m[3])]; } } usort($lines, fn (array $a, array $b) => $a[0] <=> $b[0]); return $lines; } private function heard(): void { $this->lines = self::parse((string)@file_get_contents($this->work . '/song.json')); if ($this->lines === []) { $this->fail('Whisper n\'a rien entendu de chanté'); return; } $claude = $this->useClaude ? self::claude() : null; if ($claude === null) { $this->finish(null); return; } $this->state = 'correcting'; $numbered = []; foreach ($this->lines as $i => [, $text]) { $numbered[] = ($i + 1) . '|' . $text; } $name = pathinfo($this->song, PATHINFO_FILENAME); $prompt = "Voici les paroles de la chanson « $name », transcrites automatiquement par Whisper, une ligne chantée par ligne " . "(numéro|texte). Corrige les mots mal entendus, l'orthographe et la ponctuation, sans changer le sens, sans fusionner " . "ni couper de lignes. Réponds UNIQUEMENT avec exactement " . count($numbered) . " lignes au même format numéro|texte, " . "rien d'autre.\n\n" . implode("\n", $numbered); // no tool at all: the words heard and the file's name come from the song (anyone's), they must not make it act $this->job = $this->run([$claude, '-p', $prompt, '--tools', '', '--strict-mcp-config']); } /** @param list<string>|null $texts Claude's lines (null: Whisper's are kept) */ private function finish(?array $texts): void { $lines = $this->lines; if ($texts !== null) { foreach ($texts as $i => $t) { $lines[$i][1] = $t; } $this->corrected = true; } @mkdir($this->lyrDir, 0777, true); $this->result = $this->lyricsPath !== null ? ($this->lyricsPath)($this->song) : $this->lyrDir . '/' . pathinfo($this->song, PATHINFO_FILENAME) . '.lyr'; @mkdir(dirname($this->result), 0777, true); file_put_contents($this->result, (new Lyrics($lines))->toLyr()); $this->state = 'done'; } private function fail(string $why): void { $this->state = 'error'; $this->error = $why; $this->steps = []; } /** * The sung lines of Whisper's JSON (whisper.cpp: transcription[].offsets.from in ms; openai-whisper: segments[].start in s). * @return list<array{0: float, 1: string}> */ public static function parse(string $json): array { $data = json_decode($json, true); $out = []; if (!is_array($data)) { return []; } foreach ((array)($data['transcription'] ?? []) as $seg) { $out[] = [((float)($seg['offsets']['from'] ?? 0)) / 1000, trim((string)($seg['text'] ?? ''))]; } foreach ((array)($data['segments'] ?? []) as $seg) { $out[] = [(float)($seg['start'] ?? 0), trim((string)($seg['text'] ?? ''))]; } // no empty lines, nor what Whisper writes for music without words ([Musique], ♪...) return array_values(array_filter($out, fn (array $l) => $l[1] !== '' && !preg_match('/^[\[\(♪*].*[\]\)♪*]$|^♪+$/u', $l[1]))); } /** Claude's answer: "n|text" lines; null when it is not exactly the lines asked. @return list<string>|null */ public static function correctedLines(string $answer, int $count): ?array { $texts = []; foreach (preg_split('/\r?\n/', trim($answer)) ?: [] as $row) { if (preg_match('/^\s*(\d+)\s*\|\s?(.*)$/u', $row, $m)) { $texts[(int)$m[1] - 1] = trim($m[2]); } } if (count($texts) !== $count) { return null; } ksort($texts); return array_values($texts); } /** One line for the notices. */ public function describe(): string { return match ($this->state) { 'installing' => 'Installation de Whisper (une seule fois) : ' . ($this->download !== null ? (int)round($this->download->progress * 100) . ' %' : $this->error) . '...', 'converting' => 'Whisper : préparation du son...', 'listening' => 'Whisper écoute la chanson...', 'correcting' => 'Claude Code corrige les paroles...', 'searching' => 'Claude cherche les paroles sur Internet...', 'downloading' => 'Paroles trouvées par Claude : téléchargement...', 'done' => ($this->mode === 'internet' ? 'Paroles trouvées sur Internet (Claude)' : 'Paroles créées' . ($this->corrected ? ' (Whisper + Claude)' : ' (Whisper)')) . ' : ' . basename($this->result), 'error' => 'Paroles impossibles : ' . $this->error, default => '', }; }}