EN
apps/winzik / src / Media / Transcriber.php · commit 6576d6f
<?phpnamespace App\Media;use App\Model\Lyrics;/** * Lyrics made from the song itself: Whisper listens to it and writes each sung line with its time, then Claude Code * (when it is on the computer: `claude -p`) puts the words right, line for line (the times stay Whisper's), and the * result is a .LYR of 2001 in var/lyr. Everything runs in the background, one step after the other. * * Whisper: whisper.cpp (whisper-cli) of the computer, else faster-whisper (the same model on CTranslate2, int8, every * core: a song in a few seconds; openai-whisper took half an hour for one minute on one core), installed by WinZik the * first time in its own Python (venv in its data folder, no PyTorch), else openai-whisper of the computer. Windows * without Python: whisper.cpp's release + its "base" model. ffmpeg (Tools) makes the 16 kHz sound Whisper reads. * * States: idle | installing | converting | listening | correcting | done | error */final class Transcriber{    public const CPP_ZIP = 'https://github.com/ggml-org/whisper.cpp/releases/latest/download/whisper-bin-x64.zip';    public const MODEL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin';    public string $state = 'idle';    /** whisper: Whisper (+ Claude puts its words right); internet: Claude searches the web for the real lyrics (aligned on Whisper's times when it is there) */    public string $mode = 'whisper';    /** @var array{artist: string, title: string, duration: float} what is known of the song (for the search) */    private array $hint = ['artist' => '', 'title' => '', 'duration' => 0.0];    public string $error = '';    public string $song = '';    /** the .LYR written (done) */    public string $result = '';    public bool $corrected = false;    private ?Process $job = null;    private ?FileDownload $download = null;    /** @var list<array{0: list<string>, 1: string}> the install steps left: [command, what it does] */    private array $steps = [];    /** @var list<array{0: float, 1: string}> */    private array $lines = [];    private string $work;    /** @var (\Closure(string): string)|null where the .LYR of a song goes (WinZik::lyricsCachePath()); null: var/lyr/<name>.lyr */    public ?\Closure $lyricsPath = null;    /** @param (\Closure(list<string>): Process)|null $runner */    public function __construct(private Tools $tools, private string $dataDir, private string $lyrDir, private ?\Closure $runner = null,        public bool $useClaude = true, private ?\JumPHP\Http\Http $http = null)    {        $this->work = $dataDir . '/tmp/whisper';    }    private function run(array $command): Process    {        return $this->runner !== null ? ($this->runner)($command) : Process::start($command, $this->work . '/logs');    }    public function busy(): bool    {        return !in_array($this->state, ['idle', 'done', 'error'], true);    }    /** whisper.cpp: [binary, model] when both are there. @return array{0: string, 1: string}|null */    private function cpp(): ?array    {        $bin = Tools::onPath('whisper-cli') ?? Tools::onPath('whisper-cpp');        if ($bin === null) {            foreach ((array)glob($this->dataDir . '/bin/whisper/{,*/,*/*/}' . Tools::exe('whisper-cli'), GLOB_BRACE) as $f) {                $bin = (string)$f;            }        }        $model = null;        foreach ([$this->dataDir . '/models/ggml-base.bin', (getenv('HOME') ?: '') . '/.cache/whisper.cpp/ggml-base.bin'] as $m) {            if (is_file($m)) {                $model = $m;                break;            }        }        return $bin !== null ? [$bin, $model ?? ''] : null;    }    /** openai-whisper of the computer. */    private function openai(): ?string    {        return Tools::onPath('whisper');    }    /** The Python of WinZik's faster-whisper (var/whisper), when it is installed. */    private function faster(): ?string    {        $python = $this->dataDir . (PHP_OS_FAMILY === 'Windows' ? '/whisper/Scripts/python.exe' : '/whisper/bin/python');        $mark = $this->dataDir . '/whisper/faster-whisper.ok';        return is_file($python) && is_file($mark) ? $python : null;    }    /** What WinZik's Python runs: the 16 kHz WAV read as it is (no audio library to break), segments as JSON. */    private const SCRIPT = <<<'PY'import json, os, sys, waveimport numpy as npfrom faster_whisper import WhisperModelwith wave.open(sys.argv[1], 'rb') as w:    audio = np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16).astype(np.float32) / 32768.0model = WhisperModel(sys.argv[3], device='cpu', compute_type='int8', cpu_threads=os.cpu_count() or 4)# no voice filter: sung voices over music are not "speech" for it, everything was left outsegments, info = model.transcribe(audio, vad_filter=False, beam_size=5, condition_on_previous_text=False)out = [{'start': round(s.start, 2), 'text': s.text} for s in segments]with open(sys.argv[2], 'w', encoding='utf-8') as f:    json.dump({'language': info.language, 'segments': out}, f, ensure_ascii=False)PY;    public static function claude(): ?string    {        return Tools::onPath('claude');    }    /** Whisper can listen now (nothing to install). */    public function ready(): bool    {        $cpp = $this->cpp();        return ($cpp !== null && $cpp[1] !== '') || $this->faster() !== null || $this->openai() !== null;    }    /**     * Lyrics for this song: Whisper (installed first when needed), then Claude, then the .LYR; or ($mode internet)     * Claude's search on the web, aligned on Whisper when it is installed.     * @param array{artist?: string, title?: string, duration?: float} $hint     */    public function start(string $song, string $mode = 'whisper', array $hint = []): bool    {        if ($this->busy() || !is_file($song)) {            return false;        }        $this->mode = $mode === 'internet' ? 'internet' : 'whisper';        $this->hint = ['artist' => (string)($hint['artist'] ?? ''), 'title' => (string)($hint['title'] ?? pathinfo($song, PATHINFO_FILENAME)),            'duration' => (float)($hint['duration'] ?? 0)];        $this->song = $song;        $this->error = $this->result = '';        $this->corrected = false;        $this->lines = [];        @mkdir($this->work, 0777, true);        if ($this->mode === 'internet') {            if (self::claude() === null) {                $this->fail('Claude Code n\'est pas installé sur cet ordinateur (claude)');                return false;            }            $this->askInternet();     // Claude finds where they are, WinZik takes them there            return true;        }        if ($this->tools->program('ffmpeg') === null) {            $this->fail('ffmpeg pas encore prêt');            return false;        }        if ($this->ready()) {            $this->convert();        } else {            $this->install();        }        return true;    }    private function install(): void    {        $this->state = 'installing';        $cpp = $this->cpp();        if ($cpp !== null && $cpp[1] === '') {            // whisper.cpp is there, not its model            $this->download = new FileDownload(self::MODEL, $this->dataDir . '/models/ggml-base.bin');            return;        }        $python = PHP_OS_FAMILY === 'Windows' ? (Tools::onPath('py') ?? Tools::onPath('python')) : Tools::onPath('python3');        if (PHP_OS_FAMILY === 'Windows' && $python === null) {            $this->download = new FileDownload(self::CPP_ZIP, $this->work . '/whisper-bin-x64.zip');            return;        }        if ($python === null) {            $this->fail('Whisper absent : installez python3 (WinZik y installe faster-whisper) ou whisper.cpp (whisper-cli)');            return;        }        $venv = $this->dataDir . '/whisper';        $pip = PHP_OS_FAMILY === 'Windows' ? "$venv/Scripts/pip.exe" : "$venv/bin/pip";        $this->steps = [            [[$python, '-m', 'venv', $venv], 'Python de Whisper'],            [[$pip, 'install', '--quiet', 'faster-whisper'], 'faster-whisper'],        ];        $this->nextStep();    }    private function nextStep(): void    {        $step = array_shift($this->steps);        if ($step === null && is_dir($this->dataDir . '/whisper') && !is_file($this->dataDir . '/whisper/faster-whisper.ok')) {            @touch($this->dataDir . '/whisper/faster-whisper.ok');        }        if ($step === null) {            $this->ready() ? $this->convert() : $this->fail("Whisper n'a pas pu être installé");            return;        }        $this->error = $step[1];        $this->job = $this->run($step[0]);    }    /** ffmpeg: the song as 16 kHz mono WAV, what Whisper reads. */    private function convert(): void    {        $this->state = 'converting';        $this->job = $this->run([(string)$this->tools->program('ffmpeg'), '-y', '-v', 'error', '-i', $this->song, '-vn', '-ac', '1', '-ar', '16000',            '-c:a', 'pcm_s16le', $this->work . '/song.wav']);    }    private function listen(): void    {        $this->state = 'listening';        @unlink($this->work . '/song.json');        $cpp = $this->cpp();        if ($cpp !== null && $cpp[1] !== '') {            $this->job = $this->run([$cpp[0], '-m', $cpp[1], '-f', $this->work . '/song.wav', '-l', 'auto', '-oj', '-of', $this->work . '/song',                '-t', (string)max(2, min(8, (int)(getenv('NUMBER_OF_PROCESSORS') ?: 4)))]);            return;        }        $faster = $this->faster();        if ($faster !== null) {            file_put_contents($this->dataDir . '/whisper/winzik-whisper.py', self::SCRIPT);            $this->job = $this->run([$faster, $this->dataDir . '/whisper/winzik-whisper.py', $this->work . '/song.wav', $this->work . '/song.json', 'base']);            return;        }        $this->job = $this->run([(string)$this->openai(), $this->work . '/song.wav', '--model', 'base', '--output_format', 'json',            '--output_dir', $this->work, '--verbose', 'False', '--fp16', 'False']);    }    /** Quitting: whisper / ffmpeg / the install ended now (nothing left running alone); again: nothing. */    public function stop(): void    {        if ($this->job !== null) {            $this->job->stop();            $this->job->cleanup();            $this->job = null;        }        $this->download = null;        $this->steps = [];        if ($this->busy()) {            $this->state = 'idle';        }    }    /** Each frame. */    public function poll(): void    {        if ($this->download !== null) {            if ($this->download->poll()) {                $d = $this->download;                $this->download = null;                if (!$d->ok()) {                    $this->fail('téléchargement de Whisper : ' . $d->error);                } elseif (str_ends_with($d->target, '.zip')) {                    // whisper.cpp for Windows: unpacked, then its model                    @mkdir($this->dataDir . '/bin/whisper', 0777, true);                    $this->job = $this->run(['tar', '-xf', $d->target, '-C', $this->dataDir . '/bin/whisper']);                    $this->steps = [];                } else {                    $this->ready() ? $this->convert() : $this->install();                }            }            return;        }        if ($this->job === null) {            return;        }        $this->job->poll();        if (!$this->job->done()) {            return;        }        $job = $this->job;        $this->job = null;        $ok = $job->ok();        $out = $job->output();        $why = $job->error();        $job->cleanup();        match ($this->state) {            'installing' => $ok ? ($this->cpp() !== null && $this->cpp()[1] === '' ? $this->install() : $this->nextStep())                : $this->fail('installation de Whisper : ' . ($why ?: $this->error)),            'converting' => $ok ? $this->listen() : $this->fail('ffmpeg : ' . $why),            'listening' => $ok ? $this->heard() : $this->fail('Whisper : ' . ($why ?: 'échec')),            'correcting' => $this->finish($ok ? self::correctedLines($out, count($this->lines)) : null),            'searching' => $ok ? $this->fetchSources(self::sources($out)) : $this->fail('Claude : ' . ($why ?: 'échec')),            default => null,        };    }    /**     * Claude Code searches the web (WebSearch, WebFetch) for WHERE the synchronised lyrics of the song are: LRCLIB entries,     * .lrc files. It does not write them (they belong to their authors: it would not anyway): WinZik takes them there.     */    private function askInternet(): void    {        $this->state = 'searching';        $who = trim($this->hint['artist'] . ' - ' . $this->hint['title'], ' -');        $length = $this->hint['duration'] > 0 ? sprintf(' (durée %d:%02d)', intdiv((int)$this->hint['duration'], 60), (int)$this->hint['duration'] % 60) : '';        $prompt = "Trouve sur Internet où sont publiées les paroles synchronisées (format LRC, avec les temps) de la chanson « $who »$length. "            . "Cherche d'abord sur lrclib.net (son API publique : https://lrclib.net/api/search?q=... ; essaie plusieurs orthographes de l'artiste et du titre), "            . "puis ailleurs (fichiers .lrc). Ne recopie pas les paroles : réponds UNIQUEMENT avec où les prendre, une source par ligne, la meilleure d'abord, "            . "au format LRCLIB:<identifiant numérique> ou URL:<adresse directe d'un fichier .lrc>. Si tu ne trouves rien, réponds seulement : INTROUVABLE";        // only these two tools, no MCP server: the pages it reads (and the song's tags in the prompt) could ask it for more        $this->job = $this->run([(string)self::claude(), '-p', $prompt, '--tools', 'WebSearch,WebFetch', '--allowedTools', 'WebSearch,WebFetch', '--strict-mcp-config']);    }    /** @return list<string> the sources Claude gave: "lrclib:123", "url:https://..." */    public static function sources(string $answer): array    {        $out = [];        foreach (preg_split('/\r?\n/', $answer) ?: [] as $row) {            if (preg_match('/LRCLIB\s*:\s*(\d+)/i', $row, $m)) {                $out[] = 'lrclib:' . $m[1];            } elseif (preg_match('#URL\s*:\s*(https?://\S+)#i', $row, $m) && self::publicUrl($m[1])) {                $out[] = 'url:' . $m[1];            }        }        return array_values(array_unique($out));    }    /**     * An address WinZik may fetch for Claude: https, on a named server of the Internet. Claude's answer comes from pages     * of the web (they can tell it what to answer): never this computer or its local network (a router, a local service).     */    public static function publicUrl(string $url): bool    {        $u = parse_url($url);        if (!is_array($u) || strtolower((string)($u['scheme'] ?? '')) !== 'https' || !isset($u['host']) || isset($u['user']) || isset($u['pass'])) {            return false;        }        $host = strtolower(trim((string)$u['host'], '[]'));        if ($host === '' || $host === 'localhost' || str_ends_with($host, '.localhost') || str_ends_with($host, '.local') || !str_contains($host, '.')) {            return false;        }        if (filter_var($host, FILTER_VALIDATE_IP) !== false) {            return filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE) !== false;        }        // a name, not an address written another way ("0x7f.1", "2130706433": this computer for the resolver)        return (bool)preg_match('/^[a-z0-9.-]+$/', $host) && !preg_match('/^(0x[0-9a-f]*|[0-9]+)(\.(0x[0-9a-f]*|[0-9]+))*\.?$/', $host);    }    /** The sources one after the other until one gives synchronised lyrics. @param list<string> $sources */    private function fetchSources(array $sources): void    {        $source = array_shift($sources);        if ($source === null || $this->http === null) {            $this->fail($this->http === null ? 'pas de réseau ici' : "Claude n'a pas trouvé de paroles synchronisées pour ce morceau");            return;        }        $this->state = 'downloading';        $url = str_starts_with($source, 'lrclib:') ? 'https://lrclib.net/api/get/' . substr($source, 7) : substr($source, 4);        $this->http->request('GET', $url, null, ['User-Agent' => 'WinZik (https://jumphp.net/winzik)'], 20)->then(function (\JumPHP\Http\HttpResponse $r) use ($sources, $source): void {            $lrc = $r->body;            if (str_starts_with($source, 'lrclib:')) {                $data = $r->ok() ? $r->json() : null;                $lrc = is_array($data) ? (string)($data['syncedLyrics'] ?? '') : '';            }            $lyrics = $r->ok() ? Lrclib::fromLrc($lrc) : null;            if ($lyrics === null) {                $this->fetchSources($sources);                return;            }            $this->lines = $lyrics->lines;            $this->corrected = true;            $this->finish(null);        }, fn () => $this->fetchSources($sources));    }    /** "12.40|a line" (or "[00:12.40] a line") as the lines of a .LYR; [] when it found nothing. @return list<array{0: float, 1: string}> */    public static function timedLines(string $answer): array    {        $lines = [];        foreach (preg_split('/\r?\n/', trim($answer)) ?: [] as $row) {            if (preg_match('/^\s*(\d+(?:[.,]\d+)?)\s*\|\s*(.+)$/u', $row, $m)) {                $lines[] = [(float)str_replace(',', '.', $m[1]), trim($m[2])];            } elseif (preg_match('/^\s*\[(\d+):(\d+(?:[.:]\d+)?)\]\s*(.+)$/u', $row, $m)) {                $lines[] = [(int)$m[1] * 60 + (float)str_replace(':', '.', $m[2]), trim($m[3])];            }        }        usort($lines, fn (array $a, array $b) => $a[0] <=> $b[0]);        return $lines;    }    private function heard(): void    {        $this->lines = self::parse((string)@file_get_contents($this->work . '/song.json'));        if ($this->lines === []) {            $this->fail('Whisper n\'a rien entendu de chanté');            return;        }        $claude = $this->useClaude ? self::claude() : null;        if ($claude === null) {            $this->finish(null);            return;        }        $this->state = 'correcting';        $numbered = [];        foreach ($this->lines as $i => [, $text]) {            $numbered[] = ($i + 1) . '|' . $text;        }        $name = pathinfo($this->song, PATHINFO_FILENAME);        $prompt = "Voici les paroles de la chanson « $name », transcrites automatiquement par Whisper, une ligne chantée par ligne "            . "(numéro|texte). Corrige les mots mal entendus, l'orthographe et la ponctuation, sans changer le sens, sans fusionner "            . "ni couper de lignes. Réponds UNIQUEMENT avec exactement " . count($numbered) . " lignes au même format numéro|texte, "            . "rien d'autre.\n\n" . implode("\n", $numbered);        // no tool at all: the words heard and the file's name come from the song (anyone's), they must not make it act        $this->job = $this->run([$claude, '-p', $prompt, '--tools', '', '--strict-mcp-config']);    }    /** @param list<string>|null $texts Claude's lines (null: Whisper's are kept) */    private function finish(?array $texts): void    {        $lines = $this->lines;        if ($texts !== null) {            foreach ($texts as $i => $t) {                $lines[$i][1] = $t;            }            $this->corrected = true;        }        @mkdir($this->lyrDir, 0777, true);        $this->result = $this->lyricsPath !== null ? ($this->lyricsPath)($this->song) : $this->lyrDir . '/' . pathinfo($this->song, PATHINFO_FILENAME) . '.lyr';        @mkdir(dirname($this->result), 0777, true);        file_put_contents($this->result, (new Lyrics($lines))->toLyr());        $this->state = 'done';    }    private function fail(string $why): void    {        $this->state = 'error';        $this->error = $why;        $this->steps = [];    }    /**     * The sung lines of Whisper's JSON (whisper.cpp: transcription[].offsets.from in ms; openai-whisper: segments[].start in s).     * @return list<array{0: float, 1: string}>     */    public static function parse(string $json): array    {        $data = json_decode($json, true);        $out = [];        if (!is_array($data)) {            return [];        }        foreach ((array)($data['transcription'] ?? []) as $seg) {            $out[] = [((float)($seg['offsets']['from'] ?? 0)) / 1000, trim((string)($seg['text'] ?? ''))];        }        foreach ((array)($data['segments'] ?? []) as $seg) {            $out[] = [(float)($seg['start'] ?? 0), trim((string)($seg['text'] ?? ''))];        }        // no empty lines, nor what Whisper writes for music without words ([Musique], ♪...)        return array_values(array_filter($out, fn (array $l) => $l[1] !== '' && !preg_match('/^[\[\(♪*].*[\]\)♪*]$|^♪+$/u', $l[1])));    }    /** Claude's answer: "n|text" lines; null when it is not exactly the lines asked. @return list<string>|null */    public static function correctedLines(string $answer, int $count): ?array    {        $texts = [];        foreach (preg_split('/\r?\n/', trim($answer)) ?: [] as $row) {            if (preg_match('/^\s*(\d+)\s*\|\s?(.*)$/u', $row, $m)) {                $texts[(int)$m[1] - 1] = trim($m[2]);            }        }        if (count($texts) !== $count) {            return null;        }        ksort($texts);        return array_values($texts);    }    /** One line for the notices. */    public function describe(): string    {        return match ($this->state) {            'installing' => 'Installation de Whisper (une seule fois) : ' . ($this->download !== null ? (int)round($this->download->progress * 100) . ' %' : $this->error) . '...',            'converting' => 'Whisper : préparation du son...',            'listening' => 'Whisper écoute la chanson...',            'correcting' => 'Claude Code corrige les paroles...',            'searching' => 'Claude cherche les paroles sur Internet...',            'downloading' => 'Paroles trouvées par Claude : téléchargement...',            'done' => ($this->mode === 'internet' ? 'Paroles trouvées sur Internet (Claude)' : 'Paroles créées' . ($this->corrected ? ' (Whisper + Claude)' : ' (Whisper)')) . ' : ' . basename($this->result),            'error' => 'Paroles impossibles : ' . $this->error,            default => '',        };    }}