diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.ts index 8ee305a615..885306fff3 100644 --- a/packages/cli/src/whisper/normalize.test.ts +++ b/packages/cli/src/whisper/normalize.test.ts @@ -292,6 +292,53 @@ Render video. Built for agents. expect(cues).toEqual([{ text: "你好世界", start: 0, end: 1 }]); }); + it("keeps phrase-level CJK entries as separate cues", () => { + // Chinese has no inter-word spaces, so the whitespace test cannot see + // that these are phrases; they used to collapse into one cue spanning + // the whole transcript. + const cues = wordsToCues([ + { text: "这是第一个句子", start: 0, end: 2 }, + { text: "这是第二个句子", start: 2, end: 4 }, + { text: "这是第三个句子", start: 4, end: 6 }, + ]); + expect(cues).toHaveLength(3); + expect(cues[0]).toEqual({ + text: "这是第一个句子", + start: 0, + end: 2, + }); + }); + + it("keeps phrase-level Thai entries as separate cues", () => { + const cues = wordsToCues([ + { text: "สวัสดีครับ", start: 0, end: 2 }, + { text: "ยินดีต้อนรับ", start: 2, end: 4 }, + ]); + expect(cues).toHaveLength(2); + }); + + it("treats entries at the length threshold as phrases", () => { + // Four characters is the boundary: at or above it the entries are read as + // phrase-level cues, below it as word-level tokens. + const cues = wordsToCues([ + { text: "你好世界", start: 0, end: 2 }, + { text: "谢谢大家", start: 2, end: 4 }, + ]); + expect(cues).toHaveLength(2); + }); + + it("still groups word-level CJK tokens into cues", () => { + // The mirror of the case above: short per-token entries are word-level + // whisper output and must still be joined. + const cues = wordsToCues([ + { text: "你", start: 0, end: 0.3 }, + { text: "好", start: 0.3, end: 0.6 }, + { text: "世", start: 0.6, end: 0.9 }, + { text: "界", start: 0.9, end: 1.2 }, + ]); + expect(cues).toEqual([{ text: "你好世界", start: 0, end: 1.2 }]); + }); + it("preserves single-word cue boundaries when preGrouped", () => { // Phrase-level cues without internal whitespace (one-word or CJK captions) // must not merge — auto-detection can't see them, so the caller forces it. diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.ts index 2599d8ab9b..4ad7747d30 100644 --- a/packages/cli/src/whisper/normalize.ts +++ b/packages/cli/src/whisper/normalize.ts @@ -356,6 +356,47 @@ function entriesToCues(words: Word[]): Cue[] { // inter-word spaces. const CJK_CHAR = /[ -〿぀-ヿ㐀-䶿一-鿿豈-﫿＀-￯]/; +// Scripts written without inter-word spaces: the CJK ranges above plus Thai, +// Lao, Myanmar and Khmer. Used only to decide whether entries are already +// phrase-level — the whitespace test below cannot answer that for these +// scripts. Hangul is excluded for the same reason as in CJK_CHAR. +const SPACELESS_SCRIPT_CHAR = /[฀-๿຀-໿က-႟ក-៿ -〿぀-ヿ㐀-䶿一-鿿豈-﫿＀-￯]/; + +// Whisper emits word-level tokens for spaceless scripts one or two characters +// at a time, while a phrase-level cue runs to several times that. Four is +// comfortably above the token case and below any real caption. +const SPACELESS_PHRASE_MIN_CHARS = 4; + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length === 0) return 0; + if (sorted.length % 2 === 1) return sorted[mid] ?? 0; + return ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2; +} + +/** + * Whether the entries are already grouped into phrases. + * + * Any entry containing internal whitespace is a multi-word phrase, which + * settles it for space-separated scripts. Chinese, Japanese, Thai and the + * other spaceless scripts never satisfy that test, so their phrase-level + * transcripts used to be re-grouped into a single cue covering the whole + * clip. For those, fall back to entry length instead. + * + * The median, rather than `some` or `every`, keeps one long token from + * declaring word-level input pre-grouped and a couple of short cues (a bare + * yes or no) from declaring a real transcript word-level. + */ +function inferPreGrouped(words: Word[]): boolean { + if (words.some((w) => /\s/.test(w.text.trim()))) return true; + + const spaceless = words.filter((w) => SPACELESS_SCRIPT_CHAR.test(w.text)); + if (spaceless.length === 0) return false; + + return median(spaceless.map((w) => w.text.trim().length)) >= SPACELESS_PHRASE_MIN_CHARS; +} + /** Join two adjacent tokens, omitting the space across a CJK boundary. */ function joinTokens(left: string, right: string): string { const a = left.at(-1) ?? ""; @@ -367,10 +408,10 @@ function joinTokens(left: string, right: string): string { export function wordsToCues(words: Word[], opts: WordsToCuesOptions = {}): Cue[] { // Phrase-level transcripts (imported .srt/.vtt cues) must keep their existing // cue boundaries — re-grouping would merge distinct captions and lose timing. - // The caller can force this via `preGrouped`; otherwise infer it from the data - // (any entry containing internal whitespace is a multi-word phrase, so the - // whole transcript is phrase-level rather than word-level whisper output). - const preGrouped = opts.preGrouped ?? words.some((w) => /\s/.test(w.text.trim())); + // The caller can force this via `preGrouped`; otherwise infer it from the + // data — internal whitespace for space-separated scripts, entry length for + // the scripts that have no inter-word spaces to look for. + const preGrouped = opts.preGrouped ?? inferPreGrouped(words); if (preGrouped) return entriesToCues(words); const maxChars = opts.maxChars ?? 42;