Sharing some code using fast whisper + spacy + PunctuationModel. need some refinement.

[search620]

import logging
import spacy
import srt
from datetime import timedelta
from faster_whisper import WhisperModel
from deepmultilingualpunctuation import PunctuationModel

# Set up logging
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.CRITICAL)

# Control area
control = {
    'transcribe_audio': True,
    'add_punctuation': True,
    'split_sentences': True,
    'create_srt': True,
    'use_max_duration': False,
}

parameters = {
    'whisper_model_size': "large-v2",
    'device': "cuda",
    'compute_type': "float16",
    'file_path': r "file_path_here",
    'beam_size': 5,
    'word_timestamps': True,
    'max_duration': 4,
    'gap_threshold': 0.5  
}

transcribed_words = []
timestamps = []

if control['transcribe_audio']:
    logging.info("1. Transcribing audio to text using WhisperModel...")
    whisper_model = WhisperModel(parameters['whisper_model_size'], device=parameters['device'], compute_type=parameters['compute_type'])
    segments, info = whisper_model.transcribe(parameters['file_path'], beam_size=parameters['beam_size'], word_timestamps=parameters['word_timestamps'])

    logging.info("Detected language '%s' with probability %f" % (info.language, info.language_probability))

    for segment in segments:
        logging.info("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
        if segment.words is not None:
            segment_words = [word.word for word in segment.words]
            transcribed_words.extend(segment_words)
            timestamps.extend([(word.start, word.end) for word in segment.words])

    logging.info("1. Transcription completed.")

if control['add_punctuation']:
    logging.info("2. Adding punctuation to the transcribed text using PunctuationModel...")
    punctuation_model = PunctuationModel()
    punctuated_text = punctuation_model.restore_punctuation(' '.join(transcribed_words))
    logging.info("2. Punctuation added.")

sentence_timestamps = []

if control['split_sentences']:
    logging.info("3. Splitting text into sentences and adding timestamps...")
    nlp = spacy.load('en_core_web_lg')
    doc = nlp(punctuated_text)

    word_counter = 0
    for sent in doc.sents:
        sentence_words = sent.text.split()
        words_in_sentence = len(sentence_words)
        start_time = timestamps[word_counter][0]
        end_time = timestamps[word_counter + words_in_sentence - 1][1]

        # New sentences and timestamps storage
        new_sentences = []
        new_timestamps = []

        partial_sentence = []
        partial_start_time = start_time

        for word_index in range(words_in_sentence):
            partial_end_time = timestamps[word_counter + word_index][1]
            # Check for a large gap
            if word_index < words_in_sentence - 1 and timestamps[word_counter + word_index + 1][0] - partial_end_time > parameters['gap_threshold']:
                new_sentences.append(' '.join(partial_sentence))
                new_timestamps.append((partial_start_time, partial_end_time))
                partial_start_time = timestamps[word_counter + word_index + 1][0]
                partial_sentence = [sentence_words[word_index + 1]]
            else:
                partial_sentence.append(sentence_words[word_index])

        if partial_sentence:  # Handle the last segment
            new_sentences.append(' '.join(partial_sentence))
            new_timestamps.append((partial_start_time, partial_end_time))

        # Update the original sentences and timestamps with the new ones
        for new_sentence, new_timestamp in zip(new_sentences, new_timestamps):
            sentence_timestamps.append((new_sentence, *new_timestamp))

        word_counter += words_in_sentence

    logging.info("3. Sentence splitting and timestamping completed.")

if control['create_srt']:
    logging.info("4. Creating and saving SRT file...")
    srt_content = []
    for i, (sentence, start_time, end_time) in enumerate(sentence_timestamps):
        srt_content.append(srt.Subtitle(i, timedelta(seconds=start_time), timedelta(seconds=end_time), sentence))
    srt_str = srt.compose(srt_content)

    logging.info("The content of the SRT file will be: ")
    logging.info(srt_str)

    if parameters['file_path'].endswith('.mp3'):
        output_file_path = parameters['file_path'].replace(".mp3", ".srt")
    elif parameters['file_path'].endswith('.wav'):
        output_file_path = parameters['file_path'].replace(".wav", ".srt")
    elif parameters['file_path'].endswith('.mkv'):  # Added condition to handle .mkv files
        output_file_path = parameters['file_path'].replace(".mkv", ".srt")
    elif parameters['file_path'].endswith('.mp4'):  # Added condition to handle .mp4 files
        output_file_path = parameters['file_path'].replace(".mp4", ".srt")

    with open(output_file_path, "w") as f:
        f.write(srt_str)

    logging.info(f"4. SRT file saved to {output_file_path}.")

[etlweather]

Hi there. A description of what you are trying to acheive, how adding Spacy and PunctuationModel helped or not, etc. would be nice to make this an actual discussion.