v0.1711 - cookie handling functionality & config options

FlyingFathead · FlyingFathead · commit e26eacd3b2c9 · 2025-01-25T02:18:26.000+02:00
diff --git a/README.md b/README.md
@@ -196,6 +196,9 @@ After launching your bot successfully, you can interact with it via Telegram (se
 - `/language` - set the model's transcription language (`auto` =  autodetect); if you know the language spoken in the audio, setting the transcription language manually with this command may improve both transcription speed and accuracy.
 
 ## Changes
+- v0.1711 - **🍪 Cookie handling is here!**
+   - see `config.ini` for `yt-dlp` options to set up your cookies
+   - this will make it easier to enable seamless operation with no interruptions with some services
 - v0.1710 - rewrite for chunking logic when sending as messages & re-encoding tool
    - better step-by-step logging, better error catching, better fitting into TG message limits with fallbacks
    - again; please refer to i.e. [Issue #7](https://github.com/FlyingFathead/whisper-transcriber-telegram-bot/issues/7) (and open up a new issue if necessary) if the problem persists
diff --git a/config/config.ini b/config/config.ini
@@ -68,24 +68,34 @@ cooldown_seconds = 10
 max_requests_per_minute = 5
 
 [YTDLPSettings]
-# use your own `cookies.txt` (true/false)
+# use your own `cookies.txt` file (true/false)
 # this is sometimes required for sites that require login
 # or, in some cases, with sites like YouTube that don't like downloaders.
-use_cookies = False
+# (will be skipped if the file isn't found)
+use_cookies_file = false
 cookies_file = config/cookies.txt
-
+# use cookies from your browser (to bypass yt-dlp woes)
+use_browser_cookies = true
+# your browser type
+browser_type = firefox
+# your browser cookies profile location, leave blank to use default location
+# (can be set either as a /path/to/profile.default or env var to read the location from)
+browser_cookies_profile = $TRANSCRIBERBOT_BROWSER_COOKIE_LOCATION
+# options for cache control
+# Use no cache dir (true/false); might work better in some use scenarios
+no_cache_dir = True
+# cache dir location, leave empty for default
+custom_cache_dir = 
 # some media sites don't always work well with yt-dlp's audio download feature
 # for compatibility, it's recommended to enable the flag below (true)
 download_original_video_for_domains_active = true
-
 # list your sites below to download original videos from, comma separated.
 # example:
 # download_original_video_domains = site1.com, site2.com, site3.com
 # these are the sites we use to download original videos from
 # (i.e. rumble.com has been a site that's been widely reported as having broken downloads; 
 # hence the video-only download method for that site and others alike.)
 download_original_video_domains = rumble.com
-
 # use worst video quality (true/false)
 # this is usually recommended, because we will only need the _audio_ for transcription.
 # adding a high-quality video will cause massive file size increases.
diff --git a/src/main.py b/src/main.py
@@ -3,7 +3,7 @@
 # openai-whisper transcriber-bot for Telegram
 
 # version of this program
-version_number = "0.1710"
+version_number = "0.1711"
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # https://github.com/FlyingFathead/whisper-transcriber-telegram-bot/
@@ -30,7 +30,7 @@
 # Adjust import paths based on new structure
 from transcription_handler import process_url_message, set_user_model, get_whisper_model, transcribe_audio, get_best_gpu, get_audio_duration, estimate_transcription_time, format_duration, get_whisper_language, set_user_language
 from utils.bot_token import get_bot_token
-from utils.utils import print_startup_message, safe_split_message
+from utils.utils import print_startup_message, safe_split_message, hz_line
 from config_loader import ConfigLoader  # Import ConfigLoader
 
 # Configure basic logging
@@ -53,6 +53,35 @@
 # Initialize the lock outside of your function to ensure it's shared across all invocations.
 queue_lock = asyncio.Lock()
 
+# log cookies
+def log_cookies_config():
+    """Log the status of the cookie-related settings at startup."""
+    config = ConfigLoader.get_config()
+
+    use_cookies_file = config.getboolean('YTDLPSettings', 'use_cookies_file', fallback=False)
+    cookies_file = config.get('YTDLPSettings', 'cookies_file', fallback='config/cookies.txt')
+
+    use_browser_cookies = config.getboolean('YTDLPSettings', 'use_browser_cookies', fallback=False)
+    browser_type = config.get('YTDLPSettings', 'browser_type', fallback='firefox')
+    raw_browser_cookies_profile = config.get('YTDLPSettings', 'browser_cookies_profile', fallback='')
+
+    # Expand environment variable if needed
+    expanded_profile = raw_browser_cookies_profile
+    if use_browser_cookies and raw_browser_cookies_profile.startswith('$'):
+        env_var_name = raw_browser_cookies_profile[1:]
+        env_val = os.getenv(env_var_name, '')
+        if env_val:
+            expanded_profile = env_val
+
+    # hz_line()
+    logger.info("--- yt-dlp cookie settings at startup ---")
+    logger.info(f"use_cookies_file       = {use_cookies_file}")
+    logger.info(f"cookies_file          = {cookies_file}")
+    logger.info(f"use_browser_cookies   = {use_browser_cookies}")
+    logger.info(f"browser_type          = {browser_type}")
+    logger.info(f"browser_cookies_profile = {raw_browser_cookies_profile}  [expanded -> {expanded_profile}]")
+    hz_line()
+
 class AllowedFileFilter(filters.MessageFilter):
     def __init__(self, allowed_formats):
         super().__init__()
@@ -835,6 +864,9 @@ async def ping_owners_on_startup(app: Application):
 if __name__ == '__main__':
     print_startup_message(version_number)  # Print startup message
 
+    # Log ytdlp cookie info right away
+    log_cookies_config()
+
     # Read the update settings from config
     config = ConfigLoader.get_config()
     check_for_updates = config.getboolean('UpdateSettings', 'CheckForYTDLPUpdates', fallback=False)
diff --git a/src/transcription_handler.py b/src/transcription_handler.py
@@ -178,10 +178,65 @@ def split_message(message, max_length=3500):
 async def download_audio(url, audio_path):
     config = ConfigLoader.get_config()
     ytdlp_settings = ConfigLoader.get_ytdlp_domain_settings()
-    use_cookies = config.getboolean('YTDLPSettings', 'use_cookies', fallback=False)
+
+    # --- Added verbose logging for cookie usage ---
+    use_cookies_file = config.getboolean('YTDLPSettings', 'use_cookies_file', fallback=False)
     cookies_file = config.get('YTDLPSettings', 'cookies_file', fallback='config/cookies.txt')
+    if use_cookies_file:
+        logger.info("Cookie usage is enabled (use_cookies_file=True).")
+        logger.info(f"Expected cookies file path: {cookies_file}")
+        if os.path.exists(cookies_file):
+            logger.info("Cookie file found and will be used for yt-dlp.")
+        else:
+            logger.warning(
+                "Cookie file usage is enabled, but the specified cookies file "
+                f"does not exist at: {cookies_file}. Please check config.ini."
+            )
+    else:
+        logger.info("Cookie file usage is disabled from config (use_cookies_file=False).")
+
+    # read config options
+    use_cookies_file = config.getboolean('YTDLPSettings', 'use_cookies_file', fallback=False)
+    cookies_file = config.get('YTDLPSettings', 'cookies_file', fallback='config/cookies.txt')
+
+    use_browser_cookies = config.getboolean('YTDLPSettings', 'use_browser_cookies', fallback=False)
+    browser_type = config.get('YTDLPSettings', 'browser_type', fallback='firefox')
+    browser_cookies_profile = config.get('YTDLPSettings', 'browser_cookies_profile', fallback='')
+
+    no_cache_dir = config.getboolean('YTDLPSettings', 'no_cache_dir', fallback=False)
+    custom_cache_dir = config.get('YTDLPSettings', 'custom_cache_dir', fallback='')
     use_worst_video_quality = config.getboolean('YTDLPSettings', 'use_worst_video_quality', fallback=True)
-    
+
+    # Log what was found
+    logger.info(f"use_cookies_file={use_cookies_file}, cookies_file={cookies_file}")
+    logger.info(f"use_browser_cookies={use_browser_cookies}, browser_type={browser_type}, browser_cookies_profile={browser_cookies_profile}")
+    logger.info(f"no_cache_dir={no_cache_dir}, custom_cache_dir={custom_cache_dir}, use_worst_video_quality={use_worst_video_quality}")
+
+    # -- Mutually exclusive check (optional) --
+    if use_cookies_file and use_browser_cookies:
+        # Decide which to prefer, or raise an error
+        logger.warning("Both 'use_cookies_file' and 'use_browser_cookies' are true! Defaulting to 'use_browser_cookies' and ignoring the cookies file.")
+        # Or: raise Exception("Cannot use both cookie-file and browser-cookies at once.")
+
+    # Expand environment variable if browser_cookies_profile starts with '$'
+    actual_browser_profile = browser_cookies_profile
+    if use_browser_cookies and browser_cookies_profile.startswith('$'):
+        env_var_name = browser_cookies_profile[1:]
+        env_val = os.getenv(env_var_name, '')
+        if env_val:
+            logger.info(f"Resolved browser profile from env var {env_var_name}: {env_val}")
+            actual_browser_profile = env_val
+        else:
+            logger.warning(f"Environment variable {env_var_name} not set or empty; browser cookies may fail.")
+            # Optionally raise or fallback
+
+    # Check for existence of cookies_file if we’re using a file
+    if use_cookies_file:
+        if os.path.exists(cookies_file):
+            logger.info(f"Cookies file found: {cookies_file}")
+        else:
+            logger.warning(f"Cookies file not found: {cookies_file}")
+
     parsed_url = urlparse(url)
     domain = parsed_url.netloc.lower()
     if domain.startswith('www.'):
@@ -198,8 +253,26 @@ async def download_audio(url, audio_path):
             "--dump-json",
             url
         ]
-        if use_cookies and os.path.exists(cookies_file):
-            command.extend(["--cookies", cookies_file])
+
+        # Apply cache settings based on config
+        if no_cache_dir:
+            logger.info("Disabling yt-dlp cache via config.ini (no_cache_dir=true).")
+            command.insert(1, "--no-cache-dir")  # Insert after 'yt-dlp'
+        elif custom_cache_dir:
+            logger.info(f"Using custom yt-dlp cache directory: {custom_cache_dir}")
+            command.insert(1, f"--cache-dir={custom_cache_dir}")
+
+        # if use_cookies_file and os.path.exists(cookies_file):
+        #     command.extend(["--cookies", cookies_file])
+
+        # Add cookies (only if we do NOT also prefer the browser cookie approach)
+        if use_browser_cookies and not use_cookies_file:
+            # Add --cookies-from-browser
+            command.extend(["--cookies-from-browser", f"{browser_type}:{actual_browser_profile}"])
+        elif use_cookies_file:
+            # Add --cookies <file>
+            if os.path.exists(cookies_file):
+                command.extend(["--cookies", cookies_file])
 
         process = await asyncio.create_subprocess_exec(
             *command,
@@ -257,6 +330,7 @@ def get_format_sort_key(fmt):
         if not selected_format_id:
             raise Exception("Could not determine selected format ID.")
 
+        logger.info(f"yt-dlp command: {command}")
         logger.info(f"Selected format ID: {selected_format_id}")
 
         # Step 3: Download video using the selected format
@@ -270,9 +344,26 @@ def get_format_sort_key(fmt):
             "--output", video_output_template,
             url
         ]
-        if use_cookies and os.path.exists(cookies_file):
-            command.extend(["--cookies", cookies_file])
 
+        # if use_cookies_file and os.path.exists(cookies_file):
+        #     command.extend(["--cookies", cookies_file])
+
+        # apply the cache logic
+        if no_cache_dir:
+            logger.info("Disabling yt-dlp cache via config.ini (no_cache_dir=true).")
+            command.insert(1, "--no-cache-dir")
+        elif custom_cache_dir:
+            logger.info(f"Using custom yt-dlp cache directory: {custom_cache_dir}")
+            command.insert(1, f"--cache-dir={custom_cache_dir}")
+
+        # cookies again
+        if use_browser_cookies and not use_cookies_file:
+            command.extend(["--cookies-from-browser", f"{browser_type}:{actual_browser_profile}"])
+        elif use_cookies_file:
+            if os.path.exists(cookies_file):
+                command.extend(["--cookies", cookies_file])
+
+        logger.info(f"Final yt-dlp command: {command}")
         logger.info("Downloading the selected quality video with audio...")
     else:
         # Download audio-only as mp3
@@ -283,8 +374,25 @@ def get_format_sort_key(fmt):
             "--output", audio_path,
             url
         ]
-        if use_cookies and os.path.exists(cookies_file):
+
+        # if use_cookies_file and os.path.exists(cookies_file):
+        #     command.extend(["--cookies", cookies_file])
+
+        # apply the cache logic
+        if no_cache_dir:
+            logger.info("Disabling yt-dlp cache via config.ini (no_cache_dir=true).")
+            command.insert(1, "--no-cache-dir")
+        elif custom_cache_dir:
+            logger.info(f"Using custom yt-dlp cache directory: {custom_cache_dir}")
+            command.insert(1, f"--cache-dir={custom_cache_dir}")
+
+        # cookies
+        if use_browser_cookies and not use_cookies_file:
+            command.extend(["--cookies-from-browser", f"{browser_type}:{actual_browser_profile}"])
+        elif use_cookies_file and os.path.exists(cookies_file):
             command.extend(["--cookies", cookies_file])
+
+        logger.info(f"Final yt-dlp command: {command}")
         logger.info("Downloading audio-only...")
 
     # Start the subprocess
@@ -821,9 +929,49 @@ def format_duration(duration):
 
 # Fetch details for videos
 async def fetch_video_details(url, max_retries=3, base_delay=5, command_timeout=30):
-    command = ["yt-dlp", "--user-agent",
-               "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
-               "--dump-json", url]
+    # command = ["yt-dlp", "--user-agent",
+    #            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
+    #            "--dump-json", url]
+
+    config = ConfigLoader.get_config()
+
+    # read your YTDLPSettings
+    use_cookies_file = config.getboolean('YTDLPSettings', 'use_cookies_file', fallback=False)
+    cookies_file = config.get('YTDLPSettings', 'cookies_file', fallback='config/cookies.txt')
+    use_browser_cookies = config.getboolean('YTDLPSettings', 'use_browser_cookies', fallback=False)
+    browser_type = config.get('YTDLPSettings', 'browser_type', fallback='firefox')
+    browser_cookies_profile = config.get('YTDLPSettings', 'browser_cookies_profile', fallback='')
+
+    # expand env var if needed
+    actual_browser_profile = browser_cookies_profile
+    if use_browser_cookies and browser_cookies_profile.startswith('$'):
+        env_var_name = browser_cookies_profile[1:]
+        env_val = os.getenv(env_var_name, '')
+        if env_val:
+            actual_browser_profile = env_val
+
+    command = [
+        "yt-dlp",
+        "--user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...",
+        "--dump-json",
+        url
+    ]
+
+    # if you prefer browser cookies over file
+    if use_browser_cookies and not use_cookies_file:
+        command.extend(["--cookies-from-browser", f"{browser_type}:{actual_browser_profile}"])
+    elif use_cookies_file and os.path.exists(cookies_file):
+        command.extend(["--cookies", cookies_file])
+
+    # optional: handle no_cache_dir / custom_cache_dir too if you want
+    no_cache_dir = config.getboolean('YTDLPSettings', 'no_cache_dir', fallback=False)
+    custom_cache_dir = config.get('YTDLPSettings', 'custom_cache_dir', fallback='')
+    if no_cache_dir:
+        command.insert(1, "--no-cache-dir")
+    elif custom_cache_dir:
+        command.insert(1, f"--cache-dir={custom_cache_dir}")
+
+    logger.info(f"fetch_video_details command: {command}")
 
     last_stderr_output = ""
 
