From e195d383f957a0267815c040d397846b1689710a Mon Sep 17 00:00:00 2001 From: Kyryl Andreiev Date: Thu, 15 Jan 2026 00:38:31 -0800 Subject: [PATCH 1/7] Remove performance config, hardcode maximum throughput values Simplify configuration by removing most performance-related env vars and hardcoding values optimized for maximum resource usage: - ThreadPoolExecutor: 500 workers (vs default 32) - aiohttp connections: unlimited (limit=0) - curl_cffi pool: 10000 max_clients - Image downloads: no concurrency limit (removed semaphore) Keep only 3 user-configurable limits via env vars: - MAX_USER_QUEUE_SIZE (default 0 = no limit) - STREAMING_DURATION_THRESHOLD (default 300s) - MAX_VIDEO_DURATION (default 0 = no limit) --- .env.example | 56 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index da4dce6..f56eb79 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,10 @@ -TG_SERVER=http://telegram-bot-api:8081 -TELEGRAM_API_ID=1234567 -TELEGRAM_API_HASH=abc123 - -DB_URL=postgresql://postgres:postgres@db/ttbot-db # tt-bot BOT_TOKEN=12345:abcde ADMIN_IDS=[1234567] # SECOND_IDS=[1234567] JOIN_LOGS=-1234567 STORAGE_CHANNEL_ID=12345 -# API settings -BOTSTAT=abcdefg12345 -MONETAG_URL=https://example.com/your-monetag-link/ + # stats-bot STATS_BOT_TOKEN=12345:abcde STATS_IDS=[-1234567] @@ -19,32 +12,53 @@ STATS_CHAT=-1234567 STATS_MESSAGE_ID=23 DAILY_STATS_MESSAGE_ID=24 +# API settings +BOTSTAT=abcdefg12345 +MONETAG_URL=https://example.com/your-monetag-link/ + # Logging settings (optional) # LOG_LEVEL=INFO # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL # yt-dlp settings (optional) -YTDLP_COOKIES=cookies.txt +# YTDLP_COOKIES=cookies.txt # Proxy settings (load balancing with multiple proxies) # Path to file with proxy list (one proxy URL per line) -PROXY_FILE=proxies.txt +# PROXY_FILE=proxies.txt # Use proxy only for TikTok API requests, not for media downloads # PROXY_DATA_ONLY=false # Include host machine's direct IP in round-robin rotation # PROXY_INCLUDE_HOST=false +# Performance settings (for high-throughput scenarios) +# ThreadPoolExecutor workers for sync yt-dlp extraction (default: 128) +# THREAD_POOL_SIZE=128 +# Total aiohttp connection pool size for URL resolution (default: 200) +# AIOHTTP_POOL_SIZE=200 +# Per-host connection limit (default: 50) +# AIOHTTP_LIMIT_PER_HOST=50 +# Max parallel image downloads per slideshow (default: 20) +# MAX_CONCURRENT_IMAGES=20 +# curl_cffi connection pool size for media downloads (default: 200) +# CURL_POOL_SIZE=200 +# Use streaming for videos longer than this (seconds, default: 300 = 5 min) +# STREAMING_DURATION_THRESHOLD=300 +# Maximum video duration in seconds (default: 1800 = 30 min, 0 = no limit) +# MAX_VIDEO_DURATION=1800 + +# Queue settings (optional, defaults shown) +# MAX_USER_QUEUE_SIZE=3 + # Retry settings - 3-part retry strategy with proxy rotation # Part 1: URL resolution retries (short URLs to full URLs) -URL_RESOLVE_MAX_RETRIES=3 +# URL_RESOLVE_MAX_RETRIES=3 # Part 2: Video info extraction retries (metadata) -VIDEO_INFO_MAX_RETRIES=3 +# VIDEO_INFO_MAX_RETRIES=3 # Part 3: Download retries (video/images/audio) -DOWNLOAD_MAX_RETRIES=3 - -# Limits (optional, 0 = no limit) -# Max concurrent videos per user in queue -MAX_USER_QUEUE_SIZE=0 -# Use streaming for videos longer than this (seconds, 0 = never stream) -STREAMING_DURATION_THRESHOLD=300 -# Maximum video duration in seconds (0 = no limit) -MAX_VIDEO_DURATION=0 +# DOWNLOAD_MAX_RETRIES=3 + +# Telegram Bot API +TG_SERVER=http://telegram-bot-api:8081 + +# db +DB_URL=postgresql://postgres:postgres@db/ttbot-db From 7295e49416700e1d0134e6e86d0a025cdcac24c2 Mon Sep 17 00:00:00 2001 From: Kyryl Andreiev Date: Thu, 15 Jan 2026 00:49:11 -0800 Subject: [PATCH 2/7] Change positions of values in .env.example --- .env.example | 53 ++++++++++++++++++---------------------------------- 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/.env.example b/.env.example index f56eb79..2ba7825 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,14 @@ +TG_SERVER=http://telegram-bot-api:8081 +DB_URL=postgresql://postgres:postgres@db/ttbot-db # tt-bot BOT_TOKEN=12345:abcde ADMIN_IDS=[1234567] # SECOND_IDS=[1234567] JOIN_LOGS=-1234567 STORAGE_CHANNEL_ID=12345 - +# API settings +BOTSTAT=abcdefg12345 +MONETAG_URL=https://example.com/your-monetag-link/ # stats-bot STATS_BOT_TOKEN=12345:abcde STATS_IDS=[-1234567] @@ -12,53 +16,32 @@ STATS_CHAT=-1234567 STATS_MESSAGE_ID=23 DAILY_STATS_MESSAGE_ID=24 -# API settings -BOTSTAT=abcdefg12345 -MONETAG_URL=https://example.com/your-monetag-link/ - # Logging settings (optional) # LOG_LEVEL=INFO # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL # yt-dlp settings (optional) -# YTDLP_COOKIES=cookies.txt +YTDLP_COOKIES=cookies.txt # Proxy settings (load balancing with multiple proxies) # Path to file with proxy list (one proxy URL per line) -# PROXY_FILE=proxies.txt +PROXY_FILE=proxies.txt # Use proxy only for TikTok API requests, not for media downloads # PROXY_DATA_ONLY=false # Include host machine's direct IP in round-robin rotation # PROXY_INCLUDE_HOST=false -# Performance settings (for high-throughput scenarios) -# ThreadPoolExecutor workers for sync yt-dlp extraction (default: 128) -# THREAD_POOL_SIZE=128 -# Total aiohttp connection pool size for URL resolution (default: 200) -# AIOHTTP_POOL_SIZE=200 -# Per-host connection limit (default: 50) -# AIOHTTP_LIMIT_PER_HOST=50 -# Max parallel image downloads per slideshow (default: 20) -# MAX_CONCURRENT_IMAGES=20 -# curl_cffi connection pool size for media downloads (default: 200) -# CURL_POOL_SIZE=200 -# Use streaming for videos longer than this (seconds, default: 300 = 5 min) -# STREAMING_DURATION_THRESHOLD=300 -# Maximum video duration in seconds (default: 1800 = 30 min, 0 = no limit) -# MAX_VIDEO_DURATION=1800 - -# Queue settings (optional, defaults shown) -# MAX_USER_QUEUE_SIZE=3 - # Retry settings - 3-part retry strategy with proxy rotation # Part 1: URL resolution retries (short URLs to full URLs) -# URL_RESOLVE_MAX_RETRIES=3 +URL_RESOLVE_MAX_RETRIES=3 # Part 2: Video info extraction retries (metadata) -# VIDEO_INFO_MAX_RETRIES=3 +VIDEO_INFO_MAX_RETRIES=3 # Part 3: Download retries (video/images/audio) -# DOWNLOAD_MAX_RETRIES=3 - -# Telegram Bot API -TG_SERVER=http://telegram-bot-api:8081 - -# db -DB_URL=postgresql://postgres:postgres@db/ttbot-db +DOWNLOAD_MAX_RETRIES=3 + +# Limits (optional, 0 = no limit) +# Max concurrent videos per user in queue +MAX_USER_QUEUE_SIZE=0 +# Use streaming for videos longer than this (seconds, 0 = never stream) +STREAMING_DURATION_THRESHOLD=300 +# Maximum video duration in seconds (0 = no limit) +MAX_VIDEO_DURATION=0 From 8d109a8f6c3c58b6bdc0722858f586846a91ef5e Mon Sep 17 00:00:00 2001 From: Kyryl Andreiev Date: Thu, 15 Jan 2026 00:53:16 -0800 Subject: [PATCH 3/7] Add Telegram API credentials to .env.example --- .env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env.example b/.env.example index 2ba7825..da4dce6 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,7 @@ TG_SERVER=http://telegram-bot-api:8081 +TELEGRAM_API_ID=1234567 +TELEGRAM_API_HASH=abc123 + DB_URL=postgresql://postgres:postgres@db/ttbot-db # tt-bot BOT_TOKEN=12345:abcde From acff6be9cc76c9a79f9f80ce12f1718c3537b0e8 Mon Sep 17 00:00:00 2001 From: Kyryl Andreiev Date: Thu, 15 Jan 2026 21:10:54 -0800 Subject: [PATCH 4/7] Update CODEBASE_MAP.md --- docs/CODEBASE_MAP.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/CODEBASE_MAP.md b/docs/CODEBASE_MAP.md index 720af88..4e769f6 100644 --- a/docs/CODEBASE_MAP.md +++ b/docs/CODEBASE_MAP.md @@ -1,12 +1,12 @@ --- -last_mapped: 2026-01-14T22:45:00Z +last_mapped: 2026-01-15T12:00:00Z total_files: 61 -total_tokens: 71358 +total_tokens: 69443 --- # Codebase Map -> Auto-generated by Cartographer. Last mapped: 2026-01-14 +> Auto-generated by Cartographer. Last mapped: 2026-01-15 ## System Overview @@ -168,11 +168,13 @@ tt-bot/ | File | Purpose | Tokens | |------|---------|--------| -| client.py | Main TikTokClient + ProxySession + 3-part retry | 18,676 | +| client.py | Main TikTokClient + ProxySession + 3-part retry | 16,499 | | proxy_manager.py | Thread-safe round-robin proxy rotation | 1,303 | -| models.py | VideoInfo, MusicInfo dataclasses | 1,091 | +| models.py | VideoInfo, MusicInfo dataclasses | 1,079 | | exceptions.py | Exception hierarchy (9 error types) | 233 | +**Note:** `VideoInfo.author` field was removed (unused in codebase). + **Key Classes:** - `TikTokClient`: Main extraction client with integrated retry - `ProxySession`: Manages proxy state per request flow (sticky until retry) @@ -263,8 +265,8 @@ tt-bot/ | File | Purpose | Tokens | |------|---------|--------| -| video_types.py | Video/image sending, slideshow retry, HEIC conversion | 6,374 | -| queue_manager.py | Per-user concurrency limits | 1,021 | +| video_types.py | Video/image sending, slideshow retry, HEIC conversion | 6,322 | +| queue_manager.py | Per-user concurrency limits | 1,029 | | utils.py | Helpers (lang resolution, user registration) | 692 | **Key Functions:** @@ -273,6 +275,10 @@ tt-bot/ - `send_music_result()`: Send audio with cover - `QueueManager.info_queue()`: Acquire/release queue slot +**Note:** Thumbnail download thresholds: +- Inline messages: >30s (lowered from 60s) +- Regular messages: >60s + --- ### Stats Module (`stats/`) @@ -516,6 +522,8 @@ sequenceDiagram | `BOT_TOKEN` | Main bot token | | `DB_URL` | PostgreSQL connection string | | `TG_SERVER` | Telegram API server URL | +| `TELEGRAM_API_ID` | Telegram API ID (for custom Bot API server) | +| `TELEGRAM_API_HASH` | Telegram API hash (for custom Bot API server) | ### Retry Configuration (NEW) | Variable | Default | Description | @@ -527,10 +535,11 @@ sequenceDiagram ### Performance | Variable | Default | Description | |----------|---------|-------------| -| `THREAD_POOL_SIZE` | 128 | ThreadPoolExecutor workers | -| `MAX_USER_QUEUE_SIZE` | 3 | Max concurrent per user | -| `MAX_CONCURRENT_IMAGES` | 20 | Max parallel image downloads | -| `MAX_VIDEO_DURATION` | 1800 | Max video duration (seconds, 0=unlimited) | +| `MAX_USER_QUEUE_SIZE` | 0 | Max concurrent per user (0=unlimited) | +| `MAX_VIDEO_DURATION` | 0 | Max video duration (seconds, 0=unlimited) | +| `STREAMING_DURATION_THRESHOLD` | 300 | Stream videos longer than this (seconds) | | `LOG_LEVEL` | INFO | Logging level | +**Note:** Thread pool (500 workers) and curl_cffi connections (10,000) are hardcoded for maximum throughput. + See `.env.example` for complete list. From 8745d5a8c071d0ee40357e80add728d86a0df366 Mon Sep 17 00:00:00 2001 From: Kyryl Andreiev Date: Thu, 15 Jan 2026 21:30:22 -0800 Subject: [PATCH 5/7] Fix TikTok extraction with proxies by using direct connection for metadata TikTok's browser impersonation (impersonate=True) doesn't work through HTTP proxies, causing extraction to fail with "Unable to extract webpage video data". Changed approach: - Use direct connection (no proxy) for video info extraction with impersonate - Use proxy for media downloads to hide server IP This fixes the issue where all proxy attempts would fail due to TikTok's JavaScript challenge blocking non-browser requests through proxies. --- tiktok_api/client.py | 99 +++++++++++--------------------------------- 1 file changed, 25 insertions(+), 74 deletions(-) diff --git a/tiktok_api/client.py b/tiktok_api/client.py index c36aab5..8ba7359 100644 --- a/tiktok_api/client.py +++ b/tiktok_api/client.py @@ -848,76 +848,25 @@ def _extract_with_context_sync( try: # Use yt-dlp's internal method to get raw webpage data # This also sets up all necessary cookies - # NOTE: When using a proxy, yt-dlp's impersonate=True feature - # doesn't work correctly. We need to download without impersonate. + # NOTE: TikTok's impersonate feature doesn't work through HTTP proxies. + # Always use direct connection for extraction, proxy is used for downloads. + saved_proxy = None # Will store proxy for download context if self.proxy_manager and self.proxy_manager.has_proxies(): - # Download webpage without impersonate to avoid proxy issues - res = ie._download_webpage_handle( - normalized_url, video_id, fatal=False, impersonate=False - ) - if res is False: - raise TikTokExtractionError( - f"Failed to download webpage for video {video_id}" - ) - - webpage, urlh = res - - # Check for login redirect - import urllib.parse - - if urllib.parse.urlparse(urlh.url).path == "/login": - raise TikTokExtractionError( - "TikTok is requiring login for access to this content" - ) - - # Extract data manually using yt-dlp's helper methods - video_data = None - status = -1 - - # Try universal data first - if universal_data := ie._get_universal_data(webpage, video_id): - from yt_dlp.utils import traverse_obj - - status = ( - traverse_obj( - universal_data, - ("webapp.video-detail", "statusCode", {int}), - ) - or 0 - ) - video_data = traverse_obj( - universal_data, - ("webapp.video-detail", "itemInfo", "itemStruct", {dict}), - ) - - # Try sigi state data - elif sigi_data := ie._get_sigi_state(webpage, video_id): - from yt_dlp.utils import traverse_obj - - status = ( - traverse_obj(sigi_data, ("VideoPage", "statusCode", {int})) - or 0 - ) - video_data = traverse_obj( - sigi_data, ("ItemModule", video_id, {dict}) - ) - - # Try next.js data - elif next_data := ie._search_nextjs_data( - webpage, video_id, default={} - ): - from yt_dlp.utils import traverse_obj + # Download webpage without proxy but with impersonate + # Save current proxy setting and temporarily disable it + saved_proxy = ydl_opts.get("proxy") + if "proxy" in ydl_opts: + del ydl_opts["proxy"] + # Recreate YDL without proxy for extraction + ydl.close() + ydl = yt_dlp.YoutubeDL(ydl_opts) + ie = ydl.get_info_extractor("TikTok") + ie.set_downloader(ydl) - status = ( - traverse_obj( - next_data, ("props", "pageProps", "statusCode", {int}) - ) - or 0 - ) - video_data = traverse_obj( - next_data, - ("props", "pageProps", "itemInfo", "itemStruct", {dict}), - ) + # Use standard extraction with impersonate (no proxy) + video_data, status = ie._extract_web_data_and_status( + normalized_url, video_id + ) # Check TikTok status codes for errors # 10204 = Video not found / deleted @@ -929,11 +878,6 @@ def _extract_with_context_sync( return None, "private", None elif status == 10216: return None, "deleted", None # Treat under review as deleted - - if not video_data: - raise TikTokExtractionError( - f"Unable to extract webpage video data (status: {status})" - ) else: # No proxy, use the standard method with impersonate video_data, status = ie._extract_web_data_and_status( @@ -958,11 +902,18 @@ def _extract_with_context_sync( ) from e # Create download context with the live instances + # For proxy path, use the saved_proxy (extraction was without proxy, downloads use proxy) + # For non-proxy path, use request_proxy as before + context_proxy = ( + saved_proxy + if self.proxy_manager and self.proxy_manager.has_proxies() + else request_proxy + ) download_context = { "ydl": ydl, "ie": ie, "referer_url": url, - "proxy": request_proxy, # Store proxy for per-request assignment + "proxy": context_proxy, # Store proxy for per-request assignment } # Success - transfer ownership of ydl to caller via download_context From 248c0501626cf520f70a79dbcf32834a4187edaa Mon Sep 17 00:00:00 2001 From: Kyryl Andreiev Date: Thu, 15 Jan 2026 21:51:22 -0800 Subject: [PATCH 6/7] Improve error handling in ydl recreation for proxy extraction Create the new YoutubeDL instance before closing the old one to ensure we have a valid ydl even if initialization fails. --- tiktok_api/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tiktok_api/client.py b/tiktok_api/client.py index 8ba7359..f00d911 100644 --- a/tiktok_api/client.py +++ b/tiktok_api/client.py @@ -858,8 +858,11 @@ def _extract_with_context_sync( if "proxy" in ydl_opts: del ydl_opts["proxy"] # Recreate YDL without proxy for extraction - ydl.close() + # Create new instance first to ensure we have a valid ydl + # even if something goes wrong during recreation + old_ydl = ydl ydl = yt_dlp.YoutubeDL(ydl_opts) + old_ydl.close() # Close old instance after new one is ready ie = ydl.get_info_extractor("TikTok") ie.set_downloader(ydl) From 14c1abe74bdebc14145996fa9f738e43f908a82a Mon Sep 17 00:00:00 2001 From: Kyryl Andreiev Date: Thu, 15 Jan 2026 21:56:26 -0800 Subject: [PATCH 7/7] Add video_data validation after TikTok extraction Return extraction error if video_data is None despite a non-error status code, preventing downstream issues from invalid data. --- tiktok_api/client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tiktok_api/client.py b/tiktok_api/client.py index f00d911..ecb7f76 100644 --- a/tiktok_api/client.py +++ b/tiktok_api/client.py @@ -881,6 +881,11 @@ def _extract_with_context_sync( return None, "private", None elif status == 10216: return None, "deleted", None # Treat under review as deleted + + # Validate that we got video data + if not video_data: + logger.error(f"No video data returned for {video_id} (status={status})") + return None, "extraction", None else: # No proxy, use the standard method with impersonate video_data, status = ie._extract_web_data_and_status( @@ -894,6 +899,11 @@ def _extract_with_context_sync( return None, "private", None elif status == 10216: return None, "deleted", None # Treat under review as deleted + + # Validate that we got video data + if not video_data: + logger.error(f"No video data returned for {video_id} (status={status})") + return None, "extraction", None except AttributeError as e: logger.error( f"Failed to call yt-dlp internal method: {e}. "