Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 21 additions & 35 deletions .env.example
Original file line number Diff line number Diff line change
@@ -1,64 +1,50 @@
TG_SERVER=http://telegram-bot-api:8081
TELEGRAM_API_ID=1234567
TELEGRAM_API_HASH=abc123

DB_URL=postgresql://postgres:postgres@db/ttbot-db
# tt-bot
BOT_TOKEN=12345:abcde
ADMIN_IDS=[1234567]
# SECOND_IDS=[1234567]
JOIN_LOGS=-1234567
STORAGE_CHANNEL_ID=12345

# API settings
BOTSTAT=abcdefg12345
MONETAG_URL=https://example.com/your-monetag-link/
# stats-bot
STATS_BOT_TOKEN=12345:abcde
STATS_IDS=[-1234567]
STATS_CHAT=-1234567
STATS_MESSAGE_ID=23
DAILY_STATS_MESSAGE_ID=24

# API settings
BOTSTAT=abcdefg12345
MONETAG_URL=https://example.com/your-monetag-link/

# Logging settings (optional)
# LOG_LEVEL=INFO # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL

# yt-dlp settings (optional)
# YTDLP_COOKIES=cookies.txt
YTDLP_COOKIES=cookies.txt

# Proxy settings (load balancing with multiple proxies)
# Path to file with proxy list (one proxy URL per line)
# PROXY_FILE=proxies.txt
PROXY_FILE=proxies.txt
# Use proxy only for TikTok API requests, not for media downloads
# PROXY_DATA_ONLY=false
# Include host machine's direct IP in round-robin rotation
# PROXY_INCLUDE_HOST=false

# Performance settings (for high-throughput scenarios)
# ThreadPoolExecutor workers for sync yt-dlp extraction (default: 128)
# THREAD_POOL_SIZE=128
# Total aiohttp connection pool size for URL resolution (default: 200)
# AIOHTTP_POOL_SIZE=200
# Per-host connection limit (default: 50)
# AIOHTTP_LIMIT_PER_HOST=50
# Max parallel image downloads per slideshow (default: 20)
# MAX_CONCURRENT_IMAGES=20
# curl_cffi connection pool size for media downloads (default: 200)
# CURL_POOL_SIZE=200
# Use streaming for videos longer than this (seconds, default: 300 = 5 min)
# STREAMING_DURATION_THRESHOLD=300
# Maximum video duration in seconds (default: 1800 = 30 min, 0 = no limit)
# MAX_VIDEO_DURATION=1800

# Queue settings (optional, defaults shown)
# MAX_USER_QUEUE_SIZE=3

# Retry settings - 3-part retry strategy with proxy rotation
# Part 1: URL resolution retries (short URLs to full URLs)
# URL_RESOLVE_MAX_RETRIES=3
URL_RESOLVE_MAX_RETRIES=3
# Part 2: Video info extraction retries (metadata)
# VIDEO_INFO_MAX_RETRIES=3
VIDEO_INFO_MAX_RETRIES=3
# Part 3: Download retries (video/images/audio)
# DOWNLOAD_MAX_RETRIES=3

# Telegram Bot API
TG_SERVER=http://telegram-bot-api:8081

# db
DB_URL=postgresql://postgres:postgres@db/ttbot-db
DOWNLOAD_MAX_RETRIES=3

# Limits (optional, 0 = no limit)
# Max concurrent videos per user in queue
MAX_USER_QUEUE_SIZE=0
# Use streaming for videos longer than this (seconds, 0 = never stream)
STREAMING_DURATION_THRESHOLD=300
# Maximum video duration in seconds (0 = no limit)
MAX_VIDEO_DURATION=0
22 changes: 4 additions & 18 deletions data/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,7 @@ class ProxyConfig(TypedDict):
class PerformanceConfig(TypedDict):
"""Type definition for performance configuration."""

thread_pool_size: int # ThreadPoolExecutor workers for sync yt-dlp calls
aiohttp_pool_size: int # Total aiohttp connection pool size (legacy, for redirects)
aiohttp_limit_per_host: int # Per-host connection limit
max_concurrent_images: int # Max parallel image downloads per slideshow
curl_pool_size: int # curl_cffi max_clients for media downloads
streaming_duration_threshold: (
int # Use streaming for videos longer than this (seconds)
)
streaming_duration_threshold: int # Use streaming for videos longer than this (seconds), 0 = never
max_video_duration: int # Maximum video duration in seconds (0 = no limit)


Expand Down Expand Up @@ -173,7 +166,7 @@ class Config(TypedDict):
"daily_stats_message_id": os.getenv("DAILY_STATS_MESSAGE_ID", "0"),
},
"queue": {
"max_user_queue_size": _parse_int_env("MAX_USER_QUEUE_SIZE", 3),
"max_user_queue_size": _parse_int_env("MAX_USER_QUEUE_SIZE", 0), # 0 = no limit
},
"retry": {
"url_resolve_max_retries": _parse_int_env("URL_RESOLVE_MAX_RETRIES", 3),
Expand All @@ -186,15 +179,8 @@ class Config(TypedDict):
"include_host": os.getenv("PROXY_INCLUDE_HOST", "false").lower() == "true",
},
"performance": {
"thread_pool_size": _parse_int_env("THREAD_POOL_SIZE", 128),
"aiohttp_pool_size": _parse_int_env("AIOHTTP_POOL_SIZE", 200),
"aiohttp_limit_per_host": _parse_int_env("AIOHTTP_LIMIT_PER_HOST", 50),
"max_concurrent_images": _parse_int_env("MAX_CONCURRENT_IMAGES", 20),
"curl_pool_size": _parse_int_env("CURL_POOL_SIZE", 200),
"streaming_duration_threshold": _parse_int_env(
"STREAMING_DURATION_THRESHOLD", 300
),
"max_video_duration": _parse_int_env("MAX_VIDEO_DURATION", 1800), # 30 minutes
"streaming_duration_threshold": _parse_int_env("STREAMING_DURATION_THRESHOLD", 300),
"max_video_duration": _parse_int_env("MAX_VIDEO_DURATION", 0), # 0 = no limit
},
"logging": {
"log_level": _parse_log_level("LOG_LEVEL", "INFO"),
Expand Down
4 changes: 1 addition & 3 deletions handlers/get_music.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,10 @@ async def send_tiktok_sound(callback_query: CallbackQuery):
chat_id = call_msg.chat.id
video_id = callback_query.data.lstrip("id/")
status_message = None
# Api init with proxy support and performance settings
# Api init with proxy support
api = TikTokClient(
proxy_manager=ProxyManager.get_instance(),
data_only_proxy=config["proxy"]["data_only"],
aiohttp_pool_size=config["performance"]["aiohttp_pool_size"],
aiohttp_limit_per_host=config["performance"]["aiohttp_limit_per_host"],
)
# Group chat set
group_chat = call_msg.chat.type != "private"
Expand Down
24 changes: 12 additions & 12 deletions handlers/get_video.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,10 @@ def try_again_button(lang: str):

@video_router.message(F.text)
async def send_tiktok_video(message: Message):
# Api init with proxy support and performance settings
# Api init with proxy support
api = TikTokClient(
proxy_manager=ProxyManager.get_instance(),
data_only_proxy=config["proxy"]["data_only"],
aiohttp_pool_size=config["performance"]["aiohttp_pool_size"],
aiohttp_limit_per_host=config["performance"]["aiohttp_limit_per_host"],
)
# Status message var
status_message = False
Expand Down Expand Up @@ -77,15 +75,17 @@ async def send_tiktok_video(message: Message):
await message.reply(locale[lang]["link_error"])
return

# Check per-user queue limit before proceeding
user_queue_count = queue.get_user_queue_count(message.chat.id)
if user_queue_count >= queue_config["max_user_queue_size"]:
if not group_chat:
await message.reply(
locale[lang]["error_queue_full"].format(user_queue_count),
reply_markup=try_again_button(lang),
)
return
# Check per-user queue limit before proceeding (0 = no limit)
max_queue = queue_config["max_user_queue_size"]
if max_queue > 0:
user_queue_count = queue.get_user_queue_count(message.chat.id)
if user_queue_count >= max_queue:
if not group_chat:
await message.reply(
locale[lang]["error_queue_full"].format(user_queue_count),
reply_markup=try_again_button(lang),
)
return

# Try to send initial reaction to show processing started
try:
Expand Down
9 changes: 0 additions & 9 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,6 @@
async def main() -> None:
await setup_db(config["bot"]["db_url"])

# Configure TikTokClient executor size for high throughput
# Must be called before any TikTokClient is instantiated
TikTokClient.set_executor_size(config["performance"]["thread_pool_size"])
logging.info(
f"TikTokClient configured: executor={config['performance']['thread_pool_size']} workers, "
f"aiohttp_pool={config['performance']['aiohttp_pool_size']}, "
f"limit_per_host={config['performance']['aiohttp_limit_per_host']}"
)

# Initialize proxy manager if configured
if config["proxy"]["proxy_file"]:
ProxyManager.initialize(
Expand Down
2 changes: 1 addition & 1 deletion misc/queue_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ async def acquire_info_for_user(
True if acquired successfully, False if user limit exceeded
"""
async with self._lock:
if not bypass_user_limit:
if not bypass_user_limit and self.max_user_queue > 0:
current_count = self._user_info_counts.get(user_id, 0)
if current_count >= self.max_user_queue:
logger.debug(
Expand Down
22 changes: 5 additions & 17 deletions misc/video_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,9 +284,9 @@ async def send_video_result(
if not isinstance(video_data, bytes):
raise ValueError("Video data must be bytes for inline messages")

# Download thumbnail for videos > 1 minute
# Download thumbnail for videos > 30 seconds
thumbnail = None
if video_duration and video_duration > 60:
if video_duration and video_duration > 30:
thumbnail = await download_thumbnail(video_info.cover, video_id)

# Upload to storage channel to get file_id
Expand Down Expand Up @@ -440,35 +440,23 @@ async def download_images_parallel(
image_urls: list[str],
client: TikTokClient,
video_info: VideoInfo,
max_concurrent: int | None = None,
) -> list[bytes | BaseException]:
"""
Download multiple images in parallel with concurrency limit.
Download multiple images in parallel with no concurrency limit.

Uses semaphore to prevent overwhelming TikTok CDN while maximizing throughput.
Downloads all images simultaneously for maximum throughput.
This is significantly faster than sequential downloads for slideshows.

Args:
image_urls: List of image URLs to download
client: TikTokClient instance for authenticated downloads
video_info: VideoInfo containing download context
max_concurrent: Maximum concurrent downloads. If None, uses config value.

Returns:
List of image bytes (or exceptions for failed downloads)
"""
if max_concurrent is None:
perf_config = config.get("performance")
max_concurrent = perf_config["max_concurrent_images"] if perf_config else 20

semaphore = asyncio.Semaphore(max_concurrent)

async def download_with_limit(url: str) -> bytes:
async with semaphore:
return await client.download_image(url, video_info)

return await asyncio.gather(
*[download_with_limit(url) for url in image_urls],
*[client.download_image(url, video_info) for url in image_urls],
return_exceptions=True, # Don't fail all if one fails
)

Expand Down
1 change: 0 additions & 1 deletion tiktok_api/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
>>> client = TikTokClient(proxy_manager=proxy_manager, data_only_proxy=True)
>>> try:
... video_info = await client.video("https://www.tiktok.com/@user/video/123")
... print(video_info.author)
... print(video_info.id)
... if video_info.is_video:
... print(f"Duration: {video_info.duration}s")
Expand Down
Loading