diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2a89395dc..095efa4dc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,16 +3,14 @@ name: CI on: push: branches: - - master - - dev + - main paths: - 'cyberdrop_dl/**/*.py' - 'tests/**/*.py' - '.github/workflows/ci.yaml' pull_request: branches: - - master - - dev + - main paths: - 'cyberdrop_dl/**/*.py' - 'tests/**/*.py' diff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/cli/__init__.py similarity index 72% rename from cyberdrop_dl/utils/args.py rename to cyberdrop_dl/cli/__init__.py index 556d6ad19..1eeb6f9e3 100644 --- a/cyberdrop_dl/utils/args.py +++ b/cyberdrop_dl/cli/__init__.py @@ -1,15 +1,15 @@ import dataclasses +import datetime import sys import time import warnings from argparse import SUPPRESS, ArgumentParser, BooleanOptionalAction, RawDescriptionHelpFormatter from argparse import _ArgumentGroup as ArgGroup from collections.abc import Iterable, Sequence -from datetime import date from enum import StrEnum, auto from pathlib import Path from shutil import get_terminal_size -from typing import TYPE_CHECKING, Annotated, Any, Literal, NoReturn, Self +from typing import Annotated, Any, Literal, NoReturn, Self from pydantic import BaseModel, Field, ValidationError, computed_field, field_validator, model_validator @@ -19,12 +19,6 @@ from cyberdrop_dl.models.types import HttpURL from cyberdrop_dl.utils.yaml import handle_validation_error -if TYPE_CHECKING: - from pydantic.fields import FieldInfo - - -CDL_EPILOG = "Visit the wiki for additional details: https://script-ware.gitbook.io/cyberdrop-dl" - class UIOptions(StrEnum): DISABLED = auto() @@ -37,7 +31,7 @@ class UIOptions(StrEnum): WARNING_TIMEOUT = 5 # seconds -def _check_mutually_exclusive(group: Iterable, msg: str) -> None: +def _check_mutually_exclusive(group: Iterable[Any], msg: str) -> None: if sum(1 for value in group if value) >= 2: raise ValueError(msg) @@ -81,36 +75,88 @@ def as_dict(self) -> dict[str, Any]: class CommandLineOnlyArgs(BaseModel): - links: list[HttpURL] = Field([], description="link(s) to content to download (passing multiple links is supported)") - appdata_folder: Path | None = Field(None, description="AppData folder path") - completed_after: date | None = Field( - None, description="only retry downloads that were completed on or after this date" + links: list[HttpURL] = Field( + default=[], + description="link(s) to content to download (passing multiple links is supported)", + ) + appdata_folder: Path | None = Field( + default=None, + description="AppData folder path", + ) + completed_after: datetime.date | None = Field( + default=None, + description="only retry downloads that were completed on or after this date", + ) + completed_before: datetime.date | None = Field( + default=None, + description="only retry downloads that were completed on or before this date", + ) + + config_file: Path | None = Field( + default=None, + description="path to the CDL settings.yaml file to load", ) - completed_before: date | None = Field( - None, description="only retry downloads that were completed on or before this date" + + download: bool = Field( + default=False, + description="skips UI, start download immediately", ) - config: str | None = Field(None, description="name of config to load") - config_file: Path | None = Field(None, description="path to the CDL settings.yaml file to load") - disable_cache: bool = Field(False, description="temporarily disable the requests cache") - download: bool = Field(False, description="skips UI, start download immediately") download_tiktok_audios: bool = Field( - False, description="download TikTok audios from posts and save them as separate files" + default=False, + description="download TikTok audios from posts and save them as separate files", + ) + download_tiktok_src_quality_videos: bool = Field( + default=False, + description="download TikTok videos in source quality", ) - download_tiktok_src_quality_videos: bool = Field(False, description="download TikTok videos in source quality") impersonate: Annotated[ - Literal["chrome", "edge", "safari", "safari_ios", "chrome_android", "firefox"] | bool | None, + Literal[ + "chrome", + "edge", + "safari", + "safari_ios", + "chrome_android", + "firefox", + ] + | bool + | None, CommandOptions(nargs="?", const=True), - ] = Field(None, description="Use this target as impersonation for all scrape requests") - max_items_retry: int = Field(0, description="max number of links to retry") - portrait: bool = Field(is_terminal_in_portrait(), description="force CDL to run with a vertical layout") - print_stats: bool = Field(True, description="show stats report at the end of a run") - retry_all: bool = Field(False, description="retry all downloads") - retry_failed: bool = Field(False, description="retry failed downloads") + ] = Field( + default=None, + description="Use this target as impersonation for all scrape requests", + ) + max_items_retry: int = Field( + default=0, + description="max number of links to retry", + ) + portrait: bool = Field( + default=is_terminal_in_portrait(), + description="force CDL to run with a vertical layout", + ) + print_stats: bool = Field( + default=True, + description="show stats report at the end of a run", + ) + retry_all: bool = Field( + default=False, + description="retry all downloads", + ) + retry_failed: bool = Field( + default=False, + description="retry failed downloads", + ) retry_maintenance: bool = Field( - False, description="retry download of maintenance files (bunkr). Requires files to be hashed" + default=False, + description="retry download of maintenance files (bunkr). Requires files to be hashed", + ) + show_supported_sites: bool = Field( + default=False, + description="shows a list of supported sites and exits", + ) + ui: UIOptions = Field( + default=UIOptions.FULLSCREEN, + description="DISABLED, ACTIVITY, SIMPLE or FULLSCREEN", ) - show_supported_sites: bool = Field(False, description="shows a list of supported sites and exits") - ui: UIOptions = Field(UIOptions.FULLSCREEN, description="DISABLED, ACTIVITY, SIMPLE or FULLSCREEN") @property def retry_any(self) -> bool: @@ -121,7 +167,7 @@ def fullscreen_ui(self) -> bool: return self.ui == UIOptions.FULLSCREEN @computed_field - def __computed__(self) -> dict: + def __computed__(self) -> dict[str, bool]: return {"retry_any": self.retry_any, "fullscreen_ui": self.fullscreen_ui} @model_validator(mode="after") @@ -129,9 +175,6 @@ def mutually_exclusive(self) -> Self: group1 = [self.links, self.retry_all, self.retry_failed, self.retry_maintenance] msg1 = "`--links`, '--retry-all', '--retry-maintenace' and '--retry-failed' are mutually exclusive" _check_mutually_exclusive(group1, msg1) - group2 = [self.config, self.config_file] - msg2 = "'--config' and '--config-file' are mutually exclusive" - _check_mutually_exclusive(group2, msg2) return self @field_validator("ui", mode="before") @@ -144,19 +187,17 @@ class DeprecatedArgs(BaseModel): ... class ParsedArgs(AliasModel): - cli_only_args: CommandLineOnlyArgs = CommandLineOnlyArgs() # type: ignore + cli_only_args: CommandLineOnlyArgs = CommandLineOnlyArgs() config_settings: ConfigSettings = ConfigSettings() - deprecated_args: DeprecatedArgs = DeprecatedArgs() # type: ignore + deprecated_args: DeprecatedArgs = DeprecatedArgs() global_settings: GlobalSettings = GlobalSettings() - def model_post_init(self, _) -> None: + def model_post_init(self, *_) -> None: exit_on_warning = False if self.cli_only_args.retry_all or self.cli_only_args.retry_maintenance: self.config_settings.runtime_options.ignore_history = True - warnings_to_emit = self.prepare_warnings() - if ( not self.cli_only_args.fullscreen_ui or self.cli_only_args.retry_any @@ -165,21 +206,22 @@ def model_post_init(self, _) -> None: ): self.cli_only_args.download = True - if warnings_to_emit: + if warnings_to_emit := self.prepare_warnings(): for msg in warnings_to_emit: warnings.warn(msg, DeprecationWarning, stacklevel=10) if exit_on_warning: sys.exit(1) + time.sleep(WARNING_TIMEOUT) def prepare_warnings(self) -> set[str]: - warnings_to_emit = set() + warnings_to_emit: set[str] = set() def add_warning_msg_from(field_name: str) -> None: if not field_name: return - field_info: FieldInfo = self.deprecated_args.model_fields[field_name] - warnings_to_emit.add(field_info.deprecated) + info = DeprecatedArgs.model_fields[field_name].deprecated + warnings_to_emit.add(str(info)) with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) @@ -200,22 +242,25 @@ def _add_args_from_model( full_name = prefix + name cli_name = full_name.replace("_", "-") arg_type = type(field.default) + if issubclass(arg_type, BaseModel): _add_args_from_model(parser, arg_type, cli_args=cli_args, deprecated=deprecated, prefix=f"{cli_name}.") continue + if arg_type not in (list, set, bool): arg_type = str + help_text = field.description or "" default = field.default if cli_args else SUPPRESS - default_options = {"default": default, "dest": full_name, "help": help_text} + default_options: dict[str, Any] = {"default": default, "dest": full_name, "help": help_text} for meta in field.metadata: if isinstance(meta, CommandOptions): default_options |= meta.as_dict() break name_or_flags = [f"--{cli_name}"] - alias: str = field.alias or field.validation_alias or field.serialization_alias # type: ignore - if alias and len(alias) == 1: + alias = field.alias or field.validation_alias or field.serialization_alias + if alias and len(str(alias)) == 1: name_or_flags.insert(0, f"-{alias}") if arg_type is bool: action = BooleanOptionalAction @@ -226,23 +271,28 @@ def _add_args_from_model( default_options = default_options | {"default": SUPPRESS} parser.add_argument(*name_or_flags, action=action, **default_options) continue + if cli_name == "links": _ = default_options.pop("dest") - parser.add_argument(cli_name, metavar="LINK(S)", nargs="*", action="extend", **default_options) + _ = parser.add_argument(cli_name, metavar="LINK(S)", nargs="*", action="extend", **default_options) continue + if arg_type in (list, set): - parser.add_argument(*name_or_flags, nargs="*", action="extend", **default_options) + _ = parser.add_argument(*name_or_flags, nargs="*", action="extend", **default_options) continue - parser.add_argument(*name_or_flags, type=arg_type, **default_options) + + _ = parser.add_argument(*name_or_flags, type=arg_type, **default_options) def _create_groups_from_nested_models(parser: ArgumentParser, model: type[BaseModel]) -> list[ArgGroup]: groups: list[ArgGroup] = [] for name, field in model.model_fields.items(): - submodel: type[BaseModel] = field.annotation # type: ignore + submodel = field.annotation + assert submodel and issubclass(submodel, BaseModel) submodel_group = parser.add_argument_group(name) _add_args_from_model(submodel_group, submodel) groups.append(submodel_group) + return groups @@ -266,10 +316,9 @@ def make_parser() -> tuple[ArgumentParser, dict[str, list[ArgGroup]]]: parser = ArgumentParser( description="Bulk asynchronous downloader for multiple file hosts", usage="cyberdrop-dl [OPTIONS] URL [URL...]", - epilog=CDL_EPILOG, formatter_class=CustomHelpFormatter, ) - parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}") + _ = parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}") cli_only = parser.add_argument_group("CLI-only options") _add_args_from_model(cli_only, CommandLineOnlyArgs, cli_args=True) diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py index d663a5b3f..785a5531b 100644 --- a/cyberdrop_dl/clients/scraper_client.py +++ b/cyberdrop_dl/clients/scraper_client.py @@ -121,7 +121,6 @@ async def __request_context( request_params.setdefault("max_redirects", constants.MAX_REDIRECTS) async with ( - self.client_manager.cache_control(self.client_manager._session, disabled=cache_disabled), self.client_manager._session.request(method, url, **request_params) as aio_resp, ): yield AbstractResponse.from_resp(aio_resp) diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.py index fbb36673c..c1d711864 100755 --- a/cyberdrop_dl/config/__init__.py +++ b/cyberdrop_dl/config/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import shutil from pathlib import Path from time import sleep @@ -13,8 +14,8 @@ from .global_model import GlobalSettings if TYPE_CHECKING: + from cyberdrop_dl.cli import ParsedArgs from cyberdrop_dl.utils.apprise import AppriseURL - from cyberdrop_dl.utils.args import ParsedArgs __all__ = [ "AuthSettings", @@ -36,7 +37,7 @@ def startup() -> None: global appdata, cli - from cyberdrop_dl.utils.args import parse_args + from cyberdrop_dl.cli import parse_args cli = parse_args() @@ -69,11 +70,24 @@ def mkdirs(self) -> None: dir.mkdir(parents=True, exist_ok=True) +@dataclasses.dataclass(slots=True) class Config: """Helper class to group a single config, not necessarily the current config""" + folder: Path + + apprise_file: Path + config_file: Path + + auth_config_file: Path + + auth: AuthSettings + settings: ConfigSettings + global_settings: GlobalSettings + apprise_urls: list[AppriseURL] + def __init__(self, name: str) -> None: - self.apprise_urls: list[AppriseURL] = [] + self.apprise_urls = [] self.folder = appdata.configs_dir / name self.apprise_file = self.folder / "apprise.txt" self.config_file = self.folder / "settings.yaml" @@ -82,9 +96,6 @@ def __init__(self, name: str) -> None: self.auth_config_file = auth_override else: self.auth_config_file = appdata.default_auth_config_file - self.auth: AuthSettings - self.settings: ConfigSettings - self.global_settings: GlobalSettings @staticmethod def build(name: str, auth: AuthSettings, settings: ConfigSettings, global_settings: GlobalSettings) -> Config: @@ -112,9 +123,9 @@ def _load(self) -> None: self.apprise_urls = get_apprise_urls(file=self.apprise_file) def _resolve_all_paths(self) -> None: - self.auth.resolve_paths(self.folder.name) - self.settings.resolve_paths(self.folder.name) - self.global_settings.resolve_paths(self.folder.name) + self.auth.resolve_paths() + self.settings.resolve_paths() + self.global_settings.resolve_paths() def _all_settings(self) -> tuple[ConfigSettings, AuthSettings, GlobalSettings]: return self.settings, self.auth, self.global_settings diff --git a/cyberdrop_dl/config/_common.py b/cyberdrop_dl/config/_common.py index b5cf96a7e..997bb9b7e 100755 --- a/cyberdrop_dl/config/_common.py +++ b/cyberdrop_dl/config/_common.py @@ -1,19 +1,14 @@ from pathlib import Path -from typing import Any, Self +from typing import Self -from pydantic import Field as P_Field -from pydantic.fields import _Unset +from pydantic import BaseModel from cyberdrop_dl.exceptions import InvalidYamlError -from cyberdrop_dl.models import PathAliasModel, get_model_fields +from cyberdrop_dl.models import AliasModel, get_model_fields from cyberdrop_dl.utils import yaml -def Field(default: Any, validation_alias: str = _Unset, **kwargs) -> Any: # noqa: N802 - return P_Field(default=default, validation_alias=validation_alias, **kwargs) - - -class ConfigModel(PathAliasModel): +class ConfigModel(AliasModel): @classmethod def load_file(cls, file: Path, update_if_has_string: str) -> Self: default = cls() @@ -28,12 +23,25 @@ def load_file(cls, file: Path, update_if_has_string: str) -> Self: needs_update = all_fields != set_fields or _is_in_file(update_if_has_string, file) if needs_update: - yaml.save(file, config) + config.save_to_file(file) + return config def save_to_file(self, file: Path) -> None: yaml.save(file, self) + def resolve_paths(self) -> None: + self._resolve_paths(self) + + @classmethod + def _resolve_paths(cls, model: BaseModel) -> None: + for name, value in vars(model).items(): + if isinstance(value, Path): + setattr(model, name, value.resolve()) + + elif isinstance(value, BaseModel): + cls._resolve_paths(value) + def _is_in_file(search_value: str, file: Path) -> bool: try: diff --git a/cyberdrop_dl/config/auth_model.py b/cyberdrop_dl/config/auth_model.py index cbc5faf1f..af9cb8d78 100755 --- a/cyberdrop_dl/config/auth_model.py +++ b/cyberdrop_dl/config/auth_model.py @@ -1,16 +1,15 @@ from pydantic import BaseModel +from cyberdrop_dl.config._common import ConfigModel from cyberdrop_dl.models import AliasModel -from ._common import ConfigModel, Field - class CoomerAuth(BaseModel): session: str = "" class ImgurAuth(AliasModel): - client_id: str = Field("", "imgur_client_id") + client_id: str = "" class MegaNzAuth(AliasModel): @@ -19,9 +18,9 @@ class MegaNzAuth(AliasModel): class JDownloaderAuth(AliasModel): - username: str = Field("", "jdownloader_username") - password: str = Field("", "jdownloader_password") - device: str = Field("", "jdownloader_device") + username: str = "" + password: str = "" + device: str = "" class KemonoAuth(AliasModel): @@ -29,23 +28,23 @@ class KemonoAuth(AliasModel): class GoFileAuth(AliasModel): - api_key: str = Field("", "gofile_api_key") + api_key: str = "" class PixeldrainAuth(AliasModel): - api_key: str = Field("", "pixeldrain_api_key") + api_key: str = "" class RealDebridAuth(AliasModel): - api_key: str = Field("", "realdebrid_api_key") + api_key: str = "" class AuthSettings(ConfigModel): - coomer: CoomerAuth = Field(CoomerAuth(), "Coomer") - gofile: GoFileAuth = Field(GoFileAuth(), "GoFile") - imgur: ImgurAuth = Field(ImgurAuth(), "Imgur") - jdownloader: JDownloaderAuth = Field(JDownloaderAuth(), "JDownloader") - kemono: KemonoAuth = Field(KemonoAuth(), "Kemono") - meganz: MegaNzAuth = Field(MegaNzAuth(), "MegaNz") - pixeldrain: PixeldrainAuth = Field(PixeldrainAuth(), "PixelDrain") - realdebrid: RealDebridAuth = Field(RealDebridAuth(), "RealDebrid") + coomer: CoomerAuth = CoomerAuth() + gofile: GoFileAuth = GoFileAuth() + imgur: ImgurAuth = ImgurAuth() + jdownloader: JDownloaderAuth = JDownloaderAuth() + kemono: KemonoAuth = KemonoAuth() + meganz: MegaNzAuth = MegaNzAuth() + pixeldrain: PixeldrainAuth = PixeldrainAuth() + realdebrid: RealDebridAuth = RealDebridAuth() diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.py index 38b81b89c..c8e58fdac 100755 --- a/cyberdrop_dl/config/config_model.py +++ b/cyberdrop_dl/config/config_model.py @@ -4,11 +4,11 @@ from logging import DEBUG from pathlib import Path -from pydantic import BaseModel, ByteSize, NonNegativeInt, PositiveInt, field_serializer, field_validator +from pydantic import BaseModel, ByteSize, Field, NonNegativeInt, field_serializer, field_validator from cyberdrop_dl import constants from cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE, Hashing -from cyberdrop_dl.models import HttpAppriseURL +from cyberdrop_dl.models import AliasModel, HttpAppriseURL from cyberdrop_dl.models.types import ( ByteSizeSerilized, ListNonEmptyStr, @@ -24,7 +24,7 @@ from cyberdrop_dl.utils.strings import validate_format_string from cyberdrop_dl.utils.utilities import purge_dir_tree -from ._common import ConfigModel, Field, PathAliasModel +from ._common import ConfigModel ALL_SUPPORTED_SITES = ["<>"] _SORTING_COMMON_FIELDS = { @@ -63,28 +63,23 @@ def valid_format(cls, value: str) -> str: return value -class Files(PathAliasModel): - download_folder: Path = Field(DEFAULT_DOWNLOAD_STORAGE, "d") - dump_json: bool = Field(False, "j") - input_file: Path = Field(DEFAULT_APP_STORAGE / "Configs{config}/URLs.txt", "i") +class Files(AliasModel): + download_folder: Path = Field(default=DEFAULT_DOWNLOAD_STORAGE, validation_alias="d") + dump_json: bool = Field(default=False, validation_alias="j") + input_file: Path = Field(default=DEFAULT_APP_STORAGE / "Configs{config}/URLs.txt", validation_alias="i") save_pages_html: bool = False -class Logs(PathAliasModel): - download_error_urls: LogPath = Field(Path("Download_Error_URLs.csv"), "download_error_urls_filename") - last_forum_post: LogPath = Field(Path("Last_Scraped_Forum_Posts.csv"), "last_forum_post_filename") +class Logs(AliasModel): + download_error_urls: LogPath = Path("Download_Error_URLs.csv") + last_forum_post: LogPath = Path("Last_Scraped_Forum_Posts.csv") log_folder: Path = DEFAULT_APP_STORAGE / "Configs/{config}/Logs" - log_line_width: PositiveInt = Field(240, ge=50) logs_expire_after: timedelta | None = None - main_log: MainLogPath = Field(Path("downloader.log"), "main_log_filename") + main_log: MainLogPath = Path("downloader.log") rotate_logs: bool = False - scrape_error_urls: LogPath = Field(Path("Scrape_Error_URLs.csv"), "scrape_error_urls_filename") - unsupported_urls: LogPath = Field(Path("Unsupported_URLs.csv"), "unsupported_urls_filename") - webhook: HttpAppriseURL | None = Field(None, "webhook_url") - - @property - def cdl_responses_dir(self) -> Path: - return self.main_log.parent / "cdl_responses" + scrape_error_urls: LogPath = Path("Scrape_Error_URLs.csv") + unsupported_urls: LogPath = Path("Unsupported_URLs.csv") + webhook: HttpAppriseURL | None = None @field_validator("webhook", mode="before") @classmethod @@ -107,7 +102,7 @@ def _set_output_filenames(self, now: datetime) -> None: if self.rotate_logs: new_name = f"{log_file.stem}_{current_time_file_iso}{log_file.suffix}" - log_file: Path = log_file.parent / current_time_folder_iso / new_name + log_file = log_file.parent / current_time_folder_iso / new_name setattr(self, attr, self.log_folder / log_file) log_file.parent.mkdir(exist_ok=True, parents=True) @@ -171,7 +166,7 @@ def is_valid_regex(cls, value: str | None) -> str | None: if not value: return None try: - re.compile(value) + _ = re.compile(value) except re.error as e: raise ValueError("input is not a valid regex") from e return value @@ -287,13 +282,13 @@ class DupeCleanup(BaseModel): class ConfigSettings(ConfigModel): - browser_cookies: BrowserCookies = Field(BrowserCookies(), "Browser_Cookies") - download_options: DownloadOptions = Field(DownloadOptions(), "Download_Options") - dupe_cleanup_options: DupeCleanup = Field(DupeCleanup(), "Dupe_Cleanup_Options") - file_size_limits: FileSizeLimits = Field(FileSizeLimits(), "File_Size_Limits") - media_duration_limits: MediaDurationLimits = Field(MediaDurationLimits(), "Media_Duration_Limits") - files: Files = Field(Files(), "Files") - ignore_options: IgnoreOptions = Field(IgnoreOptions(), "Ignore_Options") - logs: Logs = Field(Logs(), "Logs") - runtime_options: RuntimeOptions = Field(RuntimeOptions(), "Runtime_Options") - sorting: Sorting = Field(Sorting(), "Sorting") + browser_cookies: BrowserCookies = BrowserCookies() + download_options: DownloadOptions = DownloadOptions() + dupe_cleanup_options: DupeCleanup = DupeCleanup() + file_size_limits: FileSizeLimits = FileSizeLimits() + media_duration_limits: MediaDurationLimits = MediaDurationLimits() + files: Files = Files() + ignore_options: IgnoreOptions = IgnoreOptions() + logs: Logs = Logs() + runtime_options: RuntimeOptions = RuntimeOptions() + sorting: Sorting = Sorting() diff --git a/cyberdrop_dl/config/global_model.py b/cyberdrop_dl/config/global_model.py index 4fc30a619..a69fd25b0 100755 --- a/cyberdrop_dl/config/global_model.py +++ b/cyberdrop_dl/config/global_model.py @@ -1,5 +1,4 @@ import random -from datetime import timedelta from typing import Literal import aiohttp @@ -14,16 +13,15 @@ ) from yarl import URL -from cyberdrop_dl.config._common import ConfigModel, Field +from cyberdrop_dl.config._common import ConfigModel from cyberdrop_dl.models.types import ByteSizeSerilized, HttpURL, ListNonEmptyStr, ListPydanticURL, NonEmptyStr -from cyberdrop_dl.models.validators import falsy_as, falsy_as_none, to_bytesize, to_timedelta +from cyberdrop_dl.models.validators import falsy_as, falsy_as_none, to_bytesize MIN_REQUIRED_FREE_SPACE = to_bytesize("512MB") DEFAULT_REQUIRED_FREE_SPACE = to_bytesize("5GB") class General(BaseModel): - # TODO: Move `ssl_context` to an advance config section ssl_context: Literal["truststore", "certifi", "truststore+certifi"] | None = "truststore+certifi" disable_crawlers: ListNonEmptyStr = [] flaresolverr: HttpURL | None = None @@ -64,8 +62,6 @@ class RateLimiting(BaseModel): download_attempts: PositiveInt = 2 download_delay: NonNegativeFloat = 0.0 download_speed_limit: ByteSizeSerilized = ByteSize(0) - file_host_cache_expire_after: timedelta = timedelta(days=7) - forum_cache_expire_after: timedelta = timedelta(weeks=4) jitter: NonNegativeFloat = 0 max_simultaneous_downloads_per_domain: PositiveInt = 5 max_simultaneous_downloads: PositiveInt = 15 @@ -83,17 +79,12 @@ def model_post_init(self, *_) -> None: self._curl_timeout = self.connection_timeout if self.read_timeout is not None: self._curl_timeout = self.connection_timeout, self.read_timeout - self._aiohttp_timeout = aiohttp.ClientTimeout( + self._aiohttp_timeout: aiohttp.ClientTimeout = aiohttp.ClientTimeout( total=None, sock_connect=self.connection_timeout, sock_read=self.read_timeout, ) - @field_validator("file_host_cache_expire_after", "forum_cache_expire_after", mode="before") - @staticmethod - def parse_cache_duration(input_date: timedelta | str | int) -> timedelta | str: - return to_timedelta(input_date) - @property def total_delay(self) -> NonNegativeFloat: """download_delay + jitter""" @@ -105,10 +96,7 @@ def get_jitter(self) -> NonNegativeFloat: class UIOptions(BaseModel): - downloading_item_limit: PositiveInt = 10 refresh_rate: PositiveInt = 10 - scraping_item_limit: PositiveInt = 5 - vi_mode: bool = False class GenericCrawlerInstances(BaseModel): @@ -119,7 +107,7 @@ class GenericCrawlerInstances(BaseModel): class GlobalSettings(ConfigModel): - general: General = Field(General(), "General") - rate_limiting_options: RateLimiting = Field(RateLimiting(), "Rate_Limiting_Options") - ui_options: UIOptions = Field(UIOptions(), "UI_Options") + general: General = General() + rate_limiting_options: RateLimiting = RateLimiting() + ui_options: UIOptions = UIOptions() generic_crawlers_instances: GenericCrawlerInstances = GenericCrawlerInstances() diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.py index 04b75a74f..49c8159d9 100644 --- a/cyberdrop_dl/constants.py +++ b/cyberdrop_dl/constants.py @@ -199,4 +199,3 @@ class NotificationResult(Enum): MEDIA_EXTENSIONS = FILE_FORMATS["Audio"] | FILE_FORMATS["Videos"] | FILE_FORMATS["Images"] -DISABLE_CACHE = None diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py index cb6de7a91..b07a7b136 100644 --- a/cyberdrop_dl/director.py +++ b/cyberdrop_dl/director.py @@ -158,7 +158,7 @@ def _setup_debug_logger(manager: Manager) -> Path | None: file_io = debug_log_file_path.open("w", encoding="utf8") - file_handler = LogHandler(level=log_level, file=file_io, width=settings_data.logs.log_line_width, debug=True) + file_handler = LogHandler(level=log_level, file=file_io, width=500, debug=True) queued_logger = QueuedLogger(manager, file_handler, "debug") debug_logger.addHandler(queued_logger.handler) @@ -181,7 +181,7 @@ def _setup_main_logger(manager: Manager) -> None: constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL) logger.addHandler(constants.console_handler) - file_handler = LogHandler(level=log_level, file=file_io, width=settings_data.logs.log_line_width) + file_handler = LogHandler(level=log_level, file=file_io, width=500) queued_logger = QueuedLogger(manager, file_handler) logger.addHandler(queued_logger.handler) diff --git a/cyberdrop_dl/managers/cache_manager.py b/cyberdrop_dl/managers/cache_manager.py index dd9fb3fec..104fcab37 100644 --- a/cyberdrop_dl/managers/cache_manager.py +++ b/cyberdrop_dl/managers/cache_manager.py @@ -33,17 +33,7 @@ def load(self) -> None: self._cache = yaml.load(self.cache_file) def load_request_cache(self) -> None: - from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITES - - rate_limiting_options = self.manager.config_manager.global_settings_data.rate_limiting_options - urls_expire_after = { - "*.simpcity.su": rate_limiting_options.file_host_cache_expire_after, - } - for host in SUPPORTED_WEBSITES.values(): - match_host = f"*.{host}" if "." in host else f"*.{host}.*" - urls_expire_after[match_host] = rate_limiting_options.file_host_cache_expire_after - for forum in SUPPORTED_FORUMS.values(): - urls_expire_after[forum] = rate_limiting_options.forum_cache_expire_after + return def get(self, key: str) -> Any: """Returns the value of a key in the cache.""" diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py index 5a0401d49..7bf752250 100644 --- a/cyberdrop_dl/managers/client_manager.py +++ b/cyberdrop_dl/managers/client_manager.py @@ -176,10 +176,6 @@ def get_download_slots(self, domain: str) -> int: return min(instances, self.rate_limiting_options.max_simultaneous_downloads_per_domain) - @staticmethod - def cache_control(session: ClientSession, disabled: bool = False): - return _null_context - @staticmethod def check_curl_cffi_is_available() -> None: if _curl_import_error is None: diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py index fe6a17537..23981a74a 100644 --- a/cyberdrop_dl/managers/config_manager.py +++ b/cyberdrop_dl/managers/config_manager.py @@ -39,7 +39,7 @@ def __init__(self, manager: Manager) -> None: def startup(self) -> None: """Startup process for the config manager.""" - self.loaded_config = self.manager.parsed_args.cli_only_args.config or self.get_loaded_config() + self.loaded_config = self.get_loaded_config() self.settings = self.manager.path_manager.config_folder / self.loaded_config / "settings.yaml" self.global_settings = self.manager.path_manager.config_folder / "global_settings.yaml" self.authentication_settings = self.manager.path_manager.config_folder / "authentication.yaml" diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.py index 1a5493dea..bab9bc58e 100644 --- a/cyberdrop_dl/managers/live_manager.py +++ b/cyberdrop_dl/managers/live_manager.py @@ -8,7 +8,7 @@ from rich.live import Live from cyberdrop_dl import constants -from cyberdrop_dl.utils.args import is_terminal_in_portrait +from cyberdrop_dl.cli import is_terminal_in_portrait if TYPE_CHECKING: from collections.abc import Generator diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py index fb19b5777..ca87a508c 100644 --- a/cyberdrop_dl/managers/manager.py +++ b/cyberdrop_dl/managers/manager.py @@ -9,6 +9,7 @@ from pydantic import BaseModel from cyberdrop_dl import __version__, constants +from cyberdrop_dl.cli import ParsedArgs, parse_args from cyberdrop_dl.database import Database from cyberdrop_dl.database.transfer import transfer_v5_db_to_v6 from cyberdrop_dl.managers.cache_manager import CacheManager @@ -21,7 +22,6 @@ from cyberdrop_dl.managers.progress_manager import ProgressManager from cyberdrop_dl.managers.storage_manager import StorageManager from cyberdrop_dl.utils import ffmpeg -from cyberdrop_dl.utils.args import ParsedArgs, parse_args from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information @@ -62,7 +62,6 @@ def __init__(self, args: Sequence[str] | None = None) -> None: self.task_group: TaskGroup = field(init=False) self.scrape_mapper: ScrapeMapper = field(init=False) - self.vi_mode: bool = False self.start_time: float = perf_counter() self.downloaded_data: int = 0 self.loggers: dict[str, QueuedLogger] = {} @@ -96,31 +95,9 @@ def startup(self) -> None: self.config_manager.startup() self.args_consolidation() - self.cache_manager.load_request_cache() - self.vi_mode = self.config_manager.global_settings_data.ui_options.vi_mode self.path_manager.startup() self.log_manager = LogManager(self) - self.adjust_for_simpcity() - self.set_constants() - - def adjust_for_simpcity(self) -> None: - """Adjusts settings for SimpCity update.""" - simp_settings_adjusted = self.cache_manager.get("simp_settings_adjusted") - if not simp_settings_adjusted: - for config in self.config_manager.get_configs(): - if config != self.config_manager.loaded_config: - self.config_manager.change_config(config) - self.config_manager.settings_data.runtime_options.update_last_forum_post = True - self.config_manager.write_updated_settings_config() - - rate_limit_options = self.config_manager.global_settings_data.rate_limiting_options - if rate_limit_options.download_attempts >= 10: - rate_limit_options.download_attempts = 5 - if rate_limit_options.max_simultaneous_downloads_per_domain > 15: - rate_limit_options.max_simultaneous_downloads_per_domain = 5 - self.config_manager.write_updated_global_settings_config() - self.cache_manager.save("simp_settings_adjusted", True) """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" @@ -183,10 +160,9 @@ def args_consolidation(self) -> None: def args_logging(self) -> None: """Logs the runtime arguments.""" - auth_data: dict[str, dict] = self.config_manager.authentication_data.model_dump() auth_provided = {} - for site, auth_entries in auth_data.items(): + for site, auth_entries in self.config_manager.authentication_data.model_dump().items(): auth_provided[site] = all(auth_entries.values()) config_settings = self.config_manager.settings_data.model_copy() @@ -234,12 +210,6 @@ async def close(self) -> None: _, queued_logger = self.loggers.popitem() queued_logger.stop() - def set_constants(self) -> None: - """ - rewrite constants after config/arg manager have loaded - """ - constants.DISABLE_CACHE = self.parsed_args.cli_only_args.disable_cache - def add_or_remove_lists(cli_values: list[str], config_values: list[str]) -> None: exclude = {"+", "-"} diff --git a/cyberdrop_dl/models/__init__.py b/cyberdrop_dl/models/__init__.py index 825a75079..f34b64ca4 100755 --- a/cyberdrop_dl/models/__init__.py +++ b/cyberdrop_dl/models/__init__.py @@ -1,15 +1,15 @@ from pydantic import BaseModel -from .base_models import AliasModel, AppriseURLModel, FrozenModel, HttpAppriseURL, PathAliasModel +from .base_models import AliasModel, AppriseURLModel, FrozenModel, HttpAppriseURL def get_model_fields(model: BaseModel, *, exclude_unset: bool = True) -> set[str]: fields = set() - default_dict: dict = model.model_dump(exclude_unset=exclude_unset) - for submodel_name, submodel in default_dict.items(): + for submodel_name, submodel in model.model_dump(exclude_unset=exclude_unset).items(): for field_name in submodel: fields.add(f"{submodel_name}.{field_name}") + return fields -__all__ = ["AliasModel", "AppriseURLModel", "FrozenModel", "HttpAppriseURL", "PathAliasModel", "get_model_fields"] +__all__ = ["AliasModel", "AppriseURLModel", "FrozenModel", "HttpAppriseURL", "get_model_fields"] diff --git a/cyberdrop_dl/models/base_models.py b/cyberdrop_dl/models/base_models.py index f067f8d65..af03d43a3 100755 --- a/cyberdrop_dl/models/base_models.py +++ b/cyberdrop_dl/models/base_models.py @@ -1,7 +1,6 @@ """Pydantic models""" from collections.abc import Iterator, Mapping, Sequence -from pathlib import Path from typing import TypeVar import yarl @@ -52,19 +51,6 @@ class HttpAppriseURL(AppriseURLModel): url: Secret[HttpURL] -class PathAliasModel(AliasModel): - @staticmethod - def replace_config_and_resolve(path: Path, current_config: str) -> Path: - return Path(str(path).replace("{config}", current_config)).resolve() - - def resolve_paths(self, config_name: str) -> None: - for name, value in vars(self).items(): - if isinstance(value, Path): - setattr(self, name, self.replace_config_and_resolve(value, config_name)) - elif isinstance(value, PathAliasModel): - value.resolve_paths(config_name) - - class SequenceModel(RootModel[list[_ModelT]], Sequence[_ModelT]): def __len__(self) -> int: return len(self.root) diff --git a/cyberdrop_dl/ui/progress/file_progress.py b/cyberdrop_dl/ui/progress/file_progress.py index 8bca91d39..d2b17ed3e 100644 --- a/cyberdrop_dl/ui/progress/file_progress.py +++ b/cyberdrop_dl/ui/progress/file_progress.py @@ -25,7 +25,7 @@ class FileProgress(DequeProgress): def __init__(self, manager: Manager) -> None: self.manager = manager progress_colums = (SpinnerColumn(), "[progress.description]{task.description}", BarColumn(bar_width=None)) - visible_tasks_limit: int = manager.config_manager.global_settings_data.ui_options.downloading_item_limit + visible_tasks_limit: int = 10 horizontal_columns = ( *progress_colums, "[progress.percentage]{task.percentage:>6.2f}%", diff --git a/cyberdrop_dl/ui/progress/scraping_progress.py b/cyberdrop_dl/ui/progress/scraping_progress.py index 3b8037dee..9139c65e6 100644 --- a/cyberdrop_dl/ui/progress/scraping_progress.py +++ b/cyberdrop_dl/ui/progress/scraping_progress.py @@ -20,7 +20,7 @@ class ScrapingProgress(DequeProgress): def __init__(self, manager: Manager) -> None: self.manager = manager self._progress = Progress(SpinnerColumn(), "[progress.description]{task.description}") - visible_tasks_limit: int = manager.config_manager.global_settings_data.ui_options.scraping_item_limit + visible_tasks_limit: int = 5 super().__init__("Scraping", visible_tasks_limit) def get_queue_length(self) -> int: diff --git a/scripts/tools/update_docs.py b/scripts/tools/update_docs.py index fc3bcbfc2..6de56331e 100644 --- a/scripts/tools/update_docs.py +++ b/scripts/tools/update_docs.py @@ -1,7 +1,7 @@ from pathlib import Path from cyberdrop_dl import __version__ -from cyberdrop_dl.utils.args import CDL_EPILOG, CustomHelpFormatter, make_parser +from cyberdrop_dl.cli import CDL_EPILOG, CustomHelpFormatter, make_parser from cyberdrop_dl.utils.markdown import get_crawlers_info_as_markdown_table REPO_ROOT = Path(__file__).parents[2] diff --git a/tests/test_cli.py b/tests/test_cli.py index 82dd498ab..a6b190981 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,8 +4,8 @@ import pytest from pydantic import ValidationError +from cyberdrop_dl.cli import parse_args from cyberdrop_dl.main import _create_director, run -from cyberdrop_dl.utils.args import parse_args @pytest.mark.parametrize( @@ -75,7 +75,7 @@ def test_startup_logger_is_created_on_yaml_error(tmp_cwd: Path) -> None: def test_startup_logger_when_manager_startup_fails( tmp_cwd: Path, exception: Exception | type[Exception], exists: bool, capsys: pytest.CaptureFixture[str] ) -> None: - with mock.patch("cyberdrop_dl.managers.manager.Manager.set_constants", side_effect=exception): + with mock.patch("cyberdrop_dl.managers.manager.Manager.args_consolidation", side_effect=exception): try: run("--download") except SystemExit: diff --git a/uv.lock b/uv.lock index ab1fc854d..c1c1de2ba 100644 --- a/uv.lock +++ b/uv.lock @@ -205,6 +205,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/6b/cfa80a13437896eb8f4504ddac6dfa4ef7f1d2b2261057aa4a30003b8de6/apprise-1.9.7-py3-none-any.whl", hash = "sha256:c7640a81a1097685de66e0508e3da89f49235d566cb44bbead1dd98419bf5ee3", size = 1459879, upload-time = "2026-01-20T18:51:30.766Z" }, ] +[[package]] +name = "async-mega-py" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiolimiter" }, + { name = "pycryptodome" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/04/d7f94b5fa80ccb822ad7191110c956a926a1333f252ce1451804461bda34/async_mega_py-2.0.2.tar.gz", hash = "sha256:4fe83c594788681782c804916d1e337c18ddfb005826a8789d34cda3467b6c8e", size = 39566, upload-time = "2026-02-11T20:54:27.125Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/b9/9efdb8ed5cc2519c06dad23a05e8de3ac8e07a05688da25abaf5ceba076f/async_mega_py-2.0.2-py3-none-any.whl", hash = "sha256:9c863978e67e79569c2fc0407a793d6ef754af0de6e245b96d1f71d317d115b9", size = 48848, upload-time = "2026-02-11T20:54:24.647Z" }, +] + [[package]] name = "attrs" version = "25.4.0" @@ -704,7 +718,7 @@ wheels = [ [[package]] name = "cyberdrop-dl-patched" -version = "9.0.0" +version = "9.0.0.dev0" source = { editable = "." } dependencies = [ { name = "aiodns" }, @@ -712,6 +726,7 @@ dependencies = [ { name = "aiohttp", extra = ["speedups"] }, { name = "aiolimiter" }, { name = "aiosqlite" }, + { name = "async-mega-py" }, { name = "beautifulsoup4" }, { name = "certifi" }, { name = "curl-cffi", marker = "implementation_name == 'cpython'" }, @@ -756,6 +771,7 @@ requires-dist = [ { name = "aiohttp", extras = ["speedups"], specifier = ">=3.13.3" }, { name = "aiolimiter", specifier = ">=1.2.1" }, { name = "aiosqlite", specifier = ">=0.22.1" }, + { name = "async-mega-py", specifier = ">=2.0.2" }, { name = "beautifulsoup4", specifier = ">=4.14.3" }, { name = "certifi", specifier = ">=2026.1.4" }, { name = "curl-cffi", marker = "implementation_name == 'cpython'", specifier = ">=0.13,<0.14" },