From d005c5063d94437a5ba2f50db5bf168f73155413 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Thu, 29 Jan 2026 12:31:07 +0000 Subject: [PATCH 1/3] Support secure API/worker connections from CLI --- clients/python/coflux/__main__.py | 212 +++++++++++++++++++++++++++--- clients/python/coflux/config.py | 1 + clients/python/coflux/version.py | 5 +- clients/python/coflux/worker.py | 7 +- 4 files changed, 200 insertions(+), 25 deletions(-) diff --git a/clients/python/coflux/__main__.py b/clients/python/coflux/__main__.py index 16185743..be2fef82 100644 --- a/clients/python/coflux/__main__.py +++ b/clients/python/coflux/__main__.py @@ -23,6 +23,38 @@ T = t.TypeVar("T") +def _is_localhost(host: str) -> bool: + """ + Check if a host is localhost-like. + + Handles localhost, *.localhost, 127.0.0.1, [::1], all with optional port. + """ + # Handle IPv6 addresses in brackets (e.g., [::1]:7777) + if host.startswith("["): + bracket_end = host.find("]") + if bracket_end != -1: + return host[1:bracket_end] == "::1" + return False + + # Remove port suffix for regular hosts + hostname = host.rsplit(":", 1)[0] if ":" in host else host + + # Check for localhost, subdomains of localhost, or IPv4 loopback + return hostname == "localhost" or hostname.endswith(".localhost") or hostname == "127.0.0.1" + + +def _should_use_secure(host: str, secure: bool | None) -> bool: + """ + Determine whether to use secure connections (HTTPS/WSS). + + If secure is explicitly set, use that value. + Otherwise, infer from hostname: insecure for localhost, secure otherwise. + """ + if secure is not None: + return secure + return not _is_localhost(host) + + def _callback(_changes: set[tuple[watchfiles.Change, str]]) -> None: print("Change detected. Reloading...") @@ -36,16 +68,17 @@ def _get_default_image() -> str: def _api_request( - method: str, host: str, action: str, token: str | None, **kwargs + method: str, host: str, action: str, token: str | None, *, secure: bool, **kwargs ) -> t.Any: headers = kwargs.pop("headers", {}) if API_VERSION: headers["X-API-Version"] = API_VERSION if token: headers["Authorization"] = f"Bearer {token}" + scheme = "https" if secure else "http" with httpx.Client() as client: response = client.request( - method, f"http://{host}/api/{action}", headers=headers, **kwargs + method, f"{scheme}://{host}/api/{action}", headers=headers, **kwargs ) if response.status_code == 409: data = response.json() @@ -65,13 +98,15 @@ def _create_session( provides: dict[str, list[str]] | None = None, concurrency: int | None = None, token: str | None = None, + *, + secure: bool, ) -> str: payload: dict[str, t.Any] = {"projectId": project_id, "spaceName": space_name} if provides: payload["provides"] = provides if concurrency: payload["concurrency"] = concurrency - result = _api_request("POST", host, "create_session", token, json=payload) + result = _api_request("POST", host, "create_session", token, secure=secure, json=payload) return result["sessionId"] @@ -127,6 +162,8 @@ def _register_manifests( host: str, targets: dict[str, dict[str, tuple[models.Target, t.Callable]]], token: str | None = None, + *, + secure: bool, ) -> None: manifests = { module: { @@ -179,6 +216,7 @@ def _register_manifests( host, "register_manifests", token, + secure=secure, json={ "projectId": project_id, "spaceName": space_name, @@ -188,7 +226,7 @@ def _register_manifests( def _get_pool( - host: str, project_id: str, space_name: str, pool_name: str, token: str | None + host: str, project_id: str, space_name: str, pool_name: str, token: str | None, *, secure: bool ) -> dict | None: try: return _api_request( @@ -196,6 +234,7 @@ def _get_pool( host, "get_pool", token, + secure=secure, params={ "project": project_id, "space": space_name, @@ -237,6 +276,7 @@ def _init( space: str, host: str, token: str | None, + secure: bool, provides: dict[str, list[str]], serialiser_configs: list[config.SerialiserConfig], blob_threshold: int, @@ -248,7 +288,7 @@ def _init( try: targets = _load_modules(list(modules)) if register: - _register_manifests(project, space, host, targets, token=token) + _register_manifests(project, space, host, targets, token=token, secure=secure) # Track whether we created the session (vs it being provided externally) session_provided = session_id is not None @@ -263,7 +303,7 @@ def _init( time.sleep(delay) print("Creating session...") session_id = _create_session( - host, project, space, provides, concurrency, token=token + host, project, space, provides, concurrency, token=token, secure=secure ) print("Session created.") @@ -272,6 +312,7 @@ def _init( project, space, host, + secure, serialiser_configs, blob_threshold, blob_store_configs, @@ -364,6 +405,35 @@ def _load_config() -> config.Config: return config.Config.model_validate(_read_config(path).unwrap()) +def _parse_bool_env(name: str) -> bool | None: + """ + Parse a boolean environment variable. + + Returns True for '1', 'true', 'yes', 'on' (case-insensitive). + Returns False for '0', 'false', 'no', 'off' (case-insensitive). + Returns None if not set or empty. + Raises click.BadParameter for invalid values. + """ + value = os.environ.get(name, "").strip().lower() + if not value: + return None + if value in {"1", "true", "yes", "on"}: + return True + if value in {"0", "false", "no", "off"}: + return False + raise click.BadParameter( + f"Invalid value for {name}: {value!r}. Use: 1/0, true/false, yes/no, on/off" + ) + + +def _get_default_secure() -> bool | None: + """Get the default secure value from env or config.""" + env_value = _parse_bool_env("COFLUX_SECURE") + if env_value is not None: + return env_value + return _load_config().server.secure + + @cli.command("configure") @click.option( "-p", @@ -446,15 +516,23 @@ def spaces(): envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) def spaces_list( project: str, host: str, token: str | None, + secure: bool | None, ): """ Lists spaces. """ - spaces = _api_request("GET", host, "get_spaces", token, params={"project": project}) + use_secure = _should_use_secure(host, secure) + spaces = _api_request("GET", host, "get_spaces", token, secure=use_secure, params={"project": project}) if spaces: # TODO: draw as tree _print_table( @@ -494,6 +572,12 @@ def spaces_list( envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.option( "--base", help="The base space to inherit from", @@ -503,15 +587,17 @@ def spaces_create( project: str, host: str, token: str | None, + secure: bool | None, base: str | None, name: str, ): """ Creates a space within the project. """ + use_secure = _should_use_secure(host, secure) base_id = None if base: - spaces = _api_request("GET", host, "get_spaces", token, params={"project": project}) + spaces = _api_request("GET", host, "get_spaces", token, secure=use_secure, params={"project": project}) space_ids_by_name = {w["name"]: id for id, w in spaces.items()} base_id = space_ids_by_name.get(base) if not base_id: @@ -523,6 +609,7 @@ def spaces_create( host, "create_space", token, + secure=use_secure, json={ "projectId": project, "name": name, @@ -566,6 +653,12 @@ def spaces_create( envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.option( "--name", help="The new name of the space", @@ -584,6 +677,7 @@ def spaces_update( space: str, host: str, token: str | None, + secure: bool | None, name: str | None, base: str | None, no_base: bool, @@ -591,7 +685,8 @@ def spaces_update( """ Updates a space within the project. """ - spaces = _api_request("GET", host, "get_spaces", token, params={"project": project}) + use_secure = _should_use_secure(host, secure) + spaces = _api_request("GET", host, "get_spaces", token, secure=use_secure, params={"project": project}) space_ids_by_name = {w["name"]: id for id, w in spaces.items()} space_id = space_ids_by_name.get(space) if not space_id: @@ -616,7 +711,7 @@ def spaces_update( payload["baseId"] = None # TODO: handle response - _api_request("POST", host, "update_space", token, json=payload) + _api_request("POST", host, "update_space", token, secure=use_secure, json=payload) click.secho(f"Updated space '{name or space}'.", fg="green") @@ -655,16 +750,24 @@ def spaces_update( envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) def spaces_archive( project: str, space: str, host: str, token: str | None, + secure: bool | None, ): """ Archives a space. """ - spaces = _api_request("GET", host, "get_spaces", token, params={"project": project}) + use_secure = _should_use_secure(host, secure) + spaces = _api_request("GET", host, "get_spaces", token, secure=use_secure, params={"project": project}) space_ids_by_name = {w["name"]: id for id, w in spaces.items()} space_id = space_ids_by_name.get(space) if not space_id: @@ -675,6 +778,7 @@ def spaces_archive( host, "archive_space", token, + secure=use_secure, json={ "projectId": project, "spaceId": space_id, @@ -725,15 +829,23 @@ def pools(): envvar="COFLUX_TOKEN", default=_load_config().server.token, ) -def pools_list(project: str, space: str, host: str, token: str | None): +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) +def pools_list(project: str, space: str, host: str, token: str | None, secure: bool | None): """ Lists pools. """ + use_secure = _should_use_secure(host, secure) pools = _api_request( "GET", host, "get_pools", token, + secure=use_secure, json={"projectId": project, "spaceName": space}, ) if pools: @@ -785,6 +897,12 @@ def pools_list(project: str, space: str, host: str, token: str | None): envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.option( "modules", "-m", @@ -811,6 +929,7 @@ def pools_update( space: str, host: str, token: str | None, + secure: bool | None, modules: tuple[str, ...] | None, provides: tuple[str, ...] | None, docker_image: str | None, @@ -820,7 +939,8 @@ def pools_update( """ Updates a pool. """ - pool = _get_pool(host, project, space, name, token) or {} + use_secure = _should_use_secure(host, secure) + pool = _get_pool(host, project, space, name, token, secure=use_secure) or {} # TODO: support explicitly unsetting 'provides' (and modules, etc?) @@ -841,6 +961,7 @@ def pools_update( host, "update_pool", token, + secure=use_secure, json={ "projectId": project, "spaceName": space, @@ -884,16 +1005,24 @@ def pools_update( envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.argument("name") -def pools_delete(project: str, space: str, host: str, token: str | None, name: str): +def pools_delete(project: str, space: str, host: str, token: str | None, secure: bool | None, name: str): """ Deletes a pool. """ + use_secure = _should_use_secure(host, secure) _api_request( "POST", host, "update_pool", token, + secure=use_secure, json={"projectId": project, "spaceName": space, "poolName": name, "pool": None}, ) @@ -941,13 +1070,14 @@ def assets(): pass -def _get_asset(host: str, project_id: str, asset_id: str, token: str | None) -> dict | None: +def _get_asset(host: str, project_id: str, asset_id: str, token: str | None, *, secure: bool) -> dict | None: try: return _api_request( "GET", host, "get_asset", token, + secure=secure, params={ "project": project_id, "asset": asset_id, @@ -996,17 +1126,23 @@ def _human_size(bytes: int) -> str: envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.option( "--match", help="Glob-style matcher to filter files", ) @click.argument("id") -def assets_inspect(project: str, host: str, token: str | None, match: str | None, id: str): +def assets_inspect(project: str, host: str, token: str | None, secure: bool | None, match: str | None, id: str): """ Inspect an asset. """ - - asset = _get_asset(host, project, id, token) + use_secure = _should_use_secure(host, secure) + asset = _get_asset(host, project, id, token, secure=use_secure) if not asset: raise click.ClickException(f"Asset '{id}' not found in project") @@ -1058,6 +1194,12 @@ def assets_inspect(project: str, host: str, token: str | None, match: str | None envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.option( "--to", type=click.Path(file_okay=False, path_type=Path, resolve_path=True), @@ -1078,6 +1220,7 @@ def assets_download( project: str, host: str, token: str | None, + secure: bool | None, to: Path, force: bool, match: str | None, @@ -1086,8 +1229,8 @@ def assets_download( """ Downloads the contents of an asset. """ - - asset = _get_asset(host, project, id, token) + use_secure = _should_use_secure(host, secure) + asset = _get_asset(host, project, id, token, secure=use_secure) if not asset: raise click.ClickException(f"Asset '{id}' not found in project") @@ -1159,12 +1302,19 @@ def assets_download( envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.argument("module_name", nargs=-1) def register( project: str, space: str, host: str, token: str | None, + secure: bool | None, module_name: tuple[str, ...], ) -> None: """ @@ -1176,8 +1326,9 @@ def register( """ if not module_name: raise click.ClickException("No module(s) specified.") + use_secure = _should_use_secure(host, secure) targets = _load_modules(list(module_name)) - _register_manifests(project, space, host, targets, token=token) + _register_manifests(project, space, host, targets, token=token, secure=use_secure) click.secho("Manifest(s) registered.", fg="green") @@ -1215,6 +1366,12 @@ def register( envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.option( "--provides", help="Features that this worker provides (to be matched with features that tasks require)", @@ -1259,6 +1416,7 @@ def worker( space: str, host: str, token: str | None, + secure: bool | None, provides: tuple[str, ...] | None, session: str | None, concurrency: int, @@ -1278,12 +1436,14 @@ def worker( raise click.ClickException("No module(s) specified.") provides_ = _parse_provides(provides) config = _load_config() + use_secure = _should_use_secure(host, secure) args = (*module_name,) kwargs = { "project": project, "space": space, "host": host, "token": token, + "secure": use_secure, "provides": provides_, "serialiser_configs": config and config.serialisers, "blob_threshold": config and config.blobs and config.blobs.threshold, @@ -1340,6 +1500,12 @@ def worker( envvar="COFLUX_TOKEN", default=_load_config().server.token, ) +@click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", +) @click.argument("module") @click.argument("target") @click.argument("argument", nargs=-1) @@ -1348,6 +1514,7 @@ def submit( space: str, host: str, token: str | None, + secure: bool | None, module: str, target: str, argument: tuple[str, ...], @@ -1355,12 +1522,14 @@ def submit( """ Submit a workflow to be run. """ + use_secure = _should_use_secure(host, secure) # TODO: support overriding options? workflow = _api_request( "GET", host, "get_workflow", token, + secure=use_secure, params={ "project": project, "space": space, @@ -1377,6 +1546,7 @@ def submit( host, "submit_workflow", token, + secure=use_secure, json={ "projectId": project, "spaceName": space, diff --git a/clients/python/coflux/config.py b/clients/python/coflux/config.py index d8f27fe3..4d2605ab 100644 --- a/clients/python/coflux/config.py +++ b/clients/python/coflux/config.py @@ -7,6 +7,7 @@ class ServerConfig(pydantic.BaseModel): host: str = "localhost:7777" token: str | None = None + secure: bool | None = None class HTTPBlobStoreConfig(pydantic.BaseModel): diff --git a/clients/python/coflux/version.py b/clients/python/coflux/version.py index ea4a3d0b..57714d4b 100644 --- a/clients/python/coflux/version.py +++ b/clients/python/coflux/version.py @@ -35,14 +35,15 @@ def __init__(self, host: str, message: str): super().__init__(f"Cannot connect to server at {host}: {message}") -def check_server(host: str) -> dict: +def check_server(host: str, secure: bool) -> dict: """ Check server connectivity and version compatibility. Returns the server metadata if successful. Raises ServerConnectionError if server is unreachable. Raises VersionMismatchError if versions are incompatible. """ - url = f"http://{host}/.well-known/com.coflux" + scheme = "https" if secure else "http" + url = f"{scheme}://{host}/.well-known/com.coflux" try: with httpx.Client(timeout=10) as client: response = client.get(url) diff --git a/clients/python/coflux/worker.py b/clients/python/coflux/worker.py index 3b4914ab..99b8d68b 100644 --- a/clients/python/coflux/worker.py +++ b/clients/python/coflux/worker.py @@ -47,6 +47,7 @@ def __init__( project_id: str, space_name: str, server_host: str, + secure: bool, serialiser_configs: list[config.SerialiserConfig], blob_threshold: int, blob_store_configs: list[config.BlobStoreConfig], @@ -56,6 +57,7 @@ def __init__( self._project_id = project_id self._space_name = space_name self._server_host = server_host + self._secure = secure self._session_id = session_id self._targets = targets self._connection = server.Connection( @@ -108,10 +110,11 @@ def _subprotocols(self) -> list[str]: async def run(self) -> None: """Run the worker. Raises SessionExpiredError if session expires.""" - check_server(self._server_host) + check_server(self._server_host, self._secure) while True: print(f"Connecting ({self._server_host}, {self._project_id}, {self._space_name})...") - url = self._url("ws", "worker", self._params()) + scheme = "wss" if self._secure else "ws" + url = self._url(scheme, "worker", self._params()) try: async with websockets.connect(url, subprotocols=self._subprotocols()) as websocket: print("Connected.") From 253240cb7cf72baab5f98f4e81a0c0e30cfc7d3e Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Thu, 29 Jan 2026 12:59:16 +0000 Subject: [PATCH 2/3] Reduce duplication of CLI option definitions --- clients/python/coflux/__main__.py | 535 ++++++------------------------ 1 file changed, 93 insertions(+), 442 deletions(-) diff --git a/clients/python/coflux/__main__.py b/clients/python/coflux/__main__.py index be2fef82..f6c21e1f 100644 --- a/clients/python/coflux/__main__.py +++ b/clients/python/coflux/__main__.py @@ -434,6 +434,73 @@ def _get_default_secure() -> bool | None: return _load_config().server.secure +def _server_options(token: bool = True): + """Add server options: host, secure, and optionally token.""" + def decorator(f): + decorators = [ + click.option( + "-h", + "--host", + help="Host to connect to", + envvar="COFLUX_HOST", + default=_load_config().server.host, + show_default=True, + required=True, + ), + click.option( + "--secure/--no-secure", + "secure", + default=_get_default_secure(), + help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", + ), + ] + if token: + decorators.append( + click.option( + "--token", + help="Authentication token", + envvar="COFLUX_TOKEN", + default=_load_config().server.token, + ) + ) + for d in reversed(decorators): + f = d(f) + return f + return decorator + + +def _project_options(space: bool = False): + """Add project option, and optionally space.""" + def decorator(f): + decorators = [ + click.option( + "-p", + "--project", + help="Project ID", + envvar="COFLUX_PROJECT", + default=_load_config().project, + show_default=True, + required=True, + ), + ] + if space: + decorators.append( + click.option( + "-s", + "--space", + help="Space name", + envvar="COFLUX_SPACE", + default=_load_config().space, + show_default=True, + required=True, + ) + ) + for d in reversed(decorators): + f = d(f) + return f + return decorator + + @cli.command("configure") @click.option( "-p", @@ -492,36 +559,8 @@ def spaces(): @spaces.command("list") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options() +@_server_options() def spaces_list( project: str, host: str, @@ -548,36 +587,8 @@ def spaces_list( @spaces.command("create") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options() +@_server_options() @click.option( "--base", help="The base space to inherit from", @@ -620,45 +631,8 @@ def spaces_create( @spaces.command("update") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-s", - "--space", - help="The (current) name of the space", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() @click.option( "--name", help="The new name of the space", @@ -717,45 +691,8 @@ def spaces_update( @spaces.command("archive") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-s", - "--space", - help="Space name", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() def spaces_archive( project: str, space: str, @@ -796,45 +733,8 @@ def pools(): @pools.command("list") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-s", - "--space", - help="Space name", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() def pools_list(project: str, space: str, host: str, token: str | None, secure: bool | None): """ Lists pools. @@ -864,45 +764,8 @@ def pools_list(project: str, space: str, host: str, token: str | None, secure: b @pools.command("update") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-s", - "--space", - help="Space name", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() @click.option( "modules", "-m", @@ -972,45 +835,8 @@ def pools_update( @pools.command("delete") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-s", - "--space", - help="Space name", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() @click.argument("name") def pools_delete(project: str, space: str, host: str, token: str | None, secure: bool | None, name: str): """ @@ -1036,17 +862,9 @@ def blobs(): @blobs.command("get") -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) +@_server_options(token=False) @click.argument("key") -def blobs_get(host: str, key: str): +def blobs_get(host: str, secure: bool | None, key: str): """ Gets a blob by key and writes the content to stdout. """ @@ -1102,36 +920,8 @@ def _human_size(bytes: int) -> str: @assets.command("inspect") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options() +@_server_options() @click.option( "--match", help="Glob-style matcher to filter files", @@ -1170,36 +960,8 @@ def assets_inspect(project: str, host: str, token: str | None, secure: bool | No @assets.command("download") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options() +@_server_options() @click.option( "--to", type=click.Path(file_okay=False, path_type=Path, resolve_path=True), @@ -1269,45 +1031,8 @@ def assets_download( @cli.command("register") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-w", - "--space", - help="Space name", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() @click.argument("module_name", nargs=-1) def register( project: str, @@ -1333,45 +1058,8 @@ def register( @cli.command("worker") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-s", - "--space", - help="Space name", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() @click.option( "--provides", help="Features that this worker provides (to be matched with features that tasks require)", @@ -1467,45 +1155,8 @@ def worker( @cli.command("submit") -@click.option( - "-p", - "--project", - help="Project ID", - envvar="COFLUX_PROJECT", - default=_load_config().project, - show_default=True, - required=True, -) -@click.option( - "-s", - "--space", - help="Space name", - envvar="COFLUX_SPACE", - default=_load_config().space, - show_default=True, - required=True, -) -@click.option( - "-h", - "--host", - help="Host to connect to", - envvar="COFLUX_HOST", - default=_load_config().server.host, - show_default=True, - required=True, -) -@click.option( - "--token", - help="Authentication token", - envvar="COFLUX_TOKEN", - default=_load_config().server.token, -) -@click.option( - "--secure/--no-secure", - "secure", - default=_get_default_secure(), - help="Use secure connections (HTTPS/WSS). Inferred from host if not specified.", -) +@_project_options(space=True) +@_server_options() @click.argument("module") @click.argument("target") @click.argument("argument", nargs=-1) From 2666c8831ef5adb072f5b1784aab2f1a6e2f4717 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Thu, 29 Jan 2026 12:59:44 +0000 Subject: [PATCH 3/3] Support secure connection for blob store --- clients/python/coflux/__main__.py | 5 +++-- clients/python/coflux/blobs.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/clients/python/coflux/__main__.py b/clients/python/coflux/__main__.py index f6c21e1f..1746b15d 100644 --- a/clients/python/coflux/__main__.py +++ b/clients/python/coflux/__main__.py @@ -868,12 +868,13 @@ def blobs_get(host: str, secure: bool | None, key: str): """ Gets a blob by key and writes the content to stdout. """ + use_secure = _should_use_secure(host, secure) config = _load_config() if not config.blobs.stores: raise click.ClickException("Blob store not configured") out = click.get_binary_stream("stdout") - with BlobManager(config.blobs.stores, host) as blob_manager: + with BlobManager(config.blobs.stores, host, secure=use_secure) as blob_manager: blob = blob_manager.get(key) for chunk in iter(lambda: blob.read(64 * 1024), b""): out.write(chunk) @@ -1020,7 +1021,7 @@ def assets_download( total_size = sum(v["size"] for v in entries.values()) - with BlobManager(config.blobs.stores, host) as blob_manager: + with BlobManager(config.blobs.stores, host, secure=use_secure) as blob_manager: click.echo(f"Downloading {len(entries)} files ({_human_size(total_size)})...") # TODO: parallelise downloads with click.progressbar(entries.items(), label="") as bar: diff --git a/clients/python/coflux/blobs.py b/clients/python/coflux/blobs.py index 40c5a594..438f54c3 100644 --- a/clients/python/coflux/blobs.py +++ b/clients/python/coflux/blobs.py @@ -161,9 +161,11 @@ def upload(self, path: Path) -> str: return self.put(file) -def _create(config_: config.BlobStoreConfig, server_host: str): +def _create(config_: config.BlobStoreConfig, server_host: str, secure: bool): if config_.type == "http": - return HttpStore(config_.protocol, config_.host or server_host) + # Use secure setting if config uses default host, otherwise respect config's protocol + protocol = ("https" if secure else "http") if config_.host is None else config_.protocol + return HttpStore(protocol, config_.host or server_host) elif config_.type == "s3": return S3Store(config_.bucket, config_.prefix, config_.region) else: @@ -171,8 +173,8 @@ def _create(config_: config.BlobStoreConfig, server_host: str): class Manager: - def __init__(self, store_configs: list[config.BlobStoreConfig], server_host: str): - self._stores = [_create(c, server_host) for c in store_configs] + def __init__(self, store_configs: list[config.BlobStoreConfig], server_host: str, *, secure: bool): + self._stores = [_create(c, server_host, secure) for c in store_configs] def __enter__(self): # TODO: ?