From 4e0ea7b621b6c1cf2033f38e7acc91e6033b23a0 Mon Sep 17 00:00:00 2001 From: 0xrushi <0xrushi@gmail.com> Date: Tue, 24 Feb 2026 18:12:20 -0500 Subject: [PATCH 1/3] embedding model --- extras/openmemory-mcp/.env.template | 20 ++- extras/openmemory-mcp/README.md | 8 +- extras/openmemory-mcp/setup.sh | 185 ++++++++++++++++++--- tests/unit/test_openmemory_setup_script.py | 112 +++++++++++++ wizard.py | 158 +++++++++++++++++- 5 files changed, 449 insertions(+), 34 deletions(-) create mode 100644 tests/unit/test_openmemory_setup_script.py diff --git a/extras/openmemory-mcp/.env.template b/extras/openmemory-mcp/.env.template index 10c790bd..4bc09f57 100644 --- a/extras/openmemory-mcp/.env.template +++ b/extras/openmemory-mcp/.env.template @@ -1,11 +1,27 @@ # OpenMemory MCP Configuration # Copy this file to .env and fill in your values -# Required: OpenAI API Key for memory processing +# Required: OpenAI-compatible API key used by OpenMemory defaults OPENAI_API_KEY= +# Optional: OpenAI-compatible base URL (for local providers) +# Example: http://host.docker.internal:11434/v1 +OPENAI_BASE_URL= + +# Optional: Embedding model metadata (for local embedding setups) +OPENAI_EMBEDDING_MODEL= +OPENAI_EMBEDDING_DIMENSIONS= + +# Wizard metadata for embedding provider selection +# Supported values: openai, local +OPENMEMORY_EMBEDDINGS_PROVIDER=openai +OPENMEMORY_EMBEDDINGS_BASE_URL= +OPENMEMORY_EMBEDDINGS_MODEL= +OPENMEMORY_EMBEDDINGS_API_KEY= +OPENMEMORY_EMBEDDINGS_DIMENSIONS= + # Optional: User identifier (defaults to system username) USER=openmemory # Optional: Frontend URL (if using UI) -NEXT_PUBLIC_API_URL=http://localhost:8765 \ No newline at end of file +NEXT_PUBLIC_API_URL=http://localhost:8765 diff --git a/extras/openmemory-mcp/README.md b/extras/openmemory-mcp/README.md index 940a33e5..65d45c51 100644 --- a/extras/openmemory-mcp/README.md +++ b/extras/openmemory-mcp/README.md @@ -17,7 +17,9 @@ OpenMemory MCP is a memory service from mem0.ai that provides: ```bash cp .env.template .env -# Edit .env and add your OPENAI_API_KEY +# Edit .env and add your embedding provider settings +# - OpenAI: OPENAI_API_KEY +# - Local OpenAI-compatible: OPENAI_BASE_URL, OPENAI_API_KEY, OPENAI_EMBEDDING_MODEL, OPENAI_EMBEDDING_DIMENSIONS ``` ### 2. Start Services @@ -64,7 +66,7 @@ The deployment includes: - **MCP Server**: http://localhost:8765 - REST API: `/api/v1/memories` - MCP SSE: `/mcp/{client_name}/sse/{user_id}` - + - **Qdrant Dashboard**: http://localhost:6334/dashboard - **UI** (if enabled): http://localhost:3001 @@ -184,4 +186,4 @@ OpenMemory uses OpenAI by default. To use different models, you would need to mo - [OpenMemory Documentation](https://docs.mem0.ai/open-memory/introduction) - [MCP Protocol Spec](https://github.com/mem0ai/mem0/tree/main/openmemory) -- [Chronicle Memory Docs](../../backends/advanced/MEMORY_PROVIDERS.md) \ No newline at end of file +- [Chronicle Memory Docs](../../backends/advanced/MEMORY_PROVIDERS.md) diff --git a/extras/openmemory-mcp/setup.sh b/extras/openmemory-mcp/setup.sh index afa8cf57..8d3f0e69 100755 --- a/extras/openmemory-mcp/setup.sh +++ b/extras/openmemory-mcp/setup.sh @@ -5,6 +5,11 @@ set -euo pipefail # Parse command line arguments OPENAI_API_KEY="" +EMBEDDINGS_PROVIDER="" +LOCAL_EMBEDDINGS_BASE_URL="" +LOCAL_EMBEDDINGS_MODEL="" +LOCAL_EMBEDDINGS_API_KEY="" +LOCAL_EMBEDDINGS_DIMENSIONS="" while [[ $# -gt 0 ]]; do case $1 in @@ -12,6 +17,26 @@ while [[ $# -gt 0 ]]; do OPENAI_API_KEY="$2" shift 2 ;; + --embeddings-provider) + EMBEDDINGS_PROVIDER="$2" + shift 2 + ;; + --embeddings-base-url) + LOCAL_EMBEDDINGS_BASE_URL="$2" + shift 2 + ;; + --embeddings-model) + LOCAL_EMBEDDINGS_MODEL="$2" + shift 2 + ;; + --embeddings-api-key) + LOCAL_EMBEDDINGS_API_KEY="$2" + shift 2 + ;; + --embeddings-dimensions) + LOCAL_EMBEDDINGS_DIMENSIONS="$2" + shift 2 + ;; *) echo "Unknown argument: $1" exit 1 @@ -43,36 +68,154 @@ fi # Set restrictive permissions (owner read/write only) chmod 600 .env -# Get OpenAI API Key (prompt only if not provided via command line) -if [ -z "$OPENAI_API_KEY" ]; then +# Utility: replace env key or append if missing +upsert_env_key() { + local key="$1" + local value="$2" + local temp_file + + temp_file=$(mktemp) + awk -v key="$key" -v value="$value" ' + BEGIN { found=0 } + $0 ~ ("^" key "=") { print key "=" value; found=1; next } + { print } + END { if (!found) print key "=" value } + ' .env > "$temp_file" + mv "$temp_file" .env +} + +if [ -z "$EMBEDDINGS_PROVIDER" ]; then echo "" - echo "šŸ”‘ OpenAI API Key (required for memory extraction)" - echo "Get yours from: https://platform.openai.com/api-keys" + echo "🧩 Embedding provider" + echo "1) OpenAI embeddings" + echo "2) Local OpenAI-compatible embeddings" while true; do - read -s -r -p "OpenAI API Key: " OPENAI_API_KEY - echo # Print newline after silent input - if [ -n "$OPENAI_API_KEY" ]; then - break - fi - echo "Error: OpenAI API Key cannot be empty. Please try again." + read -r -p "Choose provider [1/2]: " provider_choice + case "$provider_choice" in + 1) + EMBEDDINGS_PROVIDER="openai" + break + ;; + 2) + EMBEDDINGS_PROVIDER="local" + break + ;; + *) + echo "Error: Please enter 1 or 2." + ;; + esac done -else - echo "āœ… OpenAI API key configured from command line" fi -# Update .env file safely using awk - replace existing line or append if missing -temp_file=$(mktemp) -awk -v key="$OPENAI_API_KEY" ' - /^OPENAI_API_KEY=/ { print "OPENAI_API_KEY=" key; found=1; next } - { print } - END { if (!found) print "OPENAI_API_KEY=" key } -' .env > "$temp_file" -mv "$temp_file" .env +if [ "$EMBEDDINGS_PROVIDER" != "openai" ] && [ "$EMBEDDINGS_PROVIDER" != "local" ]; then + echo "Error: --embeddings-provider must be 'openai' or 'local'" >&2 + exit 1 +fi + +if [ "$EMBEDDINGS_PROVIDER" = "openai" ]; then + # Get OpenAI API Key (prompt only if not provided via command line) + if [ -z "$OPENAI_API_KEY" ]; then + echo "" + echo "šŸ”‘ OpenAI API Key (required for memory extraction + embeddings)" + echo "Get yours from: https://platform.openai.com/api-keys" + while true; do + read -s -r -p "OpenAI API Key: " OPENAI_API_KEY + echo # Print newline after silent input + if [ -n "$OPENAI_API_KEY" ]; then + break + fi + echo "Error: OpenAI API Key cannot be empty. Please try again." + done + else + echo "āœ… OpenAI API key configured from command line" + fi + + upsert_env_key "OPENMEMORY_EMBEDDINGS_PROVIDER" "openai" + upsert_env_key "OPENAI_API_KEY" "$OPENAI_API_KEY" + + # Clear local embedding overrides for pure OpenAI mode + upsert_env_key "OPENAI_BASE_URL" "" + upsert_env_key "OPENAI_EMBEDDING_MODEL" "" + upsert_env_key "OPENAI_EMBEDDING_DIMENSIONS" "" + upsert_env_key "OPENMEMORY_EMBEDDINGS_BASE_URL" "" + upsert_env_key "OPENMEMORY_EMBEDDINGS_MODEL" "" + upsert_env_key "OPENMEMORY_EMBEDDINGS_API_KEY" "" + upsert_env_key "OPENMEMORY_EMBEDDINGS_DIMENSIONS" "" +else + echo "" + echo "šŸ  Local embeddings configuration (OpenAI-compatible endpoint)" + + if [ -z "$LOCAL_EMBEDDINGS_BASE_URL" ]; then + while true; do + read -r -p "Embeddings base URL (e.g. http://host.docker.internal:11434/v1): " LOCAL_EMBEDDINGS_BASE_URL + if [ -n "$LOCAL_EMBEDDINGS_BASE_URL" ]; then + break + fi + echo "Error: Base URL cannot be empty. Please try again." + done + fi + + if [ -z "$LOCAL_EMBEDDINGS_MODEL" ]; then + while true; do + read -r -p "Embeddings model name: " LOCAL_EMBEDDINGS_MODEL + if [ -n "$LOCAL_EMBEDDINGS_MODEL" ]; then + break + fi + echo "Error: Model name cannot be empty. Please try again." + done + fi + + if [ -z "$LOCAL_EMBEDDINGS_API_KEY" ]; then + while true; do + read -s -r -p "Embeddings API key: " LOCAL_EMBEDDINGS_API_KEY + echo + if [ -n "$LOCAL_EMBEDDINGS_API_KEY" ]; then + break + fi + echo "Error: API key cannot be empty. Please try again." + done + fi + + if [ -z "$LOCAL_EMBEDDINGS_DIMENSIONS" ]; then + while true; do + read -r -p "Embedding dimensions (e.g. 768): " LOCAL_EMBEDDINGS_DIMENSIONS + if [[ "$LOCAL_EMBEDDINGS_DIMENSIONS" =~ ^[0-9]+$ ]] && [ "$LOCAL_EMBEDDINGS_DIMENSIONS" -gt 0 ]; then + break + fi + echo "Error: Dimensions must be a positive integer." + done + fi + + upsert_env_key "OPENMEMORY_EMBEDDINGS_PROVIDER" "local" + + # Keep OpenAI-compatible defaults pointed at the local embeddings endpoint. + # OpenMemory reads OPENAI_API_KEY by default, and OPENAI_BASE_URL can redirect + # OpenAI client calls to local-compatible servers. + upsert_env_key "OPENAI_API_KEY" "$LOCAL_EMBEDDINGS_API_KEY" + upsert_env_key "OPENAI_BASE_URL" "$LOCAL_EMBEDDINGS_BASE_URL" + upsert_env_key "OPENAI_EMBEDDING_MODEL" "$LOCAL_EMBEDDINGS_MODEL" + upsert_env_key "OPENAI_EMBEDDING_DIMENSIONS" "$LOCAL_EMBEDDINGS_DIMENSIONS" + + # Also store explicit OpenMemory-local embedding fields for future tooling. + upsert_env_key "OPENMEMORY_EMBEDDINGS_BASE_URL" "$LOCAL_EMBEDDINGS_BASE_URL" + upsert_env_key "OPENMEMORY_EMBEDDINGS_MODEL" "$LOCAL_EMBEDDINGS_MODEL" + upsert_env_key "OPENMEMORY_EMBEDDINGS_API_KEY" "$LOCAL_EMBEDDINGS_API_KEY" + upsert_env_key "OPENMEMORY_EMBEDDINGS_DIMENSIONS" "$LOCAL_EMBEDDINGS_DIMENSIONS" +fi echo "" echo "āœ… OpenMemory MCP configured!" echo "šŸ“ Configuration saved to .env" echo "" +if [ "$EMBEDDINGS_PROVIDER" = "local" ]; then + echo "ā„¹ļø Local embeddings mode enabled" + echo " Base URL: $LOCAL_EMBEDDINGS_BASE_URL" + echo " Model: $LOCAL_EMBEDDINGS_MODEL" + echo " Dimensions: $LOCAL_EMBEDDINGS_DIMENSIONS" +else + echo "ā„¹ļø OpenAI embeddings mode enabled" +fi +echo "" echo "šŸš€ To start: docker compose up -d" echo "🌐 MCP Server: http://localhost:8765" -echo "šŸ“± Web UI: http://localhost:3001" \ No newline at end of file +echo "šŸ“± Web UI: http://localhost:3001" diff --git a/tests/unit/test_openmemory_setup_script.py b/tests/unit/test_openmemory_setup_script.py new file mode 100644 index 00000000..d03649b0 --- /dev/null +++ b/tests/unit/test_openmemory_setup_script.py @@ -0,0 +1,112 @@ +import shutil +import stat +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +OPENMEMORY_DIR = REPO_ROOT / "extras" / "openmemory-mcp" + + +def _prepare_tmp_setup(tmp_path: Path) -> Path: + setup_src = OPENMEMORY_DIR / "setup.sh" + template_src = OPENMEMORY_DIR / ".env.template" + + setup_dst = tmp_path / "setup.sh" + template_dst = tmp_path / ".env.template" + + shutil.copy2(setup_src, setup_dst) + shutil.copy2(template_src, template_dst) + + setup_dst.chmod(setup_dst.stat().st_mode | stat.S_IXUSR) + return setup_dst + + +def _read_env_map(env_path: Path) -> dict[str, str]: + data = {} + for raw_line in env_path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + data[key] = value + return data + + +def test_setup_openai_embeddings_mode_writes_expected_env(tmp_path): + setup_script = _prepare_tmp_setup(tmp_path) + + subprocess.run( + [ + "bash", + str(setup_script), + "--embeddings-provider", + "openai", + "--openai-api-key", + "sk-test-openai", + ], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + + env_map = _read_env_map(tmp_path / ".env") + assert env_map["OPENMEMORY_EMBEDDINGS_PROVIDER"] == "openai" + assert env_map["OPENAI_API_KEY"] == "sk-test-openai" + assert env_map["OPENAI_BASE_URL"] == "" + assert env_map["OPENAI_EMBEDDING_MODEL"] == "" + assert env_map["OPENAI_EMBEDDING_DIMENSIONS"] == "" + + +def test_setup_local_embeddings_mode_writes_expected_env(tmp_path): + setup_script = _prepare_tmp_setup(tmp_path) + + subprocess.run( + [ + "bash", + str(setup_script), + "--embeddings-provider", + "local", + "--embeddings-base-url", + "http://host.docker.internal:11434/v1", + "--embeddings-model", + "nomic-embed-text", + "--embeddings-api-key", + "local-key", + "--embeddings-dimensions", + "768", + ], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + + env_map = _read_env_map(tmp_path / ".env") + assert env_map["OPENMEMORY_EMBEDDINGS_PROVIDER"] == "local" + assert env_map["OPENAI_API_KEY"] == "local-key" + assert env_map["OPENAI_BASE_URL"] == "http://host.docker.internal:11434/v1" + assert env_map["OPENAI_EMBEDDING_MODEL"] == "nomic-embed-text" + assert env_map["OPENAI_EMBEDDING_DIMENSIONS"] == "768" + assert ( + env_map["OPENMEMORY_EMBEDDINGS_BASE_URL"] + == "http://host.docker.internal:11434/v1" + ) + assert env_map["OPENMEMORY_EMBEDDINGS_MODEL"] == "nomic-embed-text" + assert env_map["OPENMEMORY_EMBEDDINGS_API_KEY"] == "local-key" + assert env_map["OPENMEMORY_EMBEDDINGS_DIMENSIONS"] == "768" + + +def test_setup_rejects_invalid_embeddings_provider(tmp_path): + setup_script = _prepare_tmp_setup(tmp_path) + + result = subprocess.run( + ["bash", str(setup_script), "--embeddings-provider", "invalid-provider"], + cwd=tmp_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "--embeddings-provider must be 'openai' or 'local'" in result.stderr diff --git a/wizard.py b/wizard.py index 784fdfe4..00348056 100755 --- a/wizard.py +++ b/wizard.py @@ -250,6 +250,7 @@ def run_service_setup( langfuse_secret_key=None, langfuse_host=None, streaming_provider=None, + hardware_profile=None, ): """Execute individual service setup script""" if service_name == "advanced": @@ -302,6 +303,14 @@ def run_service_setup( # Define the speaker env path speaker_env_path = "extras/speaker-recognition/.env" + # Pass explicit hardware profile selection when provided by wizard + if hardware_profile == "strixhalo": + cmd.extend(["--pytorch-cuda-version", "strixhalo"]) + cmd.extend(["--compute-mode", "gpu"]) + console.print( + "[blue][INFO][/blue] Using AMD Strix Halo profile for speaker recognition" + ) + # HF Token should have been provided via setup_hf_token_if_needed() if hf_token: cmd.extend(["--hf-token", hf_token]) @@ -323,7 +332,7 @@ def run_service_setup( # Pass compute mode from existing .env if available compute_mode = read_env_value(speaker_env_path, "COMPUTE_MODE") - if compute_mode in ["cpu", "gpu"]: + if hardware_profile != "strixhalo" and compute_mode in ["cpu", "gpu"]: cmd.extend(["--compute-mode", compute_mode]) console.print( f"[blue][INFO][/blue] Found existing COMPUTE_MODE ({compute_mode}), reusing" @@ -332,11 +341,18 @@ def run_service_setup( # For asr-services, pass provider from wizard's transcription choice and reuse CUDA version if service_name == "asr-services": # Map wizard transcription provider to asr-services provider name - wizard_to_asr_provider = { - "vibevoice": "vibevoice", - "parakeet": "nemo", - "qwen3-asr": "qwen3-asr", - } + if hardware_profile == "strixhalo": + wizard_to_asr_provider = { + "vibevoice": "vibevoice-strixhalo", + "parakeet": "nemo-strixhalo", + "qwen3-asr": "qwen3-asr", + } + else: + wizard_to_asr_provider = { + "vibevoice": "vibevoice", + "parakeet": "nemo", + "qwen3-asr": "qwen3-asr", + } asr_provider = wizard_to_asr_provider.get(transcription_provider) if asr_provider: cmd.extend(["--provider", asr_provider]) @@ -346,7 +362,17 @@ def run_service_setup( speaker_env_path = "extras/speaker-recognition/.env" cuda_version = read_env_value(speaker_env_path, "PYTORCH_CUDA_VERSION") - if cuda_version and cuda_version in ["cu121", "cu126", "cu128"]: + if hardware_profile == "strixhalo": + cmd.extend(["--pytorch-cuda-version", "strixhalo"]) + console.print( + "[blue][INFO][/blue] Using AMD Strix Halo profile for ASR services" + ) + elif cuda_version and cuda_version in [ + "cu121", + "cu126", + "cu128", + "strixhalo", + ]: cmd.extend(["--pytorch-cuda-version", cuda_version]) console.print( f"[blue][INFO][/blue] Found existing PYTORCH_CUDA_VERSION ({cuda_version}) from speaker-recognition, reusing" @@ -362,14 +388,78 @@ def run_service_setup( # For openmemory-mcp, try to pass OpenAI API key from backend if available if service_name == "openmemory-mcp": backend_env_path = "backends/advanced/.env" + openmemory_env_path = "extras/openmemory-mcp/.env" openai_key = read_env_value(backend_env_path, "OPENAI_API_KEY") - if openai_key and not is_placeholder( + backend_openai_base_url = read_env_value( + backend_env_path, "OPENAI_BASE_URL" + ) + backend_embedding_model = read_env_value( + backend_env_path, "OPENAI_EMBEDDING_MODEL" + ) + backend_embedding_dims = read_env_value( + backend_env_path, "OPENAI_EMBEDDING_DIMENSIONS" + ) + + existing_embeddings_provider = read_env_value( + openmemory_env_path, "OPENMEMORY_EMBEDDINGS_PROVIDER" + ) + existing_embeddings_base_url = read_env_value( + openmemory_env_path, "OPENMEMORY_EMBEDDINGS_BASE_URL" + ) + existing_embeddings_model = read_env_value( + openmemory_env_path, "OPENMEMORY_EMBEDDINGS_MODEL" + ) + existing_embeddings_api_key = read_env_value( + openmemory_env_path, "OPENMEMORY_EMBEDDINGS_API_KEY" + ) + existing_embeddings_dims = read_env_value( + openmemory_env_path, "OPENMEMORY_EMBEDDINGS_DIMENSIONS" + ) + + def _has_value(value): + return value and value.strip() + + has_openai_key = _has_value(openai_key) and not is_placeholder( openai_key, "your_openai_api_key_here", "your-openai-api-key-here", "your_openai_key_here", "your-openai-key-here", + ) + + # Prefer an existing OpenMemory local embedding configuration if available. + if ( + existing_embeddings_provider == "local" + and _has_value(existing_embeddings_base_url) + and _has_value(existing_embeddings_model) + and _has_value(existing_embeddings_api_key) + and _has_value(existing_embeddings_dims) + ): + cmd.extend(["--embeddings-provider", "local"]) + cmd.extend(["--embeddings-base-url", existing_embeddings_base_url]) + cmd.extend(["--embeddings-model", existing_embeddings_model]) + cmd.extend(["--embeddings-api-key", existing_embeddings_api_key]) + cmd.extend(["--embeddings-dimensions", existing_embeddings_dims]) + console.print( + "[blue][INFO][/blue] Found existing local embeddings config for OpenMemory, reusing" + ) + elif ( + has_openai_key + and _has_value(backend_openai_base_url) + and "api.openai.com" not in backend_openai_base_url ): + # Backend appears to use a local OpenAI-compatible endpoint. + cmd.extend(["--embeddings-provider", "local"]) + cmd.extend(["--embeddings-base-url", backend_openai_base_url]) + cmd.extend(["--embeddings-api-key", openai_key]) + if _has_value(backend_embedding_model): + cmd.extend(["--embeddings-model", backend_embedding_model]) + if _has_value(backend_embedding_dims): + cmd.extend(["--embeddings-dimensions", backend_embedding_dims]) + console.print( + "[blue][INFO][/blue] Found OpenAI-compatible local endpoint in backend config, pre-filling OpenMemory local embeddings" + ) + elif has_openai_key: cmd.extend(["--openai-api-key", openai_key]) console.print( "[blue][INFO][/blue] Found existing OPENAI_API_KEY from backend config, reusing" @@ -870,6 +960,53 @@ def setup_langfuse_choice(): } +def select_hardware_profile( + selected_services, transcription_provider, streaming_provider +): + """Select hardware profile for GPU-backed optional services. + + Returns: + "strixhalo" for AMD Strix Halo profile, otherwise None. + """ + strix_capable_providers = {"parakeet", "vibevoice"} + needs_hardware_choice = ( + "speaker-recognition" in selected_services + or transcription_provider in strix_capable_providers + or streaming_provider in strix_capable_providers + ) + + if not needs_hardware_choice: + return None + + console.print("\n🧠 [bold cyan]Hardware Profile[/bold cyan]") + console.print( + "Choose target hardware for GPU services (speaker recognition and offline ASR):" + ) + choices = { + "1": "Standard (CPU/NVIDIA CUDA)", + "2": "AMD Strix Halo (ROCm, gfx1151 / Ryzen AI Max)", + } + for key, desc in choices.items(): + console.print(f" {key}) {desc}") + console.print() + + while True: + try: + choice = Prompt.ask("Enter choice", default="1") + if choice == "1": + return None + if choice == "2": + console.print( + "[green]āœ…[/green] Using AMD Strix Halo profile where supported" + ) + return "strixhalo" + console.print( + f"[red]Invalid choice. Please select from {list(choices.keys())}[/red]" + ) + except EOFError: + return None + + def main(): """Main orchestration logic""" console.print("šŸŽ‰ [bold green]Welcome to Chronicle![/bold green]\n") @@ -925,6 +1062,10 @@ def main(): selected_services.append("langfuse") # HF Token Configuration (if services require it) + hardware_profile = select_hardware_profile( + selected_services, transcription_provider, streaming_provider + ) + hf_token = setup_hf_token_if_needed(selected_services) # HTTPS Configuration (for services that need it) @@ -1101,6 +1242,7 @@ def main(): langfuse_secret_key=langfuse_secret_key, langfuse_host=langfuse_host, streaming_provider=streaming_provider, + hardware_profile=hardware_profile, ): success_count += 1 From c3fd89bdd01cf924ec5e40dedd973d89fd3bfbbd Mon Sep 17 00:00:00 2001 From: 0xrushi <0xrushi@gmail.com> Date: Wed, 25 Feb 2026 21:47:54 -0500 Subject: [PATCH 2/3] Implement prebuilt image support and enhance Docker Compose configurations - Add image: fields with CHRONICLE_REGISTRY/CHRONICLE_TAG env vars to all custom services in docker-compose files (backends/advanced, asr-services, speaker-recognition, havpe-relay) - Create scripts/push-images.sh to tag and push local images to DockerHub - Create scripts/pull-images.sh to pull and retag images for local use - Add --use-prebuilt TAG flag to services.py start command; sets CHRONICLE_REGISTRY and CHRONICLE_TAG env vars and disables build - Add unit tests for image versioning changes Co-Authored-By: Claude Sonnet 4.6 --- backends/advanced/docker-compose.yml | 4 + extras/asr-services/docker-compose.yml | 12 +- extras/havpe-relay/docker-compose.yml | 5 +- extras/speaker-recognition/docker-compose.yml | 3 +- scripts/pull-images.sh | 122 ++++++ scripts/push-images.sh | 188 +++++++++ services.py | 25 +- tests/unit/test_docker_image_versioning.py | 386 ++++++++++++++++++ 8 files changed, 735 insertions(+), 10 deletions(-) create mode 100755 scripts/pull-images.sh create mode 100755 scripts/push-images.sh create mode 100644 tests/unit/test_docker_image_versioning.py diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index 8fe1ddfe..e74811f4 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -26,6 +26,7 @@ services: socat TCP-LISTEN:18123,fork,reuseaddr TCP:100.99.62.5:8123" chronicle-backend: + image: ${CHRONICLE_REGISTRY:-}chronicle-backend:${CHRONICLE_TAG:-latest} build: context: . dockerfile: Dockerfile @@ -83,6 +84,7 @@ services: # - 1+ Stream workers (conditional based on config.yml - Deepgram/Parakeet) # Uses Python orchestrator for process management, health monitoring, and self-healing workers: + image: ${CHRONICLE_REGISTRY:-}chronicle-backend:${CHRONICLE_TAG:-latest} build: context: . dockerfile: Dockerfile @@ -131,6 +133,7 @@ services: # - Weekly: Fine-tune error detection models using user feedback # Set DEV_MODE=true in .env for 1-minute intervals (testing) annotation-cron: + image: ${CHRONICLE_REGISTRY:-}chronicle-backend:${CHRONICLE_TAG:-latest} build: context: . dockerfile: Dockerfile @@ -153,6 +156,7 @@ services: - annotation # Optional profile - enable with: docker compose --profile annotation up webui: + image: ${CHRONICLE_REGISTRY:-}chronicle-webui:${CHRONICLE_TAG:-latest} build: context: ./webui dockerfile: Dockerfile diff --git a/extras/asr-services/docker-compose.yml b/extras/asr-services/docker-compose.yml index fea49372..a1032965 100644 --- a/extras/asr-services/docker-compose.yml +++ b/extras/asr-services/docker-compose.yml @@ -24,7 +24,7 @@ services: dockerfile: providers/nemo/Dockerfile args: PYTORCH_CUDA_VERSION: ${PYTORCH_CUDA_VERSION:-cu126} - image: chronicle-asr-nemo:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-nemo:${CHRONICLE_TAG:-latest} ports: - "${ASR_PORT:-8767}:8765" volumes: @@ -58,7 +58,7 @@ services: build: context: . dockerfile: providers/faster_whisper/Dockerfile - image: chronicle-asr-faster-whisper:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-faster-whisper:${CHRONICLE_TAG:-latest} ports: - "${ASR_PORT:-8767}:8765" volumes: @@ -92,7 +92,7 @@ services: dockerfile: providers/vibevoice/Dockerfile args: PYTORCH_CUDA_VERSION: ${PYTORCH_CUDA_VERSION:-cu126} - image: chronicle-asr-vibevoice:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-vibevoice:${CHRONICLE_TAG:-latest} ports: - "${ASR_PORT:-8767}:8765" volumes: @@ -133,7 +133,7 @@ services: build: context: . dockerfile: providers/transformers/Dockerfile - image: chronicle-asr-transformers:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-transformers:${CHRONICLE_TAG:-latest} ports: - "${ASR_PORT:-8767}:8765" volumes: @@ -193,7 +193,7 @@ services: build: context: . dockerfile: providers/qwen3_asr/Dockerfile.full - image: chronicle-asr-qwen3-wrapper:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-qwen3-wrapper:${CHRONICLE_TAG:-latest} ports: - "${ASR_PORT:-8767}:8765" volumes: @@ -220,7 +220,7 @@ services: build: context: . dockerfile: providers/qwen3_asr/Dockerfile - image: chronicle-asr-qwen3-bridge:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-qwen3-bridge:${CHRONICLE_TAG:-latest} ports: - "${ASR_STREAM_PORT:-8769}:8766" environment: diff --git a/extras/havpe-relay/docker-compose.yml b/extras/havpe-relay/docker-compose.yml index 055f6492..aef52450 100644 --- a/extras/havpe-relay/docker-compose.yml +++ b/extras/havpe-relay/docker-compose.yml @@ -1,5 +1,6 @@ services: havpe-relay: + image: ${CHRONICLE_REGISTRY:-}chronicle-havpe-relay:${CHRONICLE_TAG:-latest} build: context: . dockerfile: Dockerfile @@ -21,8 +22,8 @@ services: timeout: 10s retries: 3 start_period: 10s - command: ["uv", "run", "python3", "main.py"] + command: ["uv", "run", "python3", "main.py"] extra_hosts: - "host.docker.internal:host-gateway" volumes: - - ./audio_chunks:/app/audio_chunks \ No newline at end of file + - ./audio_chunks:/app/audio_chunks diff --git a/extras/speaker-recognition/docker-compose.yml b/extras/speaker-recognition/docker-compose.yml index d70756ed..8fc0fe43 100644 --- a/extras/speaker-recognition/docker-compose.yml +++ b/extras/speaker-recognition/docker-compose.yml @@ -9,7 +9,7 @@ services: dockerfile: Dockerfile args: PYTORCH_CUDA_VERSION: ${PYTORCH_CUDA_VERSION:-cpu} - image: speaker-recognition:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-speaker:${CHRONICLE_TAG:-latest} env_file: - .env ports: @@ -63,6 +63,7 @@ services: # React Web UI web-ui: platform: linux/amd64 + image: ${CHRONICLE_REGISTRY:-}chronicle-speaker-webui:${CHRONICLE_TAG:-latest} build: context: webui dockerfile: Dockerfile diff --git a/scripts/pull-images.sh b/scripts/pull-images.sh new file mode 100755 index 00000000..caf86bbe --- /dev/null +++ b/scripts/pull-images.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# pull-images.sh — Pull Chronicle images from DockerHub and retag them locally +# +# Usage: +# DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0 +# +# After pulling, start with the prebuilt images: +# DOCKERHUB_USERNAME=myuser ./start.sh --use-prebuilt v1.0.0 + +set -euo pipefail + +# ── Colour helpers ───────────────────────────────────────────────────────────── +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +RESET='\033[0m' + +info() { echo -e "${CYAN}ā„¹ļø $*${RESET}"; } +success() { echo -e "${GREEN}āœ… $*${RESET}"; } +warn() { echo -e "${YELLOW}āš ļø $*${RESET}"; } +error() { echo -e "${RED}āŒ $*${RESET}" >&2; } + +# ── Validate inputs ──────────────────────────────────────────────────────────── +if [[ -z "${DOCKERHUB_USERNAME:-}" ]]; then + error "DOCKERHUB_USERNAME env var is required." + echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0" >&2 + exit 1 +fi + +TAG="${1:-}" +if [[ -z "$TAG" ]]; then + error "TAG argument is required." + echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/pull-images.sh v1.0.0" >&2 + exit 1 +fi + +REGISTRY="${DOCKERHUB_USERNAME}/" + +# ── Image inventory ──────────────────────────────────────────────────────────── +# Format: "local-image-name:registry-image-name" +# After pulling, each remote image is retagged to ":" +# so that docker-compose can find it when CHRONICLE_TAG= is set. +IMAGES=( + "chronicle-backend:chronicle-backend" + "chronicle-webui:chronicle-webui" + "chronicle-speaker:chronicle-speaker" + "chronicle-speaker-strixhalo:chronicle-speaker-strixhalo" + "chronicle-speaker-webui:chronicle-speaker-webui" + "chronicle-asr-nemo:chronicle-asr-nemo" + "chronicle-asr-nemo-strixhalo:chronicle-asr-nemo-strixhalo" + "chronicle-asr-faster-whisper:chronicle-asr-faster-whisper" + "chronicle-asr-vibevoice:chronicle-asr-vibevoice" + "chronicle-asr-vibevoice-strixhalo:chronicle-asr-vibevoice-strixhalo" + "chronicle-asr-transformers:chronicle-asr-transformers" + "chronicle-asr-qwen3-wrapper:chronicle-asr-qwen3-wrapper" + "chronicle-asr-qwen3-bridge:chronicle-asr-qwen3-bridge" + "chronicle-havpe-relay:chronicle-havpe-relay" +) + +echo "" +echo -e "${BOLD}Chronicle Image Pull${RESET}" +echo -e " Registry : ${REGISTRY}" +echo -e " Tag : ${TAG}" +echo "" + +# ── Pull loop ───────────────────────────────────────────────────────────────── +PULLED=() +FAILED=() + +for entry in "${IMAGES[@]}"; do + LOCAL_BASE="${entry%%:*}" + REMOTE_BASE="${REGISTRY}${entry##*:}" + REMOTE_TAG="${REMOTE_BASE}:${TAG}" + LOCAL_TAG="${LOCAL_BASE}:${TAG}" + + echo -e "${CYAN}── ${entry##*:}${RESET}" + info " Pulling ← ${REMOTE_TAG}" + + if docker pull "${REMOTE_TAG}"; then + # Retag to local name so docker-compose finds it with CHRONICLE_TAG= + info " Retagging → ${LOCAL_TAG}" + docker tag "${REMOTE_TAG}" "${LOCAL_TAG}" + success " Ready as ${LOCAL_TAG}" + PULLED+=("${entry##*:}") + else + warn " Not found on DockerHub — skipping (this service may not have been pushed)" + FAILED+=("${entry##*:}") + fi + echo "" +done + +# ── Summary ─────────────────────────────────────────────────────────────────── +echo -e "${BOLD}── Summary ──────────────────────────────────────────────────────${RESET}" +printf "%-40s %s\n" "Image" "Status" +printf "%-40s %s\n" "─────────────────────────────────────" "──────" + +for entry in "${IMAGES[@]}"; do + IMAGE_NAME="${entry##*:}" + STATUS="" + for p in "${PULLED[@]}"; do + [[ "$p" == "$IMAGE_NAME" ]] && STATUS="${GREEN}pulled${RESET}" && break + done + for f in "${FAILED[@]}"; do + [[ "$f" == "$IMAGE_NAME" ]] && STATUS="${YELLOW}not found${RESET}" && break + done + [[ -z "$STATUS" ]] && STATUS="${YELLOW}not found${RESET}" + printf "%-40s " "${IMAGE_NAME}" + echo -e "${STATUS}" +done + +echo "" +if [[ ${#PULLED[@]} -gt 0 ]]; then + success "Pulled ${#PULLED[@]} image(s) tagged as ${TAG}" + echo "" + echo "Start services with prebuilt images:" + echo -e " ${BOLD}DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./start.sh --use-prebuilt ${TAG}${RESET}" +fi +if [[ ${#FAILED[@]} -gt 0 ]]; then + warn "${#FAILED[@]} image(s) not found on DockerHub (these services will fall back to local builds)" +fi diff --git a/scripts/push-images.sh b/scripts/push-images.sh new file mode 100755 index 00000000..0cf545bc --- /dev/null +++ b/scripts/push-images.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# push-images.sh — Tag and push locally-built Chronicle images to DockerHub +# +# Usage: +# DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 "stable before refactor" +# +# Requirements: +# - Images must already be built locally (run ./start.sh --build first) +# - DOCKERHUB_USERNAME env var must be set +# - Must be logged in to DockerHub (docker login) + +set -euo pipefail + +# ── Colour helpers ───────────────────────────────────────────────────────────── +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +RESET='\033[0m' + +info() { echo -e "${CYAN}ā„¹ļø $*${RESET}"; } +success() { echo -e "${GREEN}āœ… $*${RESET}"; } +warn() { echo -e "${YELLOW}āš ļø $*${RESET}"; } +error() { echo -e "${RED}āŒ $*${RESET}" >&2; } + +# ── Validate inputs ──────────────────────────────────────────────────────────── +if [[ -z "${DOCKERHUB_USERNAME:-}" ]]; then + error "DOCKERHUB_USERNAME env var is required." + echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 'description'" >&2 + exit 1 +fi + +TAG="${1:-}" +if [[ -z "$TAG" ]]; then + error "TAG argument is required." + echo " Example: DOCKERHUB_USERNAME=myuser ./scripts/push-images.sh v1.0.0 'description'" >&2 + exit 1 +fi + +DESCRIPTION="${2:-}" + +REGISTRY="${DOCKERHUB_USERNAME}/" + +# ── Image inventory ──────────────────────────────────────────────────────────── +# Format: "local-image-name:registry-image-name" +# local-image-name = what docker-compose builds to locally (with empty CHRONICLE_REGISTRY) +# registry-image-name = what gets pushed to DockerHub +IMAGES=( + "chronicle-backend:chronicle-backend" + "chronicle-webui:chronicle-webui" + "chronicle-speaker:chronicle-speaker" + "chronicle-speaker-strixhalo:chronicle-speaker-strixhalo" + "chronicle-speaker-webui:chronicle-speaker-webui" + "chronicle-asr-nemo:chronicle-asr-nemo" + "chronicle-asr-nemo-strixhalo:chronicle-asr-nemo-strixhalo" + "chronicle-asr-faster-whisper:chronicle-asr-faster-whisper" + "chronicle-asr-vibevoice:chronicle-asr-vibevoice" + "chronicle-asr-vibevoice-strixhalo:chronicle-asr-vibevoice-strixhalo" + "chronicle-asr-transformers:chronicle-asr-transformers" + "chronicle-asr-qwen3-wrapper:chronicle-asr-qwen3-wrapper" + "chronicle-asr-qwen3-bridge:chronicle-asr-qwen3-bridge" + "chronicle-havpe-relay:chronicle-havpe-relay" +) + +# ── Collect git info ─────────────────────────────────────────────────────────── +GIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") +DATE=$(date +%Y-%m-%d) + +echo "" +echo -e "${BOLD}Chronicle Image Push${RESET}" +echo -e " Registry : ${REGISTRY}" +echo -e " Tag : ${TAG}" +echo -e " Git SHA : ${GIT_SHA}" +echo -e " Date : ${DATE}" +[[ -n "$DESCRIPTION" ]] && echo -e " Desc : ${DESCRIPTION}" +echo "" + +# ── Push loop ───────────────────────────────────────────────────────────────── +PUSHED=() +SKIPPED=() + +for entry in "${IMAGES[@]}"; do + LOCAL_NAME="${entry%%:*}:latest" + REMOTE_BASE="${REGISTRY}${entry##*:}" + REMOTE_TAG="${REMOTE_BASE}:${TAG}" + REMOTE_LATEST="${REMOTE_BASE}:latest" + + echo -e "${CYAN}── ${LOCAL_NAME}${RESET}" + + # Check if image exists locally + if ! docker image inspect "${LOCAL_NAME}" > /dev/null 2>&1; then + warn " Not found locally — skipping (run ./start.sh --build to build it first)" + SKIPPED+=("${LOCAL_NAME}") + continue + fi + + # Tag and push versioned tag + info " Tagging → ${REMOTE_TAG}" + docker tag "${LOCAL_NAME}" "${REMOTE_TAG}" + + info " Pushing → ${REMOTE_TAG}" + if docker push "${REMOTE_TAG}"; then + success " Pushed ${REMOTE_TAG}" + else + error " Failed to push ${REMOTE_TAG}" + SKIPPED+=("${LOCAL_NAME}") + continue + fi + + # Also tag and push :latest + info " Tagging → ${REMOTE_LATEST}" + docker tag "${LOCAL_NAME}" "${REMOTE_LATEST}" + + info " Pushing → ${REMOTE_LATEST}" + if docker push "${REMOTE_LATEST}"; then + success " Pushed ${REMOTE_LATEST}" + else + warn " Failed to push :latest tag (versioned tag was already pushed)" + fi + + PUSHED+=("${entry##*:}") + echo "" +done + +# ── Update releases.json ─────────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASES_FILE="${SCRIPT_DIR}/releases.json" + +# Build JSON array of pushed image names +IMAGES_JSON="[" +for i in "${!PUSHED[@]}"; do + [[ $i -gt 0 ]] && IMAGES_JSON+="," + IMAGES_JSON+="\"${PUSHED[$i]}\"" +done +IMAGES_JSON+="]" + +NEW_ENTRY="{\"tag\":\"${TAG}\",\"description\":\"${DESCRIPTION}\",\"date\":\"${DATE}\",\"git_sha\":\"${GIT_SHA}\",\"images_pushed\":${IMAGES_JSON}}" + +if [[ -f "$RELEASES_FILE" ]]; then + # Append to existing array + EXISTING=$(cat "$RELEASES_FILE") + # Strip trailing ] and append new entry + TRIMMED="${EXISTING%]}" + # Handle empty array + if [[ "$TRIMMED" == "[" ]]; then + echo "[${NEW_ENTRY}]" > "$RELEASES_FILE" + else + echo "${TRIMMED},${NEW_ENTRY}]" > "$RELEASES_FILE" + fi +else + echo "[${NEW_ENTRY}]" > "$RELEASES_FILE" +fi + +success "Recorded release in scripts/releases.json" + +# ── Summary ─────────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}── Summary ──────────────────────────────────────────────────────${RESET}" +printf "%-40s %s\n" "Image" "Status" +printf "%-40s %s\n" "─────────────────────────────────────" "──────" + +for entry in "${IMAGES[@]}"; do + IMAGE_NAME="${entry##*:}" + LOCAL_NAME="${entry%%:*}:latest" + STATUS="" + for p in "${PUSHED[@]}"; do + [[ "$p" == "$IMAGE_NAME" ]] && STATUS="${GREEN}pushed${RESET}" && break + done + for s in "${SKIPPED[@]}"; do + [[ "$s" == "$LOCAL_NAME" ]] && STATUS="${YELLOW}skipped${RESET}" && break + done + [[ -z "$STATUS" ]] && STATUS="${YELLOW}skipped${RESET}" + printf "%-40s " "${IMAGE_NAME}" + echo -e "${STATUS}" +done + +echo "" +if [[ ${#PUSHED[@]} -gt 0 ]]; then + success "Pushed ${#PUSHED[@]} image(s) as ${TAG}" + echo "" + echo "To restore this snapshot:" + echo " DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./scripts/pull-images.sh ${TAG}" + echo " DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} ./start.sh --use-prebuilt ${TAG}" +fi +if [[ ${#SKIPPED[@]} -gt 0 ]]; then + warn "${#SKIPPED[@]} image(s) were skipped (not found locally)" +fi diff --git a/services.py b/services.py index 664582e1..0c0f7ba8 100755 --- a/services.py +++ b/services.py @@ -5,6 +5,7 @@ """ import argparse +import os import subprocess from pathlib import Path @@ -12,6 +13,7 @@ from dotenv import dotenv_values from rich.console import Console from rich.table import Table + from setup_utils import read_env_value console = Console() @@ -621,6 +623,11 @@ def main(): action="store_true", help="Force recreate containers even if unchanged", ) + start_parser.add_argument( + "--use-prebuilt", + metavar="TAG", + help="Use prebuilt images from DockerHub instead of building (requires prior pull-images.sh run)", + ) # Stop command stop_parser = subparsers.add_parser("stop", help="Stop services") @@ -683,7 +690,23 @@ def main(): ) return - start_services(services, args.build, args.force_recreate) + if args.use_prebuilt: + dockerhub_user = os.environ.get("DOCKERHUB_USERNAME", "") + if not dockerhub_user: + console.print( + "[red]āŒ DOCKERHUB_USERNAME env var required with --use-prebuilt[/red]" + ) + return + os.environ["CHRONICLE_REGISTRY"] = f"{dockerhub_user}/" + os.environ["CHRONICLE_TAG"] = args.use_prebuilt + console.print( + f"[cyan]ā„¹ļø Using prebuilt images: {dockerhub_user}/*:{args.use_prebuilt}[/cyan]" + ) + build_flag = False + else: + build_flag = args.build + + start_services(services, build_flag, args.force_recreate) elif args.command == "stop": if args.all: diff --git a/tests/unit/test_docker_image_versioning.py b/tests/unit/test_docker_image_versioning.py new file mode 100644 index 00000000..6bea00c3 --- /dev/null +++ b/tests/unit/test_docker_image_versioning.py @@ -0,0 +1,386 @@ +"""Tests for the Docker image versioning & DockerHub deployment feature. + +Covers three areas without requiring Docker or network access: + 1. services.py --use-prebuilt flag (argument parsing + env-var injection) + 2. docker-compose.yml files contain the expected image: fields + 3. push-images.sh / pull-images.sh reject missing inputs +""" + +import importlib +import os +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS_DIR = REPO_ROOT / "scripts" + +# --------------------------------------------------------------------------- +# Helper: import services.py from the repo root (it lives there, not in a pkg). +# services.py depends on python-dotenv and rich which may not be installed in +# the lightweight test environment, so we stub them at sys.modules level. +# --------------------------------------------------------------------------- + + +def _stub_missing(name: str, attrs: dict): + """Insert a minimal fake module under *name* if it isn't already importable.""" + if name in sys.modules: + return + fake = MagicMock() + for k, v in attrs.items(): + setattr(fake, k, v) + sys.modules[name] = fake + + +def _import_services(): + # Stub third-party deps that aren't installed in the bare test runner + _stub_missing("dotenv", {"dotenv_values": lambda path: {}}) + _stub_missing("rich", {}) + _stub_missing("rich.console", {"Console": MagicMock}) + _stub_missing("rich.table", {"Table": MagicMock}) + _stub_missing("setup_utils", {"read_env_value": lambda *a, **kw: None}) + + spec = importlib.util.spec_from_file_location("services", REPO_ROOT / "services.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# =========================================================================== +# 1. services.py — --use-prebuilt flag +# =========================================================================== + + +class TestUsePrebuiltFlag: + """Test the --use-prebuilt TAG argument added to services.py start command.""" + + def _run_start(self, argv, env_override=None): + """Import and run services.main() with given argv + env, mocking side effects.""" + services_mod = _import_services() + + env = {**os.environ, **(env_override or {})} + # Remove CHRONICLE_* vars so we start clean + env.pop("CHRONICLE_REGISTRY", None) + env.pop("CHRONICLE_TAG", None) + + captured_calls = {} + + def fake_start_services(service_list, build, force_recreate): + captured_calls["service_list"] = service_list + captured_calls["build"] = build + captured_calls["force_recreate"] = force_recreate + # Capture env vars here while patch.dict is still active + captured_calls["CHRONICLE_REGISTRY"] = os.environ.get("CHRONICLE_REGISTRY") + captured_calls["CHRONICLE_TAG"] = os.environ.get("CHRONICLE_TAG") + + with ( + patch.object( + services_mod, "start_services", side_effect=fake_start_services + ), + patch.object(services_mod, "check_service_configured", return_value=True), + patch.object(services_mod, "ensure_docker_network", return_value=True), + patch.object( + services_mod, "_langfuse_enabled_in_backend", return_value=False + ), + patch.dict(os.environ, env, clear=True), + patch.object(sys, "argv", argv), + ): + services_mod.main() + + return captured_calls + + def test_use_prebuilt_argument_is_accepted(self): + """--use-prebuilt is a valid argument that doesn't cause a parse error.""" + services_mod = _import_services() + parser_ns = services_mod.__dict__ # just ensure no AttributeError below + # Actually drive it through argparse without executing side effects + import argparse + + # Reconstruct the parser by running main under mocks and catching SystemExit + with ( + patch.object(services_mod, "start_services"), + patch.object(services_mod, "check_service_configured", return_value=True), + patch.object(services_mod, "ensure_docker_network", return_value=True), + patch.object( + services_mod, "_langfuse_enabled_in_backend", return_value=False + ), + patch.dict(os.environ, {"DOCKERHUB_USERNAME": "testuser"}, clear=False), + patch.object( + sys, + "argv", + ["services.py", "start", "backend", "--use-prebuilt", "v1.0.0"], + ), + ): + # Should not raise + services_mod.main() + + def test_use_prebuilt_sets_chronicle_registry_env_var(self): + """CHRONICLE_REGISTRY is set to '{user}/' when --use-prebuilt is used.""" + calls = self._run_start( + ["services.py", "start", "--all", "--use-prebuilt", "v1.0.0"], + env_override={"DOCKERHUB_USERNAME": "myuser"}, + ) + assert calls.get("CHRONICLE_REGISTRY") == "myuser/" + + def test_use_prebuilt_sets_chronicle_tag_env_var(self): + """CHRONICLE_TAG is set to the supplied tag when --use-prebuilt is used.""" + calls = self._run_start( + ["services.py", "start", "--all", "--use-prebuilt", "v2.3.4"], + env_override={"DOCKERHUB_USERNAME": "myuser"}, + ) + assert calls.get("CHRONICLE_TAG") == "v2.3.4" + + def test_use_prebuilt_disables_build_flag(self): + """start_services is called with build=False when --use-prebuilt is used.""" + calls = self._run_start( + ["services.py", "start", "--all", "--use-prebuilt", "v1.0.0"], + env_override={"DOCKERHUB_USERNAME": "myuser"}, + ) + assert calls.get("build") is False + + def test_use_prebuilt_without_dockerhub_username_returns_early(self): + """Missing DOCKERHUB_USERNAME with --use-prebuilt exits without calling start_services.""" + services_mod = _import_services() + called = [] + + def fake_start(service_list, build, force_recreate): + called.append(True) + + with ( + patch.object(services_mod, "start_services", side_effect=fake_start), + patch.object(services_mod, "check_service_configured", return_value=True), + patch.object(services_mod, "ensure_docker_network", return_value=True), + patch.object( + services_mod, "_langfuse_enabled_in_backend", return_value=False + ), + patch.dict(os.environ, {}, clear=True), # no DOCKERHUB_USERNAME + patch.object( + sys, + "argv", + ["services.py", "start", "--all", "--use-prebuilt", "v1.0.0"], + ), + ): + services_mod.main() + + assert ( + not called + ), "start_services must not be called when DOCKERHUB_USERNAME is missing" + + def test_build_flag_still_works_without_use_prebuilt(self): + """--build flag still passes build=True to start_services when --use-prebuilt is absent.""" + calls = self._run_start( + ["services.py", "start", "--all", "--build"], + env_override={}, + ) + assert calls.get("build") is True + + def test_normal_start_without_prebuilt_does_not_set_chronicle_vars(self): + """CHRONICLE_REGISTRY and CHRONICLE_TAG are NOT set for a normal start.""" + env = {**os.environ} + env.pop("CHRONICLE_REGISTRY", None) + env.pop("CHRONICLE_TAG", None) + + with patch.dict(os.environ, env, clear=True): + self._run_start(["services.py", "start", "--all"]) + assert "CHRONICLE_REGISTRY" not in os.environ + assert "CHRONICLE_TAG" not in os.environ + + +# =========================================================================== +# 2. docker-compose YAML validation +# =========================================================================== + + +def _load_compose(relative_path: str) -> dict: + path = REPO_ROOT / relative_path + with open(path) as f: + return yaml.safe_load(f) + + +def _image_for(compose: dict, service: str) -> str | None: + return compose.get("services", {}).get(service, {}).get("image") + + +def _has_chronicle_vars(image_str: str | None) -> bool: + """True when the image field uses both CHRONICLE_REGISTRY and CHRONICLE_TAG.""" + if image_str is None: + return False + return "CHRONICLE_REGISTRY" in image_str and "CHRONICLE_TAG" in image_str + + +class TestBackendDockerComposeImages: + COMPOSE = _load_compose("backends/advanced/docker-compose.yml") + + def test_chronicle_backend_has_image_field(self): + assert _has_chronicle_vars(_image_for(self.COMPOSE, "chronicle-backend")) + + def test_workers_has_image_field(self): + assert _has_chronicle_vars(_image_for(self.COMPOSE, "workers")) + + def test_annotation_cron_has_image_field(self): + assert _has_chronicle_vars(_image_for(self.COMPOSE, "annotation-cron")) + + def test_webui_has_image_field(self): + assert _has_chronicle_vars(_image_for(self.COMPOSE, "webui")) + + def test_backend_services_share_same_image_name(self): + """chronicle-backend, workers, and annotation-cron should use the same image.""" + backend_img = _image_for(self.COMPOSE, "chronicle-backend") + workers_img = _image_for(self.COMPOSE, "workers") + cron_img = _image_for(self.COMPOSE, "annotation-cron") + assert backend_img == workers_img == cron_img + + def test_webui_uses_different_image_from_backend(self): + backend_img = _image_for(self.COMPOSE, "chronicle-backend") + webui_img = _image_for(self.COMPOSE, "webui") + assert backend_img != webui_img + + def test_image_names_with_defaults_are_local(self): + """With empty env vars the image names should have no registry prefix.""" + for service in ("chronicle-backend", "webui"): + image = _image_for(self.COMPOSE, service) + # The default expansion of ${CHRONICLE_REGISTRY:-} is "" + # so the name should start with "chronicle-" + assert image is not None + assert "chronicle-" in image + + +class TestSpeakerRecognitionDockerComposeImages: + COMPOSE = _load_compose("extras/speaker-recognition/docker-compose.yml") + + def test_speaker_service_has_chronicle_image(self): + assert _has_chronicle_vars(_image_for(self.COMPOSE, "speaker-service")) + + def test_web_ui_has_chronicle_image(self): + assert _has_chronicle_vars(_image_for(self.COMPOSE, "web-ui")) + + def test_caddy_image_is_not_changed(self): + """Third-party caddy image must stay unchanged (no chronicle vars).""" + caddy_img = _image_for(self.COMPOSE, "caddy") + assert caddy_img is not None + assert "CHRONICLE_REGISTRY" not in (caddy_img or "") + + +class TestAsrServicesDockerComposeImages: + COMPOSE = _load_compose("extras/asr-services/docker-compose.yml") + + @pytest.mark.parametrize( + "service", + [ + "nemo-asr", + "faster-whisper-asr", + "vibevoice-asr", + "transformers-asr", + "qwen3-asr-wrapper", + "qwen3-asr-bridge", + ], + ) + def test_asr_service_has_chronicle_image(self, service): + assert _has_chronicle_vars( + _image_for(self.COMPOSE, service) + ), f"Service '{service}' is missing CHRONICLE_REGISTRY/CHRONICLE_TAG in image: field" + + def test_all_asr_images_are_distinct(self): + """Each ASR service must resolve to a different image name.""" + services = [ + "nemo-asr", + "faster-whisper-asr", + "vibevoice-asr", + "transformers-asr", + "qwen3-asr-wrapper", + "qwen3-asr-bridge", + ] + images = [_image_for(self.COMPOSE, s) for s in services] + assert len(images) == len( + set(images) + ), "ASR service image names must all be unique" + + +class TestHavpeRelayDockerComposeImages: + COMPOSE = _load_compose("extras/havpe-relay/docker-compose.yml") + + def test_havpe_relay_has_chronicle_image(self): + assert _has_chronicle_vars(_image_for(self.COMPOSE, "havpe-relay")) + + +# =========================================================================== +# 3. Bash script input validation +# =========================================================================== + + +class TestPushScriptValidation: + """push-images.sh must reject missing inputs without running docker.""" + + SCRIPT = SCRIPTS_DIR / "push-images.sh" + + def _run(self, args: list[str], env_override: dict | None = None): + env = {**os.environ, **(env_override or {})} + env.pop("DOCKERHUB_USERNAME", None) # start clean + if env_override: + env.update(env_override) + return subprocess.run( + ["bash", str(self.SCRIPT)] + args, + env=env, + capture_output=True, + text=True, + ) + + def test_exits_nonzero_without_dockerhub_username(self): + result = self._run(["v1.0.0"]) + assert result.returncode != 0 + + def test_error_message_mentions_dockerhub_username(self): + result = self._run(["v1.0.0"]) + assert "DOCKERHUB_USERNAME" in result.stderr + + def test_exits_nonzero_without_tag(self): + result = self._run([], env_override={"DOCKERHUB_USERNAME": "testuser"}) + assert result.returncode != 0 + + def test_error_message_mentions_tag_when_tag_missing(self): + result = self._run([], env_override={"DOCKERHUB_USERNAME": "testuser"}) + assert "TAG" in result.stderr + + def test_script_is_executable(self): + assert os.access(self.SCRIPT, os.X_OK), "push-images.sh must be executable" + + +class TestPullScriptValidation: + """pull-images.sh must reject missing inputs without running docker.""" + + SCRIPT = SCRIPTS_DIR / "pull-images.sh" + + def _run(self, args: list[str], env_override: dict | None = None): + env = {**os.environ, **(env_override or {})} + env.pop("DOCKERHUB_USERNAME", None) + if env_override: + env.update(env_override) + return subprocess.run( + ["bash", str(self.SCRIPT)] + args, + env=env, + capture_output=True, + text=True, + ) + + def test_exits_nonzero_without_dockerhub_username(self): + result = self._run(["v1.0.0"]) + assert result.returncode != 0 + + def test_error_message_mentions_dockerhub_username(self): + result = self._run(["v1.0.0"]) + assert "DOCKERHUB_USERNAME" in result.stderr + + def test_exits_nonzero_without_tag(self): + result = self._run([], env_override={"DOCKERHUB_USERNAME": "testuser"}) + assert result.returncode != 0 + + def test_error_message_mentions_tag_when_tag_missing(self): + result = self._run([], env_override={"DOCKERHUB_USERNAME": "testuser"}) + assert "TAG" in result.stderr + + def test_script_is_executable(self): + assert os.access(self.SCRIPT, os.X_OK), "pull-images.sh must be executable" From 07740fcd3ec472917bd7cf592b5d0eab397ed9f2 Mon Sep 17 00:00:00 2001 From: 0xrushi <0xrushi@gmail.com> Date: Fri, 27 Feb 2026 21:42:05 -0500 Subject: [PATCH 3/3] Update Docker Compose configurations to use environment variables for image tags - Modified `docker-compose.yml` files for ASR services and speaker recognition to utilize `${CHRONICLE_REGISTRY}` and `${CHRONICLE_TAG}` for image definitions, enhancing flexibility in image management. - Updated `push-images.sh` script to include the new ASR service images, ensuring they are correctly tagged and pushed to the registry. --- extras/asr-services/docker-compose.yml | 6 +++--- extras/speaker-recognition/docker-compose.yml | 2 +- scripts/push-images.sh | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/extras/asr-services/docker-compose.yml b/extras/asr-services/docker-compose.yml index f59f0eaf..fb2c016a 100644 --- a/extras/asr-services/docker-compose.yml +++ b/extras/asr-services/docker-compose.yml @@ -163,7 +163,7 @@ services: build: context: . dockerfile: providers/qwen3_asr/Dockerfile.vllm - image: chronicle-asr-qwen3-vllm:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-qwen3-vllm:${CHRONICLE_TAG:-latest} ports: - "${ASR_VLLM_PORT:-8768}:8000" volumes: @@ -240,7 +240,7 @@ services: build: context: . dockerfile: providers/vibevoice/Dockerfile.strixhalo - image: chronicle-asr-vibevoice-strixhalo:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-vibevoice-strixhalo:${CHRONICLE_TAG:-latest} ports: - "${ASR_PORT:-8767}:8765" volumes: @@ -285,7 +285,7 @@ services: build: context: . dockerfile: providers/nemo/Dockerfile.strixhalo - image: chronicle-asr-nemo-strixhalo:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-asr-nemo-strixhalo:${CHRONICLE_TAG:-latest} ports: - "${ASR_PORT:-8767}:8765" volumes: diff --git a/extras/speaker-recognition/docker-compose.yml b/extras/speaker-recognition/docker-compose.yml index 756f862e..89cc58c0 100644 --- a/extras/speaker-recognition/docker-compose.yml +++ b/extras/speaker-recognition/docker-compose.yml @@ -67,7 +67,7 @@ services: build: context: . dockerfile: Dockerfile.strixhalo - image: speaker-recognition-strixhalo:latest + image: ${CHRONICLE_REGISTRY:-}chronicle-speaker-strixhalo:${CHRONICLE_TAG:-latest} devices: - /dev/kfd:/dev/kfd - /dev/dri:/dev/dri diff --git a/scripts/push-images.sh b/scripts/push-images.sh index 0cf545bc..f91c2d82 100755 --- a/scripts/push-images.sh +++ b/scripts/push-images.sh @@ -58,6 +58,7 @@ IMAGES=( "chronicle-asr-vibevoice:chronicle-asr-vibevoice" "chronicle-asr-vibevoice-strixhalo:chronicle-asr-vibevoice-strixhalo" "chronicle-asr-transformers:chronicle-asr-transformers" + "chronicle-asr-qwen3-vllm:chronicle-asr-qwen3-vllm" "chronicle-asr-qwen3-wrapper:chronicle-asr-qwen3-wrapper" "chronicle-asr-qwen3-bridge:chronicle-asr-qwen3-bridge" "chronicle-havpe-relay:chronicle-havpe-relay"