diff --git a/.env b/.env deleted file mode 100644 index 88aa79d..0000000 --- a/.env +++ /dev/null @@ -1,11 +0,0 @@ -# MongoDB (root account for initial setup) -MONGO_ROOT_USER=admin -MONGO_ROOT_PASSWORD=admin123 -MONGO_DB=sacred - -# MinIO (S3 + console) -MINIO_ROOT_USER=minioadmin -MINIO_ROOT_PASSWORD=minioadmin - -# Host port for Omniboard (internal port is 9000) -OMNIBOARD_HOST_PORT=9004 \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..84aecf6 --- /dev/null +++ b/.env.example @@ -0,0 +1,21 @@ +# .env.example - Optional environment file for AltarDocker +# +# docker-compose.yml has sensible defaults built-in! +# You only need this .env file if you want to customize settings. +# +# To use: Copy this file to .env and modify the values below +# Without .env: docker-compose.yml uses defaults (sacred, 27017, minio_admin, etc.) + +# MongoDB Configuration +MONGO_DB=sacred +MONGO_PORT=27017 + +# MinIO Configuration (S3-compatible storage) +MINIO_ROOT_USER=minio_admin +MINIO_ROOT_PASSWORD=changeme123 +MINIO_S3_PORT=9000 +MINIO_CONSOLE_PORT=9001 + +# Volume Paths (where data is stored) +MONGO_DATA_PATH=./data/mongo/ +MINIO_DATA_PATH=./data/minio/ diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..d317f2a --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,53 @@ +name: Validate Docker Compose + +on: + push: + branches: [ main, no_credentials ] + pull_request: + branches: [ main ] + +jobs: + validate-compose: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Validate docker-compose.yml syntax + run: | + docker compose -f docker-compose.yml config > /dev/null + echo "✅ docker-compose.yml is valid" + + - name: Check .env.example has all required variables + run: | + echo "Checking required variables in .env.example..." + grep -q "MONGO_DB=" .env.example || (echo "❌ Missing MONGO_DB" && exit 1) + grep -q "MONGO_PORT=" .env.example || (echo "❌ Missing MONGO_PORT" && exit 1) + grep -q "MINIO_ROOT_USER=" .env.example || (echo "❌ Missing MINIO_ROOT_USER" && exit 1) + grep -q "MINIO_ROOT_PASSWORD=" .env.example || (echo "❌ Missing MINIO_ROOT_PASSWORD" && exit 1) + grep -q "MONGO_DATA_PATH=" .env.example || (echo "❌ Missing MONGO_DATA_PATH" && exit 1) + grep -q "MINIO_DATA_PATH=" .env.example || (echo "❌ Missing MINIO_DATA_PATH" && exit 1) + echo "✅ All required variables present" + + - name: Test docker-compose.yml with defaults (no .env) + run: | + # Test without .env file to verify defaults work + docker compose -f docker-compose.yml up --dry-run mongo + echo "✅ docker-compose.yml works with defaults (no .env)" + + - name: Test docker-compose.yml with .env overrides + run: | + cp .env.example .env + # Test all services + docker compose -f docker-compose.yml up --dry-run + echo "✅ docker-compose.yml all services with .env valid" + + - name: Pull Docker images to verify they exist + run: | + docker pull mongo:6 + docker pull quay.io/minio/minio:latest + echo "✅ All Docker images are accessible" diff --git a/DEPLOY.md b/DEPLOY.md index ecfe593..f99c626 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -1,219 +1,81 @@ -# AltarDocker — Local Deployment Guide +# AltarDocker — Deployment Guide -This guide helps you start a minimal stack locally using Docker Compose: +Complete guide for deploying MongoDB and MinIO infrastructure for Sacred experiment tracking. + +**Services included:** - MongoDB for experiment metadata -- MinIO (S3-compatible) for raw data (optional) -- Omniboard connected to MongoDB -- AltarExtractor for browsing Sacred experiments (optional) +- MinIO for S3-compatible object storage + +> **Note:** Omniboard is managed by [AltarViewer](../AltarViewer), AltarExtractor is [deployed separately](../AltarExtractor). --- ## Prerequisites -- Docker >= 24 and Docker Compose v2 installed (https://docs.docker.com/engine/install/) -- At least 2 GB of free disk space (more depending on the size of your data) +- Docker >= 24 and Docker Compose v2 installed +- At least 2 GB free disk space --- -## Project Layout -``` -AltarDocker/ -├─ .env # Environment variables (create this) -├─ docker-compose.yml # Docker Compose configuration -├─ DEPLOY.md # This file -├─ MANAGE_USERS.md # User management guide -└─ mongo_dump.sh # Backup script example -``` +## 1) (Optional) Customize Settings ---- +**The compose file works without a `.env` file!** It has built-in defaults: +- MongoDB: `sacred` database on port `27017` +- MinIO: `minio_admin` / `changeme123` on ports `9000` and `9001` +- Data stored in `./data/mongo/` and `./data/minio/` + +**To customize**, copy `.env.example` to `.env` and modify the values: -## 1) Create the `.env` file +```bash +cp .env.example .env +# Edit .env with your preferred values +``` -Create a `.env` file in this folder with strong passwords: +The `.env.example` file contains: ```dotenv -# MongoDB (root account for initial setup) -MONGO_ROOT_USER=admin -MONGO_ROOT_PASSWORD=change_me_mongo_password +# MongoDB Configuration MONGO_DB=sacred +MONGO_PORT=27017 -# MinIO (S3 + console) +# MinIO Configuration MINIO_ROOT_USER=minio_admin -MINIO_ROOT_PASSWORD=change_me_minio_password - -# Host port for Omniboard (internal port is 9000) -OMNIBOARD_HOST_PORT=9004 +MINIO_ROOT_PASSWORD=your_secure_password +MINIO_S3_PORT=9000 +MINIO_CONSOLE_PORT=9001 -# Host port for AltarExtractor (internal port is 8050) -EXTRACTOR_HOST_PORT=8050 +# Volume Paths (where data is stored) +MONGO_DATA_PATH=./data/mongo/ +MINIO_DATA_PATH=./data/minio/ ``` --- -## 2) Edit `docker-compose.yml` - -The data for MinIO and MongoDB will be saved in the folders specified in `volumes` (outside the Docker container). Update the volume paths to your preferred locations: - -```yaml -services: - mongo: - image: mongo:6 - container_name: mongo - restart: unless-stopped - environment: - MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USER} - MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD} - MONGO_INITDB_DATABASE: ${MONGO_DB} - ports: - - "27017:27017" - volumes: - - /path/to/your/mongo_data:/data/db # Change this path - - minio: - image: quay.io/minio/minio:latest - container_name: minio - restart: unless-stopped - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} - ports: - - "9000:9000" # S3 API - - "9001:9001" # Web console - volumes: - - /path/to/your/minio_data:/data # Change this path - profiles: - - minio - - omniboard: - image: vivekratnavel/omniboard:latest - container_name: omniboard - restart: unless-stopped - depends_on: - - mongo - ports: - - "${OMNIBOARD_HOST_PORT}:9000" - command: [ - "--mu", - "mongodb://${MONGO_ROOT_USER}:${MONGO_ROOT_PASSWORD}@mongo:27017/?authSource=admin&authMechanism=SCRAM-SHA-1", - "${MONGO_DB}" - ] - - extractor: - build: - context: ../AltarExtractor - dockerfile: Dockerfile - container_name: altar-extractor - restart: unless-stopped - depends_on: - - mongo - ports: - - "${EXTRACTOR_HOST_PORT:-8050}:8050" - environment: - - PORT=8050 - - SACRED_DB_NAME=${MONGO_DB} - profiles: - - extractor - -volumes: - mongo_data: - minio_data: -``` - ---- +## 2) Start the Services -## 3) Start the Docker containers - -### Basic stack (MongoDB + Omniboard) ```bash docker compose up -d docker ps ``` -### With MinIO -```bash -docker compose --profile minio up -d -docker ps -``` - -### With AltarExtractor -```bash -docker compose --profile extractor up -d -docker ps -``` - -### With both MinIO and AltarExtractor -```bash -docker compose --profile minio --profile extractor up -d -docker ps -``` - > **Note:** On Linux, you may need to prefix commands with `sudo`. -### URLs +**All services start automatically:** +- MongoDB on port 27017 +- MinIO S3 API on port 9000 +- MinIO Console on port 9001 + +### Access URLs | Service | URL | |----------------|----------------------------------------------------------| -| MongoDB | `mongodb://localhost:27017` (authenticate with root) | +| MongoDB | `mongodb://localhost:27017` | | MinIO S3 API | http://localhost:9000 | | MinIO Console | http://localhost:9001 | -| Omniboard | http://localhost:9004 (or your `OMNIBOARD_HOST_PORT`) | -| AltarExtractor | http://localhost:8050 (or your `EXTRACTOR_HOST_PORT`) | - ---- - -## 4) Connecting AltarExtractor to MongoDB - -When you open AltarExtractor in your browser, you need to configure the MongoDB connection: - -1. Open http://localhost:8050 (or your configured `EXTRACTOR_HOST_PORT`) -2. In the "Database credentials" section, enter: - - **Host**: `mongo` (the Docker service name, not localhost) - - **Port**: `27017` - - **Username**: your `MONGO_ROOT_USER` value - - **Password**: your `MONGO_ROOT_PASSWORD` value - - **Auth source**: `admin` -3. Enter your database name (e.g., `sacred`) -4. Click "Connect" - -> **Tip:** Check "Save credentials" to remember your connection settings in browser local storage. --- -## 5) (Recommended) Create an app-specific MongoDB user -```bash -docker exec -it mongo mongosh -u admin -p your_password --authenticationDatabase admin -``` - -Inside mongosh: -```javascript -use sacred -db.createUser({ - user: "sacred_rw", - pwd: "change_me", - roles: [ { role: "readWrite", db: "sacred" } ] -}) -``` - -Then update Omniboard to use this user (edit `docker-compose.yml`): -```yaml -command: [ - "--mu", - "mongodb://sacred_rw:change_me@mongo:27017/?authSource=sacred&authMechanism=SCRAM-SHA-1", - "sacred" -] -``` - -Restart Omniboard: -```bash -docker compose up -d omniboard -``` - -> **Note:** On Linux, prefix with `sudo` if needed. - ---- - -## 6) Backups (Linux) +## Backups (Linux) ### MongoDB backup script @@ -222,7 +84,7 @@ Edit the `mongo_dump.sh` file: #!/bin/bash # Dumps root repository -BASE_DIR="/home/user/mongodumps/dump" # Change to your backup location +BAS5_DIR="/home/user/mongodumps/dump" # Change to your backup location # Timestamp (e.g., 2025-07-07) DATE_STR=$(date +%Y-%m-%d) @@ -256,7 +118,7 @@ Add: 0 2 * * * bash /path/to/mongo_dump.sh >> /path/to/cron.log 2>&1 ``` ---- +---6 ## 7) Cleanup @@ -270,14 +132,14 @@ docker compose down ## Troubleshooting -- **Omniboard cannot connect?** Check the `--mu` URI (authSource, host, port) and that MongoDB is reachable. -- **Port conflict?** Change `OMNIBOARD_HOST_PORT`, `EXTRACTOR_HOST_PORT`, or MinIO mappings in `docker-compose.yml`. +- **Omniboard cannot connect?** Use [AltarViewer](../AltarViewer) to launch Omniboard properly configured for your database. +- **Port conflict?** Change `MONGO_PORT` or MinIO port mappings in `docker-compose.yml`. - **S3 SDKs:** Use endpoint `http://localhost:9000` with MinIO credentials. -- **AltarExtractor can't connect?** Make sure to use `mongo` as the host (Docker service name), not `localhost`. --- ## Related -- [AltarExtractor](https://github.com/DreamRepo/AltarExtractor) — Browse and filter Sacred experiments in a web UI +- [AltarExtractor](https://github.com/DreamRepo/AltarExtractor) — Browse and filter Sacred experiments (standalone deployment) - [AltarSender](https://github.com/DreamRepo/AltarSender) — GUI to send experiments to Sacred and MinIO +- [AltarViewer](https://github.com/DreamRepo/AltarViewer) — Launch Omniboard configured for your database diff --git a/QUICKSTART-GUI.md b/QUICKSTART-GUI.md new file mode 100644 index 0000000..801aa48 --- /dev/null +++ b/QUICKSTART-GUI.md @@ -0,0 +1,76 @@ +# AltarDocker - Quick Start for Docker Desktop Users + +**No coding or terminal required!** This guide is for users who prefer Docker Desktop's graphical interface. + +## Prerequisites + +1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) +2. Make sure Docker Desktop is running + +## Simple Setup (3 Steps) + +### 1. Download the File + +Download `docker-compose.yml` from the [AltarDocker repository](https://github.com/DreamRepo/AltarDocker) to a folder on your computer. +Copy the full path of this folder. + +![image](https://github.com/DreamRepo/AltarDocker/compose_path.png) + + +### 2. Deploy with Docker Desktop + +**Using Docker Desktop GUI - if you find the option** +1. Open Docker Desktop +2. Click the **"+"** button or **"Import"** in the Containers tab +3. Select your `docker-compose.yml` file +4. Click **"Run"** + +**Using Terminal:** +```bash +docker compose -f path/to/your/folder/docker-compose.yml up -d +``` +![video](https://github.com/DreamRepo/AltarDocker/install_altardocker.mp4) + +### 3. Access Your Services + +- **MinIO Console**: http://localhost:9001 + - Username: `minio_admin`, Password: `changeme123` +- **MongoDB**: `mongodb://localhost:27017` + +That's it! Your services are running with sensible defaults. + +## Want to Customize? + +**Simple method** - Edit the compose file directly: +- Open `docker-compose.yml` in Notepad/VS Code +- Find lines with `:-` (e.g., `:-changeme123`) +- Change the value after `:-` to customize + +**Advanced method** - See [DEPLOY.md](DEPLOY.md) for: +- Using `.env` files +- Changing ports +- Custom data storage paths +- Authentication setup + +## Managing Services + +In Docker Desktop: +- **View status**: Go to "Containers" tab → see `mongo_altar` (and `minio_altar`) +- **Stop**: Click the Stop button +- **Restart**: Click the Start button +- **View logs**: Click the container name + +## Troubleshooting + +**Port conflict?** +- Edit compose file and change `:-27017` to `:-27077` (see customization above) + +**Can't access services?** +- Check containers are "Running" in Docker Desktop +- See [DEPLOY.md](DEPLOY.md) for detailed troubleshooting + +## Next Steps + +- Download [AltarSender](https://github.com/DreamRepo/AltarSender/releases) to upload experiments +- Download [AltarViewer](https://github.com/DreamRepo/AltarViewer/releases) to visualize your data +- Visit [AltarExtractor](https://github.com/DreamRepo/AltarExtractor) to analyze experiments diff --git a/README.md b/README.md index 2598ee9..2f2c6fa 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,72 @@ # AltarDocker -Docker Compose stack for running Sacred ML experiment tracking infrastructure locally. +Docker Compose stack for running Sacred experiment tracking infrastructure locally. ## What's Included | Service | Description | Port | |---------|-------------|------| | **MongoDB** | Stores experiment metadata | 27017 | -| **Omniboard** | Web dashboard for Sacred experiments | 9004 | -| **MinIO** | S3-compatible object storage (optional) | 9000, 9001 | -| **AltarExtractor** | Browse and filter experiments (optional) | 8050 | +| **MinIO** | S3-compatible object storage | 9000, 9001 | -## Quick Start +**Default configuration:** +- MongoDB: `localhost:27017`, database: `sacred` +- MinIO: S3 API `localhost:9000`, Console `localhost:9001` + - Credentials: `minio_admin` / `changeme123` -1. **Create `.env` file** with your credentials: +## Installation + +Choose your guide based on your needs: + +- **[QUICKSTART-GUI.md](QUICKSTART-GUI.md)** — Easy install with Docker Desktop (no command line) +- **[DEPLOY.md](DEPLOY.md)** — Complete deployment guide with customization options +- **[MANAGE_USERS.md](MANAGE_USERS.md)** — MongoDB and MinIO user management + +## Access the Services + +- **MongoDB**: `mongodb://localhost:27017` +- **MinIO Console**: http://localhost:9001 +- **Use with:** + - [AltarSender](../AltarSender) — Upload experiments + - [AltarViewer](../AltarViewer) — View experiments with Omniboard + - [AltarExtractor](../AltarExtractor) — Analyze and export data + + +## For developers: + +### Adding MongoDB Authentication + +By default, MongoDB runs without authentication. To add authentication: + +1. **Add to your `.env` file:** ```dotenv MONGO_ROOT_USER=admin MONGO_ROOT_PASSWORD=your_secure_password - MONGO_DB=sacred - MINIO_ROOT_USER=minio_admin - MINIO_ROOT_PASSWORD=your_minio_password - OMNIBOARD_HOST_PORT=9004 - EXTRACTOR_HOST_PORT=8050 ``` -2. **Start the stack:** +2. **Update the mongo service in `docker-compose.yml`** to use these variables: + ```yaml + mongo: + image: mongo:6 + container_name: mongo_altar + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USER} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD} + MONGO_INITDB_DATABASE: ${MONGO_DB} + ports: + - "${MONGO_PORT}:27017" + volumes: + - ${MONGO_DATA_PATH}:/data/db + ``` + +3. **Recreate the container:** ```bash - # Basic (MongoDB + Omniboard) + docker compose down docker compose up -d - - # With MinIO - docker compose --profile minio up -d - - # With AltarExtractor - docker compose --profile extractor up -d - - # Full stack - docker compose --profile minio --profile extractor up -d ``` -3. **Access the services:** - - Omniboard: http://localhost:9004 - - AltarExtractor: http://localhost:8050 - - MinIO Console: http://localhost:9001 - -## Documentation - -- [DEPLOY.md](DEPLOY.md) — Full deployment guide with detailed configuration -- [MANAGE_USERS.md](MANAGE_USERS.md) — MongoDB and MinIO user management +> **Note:** After enabling authentication, you'll need to update connection strings in AltarSender, AltarViewer, and AltarExtractor to include the username and password. ## Requirements @@ -57,8 +76,9 @@ Docker Compose stack for running Sacred ML experiment tracking infrastructure lo ## Related -- [AltarExtractor](https://github.com/DreamRepo/AltarExtractor) — Browse and filter Sacred experiments in a web UI +- [AltarExtractor](https://github.com/DreamRepo/AltarExtractor) — Browse and filter Sacred experiments (standalone deployment) - [AltarSender](https://github.com/DreamRepo/AltarSender) — GUI to send experiments to Sacred and MinIO +- [AltarViewer](https://github.com/DreamRepo/AltarViewer) — Launch Omniboard configured for your database ## License diff --git a/docker-compose.yml b/docker-compose.yml index 254b123..7e4496d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,65 +1,25 @@ - services: mongo: image: mongo:6 - container_name: mongo + container_name: mongo_altar restart: unless-stopped environment: - MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USER} - MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD} - MONGO_INITDB_DATABASE: ${MONGO_DB} + MONGO_INITDB_DATABASE: ${MONGO_DB:-sacred} ports: - - "27017:27017" + - "${MONGO_PORT:-27017}:27017" volumes: - - C:/Users/WilliamGaultier/Desktop/datadatadata/mongo_data:/data/db + - ${MONGO_DATA_PATH:-./data/mongo}:/data/db minio: image: quay.io/minio/minio:latest - container_name: minio + container_name: minio_altar restart: unless-stopped command: server /data --console-address ":9001" environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio_admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-changeme123} ports: - - "9000:9000" # S3 API - - "9001:9001" # Web console + - "${MINIO_S3_PORT:-9000}:9000" # S3 API + - "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console volumes: - - C:/Users/WilliamGaultier/Desktop/datadatadata/minio_data:/data # Change this path to your desired host directory - profiles: - - minio - - omniboard: - image: vivekratnavel/omniboard:latest - container_name: omniboard - restart: unless-stopped - depends_on: - - mongo - ports: - - "${OMNIBOARD_HOST_PORT}:9000" - command: [ - "--mu", - "mongodb://${MONGO_ROOT_USER}:${MONGO_ROOT_PASSWORD}@mongo:27017/?authSource=admin&authMechanism=SCRAM-SHA-1", - "${MONGO_DB}" - ] - - extractor: - build: - context: ../AltarExtractor - dockerfile: Dockerfile - container_name: altar-extractor - restart: unless-stopped - depends_on: - - mongo - ports: - - "${EXTRACTOR_HOST_PORT:-8050}:8050" - environment: - - PORT=8050 - - SACRED_DB_NAME=${MONGO_DB} - profiles: - - extractor - -volumes: - mongo_data: - minio_data: - + - ${MINIO_DATA_PATH:-./data/minio}:/data diff --git a/images/compose_path.png b/images/compose_path.png new file mode 100644 index 0000000..606365b Binary files /dev/null and b/images/compose_path.png differ diff --git a/images/install_altardocker.mp4 b/images/install_altardocker.mp4 new file mode 100644 index 0000000..c8d9153 Binary files /dev/null and b/images/install_altardocker.mp4 differ