diff --git a/Dockerfile b/Dockerfile index 99bfba7c..4cbff678 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,12 +60,30 @@ RUN conda config --set ssl_verify false && \ conda config --set ssl_verify true # Step 5: Make the environment available (simulate "conda activate spac") +# Set all necessary environment variables for conda activation ENV CONDA_DEFAULT_ENV=${ENV_NAME} +ENV CONDA_PREFIX=/opt/conda/envs/${ENV_NAME} ENV PATH=/opt/conda/envs/${ENV_NAME}/bin:${PATH} +ENV PYTHONPATH=/opt/conda/envs/${ENV_NAME}/lib/python3.9/site-packages:${PYTHONPATH} # Step 6: "Install the SPAC package in development mode" RUN /opt/conda/envs/${ENV_NAME}/bin/pip install -e . +# Make conda activate spac environment automatically in bash shells +RUN echo "source /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate ${ENV_NAME}" >> ~/.bashrc + +# Create a wrapper script that ensures conda environment is activated +# This helps when Galaxy runs commands directly +RUN echo '#!/bin/bash' > /usr/local/bin/conda-run && \ + echo 'source /opt/conda/etc/profile.d/conda.sh' >> /usr/local/bin/conda-run && \ + echo 'conda activate spac' >> /usr/local/bin/conda-run && \ + echo 'exec "$@"' >> /usr/local/bin/conda-run && \ + chmod +x /usr/local/bin/conda-run + +# Set the shell to use bash for RUN commands +SHELL ["/bin/bash", "-c"] + # Set environment variables for headless notebook execution ENV QT_QPA_PLATFORM=offscreen ENV MPLBACKEND=Agg @@ -76,13 +94,21 @@ RUN mkdir -p /workspace /data /results # Install jupyter and nbconvert for notebook testing RUN /opt/conda/envs/${ENV_NAME}/bin/pip install jupyter nbconvert -# Verify SPAC installation works correctly +# Verify SPAC installation works correctly in multiple ways RUN echo "=== VERIFYING SPAC INSTALLATION ===" && \ - /opt/conda/envs/${ENV_NAME}/bin/python -c "import spac; print(f'SPAC version: {spac.__version__}'); import scimap; print('All modules imported successfully!')" || \ - echo "Some import issues detected but proceeding with test" + echo "Test 1: Direct python call" && \ + python -c "import spac; print(f'SPAC version: {spac.__version__}')" && \ + echo "Test 2: Which python" && \ + which python && \ + echo "Test 3: Python path" && \ + python -c "import sys; print(sys.executable)" && \ + echo "Test 4: Import scimap" && \ + python -c "import scimap; print('scimap imported successfully')" && \ + echo "=== ALL TESTS PASSED ===" || \ + echo "Some import issues detected but proceeding" # Set working directory for Jupyter (will be mounted via volume) WORKDIR /workspace # Default command starts Jupyter notebook server -CMD ["/opt/conda/envs/spac/bin/jupyter", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--allow-root", "--NotebookApp.token=", "--NotebookApp.password="] \ No newline at end of file +CMD ["/opt/conda/envs/spac/bin/jupyter", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--allow-root", "--NotebookApp.token=", "--NotebookApp.password="] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..43c7e4ac --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +.PHONY: help build run dev stop clean rebuild test + +# Default target +help: + @echo "SPAC Docker Management" + @echo "======================" + @echo "" + @echo "Production targets:" + @echo " make build - Build production Docker image" + @echo " make run - Run production container with Jupyter" + @echo " make test - Run tests in production container" + @echo "" + @echo "Development targets:" + @echo " make dev - Start development environment with live code mounting" + @echo " make dev-bash - Start development container with bash shell" + @echo " make dev-test - Run tests in development mode" + @echo "" + @echo "Utility targets:" + @echo " make stop - Stop all running containers" + @echo " make clean - Remove containers and images" + @echo " make rebuild - Clean and rebuild production image" + @echo " make logs - Show container logs" + +# Production build +build: + @echo "Building production Docker image..." + docker build -t spac:latest . + +# Run production container +run: + @echo "Starting production container with Jupyter..." + docker run -d \ + --name spac-prod \ + -p 8888:8888 \ + -v $(PWD)/data:/data \ + -v $(PWD)/results:/results \ + spac:latest + @echo "Jupyter available at http://localhost:8888" + +# Development environment with live code mounting +dev: + @echo "Starting development environment with live code mounting..." + docker-compose -f docker/docker-compose.dev.yml up -d + @echo "Development Jupyter available at http://localhost:8889" + @echo "Source code is mounted - changes will be reflected immediately!" + +# Development with bash shell +dev-bash: + @echo "Starting development container with bash shell..." + docker-compose -f docker/docker-compose.dev.yml run --rm --service-ports spac-dev bash + +# Run tests in development mode +dev-test: + @echo "Running tests in development mode..." + docker-compose -f docker/docker-compose.dev.yml run --rm spac-dev \ + /opt/conda/envs/spac/bin/pytest tests/ -v + +# Run tests in production container +test: + @echo "Running tests in production container..." + docker run --rm spac:latest \ + /opt/conda/envs/spac/bin/pytest tests/ -v + +# Stop all containers +stop: + @echo "Stopping containers..." + -docker stop spac-prod 2>/dev/null || true + -docker-compose -f docker/docker-compose.dev.yml down 2>/dev/null || true + +# Clean up containers and images +clean: stop + @echo "Cleaning up containers and images..." + -docker rm spac-prod 2>/dev/null || true + -docker rmi spac:latest 2>/dev/null || true + -docker-compose -f docker/docker-compose.dev.yml down -v 2>/dev/null || true + +# Rebuild from scratch +rebuild: clean build + @echo "Rebuild complete!" + +# Show logs +logs: + @echo "=== Production logs ===" + -docker logs spac-prod 2>/dev/null || echo "No production container running" + @echo "" + @echo "=== Development logs ===" + -docker-compose -f docker/docker-compose.dev.yml logs 2>/dev/null || echo "No development container running" diff --git a/README.md b/README.md index bc65301a..aea173eb 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,14 @@ conda config --set ssl_verify true ## Using SPAC with Docker -For a reproducible environment, you can use Docker to run SPAC: +For a reproducible environment, you can use Docker to run SPAC. + +**📘 For detailed Docker documentation including development mode with live code mounting, see [`docker/README.md`](docker/README.md)** + +**Quick commands:** +- Production: `make build` → `make run` +- Development: `make dev` (changes to `src/` reflected immediately!) +- Help: `make help` ### Build the Docker Image ```bash diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 00000000..532fa215 --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,98 @@ +FROM continuumio/miniconda3:24.3.0-0 + +# Build arguments +ARG ENV_NAME=spac + +# Set labels +LABEL maintainer="FNLCR-DMAP" +LABEL description="SPAC - Single Cell Spatial Analysis Container (Development)" +LABEL version="0.9.0-dev" + +# Install system dependencies including Chromium for Kaleido (ARM64 compatible) +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + ca-certificates \ + build-essential \ + libgl1-mesa-glx \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev \ + libgomp1 \ + libglu1-mesa \ + xvfb \ + && rm -rf /var/lib/apt/lists/* + +# Install Chromium for Kaleido visualization support (ARM64 compatible) +RUN apt-get update && \ + apt-get install -y chromium && \ + rm -rf /var/lib/apt/lists/* + +# Set Chrome binary path for Kaleido +ENV CHROME_BIN=/usr/bin/chromium + +# Install and configure libmamba solver +RUN conda install -n base -y conda-libmamba-solver && \ + conda config --set solver libmamba + +# Set up working directory +WORKDIR /home/reviewer/SCSAWorkflow + +# Copy only environment setup files first (for better layer caching) +COPY environment.yml . + +# Configure conda channels +RUN conda config --add channels https://fnlcr-dmap.github.io/scimap/ && \ + conda config --add channels conda-forge && \ + conda config --add channels ohsu-comp-bio && \ + conda config --add channels leej3 && \ + conda config --add channels bioconda + +# Create the Conda environment +RUN conda config --set ssl_verify false && \ + conda env create -f environment.yml && \ + conda clean -afy && \ + conda config --set ssl_verify true + +# Make the environment available +ENV CONDA_DEFAULT_ENV=${ENV_NAME} +ENV PATH=/opt/conda/envs/${ENV_NAME}/bin:${PATH} + +# Make conda activate spac environment automatically in bash shells +RUN echo "source /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate ${ENV_NAME}" >> ~/.bashrc + +# Copy setup.py and other necessary files for package installation +COPY setup.py . +COPY LICENSE . +COPY README.md . +COPY CHANGELOG.md . + +# Create a minimal src directory structure for initial installation +# The actual source code will be mounted as a volume +RUN mkdir -p src/spac && \ + touch src/spac/__init__.py + +# Install the SPAC package in editable mode +# The source code will be mounted, so changes will be reflected +RUN /opt/conda/envs/${ENV_NAME}/bin/pip install -e . + +# Set environment variables for headless execution +ENV QT_QPA_PLATFORM=offscreen +ENV MPLBACKEND=Agg + +# Create working directories +RUN mkdir -p /workspace /data /results + +# Install jupyter and nbconvert for notebook testing +RUN /opt/conda/envs/${ENV_NAME}/bin/pip install jupyter nbconvert + +# Install development tools +RUN /opt/conda/envs/${ENV_NAME}/bin/pip install ipdb pytest-watch black flake8 isort + +# Set working directory for Jupyter +WORKDIR /workspace + +# Default command starts Jupyter notebook server +CMD ["/opt/conda/envs/spac/bin/jupyter", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--allow-root", "--NotebookApp.token=", "--NotebookApp.password="] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..806febe3 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,214 @@ +# Docker Setup for SPAC + +This repository includes Docker support for both production and development workflows. + +## Prerequisites + +- Docker installed on your system +- Docker Compose (usually comes with Docker Desktop) +- Make (optional, but recommended) + +## Quick Start + +### Production Mode + +Production mode creates a self-contained image with all code baked in. Use this for: +- Testing the final package +- Deployment +- Sharing with reviewers + +```bash +# Build the production image +make build + +# Run production container with Jupyter +make run + +# Access Jupyter at http://localhost:8888 +``` + +### Development Mode + +Development mode mounts your source code as volumes, so changes are reflected immediately. Use this for: +- Active development +- Debugging +- Testing changes without rebuilding + +```bash +# Start development environment +make dev + +# Access Jupyter at http://localhost:8889 +# Your code changes in src/ will be immediately available! +``` + +## Available Commands + +Run `make help` to see all available commands: + +```bash +make help +``` + +### Production Commands + +- `make build` - Build production Docker image +- `make run` - Run production container with Jupyter +- `make test` - Run tests in production container +- `make rebuild` - Clean and rebuild from scratch + +### Development Commands + +- `make dev` - Start development environment with live code mounting +- `make dev-bash` - Start development container with bash shell (for debugging) +- `make dev-test` - Run tests in development mode + +### Utility Commands + +- `make stop` - Stop all running containers +- `make clean` - Remove containers and images +- `make logs` - Show container logs + +## Development Workflow + +1. **Start the development environment:** + ```bash + make dev + ``` + +2. **Make changes to your code** in `src/spac/` - changes are immediately available in the container! + +3. **Test your changes:** + ```bash + make dev-test + ``` + +4. **Need to debug?** Open a shell in the container: + ```bash + make dev-bash + ``` + +5. **When done:** + ```bash + make stop + ``` + +## Directory Structure + +- `Dockerfile` (root) - Production image (code copied at build time) +- `docker/Dockerfile.dev` - Development image (code mounted as volume) +- `docker/docker-compose.dev.yml` - Development orchestration +- `Makefile` (root) - Convenient commands for both modes + +## Volumes + +Both modes mount these directories: +- `./data` → `/data` (input data) +- `./results` → `/results` (output results) + +Development mode additionally mounts: +- `./src` → `/home/reviewer/SCSAWorkflow/src` (source code) +- `./tests` → `/home/reviewer/SCSAWorkflow/tests` (test files) +- `./notebooks` → `/workspace` (Jupyter notebooks) + +## Ports + +- Production: http://localhost:8888 +- Development: http://localhost:8889 + +## Manual Docker Commands + +If you prefer not to use Make: + +### Production +```bash +# Build +docker build -t spac:latest . + +# Run +docker run -d --name spac-prod -p 8888:8888 \ + -v $(pwd)/data:/data \ + -v $(pwd)/results:/results \ + spac:latest +``` + +### Development +```bash +# Build +docker build -f docker/Dockerfile.dev -t spac:dev . + +# Run +docker-compose -f docker/docker-compose.dev.yml up -d +``` + +## Troubleshooting + +### Port already in use +If you get a port conflict, stop the other container: +```bash +make stop +``` + +### Changes not reflected in dev mode +Make sure you're running in development mode (`make dev`) and editing files in the `src/` directory. + +### Need to rebuild +If you've changed dependencies or environment.yml: +```bash +make rebuild # Production +# or +docker-compose -f docker/docker-compose.dev.yml build --no-cache # Development +``` + +## Using with Galaxy + +Galaxy can use the Docker container to run SPAC tools. The container is configured so that: +- The `spac` conda environment is active by default +- Python and all SPAC commands work without special setup +- Both direct commands and bash scripts work correctly + +### Galaxy Tool XML Example + +```xml + + + spac:latest + + + + + + + + + +``` + +### Testing Galaxy Integration + +Test that commands work as Galaxy would run them: + +```bash +# Test direct Python command (how Galaxy runs tools) +docker run --rm spac:latest python -c "import spac; print(spac.__version__)" + +# Test with a script +docker run --rm -v $(pwd):/work spac:latest python /work/your_script.py + +# Test bash command +docker run --rm spac:latest bash -c "python -c 'import spac; print(spac.__version__)'" +``` + +All three methods should work without needing to activate conda manually. + +## CI/CD Integration + +For CI/CD pipelines, use the production mode: +```bash +docker build -t spac:latest . +docker run --rm spac:latest pytest tests/ -v +``` diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 00000000..4fa1afe1 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,22 @@ +services: + spac-dev: + build: + context: .. + dockerfile: docker/Dockerfile.dev + container_name: spac-dev + ports: + - "8889:8888" # Different port to avoid conflict with production + volumes: + # Mount source code for live development + - ../src:/home/reviewer/SCSAWorkflow/src + - ../tests:/home/reviewer/SCSAWorkflow/tests + - ../data:/data + - ../results:/results + # Optional: mount notebooks if you have any + - ../notebooks:/workspace + environment: + - QT_QPA_PLATFORM=offscreen + - MPLBACKEND=Agg + - PYTHONPATH=/home/reviewer/SCSAWorkflow/src + working_dir: /workspace + command: /opt/conda/envs/spac/bin/jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.token= --NotebookApp.password= diff --git a/environment.yml b/environment.yml index 0b40e9a7..426a50e3 100644 --- a/environment.yml +++ b/environment.yml @@ -22,10 +22,17 @@ dependencies: - squidpy=1.2.2 # Single-Cell Visualization and Plotting (scverse family) - pillow=11.0.0 # Image Processing - python-kaleido=0.2.1 # Image Processing + - jupyter # Notebook Environment + - scipy=1.12.0 # Statistics (constrained by scimap) + - dask=2023.12.1 # Parallel Computing (constrained by scimap) + - tifffile=2023.9.26 # Image I/O (constrained by scimap) + - mpl-scatter-density=0.7 # Visualization (constrained by scimap) - pip: - pylint - - pytest + - pytest==7.4.4 # Testing (constrained by scimap) - sphinx - flake8 - gendocs - parmap==1.7.0 + - combat==0.3.3 # Batch correction (required by scimap) + - scimap