diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 527015042..178741404 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,7 +50,14 @@ jobs: run: | SKIP_BUILD=1 just docker-serve prod -d sleep 5 - just docker-smoke-test || { docker logs docker_prod_1; exit 1; } + just docker-smoke-test || { + echo "Smoke test failed, attempting to show logs for all containers:" + for c in $(docker ps -a --format '{{.Names}}'); do + echo "Logs for $c:"; + docker logs "$c" || true; + done + exit 1 + } - name: Publish docker image run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b9efa4c47..fdc183e6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,7 +72,14 @@ jobs: run: | just docker-serve prod -d sleep 5 - just docker-smoke-test || { docker logs docker_prod_1; exit 1; } + just docker-smoke-test || { + echo "Smoke test failed, attempting to show logs for all containers:" + for c in $(docker ps -a --format '{{.Names}}'); do + echo "Logs for $c:"; + docker logs "$c" || true; + done + exit 1 + } - name: Save docker image run: | diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..584ab1a40 --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +release: /usr/bin/env bash /app/deploy/release.sh +web: gunicorn --config /app/deploy/gunicorn/conf.py opencodelists.wsgi diff --git a/deploy/release.sh b/deploy/release.sh new file mode 100755 index 000000000..757d4fd78 --- /dev/null +++ b/deploy/release.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# Gets executed during dokku's release phase as specified in Procfile. +# This is where dokku recommends running db migrations. + +set -euo pipefail + +./manage.py check --deploy +./manage.py migrate diff --git a/docker/Dockerfile b/docker/Dockerfile index 93e84a98f..a0818bb55 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -171,8 +171,6 @@ COPY . /app RUN TRUD_API_KEY=dummy-key SECRET_KEY=dummy-key \ python /app/manage.py collectstatic --no-input -ENTRYPOINT ["/app/docker/entrypoints/prod.sh"] - # We set command rather than entrypoint, to make it easier to run different # things from the cli CMD ["gunicorn", "--config", "/app/deploy/gunicorn/conf.py", "opencodelists.wsgi"] diff --git a/docker/entrypoints/prod.sh b/docker/entrypoints/prod.sh deleted file mode 100755 index f330899f5..000000000 --- a/docker/entrypoints/prod.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -./manage.py check --deploy -./manage.py migrate - -exec "$@" diff --git a/docker/justfile b/docker/justfile index 6579e56f5..60aea37a1 100644 --- a/docker/justfile +++ b/docker/justfile @@ -65,8 +65,8 @@ _create_storage env: SCRIPT_DIR={{justfile_directory()}} if ! test -d "$SCRIPT_DIR/storage"; then mkdir -p "$SCRIPT_DIR/storage" - sudo chown 10003:10003 "$SCRIPT_DIR/storage" fi + sudo chown 10003:10003 "$SCRIPT_DIR/storage" fi @@ -86,7 +86,10 @@ exec env="dev" *args="bash": # run a basic functional smoke test against a running opencodelists -smoke-test host="http://localhost:7000": +smoke-test host="http://localhost:7000" env="prod": #!/bin/bash set -eu + echo "Running release script (check and migrations)..." + docker compose exec {{ env }} bash /app/deploy/release.sh + echo "Testing service connectivity..." curl -I {{ host }} -s --compressed --fail --retry 20 --retry-delay 1 --retry-all-errors diff --git a/docs/adr/0005-migrations-during-dokku-in-release-phase.md b/docs/adr/0005-migrations-during-dokku-in-release-phase.md new file mode 100644 index 000000000..80b56c0d3 --- /dev/null +++ b/docs/adr/0005-migrations-during-dokku-in-release-phase.md @@ -0,0 +1,62 @@ +# ADR: Run Django Migrations in Dokku Release Phase + +Date: 2025-07 + +## Status + +Accepted + +## Context + +Previously, we had a `prod.sh` script that ran the database migrations and then called `exec "$@"` in order to execute whatever command was passed as arguments to the script. The Dockerfile was + +```docker +ENTRYPOINT ["/app/docker/entrypoints/prod.sh"] +CMD ["gunicorn", "--config", "/app/deploy/gunicorn/conf.py", "opencodelists.wsgi"] +``` + +Meaning that when the docker container was run, without additional arguments, this command would execute: + +```bash +prod.sh gunicorn --config /app/deploy/gunicorn/conf.py opencodelists.wsgi +``` +Causing the migrations to run, and gunicorn to start. You could also pass a command to `docker run` which would then execute (instead of gunicorn) after the migrations. + +With this setup, migrations were run every time the container started, including during the web process startup and health checks. This approach had caused issues, including an outage to job-server where migrations were not fully applied before the health checks timed out. + +[Dokku recommend](https://dokku.com/docs/advanced-usage/deployment-tasks/) running database migrations in the `release` phase, which is executed before the new web process is started. This ensures migrations are applied atomically and safely, and that the web process only starts after the database schema is up to date. + +## Decision + +- **Create** a `release.sh` script that runs the migrations and checks: + ```bash + #!/bin/bash + set -e + + ./manage.py check --deploy + ./manage.py migrate + ``` +- **Move** the migration and check logic to the `release` phase in the `Procfile` and add the `web` command: + ``` + release: /usr/bin/env bash /app/deploy/release.sh + web: gunicorn --config /app/deploy/gunicorn/conf.py opencodelists.wsgi + ``` +- **Remove** the `ENTRYPOINT` from the production Docker image. +- **Delete** the now-obsolete `prod.sh` entrypoint script. +- **Keep** the `CMD` in the Dockerfile so `docker run` will still start Gunicorn by default, but can be overriden + +## Consequences + +- Migrations are now run in the correct place (Dokku's release phase), before the web process starts. +- Deploys are safer and more predictable, with no risk of the health checks failing because they start at the same time as the container starts. +- No scripts or documentation referenced or used the old `prod.sh` entrypoint, so no further changes were needed following its deletion. +- The dev docker image overwrites the ENTRYPOINT and so is unaffected by its removal +- The `CMD` in the Dockerfile remains, allowing for overriding of the command when running the container. + + +## Changes + +- `docker/Dockerfile`: Removed ENTRYPOINT, kept CMD for Gunicorn. +- `docker/entrypoints/prod.sh`: Deleted. +- `deploy/release.sh`: Created for running migrations and checks. +- `Procfile`: Added release phase for migrations and checks, and web phase for Gunicorn. diff --git a/justfile b/justfile index bcae3e0b5..e3e89b4c3 100644 --- a/justfile +++ b/justfile @@ -342,8 +342,8 @@ docker-exec *args="bash": _env # run tests in docker container -docker-smoke-test host="http://localhost:7000": _env - {{ just_executable() }} docker/smoke-test {{ host }} +docker-smoke-test host="http://localhost:7000" env="prod": _env + {{ just_executable() }} docker/smoke-test {{ host }} {{env}} # check migrations in the dev docker container