From eb2fd79ca4f0be248ec1fc461e3b6db4791234c2 Mon Sep 17 00:00:00 2001 From: "Andrei V. Lepikhov" Date: Wed, 31 Dec 2025 11:32:55 +0100 Subject: [PATCH 01/10] Improve Spockbench validation to catch silent failures Previously, Spockbench errors would fail silently because: 1. run-tests.sh had 'set -euo pipefail' commented out 2. check-outputs.sh only validated container exit codes 3. PostgreSQL logs were never checked for errors This meant the old log_old_value syntax could have been failing silently without triggering test failures in GitHub Actions. Changes: 1. run-tests.sh: - Uncommented 'set -euo pipefail' to fail fast on errors - Now SQL errors will immediately terminate the script 2. check-outputs.sh: - Added validation that containers actually exist - Added PostgreSQL log error checking - Scans pg logs for ERROR entries (excluding benign "relation does not exist") - Shows container logs on failure for debugging - Provides clear pass/fail feedback This ensures that: - SQL syntax errors are caught immediately - Database errors don't go unnoticed - Test failures are visible in CI/CD - Debugging is easier with automatic log output Now Spockbench will fail until the following commits fix detected issues. --- tests/docker/check-outputs.sh | 27 ++++++++++++++++++++++++++- tests/docker/run-tests.sh | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/docker/check-outputs.sh b/tests/docker/check-outputs.sh index fe2b636b..53b91154 100755 --- a/tests/docker/check-outputs.sh +++ b/tests/docker/check-outputs.sh @@ -4,11 +4,36 @@ set -euo pipefail containers=$(docker ps -aq --filter "label=com.docker.compose.project=tests"); +if [ -z "$containers" ]; then + echo "FAIL: No containers found with label 'com.docker.compose.project=tests'" + exit 1 +fi + for cid in $containers; do + name=$(docker inspect -f '{{ .Name }}' "$cid" | sed 's|/||') exit_code=$(docker inspect -f '{{ .State.ExitCode }}' "$cid") + + # Check exit code if [ "$exit_code" -ne 0 ]; then - name=$(docker inspect -f '{{ .Name }}' "$cid" | sed 's|/||') echo "FAIL: Container '$name' exited with code $exit_code" + docker logs "$cid" 2>&1 | tail -50 exit 1 fi + + # Check PostgreSQL logs for errors + if docker exec "$cid" test -d /home/pgedge/pgedge/data 2>/dev/null; then + pg_log=$(docker exec "$cid" find /home/pgedge/pgedge/data -name "postgresql-*.log" -o -name "*.log" 2>/dev/null | head -1) + if [ -n "$pg_log" ]; then + error_count=$(docker exec "$cid" grep -i "ERROR:" "$pg_log" 2>/dev/null | grep -v "ERROR: relation.*does not exist" | wc -l || echo 0) + if [ "$error_count" -gt 0 ]; then + echo "FAIL: Container '$name' has $error_count ERROR(s) in PostgreSQL logs" + docker exec "$cid" grep -i "ERROR:" "$pg_log" | grep -v "ERROR: relation.*does not exist" | tail -20 + exit 1 + fi + fi + fi + + echo "✓ Container '$name' passed all checks" done + +echo "✓ All containers passed validation" diff --git a/tests/docker/run-tests.sh b/tests/docker/run-tests.sh index 7fa13a08..dff0f5e1 100755 --- a/tests/docker/run-tests.sh +++ b/tests/docker/run-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -#set -euo pipefail +set -euo pipefail peer_names=$1 From 63f00eb2b6765521e3b74307d041b1007bd7595a Mon Sep 17 00:00:00 2001 From: "Andrei V. Lepikhov" Date: Wed, 31 Dec 2025 14:41:51 +0100 Subject: [PATCH 02/10] 1 --- .github/workflows/spockbench.yml | 22 +++++++++++----------- tests/docker/entrypoint.sh | 1 - 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/spockbench.yml b/.github/workflows/spockbench.yml index cc0876a3..58528cb0 100644 --- a/.github/workflows/spockbench.yml +++ b/.github/workflows/spockbench.yml @@ -55,6 +55,17 @@ jobs: --build-arg PGVER=${{ matrix.pgver }} \ -t spock -f tests/docker/Dockerfile-step-1.el9 . + - name: Start docker + run: | + cd ${GITHUB_WORKSPACE}/tests/docker/ + echo PG_VER=${{ matrix.pgver }} >> pgedge.env + docker compose up + + - name: Check spockbench output + run: | + cd ${GITHUB_WORKSPACE}/tests/docker + ./check-outputs.sh + - name: Run regression tests run: | REG_CT_NAME="spock-regress-${{ matrix.pgver }}-${{ github.run_id }}-${{ github.run_attempt }}" @@ -104,14 +115,3 @@ jobs: if-no-files-found: ignore retention-days: 7 - - name: Start docker - run: | - cd ${GITHUB_WORKSPACE}/tests/docker/ - echo PG_VER=${{ matrix.pgver }} >> pgedge.env - docker compose up - - - name: Check spockbench output - run: | - cd ${GITHUB_WORKSPACE}/tests/docker - ./check-outputs.sh - diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index e446d2aa..32795d3d 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -16,7 +16,6 @@ function wait_for_pg() done } -. /home/pgedge/pgedge/pg$PGVER/pg$PGVER.env . /home/pgedge/.bashrc echo "==========Installing Spockbench==========" From cab64e12477758c605a26481b31cd178b44ebd8a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 31 Dec 2025 13:52:07 +0000 Subject: [PATCH 03/10] Add Spockbench artifact collection Collect postgresql.conf, logfile, and docker logs from all containers (n1, n2, n3) after Spockbench completes. This enables debugging of test failures by providing access to PostgreSQL configuration and logs. Artifacts are collected even if tests fail (if: always()) and work with stopped containers. They are uploaded with 7-day retention for each PostgreSQL version. --- .github/workflows/spockbench.yml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/spockbench.yml b/.github/workflows/spockbench.yml index 58528cb0..d0d9dc02 100644 --- a/.github/workflows/spockbench.yml +++ b/.github/workflows/spockbench.yml @@ -66,6 +66,50 @@ jobs: cd ${GITHUB_WORKSPACE}/tests/docker ./check-outputs.sh + - name: Collect Spockbench artifacts (logs and configs) + if: ${{ always() }} + run: | + mkdir -p ${GITHUB_WORKSPACE}/spockbench-artifacts + cd ${GITHUB_WORKSPACE}/tests/docker/ + + # Collect from all containers (even stopped ones) + for container in n1 n2 n3; do + echo "Collecting artifacts from $container" + + # Copy PostgreSQL logfile + docker cp "$container":/home/pgedge/pgedge/data/pg${{ matrix.pgver }}/logfile \ + "${GITHUB_WORKSPACE}/spockbench-artifacts/${container}-logfile.log" 2>/dev/null || \ + echo "Warning: No logfile in $container" + + # Copy postgresql.conf + docker cp "$container":/home/pgedge/pgedge/data/pg${{ matrix.pgver }}/postgresql.conf \ + "${GITHUB_WORKSPACE}/spockbench-artifacts/${container}-postgresql.conf" 2>/dev/null || \ + echo "Warning: No postgresql.conf in $container" + + # Copy docker container logs + docker logs "$container" > "${GITHUB_WORKSPACE}/spockbench-artifacts/${container}-docker.log" 2>&1 || \ + echo "Warning: Could not get docker logs for $container" + done + + # List what we collected + echo "Collected artifacts:" + ls -lah ${GITHUB_WORKSPACE}/spockbench-artifacts/ || echo "No artifacts collected" + + - name: Upload Spockbench artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: spockbench-${{ matrix.pgver }} + path: spockbench-artifacts/** + if-no-files-found: ignore + retention-days: 7 + + - name: Stop Spockbench containers + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE}/tests/docker/ + docker compose down -v || true + - name: Run regression tests run: | REG_CT_NAME="spock-regress-${{ matrix.pgver }}-${{ github.run_id }}-${{ github.run_attempt }}" From f55ba26ce27056de9a759f801a39fa8d1414a01f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 31 Dec 2025 14:00:19 +0000 Subject: [PATCH 04/10] Add docker compose build step and fix build context Added 'docker compose build' before 'docker compose up' to ensure the image is built with the correct configuration. Also fixed docker-compose.yml build context from '.' to '../..' so it points to the repository root, matching what the Dockerfile expects (COPY . /home/pgedge/spock needs repo root as context). This ensures all three containers (n1, n2, n3) are built with the correct source code and configuration. --- .github/workflows/spockbench.yml | 1 + tests/docker/docker-compose.yml | 10 ++++++++-- tests/docker/entrypoint.sh | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spockbench.yml b/.github/workflows/spockbench.yml index d0d9dc02..c59e2b7d 100644 --- a/.github/workflows/spockbench.yml +++ b/.github/workflows/spockbench.yml @@ -59,6 +59,7 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/tests/docker/ echo PG_VER=${{ matrix.pgver }} >> pgedge.env + docker compose build --build-arg PGVER=${{ matrix.pgver }} docker compose up - name: Check spockbench output diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index 0881a226..235057e5 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -4,8 +4,8 @@ services: hostname: n1 image: spock build: - context: . - dockerfile: Dockerfile-step-1.el9 + context: ../.. + dockerfile: tests/docker/Dockerfile-step-1.el9 environment: - HOSTNAME=n1 - PEER_NAMES=n2,n3 @@ -23,6 +23,9 @@ services: container_name: n2 hostname: n2 image: spock + build: + context: ../.. + dockerfile: tests/docker/Dockerfile-step-1.el9 environment: - HOSTNAME=n2 - PEER_NAMES=n1,n3 @@ -40,6 +43,9 @@ services: container_name: n3 hostname: n3 image: spock + build: + context: ../.. + dockerfile: tests/docker/Dockerfile-step-1.el9 environment: - HOSTNAME=n3 - PEER_NAMES=n1,n2 diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index 32795d3d..cbf867b0 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -23,6 +23,7 @@ cd ~/spockbench sudo python3 setup.py install cd ~/pgedge + sed -i '/log_min_messages/s/^#//g' data/pg$PGVER/postgresql.conf sed -i -e '/log_min_messages =/ s/= .*/= debug1/' data/pg$PGVER/postgresql.conf ./pgedge restart From d74de8525c675c30b4fa025ca06d7bae85af9f96 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 31 Dec 2025 15:44:01 +0000 Subject: [PATCH 05/10] Add comprehensive PostgreSQL initialization in entrypoint.sh Replace pgedge commands with direct PostgreSQL initialization using settings from regress-postgresql.conf. This ensures proper pgdata creation and configuration when the Docker image is used with the default entrypoint. Changes: - Use initdb directly instead of pgedge for initialization - Configure postgresql.conf with all required Spock settings: * shared_preload_libraries, wal_level, max_wal_senders * max_replication_slots, max_worker_processes * track_commit_timestamp, max_locks_per_transaction - Configure pg_hba.conf for network replication access - Create database user and database - Use pg_ctl for start/restart instead of pgedge commands This initialization only runs when entrypoint.sh is used (spockbench via docker-compose). Regression and TAP tests override the entrypoint so they are unaffected. --- tests/docker/entrypoint.sh | 71 +++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index cbf867b0..49356178 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -24,18 +24,79 @@ sudo python3 setup.py install cd ~/pgedge -sed -i '/log_min_messages/s/^#//g' data/pg$PGVER/postgresql.conf -sed -i -e '/log_min_messages =/ s/= .*/= debug1/' data/pg$PGVER/postgresql.conf -./pgedge restart +# Initialize PostgreSQL if not already done +if [ ! -d "data/pg$PGVER" ]; then + echo "==========Initializing PostgreSQL $PGVER==========" -wait_for_pg + # Initialize the database cluster + pg${PGVER}/bin/initdb -D data/pg${PGVER} --encoding=UTF8 --locale=C + + # Configure PostgreSQL with settings from regress-postgresql.conf + cat >> data/pg${PGVER}/postgresql.conf <> data/pg${PGVER}/pg_hba.conf </dev/null || true + pg${PGVER}/bin/createdb -h /tmp -O $DBUSER $DBNAME 2>/dev/null || true + + echo "==========PostgreSQL $PGVER initialized successfully==========" +else + # PostgreSQL already initialized, just start it + pg${PGVER}/bin/pg_ctl -D data/pg${PGVER} -l data/pg${PGVER}/logfile start + wait_for_pg +fi + +# Ensure environment variables are set +export DBUSER=${DBUSER:-pgedge} +export DBNAME=${DBNAME:-demo} psql -h /tmp -U $DBUSER -d $DBNAME -c "drop extension spock;" psql -h /tmp -U $DBUSER -d $DBNAME -c "drop schema public cascade;" psql -h /tmp -U $DBUSER -d $DBNAME -c "create schema public;" psql -h /tmp -U $DBUSER -d $DBNAME -c "create extension spock;" -./pgedge restart +# Restart PostgreSQL to apply all settings +pg${PGVER}/bin/pg_ctl -D data/pg${PGVER} restart -l data/pg${PGVER}/logfile wait_for_pg From 272608486926e3223e59246d092d2ac789c7d23b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 31 Dec 2025 15:51:28 +0000 Subject: [PATCH 06/10] Add local Spockbench test runner script Created run-spockbench-local.sh to run Spockbench tests locally without the full CI workflow (no regression/TAP tests). Features: - Takes PostgreSQL version as argument (defaults to 17) - Builds Docker image and containers - Runs docker compose up to execute tests - Validates output with check-outputs.sh - Collects artifacts (logs, configs) to local directory - Cleans up containers after completion Usage: ./tests/docker/run-spockbench-local.sh [PGVER] Example: ./tests/docker/run-spockbench-local.sh 18 --- tests/docker/run-spockbench-local.sh | 118 +++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100755 tests/docker/run-spockbench-local.sh diff --git a/tests/docker/run-spockbench-local.sh b/tests/docker/run-spockbench-local.sh new file mode 100755 index 00000000..b9fb549f --- /dev/null +++ b/tests/docker/run-spockbench-local.sh @@ -0,0 +1,118 @@ +#!/bin/bash +set -e + +# Local Spockbench test runner +# Usage: ./run-spockbench-local.sh [PGVER] +# Example: ./run-spockbench-local.sh 17 + +PGVER=${1:-17} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +echo "==========================================" +echo "Running Spockbench for PostgreSQL ${PGVER}" +echo "==========================================" + +# Set environment variables +export GITHUB_WORKSPACE="${REPO_ROOT}" +export PG_VER="${PGVER}" + +# Clean up any previous runs +echo "" +echo "==> Cleaning up previous runs..." +cd "${SCRIPT_DIR}" +docker compose down -v 2>/dev/null || true + +# Build the Docker image +echo "" +echo "==> Building Docker image..." +cd "${REPO_ROOT}" +docker build \ + --build-arg PGVER=${PGVER} \ + -t spock \ + -f tests/docker/Dockerfile-step-1.el9 \ + . + +# Prepare environment file +echo "" +echo "==> Preparing environment..." +cd "${SCRIPT_DIR}" +echo "PG_VER=${PGVER}" > pgedge.env + +# Build with docker compose +echo "" +echo "==> Building with docker compose..." +docker compose build --build-arg PGVER=${PGVER} + +# Run Spockbench +echo "" +echo "==> Starting Spockbench containers..." +echo "This will run the tests. Watch the output below." +echo "==========================================" +docker compose up + +# Check for completion +COMPOSE_EXIT_CODE=$? + +echo "" +echo "==========================================" +if [ $COMPOSE_EXIT_CODE -eq 0 ]; then + echo "==> Checking test outputs..." + ./check-outputs.sh + CHECK_EXIT_CODE=$? + + if [ $CHECK_EXIT_CODE -eq 0 ]; then + echo "✓ Spockbench tests PASSED" + else + echo "✗ Spockbench tests FAILED (check-outputs.sh returned $CHECK_EXIT_CODE)" + fi +else + echo "✗ Docker compose exited with code $COMPOSE_EXIT_CODE" + CHECK_EXIT_CODE=$COMPOSE_EXIT_CODE +fi + +# Collect artifacts +echo "" +echo "==> Collecting artifacts..." +ARTIFACT_DIR="${REPO_ROOT}/spockbench-artifacts-local" +mkdir -p "${ARTIFACT_DIR}" + +for container in n1 n2 n3; do + echo "Collecting from $container..." + + # Copy PostgreSQL logfile + docker cp "$container:/home/pgedge/pgedge/data/pg${PGVER}/logfile" \ + "${ARTIFACT_DIR}/${container}-logfile.log" 2>/dev/null && \ + echo " ✓ Logfile saved" || \ + echo " ✗ No logfile found" + + # Copy postgresql.conf + docker cp "$container:/home/pgedge/pgedge/data/pg${PGVER}/postgresql.conf" \ + "${ARTIFACT_DIR}/${container}-postgresql.conf" 2>/dev/null && \ + echo " ✓ postgresql.conf saved" || \ + echo " ✗ No postgresql.conf found" + + # Copy docker container logs + docker logs "$container" > "${ARTIFACT_DIR}/${container}-docker.log" 2>&1 && \ + echo " ✓ Docker logs saved" || \ + echo " ✗ Could not get docker logs" +done + +echo "" +echo "Artifacts saved to: ${ARTIFACT_DIR}" +ls -lh "${ARTIFACT_DIR}/" + +# Cleanup +echo "" +echo "==> Cleaning up containers..." +docker compose down -v || true + +echo "" +echo "==========================================" +if [ $CHECK_EXIT_CODE -eq 0 ]; then + echo "✓ Spockbench completed successfully!" + exit 0 +else + echo "✗ Spockbench failed. Check artifacts in: ${ARTIFACT_DIR}" + exit 1 +fi From 22a114ad27cf12e6a4f5f175f81e8c919d4b58c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 31 Dec 2025 16:01:05 +0000 Subject: [PATCH 07/10] Fix spock extension drop error on first run Changed 'drop extension spock' to 'drop extension if exists spock' to avoid error when the extension hasn't been created yet during initial setup. During first run: - PostgreSQL is initialized without the spock extension - Then we try to drop it (which fails) - Then we create it This fix allows the drop to succeed even if the extension doesn't exist yet. --- tests/docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index 49356178..a7475389 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -90,7 +90,7 @@ fi export DBUSER=${DBUSER:-pgedge} export DBNAME=${DBNAME:-demo} -psql -h /tmp -U $DBUSER -d $DBNAME -c "drop extension spock;" +psql -h /tmp -U $DBUSER -d $DBNAME -c "drop extension if exists spock;" psql -h /tmp -U $DBUSER -d $DBNAME -c "drop schema public cascade;" psql -h /tmp -U $DBUSER -d $DBNAME -c "create schema public;" psql -h /tmp -U $DBUSER -d $DBNAME -c "create extension spock;" From 5348366855070cd1a5b5f4138885ebc7e6a32584 Mon Sep 17 00:00:00 2001 From: "Andrei V. Lepikhov" Date: Wed, 31 Dec 2025 18:16:47 +0100 Subject: [PATCH 08/10] 1 --- tests/docker/entrypoint.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index a7475389..85b2895f 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -90,10 +90,9 @@ fi export DBUSER=${DBUSER:-pgedge} export DBNAME=${DBNAME:-demo} -psql -h /tmp -U $DBUSER -d $DBNAME -c "drop extension if exists spock;" -psql -h /tmp -U $DBUSER -d $DBNAME -c "drop schema public cascade;" -psql -h /tmp -U $DBUSER -d $DBNAME -c "create schema public;" -psql -h /tmp -U $DBUSER -d $DBNAME -c "create extension spock;" +# This code executes on a fresh system that means we have a clean Postgres +# instance. +psql -h /tmp -U $DBUSER -d $DBNAME -c "CREATE EXTENSION spock;" # Restart PostgreSQL to apply all settings pg${PGVER}/bin/pg_ctl -D data/pg${PGVER} restart -l data/pg${PGVER}/logfile @@ -101,10 +100,10 @@ pg${PGVER}/bin/pg_ctl -D data/pg${PGVER} restart -l data/pg${PGVER}/logfile wait_for_pg echo "==========Assert Spock version is the latest==========" -expected_line=$(grep '#define SPOCK_VERSION' /home/pgedge/spock/spock.h) +expected_line=$(grep '#define SPOCK_VERSION' /home/pgedge/spock/include/spock.h) expected_version=$(echo "$expected_line" | grep -oP '"\K[0-9]+\.[0-9]+\.[0-9]+') expected_major=${expected_version%%.*} -actual_version=$(psql -U $DBUSER -d $DBNAME -X -t -A -c "select spock.spock_version()") +actual_version=$(psql -U $DBUSER -d $DBNAME -X -t -A -c "SELECT spock.spock_version()") actual_major=${actual_version%%.*} if (( actual_major >= expected_major )); then From 095b01385779cb2f208d7bfb9514ed8a8d83d2dd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 31 Dec 2025 17:34:49 +0000 Subject: [PATCH 09/10] Add SPOCK_SOURCE_DIR environment variable Defined SPOCK_SOURCE_DIR=/home/pgedge/spock as an environment variable instead of hardcoding the path throughout the codebase. Changes: - Added ENV SPOCK_SOURCE_DIR in Dockerfile-step-1.el9 - Updated COPY, RUN, and WORKDIR to use ${SPOCK_SOURCE_DIR} - Updated entrypoint.sh to use ${SPOCK_SOURCE_DIR} when accessing spock.h Benefits: - Single source of truth for Spock source location - Easier to change path if needed - More maintainable code --- tests/docker/Dockerfile-step-1.el9 | 9 +++++---- tests/docker/entrypoint.sh | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/docker/Dockerfile-step-1.el9 b/tests/docker/Dockerfile-step-1.el9 index 060ef0ad..c597af3b 100644 --- a/tests/docker/Dockerfile-step-1.el9 +++ b/tests/docker/Dockerfile-step-1.el9 @@ -10,10 +10,11 @@ ENV PGVER=$PGVER ENV PATH="/home/pgedge/pgedge/pg${PGVER}/bin:${PATH}" ENV LD_LIBRARY_PATH="/home/pgedge/pgedge/pg${PGVER}/lib:${LD_LIBRARY_PATH}" ENV PG_CONFIG="/home/pgedge/pgedge/pg${PGVER}/bin/pg_config" +ENV SPOCK_SOURCE_DIR="/home/pgedge/spock" # Copy spock source code and set proper ownership -COPY . /home/pgedge/spock -RUN chown -R pgedge:pgedge /home/pgedge/spock +COPY . ${SPOCK_SOURCE_DIR} +RUN chown -R pgedge:pgedge ${SPOCK_SOURCE_DIR} WORKDIR /home/pgedge @@ -37,7 +38,7 @@ RUN echo "Setting up pgedge..." && \ WORKDIR /home/pgedge/postgres -RUN for patchfile in /home/pgedge/spock/patches/${PGVER}/*; do \ +RUN for patchfile in ${SPOCK_SOURCE_DIR}/patches/${PGVER}/*; do \ patch -p1 --verbose < $patchfile; \ done @@ -72,7 +73,7 @@ RUN echo "==========Compiling Modified PostgreSQL==========" && \ make -C contrib install # Compile Spock -WORKDIR /home/pgedge/spock +WORKDIR ${SPOCK_SOURCE_DIR} RUN echo "==========Compiling Spock==========" && \ make clean && \ make -j${MAKE_JOBS} && \ diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index 85b2895f..40422d7e 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -100,7 +100,7 @@ pg${PGVER}/bin/pg_ctl -D data/pg${PGVER} restart -l data/pg${PGVER}/logfile wait_for_pg echo "==========Assert Spock version is the latest==========" -expected_line=$(grep '#define SPOCK_VERSION' /home/pgedge/spock/include/spock.h) +expected_line=$(grep '#define SPOCK_VERSION' ${SPOCK_SOURCE_DIR}/include/spock.h) expected_version=$(echo "$expected_line" | grep -oP '"\K[0-9]+\.[0-9]+\.[0-9]+') expected_major=${expected_version%%.*} actual_version=$(psql -U $DBUSER -d $DBNAME -X -t -A -c "SELECT spock.spock_version()") From fc8f0a474866d1d3efe76bda658909053971f182 Mon Sep 17 00:00:00 2001 From: "Andrei V. Lepikhov" Date: Tue, 6 Jan 2026 12:14:22 +0100 Subject: [PATCH 10/10] 2 --- .github/workflows/spockbench.yml | 7 +++ tests/docker/Dockerfile-step-1.el9 | 68 ++++-------------------------- tests/docker/README.md | 44 ++++++++++++++++++- 3 files changed, 58 insertions(+), 61 deletions(-) diff --git a/.github/workflows/spockbench.yml b/.github/workflows/spockbench.yml index c59e2b7d..842ddf64 100644 --- a/.github/workflows/spockbench.yml +++ b/.github/workflows/spockbench.yml @@ -54,13 +54,20 @@ jobs: docker build \ --build-arg PGVER=${{ matrix.pgver }} \ -t spock -f tests/docker/Dockerfile-step-1.el9 . + ls /home/pgedge/ + ls /home/pgedge/pgedge + ls /home/pgedge/pgedge/pg${{ matrix.pgver }} - name: Start docker run: | cd ${GITHUB_WORKSPACE}/tests/docker/ + # XXX: Do we really need PG_VER variable and pgedge.env here? echo PG_VER=${{ matrix.pgver }} >> pgedge.env docker compose build --build-arg PGVER=${{ matrix.pgver }} docker compose up + ls /home/pgedge/ + ls /home/pgedge/pgedge + ls /home/pgedge/pgedge/pg${{ matrix.pgver }} - name: Check spockbench output run: | diff --git a/tests/docker/Dockerfile-step-1.el9 b/tests/docker/Dockerfile-step-1.el9 index c597af3b..7564cd27 100644 --- a/tests/docker/Dockerfile-step-1.el9 +++ b/tests/docker/Dockerfile-step-1.el9 @@ -1,5 +1,8 @@ FROM ghcr.io/pgedge/base-test-image:latest +# Base image already has Rocky Linux, PostgreSQL, and pgedge installed +# We only need to compile Spock extension against the existing PostgreSQL + # Base image ends as USER pgedge, but we need root for installation USER root @@ -7,73 +10,18 @@ ARG PGVER ARG MAKE_JOBS=4 ENV PGVER=$PGVER + ENV PATH="/home/pgedge/pgedge/pg${PGVER}/bin:${PATH}" ENV LD_LIBRARY_PATH="/home/pgedge/pgedge/pg${PGVER}/lib:${LD_LIBRARY_PATH}" ENV PG_CONFIG="/home/pgedge/pgedge/pg${PGVER}/bin/pg_config" ENV SPOCK_SOURCE_DIR="/home/pgedge/spock" # Copy spock source code and set proper ownership -COPY . ${SPOCK_SOURCE_DIR} -RUN chown -R pgedge:pgedge ${SPOCK_SOURCE_DIR} - -WORKDIR /home/pgedge - -# Determine PostgreSQL version and clone repository -RUN LATEST_TAG=$(git ls-remote --tags https://github.com/postgres/postgres.git | \ - grep "refs/tags/REL_${PGVER}_" | \ - sed 's|.*refs/tags/||' | \ - tr '_' '.' | \ - sort -V | \ - tail -n 1 | \ - tr '.' '_') && \ - echo "Using PostgreSQL tag: $LATEST_TAG" && \ - git clone --branch $LATEST_TAG --depth 1 https://github.com/postgres/postgres /home/pgedge/postgres && \ - chmod -R a+w /home/pgedge/postgres - -# Install pgedge (run as pgedge user, not root) -RUN echo "Setting up pgedge..." && \ - curl -fsSL https://pgedge-download.s3.amazonaws.com/REPO/install.py -o /home/pgedge/install.py && \ - chown pgedge:pgedge /home/pgedge/install.py && \ - su - pgedge -c "python3 /home/pgedge/install.py" - -WORKDIR /home/pgedge/postgres - -RUN for patchfile in ${SPOCK_SOURCE_DIR}/patches/${PGVER}/*; do \ - patch -p1 --verbose < $patchfile; \ - done - -# Compile PostgreSQL -RUN echo "==========Compiling Modified PostgreSQL==========" && \ - ./configure \ - --prefix="/home/pgedge/pgedge/pg${PGVER}" \ - --disable-rpath \ - --with-zstd \ - --with-lz4 \ - --with-icu \ - --with-libxslt \ - --with-libxml \ - --with-uuid=ossp \ - --with-gssapi \ - --with-ldap \ - --with-pam \ - --enable-debug \ - --enable-dtrace \ - --with-llvm \ - --with-openssl \ - --with-systemd \ - --enable-tap-tests \ - --with-python \ - --enable-cassert \ - PYTHON=/usr/bin/python3.9 \ - BITCODE_CFLAGS="-gdwarf-5 -O0 -fforce-dwarf-frame" \ - CFLAGS="-g -O0" && \ - make -j${MAKE_JOBS} && \ - make -C contrib -j${MAKE_JOBS} && \ - make install && \ - make -C contrib install +COPY . /home/pgedge/spock +RUN chown -R pgedge:pgedge /home/pgedge/spock -# Compile Spock -WORKDIR ${SPOCK_SOURCE_DIR} +# Compile Spock against the pre-installed PostgreSQL +WORKDIR /home/pgedge/spock RUN echo "==========Compiling Spock==========" && \ make clean && \ make -j${MAKE_JOBS} && \ diff --git a/tests/docker/README.md b/tests/docker/README.md index 0702691e..4b7709bc 100644 --- a/tests/docker/README.md +++ b/tests/docker/README.md @@ -1 +1,43 @@ -# see [pgedge-docker repo](https://github.com/pgedge/pgedge-docker) repo for container guidance +# Docker infrastructure for the Spock testing + +This document describes how the Docker infrastructure is organized, how to use it for +testing locally and as a part of a GitHub Actions workflow. + +## Structure + +The Spock Docker environment provides one `stable` image cached by the URL ghcr.io/pgedge/base-test-image:latest and a `floating` one, created on the fly during the tests. + +The `stable` image is created according to the `Dockerfile-base.el9` file and should be built manually by the `cache-base-image.yml` workflow. It contains a Rocky Linux image with a complete set of libraries necessary for the Postgres build and passing regression and TAP tests. + +The `floating` image is defined by the `Dockerfile-step-1.el9` file and is intended to be used in tests. + +### Main parameters of the `stable` image +* Base OS: Rocky Linux 9 +* Default user: `pgedge` (with sudo privileges, password: `asdf`) +* Current working directory: `/home/pgedge` +* SSH keys: id_ed25519 and corresponding id_ed25519.pub in the folder `~/.ssh` are provided (authorized for passwordless SSH as pgedge user) +* Includes complete PostgreSQL build dependencies (LLVM, ICU, SSL, etc.) +* Testing tools: Perl Test::More, SSH server/client configured + +### Main parameters of the `floating` image +* Based on the `stable` image +* Build argument: `PGVER` - PostgreSQL version to build against +* Build argument: `MAKE_JOBS=4` - number of parallel make jobs (default: 4) +* Default user: `pgedge` +* Defined environment variables: + * `PGVER` - PostgreSQL version + * `PATH` - includes `/home/pgedge/pgedge/pg${PGVER}/bin` + * `LD_LIBRARY_PATH` - includes `/home/pgedge/pgedge/pg${PGVER}/lib` + * `PG_CONFIG` - points to `/home/pgedge/pgedge/pg${PGVER}/bin/pg_config` + * `SPOCK_SOURCE_DIR` - set to `/home/pgedge/spock` +* Includes compiled Spock extension installed against PostgreSQL +* Test scripts from `tests/docker/*.sh` are available in `/home/pgedge/` +* Default entrypoint: `/home/pgedge/entrypoint.sh` + +## Notes + +* If you need an extra package, add it to the `Dockerfile-base.el9` file, updating the library list. Afterwards, re-run the workflow to rebuild the image. + +## References + +* See the [pgedge-docker repo](https://github.com/pgedge/pgedge-docker) for container guidance