From 62e8c02d59233b9b4e4c07c419f9b88256186758 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 22 Jan 2026 02:24:17 +0000 Subject: [PATCH 1/4] feat: Add seed.pgpm() adapter and pgpm integration tests - Add PgpmSeedAdapter that calls pgpm CLI via subprocess - Pass database connection info via PGHOST/PGPORT/PGDATABASE/PGUSER/PGPASSWORD env vars - Add pre-scaffolded pgpm workspace fixture with test-module - Add test_pgpm_integration.py with tests for pgpm deploy and @pgpm/faker - Add test-pgpm.yml workflow that: - Installs pgpm CLI globally - Installs @pgpm/faker in the test fixture - Runs pgpm admin-users bootstrap - Runs pgpm integration tests This demonstrates Option 1: using pgpm as a CLI tool from Python to run database migrations without rewriting pgpm in Python. --- .github/workflows/test-pgpm.yml | 89 +++++++++++ src/pysql_test/seed/__init__.py | 3 + src/pysql_test/seed/pgpm.py | 140 ++++++++++++++++++ .../test-module/deploy/schemas/test_app.sql | 7 + .../packages/test-module/package.json | 7 + .../packages/test-module/pgpm.plan | 5 + .../test-module/revert/schemas/test_app.sql | 7 + .../packages/test-module/test-module.control | 7 + .../test-module/verify/schemas/test_app.sql | 7 + tests/fixtures/pgpm-workspace/pgpm.json | 5 + tests/test_pgpm_integration.py | 96 ++++++++++++ 11 files changed, 373 insertions(+) create mode 100644 .github/workflows/test-pgpm.yml create mode 100644 src/pysql_test/seed/pgpm.py create mode 100644 tests/fixtures/pgpm-workspace/packages/test-module/deploy/schemas/test_app.sql create mode 100644 tests/fixtures/pgpm-workspace/packages/test-module/package.json create mode 100644 tests/fixtures/pgpm-workspace/packages/test-module/pgpm.plan create mode 100644 tests/fixtures/pgpm-workspace/packages/test-module/revert/schemas/test_app.sql create mode 100644 tests/fixtures/pgpm-workspace/packages/test-module/test-module.control create mode 100644 tests/fixtures/pgpm-workspace/packages/test-module/verify/schemas/test_app.sql create mode 100644 tests/fixtures/pgpm-workspace/pgpm.json create mode 100644 tests/test_pgpm_integration.py diff --git a/.github/workflows/test-pgpm.yml b/.github/workflows/test-pgpm.yml new file mode 100644 index 0000000..9d362a5 --- /dev/null +++ b/.github/workflows/test-pgpm.yml @@ -0,0 +1,89 @@ +name: pgpm Integration Tests + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-pgpm-tests + cancel-in-progress: true + +env: + PGPM_VERSION: '2.7.9' + +jobs: + test-pgpm: + runs-on: ubuntu-latest + + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: password + + services: + pg_db: + image: ghcr.io/constructive-io/docker/postgres-plus:17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Cache pgpm CLI + uses: actions/cache@v4 + with: + path: ~/.npm + key: pgpm-${{ runner.os }}-${{ env.PGPM_VERSION }} + + - name: Install pgpm CLI globally + run: npm install -g pgpm@${{ env.PGPM_VERSION }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: latest + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Install Python dependencies + run: poetry install + + - name: Install @pgpm/faker in test fixture + run: | + cd tests/fixtures/pgpm-workspace/packages/test-module + pgpm install @pgpm/faker + + - name: Seed pg and app_user + run: | + pgpm admin-users bootstrap --yes + pgpm admin-users add --test --yes + + - name: Run pgpm integration tests + run: poetry run pytest tests/test_pgpm_integration.py -v diff --git a/src/pysql_test/seed/__init__.py b/src/pysql_test/seed/__init__.py index 56d1482..97b70f4 100644 --- a/src/pysql_test/seed/__init__.py +++ b/src/pysql_test/seed/__init__.py @@ -5,13 +5,16 @@ - sqlfile: Execute raw SQL files - fn: Run custom Python functions - compose: Combine multiple adapters +- pgpm: Run pgpm migrations (requires pgpm CLI) """ from pysql_test.seed.adapters import compose, fn +from pysql_test.seed.pgpm import pgpm from pysql_test.seed.sql import sqlfile __all__ = [ "sqlfile", "fn", "compose", + "pgpm", ] diff --git a/src/pysql_test/seed/pgpm.py b/src/pysql_test/seed/pgpm.py new file mode 100644 index 0000000..645574f --- /dev/null +++ b/src/pysql_test/seed/pgpm.py @@ -0,0 +1,140 @@ +""" +pgpm seed adapter for pysql-test. + +Provides integration with pgpm (PostgreSQL Package Manager) for running +database migrations as part of test seeding. + +Requires pgpm to be installed globally: npm install -g pgpm +""" + +from __future__ import annotations + +import logging +import os +import subprocess +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pysql_test.types import SeedContext + +logger = logging.getLogger(__name__) + + +class PgpmSeedAdapter: + """ + Seed adapter that runs pgpm deploy to apply migrations. + + This adapter calls the pgpm CLI via subprocess, passing the database + connection info via environment variables. + + Usage: + adapter = PgpmSeedAdapter(module_path="./my-module") + adapter.seed(ctx) + """ + + def __init__( + self, + module_path: str | None = None, + deploy_args: list[str] | None = None, + cache: bool = False, + ) -> None: + """ + Initialize the pgpm seed adapter. + + Args: + module_path: Path to the pgpm module directory (defaults to cwd) + deploy_args: Additional arguments to pass to pgpm deploy + cache: Whether to enable caching (not yet implemented) + """ + self._module_path = module_path + self._deploy_args = deploy_args or [] + self._cache = cache + + def seed(self, ctx: SeedContext) -> None: + """ + Run pgpm deploy to apply migrations. + + Args: + ctx: Seed context containing pg client and config + + Raises: + RuntimeError: If pgpm deploy fails + """ + config = ctx["config"] + + # Build environment with database connection info + env = os.environ.copy() + env["PGHOST"] = config.get("host", "localhost") + env["PGPORT"] = str(config.get("port", 5432)) + env["PGDATABASE"] = config["database"] + env["PGUSER"] = config.get("user", "postgres") + if "password" in config: + env["PGPASSWORD"] = config["password"] + + # Determine working directory + cwd = self._module_path or os.getcwd() + + # Build pgpm deploy command + cmd = ["pgpm", "deploy", "--yes"] + cmd.extend(self._deploy_args) + + logger.info(f"Running pgpm deploy in {cwd}") + logger.debug(f"Command: {' '.join(cmd)}") + logger.debug(f"Database: {config['database']}") + + try: + result = subprocess.run( + cmd, + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=False, + ) + + if result.returncode != 0: + error_msg = result.stderr or result.stdout or "Unknown error" + logger.error(f"pgpm deploy failed: {error_msg}") + raise RuntimeError(f"pgpm deploy failed: {error_msg}") + + logger.info("pgpm deploy completed successfully") + if result.stdout: + logger.debug(f"pgpm output: {result.stdout}") + + except FileNotFoundError as err: + raise RuntimeError( + "pgpm not found. Install it with: npm install -g pgpm" + ) from err + + +def pgpm( + module_path: str | None = None, + deploy_args: list[str] | None = None, + cache: bool = False, +) -> PgpmSeedAdapter: + """ + Create a pgpm seed adapter. + + This adapter runs pgpm deploy to apply database migrations as part of + test seeding. Requires pgpm to be installed globally. + + Args: + module_path: Path to the pgpm module directory (defaults to cwd) + deploy_args: Additional arguments to pass to pgpm deploy + cache: Whether to enable caching + + Returns: + A PgpmSeedAdapter instance + + Example: + # Deploy migrations from a specific module + seed_adapters = [ + seed.pgpm(module_path="./packages/my-module") + ] + + # Deploy with additional arguments + seed_adapters = [ + seed.pgpm(module_path="./my-module", deploy_args=["--verbose"]) + ] + """ + return PgpmSeedAdapter(module_path=module_path, deploy_args=deploy_args, cache=cache) diff --git a/tests/fixtures/pgpm-workspace/packages/test-module/deploy/schemas/test_app.sql b/tests/fixtures/pgpm-workspace/packages/test-module/deploy/schemas/test_app.sql new file mode 100644 index 0000000..8796001 --- /dev/null +++ b/tests/fixtures/pgpm-workspace/packages/test-module/deploy/schemas/test_app.sql @@ -0,0 +1,7 @@ +-- Deploy test-module:schemas/test_app to pg + +BEGIN; + +CREATE SCHEMA test_app; + +COMMIT; diff --git a/tests/fixtures/pgpm-workspace/packages/test-module/package.json b/tests/fixtures/pgpm-workspace/packages/test-module/package.json new file mode 100644 index 0000000..f9a2012 --- /dev/null +++ b/tests/fixtures/pgpm-workspace/packages/test-module/package.json @@ -0,0 +1,7 @@ +{ + "name": "test-module", + "version": "0.0.1", + "description": "Test module for pysql-test pgpm integration", + "author": "pysql-test", + "license": "MIT" +} diff --git a/tests/fixtures/pgpm-workspace/packages/test-module/pgpm.plan b/tests/fixtures/pgpm-workspace/packages/test-module/pgpm.plan new file mode 100644 index 0000000..a1bc5fe --- /dev/null +++ b/tests/fixtures/pgpm-workspace/packages/test-module/pgpm.plan @@ -0,0 +1,5 @@ +%syntax-version=1.0.0 +%project=test-module +%uri=test-module + +schemas/test_app 2026-01-22T00:00:00Z pysql-test # add test_app schema diff --git a/tests/fixtures/pgpm-workspace/packages/test-module/revert/schemas/test_app.sql b/tests/fixtures/pgpm-workspace/packages/test-module/revert/schemas/test_app.sql new file mode 100644 index 0000000..7672dfc --- /dev/null +++ b/tests/fixtures/pgpm-workspace/packages/test-module/revert/schemas/test_app.sql @@ -0,0 +1,7 @@ +-- Revert test-module:schemas/test_app from pg + +BEGIN; + +DROP SCHEMA IF EXISTS test_app CASCADE; + +COMMIT; diff --git a/tests/fixtures/pgpm-workspace/packages/test-module/test-module.control b/tests/fixtures/pgpm-workspace/packages/test-module/test-module.control new file mode 100644 index 0000000..6abbbd9 --- /dev/null +++ b/tests/fixtures/pgpm-workspace/packages/test-module/test-module.control @@ -0,0 +1,7 @@ +# test-module extension +comment = 'test-module extension for pysql-test' +default_version = '0.0.1' +module_pathname = '$libdir/test-module' +requires = 'plpgsql' +relocatable = false +superuser = false diff --git a/tests/fixtures/pgpm-workspace/packages/test-module/verify/schemas/test_app.sql b/tests/fixtures/pgpm-workspace/packages/test-module/verify/schemas/test_app.sql new file mode 100644 index 0000000..8936cfc --- /dev/null +++ b/tests/fixtures/pgpm-workspace/packages/test-module/verify/schemas/test_app.sql @@ -0,0 +1,7 @@ +-- Verify test-module:schemas/test_app on pg + +BEGIN; + +SELECT pg_catalog.has_schema_privilege('test_app', 'usage'); + +ROLLBACK; diff --git a/tests/fixtures/pgpm-workspace/pgpm.json b/tests/fixtures/pgpm-workspace/pgpm.json new file mode 100644 index 0000000..e251a6b --- /dev/null +++ b/tests/fixtures/pgpm-workspace/pgpm.json @@ -0,0 +1,5 @@ +{ + "packages": [ + "packages/*" + ] +} diff --git a/tests/test_pgpm_integration.py b/tests/test_pgpm_integration.py new file mode 100644 index 0000000..30d7cc1 --- /dev/null +++ b/tests/test_pgpm_integration.py @@ -0,0 +1,96 @@ +""" +pgpm integration tests for pysql-test. + +These tests demonstrate using seed.pgpm() to run pgpm migrations +as part of test database seeding. + +Requires: +- pgpm CLI installed: npm install -g pgpm +- @pgpm/faker installed in the test fixture: pgpm install @pgpm/faker +""" + +from pathlib import Path + +import pytest + +from pysql_test import get_connections, seed + +# Path to the pre-scaffolded pgpm workspace fixture +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "pgpm-workspace" / "packages" / "test-module" + + +@pytest.fixture +def pgpm_db(): + """ + Create an isolated test database with pgpm migrations applied. + + This fixture uses seed.pgpm() to run pgpm deploy, which applies + all migrations from the test-module fixture. + """ + conn = get_connections( + seed_adapters=[ + seed.pgpm(module_path=str(FIXTURE_PATH)) + ] + ) + db = conn.db + db.before_each() + yield db + db.after_each() + conn.teardown() + + +def test_pgpm_creates_schema(pgpm_db): + """Test that pgpm deploy creates the test_app schema.""" + result = pgpm_db.one(""" + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name = 'test_app' + """) + assert result["schema_name"] == "test_app" + + +def test_pgpm_faker_available(pgpm_db): + """ + Test that @pgpm/faker is available after pgpm deploy. + + This test verifies that pgpm install @pgpm/faker was run + and the faker schema/functions are available. + """ + # Check if faker schema exists (installed via pgpm install @pgpm/faker) + result = pgpm_db.one_or_none(""" + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name = 'faker' + """) + + if result is None: + pytest.skip("@pgpm/faker not installed - run: cd tests/fixtures/pgpm-workspace/packages/test-module && pgpm install @pgpm/faker") + + assert result["schema_name"] == "faker" + + +def test_pgpm_faker_city_function(pgpm_db): + """ + Test that faker.city() function works. + + This demonstrates the full pgpm integration flow: + 1. pysql-test creates isolated database + 2. seed.pgpm() runs pgpm deploy + 3. pgpm deploys test-module + @pgpm/faker dependency + 4. Test can use faker functions + """ + # Check if faker schema exists first + schema_exists = pgpm_db.one_or_none(""" + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name = 'faker' + """) + + if schema_exists is None: + pytest.skip("@pgpm/faker not installed - run: cd tests/fixtures/pgpm-workspace/packages/test-module && pgpm install @pgpm/faker") + + # Test faker.city() function with Michigan state code + result = pgpm_db.one("SELECT faker.city('MI') as city") + assert result["city"] is not None + assert isinstance(result["city"], str) + assert len(result["city"]) > 0 From 7cda6a3ed1f78cb14e763611d3b941ed2bc94cc8 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 22 Jan 2026 02:31:20 +0000 Subject: [PATCH 2/4] fix: Skip pgpm tests in regular workflow and add verbose logging - Exclude tests/test_pgpm_integration.py from regular test workflow (pgpm CLI not installed in that workflow) - Add --verbose flag to pgpm deploy for better debugging - Log stdout and stderr from pgpm deploy for troubleshooting --- .github/workflows/test.yml | 2 +- src/pysql_test/seed/pgpm.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f6e70fa..6a00415 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,4 +65,4 @@ jobs: run: poetry run mypy src --ignore-missing-imports - name: Run tests - run: poetry run pytest -v + run: poetry run pytest -v --ignore=tests/test_pgpm_integration.py diff --git a/src/pysql_test/seed/pgpm.py b/src/pysql_test/seed/pgpm.py index 645574f..8c5c295 100644 --- a/src/pysql_test/seed/pgpm.py +++ b/src/pysql_test/seed/pgpm.py @@ -75,7 +75,7 @@ def seed(self, ctx: SeedContext) -> None: cwd = self._module_path or os.getcwd() # Build pgpm deploy command - cmd = ["pgpm", "deploy", "--yes"] + cmd = ["pgpm", "deploy", "--yes", "--verbose"] cmd.extend(self._deploy_args) logger.info(f"Running pgpm deploy in {cwd}") @@ -99,7 +99,9 @@ def seed(self, ctx: SeedContext) -> None: logger.info("pgpm deploy completed successfully") if result.stdout: - logger.debug(f"pgpm output: {result.stdout}") + logger.info(f"pgpm output: {result.stdout}") + if result.stderr: + logger.info(f"pgpm stderr: {result.stderr}") except FileNotFoundError as err: raise RuntimeError( From 4336986aad4f91f4486433b75c113cca2c580f02 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 22 Jan 2026 02:34:30 +0000 Subject: [PATCH 3/4] debug: Enable verbose logging in pgpm integration tests Add -s and --log-cli-level=INFO to pytest to capture pgpm deploy output --- .github/workflows/test-pgpm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pgpm.yml b/.github/workflows/test-pgpm.yml index 9d362a5..45233d5 100644 --- a/.github/workflows/test-pgpm.yml +++ b/.github/workflows/test-pgpm.yml @@ -86,4 +86,4 @@ jobs: pgpm admin-users add --test --yes - name: Run pgpm integration tests - run: poetry run pytest tests/test_pgpm_integration.py -v + run: poetry run pytest tests/test_pgpm_integration.py -v -s --log-cli-level=INFO From 3688b002db7c3839eb993b762ab5ee7b4dfc272f Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 22 Jan 2026 02:38:00 +0000 Subject: [PATCH 4/4] fix: Add --package flag to pgpm deploy to avoid interactive prompt The pgpm deploy command was showing an interactive prompt to choose a package instead of actually deploying. This fix adds a 'package' parameter to the seed.pgpm() adapter that passes --package to pgpm deploy, avoiding the interactive prompt in CI/non-interactive environments. --- src/pysql_test/seed/pgpm.py | 13 ++++++++++--- tests/test_pgpm_integration.py | 3 ++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/pysql_test/seed/pgpm.py b/src/pysql_test/seed/pgpm.py index 8c5c295..47b85a3 100644 --- a/src/pysql_test/seed/pgpm.py +++ b/src/pysql_test/seed/pgpm.py @@ -35,6 +35,7 @@ class PgpmSeedAdapter: def __init__( self, module_path: str | None = None, + package: str | None = None, deploy_args: list[str] | None = None, cache: bool = False, ) -> None: @@ -43,10 +44,12 @@ def __init__( Args: module_path: Path to the pgpm module directory (defaults to cwd) + package: Package name to deploy (avoids interactive prompt) deploy_args: Additional arguments to pass to pgpm deploy cache: Whether to enable caching (not yet implemented) """ self._module_path = module_path + self._package = package self._deploy_args = deploy_args or [] self._cache = cache @@ -76,6 +79,8 @@ def seed(self, ctx: SeedContext) -> None: # Build pgpm deploy command cmd = ["pgpm", "deploy", "--yes", "--verbose"] + if self._package: + cmd.extend(["--package", self._package]) cmd.extend(self._deploy_args) logger.info(f"Running pgpm deploy in {cwd}") @@ -111,6 +116,7 @@ def seed(self, ctx: SeedContext) -> None: def pgpm( module_path: str | None = None, + package: str | None = None, deploy_args: list[str] | None = None, cache: bool = False, ) -> PgpmSeedAdapter: @@ -122,6 +128,7 @@ def pgpm( Args: module_path: Path to the pgpm module directory (defaults to cwd) + package: Package name to deploy (avoids interactive prompt) deploy_args: Additional arguments to pass to pgpm deploy cache: Whether to enable caching @@ -131,12 +138,12 @@ def pgpm( Example: # Deploy migrations from a specific module seed_adapters = [ - seed.pgpm(module_path="./packages/my-module") + seed.pgpm(module_path="./packages/my-module", package="my-module") ] # Deploy with additional arguments seed_adapters = [ - seed.pgpm(module_path="./my-module", deploy_args=["--verbose"]) + seed.pgpm(module_path="./my-module", package="my-module", deploy_args=["--verbose"]) ] """ - return PgpmSeedAdapter(module_path=module_path, deploy_args=deploy_args, cache=cache) + return PgpmSeedAdapter(module_path=module_path, package=package, deploy_args=deploy_args, cache=cache) diff --git a/tests/test_pgpm_integration.py b/tests/test_pgpm_integration.py index 30d7cc1..b095d01 100644 --- a/tests/test_pgpm_integration.py +++ b/tests/test_pgpm_integration.py @@ -17,6 +17,7 @@ # Path to the pre-scaffolded pgpm workspace fixture FIXTURE_PATH = Path(__file__).parent / "fixtures" / "pgpm-workspace" / "packages" / "test-module" +PACKAGE_NAME = "test-module" @pytest.fixture @@ -29,7 +30,7 @@ def pgpm_db(): """ conn = get_connections( seed_adapters=[ - seed.pgpm(module_path=str(FIXTURE_PATH)) + seed.pgpm(module_path=str(FIXTURE_PATH), package=PACKAGE_NAME) ] ) db = conn.db