diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 773e01b05..4a3c40612 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -61,13 +61,6 @@ jobs: run: | pnpm build:all - - name: Typecheck pglite - working-directory: ${{ github.workspace }}/packages/pglite - run: pnpm typecheck - - name: Test pglite - working-directory: ${{ github.workspace }}/packages/pglite - run: pnpm test - - name: Upload PGlite Interim to Github artifacts id: upload-pglite-interim-build-files uses: actions/upload-artifact@v4 @@ -84,6 +77,26 @@ jobs: path: ./packages/pglite-tools/release/** retention-days: 60 + - name: Upload pglite-postgis build artifacts to Github artifacts + id: upload-pglite-postgis-release-files + uses: actions/upload-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/** + retention-days: 60 + + - name: Typecheck pglite + working-directory: ${{ github.workspace }}/packages/pglite + run: pnpm typecheck + + - name: Test pglite + working-directory: ${{ github.workspace }}/packages/pglite + run: pnpm test + + - name: Test pglite-postgis + working-directory: ${{ github.workspace }}/packages/pglite-postgis + run: pnpm test + build-and-test-pglite: name: Build and Test packages/pglite runs-on: blacksmith-32vcpu-ubuntu-2204 @@ -119,6 +132,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release + - name: Install dependencies run: | pnpm install --frozen-lockfile @@ -165,7 +184,7 @@ jobs: with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' - body-includes: '- PGlite with node:' + body-includes: '- PGlite with node' - name: Create or update build outputs comment uses: peter-evans/create-or-update-comment@v4 @@ -205,6 +224,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release/ + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -225,7 +250,14 @@ jobs: uses: actions/upload-artifact@v4 with: name: pglite-tools-dist-node-v${{ matrix.node }} - path: ./packages/pglite-tools/dist/* + path: ./packages/pglite-tools/dist/* + + - name: Upload pglite-postgis distribution artifact + id: upload-pglite-postgis-dist + uses: actions/upload-artifact@v4 + with: + name: pglite-postgis-dist-node-v${{ matrix.node }} + path: ./packages/pglite-postgis/dist/* publish-website-with-demos: name: Publish website with demos @@ -252,7 +284,13 @@ jobs: uses: actions/download-artifact@v4 with: name: pglite-tools-release-files-node-v20.x - path: ./packages/pglite-tools/release + path: ./packages/pglite-tools/release + + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ - name: Download PGlite build artifacts uses: actions/download-artifact@v4 @@ -260,6 +298,12 @@ jobs: name: pglite-dist-node-v20.x path: ./packages/pglite/dist/ + - name: Download pglite-postgis dist artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-dist-node-v20.x + path: ./packages/pglite-postgis/dist/ + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -313,7 +357,7 @@ jobs: with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' - body-includes: Built bundles + body-includes: "- Demos:" - name: Create or update build outputs comment uses: peter-evans/create-or-update-comment@v4 @@ -368,6 +412,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release/ + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ + - run: pnpm install --frozen-lockfile - run: pnpm --filter "./packages/**" build - name: Create Release Pull Request or Publish diff --git a/docs/extensions/extensions.data.ts b/docs/extensions/extensions.data.ts index aded5edbf..efc4fb07f 100644 --- a/docs/extensions/extensions.data.ts +++ b/docs/extensions/extensions.data.ts @@ -595,12 +595,31 @@ const baseExtensions: Extension[] = [ importName: 'age', size: 141551, }, + { + name: 'PostGIS', + description: ` + *** EXPERIMENTAL *** + PostGIS extends the capabilities of the PostgreSQL relational database by adding + support for storing, indexing, and querying geospatial data. + + *Follow this issue for the status: TODOP + + *No GDAL support atm. + `, + shortDescription: 'Storing, indexing, and querying geospatial data.', + docs: 'postgis.net', + tags: ['postgres extension', 'experimental'], + importPath: '@electric-sql/pglite-postgis', + importName: 'postgis', + size: 7901736, + }, ] const tags = [ 'postgres extension', 'pglite plugin', 'postgres/contrib', + 'experimental', ] as const export type Tag = (typeof tags)[number] diff --git a/docs/package.json b/docs/package.json index 03864e194..95ad3ef61 100644 --- a/docs/package.json +++ b/docs/package.json @@ -20,6 +20,7 @@ "dependencies": { "@electric-sql/pglite": "workspace:*", "@electric-sql/pglite-repl": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*", "@uiw/codemirror-theme-github": "^4.23.0", "dedent": "^1.5.3" } diff --git a/docs/repl/allExtensions.ts b/docs/repl/allExtensions.ts index cab6fafe1..c874a7ab6 100644 --- a/docs/repl/allExtensions.ts +++ b/docs/repl/allExtensions.ts @@ -35,3 +35,4 @@ export { unaccent } from '@electric-sql/pglite/contrib/unaccent' export { uuid_ossp } from '@electric-sql/pglite/contrib/uuid_ossp' export { vector } from '@electric-sql/pglite/vector' export { age } from '@electric-sql/pglite/age' +export { postgis } from '@electric-sql/pglite-postgis' diff --git a/package.json b/package.json index 22732421d..36b8dfc88 100644 --- a/package.json +++ b/package.json @@ -12,10 +12,11 @@ "ci:publish": "pnpm changeset publish", "ts:build": "pnpm -r --filter \"./packages/**\" build", "ts:build:debug": "DEBUG=true pnpm ts:build", + "wasm:copy-postgis": "mkdir -p ./packages/pglite-postgis/release && cp ./postgres-pglite/dist/extensions/postgis/postgis.tar.gz ./packages/pglite-postgis/release", "wasm:copy-initdb": "mkdir -p ./packages/pglite/release && cp ./postgres-pglite/dist/bin/initdb.* ./packages/pglite/release", "wasm:copy-pgdump": "mkdir -p ./packages/pglite-tools/release && cp ./postgres-pglite/dist/bin/pg_dump.* ./packages/pglite-tools/release", "wasm:copy-pglite": "mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/", - "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump && pnpm wasm:copy-initdb", + "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump && pnpm wasm:copy-initdb && pnpm wasm:copy-postgis", "wasm:build:debug": "DEBUG=true pnpm wasm:build", "build:all": "pnpm wasm:build && pnpm ts:build", "build:all:debug": "DEBUG=true pnpm build:all" diff --git a/packages/pglite-postgis/.gitignore b/packages/pglite-postgis/.gitignore new file mode 100644 index 000000000..ef8abb0e7 --- /dev/null +++ b/packages/pglite-postgis/.gitignore @@ -0,0 +1,2 @@ +release/* +dist \ No newline at end of file diff --git a/packages/pglite-postgis/README.md b/packages/pglite-postgis/README.md new file mode 100644 index 000000000..58ca6f903 --- /dev/null +++ b/packages/pglite-postgis/README.md @@ -0,0 +1,51 @@ +# @electric-sql/pglite-postgis + +*** EXPERIMENTAL *** + +PostGIS extension for [PGlite](https://pglite.dev). This is an experimental release, use at your own risk. + +## Installation + +```bash +npm install @electric-sql/pglite-postgis +``` + +## Usage + +```typescript +import { PGlite } from '@electric-sql/pglite' +import { postgis } from '@electric-sql/pglite-postgis' + +const pg = new PGlite({ + extensions: { + postgis, + }, +}) + +await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + +// Create a table with geometry columns +await pg.exec(` + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + location GEOMETRY(Point, 4326) + ); +`) + +// Insert data +await pg.query(` + INSERT INTO cities (name, location) + VALUES ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)) +`) + +// Query with spatial functions +const result = await pg.query(` + SELECT name, ST_AsText(location) as location + FROM cities +`) +``` + +## License + +Apache-2.0 \ No newline at end of file diff --git a/packages/pglite-postgis/eslint.config.js b/packages/pglite-postgis/eslint.config.js new file mode 100644 index 000000000..e1dc3262d --- /dev/null +++ b/packages/pglite-postgis/eslint.config.js @@ -0,0 +1,21 @@ +import globals from 'globals' +import rootConfig from '../../eslint.config.js' + +export default [ + ...rootConfig, + { + ignores: ['release/**/*', 'dist/**/*'], + }, + { + languageOptions: { + globals: { + ...globals.browser, + ...globals.node, + }, + }, + rules: { + ...rootConfig.rules, + '@typescript-eslint/no-explicit-any': 'off', + }, + }, +] \ No newline at end of file diff --git a/packages/pglite-postgis/package.json b/packages/pglite-postgis/package.json new file mode 100644 index 000000000..46a299049 --- /dev/null +++ b/packages/pglite-postgis/package.json @@ -0,0 +1,65 @@ +{ + "name": "@electric-sql/pglite-postgis", + "version": "0.0.1", + "description": "PostGIS extension for PGlite", + "author": "Electric DB Limited", + "homepage": "https://pglite.dev", + "license": "Apache-2.0", + "keywords": [ + "postgres", + "sql", + "database", + "wasm", + "pglite", + "postgis", + "gis", + "geospatial" + ], + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + } + }, + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/electric-sql/pglite.git", + "directory": "packages/pglite-postgis" + }, + "scripts": { + "build": "tsup", + "check:exports": "attw . --pack --profile node16", + "lint": "eslint ./src ./tests --report-unused-disable-directives --max-warnings 0", + "format": "prettier --write ./src ./tests", + "typecheck": "tsc", + "stylecheck": "pnpm lint && prettier --check ./src ./tests", + "test": "vitest", + "prepublishOnly": "pnpm check:exports" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.18.1", + "@electric-sql/pglite": "workspace:*", + "@types/node": "^20.16.11", + "vitest": "^2.1.2" + }, + "peerDependencies": { + "@electric-sql/pglite": "workspace:*" + } +} \ No newline at end of file diff --git a/packages/pglite-postgis/src/index.ts b/packages/pglite-postgis/src/index.ts new file mode 100644 index 000000000..7638fbd62 --- /dev/null +++ b/packages/pglite-postgis/src/index.ts @@ -0,0 +1,17 @@ +import type { + Extension, + ExtensionSetupResult, + PGliteInterface, +} from '@electric-sql/pglite' + +const setup = async (_pg: PGliteInterface, emscriptenOpts: any) => { + return { + emscriptenOpts, + bundlePath: new URL('../release/postgis.tar.gz', import.meta.url), + } satisfies ExtensionSetupResult +} + +export const postgis = { + name: 'postgis', + setup, +} satisfies Extension diff --git a/packages/pglite-postgis/tests/postgis.test.ts b/packages/pglite-postgis/tests/postgis.test.ts new file mode 100644 index 000000000..c38ba765e --- /dev/null +++ b/packages/pglite-postgis/tests/postgis.test.ts @@ -0,0 +1,284 @@ +import { describe, it, expect } from 'vitest' +import { PGlite } from '@electric-sql/pglite' +import { postgis } from '../src/index.js' + +describe(`postgis`, () => { + it('basic', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE vehicle_location ( + time TIMESTAMPTZ NOT NULL, + vehicle_id INT NOT NULL, + location GEOGRAPHY(POINT, 4326) +); + `) + const inserted = await pg.query(`INSERT INTO vehicle_location VALUES + ('2023-05-29 20:00:00', 1, 'POINT(15.3672 -87.7231)'), + ('2023-05-30 20:00:00', 1, 'POINT(15.3652 -80.7331)'), + ('2023-05-31 20:00:00', 1, 'POINT(15.2672 -85.7431)');`) + + expect(inserted.affectedRows).toEqual(3) + }), + it('cities', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + location GEOMETRY(Point, 4326) +); + `) + const inserted = await pg.query(`INSERT INTO cities (name, location) +VALUES + ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)), + ('Los Angeles', ST_GeomFromText('POINT(-118.2437 34.0522)', 4326)), + ('Chicago', ST_GeomFromText('POINT(-87.6298 41.8781)', 4326));`) + + expect(inserted.affectedRows).toEqual(3) + + const cities = await pg.query(`WITH state_boundary AS ( + SELECT ST_GeomFromText( + 'POLYGON((-91 36, -91 43, -87 43, -87 36, -91 36))', 4326 + ) AS geom +) +SELECT c.name +FROM cities c, state_boundary s +WHERE ST_Within(c.location, s.geom);`) + + expect(cities.affectedRows).toBe(0) + expect(cities.rows[0]).toEqual({ + name: 'Chicago', + }) + }) +}) + +it('areas', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + const area1 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(geom) * 0.3048 ^ 2 sqm + from ( + select 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery;`) + + expect(area1).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27208552, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area2 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(ST_Transform(geom, 26986)) As sqm + from ( + select + 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery; + + -- Cleanup test schema + -- DROP SCHEMA postgis_test CASCADE; + `) + + expect(area2).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27243061926092, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area3 = await pg.exec(` + select ST_Area(geog) / 0.3048 ^ 2 sqft_spheroid, + ST_Area(geog, false) / 0.3048 ^ 2 sqft_sphere, + ST_Area(geog) sqm_spheroid + from ( + select ST_Transform( + 'SRID=2249;POLYGON((743238 2967416,743238 2967450,743265 2967450,743265.625 2967416,743238 2967416))'::geometry, + 4326 + ) :: geography geog + ) as subquery; + `) + + expect(area3).toEqual([ + { + rows: [ + { + sqft_spheroid: 928.6844047556697, + sqft_sphere: 926.609762750544, + sqm_spheroid: 86.27760440239217, + }, + ], + fields: [ + { + name: 'sqft_spheroid', + dataTypeID: 701, + }, + { + name: 'sqft_sphere', + dataTypeID: 701, + }, + { + name: 'sqm_spheroid', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) +}) + +it('ST_Polygonize', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + const res = await pg.exec(` + WITH data(geom) AS (VALUES + ('LINESTRING (180 40, 30 20, 20 90)'::geometry) + ,('LINESTRING (180 40, 160 160)'::geometry) + ,('LINESTRING (80 60, 120 130, 150 80)'::geometry) + ,('LINESTRING (80 60, 150 80)'::geometry) + ,('LINESTRING (20 90, 70 70, 80 130)'::geometry) + ,('LINESTRING (80 130, 160 160)'::geometry) + ,('LINESTRING (20 90, 20 160, 70 190)'::geometry) + ,('LINESTRING (70 190, 80 130)'::geometry) + ,('LINESTRING (70 190, 160 160)'::geometry) + ) + SELECT ST_AsText( ST_Polygonize( geom )) + FROM data; + `) + + expect(res).toEqual([ + { + rows: [ + { + st_astext: + 'GEOMETRYCOLLECTION(POLYGON((180 40,30 20,20 90,70 70,80 130,160 160,180 40),(150 80,120 130,80 60,150 80)),POLYGON((80 60,120 130,150 80,80 60)),POLYGON((80 130,70 70,20 90,20 160,70 190,80 130)),POLYGON((160 160,80 130,70 190,160 160)))', + }, + ], + fields: [ + { + name: 'st_astext', + dataTypeID: 25, + }, + ], + affectedRows: 0, + }, + ]) +}) + +it('complex1', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + await pg.exec(` + -- Create test schema + -- CREATE SCHEMA IF NOT EXISTS postgis_test; + -- SET search_path TO postgis_test; + + -- Create a table with geometry columns + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + population INTEGER, + geom GEOMETRY(Point, 4326) + );`) + + await pg.exec(` + CREATE TABLE rivers ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + geom GEOMETRY(LineString, 4326) + ); + + -- Insert sample data + INSERT INTO cities (name, population, geom) VALUES + ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), + ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), + ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), + ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); + + INSERT INTO rivers (name, geom) VALUES + ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(2.1, 48.8), + ST_MakePoint(2.35, 48.85), + ST_MakePoint(2.45, 48.9) + ]), 4326)), + ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(13.1, 52.4), + ST_MakePoint(13.35, 52.5), + ST_MakePoint(13.45, 52.52) + ]), 4326)); + + -- Create spatial index + CREATE INDEX idx_cities_geom ON cities USING GIST (geom); + CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); + + -- Query: Find cities within 10 km of any river + SELECT + c.name AS city, + r.name AS river, + ST_Distance(c.geom::geography, r.geom::geography) AS distance_km + FROM cities c + JOIN rivers r + ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) + ORDER BY distance_km; + + `) +}) diff --git a/packages/pglite-postgis/tsconfig.json b/packages/pglite-postgis/tsconfig.json new file mode 100644 index 000000000..27611a8fd --- /dev/null +++ b/packages/pglite-postgis/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src", "tsup.config.ts", "vitest.config.ts"] +} \ No newline at end of file diff --git a/packages/pglite-postgis/tsup.config.ts b/packages/pglite-postgis/tsup.config.ts new file mode 100644 index 000000000..990f7a527 --- /dev/null +++ b/packages/pglite-postgis/tsup.config.ts @@ -0,0 +1,25 @@ +import { cpSync } from 'fs' +import { resolve } from 'path' +import { defineConfig } from 'tsup' + +const entryPoints = ['src/index.ts'] + +const minify = process.env.DEBUG === 'true' ? false : true + +export default defineConfig([ + { + entry: entryPoints, + sourcemap: true, + dts: { + entry: entryPoints, + resolve: true, + }, + clean: true, + minify: minify, + shims: true, + format: ['esm', 'cjs'], + onSuccess: async () => { + cpSync(resolve('release/postgis.tar.gz'), resolve('dist/postgis.tar.gz')) + }, + }, +]) \ No newline at end of file diff --git a/packages/pglite-postgis/vitest.config.ts b/packages/pglite-postgis/vitest.config.ts new file mode 100644 index 000000000..378f02808 --- /dev/null +++ b/packages/pglite-postgis/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + testTimeout: 30000, + }, +}) \ No newline at end of file diff --git a/packages/pglite-socket/package.json b/packages/pglite-socket/package.json index 8d96d3587..028821306 100644 --- a/packages/pglite-socket/package.json +++ b/packages/pglite-socket/package.json @@ -57,6 +57,7 @@ "@arethetypeswrong/cli": "^0.18.1", "@electric-sql/pg-protocol": "workspace:*", "@electric-sql/pglite": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*", "@types/emscripten": "^1.41.1", "@types/node": "^20.16.11", "pg": "^8.14.0", @@ -65,6 +66,7 @@ "vitest": "^1.3.1" }, "peerDependencies": { - "@electric-sql/pglite": "workspace:*" + "@electric-sql/pglite": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*" } } diff --git a/packages/pglite-socket/src/scripts/server.ts b/packages/pglite-socket/src/scripts/server.ts index b7ee83c00..e012317c1 100755 --- a/packages/pglite-socket/src/scripts/server.ts +++ b/packages/pglite-socket/src/scripts/server.ts @@ -43,7 +43,7 @@ const args = parseArgs({ type: 'string', short: 'e', default: undefined, - help: 'Comma-separated list of extensions to load (e.g., vector,pgcrypto)', + help: 'Comma-separated list of extensions to load (e.g., vector,pgcrypto,postgis etc.)', }, run: { type: 'string', diff --git a/packages/pglite/src/extensionUtils.ts b/packages/pglite/src/extensionUtils.ts index ea225566d..071f6f987 100644 --- a/packages/pglite/src/extensionUtils.ts +++ b/packages/pglite/src/extensionUtils.ts @@ -53,7 +53,8 @@ export async function loadExtensionBundle( export async function loadExtensions( mod: PostgresMod, log: (...args: any[]) => void, -) { +): Promise { + const promises = new Array>() for (const ext in mod.pg_extensions) { let blob try { @@ -64,11 +65,12 @@ export async function loadExtensions( } if (blob) { const bytes = new Uint8Array(await blob.arrayBuffer()) - loadExtension(mod, ext, bytes, log) + promises.push(...loadExtension(mod, ext, bytes, log)) } else { console.error('Could not get binary data for extension:', ext) } } + return Promise.all(promises) } function loadExtension( @@ -76,41 +78,65 @@ function loadExtension( _ext: string, bytes: Uint8Array, log: (...args: any[]) => void, -) { +): Promise[] { + const soPreloadPromises: Promise[] = [] const data = tinyTar.untar(bytes) - data.forEach((file: any) => { - if (!file.name.startsWith('.')) { - const filePath = mod.WASM_PREFIX + '/' + file.name - if (file.name.endsWith('.so')) { - const extOk = (...args: any[]) => { - log('pgfs:ext OK', filePath, args) - } - const extFail = (...args: any[]) => { - log('pgfs:ext FAIL', filePath, args) - } - mod.FS.createPreloadedFile( - dirname(filePath), - file.name.split('/').pop()!.slice(0, -3), - file.data as any, // There is a type error in Emscripten's FS.createPreloadedFile, this excepts a Uint8Array, but the type is defined as any - true, - true, - extOk, - extFail, - false, - ) + data.forEach((entry: any) => { + if (entry.name.endsWith('/')) { + const dirPath = `${mod.WASM_PREFIX}/${entry.name}` + if (mod.FS.analyzePath(dirPath).exists === false) { + mod.FS.mkdirTree(dirPath) + } + } else if (!entry.name.startsWith('.')) { + const filePath = mod.WASM_PREFIX + '/' + entry.name + if (entry.name.endsWith('.so')) { + const soName = entry.name.split('/').pop()! // e.g. 'postgis-3.so' + const dirPath = dirname(filePath) + // Wrap createPreloadedFile in a Promise so loadExtensions can await the + // async WASM compilation done by Emscripten's wasm preload plugin. + // The plugin calls extOk only after preloadedWasm[path] is set, so + // awaiting this ensures dlopen finds the pre-compiled module. + const soPreload = new Promise((resolve, _reject) => { + const extOk = (...args: any[]) => { + log('pgfs:ext OK', filePath, args) + resolve() + } + const extFail = (...args: any[]) => { + log('pgfs:ext FAIL', filePath, args) + // hope for the best: it's not the end even if we were unable to preload a file + // emscripten will try again if/when needed and do a wasm.compile on the main thread + resolve() + // _reject(new Error(`Failed to preload ${filePath}`)) + } + // Keep the .so suffix so Emscripten's wasm preload plugin canHandle() matches, + // triggering async WebAssembly.instantiate. The compiled module is stored in + // preloadedWasm under the path with .so. + mod.FS.createPreloadedFile( + dirPath, + soName, + entry.data as any, // There is a type error in Emscripten's FS.createPreloadedFile, this excepts a Uint8Array, but the type is defined as any + true, + true, + extOk, + extFail, + false, + ) + }) + soPreloadPromises.push(soPreload) } else { try { const dirPath = filePath.substring(0, filePath.lastIndexOf('/')) if (mod.FS.analyzePath(dirPath).exists === false) { mod.FS.mkdirTree(dirPath) } - mod.FS.writeFile(filePath, file.data) + mod.FS.writeFile(filePath, entry.data) } catch (e) { console.error(`Error writing file ${filePath}`, e) } } } }) + return soPreloadPromises } function dirname(path: string) { diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index 5856ab76e..226af5ee5 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -399,6 +399,13 @@ export class PGlite mod.ENV.TZ = 'UTC' mod.ENV.PGTZ = 'UTC' mod.ENV.PGCLIENTENCODING = 'UTF8' + + // some extensions might need their own ENV variables + for (const [extName] of Object.entries(this.#extensions)) { + if (extName === 'postgis') { + mod.ENV.PROJ_DATA = `${WASM_PREFIX}/share/proj` + } + } }, ], } diff --git a/packages/pglite/src/postgresMod.ts b/packages/pglite/src/postgresMod.ts index 5a7ba1339..c1520d93e 100644 --- a/packages/pglite/src/postgresMod.ts +++ b/packages/pglite/src/postgresMod.ts @@ -27,7 +27,6 @@ export interface PostgresMod pg_extensions: Record> UTF8ToString: (ptr: number, maxBytesToRead?: number) => string stringToUTF8OnStack: (s: string) => number - _pgl_shutdown: () => void _pgl_set_system_fn: (system_fn: number) => void _pgl_set_popen_fn: (popen_fn: number) => void _pgl_set_pclose_fn: (pclose_fn: number) => void diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ee45078b5..162c10daa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: '@electric-sql/pglite': specifier: workspace:* version: link:../packages/pglite + '@electric-sql/pglite-postgis': + specifier: workspace:* + version: link:../packages/pglite-postgis '@electric-sql/pglite-repl': specifier: workspace:* version: link:../packages/pglite-repl @@ -202,6 +205,20 @@ importers: specifier: ^2.1.2 version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) + packages/pglite-postgis: + devDependencies: + '@arethetypeswrong/cli': + specifier: ^0.18.1 + version: 0.18.1 + '@electric-sql/pglite': + specifier: workspace:* + version: link:../pglite + '@types/node': + specifier: ^20.16.11 + version: 20.16.11 + vitest: + specifier: ^2.1.2 + version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) packages/pglite-react: devDependencies: @@ -341,6 +358,9 @@ importers: '@electric-sql/pglite': specifier: workspace:* version: link:../pglite + '@electric-sql/pglite-postgis': + specifier: workspace:* + version: link:../pglite-postgis '@types/emscripten': specifier: ^1.41.1 version: 1.41.1 diff --git a/postgres-pglite b/postgres-pglite index 6c76f5e2b..0d7e5cd9c 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 6c76f5e2b5468de4464bd37a69ed4d1bff0cab82 +Subproject commit 0d7e5cd9c37c6217cb25548d036d87227aa6c4ac