New PGlite

.github/workflows/build_and_test.yml

@@ -165,7 +165,7 @@ jobs:
         with:
           issue-number: ${{ github.event.pull_request.number }}
           comment-author: 'github-actions[bot]'
-          body-includes: '- PGlite:'
+          body-includes: '- PGlite with node'
 
       - name: Create or update build outputs comment
         uses: peter-evans/create-or-update-comment@v4
@@ -175,7 +175,7 @@ jobs:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ github.event.pull_request.number }}
           body: |
-            - PGlite: ${{ steps.upload-pglite-package.outputs.artifact-url }}
+            - PGlite with node v${{ matrix.node }}: ${{ steps.upload-pglite-package.outputs.artifact-url }}
           edit-mode: append
 
   build-and-test-pglite-dependents:
@@ -204,7 +204,7 @@ jobs:
         with:
           name: pglite-tools-release-files-node-v20.x
           path: ./packages/pglite-tools/release/
-      
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
@@ -252,7 +252,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: pglite-tools-release-files-node-v20.x
-          path: ./packages/pglite-tools/release          
+          path: ./packages/pglite-tools/release       
 
       - name: Download PGlite build artifacts
         uses: actions/download-artifact@v4
@@ -313,7 +313,7 @@ jobs:
         with:
           issue-number: ${{ github.event.pull_request.number }}
           comment-author: 'github-actions[bot]'
-          body-includes: Built bundles
+          body-includes: "- Demos:"
 
       - name: Create or update build outputs comment
         uses: peter-evans/create-or-update-comment@v4

docs/extensions/development.md

@@ -84,18 +84,18 @@ $ git checkout -b myghname/myawesomeextension
 PGlite's backend code is in the repo [postgres-pglite](https://github.com/electric-sql/postgres-pglite) and is downloaded as a submodule dependency of the main repo. You will add your extension's code as a new submodule dependency:
 
 ```
-$ cd postgres-pglite/pglite
+$ cd postgres-pglite/pglite/other_extensions
 $ git submodule add <myawesomeextension_url>
 ```
 
-This **should** create a new folder `postgres-pglite/pglite/myawesomeextension` where the extension code has been downloaded. Check it:
+This **should** create a new folder `postgres-pglite/pglite/other_extensions/myawesomeextension` where the extension code has been downloaded. Check it:
 
 ```
 $ ls -lah myawesomeextension
 <the extension files should be listed here>
 ```
 
-Now append the **folder name** to `SUBDIRS` inside `postgres-pglite/pglite/Makefile`:
+Now append the **folder name** to `SUBDIRS` inside `postgres-pglite/pglite/other_extensions/Makefile`:
 
 ```
 SUBDIRS = \

docs/repl/allExtensions.ts

@@ -18,6 +18,7 @@ export { ltree } from '@electric-sql/pglite/contrib/ltree'
 export { pageinspect } from '@electric-sql/pglite/contrib/pageinspect'
 export { pg_buffercache } from '@electric-sql/pglite/contrib/pg_buffercache'
 export { pg_freespacemap } from '@electric-sql/pglite/contrib/pg_freespacemap'
+export { pg_hashids } from '@electric-sql/pglite/pg_hashids'
 export { pg_surgery } from '@electric-sql/pglite/contrib/pg_surgery'
 export { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'
 export { pg_uuidv7 } from '@electric-sql/pglite/pg_uuidv7'
@@ -33,5 +34,4 @@ export { tsm_system_time } from '@electric-sql/pglite/contrib/tsm_system_time'
 export { unaccent } from '@electric-sql/pglite/contrib/unaccent'
 export { uuid_ossp } from '@electric-sql/pglite/contrib/uuid_ossp'
 export { vector } from '@electric-sql/pglite/vector'
-export { pg_hashids } from '@electric-sql/pglite/pg_hashids'
 export { age } from '@electric-sql/pglite/age'

package.json

@@ -12,9 +12,10 @@
     "ci:publish": "pnpm changeset publish",
     "ts:build": "pnpm -r --filter \"./packages/**\" build",
     "ts:build:debug": "DEBUG=true pnpm ts:build",
+    "wasm:copy-initdb": "mkdir -p ./packages/pglite/release && cp ./postgres-pglite/dist/bin/initdb.* ./packages/pglite/release",
     "wasm:copy-pgdump": "mkdir -p ./packages/pglite-tools/release && cp ./postgres-pglite/dist/bin/pg_dump.* ./packages/pglite-tools/release",
     "wasm:copy-pglite": "mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/",
-    "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump",
+    "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump && pnpm wasm:copy-initdb",
     "wasm:build:debug": "DEBUG=true pnpm wasm:build",
     "build:all": "pnpm wasm:build && pnpm ts:build",
     "build:all:debug": "DEBUG=true pnpm build:all"

packages/pglite-react/test-setup.ts

@@ -1,5 +1,18 @@
 import { afterEach } from 'vitest'
 import { cleanup } from '@testing-library/react'
 
+// Polyfill File.prototype.arrayBuffer for jsdom
+// jsdom's File implementation doesn't properly support arrayBuffer()
+if (typeof File !== 'undefined' && !File.prototype.arrayBuffer) {
+  File.prototype.arrayBuffer = function () {
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader()
+      reader.onload = () => resolve(reader.result as ArrayBuffer)
+      reader.onerror = () => reject(reader.error)
+      reader.readAsArrayBuffer(this)
+    })
+  }
+}
+
 // https://testing-library.com/docs/react-testing-library/api#cleanup
 afterEach(() => cleanup())

packages/pglite-socket/src/index.ts

@@ -93,21 +93,23 @@ class QueryQueueManager {
         `processing query from handler #${query.handlerId} (waited ${waitTime}ms)`,
       )
 
+      let result
       try {
         // Execute the query with exclusive access to PGlite
-        const result = await this.db.runExclusive(async () => {
+        result = await this.db.runExclusive(async () => {
           return await this.db.execProtocolRaw(query.message)
         })
-
-        this.log(
-          `query from handler #${query.handlerId} completed, ${result.length} bytes`,
-        )
-        this.lastHandlerId = query.handlerId
-        query.resolve(result)
       } catch (error) {
         this.log(`query from handler #${query.handlerId} failed:`, error)
         query.reject(error as Error)
+        return
       }
+
+      this.log(
+        `query from handler #${query.handlerId} completed, ${result.length} bytes`,
+      )
+      this.lastHandlerId = query.handlerId
+      query.resolve(result)
     }
 
     this.processing = false

packages/pglite-tools/src/pgDumpModFactory.ts

@@ -22,16 +22,16 @@ export interface PgDumpMod
   FS: FS
   WASM_PREFIX: string
   INITIAL_MEMORY: number
-  _set_read_write_cbs: (read_cb: number, write_cb: number) => void
+  _pgl_set_rw_cbs: (read_cb: number, write_cb: number) => void
   addFunction: (
     cb: (ptr: any, length: number) => void,
     signature: string,
   ) => number
   removeFunction: (f: number) => void
-  _main: (args: string[]) => number
   onExit: (status: number) => void
   print: (test: string) => void
   printErr: (text: string) => void
+  callMain: (args?: string[]) => number
 }
 
 type PgDumpFactory<T extends PgDumpMod = PgDumpMod> = (

packages/pglite-tools/src/pg_dump.ts

@@ -36,11 +36,10 @@ async function execPgDump({
   args: string[]
 }): Promise<ExecResult> {
   let pgdump_write, pgdump_read
-  let exitStatus = 0
+  let exitCode = 0
   let stderrOutput: string = ''
   let stdoutOutput: string = ''
   const emscriptenOpts: Partial<PgDumpMod> = {
-    arguments: args,
     noExitRuntime: false,
     print: (text) => {
       stdoutOutput += text
@@ -49,7 +48,7 @@ async function execPgDump({
       stderrOutput += text
     },
     onExit: (status: number) => {
-      exitStatus = status
+      exitCode = status
     },
     preRun: [
       (mod: PgDumpMod) => {
@@ -83,7 +82,8 @@ async function execPgDump({
             return length
           }, 'iii')
 
-          mod._set_read_write_cbs(pgdump_read, pgdump_write)
+          mod._pgl_set_rw_cbs(pgdump_read, pgdump_write)
+
           // default $HOME in emscripten is /home/web_user
           mod.FS.chmod('/home/web_user/.pgpass', 0o0600) // https://www.postgresql.org/docs/current/libpq-pgpass.html
         }
@@ -92,13 +92,14 @@ async function execPgDump({
   }
 
   const mod = await PgDumpModFactory(emscriptenOpts)
+  mod.callMain(args)
   let fileContents = ''
-  if (!exitStatus) {
+  if (!exitCode) {
     fileContents = mod.FS.readFile(dumpFilePath, { encoding: 'utf8' })
   }
 
   return {
-    exitCode: exitStatus,
+    exitCode,
     fileContents,
     stderr: stderrOutput,
     stdout: stdoutOutput,
@@ -108,12 +109,17 @@ async function execPgDump({
 interface PgDumpOptions {
   pg: PGlite
   args?: string[]
+  database?: string
   fileName?: string
   verbose?: boolean
 }
 
 /**
  * Execute pg_dump
+ * @param pg - The PGlite instance
+ * @param args - The arguments to pass to pg_dump
+ * @param fileName - The name of the file to write the dump to (dump.sql by default)
+ * @returns The file containing the dump
  */
 export async function pgDump({
   pg,
@@ -123,25 +129,35 @@ export async function pgDump({
   const getSearchPath = await pg.query<{ search_path: string }>(
     'SHOW SEARCH_PATH;',
   )
-  const search_path = getSearchPath.rows[0].search_path
+  const searchPath = getSearchPath.rows[0].search_path
 
   const baseArgs = [
     '-U',
-    'postgres',
+    'web_user',
     '--inserts',
     '-j',
     '1',
     '-f',
     dumpFilePath,
-    'postgres',
   ]
 
   const execResult = await execPgDump({
     pg,
     args: [...(args ?? []), ...baseArgs],
   })
 
-  pg.exec(`DEALLOCATE ALL; SET SEARCH_PATH = ${search_path}`)
+  await pg.exec(`DEALLOCATE ALL`)
+  await pg.exec(`SET SEARCH_PATH = ${searchPath}`)
+  const newSearchPath = await pg.query<{ search_path: string }>(
+    'SHOW SEARCH_PATH;',
+  )
+  if (newSearchPath.rows[0].search_path !== searchPath) {
+    console.warn(
+      `Warning: search_path has been changed from ${searchPath} to ${newSearchPath}`,
+      searchPath,
+      newSearchPath,
+    )
+  }
 
   if (execResult.exitCode !== 0) {
     throw new Error(

packages/pglite-vue/test-setup.ts

@@ -1,3 +1,16 @@
 import { install } from 'vue-demi'
 
+// Polyfill File.prototype.arrayBuffer for jsdom
+// jsdom's File implementation doesn't properly support arrayBuffer()
+if (typeof File !== 'undefined' && !File.prototype.arrayBuffer) {
+  File.prototype.arrayBuffer = function () {
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader()
+      reader.onload = () => resolve(reader.result as ArrayBuffer)
+      reader.onerror = () => reject(reader.error)
+      reader.readAsArrayBuffer(this)
+    })
+  }
+}
+
 install()

packages/pglite/examples/basic.html

@@ -21,7 +21,7 @@ <h1>PGlite Basic Example</h1>
         console.log('Starting...')
 
         // In-memory database:
-        const pg = new PGlite()
+        const pg = await PGlite.create()
         // Or, stored in indexedDB:
         // const pg = new PGlite('pgdata');
 

packages/pglite/package.json

@@ -159,7 +159,22 @@
   "type": "module",
   "types": "dist/index.d.ts",
   "files": [
-    "dist"
+    "dist",
+    "!dist/auth_delay.tar.gz",
+    "!dist/basebackup_to_shell.tar.gz",
+    "!dist/basic_archive.tar.gz",
+    "!dist/dblink.tar.gz",
+    "!dist/intagg.tar.gz",
+    "!dist/oid2name.tar.gz",
+    "!dist/pg_prewarm.tar.gz",
+    "!dist/pg_stat_statements.tar.gz",
+    "!dist/pglite.html",
+    "!dist/pgrowlocks.tar.gz",
+    "!dist/pgstattuple.tar.gz",
+    "!dist/spi.tar.gz",
+    "!dist/test_decoding.tar.gz",
+    "!dist/vacuumlo.tar.gz",
+    "!dist/xml2.tar.gz"
   ],
   "repository": {
     "type": "git",