feat: pgboss upgrade MAPCO-9745

.github/CODEOWNERS

@@ -1 +1 @@
-*  @netanelC @CptSchnitz @shimoncohen
+*       @MapColonies/vector-team

.github/workflows/pull_request.yaml

@@ -64,7 +64,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:14
+        image: postgres:15
         env:
           POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }}
           POSTGRES_USER: ${{ env.DB_USERNAME }}

README.md

@@ -52,7 +52,16 @@ each item on this queue is a metatile in the size of metatile property located i
 - `QUEUE_JOBS_RETRY_BACKOFF` - Default: true. Enables exponential backoff retries based on retryDelay instead of a fixed delay. Sets initial retryDelay to 1 if not set.
 - `QUEUE_JOBS_RETRY_LIMIT` - Default: 3. Max number of retries of failed jobs.
 - `QUEUE_JOBS_RETRY_DELAY_SECONDS` - Default: 60. Delay between retries of failed jobs, in seconds.
-- `QUEUE_JOBS_RETENTION_HOURS` - Default: 87660. How many hours a job may be in created or retry state before it's archived. Must be >=1
+- `QUEUE_JOBS_RETENTION_SECONDS` - Default: 315576000. How many seconds a job may be in created or retry state before it's archived. Must be >=1
+- `DB_CREATE_SCHEMA` - Default: false. enable pg-boss schema creation at startup
+- `DB_RUN_MIGRATE` - Default: false. enable pg-boss schema migrations at startup
 - `APP_CONSUME_CONDITION_ENABLED` - Pre-request consumption flag for conditions validation, current condition is tiles queue size (see `APP_CONSUME_CONDITION_TILES_QUEUE_SIZE_LIMIT`)
 - `APP_CONSUME_CONDITION_TILES_QUEUE_SIZE_LIMIT` - The max number of tiles in the tiles queue allowed before the request consumption
-- `APP_CONSUME_CONDITION_CHECK_INTERVAL_SEC` - upon invalid match to the consume condition, the duration in seconds to wait until the next consume condition validation
+- `APP_CONSUME_CONDITION_CHECK_INTERVAL_SEC` - upon invalid match to the consume condition, the duration in seconds to wait until the next consume condition validation
+
+## pg-boss migrations
+pg-boss v12 requires its database schema to be created and migrated before starting the service. Run migrations with the pg-boss CLI using your database connection (or `PGBOSS_*` environment variables):
+
+```bash
+npx pg-boss migrate --connection-string postgres://user:pass@host:5432/database --schema pgboss
+```

config/default.json

@@ -52,14 +52,16 @@
     "retryBackoff": true,
     "retryLimit": 3,
     "retryDelaySeconds": 60,
-    "retentionHours": 87660
+    "retentionSeconds": 315576000
   },
   "db": {
     "host": "localhost",
     "port": 5432,
     "username": "postgres",
     "password": "postgres",
     "schema": "pgboss",
+    "createSchema": true,
+    "migrate": true,
     "ssl": {
       "enabled": false,
       "ca": "",

helm/templates/configmap.yaml

@@ -24,14 +24,16 @@ data:
   DB_HOST: {{ .host }}
   DB_NAME: {{ .database }}
   DB_SCHEMA: {{ .schema }}
+  DB_CREATE_SCHEMA: {{ .createSchema | quote }}
+  DB_RUN_MIGRATE: {{ .migrate | quote }}
   DB_PORT: {{ .port | quote }}
   {{- end -}}
   {{- with .Values.queueConfig }}
   QUEUE_JOBS_EXPIRE_IN_SECONDS: {{ .expireInSeconds | quote }}
   QUEUE_JOBS_RETRY_BACKOFF: {{ .retryBackoff | quote }}
   QUEUE_JOBS_RETRY_LIMIT: {{ .retryLimit | quote }}
   QUEUE_JOBS_RETRY_DELAY_SECONDS: {{ .retryDelaySeconds | quote }}
-  QUEUE_JOBS_RETENTION_HOURS: {{ .retentionHours | quote }}
+  QUEUE_JOBS_RETENTION_SECONDS: {{ .retentionSeconds | quote }}
   {{- end -}}
   FORCE_API_TILES: {{ .Values.env.force.api | quote }}
   FORCE_EXPIRED_TILES: {{ .Values.env.force.expiredTiles | quote }}

helm/values.yaml

@@ -61,14 +61,16 @@ queueConfig:
   retryBackoff: true
   retryLimit: 3
   retryDelaySeconds: 60
-  retentionHours: 87660
+  retentionSeconds: 315576000
 
 dbConfig:
   host: localhost
   username: postgres
   password: postgres
   database: metatile-queue-populator
   schema: pgboss
+  createSchema: true
+  migrate: true
   port: 5432
   sslAuth:
     enabled: false

package.json

@@ -43,7 +43,7 @@
     "@map-colonies/js-logger": "^1.0.1",
     "@map-colonies/openapi-express-viewer": "^3.0.0",
     "@map-colonies/read-pkg": "0.0.1",
-    "@map-colonies/schemas": "^1.17.0",
+    "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-36806c74374e95e9771f7673691d2fd33cb2337e.tgz",
     "@map-colonies/telemetry": "^10.0.1",
     "@map-colonies/tile-calc": "0.1.3",
     "@map-colonies/tsconfig": "^1.0.1",
@@ -55,7 +55,7 @@
     "express": "^4.19.2",
     "express-openapi-validator": "^5.0.4",
     "http-status-codes": "^2.2.0",
-    "pg-boss": "^7.1.0",
+    "pg-boss": "^12.8.0",
     "prom-client": "^15.1.2",
     "reflect-metadata": "^0.1.13",
     "snake-case": "^3.0.4",

src/common/config.ts

@@ -1,8 +1,8 @@
 import { type ConfigInstance, config } from '@map-colonies/config';
-import { vectorMetatileQueuePopulatorFullV1, type vectorMetatileQueuePopulatorFullV1Type } from '@map-colonies/schemas';
+import { vectorMetatileQueuePopulatorFullV2, type vectorMetatileQueuePopulatorFullV2Type } from '@map-colonies/schemas';
 
 // Choose here the type of the config instance and import this type from the entire application
-type ConfigType = ConfigInstance<vectorMetatileQueuePopulatorFullV1Type>;
+type ConfigType = ConfigInstance<vectorMetatileQueuePopulatorFullV2Type>;
 
 let configInstance: ConfigType | undefined;
 
@@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined;
  */
 async function initConfig(offlineMode?: boolean): Promise<void> {
   configInstance = await config({
-    schema: vectorMetatileQueuePopulatorFullV1,
+    schema: vectorMetatileQueuePopulatorFullV2,
     offlineMode,
   });
 }

src/common/constants.ts

@@ -21,5 +21,6 @@ export const ON_SIGNAL = Symbol('onSignal');
 
 export const CONSUME_AND_POPULATE_FACTORY = Symbol('consumeAndPopulateFactory');
 export const JOB_QUEUE_PROVIDER = Symbol('JobQueueProvider');
+export const QUEUE_NAMES = Symbol('QueueNames');
 
 export const MILLISECONDS_IN_SECOND = 1000;