From c220c13b2b67ea93923aa832465fb8450e35da31 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:14:02 +0000 Subject: [PATCH 1/6] feat: created v2 MQP schemas --- .../full/v2.schema.json | 45 ++++ .../shared/v2.schema.json | 208 ++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 schemas/vector/metatileQueuePopulator/full/v2.schema.json create mode 100644 schemas/vector/metatileQueuePopulator/shared/v2.schema.json diff --git a/schemas/vector/metatileQueuePopulator/full/v2.schema.json b/schemas/vector/metatileQueuePopulator/full/v2.schema.json new file mode 100644 index 00000000..ff14c57a --- /dev/null +++ b/schemas/vector/metatileQueuePopulator/full/v2.schema.json @@ -0,0 +1,45 @@ +{ + "$id": "https://mapcolonies.com/vector/metatileQueuePopulator/full/v2", + "type": "object", + "title": "metatileQueuePopulatorFullV2", + "description": "Metatile Queue Populator's full schema", + "unevaluatedProperties": false, + "allOf": [ + { + "$ref": "https://mapcolonies.com/vector/metatileQueuePopulator/shared/v2" + }, + { + "type": "object", + "required": ["db"], + "properties": { + "db": { + "$ref": "https://mapcolonies.com/common/db/full/v2" + } + } + }, + { + "$ref": "#/definitions/appSchema" + } + ], + "definitions": { + "appSchema": { + "type": "object", + "required": ["app"], + "properties": { + "app": { + "description": "app configuration", + "type": "object", + "default": {}, + "properties": { + "projectName": { + "type": "string", + "description": "The name of the project", + "x-env-value": "APP_PROJECT_NAME" + } + }, + "required": ["projectName"] + } + } + } + } +} diff --git a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json new file mode 100644 index 00000000..33529326 --- /dev/null +++ b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json @@ -0,0 +1,208 @@ +{ + "$id": "https://mapcolonies.com/vector/metatileQueuePopulator/shared/v2", + "type": "object", + "title": "metatileQueuePopulatorSharedV2", + "description": "Metatile Queue Populator's shared schema", + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/boilerplate/v2" + }, + { + "properties": { + "telemetry": { + "$ref": "#/definitions/metricsSchema" + } + } + }, + { + "type": "object", + "required": ["db"], + "properties": { + "db": { + "$ref": "https://mapcolonies.com/common/db/partial/v2" + } + } + }, + { + "$ref": "#/definitions/appSchema" + }, + { + "$ref": "#/definitions/queueSchema" + } + ], + "definitions": { + "appSchema": { + "type": "object", + "required": ["app"], + "properties": { + "app": { + "description": "app configuration", + "type": "object", + "properties": { + "enableRequestQueueHandling": { + "type": "boolean", + "description": "Flag to enable request queue handling", + "x-env-value": "APP_ENABLE_REQUEST_QUEUE_HANDLING", + "default": true + }, + "requestQueueCheckIntervalSec": { + "type": "number", + "description": "Interval in seconds to check the request queue", + "x-env-value": "APP_REQUEST_QUEUE_CHECK_INTERVAL_SEC", + "default": 30, + "exclusiveMinimum": 0 + }, + "tilesBatchSize": { + "type": "number", + "description": "Size of the tiles batch", + "x-env-value": "APP_TILES_BATCH_SIZE", + "default": 10000, + "exclusiveMinimum": 0 + }, + "metatileSize": { + "type": "number", + "description": "Size of the metatile", + "x-env-value": "APP_METATILE_SIZE", + "default": 8, + "exclusiveMinimum": 0 + }, + "consumeCondition": { + "type": "object", + "description": "Configuration for consume delay", + "properties": { + "enabled": { + "type": "boolean", + "description": "Flag to enable consume delay", + "x-env-value": "APP_CONSUME_CONDITION_ENABLED", + "default": false + }, + "conditionCheckIntervalSec": { + "type": "number", + "description": "number of seconds for consume delay", + "x-env-value": "APP_CONSUME_CONDITION_CHECK_INTERVAL_SEC", + "exclusiveMinimum": 0 + }, + "tilesQueueSizeLimit": { + "type": "number", + "description": "Limit of tiles queue size", + "x-env-value": "APP_CONSUME_CONDITION_TILES_QUEUE_SIZE_LIMIT", + "exclusiveMinimum": 0 + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["conditionCheckIntervalSec", "tilesQueueSizeLimit"] + }, + "else": { + "allOf": [{ "not": { "required": ["conditionCheckIntervalSec"] } }, { "not": { "required": ["tilesQueueSizeLimit"] } }], + "properties": { + "enabled": { + "const": false + } + } + } + }, + "force": { + "type": "object", + "description": "Force configuration", + "properties": { + "api": { + "type": "boolean", + "description": "Flag to force API", + "x-env-value": "FORCE_API", + "default": false + }, + "expiredTiles": { + "type": "boolean", + "description": "Flag to force expired tiles", + "x-env-value": "FORCE_EXPIRED_TILES", + "default": false + } + } + } + }, + "required": ["enableRequestQueueHandling", "requestQueueCheckIntervalSec", "tilesBatchSize", "metatileSize", "consumeCondition", "force"] + } + } + }, + "queueSchema": { + "type": "object", + "required": ["queue"], + "properties": { + "queue": { + "description": "queue configuration", + "type": "object", + "default": {}, + "properties": { + "expireInSeconds": { + "type": "number", + "description": "Time in seconds before the queue expires", + "x-env-value": "QUEUE_JOBS_EXPIRE_IN_SECONDS", + "default": 600, + "exclusiveMinimum": 0 + }, + "retryBackoff": { + "type": "boolean", + "description": "Flag to enable retry backoff", + "x-env-value": "QUEUE_JOBS_RETRY_BACKOFF", + "default": true + }, + "retryLimit": { + "type": "number", + "description": "Maximum number of retry attempts", + "x-env-value": "QUEUE_JOBS_RETRY_LIMIT", + "default": 3, + "exclusiveMinimum": 0 + }, + "retryDelaySeconds": { + "type": "number", + "description": "Delay in seconds between retry attempts", + "x-env-value": "QUEUE_JOBS_RETRY_DELAY_SECONDS", + "default": 60, + "exclusiveMinimum": 0 + }, + "retentionHours": { + "type": "number", + "description": "Time in hours to retain the queue", + "x-env-value": "QUEUE_JOBS_RETENTION_HOURS", + "default": 87660, + "exclusiveMinimum": 0 + } + }, + "required": ["expireInSeconds", "retryBackoff", "retryLimit", "retryDelaySeconds", "retentionHours"] + } + } + }, + "metricsSchema": { + "type": "object", + "required": ["metrics"], + "properties": { + "metrics": { + "description": "metrics configuration", + "type": "object", + "default": {}, + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "buckets": { + "type": "array", + "items": { + "type": "number" + }, + "default": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 50, 250, 500] + } + }, + "required": ["enabled", "buckets"] + } + } + } + } +} From cab0bb5affef4ecef06f77d1ba558e5c9865b837 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:14:38 +0000 Subject: [PATCH 2/6] feat: added create schema and migrate config --- .../shared/v2.schema.json | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json index 33529326..81f56989 100644 --- a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json +++ b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json @@ -19,7 +19,7 @@ "required": ["db"], "properties": { "db": { - "$ref": "https://mapcolonies.com/common/db/partial/v2" + "$ref": "#/definitions/dbSchema" } } }, @@ -203,6 +203,33 @@ "required": ["enabled", "buckets"] } } + }, + "dbSchema": { + "type": "object", + "required": ["db"], + "properties": { + "db": { + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/db/partial/v1" + }, + { + "createSchema": { + "type": "boolean", + "description": "Flag to enable schema creation if not exists", + "x-env-value": "DB_CREATE_SCHEMA", + "default": false + }, + "migrate": { + "type": "boolean", + "description": "Flag to enable migrations on startup", + "x-env-value": "DB_RUN_MIGRATE", + "default": false + } + } + ] + } + } } } } From 58631ca2fad0456721c1d59dbc3aaeb497459b32 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:14:48 +0000 Subject: [PATCH 3/6] feat: added v2 configs --- .../shared/v2.configs.json | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 schemas/vector/metatileQueuePopulator/shared/v2.configs.json diff --git a/schemas/vector/metatileQueuePopulator/shared/v2.configs.json b/schemas/vector/metatileQueuePopulator/shared/v2.configs.json new file mode 100644 index 00000000..92d79f9a --- /dev/null +++ b/schemas/vector/metatileQueuePopulator/shared/v2.configs.json @@ -0,0 +1,73 @@ +[ + { + "name": "vector-metatilequeuepopulator-shared", + "value": { + "telemetry": { + "shared": { + "serviceName": "metatileQueuePopulator" + }, + "tracing": { + "isEnabled": false + }, + "logger": { + "level": "info", + "prettyPrint": false + }, + "metrics": { + "enabled": true, + "buckets": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 50, 250, 500] + } + }, + "openapiConfig": { + "filePath": "./openapi3.yaml", + "basePath": "/docs", + "rawPath": "/api", + "uiPath": "/api" + }, + "server": { + "port": 8080, + "request": { + "payload": { + "limit": "1mb" + } + }, + "response": { + "compression": { + "enabled": true, + "options": null + } + } + }, + "app": { + "consumeCondition": { + "enabled": true, + "conditionCheckIntervalSec": 60, + "tilesQueueSizeLimit": 400000 + }, + "enableRequestQueueHandling": true, + "force": { + "api": false, + "expiredTiles": false + }, + "metatileSize": 8, + "requestQueueCheckIntervalSec": 30, + "tilesBatchSize": 10000 + }, + "db": { + "$ref": { "configName": "db-connection", "version": "latest", "schemaId": "https://mapcolonies.com/common/db/partial/v1" }, + "ssl": { + "enabled": false + }, + "createSchema": false, + "migrate": false + }, + "queue": { + "expireInSeconds": 600, + "retryBackoff": true, + "retryLimit": 3, + "retryDelaySeconds": 60, + "retentionHours": 87660 + } + } + } +] From 996fc5c83a153bbbf20a380523767a9d466ff737 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:23:03 +0000 Subject: [PATCH 4/6] fix: db schema ref --- .../shared/v2.schema.json | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json index 81f56989..b5774dab 100644 --- a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json +++ b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json @@ -17,11 +17,7 @@ { "type": "object", "required": ["db"], - "properties": { - "db": { - "$ref": "#/definitions/dbSchema" - } - } + "$ref": "#/definitions/dbSchema" }, { "$ref": "#/definitions/appSchema" @@ -100,7 +96,18 @@ "required": ["conditionCheckIntervalSec", "tilesQueueSizeLimit"] }, "else": { - "allOf": [{ "not": { "required": ["conditionCheckIntervalSec"] } }, { "not": { "required": ["tilesQueueSizeLimit"] } }], + "allOf": [ + { + "not": { + "required": ["conditionCheckIntervalSec"] + } + }, + { + "not": { + "required": ["tilesQueueSizeLimit"] + } + } + ], "properties": { "enabled": { "const": false @@ -214,17 +221,20 @@ "$ref": "https://mapcolonies.com/common/db/partial/v1" }, { - "createSchema": { - "type": "boolean", - "description": "Flag to enable schema creation if not exists", - "x-env-value": "DB_CREATE_SCHEMA", - "default": false - }, - "migrate": { - "type": "boolean", - "description": "Flag to enable migrations on startup", - "x-env-value": "DB_RUN_MIGRATE", - "default": false + "type": "object", + "properties": { + "createSchema": { + "type": "boolean", + "description": "Flag to enable schema creation if not exists", + "x-env-value": "DB_CREATE_SCHEMA", + "default": false + }, + "migrate": { + "type": "boolean", + "description": "Flag to enable migrations on startup", + "x-env-value": "DB_RUN_MIGRATE", + "default": false + } } } ] From 623fcb0273a40736510561559ec1a49e31f60e1b Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:32:48 +0000 Subject: [PATCH 5/6] fix: changed retentionHours to retentionSeconds --- schemas/vector/metatileQueuePopulator/shared/v2.schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json index b5774dab..fc50f9ae 100644 --- a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json +++ b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json @@ -174,10 +174,10 @@ "default": 60, "exclusiveMinimum": 0 }, - "retentionHours": { + "retentionSeconds": { "type": "number", - "description": "Time in hours to retain the queue", - "x-env-value": "QUEUE_JOBS_RETENTION_HOURS", + "description": "Time in seconds to retain the queue", + "x-env-value": "QUEUE_JOBS_RETENTION_SECONDS", "default": 87660, "exclusiveMinimum": 0 } From 7eb52fdaac0dc0037a8a1f8182a4318b2b2cb87b Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:57:00 +0000 Subject: [PATCH 6/6] fix: changed retentionHours leftovers --- schemas/vector/metatileQueuePopulator/shared/v2.configs.json | 2 +- schemas/vector/metatileQueuePopulator/shared/v2.schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schemas/vector/metatileQueuePopulator/shared/v2.configs.json b/schemas/vector/metatileQueuePopulator/shared/v2.configs.json index 92d79f9a..9f480c5b 100644 --- a/schemas/vector/metatileQueuePopulator/shared/v2.configs.json +++ b/schemas/vector/metatileQueuePopulator/shared/v2.configs.json @@ -66,7 +66,7 @@ "retryBackoff": true, "retryLimit": 3, "retryDelaySeconds": 60, - "retentionHours": 87660 + "retentionSeconds": 87660 } } } diff --git a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json index fc50f9ae..e0df83d2 100644 --- a/schemas/vector/metatileQueuePopulator/shared/v2.schema.json +++ b/schemas/vector/metatileQueuePopulator/shared/v2.schema.json @@ -182,7 +182,7 @@ "exclusiveMinimum": 0 } }, - "required": ["expireInSeconds", "retryBackoff", "retryLimit", "retryDelaySeconds", "retentionHours"] + "required": ["expireInSeconds", "retryBackoff", "retryLimit", "retryDelaySeconds", "retentionSeconds"] } } },