From 29735c34415d2f02d807e362bcd30ca2211e874d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 12:48:30 +0300 Subject: [PATCH 01/13] feat: created schemas --- .../vector/retiler/{ => full}/v1.schema.json | 88 +++---- schemas/vector/retiler/shared/v1.schema.json | 219 ++++++++++++++++++ 2 files changed, 251 insertions(+), 56 deletions(-) rename schemas/vector/retiler/{ => full}/v1.schema.json (81%) create mode 100644 schemas/vector/retiler/shared/v1.schema.json diff --git a/schemas/vector/retiler/v1.schema.json b/schemas/vector/retiler/full/v1.schema.json similarity index 81% rename from schemas/vector/retiler/v1.schema.json rename to schemas/vector/retiler/full/v1.schema.json index 7919939c..9c9dd8d7 100644 --- a/schemas/vector/retiler/v1.schema.json +++ b/schemas/vector/retiler/full/v1.schema.json @@ -1,11 +1,11 @@ { - "$id": "https://mapcolonies.com/vector/retiler/v1", + "$id": "https://mapcolonies.com/vector/retiler/full/v1", "type": "object", - "title": "vectorRetilerSchemaV1", - "description": "Vector's Retiler schema", + "title": "vectorRetilerFullSchemaV1", + "description": "Vector's Retiler full schema", "allOf": [ { - "$ref": "https://mapcolonies.com/common/boilerplate/v2" + "$ref": "https://mapcolonies.com/vector/retiler/shared/v1" }, { "type": "object", @@ -14,6 +14,7 @@ "app": { "type": "object", "description": "Application configuration", + "unevaluatedProperties": false, "required": ["queueName", "map", "project", "tilesStorage", "jobQueue", "parallelism"], "properties": { "project": { @@ -47,19 +48,12 @@ "type": "object", "description": "Tiles storage configuration", "required": ["providers", "layout"], - "unevaluatedProperties": false, "properties": { "providers": { "$ref": "#/definitions/tilesStorageProviders" }, "layout": { "$ref": "#/definitions/tilesStorageLayout" - }, - "shouldFilterBlankTiles": { - "type": "boolean", - "description": "Should filter blank tiles", - "default": false, - "x-env-value": "TILES_STORAGE_SHOULD_FILTER_BLANK_TILES" } } } @@ -132,27 +126,12 @@ }, "jobQueue": { "type": "object", - "required": ["waitTimeout", "pgBoss"], + "required": ["pgBoss"], "properties": { - "waitTimeout": { - "type": "integer", - "description": "Job queue wait timeout in milliseconds", - "x-env-value": "QUEUE_WAIT_TIMEOUT" - }, "pgBoss": { "allOf": [ { "$ref": "https://mapcolonies.com/common/db/full/v2" - }, - { - "type": "object", - "properties": { - "noSupervisor": { - "type": "boolean", - "default": true, - "description": "Disable pgBoss supervisor" - } - } } ] } @@ -161,7 +140,7 @@ "map": { "type": "object", "description": "DigitalMap provider configuration", - "required": ["provider", "url", "format", "wms", "client"], + "required": ["provider", "url", "wms"], "properties": { "provider": { "enum": ["wms", "arcgis"], @@ -173,26 +152,8 @@ "description": "DigitalMap provider URL", "x-env-value": "MAP_URL" }, - "format": { - "enum": ["jpg", "jpeg", "image/jpeg", "image/png", "image/webp"], - "description": "DigitalMap provider output format", - "default": "image/png", - "x-env-value": "MAP_FORMAT" - }, - "client": { - "type": "object", - "required": ["timeoutMs"], - "properties": { - "timeoutMs": { - "type": "integer", - "description": "DigitalMap provider client timeout in milliseconds", - "x-env-value": "MAP_CLIENT_TIMEOUT_MS" - } - } - }, "wms": { "type": "object", - "required": ["version", "layers", "styles"], "properties": { "version": { "$comment": "Should I use enum instead of string?", @@ -220,12 +181,33 @@ } } } + }, + "if": { + "properties": { + "provider": { + "const": "wms" + } + }, + "then": { + "required": ["wms"], + "properties": { + "wms": { + "required": ["version", "layers", "styles"] + } + } + }, + "else": { + "properties": { + "wms": { + "required": ["version"] + } + } + } } }, "s3TilesStorageProvider": { "type": "object", "required": ["kind", "endpoint", "bucketName", "region", "credentials"], - "unevaluatedProperties": false, "properties": { "kind": { "type": "string", @@ -250,7 +232,7 @@ "credentials": { "type": "object", "required": ["accessKeyId", "secretAccessKey"], - "unevaluatedProperties": false, + "properties": { "accessKeyId": { "type": "string" @@ -265,7 +247,6 @@ "fsTilesStorageProvider": { "type": "object", "required": ["kind", "basePath"], - "unevaluatedProperties": false, "description": "File system tiles storage provider", "properties": { "kind": { @@ -280,6 +261,7 @@ }, "tilesStorageProviders": { "type": "array", + "minItems": 1, "items": { "anyOf": [ { @@ -296,18 +278,12 @@ "tilesStorageLayout": { "type": "object", "required": ["format"], - "unevaluatedProperties": false, "properties": { "format": { "type": "string", + "pattern": "^.+\\{z\\}\\/\\{x\\}\\/\\{y\\}\\..+$", "description": "Tiles storage layout format", "x-env-value": "TILES_STORAGE_LAYOUT_FORMAT" - }, - "shouldFlipY": { - "type": "boolean", - "description": "Should flip Y", - "default": true, - "x-env-value": "TILES_STORAGE_LAYOUT_SHOULD_FLIP_Y" } } }, diff --git a/schemas/vector/retiler/shared/v1.schema.json b/schemas/vector/retiler/shared/v1.schema.json new file mode 100644 index 00000000..35ad4805 --- /dev/null +++ b/schemas/vector/retiler/shared/v1.schema.json @@ -0,0 +1,219 @@ +{ + "$id": "https://mapcolonies.com/vector/retiler/shared/v1", + "type": "object", + "title": "vectorRetilerSharedSchemaV1", + "description": "Vector's Retiler schema", + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/boilerplate/v2" + }, + { + "type": "object", + "required": ["app"], + "properties": { + "app": { + "type": "object", + "description": "Application configuration", + "required": ["map", "tilesStorage", "jobQueue", "parallelism"], + "properties": { + "forceProcess": { + "type": "boolean", + "description": "Force process all tiles", + "default": false, + "x-env-value": "APP_FORCE_PROCESS" + }, + "parallelism": { + "type": "integer", + "description": "Number of parallel jobs", + "default": 1, + "x-env-value": "APP_PARALLELISM" + }, + "jobQueue": { + "$ref": "#/definitions/jobQueue" + }, + "map": { + "$ref": "#/definitions/map" + }, + "tilesStorage": { + "type": "object", + "description": "Tiles storage configuration", + "required": ["layout"], + "properties": { + "layout": { + "$ref": "#/definitions/tilesStorageLayout" + }, + "shouldFilterBlankTiles": { + "type": "boolean", + "description": "Should filter blank tiles", + "default": false, + "x-env-value": "TILES_STORAGE_SHOULD_FILTER_BLANK_TILES" + } + } + } + } + } + } + }, + { + "type": "object", + "required": ["detiler"], + "properties": { + "detiler": { + "type": "object", + "description": "Detiler configuration", + "required": ["enabled"], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable Detiler", + "default": false, + "x-env-value": "DETILER_ENABLED" + }, + "proceedOnFailure": { + "type": "boolean", + "default": false, + "x-env-value": "DETILER_PROCEED_ON_FAILURE" + }, + "client": { + "$ref": "#/definitions/detilerClient" + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["client"] + }, + "else": { + "properties": { + "enabled": { + "const": false + } + } + } + } + } + } + ], + "definitions": { + "jobQueue": { + "type": "object", + "required": ["waitTimeout"], + "properties": { + "waitTimeout": { + "type": "integer", + "default": 60000, + "description": "Job queue wait timeout in milliseconds", + "x-env-value": "QUEUE_WAIT_TIMEOUT" + }, + "pgBoss": { + "allOf": [ + { + "type": "object", + "properties": { + "noSupervisor": { + "type": "boolean", + "default": true, + "description": "Disable pgBoss supervisor" + } + } + } + ] + } + } + }, + "map": { + "type": "object", + "description": "DigitalMap provider configuration", + "required": ["format", "client"], + "properties": { + "format": { + "enum": ["jpg", "jpeg", "image/jpeg", "image/png", "image/webp"], + "description": "DigitalMap provider output format", + "default": "image/png", + "x-env-value": "MAP_FORMAT" + }, + "client": { + "type": "object", + "required": ["timeoutMs"], + "properties": { + "timeoutMs": { + "type": "integer", + "description": "DigitalMap provider client timeout in milliseconds", + "x-env-value": "MAP_CLIENT_TIMEOUT_MS" + } + } + } + } + }, + "tilesStorageLayout": { + "type": "object", + "required": ["format"], + "properties": { + "shouldFlipY": { + "type": "boolean", + "description": "Should flip Y", + "default": true, + "x-env-value": "TILES_STORAGE_LAYOUT_SHOULD_FLIP_Y" + } + } + }, + "detilerClientRetryStrategy": { + "type": "object", + "required": ["retries", "shouldResetTimeout", "isExponential", "delay"], + "properties": { + "retries": { + "type": "integer", + "description": "Number of retries", + "default": 3, + "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_RETRIES" + }, + "shouldResetTimeout": { + "type": "boolean", + "description": "Should reset timeout", + "default": true, + "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_SHOULD_RESET_TIMEOUT" + }, + "isExponential": { + "type": "boolean", + "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_IS_EXPONENTIAL" + }, + "delay": { + "type": "integer", + "description": "Delay in milliseconds", + "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_DELAY" + } + } + }, + "detilerClient": { + "type": "object", + "required": ["url"], + "properties": { + "url": { + "type": "string", + "description": "Detiler client URL", + "x-env-value": "DETILER_CLIENT_URL" + }, + "timeout": { + "type": "integer", + "description": "Detiler client timeout in milliseconds", + "default": 60000, + "x-env-value": "DETILER_CLIENT_TIMEOUT" + }, + "enableRetryStrategy": { + "type": "boolean", + "default": false, + "description": "Enable retry strategy", + "x-env-value": "DETILER_CLIENT_ENABLE_RETRY_STRATEGY" + }, + "retryStrategy": { + "$ref": "#/definitions/detilerClientRetryStrategy" + } + } + } + } +} From c59a1f7b79e38216d9bea3b68ecbdc67bc5fc7d1 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 13:25:37 +0300 Subject: [PATCH 02/13] feat: changed s3 to reference s3 schema. removed detiler as it is shared --- schemas/vector/retiler/full/v1.schema.json | 161 ++------------------- 1 file changed, 12 insertions(+), 149 deletions(-) diff --git a/schemas/vector/retiler/full/v1.schema.json b/schemas/vector/retiler/full/v1.schema.json index 9c9dd8d7..b94f5164 100644 --- a/schemas/vector/retiler/full/v1.schema.json +++ b/schemas/vector/retiler/full/v1.schema.json @@ -60,50 +60,6 @@ } } } - }, - { - "type": "object", - "required": ["detiler"], - "properties": { - "detiler": { - "type": "object", - "description": "Detiler configuration", - "required": ["enabled"], - "properties": { - "enabled": { - "type": "boolean", - "description": "Enable Detiler", - "default": false, - "x-env-value": "DETILER_ENABLED" - }, - "proceedOnFailure": { - "type": "boolean", - "default": false, - "x-env-value": "DETILER_PROCEED_ON_FAILURE" - }, - "client": { - "$ref": "#/definitions/detilerClient" - } - }, - "if": { - "properties": { - "enabled": { - "const": true - } - } - }, - "then": { - "required": ["client"] - }, - "else": { - "properties": { - "enabled": { - "const": false - } - } - } - } - } } ], "definitions": { @@ -154,6 +110,7 @@ }, "wms": { "type": "object", + "required": ["version"], "properties": { "version": { "$comment": "Should I use enum instead of string?", @@ -187,62 +144,21 @@ "provider": { "const": "wms" } - }, - "then": { - "required": ["wms"], - "properties": { - "wms": { - "required": ["version", "layers", "styles"] - } - } - }, - "else": { - "properties": { - "wms": { - "required": ["version"] - } + } + }, + "then": { + "required": ["wms"], + "properties": { + "wms": { + "type": "object", + "required": ["layers", "styles"] } } - } + }, + "else": {} }, "s3TilesStorageProvider": { - "type": "object", - "required": ["kind", "endpoint", "bucketName", "region", "credentials"], - "properties": { - "kind": { - "type": "string", - "const": "s3" - }, - "endpoint": { - "type": "string", - "description": "S3 endpoint" - }, - "bucketName": { - "type": "string", - "description": "S3 bucket name" - }, - "region": { - "type": "string", - "description": "S3 region" - }, - "forcePathStyle": { - "type": "boolean", - "description": "Force path style" - }, - "credentials": { - "type": "object", - "required": ["accessKeyId", "secretAccessKey"], - - "properties": { - "accessKeyId": { - "type": "string" - }, - "secretAccessKey": { - "type": "string" - } - } - } - } + "$ref": "https://mapcolonies.com/common/s3/full/v1" }, "fsTilesStorageProvider": { "type": "object", @@ -286,59 +202,6 @@ "x-env-value": "TILES_STORAGE_LAYOUT_FORMAT" } } - }, - "detilerClientRetryStrategy": { - "type": "object", - "required": ["retries", "shouldResetTimeout", "isExponential", "delay"], - "properties": { - "retries": { - "type": "integer", - "description": "Number of retries", - "default": 3, - "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_RETRIES" - }, - "shouldResetTimeout": { - "type": "boolean", - "description": "Should reset timeout", - "default": true, - "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_SHOULD_RESET_TIMEOUT" - }, - "isExponential": { - "type": "boolean", - "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_IS_EXPONENTIAL" - }, - "delay": { - "type": "integer", - "description": "Delay in milliseconds", - "x-env-value": "DETILER_CLIENT_RETRY_STRATEGY_DELAY" - } - } - }, - "detilerClient": { - "type": "object", - "required": ["url"], - "properties": { - "url": { - "type": "string", - "description": "Detiler client URL", - "x-env-value": "DETILER_CLIENT_URL" - }, - "timeout": { - "type": "integer", - "description": "Detiler client timeout in milliseconds", - "default": 60000, - "x-env-value": "DETILER_CLIENT_TIMEOUT" - }, - "enableRetryStrategy": { - "type": "boolean", - "default": false, - "description": "Enable retry strategy", - "x-env-value": "DETILER_CLIENT_ENABLE_RETRY_STRATEGY" - }, - "retryStrategy": { - "$ref": "#/definitions/detilerClientRetryStrategy" - } - } } } } From 935feefe194ff7bda2c5349fa8ea744fb2dc2f1a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 13:33:01 +0300 Subject: [PATCH 03/13] fix: added back missing kind s3 const --- schemas/vector/retiler/full/v1.schema.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/schemas/vector/retiler/full/v1.schema.json b/schemas/vector/retiler/full/v1.schema.json index b94f5164..a1abc1ac 100644 --- a/schemas/vector/retiler/full/v1.schema.json +++ b/schemas/vector/retiler/full/v1.schema.json @@ -158,7 +158,22 @@ "else": {} }, "s3TilesStorageProvider": { - "$ref": "https://mapcolonies.com/common/s3/full/v1" + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/s3/full/v1" + }, + { + "type": "object", + "required": ["kind", "bucket"], + "description": "S3 tiles storage provider", + "properties": { + "kind": { + "type": "string", + "const": "s3" + } + } + } + ] }, "fsTilesStorageProvider": { "type": "object", From a267d310894dca91f47d9802a971b63c0f7c1f47 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 13:45:53 +0300 Subject: [PATCH 04/13] fix: set unevaluatedProperties in top level. reference s3 v2 schema --- schemas/vector/retiler/full/v1.schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schemas/vector/retiler/full/v1.schema.json b/schemas/vector/retiler/full/v1.schema.json index a1abc1ac..e15ef530 100644 --- a/schemas/vector/retiler/full/v1.schema.json +++ b/schemas/vector/retiler/full/v1.schema.json @@ -3,6 +3,7 @@ "type": "object", "title": "vectorRetilerFullSchemaV1", "description": "Vector's Retiler full schema", + "unevaluatedProperties": false, "allOf": [ { "$ref": "https://mapcolonies.com/vector/retiler/shared/v1" @@ -14,7 +15,6 @@ "app": { "type": "object", "description": "Application configuration", - "unevaluatedProperties": false, "required": ["queueName", "map", "project", "tilesStorage", "jobQueue", "parallelism"], "properties": { "project": { @@ -160,7 +160,7 @@ "s3TilesStorageProvider": { "allOf": [ { - "$ref": "https://mapcolonies.com/common/s3/full/v1" + "$ref": "https://mapcolonies.com/common/s3/full/v2" }, { "type": "object", From e06cc9449c7894c1954174ea3c99b1004cade800 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 16:19:43 +0300 Subject: [PATCH 05/13] fix: removed not needed required --- schemas/vector/retiler/full/v1.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/vector/retiler/full/v1.schema.json b/schemas/vector/retiler/full/v1.schema.json index e15ef530..d76ad262 100644 --- a/schemas/vector/retiler/full/v1.schema.json +++ b/schemas/vector/retiler/full/v1.schema.json @@ -164,7 +164,7 @@ }, { "type": "object", - "required": ["kind", "bucket"], + "required": ["kind"], "description": "S3 tiles storage provider", "properties": { "kind": { From 8e91b283f79a7164f6e8a015211529091df6e953 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 16:33:15 +0300 Subject: [PATCH 06/13] feat: created retiler shared config --- schemas/vector/retiler/shared/v1.configs.json | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 schemas/vector/retiler/shared/v1.configs.json diff --git a/schemas/vector/retiler/shared/v1.configs.json b/schemas/vector/retiler/shared/v1.configs.json new file mode 100644 index 00000000..add57385 --- /dev/null +++ b/schemas/vector/retiler/shared/v1.configs.json @@ -0,0 +1,57 @@ +[ + { + "name": "vector-retiler-shared", + "value": { + "telemetry": { + "shared": { + "serviceName": "retiler" + }, + "tracing": { + "isEnabled": false + }, + "logger": { + "level": "info", + "prettyPrint": false + } + }, + "openapiConfig": { + "filePath": "./openapi3.yaml", + "basePath": "/docs", + "rawPath": "/api", + "uiPath": "/api" + }, + "server": { + "port": 8080, + "request": { + "payload": { + "limit": "1mb" + } + }, + "response": {} + }, + "app": { + "forceProcess": false, + "jobQueue": { + "pgBoss": { "noSupervisor": true }, + "waitTimeout": 60000 + }, + "map": { + "format": "image/png", + "client": { + "timeoutMs": 600000 + } + }, + "parallelism": 15, + "tilesStorage": { + "layout": { + "shouldFlipY": true + }, + "shouldFilterBlankTiles": true + } + }, + "detiler": { + "enabled": false + } + } + } +] From fdaf6bcd32b0e87ce3504a997c886ee044dcc255 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 16:33:37 +0300 Subject: [PATCH 07/13] fix: removed unnecessary required --- schemas/vector/retiler/shared/v1.schema.json | 1 - 1 file changed, 1 deletion(-) diff --git a/schemas/vector/retiler/shared/v1.schema.json b/schemas/vector/retiler/shared/v1.schema.json index 35ad4805..3573e576 100644 --- a/schemas/vector/retiler/shared/v1.schema.json +++ b/schemas/vector/retiler/shared/v1.schema.json @@ -152,7 +152,6 @@ }, "tilesStorageLayout": { "type": "object", - "required": ["format"], "properties": { "shouldFlipY": { "type": "boolean", From 2d14e756d55c471ffa4e1fce86e7953c674ccd6b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 17:00:23 +0300 Subject: [PATCH 08/13] feat: changed mqp to shared-full schema design. created default config --- .../full/v1.schema.json | 45 ++++++++++++ .../shared/v1.configs.json | 71 +++++++++++++++++++ .../{ => shared}/v1.schema.json | 27 ++----- 3 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 schemas/vector/metatileQueuePopulator/full/v1.schema.json create mode 100644 schemas/vector/metatileQueuePopulator/shared/v1.configs.json rename schemas/vector/metatileQueuePopulator/{ => shared}/v1.schema.json (88%) diff --git a/schemas/vector/metatileQueuePopulator/full/v1.schema.json b/schemas/vector/metatileQueuePopulator/full/v1.schema.json new file mode 100644 index 00000000..8bb07252 --- /dev/null +++ b/schemas/vector/metatileQueuePopulator/full/v1.schema.json @@ -0,0 +1,45 @@ +{ + "$id": "https://mapcolonies.com/vector/metatileQueuePopulator/full/v1", + "type": "object", + "title": "metatileQueuePopulatorV1", + "description": "Metatile Queue Populator's full schema", + "unevaluatedProperties": false, + "allOf": [ + { + "$ref": "https://mapcolonies.com/vector/metatileQueuePopulator/shared/v1" + }, + { + "type": "object", + "required": ["db"], + "properties": { + "db": { + "$ref": "https://mapcolonies.com/common/db/full/v2" + } + } + }, + { + "$ref": "#/definitions/appSchema" + } + ], + "definitions": { + "appSchema": { + "type": "object", + "required": ["app"], + "properties": { + "app": { + "description": "app configuration", + "type": "object", + "default": {}, + "properties": { + "projectName": { + "type": "string", + "description": "The name of the project", + "x-env-value": "APP_PROJECT_NAME" + } + }, + "required": ["projectName"] + } + } + } + } +} diff --git a/schemas/vector/metatileQueuePopulator/shared/v1.configs.json b/schemas/vector/metatileQueuePopulator/shared/v1.configs.json new file mode 100644 index 00000000..aeb6d3e7 --- /dev/null +++ b/schemas/vector/metatileQueuePopulator/shared/v1.configs.json @@ -0,0 +1,71 @@ +[ + { + "name": "vector-metatileQueuePopulator-shared", + "value": { + "telemetry": { + "shared": { + "serviceName": "metatileQueuePopulator" + }, + "tracing": { + "isEnabled": false + }, + "logger": { + "level": "info", + "prettyPrint": false + }, + "metrics": { + "enabled": true, + "buckets": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 50, 250, 500] + } + }, + "openapiConfig": { + "filePath": "./openapi3.yaml", + "basePath": "/docs", + "rawPath": "/api", + "uiPath": "/api" + }, + "server": { + "port": 8080, + "request": { + "payload": { + "limit": "1mb" + } + }, + "response": { + "compression": { + "enabled": true, + "options": null + } + } + }, + "app": { + "consumeDelay": { + "enabled": true, + "delaySec": 60, + "tilesQueueSizeLimit": 400000 + }, + "enableRequestQueueHandling": true, + "force": { + "api": false, + "expiredTiles": false + }, + "metatileSize": 8, + "requestQueueCheckIntervalSec": 30, + "tilesBatchSize": 10000 + }, + "db": { + "$ref": { "configName": "db-connection", "version": "latest" }, + "ssl": { + "enabled": false + } + }, + "queue": { + "expireInSeconds": 600, + "retryBackoff": true, + "retryLimit": 3, + "retryDelaySeconds": 60, + "retentionHours": 87660 + } + } + } +] diff --git a/schemas/vector/metatileQueuePopulator/v1.schema.json b/schemas/vector/metatileQueuePopulator/shared/v1.schema.json similarity index 88% rename from schemas/vector/metatileQueuePopulator/v1.schema.json rename to schemas/vector/metatileQueuePopulator/shared/v1.schema.json index 1cec9e63..43aec391 100644 --- a/schemas/vector/metatileQueuePopulator/v1.schema.json +++ b/schemas/vector/metatileQueuePopulator/shared/v1.schema.json @@ -1,8 +1,8 @@ { - "$id": "https://mapcolonies.com/vector/metatileQueuePopulator/v1", + "$id": "https://mapcolonies.com/vector/metatileQueuePopulator/shared/v1", "type": "object", - "title": "metatileQueuePopulatorV1", - "description": "Metatile Queue Populator's schema", + "title": "metatileQueuePopulatorSharedV1", + "description": "Metatile Queue Populator's shared schema", "allOf": [ { "$ref": "https://mapcolonies.com/common/boilerplate/v2" @@ -19,7 +19,7 @@ "required": ["db"], "properties": { "db": { - "$ref": "https://mapcolonies.com/common/db/full/v1" + "$ref": "https://mapcolonies.com/common/db/partial/v1" } } }, @@ -38,14 +38,7 @@ "app": { "description": "app configuration", "type": "object", - "default": {}, - "unevaluatedProperties": false, "properties": { - "projectName": { - "type": "string", - "description": "The name of the project", - "x-env-value": "APP_PROJECT_NAME" - }, "enableRequestQueueHandling": { "type": "boolean", "description": "Flag to enable request queue handling", @@ -121,15 +114,7 @@ } } }, - "required": [ - "projectName", - "enableRequestQueueHandling", - "requestQueueCheckIntervalSec", - "tilesBatchSize", - "metatileSize", - "consumeDelay", - "force" - ] + "required": ["enableRequestQueueHandling", "requestQueueCheckIntervalSec", "tilesBatchSize", "metatileSize", "consumeDelay", "force"] } } }, @@ -141,7 +126,6 @@ "description": "queue configuration", "type": "object", "default": {}, - "unevaluatedProperties": false, "properties": { "expireInSeconds": { "type": "integer", @@ -186,7 +170,6 @@ "description": "metrics configuration", "type": "object", "default": {}, - "unevaluatedProperties": false, "properties": { "enabled": { "type": "boolean", From 02cfc613894cb403ccef459398ed28834bbf0fcd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 17:13:42 +0300 Subject: [PATCH 09/13] fix: title name --- schemas/vector/metatileQueuePopulator/full/v1.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/vector/metatileQueuePopulator/full/v1.schema.json b/schemas/vector/metatileQueuePopulator/full/v1.schema.json index 8bb07252..f9466b99 100644 --- a/schemas/vector/metatileQueuePopulator/full/v1.schema.json +++ b/schemas/vector/metatileQueuePopulator/full/v1.schema.json @@ -1,7 +1,7 @@ { "$id": "https://mapcolonies.com/vector/metatileQueuePopulator/full/v1", "type": "object", - "title": "metatileQueuePopulatorV1", + "title": "metatileQueuePopulatorFullV1", "description": "Metatile Queue Populator's full schema", "unevaluatedProperties": false, "allOf": [ From a885d84b5fb252a73ce06b4391ce0974b8977a71 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 17:14:41 +0300 Subject: [PATCH 10/13] fix: added back missing else statement --- .../vector/metatileQueuePopulator/shared/v1.schema.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/schemas/vector/metatileQueuePopulator/shared/v1.schema.json b/schemas/vector/metatileQueuePopulator/shared/v1.schema.json index 43aec391..90ac1368 100644 --- a/schemas/vector/metatileQueuePopulator/shared/v1.schema.json +++ b/schemas/vector/metatileQueuePopulator/shared/v1.schema.json @@ -93,6 +93,13 @@ }, "then": { "required": ["delaySec", "tilesQueueSizeLimit"] + }, + "else": { + "properties": { + "enabled": { + "const": false + } + } } }, "force": { From 0cf8a23531915e7f35212371b4188662c2869afc Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 16 Apr 2025 17:57:48 +0300 Subject: [PATCH 11/13] fix: changed tilesStorage.providers to be shared as all tiles stored in the same bucket --- schemas/vector/retiler/full/v1.schema.json | 54 +------------ schemas/vector/retiler/shared/v1.configs.json | 1 + schemas/vector/retiler/shared/v1.schema.json | 75 ++++++++++++++++++- 3 files changed, 76 insertions(+), 54 deletions(-) diff --git a/schemas/vector/retiler/full/v1.schema.json b/schemas/vector/retiler/full/v1.schema.json index d76ad262..9f4c0e12 100644 --- a/schemas/vector/retiler/full/v1.schema.json +++ b/schemas/vector/retiler/full/v1.schema.json @@ -47,11 +47,8 @@ "tilesStorage": { "type": "object", "description": "Tiles storage configuration", - "required": ["providers", "layout"], + "required": ["layout"], "properties": { - "providers": { - "$ref": "#/definitions/tilesStorageProviders" - }, "layout": { "$ref": "#/definitions/tilesStorageLayout" } @@ -157,55 +154,6 @@ }, "else": {} }, - "s3TilesStorageProvider": { - "allOf": [ - { - "$ref": "https://mapcolonies.com/common/s3/full/v2" - }, - { - "type": "object", - "required": ["kind"], - "description": "S3 tiles storage provider", - "properties": { - "kind": { - "type": "string", - "const": "s3" - } - } - } - ] - }, - "fsTilesStorageProvider": { - "type": "object", - "required": ["kind", "basePath"], - "description": "File system tiles storage provider", - "properties": { - "kind": { - "type": "string", - "const": "fs" - }, - "basePath": { - "type": "string", - "description": "File system path" - } - } - }, - "tilesStorageProviders": { - "type": "array", - "minItems": 1, - "items": { - "anyOf": [ - { - "$ref": "#/definitions/s3TilesStorageProvider" - }, - { - "$ref": "#/definitions/fsTilesStorageProvider" - } - ] - }, - "x-env-value": "TILES_STORAGE_PROVIDERS", - "x-env-format": "json" - }, "tilesStorageLayout": { "type": "object", "required": ["format"], diff --git a/schemas/vector/retiler/shared/v1.configs.json b/schemas/vector/retiler/shared/v1.configs.json index add57385..159d73cf 100644 --- a/schemas/vector/retiler/shared/v1.configs.json +++ b/schemas/vector/retiler/shared/v1.configs.json @@ -43,6 +43,7 @@ }, "parallelism": 15, "tilesStorage": { + "providers": [], "layout": { "shouldFlipY": true }, diff --git a/schemas/vector/retiler/shared/v1.schema.json b/schemas/vector/retiler/shared/v1.schema.json index 3573e576..94a0f7e9 100644 --- a/schemas/vector/retiler/shared/v1.schema.json +++ b/schemas/vector/retiler/shared/v1.schema.json @@ -37,8 +37,11 @@ "tilesStorage": { "type": "object", "description": "Tiles storage configuration", - "required": ["layout"], + "required": ["layout", "providers"], "properties": { + "providers": { + "$ref": "#/definitions/tilesStorageProviders" + }, "layout": { "$ref": "#/definitions/tilesStorageLayout" }, @@ -150,6 +153,76 @@ } } }, + "s3TilesStorageProvider": { + "type": "object", + "required": ["kind", "endpoint", "bucketName", "region", "credentials"], + "unevaluatedProperties": false, + "properties": { + "kind": { + "type": "string", + "const": "s3" + }, + "endpoint": { + "type": "string", + "description": "S3 endpoint" + }, + "bucketName": { + "type": "string", + "description": "S3 bucket name" + }, + "region": { + "type": "string", + "description": "S3 region" + }, + "forcePathStyle": { + "type": "boolean", + "description": "Force path style" + }, + "credentials": { + "type": "object", + "required": ["accessKeyId", "secretAccessKey"], + "unevaluatedProperties": false, + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + } + } + } + }, + "fsTilesStorageProvider": { + "type": "object", + "required": ["kind", "basePath"], + "description": "File system tiles storage provider", + "properties": { + "kind": { + "type": "string", + "const": "fs" + }, + "basePath": { + "type": "string", + "description": "File system path" + } + } + }, + "tilesStorageProviders": { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/s3TilesStorageProvider" + }, + { + "$ref": "#/definitions/fsTilesStorageProvider" + } + ] + }, + "x-env-value": "TILES_STORAGE_PROVIDERS", + "x-env-format": "json" + }, "tilesStorageLayout": { "type": "object", "properties": { From e9049bea238c046ef28f134cb9e22012529596b0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Sun, 20 Apr 2025 15:45:05 +0300 Subject: [PATCH 12/13] fix: change name --- schemas/vector/metatileQueuePopulator/shared/v1.configs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/vector/metatileQueuePopulator/shared/v1.configs.json b/schemas/vector/metatileQueuePopulator/shared/v1.configs.json index aeb6d3e7..3e73041e 100644 --- a/schemas/vector/metatileQueuePopulator/shared/v1.configs.json +++ b/schemas/vector/metatileQueuePopulator/shared/v1.configs.json @@ -1,6 +1,6 @@ [ { - "name": "vector-metatileQueuePopulator-shared", + "name": "vector-metatilequeuepopulator-shared", "value": { "telemetry": { "shared": { From ae88b3542f8b0fd473f983035a10b2ef4acd8011 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Mon, 21 Apr 2025 17:55:54 +0300 Subject: [PATCH 13/13] fix: changed anyOf to oneOf --- schemas/vector/retiler/shared/v1.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/vector/retiler/shared/v1.schema.json b/schemas/vector/retiler/shared/v1.schema.json index 94a0f7e9..d58e71b2 100644 --- a/schemas/vector/retiler/shared/v1.schema.json +++ b/schemas/vector/retiler/shared/v1.schema.json @@ -211,7 +211,7 @@ "tilesStorageProviders": { "type": "array", "items": { - "anyOf": [ + "oneOf": [ { "$ref": "#/definitions/s3TilesStorageProvider" },