From 084495582a116591f6d4730503b4e047f2825197 Mon Sep 17 00:00:00 2001 From: RonitKissis Date: Tue, 30 Jul 2024 11:28:29 +0300 Subject: [PATCH 1/7] feat: add common schemas and ingestion trigger --- .../raster/common/httpRetry/v1.schema.json | 23 +++ .../raster/common/jobManager/v1.schema.json | 45 ++++++ .../raster/common/servicesUrls/v1.schema.json | 20 +++ .../ingestion/ingestionTrigger/v1.schema.json | 131 ++++++++++++++++++ 4 files changed, 219 insertions(+) create mode 100644 schemas/raster/common/httpRetry/v1.schema.json create mode 100644 schemas/raster/common/jobManager/v1.schema.json create mode 100644 schemas/raster/common/servicesUrls/v1.schema.json create mode 100644 schemas/raster/ingestion/ingestionTrigger/v1.schema.json diff --git a/schemas/raster/common/httpRetry/v1.schema.json b/schemas/raster/common/httpRetry/v1.schema.json new file mode 100644 index 00000000..cc54ead2 --- /dev/null +++ b/schemas/raster/common/httpRetry/v1.schema.json @@ -0,0 +1,23 @@ +{ + "description": "Http reply schema", + "$id": "https://mapcolonies.com/raster/common/httpRetry/v1", + "type": "object", + "properties": { + "attempts": { + "type": "number", + "x-env-value": "HTTP_RETRY_ATTEMPTS", + "default": 5 + }, + "delay": { + "type": "string", + "x-env-value": "HTTP_RETRY_DELAY", + "default": "exponential" + }, + "shouldResetTimeout": { + "type": "boolean", + "x-env-value": "DISABLE_HTTP_CLIENT_LOGS", + "default": true + } + }, + "required": ["attempts", "delay", "shouldResetTimeout"] +} diff --git a/schemas/raster/common/jobManager/v1.schema.json b/schemas/raster/common/jobManager/v1.schema.json new file mode 100644 index 00000000..07970db4 --- /dev/null +++ b/schemas/raster/common/jobManager/v1.schema.json @@ -0,0 +1,45 @@ +{ + "description": "Job Manager configurations schema", + "$id": "https://mapcolonies.com/raster/common/jobManager/v1", + "type": "object", + "properties": { + "jobDomain": { + "type": "string", + "x-env-value": "JOB_DOMAIN", + "default": "RASTER" + }, + "ingestionNewJobType": { + "type": "string", + "x-env-value": "INGESTION_NEW_JOB_TYPE" + }, + "initTaskType": { + "type": "string", + "x-env-value": "INIT_TASK_TYPE" + }, + "ingestionUpdateJobType": { + "type": "string", + "x-env-value": "INGESTION_UPDATE_JOB_TYPE" + }, + "ingestionSwapUpdateJobType": { + "type": "string", + "x-env-value": "INGESTION_SWAP_UPDATE_JOB_TYPE" + }, + "supportedIngestionSwapTypes": { + "type": "object", + "x-env-value": "SUPPORTED_INGESTION_SWAP_TYPES" + }, + "forbiddenJobTypesForParallelIngestion": { + "type": "object", + "x-env-value": "FORBIDDEN_TYPES_FOR_PARALLEL_INGESTION" + } + }, + "required": [ + "jobDomain", + "ingestionNewJobType", + "initTaskType", + "ingestionUpdateJobType", + "ingestionSwapUpdateJobType", + "supportedIngestionSwapTypes", + "forbiddenJobTypesForParallelIngestion" + ] +} diff --git a/schemas/raster/common/servicesUrls/v1.schema.json b/schemas/raster/common/servicesUrls/v1.schema.json new file mode 100644 index 00000000..9a09f4d8 --- /dev/null +++ b/schemas/raster/common/servicesUrls/v1.schema.json @@ -0,0 +1,20 @@ +{ + "description": "services Urls schema", + "$id": "https://mapcolonies.com/raster/common/servicesUrls/v1", + "type": "object", + "properties": { + "jobManagerURL": { + "type": "string", + "x-env-value": "JOB_MANAGER_URL" + }, + "mapProxyApiServiceUrl": { + "type": "string", + "x-env-value": "MAP_PROXY_API_SERVICE_URL" + }, + "catalogServiceURL": { + "type": "string", + "x-env-value": "CATALOG_SERVICE_URL" + } + }, + "required": ["jobManagerURL", "mapProxyApiServiceUrl", "catalogServiceURL"] +} diff --git a/schemas/raster/ingestion/ingestionTrigger/v1.schema.json b/schemas/raster/ingestion/ingestionTrigger/v1.schema.json new file mode 100644 index 00000000..d4876142 --- /dev/null +++ b/schemas/raster/ingestion/ingestionTrigger/v1.schema.json @@ -0,0 +1,131 @@ +{ + "$id": "https://mapcolonies.com/raster/ingestion/ingestionTrigger/v1", + "description": "The full ingestion trigger service schema", + "type": "object", + "required": ["services", "jobManager", "httpRetry", "validationValuesByInfo", "storageExplorer"], + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/boilerplate/v3" + }, + { + "type": "object", + "properties": { + "services": { + "$ref": "https://mapcolonies.com/raster/common/servicesUrls/v1" + } + } + }, + { + "type": "object", + "properties": { + "httpRetry": { + "$ref": "https://mapcolonies.com/raster/common/httpRetry/v1" + } + } + }, + { + "type": "object", + "properties": { + "jobManager": { + "$ref": "https://mapcolonies.com/raster/common/jobManager/v1" + } + } + }, + { + "type": "object", + "properties": { + "disableHttpClientLogs": { + "type": "boolean", + "x-env-value": "DISABLE_HTTP_CLIENT_LOGS", + "default": false + } + } + }, + { + "type": "object", + "properties": { + "storageExplorer": { + "$ref": "#/definitions/storageExplorer" + } + } + }, + { + "type": "object", + "properties": { + "validationValuesByInfo": { + "$ref": "#/definitions/validationValuesByInfo" + } + } + } + ], + "definitions": { + "storageExplorer": { + "type": "object", + "properties": { + "layerSourceDir": { + "type": "string", + "default": "/layerSources", + "x-env-value": "STORAGE_EXPLORER_LAYER_SOURCE_DIR" + }, + "displayNameDir": { + "type": "string", + "x-env-value": "STORAGE_EXPLORER_DISPLAY_NAME_DIR" + }, + "watchDirectory": { + "type": "string", + "default": "watch", + "x-env-value": "STORAGE_EXPLORER_WATCH_DIRECTORY" + }, + "validFileExtensions": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "x-env-value": "STORAGE_EXPLORER_VALID_FILE_EXTENSIONS" + } + }, + "required": ["displayNameDir", "validFileExtensions", "layerSourceDir", "watchDirectory"] + }, + "validationValuesByInfo": { + "type": "object", + "properties": { + "crs": { + "type": "array", + "items": { + "enum": [4326] + }, + "default": [4326], + "minItems": 1, + "x-env-value": "CRS" + }, + "fileFormat": { + "type": "array", + "items": { + "enum": ["GPKG"] + }, + "default": ["GPKG"], + "minItems": 1, + "x-env-value": "FILE_FORMAT" + }, + "tileSize": { + "type": "number", + "default": 256, + "const": 256, + "x-env-value": "TILE_SIZE" + }, + "resolutionFixedPointTolerance": { + "type": "number", + "default": 12, + "x-env-value": "RESOLUTION_FIXED_POINT_TOLERANCE" + }, + "extentBufferInMeters": { + "type": "number", + "default": 50, + "x-env-value": "EXTENT_BUFFER_IN_METERS" + } + }, + "required": ["crs", "fileFormat", "tileSize", "resolutionFixedPointTolerance", "extentBufferInMeters"] + } + } +} From b51ee67df072fa5fd007db60a79b825df95897a0 Mon Sep 17 00:00:00 2001 From: RonitKissis Date: Tue, 30 Jul 2024 11:31:12 +0300 Subject: [PATCH 2/7] feat: update boilerplate to v4 --- schemas/raster/ingestion/ingestionTrigger/v1.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/raster/ingestion/ingestionTrigger/v1.schema.json b/schemas/raster/ingestion/ingestionTrigger/v1.schema.json index d4876142..50d12fa0 100644 --- a/schemas/raster/ingestion/ingestionTrigger/v1.schema.json +++ b/schemas/raster/ingestion/ingestionTrigger/v1.schema.json @@ -5,7 +5,7 @@ "required": ["services", "jobManager", "httpRetry", "validationValuesByInfo", "storageExplorer"], "allOf": [ { - "$ref": "https://mapcolonies.com/common/boilerplate/v3" + "$ref": "https://mapcolonies.com/common/boilerplate/v4" }, { "type": "object", From 8ea8cc0908bb6c4f1c8082a8171ee4cbfce72592 Mon Sep 17 00:00:00 2001 From: RonitKissis Date: Tue, 30 Jul 2024 17:23:10 +0300 Subject: [PATCH 3/7] fix: delay --- schemas/raster/common/httpRetry/v1.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/raster/common/httpRetry/v1.schema.json b/schemas/raster/common/httpRetry/v1.schema.json index cc54ead2..e9777144 100644 --- a/schemas/raster/common/httpRetry/v1.schema.json +++ b/schemas/raster/common/httpRetry/v1.schema.json @@ -9,7 +9,7 @@ "default": 5 }, "delay": { - "type": "string", + "oneOf": [{ "type": "number" }, { "const": "exponential" }], "x-env-value": "HTTP_RETRY_DELAY", "default": "exponential" }, From fbdc3c5f0a965b00ee2f2474fcf0e063adba5c89 Mon Sep 17 00:00:00 2001 From: RonitKissis Date: Tue, 13 Aug 2024 17:21:26 +0300 Subject: [PATCH 4/7] fix: change to array --- schemas/raster/common/jobManager/v1.schema.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/schemas/raster/common/jobManager/v1.schema.json b/schemas/raster/common/jobManager/v1.schema.json index 07970db4..2c9d57fd 100644 --- a/schemas/raster/common/jobManager/v1.schema.json +++ b/schemas/raster/common/jobManager/v1.schema.json @@ -25,11 +25,17 @@ "x-env-value": "INGESTION_SWAP_UPDATE_JOB_TYPE" }, "supportedIngestionSwapTypes": { - "type": "object", + "type": "array", + "items": { + "type": "string" + }, "x-env-value": "SUPPORTED_INGESTION_SWAP_TYPES" }, "forbiddenJobTypesForParallelIngestion": { - "type": "object", + "type": "array", + "items": { + "type": "string" + }, "x-env-value": "FORBIDDEN_TYPES_FOR_PARALLEL_INGESTION" } }, From 3a45c242cafaece76f981185fcdac49f9a1c5a45 Mon Sep 17 00:00:00 2001 From: RonitKissis Date: Tue, 13 Aug 2024 17:25:41 +0300 Subject: [PATCH 5/7] fix: to object --- schemas/raster/common/jobManager/v1.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/raster/common/jobManager/v1.schema.json b/schemas/raster/common/jobManager/v1.schema.json index 2c9d57fd..389bea12 100644 --- a/schemas/raster/common/jobManager/v1.schema.json +++ b/schemas/raster/common/jobManager/v1.schema.json @@ -27,7 +27,7 @@ "supportedIngestionSwapTypes": { "type": "array", "items": { - "type": "string" + "type": "object" }, "x-env-value": "SUPPORTED_INGESTION_SWAP_TYPES" }, From 683c64c5b01ac3c84adeee609e28f867e794d38a Mon Sep 17 00:00:00 2001 From: RonitKissis Date: Wed, 14 Aug 2024 11:36:07 +0300 Subject: [PATCH 6/7] fix: supportedIngestionSwapTypes type --- schemas/raster/common/jobManager/v1.schema.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/schemas/raster/common/jobManager/v1.schema.json b/schemas/raster/common/jobManager/v1.schema.json index 389bea12..94804210 100644 --- a/schemas/raster/common/jobManager/v1.schema.json +++ b/schemas/raster/common/jobManager/v1.schema.json @@ -27,6 +27,15 @@ "supportedIngestionSwapTypes": { "type": "array", "items": { + "properties": { + "productType": { + "type": "string" + }, + "productSubType": { + "type": "string" + } + }, + "required": ["productType", "productSubType"], "type": "object" }, "x-env-value": "SUPPORTED_INGESTION_SWAP_TYPES" From ea0a856fe461bf2cf33c0520ba8b8042a8325e2c Mon Sep 17 00:00:00 2001 From: RonitKissis Date: Wed, 14 Aug 2024 13:11:47 +0300 Subject: [PATCH 7/7] fix: crs and gpkg type --- schemas/raster/ingestion/ingestionTrigger/v1.schema.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schemas/raster/ingestion/ingestionTrigger/v1.schema.json b/schemas/raster/ingestion/ingestionTrigger/v1.schema.json index 50d12fa0..9251b854 100644 --- a/schemas/raster/ingestion/ingestionTrigger/v1.schema.json +++ b/schemas/raster/ingestion/ingestionTrigger/v1.schema.json @@ -93,8 +93,9 @@ "crs": { "type": "array", "items": { - "enum": [4326] + "type": "number" }, + "uniqueItems": true, "default": [4326], "minItems": 1, "x-env-value": "CRS" @@ -102,7 +103,7 @@ "fileFormat": { "type": "array", "items": { - "enum": ["GPKG"] + "type": "string" }, "default": ["GPKG"], "minItems": 1,