From 3b50152294964cefc3363256e7b04bebce138113 Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Mon, 5 May 2025 08:14:50 +0000 Subject: [PATCH 1/5] feat: created schema to vector's osm-sync-tracker --- schemas/vector/osmSyncTracker/v1.schema.json | 655 +++++++++++++++++++ 1 file changed, 655 insertions(+) create mode 100644 schemas/vector/osmSyncTracker/v1.schema.json diff --git a/schemas/vector/osmSyncTracker/v1.schema.json b/schemas/vector/osmSyncTracker/v1.schema.json new file mode 100644 index 00000000..9dcc7c71 --- /dev/null +++ b/schemas/vector/osmSyncTracker/v1.schema.json @@ -0,0 +1,655 @@ +{ + "$id": "https://mapcolonies.com/vector/osmSyncTracker/v1", + "type": "object", + "title": "VectorOsmSyncTrackerSchemaV1", + "description": "osm-sync-tracker schema", + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/boilerplate/v2" + }, + { + "properties": { + "telemetry": { + "$ref": "#/definitions/metrics" + } + } + }, + { + "type": "object", + "required": ["db"], + "properties": { + "db": { + "$ref": "https://mapcolonies.com/common/db/full/v1" + } + } + }, + { + "type": "object", + "required": ["redis"], + "properties": { + "redis": { + "$ref": "https://mapcolonies.com/common/redis/v1" + } + } + }, + { + "$ref": "#/definitions/closure" + } + ], + "definitions": { + "metrics": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "x-env-value": "TELEMETRY_METRICS_ENABLED", + "default": false + }, + "url": { + "type": "string", + "x-env-value": "TELEMETRY_METRICS_URL", + "description": "The URL of the metrics server. could be http or https", + "pattern": "^https?:\/\/.*" + }, + "interval": { + "type": "integer", + "x-env-value": "TELEMETRY_METRICS_INTERVAL", + "description": "The interval in seconds to send metrics" + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["url", "interval"] + }, + "else": { + "properties": { + "enabled": { + "const": false + } + } + } + }, + "closure": { + "type": "object", + "properties": { + "uiPath": { + "type": "string", + "x-env-value": "CLOSURE_UI_PATH", + "description": "The path to the closure UI" + }, + "queues": { + "type": "object", + "properties": { + "changesets": { + "type": "object", + "properties": { + "queueOptions": { + "$ref": "#/definitions/changesetsQueueOptions" + }, + "jobOptions": { + "$ref": "#/definitions/changesetsJobOptions" + }, + "workerOptions": { + "$ref": "#/definitions/changesetsWorkerOptions" + } + } + }, + "files": { + "type": "object", + "properties": { + "queueOptions": { + "$ref": "#/definitions/filesQueueOptions" + }, + "jobOptions": { + "$ref": "#/definitions/filesJobOptions" + }, + "workerOptions": { + "$ref": "#/definitions/filesWorkerOptions" + } + } + }, + "syncs": { + "type": "object", + "properties": { + "queueOptions": { + "$ref": "#/definitions/syncsQueueOptions" + }, + "jobOptions": { + "$ref": "#/definitions/syncsJobOptions" + }, + "workerOptions": { + "$ref": "#/definitions/syncsWorkerOptions" + } + } + } + } + } + } + }, + "changesetsQueueOptions": { + "type": "object", + "properties": { + "queueOptions": { + "allOf": [ + { + "$ref": "#/definitions/queueOptions" + }, + { + "type": "object", + "properties": { + "enabledBatchJobs": { + "x-env-value": "CHANGESETS_QUEUE_ENABLED_BATCH_JOBS", + "default": true + }, + "maxBatchSize": { + "x-env-value": "CHANGESETS_QUEUE_MAX_BATCH_SIZE", + "default": 10 + } + } + } + ] + } + } + }, + "filesQueueOptions": { + "type": "object", + "properties": { + "queueOptions": { + "allOf": [ + { + "$ref": "#/definitions/queueOptions" + }, + { + "type": "object", + "properties": { + "enabledBatchJobs": { + "x-env-value": "FILES_QUEUE_ENABLED_BATCH_JOBS", + "default": false + }, + "maxBatchSize": { + "x-env-value": "FILES_QUEUE_MAX_BATCH_SIZE", + "default": 10 + } + } + } + ] + } + } + }, + "syncsQueueOptions": { + "type": "object", + "properties": { + "queueOptions": { + "allOf": [ + { + "$ref": "#/definitions/queueOptions" + }, + { + "type": "object", + "properties": { + "enabledBatchJobs": { + "x-env-value": "SYNCS_QUEUE_ENABLED_BATCH_JOBS", + "default": false + }, + "maxBatchSize": { + "x-env-value": "SYNCS_QUEUE_MAX_BATCH_SIZE" + } + } + } + ] + } + } + }, + "changesetsJobOptions": { + "allOf": [ + { + "$ref": "#/definitions/jobOptions" + }, + { + "type": "object", + "properties": { + "attempts": { + "x-env-value": "CHANGESETS_QUEUE_JOB_ATTEMPTS", + "default": 10 + }, + "delay": { + "x-env-value": "CHANGESETS_QUEUE_JOB_DELAY", + "default": 60000 + }, + "deduplicationDelay": { + "x-env-value": "CHANGESETS_QUEUE_JOB_DEDUPLICATION_DELAY", + "default": 0 + }, + "deduplicationTtl": { + "x-env-value": "CHANGESETS_QUEUE_JOB_DEDUPLICATION_TTL", + "default": 0 + }, + "backoff": { + "type": "object", + "properties": { + "type": { + "x-env-value": "CHANGESETS_QUEUE_JOB_BACKOFF_TYPE", + "default": "fixed" + }, + "delay": { + "x-env-value": "CHANGESETS_QUEUE_JOB_BACKOFF_DELAY", + "default": 60000 + } + } + } + } + } + ] + }, + "filesJobOptions": { + "allOf": [ + { + "$ref": "#/definitions/jobOptions" + }, + { + "type": "object", + "properties": { + "attempts": { + "x-env-value": "FILES_QUEUE_JOB_ATTEMPTS", + "default": 10 + }, + "delay": { + "x-env-value": "FILES_QUEUE_JOB_DELAY", + "default": 60000 + }, + "deduplicationDelay": { + "x-env-value": "FILES_QUEUE_JOB_DEDUPLICATION_DELAY", + "default": 0 + }, + "deduplicationTtl": { + "x-env-value": "FILES_QUEUE_JOB_DEDUPLICATION_TTL", + "default": 0 + }, + "backoff": { + "type": "object", + "properties": { + "type": { + "x-env-value": "FILES_QUEUE_JOB_BACKOFF_TYPE", + "default": "fixed" + }, + "delay": { + "x-env-value": "FILES_QUEUE_JOB_BACKOFF_DELAY", + "default": 60000 + } + } + } + } + } + ] + }, + "syncsJobOptions": { + "allOf": [ + { + "$ref": "#/definitions/jobOptions" + }, + { + "type": "object", + "properties": { + "attempts": { + "x-env-value": "SYNCS_QUEUE_JOB_ATTEMPTS", + "default": 10 + }, + "delay": { + "x-env-value": "SYNCS_QUEUE_JOB_DELAY", + "default": 60000 + }, + "deduplicationDelay": { + "x-env-value": "SYNCS_QUEUE_JOB_DEDUPLICATION_DELAY", + "default": 0 + }, + "deduplicationTtl": { + "x-env-value": "SYNCS_QUEUE_JOB_DEDUPLICATION_TTL", + "default": 0 + }, + "backoff": { + "type": "object", + "properties": { + "type": { + "x-env-value": "SYNCS_QUEUE_JOB_BACKOFF_TYPE", + "default": "fixed" + }, + "delay": { + "x-env-value": "SYNCS_QUEUE_JOB_BACKOFF_DELAY", + "default": 60000 + } + } + } + } + } + ] + }, + "changesetsWorkerOptions": { + "allOf": [ + { + "$ref": "#/definitions/workerOptions" + }, + { + "type": "object", + "properties": { + "concurrency": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_CONCURRENCY" + }, + "limiter": { + "type": "object", + "properties": { + "max": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_LIMITER_MAX" + }, + "duration": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_LIMITER_DURATION" + } + } + }, + "maxStalledCount": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_MAX_STALLED_COUNT" + }, + "stalledInterval": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_STALLED_INTERVAL" + }, + "removeOnComplete": { + "type": "object", + "properties": { + "age": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE" + }, + "count": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT" + } + } + }, + "removeOnFail": { + "type": "object", + "properties": { + "age": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_FAIL_AGE" + }, + "count": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT" + } + } + }, + "transactionIsolationLevel": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL" + }, + "transactionFailureDelay": { + "type": "object", + "properties": { + "minimum": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN" + }, + "maximum": { + "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX" + } + } + } + } + } + ] + }, + "filesWorkerOptions": { + "allOf": [ + { + "$ref": "#/definitions/workerOptions" + }, + { + "type": "object", + "properties": { + "concurrency": { + "x-env-value": "FILES_QUEUE_WORKER_CONCURRENCY" + }, + "limiter": { + "type": "object", + "properties": { + "max": { + "x-env-value": "FILES_QUEUE_WORKER_LIMITER_MAX" + }, + "duration": { + "x-env-value": "FILES_QUEUE_WORKER_LIMITER_DURATION" + } + } + }, + "maxStalledCount": { + "x-env-value": "FILES_QUEUE_WORKER_MAX_STALLED_COUNT" + }, + "stalledInterval": { + "x-env-value": "FILES_QUEUE_WORKER_STALLED_INTERVAL" + }, + "removeOnComplete": { + "type": "object", + "properties": { + "age": { + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE" + }, + "count": { + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT" + } + } + }, + "removeOnFail": { + "type": "object", + "properties": { + "age": { + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_FAIL_AGE" + }, + "count": { + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT" + } + } + }, + "transactionIsolationLevel": { + "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL" + }, + "transactionFailureDelay": { + "type": "object", + "properties": { + "minimum": { + "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN" + }, + "maximum": { + "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX" + } + } + } + } + } + ] + }, + "syncsWorkerOptions": { + "allOf": [ + { + "$ref": "#/definitions/workerOptions" + }, + { + "type": "object", + "properties": { + "concurrency": { + "x-env-value": "SYNCS_QUEUE_WORKER_CONCURRENCY" + }, + "limiter": { + "type": "object", + "properties": { + "max": { + "x-env-value": "SYNCS_QUEUE_WORKER_LIMITER_MAX" + }, + "duration": { + "x-env-value": "SYNCS_QUEUE_WORKER_LIMITER_DURATION" + } + } + }, + "maxStalledCount": { + "x-env-value": "SYNCS_QUEUE_WORKER_MAX_STALLED_COUNT" + }, + "stalledInterval": { + "x-env-value": "SYNCS_QUEUE_WORKER_STALLED_INTERVAL" + }, + "removeOnComplete": { + "type": "object", + "properties": { + "age": { + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE" + }, + "count": { + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT" + } + } + }, + "removeOnFail": { + "type": "object", + "properties": { + "age": { + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_FAIL_AGE" + }, + "count": { + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT" + } + } + }, + "transactionIsolationLevel": { + "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL" + }, + "transactionFailureDelay": { + "type": "object", + "properties": { + "minimum": { + "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN" + }, + "maximum": { + "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX" + } + } + } + } + } + ] + }, + "queueOptions": { + "type": "object", + "properties": { + "enabledBatchJobs": { + "type": "boolean" + }, + "maxBatchSize": { + "type": "integer" + } + }, + "if": { + "properties": { + "enabledBatchJobs": { + "const": true + } + } + }, + "then": { + "required": ["maxBatchSize"] + }, + "else": { + "properties": { + "enabledBatchJobs": { + "const": false + } + } + } + }, + "jobOptions": { + "type": "object", + "properties": { + "attempts": { + "type": "integer" + }, + "delay": { + "type": "integer" + }, + "deduplicationDelay": { + "type": "integer" + }, + "deduplicationTtl": { + "type": "integer" + }, + "backoff": { + "type": "object", + "properties": { + "type": { + "enum": ["fixed", "exponential"] + }, + "delay": { + "type": "integer" + } + } + } + } + }, + "workerOptions": { + "type": "object", + "properties": { + "concurrency": { + "type": "integer" + }, + "limiter": { + "type": "object", + "properties": { + "max": { + "type": "integer" + }, + "duration": { + "type": "integer" + } + } + }, + "maxStalledCount": { + "type": "integer" + }, + "stalledInterval": { + "type": "integer" + }, + "removeOnComplete": { + "type": "object", + "properties": { + "age": { + "type": "integer" + }, + "count": { + "type": "integer" + } + } + }, + "removeOnFail": { + "type": "object", + "properties": { + "age": { + "type": "integer" + }, + "count": { + "type": "integer" + } + } + }, + "transactionIsolationLevel": { + "type": "string", + "enum": ["READ UNCOMMITTED", "READ COMMITTED", "REPEATABLE READ", "SERIALIZABLE"] + }, + "transactionFailureDelay": { + "type": "object", + "properties": { + "minimum": { + "type": "integer" + }, + "maximum": { + "type": "integer" + } + } + } + } + } + } +} From 92934db2c4a3334d2934e959a1ccbdab1a6f415a Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Tue, 6 May 2025 10:25:02 +0000 Subject: [PATCH 2/5] fix: changed db version to v2 --- schemas/vector/osmSyncTracker/v1.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/vector/osmSyncTracker/v1.schema.json b/schemas/vector/osmSyncTracker/v1.schema.json index 9dcc7c71..8ed86b28 100644 --- a/schemas/vector/osmSyncTracker/v1.schema.json +++ b/schemas/vector/osmSyncTracker/v1.schema.json @@ -19,7 +19,7 @@ "required": ["db"], "properties": { "db": { - "$ref": "https://mapcolonies.com/common/db/full/v1" + "$ref": "https://mapcolonies.com/common/db/full/v2" } } }, From 503d88a85c8dbeb0202fef3bf64dc0a86af92d5b Mon Sep 17 00:00:00 2001 From: Niv Greenstein <88280771+NivGreenstein@users.noreply.github.com> Date: Tue, 6 May 2025 16:42:51 +0300 Subject: [PATCH 3/5] fix: fixed schema to work --- schemas/vector/osmSyncTracker/v1.configs.json | 152 +++++++++++ schemas/vector/osmSyncTracker/v1.schema.json | 246 ++++++++++-------- 2 files changed, 296 insertions(+), 102 deletions(-) create mode 100644 schemas/vector/osmSyncTracker/v1.configs.json diff --git a/schemas/vector/osmSyncTracker/v1.configs.json b/schemas/vector/osmSyncTracker/v1.configs.json new file mode 100644 index 00000000..d2a397e1 --- /dev/null +++ b/schemas/vector/osmSyncTracker/v1.configs.json @@ -0,0 +1,152 @@ +[ + { + "name": "osm-sync-tracker", + "value": { + "openapiConfig": { + "filePath": "./openapi3.yaml", + "basePath": "/docs", + "rawPath": "/api", + "uiPath": "/api" + }, + "telemetry": { + "tracing": { + "isEnabled": false + }, + "shared": {}, + "logger": { + "level": "info", + "prettyPrint": false + } + }, + "server": { + "port": 8080, + "request": { + "payload": { + "limit": "1mb" + } + }, + "response": { + "compression": { + "enabled": true, + "options": null + } + } + }, + "db": { + "type": "postgres", + "host": "localhost", + "port": 5432, + "username": "postgres", + "password": "postgres", + "enableSslAuth": false, + "sslPaths": { + "ca": "", + "key": "", + "cert": "" + }, + "database": "postgres", + "schema": "osm_sync_tracker" + }, + "redis": { + "host": "localhost", + "port": 6379, + "username": "", + "password": "", + "enableSslAuth": false, + "sslPaths": { + "ca": "", + "key": "", + "cert": "" + }, + "db": 0 + }, + "closure": { + "uiPath": "/closure", + "queues": { + "changesets": { + "queueOptions": { + "enabledBatchJobs": true, + "maxBatchSize": 10 + }, + "jobOptions": { + "attempts": 10, + "delay": 60000, + "backoff": { + "type": "fixed", + "delay": 60000 + } + }, + "workerOptions": { + "concurrency": 1, + "removeOnComplete": { + "age": 604800 + }, + "removeOnFail": { + "age": 2419200 + }, + "transactionIsolationLevel": "SERIALIZABLE", + "transactionFailureDelay": { + "minimum": 10000, + "maximum": 10000 + } + } + }, + "files": { + "queueOptions": { + "enabledBatchJobs": false + }, + "jobOptions": { + "attempts": 10, + "delay": 60000, + "backoff": { + "type": "fixed", + "delay": 60000 + } + }, + "workerOptions": { + "concurrency": 1, + "removeOnComplete": { + "age": 604800 + }, + "removeOnFail": { + "age": 2419200 + }, + "transactionIsolationLevel": "SERIALIZABLE", + "transactionFailureDelay": { + "minimum": 10000, + "maximum": 10000 + } + } + }, + "syncs": { + "queueOptions": { + "enabledBatchJobs": false + }, + "jobOptions": { + "attempts": 10, + "delay": 60000, + "backoff": { + "type": "fixed", + "delay": 60000 + } + }, + "workerOptions": { + "concurrency": 1, + "removeOnComplete": { + "age": 604800 + }, + "removeOnFail": { + "age": 2419200 + }, + "transactionIsolationLevel": "SERIALIZABLE", + "transactionFailureDelay": { + "minimum": 10000, + "maximum": 10000 + } + } + } + } + } + } + } +] diff --git a/schemas/vector/osmSyncTracker/v1.schema.json b/schemas/vector/osmSyncTracker/v1.schema.json index 8ed86b28..61be2a93 100644 --- a/schemas/vector/osmSyncTracker/v1.schema.json +++ b/schemas/vector/osmSyncTracker/v1.schema.json @@ -33,7 +33,13 @@ } }, { - "$ref": "#/definitions/closure" + "type": "object", + "required": ["closure"], + "properties": { + "closure": { + "$ref": "#/definitions/closure" + } + } } ], "definitions": { @@ -77,17 +83,23 @@ }, "closure": { "type": "object", + "required": ["uiPath", "queues"], "properties": { "uiPath": { "type": "string", - "x-env-value": "CLOSURE_UI_PATH", - "description": "The path to the closure UI" + "description": "The path to the closure UI", + "default": "/closure", + "pattern": "^/.*", + "x-env-value": "CLOSURE_UI_PATH" }, "queues": { "type": "object", + "required": ["changesets", "files", "syncs"], + "additionalProperties": false, "properties": { "changesets": { "type": "object", + "required": ["queueOptions", "jobOptions", "workerOptions"], "properties": { "queueOptions": { "$ref": "#/definitions/changesetsQueueOptions" @@ -102,6 +114,7 @@ }, "files": { "type": "object", + "required": ["queueOptions", "jobOptions", "workerOptions"], "properties": { "queueOptions": { "$ref": "#/definitions/filesQueueOptions" @@ -116,6 +129,7 @@ }, "syncs": { "type": "object", + "required": ["queueOptions", "jobOptions", "workerOptions"], "properties": { "queueOptions": { "$ref": "#/definitions/syncsQueueOptions" @@ -133,78 +147,64 @@ } }, "changesetsQueueOptions": { - "type": "object", - "properties": { - "queueOptions": { - "allOf": [ - { - "$ref": "#/definitions/queueOptions" + "allOf": [ + { + "$ref": "#/definitions/queueOptions" + }, + { + "type": "object", + "properties": { + "enabledBatchJobs": { + "x-env-value": "CHANGESETS_QUEUE_ENABLED_BATCH_JOBS", + "default": true }, - { - "type": "object", - "properties": { - "enabledBatchJobs": { - "x-env-value": "CHANGESETS_QUEUE_ENABLED_BATCH_JOBS", - "default": true - }, - "maxBatchSize": { - "x-env-value": "CHANGESETS_QUEUE_MAX_BATCH_SIZE", - "default": 10 - } - } + "maxBatchSize": { + "x-env-value": "CHANGESETS_QUEUE_MAX_BATCH_SIZE", + "default": 10 } - ] + } } - } + ] }, "filesQueueOptions": { - "type": "object", - "properties": { - "queueOptions": { - "allOf": [ - { - "$ref": "#/definitions/queueOptions" + "allOf": [ + { + "$ref": "#/definitions/queueOptions" + }, + { + "type": "object", + "properties": { + "enabledBatchJobs": { + "x-env-value": "FILES_QUEUE_ENABLED_BATCH_JOBS", + "default": false }, - { - "type": "object", - "properties": { - "enabledBatchJobs": { - "x-env-value": "FILES_QUEUE_ENABLED_BATCH_JOBS", - "default": false - }, - "maxBatchSize": { - "x-env-value": "FILES_QUEUE_MAX_BATCH_SIZE", - "default": 10 - } - } + "maxBatchSize": { + "x-env-value": "FILES_QUEUE_MAX_BATCH_SIZE", + "default": 10 } - ] + } } - } + ] }, "syncsQueueOptions": { - "type": "object", - "properties": { - "queueOptions": { - "allOf": [ - { - "$ref": "#/definitions/queueOptions" + "allOf": [ + { + "$ref": "#/definitions/queueOptions" + }, + { + "type": "object", + "properties": { + "enabledBatchJobs": { + "x-env-value": "SYNCS_QUEUE_ENABLED_BATCH_JOBS", + "default": false }, - { - "type": "object", - "properties": { - "enabledBatchJobs": { - "x-env-value": "SYNCS_QUEUE_ENABLED_BATCH_JOBS", - "default": false - }, - "maxBatchSize": { - "x-env-value": "SYNCS_QUEUE_MAX_BATCH_SIZE" - } - } + "maxBatchSize": { + "x-env-value": "SYNCS_QUEUE_MAX_BATCH_SIZE", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } - ] + } } - } + ] }, "changesetsJobOptions": { "allOf": [ @@ -338,33 +338,40 @@ "type": "object", "properties": { "concurrency": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_CONCURRENCY" + "x-env-value": "CHANGESETS_QUEUE_WORKER_CONCURRENCY", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "limiter": { "type": "object", "properties": { "max": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_LIMITER_MAX" + "x-env-value": "CHANGESETS_QUEUE_WORKER_LIMITER_MAX", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "duration": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_LIMITER_DURATION" + "x-env-value": "CHANGESETS_QUEUE_WORKER_LIMITER_DURATION", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, "maxStalledCount": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_MAX_STALLED_COUNT" + "x-env-value": "CHANGESETS_QUEUE_WORKER_MAX_STALLED_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "stalledInterval": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_STALLED_INTERVAL" + "x-env-value": "CHANGESETS_QUEUE_WORKER_STALLED_INTERVAL", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "removeOnComplete": { "type": "object", "properties": { "age": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE" + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "count": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT" + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, @@ -372,24 +379,29 @@ "type": "object", "properties": { "age": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_FAIL_AGE" + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_FAIL_AGE", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "count": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT" + "x-env-value": "CHANGESETS_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, "transactionIsolationLevel": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL" + "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "transactionFailureDelay": { "type": "object", "properties": { "minimum": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN" + "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "maximum": { - "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX" + "x-env-value": "CHANGESETS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } } @@ -406,33 +418,40 @@ "type": "object", "properties": { "concurrency": { - "x-env-value": "FILES_QUEUE_WORKER_CONCURRENCY" + "x-env-value": "FILES_QUEUE_WORKER_CONCURRENCY", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "limiter": { "type": "object", "properties": { "max": { - "x-env-value": "FILES_QUEUE_WORKER_LIMITER_MAX" + "x-env-value": "FILES_QUEUE_WORKER_LIMITER_MAX", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "duration": { - "x-env-value": "FILES_QUEUE_WORKER_LIMITER_DURATION" + "x-env-value": "FILES_QUEUE_WORKER_LIMITER_DURATION", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, "maxStalledCount": { - "x-env-value": "FILES_QUEUE_WORKER_MAX_STALLED_COUNT" + "x-env-value": "FILES_QUEUE_WORKER_MAX_STALLED_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "stalledInterval": { - "x-env-value": "FILES_QUEUE_WORKER_STALLED_INTERVAL" + "x-env-value": "FILES_QUEUE_WORKER_STALLED_INTERVAL", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "removeOnComplete": { "type": "object", "properties": { "age": { - "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE" + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "count": { - "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT" + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, @@ -440,24 +459,29 @@ "type": "object", "properties": { "age": { - "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_FAIL_AGE" + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_FAIL_AGE", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "count": { - "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT" + "x-env-value": "FILES_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, "transactionIsolationLevel": { - "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL" + "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "transactionFailureDelay": { "type": "object", "properties": { "minimum": { - "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN" + "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "maximum": { - "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX" + "x-env-value": "FILES_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } } @@ -474,33 +498,40 @@ "type": "object", "properties": { "concurrency": { - "x-env-value": "SYNCS_QUEUE_WORKER_CONCURRENCY" + "x-env-value": "SYNCS_QUEUE_WORKER_CONCURRENCY", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "limiter": { "type": "object", "properties": { "max": { - "x-env-value": "SYNCS_QUEUE_WORKER_LIMITER_MAX" + "x-env-value": "SYNCS_QUEUE_WORKER_LIMITER_MAX", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "duration": { - "x-env-value": "SYNCS_QUEUE_WORKER_LIMITER_DURATION" + "x-env-value": "SYNCS_QUEUE_WORKER_LIMITER_DURATION", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, "maxStalledCount": { - "x-env-value": "SYNCS_QUEUE_WORKER_MAX_STALLED_COUNT" + "x-env-value": "SYNCS_QUEUE_WORKER_MAX_STALLED_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "stalledInterval": { - "x-env-value": "SYNCS_QUEUE_WORKER_STALLED_INTERVAL" + "x-env-value": "SYNCS_QUEUE_WORKER_STALLED_INTERVAL", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "removeOnComplete": { "type": "object", "properties": { "age": { - "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE" + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_COMPLETE_AGE", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "count": { - "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT" + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_COMPLETE_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, @@ -508,24 +539,29 @@ "type": "object", "properties": { "age": { - "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_FAIL_AGE" + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_FAIL_AGE", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "count": { - "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT" + "x-env-value": "SYNCS_QUEUE_WORKER_REMOVE_ON_FAIL_COUNT", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } }, "transactionIsolationLevel": { - "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL" + "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_ISOLATION_LEVEL", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "transactionFailureDelay": { "type": "object", "properties": { "minimum": { - "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN" + "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MIN", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" }, "maximum": { - "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX" + "x-env-value": "SYNCS_QUEUE_WORKER_TRANSACTION_FAILURE_DELAY_MAX", + "$comment": "Must add some comment. See this issue for more details: https://github.com/MapColonies/schemas/issues/96" } } } @@ -537,7 +573,8 @@ "type": "object", "properties": { "enabledBatchJobs": { - "type": "boolean" + "type": "boolean", + "default": false }, "maxBatchSize": { "type": "integer" @@ -591,12 +628,14 @@ }, "workerOptions": { "type": "object", + "required": ["transactionFailureDelay"], "properties": { "concurrency": { "type": "integer" }, "limiter": { "type": "object", + "required": ["max", "duration"], "properties": { "max": { "type": "integer" @@ -636,16 +675,19 @@ }, "transactionIsolationLevel": { "type": "string", - "enum": ["READ UNCOMMITTED", "READ COMMITTED", "REPEATABLE READ", "SERIALIZABLE"] + "enum": ["READ UNCOMMITTED", "READ COMMITTED", "REPEATABLE READ", "SERIALIZABLE"], + "default": "SERIALIZABLE" }, "transactionFailureDelay": { "type": "object", "properties": { "minimum": { - "type": "integer" + "type": "integer", + "default": "10000" }, "maximum": { - "type": "integer" + "type": "integer", + "default": "10000" } } } From 2a7918c3ff457e6d799ace8b8a95ab3ff6f0afc6 Mon Sep 17 00:00:00 2001 From: Ofer <12687466+CptSchnitz@users.noreply.github.com> Date: Wed, 7 May 2025 09:25:11 +0300 Subject: [PATCH 4/5] feat(opala): added tracing config to opala cron (#94) --- .../cron/{v1.configs.json => v2.configs.json} | 6 +- schemas/infra/opala/cron/v2.schema.json | 101 ++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) rename schemas/infra/opala/cron/{v1.configs.json => v2.configs.json} (78%) create mode 100644 schemas/infra/opala/cron/v2.schema.json diff --git a/schemas/infra/opala/cron/v1.configs.json b/schemas/infra/opala/cron/v2.configs.json similarity index 78% rename from schemas/infra/opala/cron/v1.configs.json rename to schemas/infra/opala/cron/v2.configs.json index 7fbcf580..3d9abf84 100644 --- a/schemas/infra/opala/cron/v1.configs.json +++ b/schemas/infra/opala/cron/v2.configs.json @@ -6,7 +6,11 @@ "logger": { "level": "info", "prettyPrint": false - } + }, + "tracing": { + "$ref": { "configName": "common-tracing", "version": "latest" } + }, + "shared": {} }, "db": { "$ref": { "configName": "infra-opala-db", "version": "latest" } diff --git a/schemas/infra/opala/cron/v2.schema.json b/schemas/infra/opala/cron/v2.schema.json new file mode 100644 index 00000000..d50aa1cd --- /dev/null +++ b/schemas/infra/opala/cron/v2.schema.json @@ -0,0 +1,101 @@ +{ + "$id": "https://mapcolonies.com/infra/opala/cron/v2", + "description": "The opala cron service schema", + "title": "opalaCronSchemaV2", + "type": "object", + "allOf": [ + { + "type": "object", + "required": ["telemetry", "cron", "db"], + "properties": { + "telemetry": { + "$ref": "#/definitions/telemetry" + }, + "db": { + "$ref": "https://mapcolonies.com/common/db/full/v1" + }, + "cron": { + "$ref": "#/definitions/crons" + } + } + } + ], + "definitions": { + "crons": { + "type": "object", + "additionalProperties": false, + "anyOf": [ + { + "required": ["np"] + }, + { + "required": ["stage"] + }, + { + "required": ["prod"] + } + ], + "properties": { + "np": { + "$ref": "#/definitions/cron" + }, + "prod": { + "$ref": "#/definitions/cron" + }, + "stage": { + "$ref": "#/definitions/cron" + } + } + }, + "cron": { + "type": "object", + "required": ["pattern", "s3"], + "properties": { + "pattern": { + "type": "string", + "description": "The cron timing spec", + "examples": ["*/1 * * * *"] + }, + "s3": { + "type": "object", + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/s3/partial/v1" + }, + { + "type": "object", + "required": ["key", "bucket"], + "properties": { + "key": { + "type": "string", + "description": "specific key of bundle inside s3", + "x-env-value": "S3_KEY" + }, + "bucket": { + "type": "string", + "description": "specific bucket name inside s3", + "x-env-value": "S3_BUCKET" + } + } + } + ] + } + } + }, + "telemetry": { + "type": "object", + "required": ["logger", "tracing", "shared"], + "properties": { + "logger": { + "$ref": "https://mapcolonies.com/common/telemetry/logger/v1" + }, + "tracing": { + "$ref": "https://mapcolonies.com/common/telemetry/tracing/v1" + }, + "shared": { + "$ref": "https://mapcolonies.com/common/telemetry/base/v1" + } + } + } + } +} From c69b87a9da7660833a593d362e4ba52529734121 Mon Sep 17 00:00:00 2001 From: mapcolonies-devops <143094402+mapcolonies-devops@users.noreply.github.com> Date: Thu, 8 May 2025 07:48:07 +0300 Subject: [PATCH 5/5] chore(master): release 1.8.0 (#98) --- .release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index e894fd62..5eb41ee7 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1 +1 @@ -{".":"1.7.0"} +{".":"1.8.0"} diff --git a/CHANGELOG.md b/CHANGELOG.md index d8cd5229..4f9c263a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.8.0](https://github.com/MapColonies/schemas/compare/v1.7.0...v1.8.0) (2025-05-07) + + +### Features + +* **opala:** added tracing config to opala cron ([#94](https://github.com/MapColonies/schemas/issues/94)) ([6d86a6c](https://github.com/MapColonies/schemas/commit/6d86a6cfca86c27a2f2dd44a1a94e149e3c362e7)) + ## [1.7.0](https://github.com/MapColonies/schemas/compare/v1.6.0...v1.7.0) (2025-04-29) diff --git a/package-lock.json b/package-lock.json index b4bd269f..f84faf89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@map-colonies/schemas", - "version": "1.7.0", + "version": "1.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@map-colonies/schemas", - "version": "1.7.0", + "version": "1.8.0", "license": "MIT", "devDependencies": { "@actions/core": "^1.10.1", diff --git a/package.json b/package.json index b6f1683a..a672fff4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@map-colonies/schemas", - "version": "1.7.0", + "version": "1.8.0", "description": "Package for all of MapColonies config schemas", "main": "./build/schemas/index.js", "repository": {