From af852bac01dbc7f4fa2e73f9c8f89e041b74eafd Mon Sep 17 00:00:00 2001 From: loothero Date: Wed, 4 Mar 2026 05:48:29 -0800 Subject: [PATCH 01/39] chore(indexer): upgrade apibara and switch to accepted finality --- indexer/indexers/summit.indexer.ts | 2 +- indexer/package.json | 10 ++--- indexer/pnpm-lock.yaml | 64 +++++++++++++++--------------- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/indexer/indexers/summit.indexer.ts b/indexer/indexers/summit.indexer.ts index 93e58b2d..bfab31a7 100644 --- a/indexer/indexers/summit.indexer.ts +++ b/indexer/indexers/summit.indexer.ts @@ -926,7 +926,7 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { return defineIndexer(StarknetStream)({ streamUrl, - finality: "pending", + finality: "accepted", startingBlock, filter: { events: [ diff --git a/indexer/package.json b/indexer/package.json index f75fb34d..02c62af6 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -27,11 +27,11 @@ "license": "MIT", "description": "Savage Summit Dojo event indexer using Apibara with PostgreSQL persistence", "dependencies": { - "@apibara/indexer": "next", - "@apibara/plugin-drizzle": "next", - "@apibara/protocol": "next", - "@apibara/starknet": "next", - "apibara": "next", + "@apibara/indexer": "2.1.0-beta.56", + "@apibara/plugin-drizzle": "2.1.0-beta.55", + "@apibara/protocol": "2.1.0-beta.56", + "@apibara/starknet": "2.1.0-beta.56", + "apibara": "2.1.0-beta.55", "drizzle-orm": "^0.38.0", "pg": "^8.13.0", "starknet": "^7.1.0" diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index d22f4d57..dbdcdeea 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -9,20 +9,20 @@ importers: .: dependencies: '@apibara/indexer': - specifier: next - version: 2.1.0-beta.54(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + specifier: 2.1.0-beta.56 + version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': - specifier: next - version: 2.1.0-beta.53(drizzle-orm@0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + specifier: 2.1.0-beta.55 + version: 2.1.0-beta.55(drizzle-orm@0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': - specifier: next - version: 2.1.0-beta.54(typescript@5.9.3) + specifier: 2.1.0-beta.56 + version: 2.1.0-beta.56(typescript@5.9.3) '@apibara/starknet': - specifier: next - version: 2.1.0-beta.54(typescript@5.9.3) + specifier: 2.1.0-beta.56 + version: 2.1.0-beta.56(typescript@5.9.3) apibara: - specifier: next - version: 2.1.0-beta.53(magicast@0.3.5)(rollup@4.57.1)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + specifier: 2.1.0-beta.55 + version: 2.1.0-beta.55(magicast@0.3.5)(rollup@4.57.1)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) drizzle-orm: specifier: ^0.38.0 version: 0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0) @@ -79,13 +79,13 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} - '@apibara/indexer@2.1.0-beta.54': - resolution: {integrity: sha512-UBrQP1/4IF8gHACbRWKwNYoIWaC7nNNdc3XcH8ucKaeGZe8ufxeyj4kYvEEq1el8viEaf/CD0JL4LhnnK5L1Hg==} + '@apibara/indexer@2.1.0-beta.56': + resolution: {integrity: sha512-i8o6RC0rCNCxComFtZowpcnThLblFn8Z+6Wf2aqZShJd/AG+CrkNwmvtdOnyrK2sZ/+y7fICKopZjnbtmhB1HA==} peerDependencies: vitest: ^1.6.0 - '@apibara/plugin-drizzle@2.1.0-beta.53': - resolution: {integrity: sha512-LxaPjhRonHjYNB/ifm84KXmdaZUnpgEdJpRUBfx2XlYafKegINpz+azGY3EuwVmmZP64ej6xyZG9bKzBEm9NbA==} + '@apibara/plugin-drizzle@2.1.0-beta.55': + resolution: {integrity: sha512-DDgg6qpOQGRgt48KGCG8bUkSdhm4hmjhpZP1iApC4YAOHeCRezWNUVKQ8hQphXd5jdWudy1oLnSxYvFkYdoxiA==} peerDependencies: '@electric-sql/pglite': '>=0.2.0' drizzle-orm: <1 @@ -96,11 +96,11 @@ packages: pg: optional: true - '@apibara/protocol@2.1.0-beta.54': - resolution: {integrity: sha512-rOCLV44Mz/mKCP6DtSRxdqOrBgC90Ev+5apMzO/0S/cQoqfd1ZhKU3Sc3qsR4+Id+MFkH9mzgIlcg2czyXW2BA==} + '@apibara/protocol@2.1.0-beta.56': + resolution: {integrity: sha512-TerNSio5rIy/3Cx3nj3OJNa/Ndzj4tcxVfsaDAoS+oU6cpPRxl1dIaiLwXQPrlMY4+pQQ8JlTQPjQhmiw4XRhg==} - '@apibara/starknet@2.1.0-beta.54': - resolution: {integrity: sha512-SW725WqTrgAASrx+XsPS3VKG3XJOIoIdERyBWZBnrBCZEXzeL2pOfgA3TvRboP0WyCxlgy93t5pWA9Kvf86VkA==} + '@apibara/starknet@2.1.0-beta.56': + resolution: {integrity: sha512-CtwI+iqFrDYBRzH/AoOCggZhgp4tpIug9sfDz+pHhoXySgbH+pN0mt4px03ICmPtb6rrSsYozVBtGoNT+zRlzA==} '@babel/code-frame@7.29.0': resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} @@ -1225,8 +1225,8 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} - apibara@2.1.0-beta.53: - resolution: {integrity: sha512-rcgGRA2glUe2p0bIkQv2vNK2pExMR2Rh4d6mDgVbNRDU7aiDAtucbtLe9L/cSjGlUOnT/BCv3P4Z2U/cStNhng==} + apibara@2.1.0-beta.55: + resolution: {integrity: sha512-N2ck7DmJZPYm+Rol3m/iXMqmmcJQi8c5spcpLlI8uZ60TTVmZU4q7ccc4pPdatCIzr7pWXlxn86KRnixeJJhgw==} hasBin: true argparse@2.0.1: @@ -2485,9 +2485,9 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 - '@apibara/indexer@2.1.0-beta.54(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/indexer@2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: - '@apibara/protocol': 2.1.0-beta.54(typescript@5.9.3) + '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) '@opentelemetry/api': 1.9.0 ci-info: 4.4.0 consola: 3.4.2 @@ -2502,10 +2502,10 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.53(drizzle-orm@0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(drizzle-orm@0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: - '@apibara/indexer': 2.1.0-beta.54(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) - '@apibara/protocol': 2.1.0-beta.54(typescript@5.9.3) + '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) drizzle-orm: 0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0) postgres-range: 1.1.4 optionalDependencies: @@ -2517,7 +2517,7 @@ snapshots: - vitest - zod - '@apibara/protocol@2.1.0-beta.54(typescript@5.9.3)': + '@apibara/protocol@2.1.0-beta.56(typescript@5.9.3)': dependencies: consola: 3.4.2 long: 5.3.2 @@ -2531,9 +2531,9 @@ snapshots: - utf-8-validate - zod - '@apibara/starknet@2.1.0-beta.54(typescript@5.9.3)': + '@apibara/starknet@2.1.0-beta.56(typescript@5.9.3)': dependencies: - '@apibara/protocol': 2.1.0-beta.54(typescript@5.9.3) + '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) '@scure/starknet': 1.1.2 abi-wan-kanabi: 2.2.4 long: 5.3.2 @@ -3198,7 +3198,7 @@ snapshots: '@scure/bip32@1.7.0': dependencies: - '@noble/curves': 1.9.1 + '@noble/curves': 1.9.7 '@noble/hashes': 1.8.0 '@scure/base': 1.2.6 @@ -3463,10 +3463,10 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 - apibara@2.1.0-beta.53(magicast@0.3.5)(rollup@4.57.1)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)): + apibara@2.1.0-beta.55(magicast@0.3.5)(rollup@4.57.1)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)): dependencies: - '@apibara/indexer': 2.1.0-beta.54(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) - '@apibara/protocol': 2.1.0-beta.54(typescript@5.9.3) + '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) '@rollup/plugin-replace': 6.0.3(rollup@4.57.1) '@rollup/plugin-virtual': 3.0.2(rollup@4.57.1) c12: 1.11.2(magicast@0.3.5) From 090da3f303b9907cd2a2a9b79070b6307c7a285d Mon Sep 17 00:00:00 2001 From: loothero Date: Wed, 4 Mar 2026 14:36:33 -0800 Subject: [PATCH 02/39] add consumables table migration The consumables table was defined in the Drizzle schema but had no corresponding SQL migration, causing "relation does not exist" errors on deployment. Co-Authored-By: Claude Opus 4.6 --- indexer/migrations/0003_consumables.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 indexer/migrations/0003_consumables.sql diff --git a/indexer/migrations/0003_consumables.sql b/indexer/migrations/0003_consumables.sql new file mode 100644 index 00000000..b63183d0 --- /dev/null +++ b/indexer/migrations/0003_consumables.sql @@ -0,0 +1,15 @@ +-- Migration: Create consumables table for ERC20 token balance tracking. +-- +-- Tracks circulating supply of 4 consumable tokens (XLIFE, ATTACK, REVIVE, POISON) +-- per owner address. Updated additively on each ERC20 Transfer event. + +CREATE TABLE IF NOT EXISTS "consumables" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "owner" text NOT NULL, + "xlife_count" integer DEFAULT 0 NOT NULL, + "attack_count" integer DEFAULT 0 NOT NULL, + "revive_count" integer DEFAULT 0 NOT NULL, + "poison_count" integer DEFAULT 0 NOT NULL, + "updated_at" timestamp DEFAULT now(), + CONSTRAINT "consumables_owner_unique" UNIQUE("owner") +); From 8f642d9650aa7e479325a685f40642a842d5b12e Mon Sep 17 00:00:00 2001 From: loothero Date: Wed, 4 Mar 2026 14:40:02 -0800 Subject: [PATCH 03/39] register consumables migration in drizzle journal The migration file was added but not registered in the meta/_journal.json, so drizzle's migration runner was skipping it on startup. Co-Authored-By: Claude Opus 4.6 --- indexer/migrations/meta/_journal.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/indexer/migrations/meta/_journal.json b/indexer/migrations/meta/_journal.json index 3ceceb88..50638ad7 100644 --- a/indexer/migrations/meta/_journal.json +++ b/indexer/migrations/meta/_journal.json @@ -22,6 +22,13 @@ "when": 1770990939402, "tag": "0002_beast_stats_flags_and_trigger_refresh", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1770990939403, + "tag": "0003_consumables", + "breakpoints": true } ] } \ No newline at end of file From 02cf96e4450540a6716111a05e693ebc9fb59b97 Mon Sep 17 00:00:00 2001 From: loothero Date: Thu, 5 Mar 2026 06:27:49 -0800 Subject: [PATCH 04/39] chore(indexer): bump drizzle-orm to 0.45.1 --- indexer/package.json | 2 +- indexer/pnpm-lock.yaml | 31 ++++++++++++++++--------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/indexer/package.json b/indexer/package.json index 02c62af6..a81d3f3c 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -32,7 +32,7 @@ "@apibara/protocol": "2.1.0-beta.56", "@apibara/starknet": "2.1.0-beta.56", "apibara": "2.1.0-beta.55", - "drizzle-orm": "^0.38.0", + "drizzle-orm": "^0.45.1", "pg": "^8.13.0", "starknet": "^7.1.0" }, diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index dbdcdeea..f72c9eee 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -13,7 +13,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(drizzle-orm@0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -24,8 +24,8 @@ importers: specifier: 2.1.0-beta.55 version: 2.1.0-beta.55(magicast@0.3.5)(rollup@4.57.1)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) drizzle-orm: - specifier: ^0.38.0 - version: 0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0) + specifier: ^0.45.1 + version: 0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0) pg: specifier: ^8.13.0 version: 8.18.0 @@ -1379,8 +1379,8 @@ packages: resolution: {integrity: sha512-U4wWit0fyZuGuP7iNmRleQyK2V8wCuv57vf5l3MnG4z4fzNTjY/U13M8owyQ5RavqvqxBifWORaR3wIUzlN64g==} hasBin: true - drizzle-orm@0.38.4: - resolution: {integrity: sha512-s7/5BpLKO+WJRHspvpqTydxFob8i1vo2rEx4pY6TGY7QSMuUfWUuzaY0DIpXCkgHOo37BaFC+SJQb99dDUXT3Q==} + drizzle-orm@0.45.1: + resolution: {integrity: sha512-Te0FOdKIistGNPMq2jscdqngBRfBpC8uMFVwqjf6gtTVJHIQ/dosgV/CLBU2N4ZJBsXL5savCba9b0YJskKdcA==} peerDependencies: '@aws-sdk/client-rds-data': '>=3' '@cloudflare/workers-types': '>=4' @@ -1390,25 +1390,25 @@ packages: '@neondatabase/serverless': '>=0.10.0' '@op-engineering/op-sqlite': '>=2' '@opentelemetry/api': ^1.4.1 - '@planetscale/database': '>=1' + '@planetscale/database': '>=1.13' '@prisma/client': '*' '@tidbcloud/serverless': '*' '@types/better-sqlite3': '*' '@types/pg': '*' - '@types/react': '>=18' '@types/sql.js': '*' + '@upstash/redis': '>=1.34.7' '@vercel/postgres': '>=0.8.0' '@xata.io/client': '*' better-sqlite3: '>=7' bun-types: '*' expo-sqlite: '>=14.0.0' + gel: '>=2' knex: '*' kysely: '*' mysql2: '>=2' pg: '>=8' postgres: '>=3' prisma: '*' - react: '>=18' sql.js: '>=1' sqlite3: '>=5' peerDependenciesMeta: @@ -1438,10 +1438,10 @@ packages: optional: true '@types/pg': optional: true - '@types/react': - optional: true '@types/sql.js': optional: true + '@upstash/redis': + optional: true '@vercel/postgres': optional: true '@xata.io/client': @@ -1452,6 +1452,8 @@ packages: optional: true expo-sqlite: optional: true + gel: + optional: true knex: optional: true kysely: @@ -1464,8 +1466,6 @@ packages: optional: true prisma: optional: true - react: - optional: true sql.js: optional: true sqlite3: @@ -2502,11 +2502,11 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(drizzle-orm@0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) - drizzle-orm: 0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0) + drizzle-orm: 0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0) postgres-range: 1.1.4 optionalDependencies: pg: 8.18.0 @@ -3654,10 +3654,11 @@ snapshots: transitivePeerDependencies: - supports-color - drizzle-orm@0.38.4(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0): + drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0): optionalDependencies: '@opentelemetry/api': 1.9.0 '@types/pg': 8.16.0 + gel: 2.2.0 pg: 8.18.0 eastasianwidth@0.2.0: {} From 6454a34673a30ceca289d1c5685e461356aaaf28 Mon Sep 17 00:00:00 2001 From: loothero Date: Thu, 5 Mar 2026 06:32:09 -0800 Subject: [PATCH 05/39] chore(indexer): bump pg and @types/pg to latest --- indexer/package.json | 4 +-- indexer/pnpm-lock.yaml | 64 +++++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/indexer/package.json b/indexer/package.json index a81d3f3c..b265cad2 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -33,13 +33,13 @@ "@apibara/starknet": "2.1.0-beta.56", "apibara": "2.1.0-beta.55", "drizzle-orm": "^0.45.1", - "pg": "^8.13.0", + "pg": "^8.20.0", "starknet": "^7.1.0" }, "devDependencies": { "@eslint/js": "^9.39.2", "@types/node": "^22.0.0", - "@types/pg": "^8.11.0", + "@types/pg": "^8.18.0", "@vitest/coverage-v8": "^3.1.1", "@vitest/eslint-plugin": "^1.6.9", "drizzle-kit": "^0.30.0", diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index f72c9eee..694eab63 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -13,7 +13,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -25,10 +25,10 @@ importers: version: 2.1.0-beta.55(magicast@0.3.5)(rollup@4.57.1)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) drizzle-orm: specifier: ^0.45.1 - version: 0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0) + version: 0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0) pg: - specifier: ^8.13.0 - version: 8.18.0 + specifier: ^8.20.0 + version: 8.20.0 starknet: specifier: ^7.1.0 version: 7.6.4 @@ -40,8 +40,8 @@ importers: specifier: ^22.0.0 version: 22.19.11 '@types/pg': - specifier: ^8.11.0 - version: 8.16.0 + specifier: ^8.18.0 + version: 8.18.0 '@vitest/coverage-v8': specifier: ^3.1.1 version: 3.2.4(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) @@ -1053,8 +1053,8 @@ packages: '@types/node@22.19.11': resolution: {integrity: sha512-BH7YwL6rA93ReqeQS1c4bsPpcfOmJasG+Fkr6Y59q83f9M1WcBRHR2vM+P9eOisYRcN3ujQoiZY8uk5W+1WL8w==} - '@types/pg@8.16.0': - resolution: {integrity: sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ==} + '@types/pg@8.18.0': + resolution: {integrity: sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==} '@typescript-eslint/eslint-plugin@8.55.0': resolution: {integrity: sha512-1y/MVSz0NglV1ijHC8OT49mPJ4qhPYjiK08YUQVbIOyu+5k862LKUHFkpKHWu//zmr7hDR2rhwUm6gnCGNmGBQ==} @@ -1995,27 +1995,27 @@ packages: pg-cloudflare@1.3.0: resolution: {integrity: sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==} - pg-connection-string@2.11.0: - resolution: {integrity: sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==} + pg-connection-string@2.12.0: + resolution: {integrity: sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==} pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} - pg-pool@3.11.0: - resolution: {integrity: sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w==} + pg-pool@3.13.0: + resolution: {integrity: sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==} peerDependencies: pg: '>=8.0' - pg-protocol@1.11.0: - resolution: {integrity: sha512-pfsxk2M9M3BuGgDOfuy37VNRRX3jmKgMjcvAcWqNDpZSf4cUmv8HSOl5ViRQFsfARFn0KuUQTgLxVMbNq5NW3g==} + pg-protocol@1.13.0: + resolution: {integrity: sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==} pg-types@2.2.0: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} - pg@8.18.0: - resolution: {integrity: sha512-xqrUDL1b9MbkydY/s+VZ6v+xiMUmOUk7SS9d/1kpyQxoJ6U9AO1oIJyUWVZojbfe5Cc/oluutcgFG4L9RDP1iQ==} + pg@8.20.0: + resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==} engines: {node: '>= 16.0.0'} peerDependencies: pg-native: '>=3.0.1' @@ -2502,14 +2502,14 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0))(pg@8.18.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) - drizzle-orm: 0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0) + drizzle-orm: 0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0) postgres-range: 1.1.4 optionalDependencies: - pg: 8.18.0 + pg: 8.20.0 transitivePeerDependencies: - bufferutil - typescript @@ -3247,10 +3247,10 @@ snapshots: dependencies: undici-types: 6.21.0 - '@types/pg@8.16.0': + '@types/pg@8.18.0': dependencies: '@types/node': 22.19.11 - pg-protocol: 1.11.0 + pg-protocol: 1.13.0 pg-types: 2.2.0 '@typescript-eslint/eslint-plugin@8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': @@ -3654,12 +3654,12 @@ snapshots: transitivePeerDependencies: - supports-color - drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(gel@2.2.0)(pg@8.18.0): + drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0): optionalDependencies: '@opentelemetry/api': 1.9.0 - '@types/pg': 8.16.0 + '@types/pg': 8.18.0 gel: 2.2.0 - pg: 8.18.0 + pg: 8.20.0 eastasianwidth@0.2.0: {} @@ -4239,15 +4239,15 @@ snapshots: pg-cloudflare@1.3.0: optional: true - pg-connection-string@2.11.0: {} + pg-connection-string@2.12.0: {} pg-int8@1.0.1: {} - pg-pool@3.11.0(pg@8.18.0): + pg-pool@3.13.0(pg@8.20.0): dependencies: - pg: 8.18.0 + pg: 8.20.0 - pg-protocol@1.11.0: {} + pg-protocol@1.13.0: {} pg-types@2.2.0: dependencies: @@ -4257,11 +4257,11 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 - pg@8.18.0: + pg@8.20.0: dependencies: - pg-connection-string: 2.11.0 - pg-pool: 3.11.0(pg@8.18.0) - pg-protocol: 1.11.0 + pg-connection-string: 2.12.0 + pg-pool: 3.13.0(pg@8.20.0) + pg-protocol: 1.13.0 pg-types: 2.2.0 pgpass: 1.0.5 optionalDependencies: From c87534639c2f208f5fa924bcbff2611f1f984fec Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Thu, 5 Mar 2026 10:51:22 -0800 Subject: [PATCH 06/39] fix(observability): address metrics review findings --- README.md | 32 +++ api/src/index.ts | 82 +++++- api/src/lib/metrics.ts | 243 ++++++++++++++++++ indexer/indexers/summit.indexer.ts | 72 ++++++ indexer/package.json | 2 + indexer/scripts/railway-metrics-summary.mjs | 271 ++++++++++++++++++++ indexer/src/lib/metrics.ts | 243 ++++++++++++++++++ scripts/check-metrics-sync.mjs | 22 ++ 8 files changed, 959 insertions(+), 8 deletions(-) create mode 100644 api/src/lib/metrics.ts create mode 100755 indexer/scripts/railway-metrics-summary.mjs create mode 100644 indexer/src/lib/metrics.ts create mode 100755 scripts/check-metrics-sync.mjs diff --git a/README.md b/README.md index b25f823f..6b8b35aa 100644 --- a/README.md +++ b/README.md @@ -93,3 +93,35 @@ PR CI is path-filtered and runs component-specific checks: - [Game Docs](https://docs.provable.games/summit) - [View & Trade Beasts](https://beast-dex.vercel.app/marketplace) - [Collect Beasts in Loot Survivor](https://lootsurvivor.io) + +## Resource Metrics from Railway Logs + +API and indexer now emit structured metric lines in logs with the prefix: + +`METRIC resource_metric_v1 {...}` + +Defaults: + +- `METRICS_ENABLED=true` in production by default +- `METRICS_INTERVAL_MS=30000` +- `DB_METRICS_INTERVAL_MS=60000` + +To summarize current RAM/CPU/DB pressure from Railway logs: + +```bash +cd indexer +pnpm metrics:snapshot -- --minutes 10 +``` + +JSON output (for agent/tool consumption): + +```bash +pnpm metrics:snapshot -- --minutes 10 --json +``` + +Verify duplicated metrics modules are in sync: + +```bash +cd indexer +pnpm metrics:check-sync +``` diff --git a/api/src/index.ts b/api/src/index.ts index 014be968..5bcc6970 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -12,7 +12,7 @@ import { v4 as uuidv4 } from "uuid"; import { eq, sql, desc, and, inArray } from "drizzle-orm"; import "dotenv/config"; -import { checkDatabaseHealth, db } from "./db/client.js"; +import { checkDatabaseHealth, db, pool } from "./db/client.js"; import { beasts, beast_owners, @@ -33,10 +33,59 @@ import { ITEM_NAME_PREFIXES, ITEM_NAME_SUFFIXES, } from "./lib/beastData.js"; +import { isMetricsEnabled, startResourceMetrics } from "./lib/metrics.js"; import { getBeastRevivalTime, getBeastCurrentLevel, normalizeAddress } from "./lib/helpers.js"; const isDevelopment = process.env.NODE_ENV !== "production"; +async function collectDbProxyMetrics() { + const [connections, databaseStats, databaseSize] = await Promise.all([ + pool.query( + ` + SELECT + count(*) FILTER (WHERE state = 'active')::bigint AS active_connections, + count(*) FILTER (WHERE state = 'idle')::bigint AS idle_connections + FROM pg_stat_activity + WHERE datname = current_database(); + ` + ), + pool.query( + ` + SELECT + xact_commit::bigint AS xact_commit, + xact_rollback::bigint AS xact_rollback, + blks_read::bigint AS blks_read, + blks_hit::bigint AS blks_hit, + temp_files::bigint AS temp_files, + temp_bytes::bigint AS temp_bytes + FROM pg_stat_database + WHERE datname = current_database(); + ` + ), + pool.query( + ` + SELECT pg_database_size(current_database())::bigint AS db_size_bytes; + ` + ), + ]); + + const connectionRow = connections.rows[0] ?? {}; + const statRow = databaseStats.rows[0] ?? {}; + const sizeRow = databaseSize.rows[0] ?? {}; + + return { + db_active_connections: Number(connectionRow.active_connections ?? 0), + db_idle_connections: Number(connectionRow.idle_connections ?? 0), + db_xact_commit: Number(statRow.xact_commit ?? 0), + db_xact_rollback: Number(statRow.xact_rollback ?? 0), + db_blks_read: Number(statRow.blks_read ?? 0), + db_blks_hit: Number(statRow.blks_hit ?? 0), + db_temp_files: Number(statRow.temp_files ?? 0), + db_temp_bytes: Number(statRow.temp_bytes ?? 0), + db_size_bytes: Number(sizeRow.db_size_bytes ?? 0), + }; +} + const app = new Hono(); // Middleware @@ -755,17 +804,34 @@ const server = serve( injectWebSocket(server); +const metricEmitters: Array<{ stop: () => void }> = []; + +if (isMetricsEnabled()) { + metricEmitters.push( + startResourceMetrics({ + service: "summit-api", + dbPoolStats: () => ({ + total: pool.totalCount, + idle: pool.idleCount, + waiting: pool.waitingCount, + }), + dbProbe: collectDbProxyMetrics, + }) + ); +} + // Graceful shutdown -process.on("SIGINT", async () => { +async function shutdown() { console.log("\nShutting down..."); + for (const emitter of metricEmitters) { + emitter.stop(); + } await getSubscriptionHub().shutdown(); + await pool.end(); process.exit(0); -}); +} -process.on("SIGTERM", async () => { - console.log("\nShutting down..."); - await getSubscriptionHub().shutdown(); - process.exit(0); -}); +process.once("SIGINT", shutdown); +process.once("SIGTERM", shutdown); export default app; diff --git a/api/src/lib/metrics.ts b/api/src/lib/metrics.ts new file mode 100644 index 00000000..ce83ae79 --- /dev/null +++ b/api/src/lib/metrics.ts @@ -0,0 +1,243 @@ +// NOTE: Keep this file byte-for-byte in sync between: +// api/src/lib/metrics.ts and indexer/src/lib/metrics.ts +// Verify with: node scripts/check-metrics-sync.mjs +import { existsSync, readFileSync } from "node:fs"; +import os from "node:os"; + +type MetricPrimitive = string | number | boolean | null; +type MetricRecord = Record; + +export interface ResourceMetricsOptions { + service: string; + environment?: string; + intervalMs?: number; + dbProbeIntervalMs?: number; + dbPoolStats?: () => { total: number; idle: number; waiting: number } | null; + dbProbe?: () => Promise>; + getExtraMetrics?: () => MetricRecord; + log?: (line: string) => void; +} + +interface CgroupPaths { + memCurrent?: string; + memMax?: string; + cpuStat?: string; + cpuUsageNs?: string; +} + +const cgroupPaths: CgroupPaths = detectCgroupPaths(); + +export function isMetricsEnabled(): boolean { + const raw = process.env.METRICS_ENABLED?.trim().toLowerCase(); + if (!raw) return process.env.NODE_ENV === "production"; + return !(raw === "0" || raw === "false" || raw === "off" || raw === "no"); +} + +export function startResourceMetrics(options: ResourceMetricsOptions): { stop: () => void } { + const intervalMs = Number(process.env.METRICS_INTERVAL_MS || options.intervalMs || 30_000); + const dbProbeIntervalMs = Number(process.env.DB_METRICS_INTERVAL_MS || options.dbProbeIntervalMs || 60_000); + const log = options.log ?? console.log; + + let inFlight = false; + let expectedTick = Date.now() + intervalMs; + let previousCpu = process.cpuUsage(); + let previousWallNs = process.hrtime.bigint(); + let nextDbProbeAt = Date.now(); + let lastDbMetrics: Record = {}; + + const timer = setInterval(async () => { + if (inFlight) return; + inFlight = true; + + try { + const now = Date.now(); + const loopLagMs = Math.max(0, now - expectedTick); + expectedTick = now + intervalMs; + + const currentWallNs = process.hrtime.bigint(); + const elapsedNs = Number(currentWallNs - previousWallNs); + previousWallNs = currentWallNs; + + const currentCpu = process.cpuUsage(); + const cpuUsage = { + user: currentCpu.user - previousCpu.user, + system: currentCpu.system - previousCpu.system, + }; + previousCpu = currentCpu; + const cpuMicros = cpuUsage.user + cpuUsage.system; + const cpuPct = elapsedNs > 0 ? (cpuMicros / (elapsedNs / 1_000)) * 100 : null; + + if (options.dbProbe && now >= nextDbProbeAt) { + nextDbProbeAt = now + dbProbeIntervalMs; + try { + lastDbMetrics = await options.dbProbe(); + } catch { + lastDbMetrics = { + ...lastDbMetrics, + db_probe_error: 1, + }; + } + } + + const memory = process.memoryUsage(); + const cgroup = readCgroupStats(); + const poolStats = options.dbPoolStats?.() ?? null; + + const payload: MetricRecord = { + schema: "resource_metric_v1", + service: options.service, + environment: + options.environment ?? + process.env.RAILWAY_ENVIRONMENT_NAME ?? + process.env.NODE_ENV ?? + "unknown", + timestamp: new Date().toISOString(), + uptime_s: Math.round(process.uptime()), + cpu_cores: os.cpus().length, + process_cpu_pct: cpuPct === null ? null : round(cpuPct, 2), + event_loop_lag_ms: round(loopLagMs, 2), + rss_bytes: memory.rss, + heap_used_bytes: memory.heapUsed, + heap_total_bytes: memory.heapTotal, + external_bytes: memory.external, + array_buffers_bytes: memory.arrayBuffers, + cgroup_mem_current_bytes: cgroup.memCurrent, + cgroup_mem_max_bytes: cgroup.memMax, + cgroup_cpu_usage_usec: cgroup.cpuUsageUsec, + cgroup_cpu_throttled_usec: cgroup.cpuThrottledUsec, + db_pool_total: poolStats?.total ?? null, + db_pool_idle: poolStats?.idle ?? null, + db_pool_waiting: poolStats?.waiting ?? null, + }; + + for (const [key, value] of Object.entries(lastDbMetrics)) { + payload[key] = value; + } + + if (options.getExtraMetrics) { + const extras = options.getExtraMetrics(); + for (const [key, value] of Object.entries(extras)) { + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + payload[key] = value; + } + } + } + + log(`METRIC resource_metric_v1 ${JSON.stringify(payload)}`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log(`METRIC resource_metric_v1 ${JSON.stringify({ schema: "resource_metric_v1", service: options.service, metric_error: message, timestamp: new Date().toISOString() })}`); + } finally { + inFlight = false; + } + }, intervalMs); + + timer.unref?.(); + + return { + stop: () => clearInterval(timer), + }; +} + +function round(value: number, places: number): number { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function detectCgroupPaths(): CgroupPaths { + const paths: CgroupPaths = {}; + + if (existsSync("/sys/fs/cgroup/memory.current")) { + paths.memCurrent = "/sys/fs/cgroup/memory.current"; + } else if (existsSync("/sys/fs/cgroup/memory/memory.usage_in_bytes")) { + paths.memCurrent = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; + } + + if (existsSync("/sys/fs/cgroup/memory.max")) { + paths.memMax = "/sys/fs/cgroup/memory.max"; + } else if (existsSync("/sys/fs/cgroup/memory/memory.limit_in_bytes")) { + paths.memMax = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; + } + + if (existsSync("/sys/fs/cgroup/cpu.stat")) { + paths.cpuStat = "/sys/fs/cgroup/cpu.stat"; + } else if (existsSync("/sys/fs/cgroup/cpu/cpu.stat")) { + paths.cpuStat = "/sys/fs/cgroup/cpu/cpu.stat"; + } + + if (existsSync("/sys/fs/cgroup/cpuacct.usage")) { + paths.cpuUsageNs = "/sys/fs/cgroup/cpuacct.usage"; + } + + return paths; +} + +function readCgroupStats(): { + memCurrent: number | null; + memMax: number | null; + cpuUsageUsec: number | null; + cpuThrottledUsec: number | null; +} { + const memCurrent = readNumber(cgroupPaths.memCurrent); + const memMax = readNumber(cgroupPaths.memMax); + + let cpuUsageUsec: number | null = null; + let cpuThrottledUsec: number | null = null; + + if (cgroupPaths.cpuStat) { + const stat = parseCpuStat(cgroupPaths.cpuStat); + cpuUsageUsec = stat.usageUsec; + cpuThrottledUsec = stat.throttledUsec; + } else if (cgroupPaths.cpuUsageNs) { + const usageNs = readNumber(cgroupPaths.cpuUsageNs); + cpuUsageUsec = usageNs === null ? null : Math.round(usageNs / 1_000); + } + + return { + memCurrent, + memMax, + cpuUsageUsec, + cpuThrottledUsec, + }; +} + +function parseCpuStat(path: string): { usageUsec: number | null; throttledUsec: number | null } { + try { + const content = readFileSync(path, "utf8"); + const lines = content.split(/\r?\n/); + let usageUsec: number | null = null; + let throttledUsec: number | null = null; + + for (const line of lines) { + const [key, rawValue] = line.trim().split(/\s+/); + if (!key || !rawValue) continue; + const value = Number(rawValue); + if (!Number.isFinite(value)) continue; + + if (key === "usage_usec") usageUsec = value; + if (key === "throttled_usec") throttledUsec = value; + if (key === "throttled_time") throttledUsec = Math.round(value / 1_000); + } + + return { usageUsec, throttledUsec }; + } catch { + return { usageUsec: null, throttledUsec: null }; + } +} + +function readNumber(path?: string): number | null { + if (!path) return null; + try { + const value = readFileSync(path, "utf8").trim(); + if (!value || value === "max") return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } catch { + return null; + } +} diff --git a/indexer/indexers/summit.indexer.ts b/indexer/indexers/summit.indexer.ts index bfab31a7..98685ecf 100644 --- a/indexer/indexers/summit.indexer.ts +++ b/indexer/indexers/summit.indexer.ts @@ -60,6 +60,7 @@ import { feltToHex, isZeroFeltAddress, } from "../src/lib/decoder.js"; +import { isMetricsEnabled, startResourceMetrics } from "../src/lib/metrics.js"; interface SummitConfig { summitContractAddress: string; @@ -867,6 +868,69 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { console.error("[Summit Indexer] Pool background connection error:", err.message); }); + const perfState = { + last_block_number: startingBlock.toString(), + last_block_events: 0, + last_block_total_ms: 0, + last_insert_ms: 0, + last_context_lookup_ms: 0, + blocks_without_events: 0, + blocks_with_events_total: 0, + events_processed_total: 0, + }; + + const metricEmitters: Array<{ stop: () => void }> = []; + if (isMetricsEnabled()) { + const pool = database.$client as { + totalCount?: number; + idleCount?: number; + waitingCount?: number; + }; + const hasPoolCounters = + typeof pool.totalCount === "number" && + typeof pool.idleCount === "number" && + typeof pool.waitingCount === "number"; + + if (!hasPoolCounters) { + console.warn( + "[Summit Indexer] Pool counters unavailable on Drizzle client; db_pool_* metrics will be null." + ); + } + + metricEmitters.push( + startResourceMetrics({ + service: "summit-indexer", + dbPoolStats: () => + hasPoolCounters + ? { + total: pool.totalCount as number, + idle: pool.idleCount as number, + waiting: pool.waitingCount as number, + } + : null, + getExtraMetrics: () => ({ + last_block_number: perfState.last_block_number, + last_block_events: perfState.last_block_events, + last_block_total_ms: perfState.last_block_total_ms, + last_insert_ms: perfState.last_insert_ms, + last_context_lookup_ms: perfState.last_context_lookup_ms, + blocks_without_events: perfState.blocks_without_events, + blocks_with_events_total: perfState.blocks_with_events_total, + events_processed_total: perfState.events_processed_total, + }), + }) + ); + } + + const stopMetricEmitters = () => { + for (const emitter of metricEmitters) { + emitter.stop(); + } + }; + process.once("beforeExit", stopMetricEmitters); + process.once("SIGINT", stopMetricEmitters); + process.once("SIGTERM", stopMetricEmitters); + // getBeast selector: starknet_keccak("getBeast") const GET_BEAST_SELECTOR = "0x0385b69551f247794fe651459651cdabc76b6cdf4abacafb5b28ceb3b1ac2e98"; @@ -2070,10 +2134,18 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { const insertStartTime = Date.now(); await executeBulkInserts(db, batches); const insertTime = Date.now() - insertStartTime; + perfState.last_block_number = block_number.toString(); + perfState.blocks_without_events = blocksWithoutEvents; // Log performance metrics for blocks with events if (events.length > 0) { const totalTime = Date.now() - blockStartTime; + perfState.last_block_events = events.length; + perfState.last_block_total_ms = totalTime; + perfState.last_insert_ms = insertTime; + perfState.last_context_lookup_ms = contextLookupTime; + perfState.blocks_with_events_total += 1; + perfState.events_processed_total += events.length; // Detailed timing breakdown // scan = pre-scan to collect token IDs diff --git a/indexer/package.json b/indexer/package.json index b265cad2..69ec3b17 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -8,6 +8,8 @@ "lint": "eslint . --cache --cache-strategy content --cache-location .cache/eslint --concurrency auto", "lint:ci": "eslint . --max-warnings=0 --report-unused-inline-configs error", "start": "apibara start", + "metrics:check-sync": "node ../scripts/check-metrics-sync.mjs", + "metrics:snapshot": "node scripts/railway-metrics-summary.mjs", "test": "vitest run", "test:coverage": "vitest run --coverage", "test:parity": "tsx scripts/test-live-beast-stats-parity.ts", diff --git a/indexer/scripts/railway-metrics-summary.mjs b/indexer/scripts/railway-metrics-summary.mjs new file mode 100755 index 00000000..3b8c7509 --- /dev/null +++ b/indexer/scripts/railway-metrics-summary.mjs @@ -0,0 +1,271 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; + +const DEFAULT_MINUTES = 10; +const DEFAULT_LINES = 2000; + +function parseArgs(argv) { + const options = { + environment: "production", + minutes: DEFAULT_MINUTES, + lines: DEFAULT_LINES, + services: [], + json: false, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--environment" || arg === "-e") { + options.environment = argv[++i] ?? options.environment; + } else if (arg === "--minutes" || arg === "-m") { + options.minutes = Number(argv[++i] ?? options.minutes); + } else if (arg === "--lines" || arg === "-n") { + options.lines = Number(argv[++i] ?? options.lines); + } else if (arg === "--service" || arg === "-s") { + options.services.push(argv[++i]); + } else if (arg === "--json") { + options.json = true; + } else if (arg === "--help" || arg === "-h") { + printHelp(); + process.exit(0); + } + } + + options.services = options.services.filter(Boolean); + return options; +} + +function printHelp() { + console.log(`Usage: node scripts/railway-metrics-summary.mjs [options]\n\nOptions:\n -e, --environment Railway environment (default: production)\n -m, --minutes Lookback window in minutes (default: 10)\n -n, --lines Max log lines per service (default: 2000)\n -s, --service Service to query (repeatable)\n --json Output JSON\n -h, --help Show help\n`); +} + +function runRailway(args) { + const result = spawnSync("railway", args, { encoding: "utf8" }); + if (result.status !== 0) { + const detail = (result.stderr || result.stdout || "").trim(); + throw new Error(detail || `railway ${args.join(" ")} failed`); + } + return result.stdout; +} + +function getServices() { + const statusRaw = runRailway(["status", "--json"]); + const status = JSON.parse(statusRaw); + const services = status?.services?.edges?.map((edge) => edge?.node?.name).filter(Boolean) ?? []; + + if (services.length === 0) { + throw new Error("No services found in linked project"); + } + + return services; +} + +function parseMetric(message) { + const marker = "METRIC resource_metric_v1 "; + const index = message.indexOf(marker); + if (index === -1) return null; + + const jsonPart = message.slice(index + marker.length).trim(); + try { + return JSON.parse(jsonPart); + } catch { + return null; + } +} + +function parseLogOutput(raw, fallbackService) { + const entries = []; + for (const line of raw.split(/\r?\n/)) { + if (!line.trim()) continue; + + try { + const parsed = JSON.parse(line); + const messageCandidate = + parsed?.message ?? parsed?.msg ?? parsed?.text ?? parsed?.line ?? parsed?.log ?? ""; + const message = typeof messageCandidate === "string" ? messageCandidate : JSON.stringify(messageCandidate); + const metric = parseMetric(message); + if (!metric) continue; + const timestamp = metric.timestamp ?? parsed?.timestamp ?? new Date().toISOString(); + const metricService = String(metric.service ?? fallbackService); + entries.push({ timestamp, service: metricService, metric }); + continue; + } catch { + const metric = parseMetric(line); + if (!metric) continue; + const timestamp = metric.timestamp ?? new Date().toISOString(); + const metricService = String(metric.service ?? fallbackService); + entries.push({ timestamp, service: metricService, metric }); + } + } + return entries; +} + +function toNumber(value) { + if (value === null || value === undefined) return null; + const num = Number(value); + return Number.isFinite(num) ? num : null; +} + +function toMB(value) { + const num = toNumber(value); + if (num === null) return null; + return num / (1024 * 1024); +} + +function buildSummary(metrics) { + const byService = new Map(); + + for (const row of metrics) { + if (!byService.has(row.service)) byService.set(row.service, []); + byService.get(row.service).push(row); + } + + const summaries = []; + for (const [service, rows] of byService.entries()) { + rows.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); + const latest = rows[rows.length - 1]?.metric; + const first = rows[0]?.metric; + if (!latest) continue; + + const rssLatestMb = toMB(latest.rss_bytes); + const rssFirstMb = first ? toMB(first.rss_bytes) : null; + const cpuPctValues = rows + .map((row) => toNumber(row.metric.process_cpu_pct)) + .filter((value) => value !== null); + const avgCpuPct = + cpuPctValues.length > 0 + ? cpuPctValues.reduce((sum, value) => sum + value, 0) / cpuPctValues.length + : null; + + summaries.push({ + service, + samples: rows.length, + timestamp: latest.timestamp ?? rows[rows.length - 1].timestamp, + rss_mb: rssLatestMb, + rss_delta_mb: rssLatestMb !== null && rssFirstMb !== null ? rssLatestMb - rssFirstMb : null, + heap_mb: toMB(latest.heap_used_bytes), + cpu_pct: toNumber(latest.process_cpu_pct), + cpu_pct_avg_window: avgCpuPct, + event_loop_lag_ms: toNumber(latest.event_loop_lag_ms), + cgroup_mem_mb: toMB(latest.cgroup_mem_current_bytes), + db_active: toNumber(latest.db_active_connections), + db_idle: toNumber(latest.db_idle_connections), + db_size_mb: toMB(latest.db_size_bytes), + db_pool_waiting: toNumber(latest.db_pool_waiting), + events_processed_total: toNumber(latest.events_processed_total), + last_block_number: latest.last_block_number ?? null, + }); + } + + return summaries.sort((a, b) => a.service.localeCompare(b.service)); +} + +function formatNumber(value, digits = 2) { + return value === null || value === undefined ? "-" : Number(value).toFixed(digits); +} + +function printTable(summaries, minutes, environment) { + console.log(`Railway resource metric snapshot (${environment}, last ${minutes}m)`); + if (summaries.length === 0) { + console.log("No resource_metric_v1 lines found in the selected window."); + return; + } + + const header = [ + "service", + "samples", + "rss_mb", + "rss_delta_mb", + "cpu_pct", + "cpu_avg", + "heap_mb", + "lag_ms", + "cgroup_mem_mb", + "db_active", + "db_waiting", + "last_block", + ]; + + console.log(header.join("\t")); + for (const row of summaries) { + console.log( + [ + row.service, + row.samples, + formatNumber(row.rss_mb), + formatNumber(row.rss_delta_mb), + formatNumber(row.cpu_pct), + formatNumber(row.cpu_pct_avg_window), + formatNumber(row.heap_mb), + formatNumber(row.event_loop_lag_ms), + formatNumber(row.cgroup_mem_mb), + formatNumber(row.db_active, 0), + formatNumber(row.db_pool_waiting, 0), + row.last_block_number ?? "-", + ].join("\t") + ); + } +} + +function main() { + const options = parseArgs(process.argv.slice(2)); + const services = options.services.length > 0 ? options.services : getServices(); + + const allMetrics = []; + const errors = []; + + for (const service of services) { + try { + const raw = runRailway([ + "logs", + "--service", + service, + "--environment", + options.environment, + "--since", + `${options.minutes}m`, + "--lines", + String(options.lines), + "--json", + ]); + allMetrics.push(...parseLogOutput(raw, service)); + } catch (error) { + errors.push({ service, error: error instanceof Error ? error.message : String(error) }); + } + } + + const summaries = buildSummary(allMetrics); + + if (options.json) { + console.log( + JSON.stringify( + { + environment: options.environment, + minutes: options.minutes, + summaries, + errors, + }, + null, + 2 + ) + ); + return; + } + + printTable(summaries, options.minutes, options.environment); + if (errors.length > 0) { + console.log("\nErrors:"); + for (const entry of errors) { + console.log(`- ${entry.service}: ${entry.error}`); + } + } +} + +try { + main(); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exit(1); +} diff --git a/indexer/src/lib/metrics.ts b/indexer/src/lib/metrics.ts new file mode 100644 index 00000000..ce83ae79 --- /dev/null +++ b/indexer/src/lib/metrics.ts @@ -0,0 +1,243 @@ +// NOTE: Keep this file byte-for-byte in sync between: +// api/src/lib/metrics.ts and indexer/src/lib/metrics.ts +// Verify with: node scripts/check-metrics-sync.mjs +import { existsSync, readFileSync } from "node:fs"; +import os from "node:os"; + +type MetricPrimitive = string | number | boolean | null; +type MetricRecord = Record; + +export interface ResourceMetricsOptions { + service: string; + environment?: string; + intervalMs?: number; + dbProbeIntervalMs?: number; + dbPoolStats?: () => { total: number; idle: number; waiting: number } | null; + dbProbe?: () => Promise>; + getExtraMetrics?: () => MetricRecord; + log?: (line: string) => void; +} + +interface CgroupPaths { + memCurrent?: string; + memMax?: string; + cpuStat?: string; + cpuUsageNs?: string; +} + +const cgroupPaths: CgroupPaths = detectCgroupPaths(); + +export function isMetricsEnabled(): boolean { + const raw = process.env.METRICS_ENABLED?.trim().toLowerCase(); + if (!raw) return process.env.NODE_ENV === "production"; + return !(raw === "0" || raw === "false" || raw === "off" || raw === "no"); +} + +export function startResourceMetrics(options: ResourceMetricsOptions): { stop: () => void } { + const intervalMs = Number(process.env.METRICS_INTERVAL_MS || options.intervalMs || 30_000); + const dbProbeIntervalMs = Number(process.env.DB_METRICS_INTERVAL_MS || options.dbProbeIntervalMs || 60_000); + const log = options.log ?? console.log; + + let inFlight = false; + let expectedTick = Date.now() + intervalMs; + let previousCpu = process.cpuUsage(); + let previousWallNs = process.hrtime.bigint(); + let nextDbProbeAt = Date.now(); + let lastDbMetrics: Record = {}; + + const timer = setInterval(async () => { + if (inFlight) return; + inFlight = true; + + try { + const now = Date.now(); + const loopLagMs = Math.max(0, now - expectedTick); + expectedTick = now + intervalMs; + + const currentWallNs = process.hrtime.bigint(); + const elapsedNs = Number(currentWallNs - previousWallNs); + previousWallNs = currentWallNs; + + const currentCpu = process.cpuUsage(); + const cpuUsage = { + user: currentCpu.user - previousCpu.user, + system: currentCpu.system - previousCpu.system, + }; + previousCpu = currentCpu; + const cpuMicros = cpuUsage.user + cpuUsage.system; + const cpuPct = elapsedNs > 0 ? (cpuMicros / (elapsedNs / 1_000)) * 100 : null; + + if (options.dbProbe && now >= nextDbProbeAt) { + nextDbProbeAt = now + dbProbeIntervalMs; + try { + lastDbMetrics = await options.dbProbe(); + } catch { + lastDbMetrics = { + ...lastDbMetrics, + db_probe_error: 1, + }; + } + } + + const memory = process.memoryUsage(); + const cgroup = readCgroupStats(); + const poolStats = options.dbPoolStats?.() ?? null; + + const payload: MetricRecord = { + schema: "resource_metric_v1", + service: options.service, + environment: + options.environment ?? + process.env.RAILWAY_ENVIRONMENT_NAME ?? + process.env.NODE_ENV ?? + "unknown", + timestamp: new Date().toISOString(), + uptime_s: Math.round(process.uptime()), + cpu_cores: os.cpus().length, + process_cpu_pct: cpuPct === null ? null : round(cpuPct, 2), + event_loop_lag_ms: round(loopLagMs, 2), + rss_bytes: memory.rss, + heap_used_bytes: memory.heapUsed, + heap_total_bytes: memory.heapTotal, + external_bytes: memory.external, + array_buffers_bytes: memory.arrayBuffers, + cgroup_mem_current_bytes: cgroup.memCurrent, + cgroup_mem_max_bytes: cgroup.memMax, + cgroup_cpu_usage_usec: cgroup.cpuUsageUsec, + cgroup_cpu_throttled_usec: cgroup.cpuThrottledUsec, + db_pool_total: poolStats?.total ?? null, + db_pool_idle: poolStats?.idle ?? null, + db_pool_waiting: poolStats?.waiting ?? null, + }; + + for (const [key, value] of Object.entries(lastDbMetrics)) { + payload[key] = value; + } + + if (options.getExtraMetrics) { + const extras = options.getExtraMetrics(); + for (const [key, value] of Object.entries(extras)) { + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + payload[key] = value; + } + } + } + + log(`METRIC resource_metric_v1 ${JSON.stringify(payload)}`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log(`METRIC resource_metric_v1 ${JSON.stringify({ schema: "resource_metric_v1", service: options.service, metric_error: message, timestamp: new Date().toISOString() })}`); + } finally { + inFlight = false; + } + }, intervalMs); + + timer.unref?.(); + + return { + stop: () => clearInterval(timer), + }; +} + +function round(value: number, places: number): number { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function detectCgroupPaths(): CgroupPaths { + const paths: CgroupPaths = {}; + + if (existsSync("/sys/fs/cgroup/memory.current")) { + paths.memCurrent = "/sys/fs/cgroup/memory.current"; + } else if (existsSync("/sys/fs/cgroup/memory/memory.usage_in_bytes")) { + paths.memCurrent = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; + } + + if (existsSync("/sys/fs/cgroup/memory.max")) { + paths.memMax = "/sys/fs/cgroup/memory.max"; + } else if (existsSync("/sys/fs/cgroup/memory/memory.limit_in_bytes")) { + paths.memMax = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; + } + + if (existsSync("/sys/fs/cgroup/cpu.stat")) { + paths.cpuStat = "/sys/fs/cgroup/cpu.stat"; + } else if (existsSync("/sys/fs/cgroup/cpu/cpu.stat")) { + paths.cpuStat = "/sys/fs/cgroup/cpu/cpu.stat"; + } + + if (existsSync("/sys/fs/cgroup/cpuacct.usage")) { + paths.cpuUsageNs = "/sys/fs/cgroup/cpuacct.usage"; + } + + return paths; +} + +function readCgroupStats(): { + memCurrent: number | null; + memMax: number | null; + cpuUsageUsec: number | null; + cpuThrottledUsec: number | null; +} { + const memCurrent = readNumber(cgroupPaths.memCurrent); + const memMax = readNumber(cgroupPaths.memMax); + + let cpuUsageUsec: number | null = null; + let cpuThrottledUsec: number | null = null; + + if (cgroupPaths.cpuStat) { + const stat = parseCpuStat(cgroupPaths.cpuStat); + cpuUsageUsec = stat.usageUsec; + cpuThrottledUsec = stat.throttledUsec; + } else if (cgroupPaths.cpuUsageNs) { + const usageNs = readNumber(cgroupPaths.cpuUsageNs); + cpuUsageUsec = usageNs === null ? null : Math.round(usageNs / 1_000); + } + + return { + memCurrent, + memMax, + cpuUsageUsec, + cpuThrottledUsec, + }; +} + +function parseCpuStat(path: string): { usageUsec: number | null; throttledUsec: number | null } { + try { + const content = readFileSync(path, "utf8"); + const lines = content.split(/\r?\n/); + let usageUsec: number | null = null; + let throttledUsec: number | null = null; + + for (const line of lines) { + const [key, rawValue] = line.trim().split(/\s+/); + if (!key || !rawValue) continue; + const value = Number(rawValue); + if (!Number.isFinite(value)) continue; + + if (key === "usage_usec") usageUsec = value; + if (key === "throttled_usec") throttledUsec = value; + if (key === "throttled_time") throttledUsec = Math.round(value / 1_000); + } + + return { usageUsec, throttledUsec }; + } catch { + return { usageUsec: null, throttledUsec: null }; + } +} + +function readNumber(path?: string): number | null { + if (!path) return null; + try { + const value = readFileSync(path, "utf8").trim(); + if (!value || value === "max") return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } catch { + return null; + } +} diff --git a/scripts/check-metrics-sync.mjs b/scripts/check-metrics-sync.mjs new file mode 100755 index 00000000..ee794a39 --- /dev/null +++ b/scripts/check-metrics-sync.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(scriptDir, ".."); +const apiPath = resolve(repoRoot, "api/src/lib/metrics.ts"); +const indexerPath = resolve(repoRoot, "indexer/src/lib/metrics.ts"); + +const apiContents = readFileSync(apiPath, "utf8"); +const indexerContents = readFileSync(indexerPath, "utf8"); + +if (apiContents !== indexerContents) { + console.error("metrics.ts files are out of sync:"); + console.error(`- ${apiPath}`); + console.error(`- ${indexerPath}`); + process.exit(1); +} + +console.log("metrics.ts files are in sync."); From 3e1df5c8ef860d8b4db03d5ef8e27b3377b42fd3 Mon Sep 17 00:00:00 2001 From: loothero Date: Thu, 5 Mar 2026 16:33:27 -0800 Subject: [PATCH 07/39] fix(api): reduce hot-path logging with sampled structured logs --- api/src/db/client.test.ts | 7 +- api/src/db/client.ts | 13 ++- api/src/index.ts | 22 ++-- api/src/lib/logging.ts | 134 +++++++++++++++++++++++++ api/src/ws/subscriptions.ts | 195 ++++++++++++++++++++++++++++++++---- 5 files changed, 333 insertions(+), 38 deletions(-) create mode 100644 api/src/lib/logging.ts diff --git a/api/src/db/client.test.ts b/api/src/db/client.test.ts index b8283f4a..c13e0c79 100644 --- a/api/src/db/client.test.ts +++ b/api/src/db/client.test.ts @@ -78,9 +78,10 @@ describe("db client environment validation", () => { await importDbClientModule(); - expect(consoleWarnSpy).toHaveBeenCalledWith( - "[DB CONFIG] DATABASE_SSL not set in production, defaulting to SSL enabled", - ); + expect(consoleWarnSpy).toHaveBeenCalledTimes(1); + const firstWarn = consoleWarnSpy.mock.calls[0]?.[0]; + expect(typeof firstWarn).toBe("string"); + expect(String(firstWarn)).toContain("\"msg\":\"db_ssl_default_enabled\""); expect(mocks.poolCtor).toHaveBeenCalledWith(expect.objectContaining({ ssl: { rejectUnauthorized: false }, })); diff --git a/api/src/db/client.ts b/api/src/db/client.ts index 93296c8d..5db01b91 100644 --- a/api/src/db/client.ts +++ b/api/src/db/client.ts @@ -5,6 +5,7 @@ import { drizzle } from "drizzle-orm/node-postgres"; import pg from "pg"; +import { log } from "../lib/logging.js"; const databaseUrl = process.env.DATABASE_URL; if (!databaseUrl) { @@ -17,7 +18,9 @@ if (typeof databaseSsl !== "undefined" && databaseSsl !== "true" && databaseSsl } if (process.env.NODE_ENV === "production" && typeof databaseSsl === "undefined") { - console.warn('[DB CONFIG] DATABASE_SSL not set in production, defaulting to SSL enabled'); + log.warn("db_ssl_default_enabled", { + reason: "DATABASE_SSL missing in production", + }); } // Create a connection pool for queries @@ -35,7 +38,9 @@ const pool = new pg.Pool({ // Handle pool errors to prevent crashes from unexpected disconnections // pg-pool will automatically replace dead clients, so we just log here pool.on("error", (err) => { - console.error("[PG POOL ERROR]", err.message); + log.error("pg_pool_error", { + message: err.message, + }); }); // Create Drizzle ORM instance @@ -52,7 +57,9 @@ export async function checkDatabaseHealth(): Promise { client.release(); return true; } catch (error) { - console.error("[DB HEALTH CHECK] Failed:", error instanceof Error ? error.message : error); + log.error("db_health_check_failed", { + error: error instanceof Error ? error.message : String(error), + }); return false; } } diff --git a/api/src/index.ts b/api/src/index.ts index 5bcc6970..1b9f07a4 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -5,7 +5,6 @@ import { Hono } from "hono"; import { compress } from "hono/compress"; import { cors } from "hono/cors"; -import { logger } from "hono/logger"; import { serve } from "@hono/node-server"; import { createNodeWebSocket } from "@hono/node-ws"; import { v4 as uuidv4 } from "uuid"; @@ -35,6 +34,7 @@ import { } from "./lib/beastData.js"; import { isMetricsEnabled, startResourceMetrics } from "./lib/metrics.js"; import { getBeastRevivalTime, getBeastCurrentLevel, normalizeAddress } from "./lib/helpers.js"; +import { createRequestLogMiddleware, log } from "./lib/logging.js"; const isDevelopment = process.env.NODE_ENV !== "production"; @@ -89,7 +89,7 @@ async function collectDbProxyMetrics() { const app = new Hono(); // Middleware -app.use("*", logger()); +app.use("*", createRequestLogMiddleware()); app.use("*", compress()); app.use( "*", @@ -765,7 +765,6 @@ app.get( return { onOpen(_event, ws) { hub.addClient(clientId, ws.raw as unknown as Parameters[1]); - console.log(`[WebSocket] Client connected: ${clientId}`); }, onMessage(event, _ws) { @@ -775,11 +774,13 @@ app.get( onClose() { hub.removeClient(clientId); - console.log(`[WebSocket] Client disconnected: ${clientId}`); }, onError(error) { - console.error(`[WebSocket] Error for client ${clientId}:`, error); + log.warn("ws_client_error", { + client_id: clientId, + error, + }); hub.removeClient(clientId); }, }; @@ -788,8 +789,7 @@ app.get( // Start server const port = parseInt(process.env.PORT || "3001", 10); - -console.log(`Starting Summit API server on port ${port}...`); +log.info("api_starting", { port }); const server = serve( { @@ -797,8 +797,10 @@ const server = serve( port, }, (info) => { - console.log(`[API] Server running at http://localhost:${info.port}`); - console.log(`[API] WebSocket available at ws://localhost:${info.port}/ws`); + log.info("api_server_ready", { + http: `http://localhost:${info.port}`, + websocket: `ws://localhost:${info.port}/ws`, + }); } ); @@ -822,7 +824,7 @@ if (isMetricsEnabled()) { // Graceful shutdown async function shutdown() { - console.log("\nShutting down..."); + log.info("api_shutdown_started"); for (const emitter of metricEmitters) { emitter.stop(); } diff --git a/api/src/lib/logging.ts b/api/src/lib/logging.ts new file mode 100644 index 00000000..7cf3404f --- /dev/null +++ b/api/src/lib/logging.ts @@ -0,0 +1,134 @@ +import type { MiddlewareHandler } from "hono"; + +type LogLevel = "debug" | "info" | "warn" | "error"; +type LogMeta = Record; + +const LEVEL_WEIGHT: Record = { + debug: 10, + info: 20, + warn: 30, + error: 40, +}; + +const DEFAULT_LEVEL: LogLevel = process.env.NODE_ENV === "production" + ? "info" + : process.env.NODE_ENV === "test" + ? "warn" + : "debug"; +const configuredLevel = (process.env.LOG_LEVEL?.toLowerCase() as LogLevel | undefined) ?? DEFAULT_LEVEL; +const currentLevel = LEVEL_WEIGHT[configuredLevel] ? configuredLevel : DEFAULT_LEVEL; +const service = process.env.RAILWAY_SERVICE_NAME ?? "summit-api"; + +function shouldLog(level: LogLevel): boolean { + return LEVEL_WEIGHT[level] >= LEVEL_WEIGHT[currentLevel]; +} + +function toSerializable(value: unknown): unknown { + if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return value; + } + + if (value instanceof Error) { + return { + name: value.name, + message: value.message, + stack: value.stack, + }; + } + + if (Array.isArray(value)) { + return value.map(toSerializable); + } + + if (typeof value === "object" && value !== null) { + const out: Record = {}; + for (const [key, child] of Object.entries(value)) { + if (typeof child === "undefined") continue; + out[key] = toSerializable(child); + } + return out; + } + + return String(value); +} + +function emit(level: LogLevel, msg: string, meta?: LogMeta): void { + if (!shouldLog(level)) return; + + const payload = { + ts: new Date().toISOString(), + level, + service, + msg, + ...(meta ? (toSerializable(meta) as LogMeta) : {}), + }; + const line = JSON.stringify(payload); + + if (level === "error") { + console.error(line); + return; + } + + if (level === "warn") { + console.warn(line); + return; + } + + console.log(line); +} + +export const log = { + debug: (msg: string, meta?: LogMeta) => emit("debug", msg, meta), + info: (msg: string, meta?: LogMeta) => emit("info", msg, meta), + warn: (msg: string, meta?: LogMeta) => emit("warn", msg, meta), + error: (msg: string, meta?: LogMeta) => emit("error", msg, meta), +}; + +export function parsePositiveInt(input: string | undefined, fallback: number): number { + if (!input) return fallback; + const parsed = Number(input); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +export function parseProbability(input: string | undefined, fallback: number): number { + if (!input) return fallback; + const parsed = Number(input); + if (!Number.isFinite(parsed)) return fallback; + if (parsed <= 0) return 0; + if (parsed >= 1) return 1; + return parsed; +} + +export function createRequestLogMiddleware(): MiddlewareHandler { + const sampleRate = parseProbability( + process.env.REQUEST_LOG_SAMPLE_RATE, + process.env.NODE_ENV === "production" ? 0.01 : 1 + ); + const slowMs = parsePositiveInt(process.env.REQUEST_LOG_SLOW_MS, 1500); + + return async (c, next) => { + const started = Date.now(); + await next(); + + const durationMs = Date.now() - started; + const status = c.res.status; + const isServerError = status >= 500; + const isClientError = status >= 400 && status < 500; + const isSlow = durationMs >= slowMs; + const sampled = Math.random() < sampleRate; + + if (!isServerError && !isClientError && !isSlow && !sampled) { + return; + } + + const level: LogLevel = isServerError ? "error" : isClientError ? "warn" : "info"; + emit(level, "http_request", { + method: c.req.method, + path: c.req.path, + status, + duration_ms: durationMs, + request_id: c.req.header("x-request-id") ?? null, + }); + }; +} diff --git a/api/src/ws/subscriptions.ts b/api/src/ws/subscriptions.ts index 9191b756..f8ad6efa 100644 --- a/api/src/ws/subscriptions.ts +++ b/api/src/ws/subscriptions.ts @@ -8,6 +8,7 @@ */ import { pool } from "../db/client.js"; +import { log, parsePositiveInt } from "../lib/logging.js"; import type pg from "pg"; interface WebSocketLike { @@ -61,49 +62,155 @@ interface EventPayload { created_at: string; } +interface HubCounters { + connections: number; + disconnections: number; + subscribes: number; + unsubscribes: number; + reconnects: number; + parseErrors: number; + messageErrors: number; + sendErrors: number; + broadcasts: Record; + delivered: Record; +} + +function createCounters(): HubCounters { + return { + connections: 0, + disconnections: 0, + subscribes: 0, + unsubscribes: 0, + reconnects: 0, + parseErrors: 0, + messageErrors: 0, + sendErrors: 0, + broadcasts: { + summit: 0, + event: 0, + }, + delivered: { + summit: 0, + event: 0, + }, + }; +} + export class SubscriptionHub { private clients: Map = new Map(); private pgClient: pg.PoolClient | null = null; private isConnected = false; private reconnectTimer: ReturnType | null = null; + private summaryTimer: ReturnType | null = null; // Exponential backoff configuration (no max limit - retries forever) private reconnectAttempts = 0; private readonly baseReconnectDelay = 1000; // 1 second private readonly maxReconnectDelay = 30000; // 30 seconds (capped) + private readonly verboseLogs = + process.env.WS_VERBOSE_LOGS === "true" || + (process.env.NODE_ENV !== "production" && process.env.NODE_ENV !== "test"); + private readonly sendErrorSampleEvery = parsePositiveInt( + process.env.WS_SEND_ERROR_SAMPLE_EVERY, + 100 + ); + private readonly summaryIntervalMs = parsePositiveInt( + process.env.WS_LOG_SUMMARY_INTERVAL_MS, + 30_000 + ); + private counters = createCounters(); + private windowCounters = createCounters(); + constructor() { this.connect(); + this.startSummaryLogs(); + } + + private startSummaryLogs(): void { + this.summaryTimer = setInterval(() => { + const window = this.windowCounters; + this.windowCounters = createCounters(); + + log.info("ws_summary", { + connected_to_pg: this.isConnected, + active_clients: this.clients.size, + reconnect_attempt: this.reconnectAttempts, + window_connections: window.connections, + window_disconnections: window.disconnections, + window_subscribes: window.subscribes, + window_unsubscribes: window.unsubscribes, + window_reconnects: window.reconnects, + window_parse_errors: window.parseErrors, + window_message_errors: window.messageErrors, + window_send_errors: window.sendErrors, + window_broadcasts: window.broadcasts, + window_delivered: window.delivered, + total_connections: this.counters.connections, + total_disconnections: this.counters.disconnections, + total_send_errors: this.counters.sendErrors, + }); + }, this.summaryIntervalMs); + + this.summaryTimer.unref?.(); + } + + private bumpCounter(key: keyof Pick< + HubCounters, + | "connections" + | "disconnections" + | "subscribes" + | "unsubscribes" + | "reconnects" + | "parseErrors" + | "messageErrors" + | "sendErrors" + >): void { + this.counters[key] += 1; + this.windowCounters[key] += 1; + } + + private bumpBroadcast(channel: Channel, delivered: number): void { + this.counters.broadcasts[channel] += 1; + this.windowCounters.broadcasts[channel] += 1; + this.counters.delivered[channel] += delivered; + this.windowCounters.delivered[channel] += delivered; } private async connect(): Promise { try { this.pgClient = await pool.connect(); this.isConnected = true; - this.reconnectAttempts = 0; // Reset on successful connection + this.reconnectAttempts = 0; - console.log("[SubscriptionHub] Connected to PostgreSQL for LISTEN"); + log.info("ws_pg_listen_connected"); this.pgClient.on("notification", (msg) => { this.handleNotification(msg); }); this.pgClient.on("error", (err) => { - console.error("[SubscriptionHub] PostgreSQL client error:", err); + log.error("ws_pg_client_error", { + error: err, + }); this.reconnect(); }); this.pgClient.on("end", () => { - console.log("[SubscriptionHub] PostgreSQL client disconnected"); + log.warn("ws_pg_client_disconnected"); this.reconnect(); }); await this.pgClient.query("LISTEN summit_update"); await this.pgClient.query("LISTEN summit_log_insert"); - console.log("[SubscriptionHub] Listening on: summit_update, summit_log_insert"); + log.info("ws_pg_listening_channels", { + channels: ["summit_update", "summit_log_insert"], + }); } catch (error) { - console.error("[SubscriptionHub] Failed to connect:", error); + log.error("ws_pg_connect_failed", { + error, + }); this.reconnect(); } } @@ -121,17 +228,18 @@ export class SubscriptionHub { this.pgClient = null; } - this.reconnectAttempts++; + this.reconnectAttempts += 1; + this.bumpCounter("reconnects"); - // Exponential backoff: 1s, 2s, 4s, 8s, 16s, 30s (capped at 30s, retries forever) const delay = Math.min( this.baseReconnectDelay * Math.pow(2, this.reconnectAttempts - 1), this.maxReconnectDelay ); - console.log( - `[SubscriptionHub] Reconnecting in ${delay / 1000}s (attempt ${this.reconnectAttempts})...` - ); + log.warn("ws_pg_reconnect_scheduled", { + attempt: this.reconnectAttempts, + delay_ms: delay, + }); this.reconnectTimer = setTimeout(() => { this.reconnectTimer = null; @@ -154,39 +262,62 @@ export class SubscriptionHub { break; } } catch (error) { - console.error("[SubscriptionHub] Failed to parse notification:", error); + this.bumpCounter("parseErrors"); + log.error("ws_notification_parse_failed", { + channel: msg.channel, + error, + }); } } private broadcast(channel: Channel, data: SummitPayload | EventPayload): void { const message = JSON.stringify({ type: channel, data }); - let sentCount = 0; + let delivered = 0; for (const [, client] of this.clients) { if (!client.channels.has(channel)) continue; this.send(client.ws, message); - sentCount++; + delivered += 1; } - console.log(`[SubscriptionHub] ${channel} broadcast to ${sentCount}/${this.clients.size} clients`); + this.bumpBroadcast(channel, delivered); } private send(ws: WebSocketLike, message: string): void { try { ws.send(message); } catch (error) { - console.error("[SubscriptionHub] Failed to send message:", error); + this.bumpCounter("sendErrors"); + if (this.windowCounters.sendErrors % this.sendErrorSampleEvery === 1) { + log.warn("ws_send_failed_sampled", { + error, + window_send_errors: this.windowCounters.sendErrors, + sample_every: this.sendErrorSampleEvery, + }); + } } } addClient(id: string, ws: WebSocketLike): void { this.clients.set(id, { ws, channels: new Set() }); - console.log(`[SubscriptionHub] Client connected: ${id} (total: ${this.clients.size})`); + this.bumpCounter("connections"); + if (this.verboseLogs) { + log.debug("ws_client_connected", { + client_id: id, + total_clients: this.clients.size, + }); + } } removeClient(id: string): void { this.clients.delete(id); - console.log(`[SubscriptionHub] Client disconnected: ${id} (total: ${this.clients.size})`); + this.bumpCounter("disconnections"); + if (this.verboseLogs) { + log.debug("ws_client_disconnected", { + client_id: id, + total_clients: this.clients.size, + }); + } } subscribe(id: string, channels: Channel[]): void { @@ -197,7 +328,13 @@ export class SubscriptionHub { client.channels.add(channel); } - console.log(`[SubscriptionHub] Client ${id} subscribed to: ${channels.join(", ")}`); + this.bumpCounter("subscribes"); + if (this.verboseLogs) { + log.debug("ws_client_subscribed", { + client_id: id, + channels, + }); + } } unsubscribe(id: string, channels: Channel[]): void { @@ -208,7 +345,13 @@ export class SubscriptionHub { client.channels.delete(channel); } - console.log(`[SubscriptionHub] Client ${id} unsubscribed from: ${channels.join(", ")}`); + this.bumpCounter("unsubscribes"); + if (this.verboseLogs) { + log.debug("ws_client_unsubscribed", { + client_id: id, + channels, + }); + } } handleMessage(id: string, message: string): void { @@ -237,7 +380,10 @@ export class SubscriptionHub { break; } } catch (error) { - console.error("[SubscriptionHub] Failed to handle message:", error); + this.bumpCounter("messageErrors"); + log.warn("ws_handle_message_failed", { + error, + }); } } @@ -253,6 +399,11 @@ export class SubscriptionHub { clearTimeout(this.reconnectTimer); } + if (this.summaryTimer) { + clearInterval(this.summaryTimer); + this.summaryTimer = null; + } + if (this.pgClient) { await this.pgClient.query("UNLISTEN *"); this.pgClient.release(); @@ -267,7 +418,7 @@ export class SubscriptionHub { } this.clients.clear(); - console.log("[SubscriptionHub] Shutdown complete"); + log.info("ws_shutdown_complete"); } } From d24118c9c1ea9df3293534405675178b3c9a72de Mon Sep 17 00:00:00 2001 From: loothero Date: Thu, 5 Mar 2026 20:03:05 -0800 Subject: [PATCH 08/39] perf(api): add thin strategic response cache --- api/.gitignore | 1 + api/AGENTS.md | 4 +- api/README.md | 13 +- api/src/index.ts | 410 ++++++++++++++++++++++---------------- api/src/lib/cache.test.ts | 216 ++++++++++++++++++++ api/src/lib/cache.ts | 195 ++++++++++++++++++ 6 files changed, 660 insertions(+), 179 deletions(-) create mode 100644 api/src/lib/cache.test.ts create mode 100644 api/src/lib/cache.ts diff --git a/api/.gitignore b/api/.gitignore index a373ee5a..7e522134 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -3,6 +3,7 @@ node_modules/ # Build output dist/ +.cache/ # Environment files .env diff --git a/api/AGENTS.md b/api/AGENTS.md index 2dda7d93..b9a4b995 100644 --- a/api/AGENTS.md +++ b/api/AGENTS.md @@ -64,7 +64,7 @@ Behavior details that affect integration: - lowercase - 66-char `0x` padded form. - No auth layer (public read API). -- No cache layer (responses are DB-backed). +- Thin in-memory SWR cache is enabled for high-traffic read endpoints. ## TypeScript and DB Settings - `tsconfig.json`: `strict: true`. @@ -94,3 +94,5 @@ Behavior details that affect integration: - `DB_POOL_MAX` (default `15`) - `PORT` (default `3001`) - `NODE_ENV` (`production` hides debug entries from `/` response) +- `API_CACHE_ENABLED` (optional; defaults to enabled in production) +- `API_CACHE_MAX_ENTRIES` (default `500`) diff --git a/api/README.md b/api/README.md index d35c138f..1b8ccfff 100644 --- a/api/README.md +++ b/api/README.md @@ -28,6 +28,8 @@ For AI-oriented coding guidance and deeper architecture notes, read `AGENTS.md` - `DB_POOL_MAX` (default `15`) - `PORT` (default `3001`) - `NODE_ENV` (`production` hides debug entries from `/` discovery payload) +- `API_CACHE_ENABLED` (optional; defaults to enabled in production) +- `API_CACHE_MAX_ENTRIES` (optional; default `500`) Production note: - API startup fails fast when `NODE_ENV=production` and `DATABASE_SSL` is unset. @@ -155,7 +157,16 @@ Realtime pipeline: - Address inputs are normalized to lowercase 66-char `0x`-padded form. - API is public read-only (no auth layer). -- No dedicated caching layer is used. +- A thin in-memory SWR cache is applied to high-traffic read endpoints: + - `/logs` + - `/beasts/stats/counts` + - `/beasts/stats/top` + - `/diplomacy` + - `/diplomacy/all` + - `/leaderboard` + - `/quest-rewards/total` + - `/consumables/supply` +- Cached responses include `X-Cache` with `HIT`, `MISS`, `STALE`, or `BYPASS`. - Graceful shutdown closes WS subscriptions/listeners on `SIGINT`/`SIGTERM`. ## Deployment Notes diff --git a/api/src/index.ts b/api/src/index.ts index 1b9f07a4..0009b876 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -2,7 +2,7 @@ * Summit API Server */ -import { Hono } from "hono"; +import { Hono, type Context } from "hono"; import { compress } from "hono/compress"; import { cors } from "hono/cors"; import { serve } from "@hono/node-server"; @@ -35,8 +35,57 @@ import { import { isMetricsEnabled, startResourceMetrics } from "./lib/metrics.js"; import { getBeastRevivalTime, getBeastCurrentLevel, normalizeAddress } from "./lib/helpers.js"; import { createRequestLogMiddleware, log } from "./lib/logging.js"; +import { + ApiResponseCache, + type CachePolicy, + createCacheKey, + parseCacheEnabled, + parseMaxEntries, + shouldBypassCache, +} from "./lib/cache.js"; const isDevelopment = process.env.NODE_ENV !== "production"; +const apiCache = new ApiResponseCache({ + enabled: parseCacheEnabled(), + maxEntries: parseMaxEntries(process.env.API_CACHE_MAX_ENTRIES), +}); + +const CACHE_POLICIES: Record< + | "logs" + | "beastsStatsCounts" + | "beastsStatsTop" + | "diplomacy" + | "diplomacyAll" + | "leaderboard" + | "questRewardsTotal" + | "consumablesSupply", + CachePolicy +> = { + logs: { freshTtlMs: 2_000, staleTtlMs: 8_000 }, + beastsStatsCounts: { freshTtlMs: 5_000, staleTtlMs: 20_000 }, + beastsStatsTop: { freshTtlMs: 3_000, staleTtlMs: 12_000 }, + diplomacy: { freshTtlMs: 15_000, staleTtlMs: 60_000 }, + diplomacyAll: { freshTtlMs: 30_000, staleTtlMs: 120_000 }, + leaderboard: { freshTtlMs: 3_000, staleTtlMs: 12_000 }, + questRewardsTotal: { freshTtlMs: 10_000, staleTtlMs: 40_000 }, + consumablesSupply: { freshTtlMs: 10_000, staleTtlMs: 40_000 }, +}; + +async function respondWithCachedJson( + c: Context, + policy: CachePolicy, + loader: () => Promise +): Promise { + if (!apiCache.enabledInRuntime || shouldBypassCache(c)) { + apiCache.noteBypass(); + c.header("X-Cache", "BYPASS"); + return c.json(await loader()); + } + + const { status, value } = await apiCache.getOrLoad(createCacheKey(c), policy, loader); + c.header("X-Cache", status); + return c.json(value); +} async function collectDbProxyMetrics() { const [connections, databaseStats, databaseSize] = await Promise.all([ @@ -406,41 +455,41 @@ app.get("/logs", async (c) => { const whereClause = conditions.length > 0 ? and(...conditions) : undefined; - // Get results - const results = await db - .select() - .from(summit_log) - .where(whereClause) - .orderBy(desc(summit_log.block_number), desc(summit_log.event_index)) - .limit(limit) - .offset(offset); - - // Get total count - const countResult = await db - .select({ count: sql`count(*)` }) - .from(summit_log) - .where(whereClause); - const total = Number(countResult[0]?.count ?? 0); + return respondWithCachedJson(c, CACHE_POLICIES.logs, async () => { + const results = await db + .select() + .from(summit_log) + .where(whereClause) + .orderBy(desc(summit_log.block_number), desc(summit_log.event_index)) + .limit(limit) + .offset(offset); + + const countResult = await db + .select({ count: sql`count(*)` }) + .from(summit_log) + .where(whereClause); + const total = Number(countResult[0]?.count ?? 0); - return c.json({ - data: results.map((r) => ({ - id: r.id, - block_number: r.block_number.toString(), - event_index: r.event_index, - category: r.category, - sub_category: r.sub_category, - data: r.data, - player: r.player, - token_id: r.token_id, - transaction_hash: r.transaction_hash, - created_at: r.created_at.toISOString(), - })), - pagination: { - limit, - offset, - total, - has_more: offset + results.length < total, - }, + return { + data: results.map((r) => ({ + id: r.id, + block_number: r.block_number.toString(), + event_index: r.event_index, + category: r.category, + sub_category: r.sub_category, + data: r.data, + player: r.player, + token_id: r.token_id, + transaction_hash: r.transaction_hash, + created_at: r.created_at.toISOString(), + })), + pagination: { + limit, + offset, + total, + has_more: offset + results.length < total, + }, + }; }); }); @@ -452,19 +501,20 @@ app.get("/logs", async (c) => { app.get("/beasts/stats/counts", async (c) => { const twentyFourHoursAgo = Math.floor(Date.now() / 1000) - 86400; - const result = await db - .select({ - total: sql`count(*)`, - alive: sql`count(*) filter (where ${beast_stats.last_death_timestamp} < ${twentyFourHoursAgo})`, - }) - .from(beast_stats); - - const { total, alive } = result[0] ?? { total: 0, alive: 0 }; + return respondWithCachedJson(c, CACHE_POLICIES.beastsStatsCounts, async () => { + const result = await db + .select({ + total: sql`count(*)`, + alive: sql`count(*) filter (where ${beast_stats.last_death_timestamp} < ${twentyFourHoursAgo})`, + }) + .from(beast_stats); - return c.json({ - total: Number(total), - alive: Number(alive), - dead: Number(total) - Number(alive), + const { total, alive } = result[0] ?? { total: 0, alive: 0 }; + return { + total: Number(total), + alive: Number(alive), + dead: Number(total) - Number(alive), + }; }); }); @@ -481,62 +531,62 @@ app.get("/beasts/stats/top", async (c) => { const limit = Math.min(parseInt(c.req.query("limit") || "25", 10), 100); const offset = parseInt(c.req.query("offset") || "0", 10); - // Get paginated results with beast metadata - const results = await db - .select({ - token_id: beast_stats.token_id, - summit_held_seconds: beast_stats.summit_held_seconds, - bonus_xp: beast_stats.bonus_xp, - last_death_timestamp: beast_stats.last_death_timestamp, - beast_id: beasts.beast_id, - prefix: beasts.prefix, - suffix: beasts.suffix, - owner: beast_owners.owner, - }) - .from(beast_stats) - .innerJoin(beasts, eq(beasts.token_id, beast_stats.token_id)) - .leftJoin(beast_owners, eq(beast_owners.token_id, beast_stats.token_id)) - .where(sql`${beast_stats.summit_held_seconds} > 0`) - .orderBy( - desc(beast_stats.summit_held_seconds), - desc(beast_stats.bonus_xp), - desc(beast_stats.last_death_timestamp) - ) - .limit(limit) - .offset(offset); - - // Get total count - const countResult = await db - .select({ count: sql`count(*)` }) - .from(beast_stats) - .where(sql`${beast_stats.summit_held_seconds} > 0`); - const total = Number(countResult[0]?.count ?? 0); + return respondWithCachedJson(c, CACHE_POLICIES.beastsStatsTop, async () => { + const results = await db + .select({ + token_id: beast_stats.token_id, + summit_held_seconds: beast_stats.summit_held_seconds, + bonus_xp: beast_stats.bonus_xp, + last_death_timestamp: beast_stats.last_death_timestamp, + beast_id: beasts.beast_id, + prefix: beasts.prefix, + suffix: beasts.suffix, + owner: beast_owners.owner, + }) + .from(beast_stats) + .innerJoin(beasts, eq(beasts.token_id, beast_stats.token_id)) + .leftJoin(beast_owners, eq(beast_owners.token_id, beast_stats.token_id)) + .where(sql`${beast_stats.summit_held_seconds} > 0`) + .orderBy( + desc(beast_stats.summit_held_seconds), + desc(beast_stats.bonus_xp), + desc(beast_stats.last_death_timestamp) + ) + .limit(limit) + .offset(offset); - return c.json({ - data: results.map((r) => { - const beastName = BEAST_NAMES[r.beast_id] ?? "Unknown"; - const prefix = ITEM_NAME_PREFIXES[r.prefix] ?? ""; - const suffix = ITEM_NAME_SUFFIXES[r.suffix] ?? ""; - const fullName = prefix && suffix ? `"${prefix} ${suffix}" ${beastName}` : beastName; + const countResult = await db + .select({ count: sql`count(*)` }) + .from(beast_stats) + .where(sql`${beast_stats.summit_held_seconds} > 0`); + const total = Number(countResult[0]?.count ?? 0); - return { - token_id: r.token_id, - summit_held_seconds: r.summit_held_seconds, - bonus_xp: r.bonus_xp, - last_death_timestamp: Number(r.last_death_timestamp), - owner: r.owner, - beast_name: beastName, - prefix, - suffix, - full_name: fullName, - }; - }), - pagination: { - limit, - offset, - total, - has_more: offset + results.length < total, - }, + return { + data: results.map((r) => { + const beastName = BEAST_NAMES[r.beast_id] ?? "Unknown"; + const prefix = ITEM_NAME_PREFIXES[r.prefix] ?? ""; + const suffix = ITEM_NAME_SUFFIXES[r.suffix] ?? ""; + const fullName = prefix && suffix ? `"${prefix} ${suffix}" ${beastName}` : beastName; + + return { + token_id: r.token_id, + summit_held_seconds: r.summit_held_seconds, + bonus_xp: r.bonus_xp, + last_death_timestamp: Number(r.last_death_timestamp), + owner: r.owner, + beast_name: beastName, + prefix, + suffix, + full_name: fullName, + }; + }), + pagination: { + limit, + offset, + total, + has_more: offset + results.length < total, + }, + }; }); }); @@ -557,35 +607,35 @@ app.get("/diplomacy", async (c) => { return c.json({ error: "prefix and suffix are required" }, 400); } - const results = await db - .select({ - token_id: beasts.token_id, - beast_id: beasts.beast_id, - prefix: beasts.prefix, - suffix: beasts.suffix, - level: beasts.level, - health: beasts.health, - owner: beast_owners.owner, - current_health: beast_stats.current_health, - bonus_health: beast_stats.bonus_health, - bonus_xp: beast_stats.bonus_xp, - summit_held_seconds: beast_stats.summit_held_seconds, - spirit: beast_stats.spirit, - luck: beast_stats.luck, - }) - .from(beasts) - .innerJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) - .leftJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) - .where( - and( - eq(beasts.prefix, prefix), - eq(beasts.suffix, suffix), - eq(beast_stats.diplomacy, true) - ) - ); - - return c.json( - results.map((r) => { + return respondWithCachedJson(c, CACHE_POLICIES.diplomacy, async () => { + const results = await db + .select({ + token_id: beasts.token_id, + beast_id: beasts.beast_id, + prefix: beasts.prefix, + suffix: beasts.suffix, + level: beasts.level, + health: beasts.health, + owner: beast_owners.owner, + current_health: beast_stats.current_health, + bonus_health: beast_stats.bonus_health, + bonus_xp: beast_stats.bonus_xp, + summit_held_seconds: beast_stats.summit_held_seconds, + spirit: beast_stats.spirit, + luck: beast_stats.luck, + }) + .from(beasts) + .innerJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) + .leftJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) + .where( + and( + eq(beasts.prefix, prefix), + eq(beasts.suffix, suffix), + eq(beast_stats.diplomacy, true) + ) + ); + + return results.map((r) => { const beastName = BEAST_NAMES[r.beast_id] ?? "Unknown"; const prefixName = ITEM_NAME_PREFIXES[r.prefix] ?? ""; const suffixName = ITEM_NAME_SUFFIXES[r.suffix] ?? ""; @@ -611,8 +661,8 @@ app.get("/diplomacy", async (c) => { luck: r.luck, owner: r.owner, }; - }) - ); + }); + }); }); /** @@ -620,52 +670,54 @@ app.get("/diplomacy", async (c) => { * Used for building diplomacy leaderboard (grouped by prefix/suffix with power calculation) */ app.get("/diplomacy/all", async (c) => { - const results = await db - .select({ - token_id: beasts.token_id, - beast_id: beasts.beast_id, - prefix: beasts.prefix, - suffix: beasts.suffix, - level: beasts.level, - bonus_xp: beast_stats.bonus_xp, - }) - .from(beasts) - .innerJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) - .where(eq(beast_stats.diplomacy, true)); - - return c.json(results); + return respondWithCachedJson(c, CACHE_POLICIES.diplomacyAll, async () => { + return db + .select({ + token_id: beasts.token_id, + beast_id: beasts.beast_id, + prefix: beasts.prefix, + suffix: beasts.suffix, + level: beasts.level, + bonus_xp: beast_stats.bonus_xp, + }) + .from(beasts) + .innerJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) + .where(eq(beast_stats.diplomacy, true)); + }); }); /** * GET /leaderboard - Get rewards leaderboard grouped by owner */ app.get("/leaderboard", async (c) => { - const results = await db - .select({ - owner: rewards_earned.owner, - amount: sql`sum(${rewards_earned.amount})`, - }) - .from(rewards_earned) - .groupBy(rewards_earned.owner) - .orderBy(sql`sum(${rewards_earned.amount}) desc`); - - return c.json( - results.map((r) => ({ + return respondWithCachedJson(c, CACHE_POLICIES.leaderboard, async () => { + const results = await db + .select({ + owner: rewards_earned.owner, + amount: sql`sum(${rewards_earned.amount})`, + }) + .from(rewards_earned) + .groupBy(rewards_earned.owner) + .orderBy(sql`sum(${rewards_earned.amount}) desc`); + + return results.map((r) => ({ owner: r.owner, amount: Number(r.amount) / 100000, - })) - ); + })); + }); }); /** * GET /quest-rewards/total - Get total quest rewards claimed */ app.get("/quest-rewards/total", async (c) => { - const result = await db - .select({ total: sql`coalesce(sum(${quest_rewards_claimed.amount}), 0)` }) - .from(quest_rewards_claimed); + return respondWithCachedJson(c, CACHE_POLICIES.questRewardsTotal, async () => { + const result = await db + .select({ total: sql`coalesce(sum(${quest_rewards_claimed.amount}), 0)` }) + .from(quest_rewards_claimed); - return c.json({ total: Number(result[0]?.total ?? 0) / 100 }); + return { total: Number(result[0]?.total ?? 0) / 100 }; + }); }); /** @@ -695,20 +747,23 @@ app.get("/adventurers/:player", async (c) => { * GET /consumables/supply - Get total circulating supply of consumable tokens */ app.get("/consumables/supply", async (c) => { - const result = await db - .select({ - xlife: sql`coalesce(sum(${consumables.xlife_count}), 0)`, - attack: sql`coalesce(sum(${consumables.attack_count}), 0)`, - revive: sql`coalesce(sum(${consumables.revive_count}), 0)`, - poison: sql`coalesce(sum(${consumables.poison_count}), 0)`, - }) - .from(consumables); - const row = result[0] ?? { xlife: 0, attack: 0, revive: 0, poison: 0 }; - return c.json({ - xlife: Number(row.xlife), - attack: Number(row.attack), - revive: Number(row.revive), - poison: Number(row.poison), + return respondWithCachedJson(c, CACHE_POLICIES.consumablesSupply, async () => { + const result = await db + .select({ + xlife: sql`coalesce(sum(${consumables.xlife_count}), 0)`, + attack: sql`coalesce(sum(${consumables.attack_count}), 0)`, + revive: sql`coalesce(sum(${consumables.revive_count}), 0)`, + poison: sql`coalesce(sum(${consumables.poison_count}), 0)`, + }) + .from(consumables); + const row = result[0] ?? { xlife: 0, attack: 0, revive: 0, poison: 0 }; + + return { + xlife: Number(row.xlife), + attack: Number(row.attack), + revive: Number(row.revive), + poison: Number(row.poison), + }; }); }); @@ -818,6 +873,7 @@ if (isMetricsEnabled()) { waiting: pool.waitingCount, }), dbProbe: collectDbProxyMetrics, + getExtraMetrics: () => apiCache.snapshot(), }) ); } diff --git a/api/src/lib/cache.test.ts b/api/src/lib/cache.test.ts new file mode 100644 index 00000000..0a34f94b --- /dev/null +++ b/api/src/lib/cache.test.ts @@ -0,0 +1,216 @@ +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Context } from "hono"; +import { + ApiResponseCache, + createCacheKey, + parseCacheEnabled, + parseMaxEntries, + shouldBypassCache, +} from "./cache.js"; + +function createContextStub(options: { + url: string; + method?: string; + headers?: Record; +}): Context { + const normalizedHeaders = new Map(); + for (const [key, value] of Object.entries(options.headers ?? {})) { + if (typeof value === "string") { + normalizedHeaders.set(key.toLowerCase(), value); + } + } + + return { + req: { + url: options.url, + method: options.method ?? "GET", + path: new URL(options.url).pathname, + header: (name: string) => normalizedHeaders.get(name.toLowerCase()), + }, + } as Context; +} + +const originalEnv = { ...process.env }; + +beforeEach(() => { + vi.useRealTimers(); + process.env = { ...originalEnv }; + delete process.env.API_CACHE_ENABLED; + delete process.env.NODE_ENV; +}); + +afterAll(() => { + vi.useRealTimers(); + process.env = originalEnv; +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("ApiResponseCache", () => { + it("serves MISS then HIT inside fresh TTL", async () => { + const cache = new ApiResponseCache({ enabled: true, maxEntries: 10 }); + const policy = { freshTtlMs: 1_000, staleTtlMs: 2_000 }; + const loader = vi.fn().mockResolvedValue({ value: 1 }); + + const first = await cache.getOrLoad("GET:/leaderboard", policy, loader); + const second = await cache.getOrLoad("GET:/leaderboard", policy, loader); + + expect(first.status).toBe("MISS"); + expect(second.status).toBe("HIT"); + expect(loader).toHaveBeenCalledTimes(1); + + expect(cache.snapshot()).toMatchObject({ + cache_entries: 1, + cache_misses: 1, + cache_hits: 1, + cache_refreshes: 1, + }); + }); + + it("serves STALE and refreshes in background with single-flight", async () => { + vi.useFakeTimers(); + + const cache = new ApiResponseCache({ enabled: true, maxEntries: 10 }); + const policy = { freshTtlMs: 1_000, staleTtlMs: 3_000 }; + let resolveRefresh: ((value: { version: number }) => void) | undefined; + const loader = vi.fn(); + loader + .mockResolvedValueOnce({ version: 1 }) + .mockImplementationOnce( + () => + new Promise<{ version: number }>((resolve) => { + resolveRefresh = resolve; + }) + ); + + await cache.getOrLoad("GET:/logs", policy, loader); + vi.advanceTimersByTime(1_200); + + const staleOne = await cache.getOrLoad("GET:/logs", policy, loader); + const staleTwo = await cache.getOrLoad("GET:/logs", policy, loader); + + expect(staleOne.status).toBe("STALE"); + expect(staleOne.value).toEqual({ version: 1 }); + expect(staleTwo.status).toBe("STALE"); + expect(staleTwo.value).toEqual({ version: 1 }); + expect(loader).toHaveBeenCalledTimes(2); + + resolveRefresh?.({ version: 2 }); + await Promise.resolve(); + + const freshAfterRefresh = await cache.getOrLoad("GET:/logs", policy, loader); + expect(freshAfterRefresh.status).toBe("HIT"); + expect(freshAfterRefresh.value).toEqual({ version: 2 }); + }); + + it("shares in-flight loader promise for concurrent misses", async () => { + const cache = new ApiResponseCache({ enabled: true, maxEntries: 10 }); + const policy = { freshTtlMs: 1_000, staleTtlMs: 2_000 }; + + let resolveLoader: ((value: { token: string }) => void) | undefined; + const loader = vi.fn( + () => + new Promise<{ token: string }>((resolve) => { + resolveLoader = resolve; + }) + ); + + const pendingA = cache.getOrLoad("GET:/beasts/stats/top?limit=25", policy, loader); + const pendingB = cache.getOrLoad("GET:/beasts/stats/top?limit=25", policy, loader); + + expect(loader).toHaveBeenCalledTimes(1); + + resolveLoader?.({ token: "shared" }); + + const [resultA, resultB] = await Promise.all([pendingA, pendingB]); + + expect(resultA.status).toBe("MISS"); + expect(resultB.status).toBe("MISS"); + expect(resultA.value).toEqual({ token: "shared" }); + expect(resultB.value).toEqual({ token: "shared" }); + }); + + it("evicts least-recently-used key when capacity is reached", async () => { + const cache = new ApiResponseCache({ enabled: true, maxEntries: 2 }); + const policy = { freshTtlMs: 5_000, staleTtlMs: 10_000 }; + + const loadFor = (key: string) => vi.fn().mockResolvedValue({ key }); + + await cache.getOrLoad("GET:/a", policy, loadFor("a")); + await cache.getOrLoad("GET:/b", policy, loadFor("b")); + await cache.getOrLoad("GET:/a", policy, loadFor("a-hit")); // Touch "a" + await cache.getOrLoad("GET:/c", policy, loadFor("c")); + + const loaderForBReload = vi.fn().mockResolvedValue({ key: "b-reloaded" }); + const bResult = await cache.getOrLoad("GET:/b", policy, loaderForBReload); + + expect(bResult.status).toBe("MISS"); + expect(loaderForBReload).toHaveBeenCalledTimes(1); + expect(cache.snapshot().cache_evictions).toBe(2); + }); +}); + +describe("cache helpers", () => { + it("parseCacheEnabled defaults to true in production", () => { + process.env.NODE_ENV = "production"; + + expect(parseCacheEnabled()).toBe(true); + }); + + it("parseCacheEnabled disables cache for explicit falsey env values", () => { + process.env.NODE_ENV = "production"; + process.env.API_CACHE_ENABLED = "false"; + + expect(parseCacheEnabled()).toBe(false); + + process.env.API_CACHE_ENABLED = "0"; + expect(parseCacheEnabled()).toBe(false); + }); + + it("parseMaxEntries falls back on invalid input", () => { + expect(parseMaxEntries(undefined)).toBe(500); + expect(parseMaxEntries("abc")).toBe(500); + expect(parseMaxEntries("0")).toBe(500); + expect(parseMaxEntries("50.7")).toBe(50); + }); + + it("createCacheKey normalizes query ordering", () => { + const context = createContextStub({ + method: "GET", + url: "https://api.example.com/logs?player=0xabc&limit=25&category=event", + }); + + const key = createCacheKey(context); + + expect(key).toBe("GET:/logs?category=event&limit=25&player=0xabc"); + }); + + it("shouldBypassCache respects no-cache directives", () => { + const requestNoCache = createContextStub({ + url: "https://api.example.com/leaderboard", + headers: { + "cache-control": "max-age=0, no-cache", + }, + }); + + const pragmaNoCache = createContextStub({ + url: "https://api.example.com/leaderboard", + headers: { + pragma: "no-cache", + }, + }); + + const normalRequest = createContextStub({ + url: "https://api.example.com/leaderboard", + headers: { + "cache-control": "max-age=60", + }, + }); + + expect(shouldBypassCache(requestNoCache)).toBe(true); + expect(shouldBypassCache(pragmaNoCache)).toBe(true); + expect(shouldBypassCache(normalRequest)).toBe(false); + }); +}); diff --git a/api/src/lib/cache.ts b/api/src/lib/cache.ts new file mode 100644 index 00000000..5939c437 --- /dev/null +++ b/api/src/lib/cache.ts @@ -0,0 +1,195 @@ +import type { Context } from "hono"; + +export type CacheStatus = "HIT" | "STALE" | "MISS" | "BYPASS"; + +export interface CachePolicy { + freshTtlMs: number; + staleTtlMs: number; +} + +export interface ApiCacheOptions { + enabled: boolean; + maxEntries: number; +} + +interface CacheEntry { + value: T; + freshUntil: number; + staleUntil: number; +} + +interface CacheSnapshot { + [key: string]: number; + cache_entries: number; + cache_hits: number; + cache_stale_hits: number; + cache_misses: number; + cache_bypasses: number; + cache_refreshes: number; + cache_refresh_errors: number; + cache_evictions: number; +} + +const DEFAULT_MAX_ENTRIES = 500; + +export class ApiResponseCache { + private readonly enabled: boolean; + private readonly maxEntries: number; + private readonly store = new Map>(); + private readonly inFlight = new Map>(); + private stats: CacheSnapshot = { + cache_entries: 0, + cache_hits: 0, + cache_stale_hits: 0, + cache_misses: 0, + cache_bypasses: 0, + cache_refreshes: 0, + cache_refresh_errors: 0, + cache_evictions: 0, + }; + + constructor(options: ApiCacheOptions) { + this.enabled = options.enabled; + this.maxEntries = Math.max(1, options.maxEntries || DEFAULT_MAX_ENTRIES); + } + + get enabledInRuntime(): boolean { + return this.enabled; + } + + snapshot(): CacheSnapshot { + return { + ...this.stats, + cache_entries: this.store.size, + }; + } + + noteBypass(): void { + this.stats.cache_bypasses += 1; + } + + async getOrLoad( + key: string, + policy: CachePolicy, + loader: () => Promise + ): Promise<{ status: Exclude; value: T }> { + const now = Date.now(); + const existing = this.store.get(key) as CacheEntry | undefined; + + if (existing) { + if (existing.freshUntil > now) { + this.stats.cache_hits += 1; + this.touch(key, existing); + return { status: "HIT", value: existing.value }; + } + + if (existing.staleUntil > now) { + this.stats.cache_stale_hits += 1; + this.touch(key, existing); + this.refreshInBackground(key, policy, loader); + return { status: "STALE", value: existing.value }; + } + } + + this.stats.cache_misses += 1; + const value = await this.loadSingleFlight(key, policy, loader); + return { status: "MISS", value }; + } + + private touch(key: string, entry: CacheEntry): void { + this.store.delete(key); + this.store.set(key, entry); + } + + private refreshInBackground( + key: string, + policy: CachePolicy, + loader: () => Promise + ): void { + if (this.inFlight.has(key)) return; + + void this.loadSingleFlight(key, policy, loader).catch(() => { + // Error is counted in loadSingleFlight; stale value remains. + }); + } + + private async loadSingleFlight( + key: string, + policy: CachePolicy, + loader: () => Promise + ): Promise { + const inFlight = this.inFlight.get(key) as Promise | undefined; + if (inFlight) return inFlight; + + const loadPromise = (async () => { + this.stats.cache_refreshes += 1; + try { + const value = await loader(); + this.set(key, value, policy); + return value; + } catch (error) { + this.stats.cache_refresh_errors += 1; + throw error; + } finally { + this.inFlight.delete(key); + } + })(); + + this.inFlight.set(key, loadPromise); + return loadPromise; + } + + private set(key: string, value: T, policy: CachePolicy): void { + const now = Date.now(); + const freshTtl = Math.max(1, policy.freshTtlMs); + const staleTtl = Math.max(freshTtl, policy.staleTtlMs); + + if (!this.store.has(key) && this.store.size >= this.maxEntries) { + const oldest = this.store.keys().next().value; + if (oldest) { + this.store.delete(oldest); + this.stats.cache_evictions += 1; + } + } + + this.store.set(key, { + value, + freshUntil: now + freshTtl, + staleUntil: now + staleTtl, + }); + } +} + +export function parseCacheEnabled(): boolean { + const raw = process.env.API_CACHE_ENABLED?.trim().toLowerCase(); + if (!raw) return process.env.NODE_ENV === "production"; + return !(raw === "0" || raw === "false" || raw === "off" || raw === "no"); +} + +export function parseMaxEntries(value: string | undefined): number { + if (!value) return DEFAULT_MAX_ENTRIES; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_MAX_ENTRIES; + return Math.floor(parsed); +} + +export function createCacheKey(c: Context): string { + const url = new URL(c.req.url); + const entries = [...url.searchParams.entries()].sort(([aKey, aValue], [bKey, bValue]) => { + if (aKey === bKey) return aValue.localeCompare(bValue); + return aKey.localeCompare(bKey); + }); + + const query = new URLSearchParams(entries).toString(); + const suffix = query ? `?${query}` : ""; + return `${c.req.method}:${c.req.path}${suffix}`; +} + +export function shouldBypassCache(c: Context): boolean { + const requestCacheControl = (c.req.header("cache-control") || "").toLowerCase(); + const pragma = (c.req.header("pragma") || "").toLowerCase(); + + return requestCacheControl.includes("no-cache") || + requestCacheControl.includes("no-store") || + pragma.includes("no-cache"); +} From b9443d5efb1f22c75c4d3353e25b426dee2f3875 Mon Sep 17 00:00:00 2001 From: loothero Date: Fri, 6 Mar 2026 04:54:54 -0800 Subject: [PATCH 09/39] perf(api,indexer): optimize hot queries and add index support --- api/AGENTS.md | 6 +- api/README.md | 13 +- api/src/db/schema.ts | 8 + api/src/index.ts | 399 ++++++++++++------- indexer/.gitignore | 1 + indexer/migrations/0004_api_perf_indexes.sql | 17 + indexer/migrations/meta/_journal.json | 9 +- indexer/src/lib/schema.ts | 8 + 8 files changed, 308 insertions(+), 153 deletions(-) create mode 100644 indexer/migrations/0004_api_perf_indexes.sql diff --git a/api/AGENTS.md b/api/AGENTS.md index b9a4b995..bb0ebd5f 100644 --- a/api/AGENTS.md +++ b/api/AGENTS.md @@ -43,9 +43,9 @@ Read [`../AGENTS.md`](../AGENTS.md) first for shared addresses/mechanics and ind - subscribe payload: `{"type":"subscribe","channels":["summit","event"]}` Query/pagination rules agents usually need: -- `/beasts/all`: `limit` default `25`, max `100`; `offset`; filters `prefix`, `suffix`, `beast_id`, `name`, `owner`; `sort` in `summit_held_seconds|level`. -- `/logs`: `limit` default `50`, max `100`; `offset`; `category`, `sub_category` (comma-separated), `player`. -- `/beasts/stats/top`: `limit` default `25`, max `100`; `offset`. +- `/beasts/all`: `limit` default `25`, max `100`; `offset`; filters `prefix`, `suffix`, `beast_id`, `name`, `owner`; `sort` in `summit_held_seconds|level`; `include_total` optional (`false` skips `count(*)`). +- `/logs`: `limit` default `50`, max `100`; `offset`; `category`, `sub_category` (comma-separated), `player`; `include_total` optional (`false` skips `count(*)`). +- `/beasts/stats/top`: `limit` default `25`, max `100`; `offset`; `include_total` optional (`false` skips `count(*)`). - `/diplomacy`: `prefix` and `suffix` required; returns HTTP `400` if missing. - Paginated routes return `{ data, pagination: { limit, offset, total, has_more } }`. diff --git a/api/README.md b/api/README.md index 1b8ccfff..2a502b96 100644 --- a/api/README.md +++ b/api/README.md @@ -90,18 +90,23 @@ curl http://localhost:3001/health ### Query Parameters and Response Shapes `GET /beasts/all` -- params: `limit` (default `25`, max `100`), `offset`, `prefix`, `suffix`, `beast_id`, `name`, `owner`, `sort` (`summit_held_seconds|level`) +- params: `limit` (default `25`, max `100`), `offset`, `prefix`, `suffix`, `beast_id`, `name`, `owner`, `sort` (`summit_held_seconds|level`), `include_total` (`true|false`, default `true`) - returns: `{ data: Beast[], pagination: { limit, offset, total, has_more } }` `GET /logs` -- params: `limit` (default `50`, max `100`), `offset`, `category`, `sub_category`, `player` +- params: `limit` (default `50`, max `100`), `offset`, `category`, `sub_category`, `player`, `include_total` (`true|false`, default `true`) - `category`/`sub_category` accept comma-separated values - returns: `{ data: LogEntry[], pagination: { limit, offset, total, has_more } }` `GET /beasts/stats/top` -- params: `limit` (default `25`, max `100`), `offset` +- params: `limit` (default `25`, max `100`), `offset`, `include_total` (`true|false`, default `true`) - returns: paginated top beasts sorted by summit hold time, bonus XP, death timestamp +`include_total=false` behavior: +- skips `count(*)` query for lower latency +- returns `pagination.total = null` +- computes `has_more` via `limit + 1` fetch strategy + `GET /diplomacy` - params: `prefix` (required), `suffix` (required) - returns HTTP `400` if either is missing @@ -158,6 +163,8 @@ Realtime pipeline: - Address inputs are normalized to lowercase 66-char `0x`-padded form. - API is public read-only (no auth layer). - A thin in-memory SWR cache is applied to high-traffic read endpoints: + - `/beasts/all` (common public list patterns) + - `/beasts/:owner` - `/logs` - `/beasts/stats/counts` - `/beasts/stats/top` diff --git a/api/src/db/schema.ts b/api/src/db/schema.ts index 60fe728b..aa91e5ca 100644 --- a/api/src/db/schema.ts +++ b/api/src/db/schema.ts @@ -40,6 +40,7 @@ export const beasts = pgTable( index("beasts_beast_id_idx").on(table.beast_id), index("beasts_prefix_idx").on(table.prefix), index("beasts_suffix_idx").on(table.suffix), + index("beasts_prefix_suffix_token_idx").on(table.prefix, table.suffix, table.token_id), index("beasts_level_idx").on(table.level.desc()), ] ); @@ -57,6 +58,7 @@ export const beast_owners = pgTable( }, (table) => [ index("beast_owners_owner_idx").on(table.owner), + index("beast_owners_owner_token_idx").on(table.owner, table.token_id), index("beast_owners_token_id_idx").on(table.token_id), ] ); @@ -118,6 +120,12 @@ export const beast_stats = pgTable( (table) => [ index("beast_stats_current_health_idx").on(table.current_health), index("beast_stats_summit_held_seconds_idx").on(table.summit_held_seconds.desc()), + index("beast_stats_top_order_idx").on( + table.summit_held_seconds.desc(), + table.bonus_xp.desc(), + table.last_death_timestamp.desc(), + table.token_id + ), index("beast_stats_updated_at_idx").on(table.updated_at.desc()), ] ); diff --git a/api/src/index.ts b/api/src/index.ts index 0009b876..cdeee7b4 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -51,6 +51,8 @@ const apiCache = new ApiResponseCache({ }); const CACHE_POLICIES: Record< + | "beastsAll" + | "beastsByOwner" | "logs" | "beastsStatsCounts" | "beastsStatsTop" @@ -61,6 +63,8 @@ const CACHE_POLICIES: Record< | "consumablesSupply", CachePolicy > = { + beastsAll: { freshTtlMs: 2_000, staleTtlMs: 8_000 }, + beastsByOwner: { freshTtlMs: 3_000, staleTtlMs: 12_000 }, logs: { freshTtlMs: 2_000, staleTtlMs: 8_000 }, beastsStatsCounts: { freshTtlMs: 5_000, staleTtlMs: 20_000 }, beastsStatsTop: { freshTtlMs: 3_000, staleTtlMs: 12_000 }, @@ -87,6 +91,17 @@ async function respondWithCachedJson( return c.json(value); } +function parseIncludeTotal(value: string | undefined): boolean { + if (!value) return true; + const normalized = value.trim().toLowerCase(); + return !( + normalized === "false" || + normalized === "0" || + normalized === "off" || + normalized === "no" + ); +} + async function collectDbProxyMetrics() { const [connections, databaseStats, databaseSize] = await Promise.all([ pool.query( @@ -175,6 +190,7 @@ app.get("/health", async (c) => { * - name: Filter by beast name search (optional, uses beast_id index) * - owner: Filter by owner address (optional, indexed) * - sort: Sort by "summit_held_seconds" or "level" (default: summit_held_seconds, both indexed) + * - include_total: Set to false to skip count(*) and return pagination.total=null */ app.get("/beasts/all", async (c) => { const limit = Math.min(parseInt(c.req.query("limit") || "25", 10), 100); @@ -183,15 +199,17 @@ app.get("/beasts/all", async (c) => { const suffix = c.req.query("suffix"); const beastId = c.req.query("beast_id"); const name = c.req.query("name"); - const owner = c.req.query("owner"); + const ownerRaw = c.req.query("owner"); const sort = c.req.query("sort") || "summit_held_seconds"; + const includeTotal = parseIncludeTotal(c.req.query("include_total")); + const owner = ownerRaw ? normalizeAddress(ownerRaw) : undefined; // Build where conditions (all filters use indexed columns) const conditions = []; if (prefix) conditions.push(eq(beasts.prefix, parseInt(prefix, 10))); if (suffix) conditions.push(eq(beasts.suffix, parseInt(suffix, 10))); if (beastId) conditions.push(eq(beasts.beast_id, parseInt(beastId, 10))); - if (owner) conditions.push(eq(beast_owners.owner, normalizeAddress(owner))); + if (owner) conditions.push(eq(beast_owners.owner, owner)); if (name) { // Find beast IDs that match the name search (uses beast_id index) const lowerName = name.toLowerCase(); @@ -204,85 +222,158 @@ app.get("/beasts/all", async (c) => { // No matches, return empty result return c.json({ data: [], - pagination: { limit, offset, total: 0, has_more: false }, + pagination: { limit, offset, total: includeTotal ? 0 : null, has_more: false }, }); } } const whereClause = conditions.length > 0 ? and(...conditions) : undefined; - // Both sort options use indexed columns - const orderByClause = sort === "level" - ? desc(beasts.level) - : desc(beast_stats.summit_held_seconds); + const loadBeastsAll = async (): Promise<{ + data: Array>; + pagination: { limit: number; offset: number; total: number | null; has_more: boolean }; + }> => { + const tokenRowsLimit = includeTotal ? limit : limit + 1; + const tokenRows = sort === "level" + ? await ( + owner + ? db + .select({ token_id: beasts.token_id }) + .from(beasts) + .innerJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) + : db.select({ token_id: beasts.token_id }).from(beasts) + ) + .where(whereClause) + .orderBy(desc(beasts.level)) + .limit(tokenRowsLimit) + .offset(offset) + : await ( + owner + ? db + .select({ token_id: beasts.token_id }) + .from(beasts) + .leftJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) + .innerJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) + : db + .select({ token_id: beasts.token_id }) + .from(beasts) + .leftJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) + ) + .where(whereClause) + .orderBy(desc(beast_stats.summit_held_seconds)) + .limit(tokenRowsLimit) + .offset(offset); - // Get paginated results - const results = await db - .select({ - token_id: beasts.token_id, - beast_id: beasts.beast_id, - prefix: beasts.prefix, - suffix: beasts.suffix, - level: beasts.level, - health: beasts.health, - shiny: beasts.shiny, - animated: beasts.animated, - bonus_health: beast_stats.bonus_health, - bonus_xp: beast_stats.bonus_xp, - summit_held_seconds: beast_stats.summit_held_seconds, - spirit: beast_stats.spirit, - luck: beast_stats.luck, - specials: beast_stats.specials, - wisdom: beast_stats.wisdom, - diplomacy: beast_stats.diplomacy, - extra_lives: beast_stats.extra_lives, - owner: beast_owners.owner, - }) - .from(beasts) - .leftJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) - .leftJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) - .where(whereClause) - .orderBy(orderByClause) - .limit(limit) - .offset(offset); - - // Get total count - const countResult = await db - .select({ count: sql`count(*)` }) - .from(beasts) - .leftJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) - .leftJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) - .where(whereClause); - const total = Number(countResult[0]?.count ?? 0); + const hasMoreWithoutTotal = !includeTotal && tokenRows.length > limit; + const pageTokenIds = tokenRows.slice(0, limit).map((row) => row.token_id); - return c.json({ - data: results.map((r) => ({ - token_id: r.token_id, - beast_id: r.beast_id, - prefix: r.prefix, - suffix: r.suffix, - level: r.level, - health: r.health, - bonus_health: r.bonus_health ?? 0, - bonus_xp: r.bonus_xp ?? 0, - summit_held_seconds: r.summit_held_seconds ?? 0, - spirit: r.spirit ?? 0, - luck: r.luck ?? 0, - specials: r.specials ?? false, - wisdom: r.wisdom ?? false, - diplomacy: r.diplomacy ?? false, - extra_lives: r.extra_lives ?? 0, - owner: r.owner, - shiny: r.shiny, - animated: r.animated, - })), - pagination: { - limit, - offset, - total, - has_more: offset + results.length < total, - }, - }); + if (pageTokenIds.length === 0) { + return { + data: [], + pagination: { + limit, + offset, + total: includeTotal ? 0 : null, + has_more: false, + }, + }; + } + + const detailRows = await db + .select({ + token_id: beasts.token_id, + beast_id: beasts.beast_id, + prefix: beasts.prefix, + suffix: beasts.suffix, + level: beasts.level, + health: beasts.health, + shiny: beasts.shiny, + animated: beasts.animated, + bonus_health: beast_stats.bonus_health, + bonus_xp: beast_stats.bonus_xp, + summit_held_seconds: beast_stats.summit_held_seconds, + spirit: beast_stats.spirit, + luck: beast_stats.luck, + specials: beast_stats.specials, + wisdom: beast_stats.wisdom, + diplomacy: beast_stats.diplomacy, + extra_lives: beast_stats.extra_lives, + owner: beast_owners.owner, + }) + .from(beasts) + .leftJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) + .leftJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) + .where(inArray(beasts.token_id, pageTokenIds)); + + const byTokenId = new Map(detailRows.map((row) => [row.token_id, row])); + const orderedRows = pageTokenIds + .map((tokenId) => byTokenId.get(tokenId)) + .filter((row): row is NonNullable => Boolean(row)); + + let total: number | null = null; + if (includeTotal) { + const countResult = owner + ? await db + .select({ count: sql`count(*)` }) + .from(beasts) + .innerJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) + .where(whereClause) + : await db + .select({ count: sql`count(*)` }) + .from(beasts) + .where(whereClause); + total = Number(countResult[0]?.count ?? 0); + } + + return { + data: orderedRows.map((r) => ({ + token_id: r.token_id, + beast_id: r.beast_id, + prefix: r.prefix, + suffix: r.suffix, + level: r.level, + health: r.health, + bonus_health: r.bonus_health ?? 0, + bonus_xp: r.bonus_xp ?? 0, + summit_held_seconds: r.summit_held_seconds ?? 0, + spirit: r.spirit ?? 0, + luck: r.luck ?? 0, + specials: r.specials ?? false, + wisdom: r.wisdom ?? false, + diplomacy: r.diplomacy ?? false, + extra_lives: r.extra_lives ?? 0, + owner: r.owner, + shiny: r.shiny, + animated: r.animated, + })), + pagination: { + limit, + offset, + total, + has_more: includeTotal + ? offset + orderedRows.length < (total ?? 0) + : hasMoreWithoutTotal, + }, + }; + }; + + const shouldCacheBeastsAll = + !owner && + !name && + !prefix && + !suffix && + !beastId && + sort === "summit_held_seconds" && + offset <= 200 && + limit <= 50; + + if (shouldCacheBeastsAll) { + return respondWithCachedJson(c, CACHE_POLICIES.beastsAll, loadBeastsAll); + } + + apiCache.noteBypass(); + c.header("X-Cache", "BYPASS"); + return c.json(await loadBeastsAll()); }); /** @@ -292,59 +383,59 @@ app.get("/beasts/all", async (c) => { app.get("/beasts/:owner", async (c) => { const owner = normalizeAddress(c.req.param("owner")); - // Get beast data with all joins including skulls - const results = await db - .select({ - // Beast NFT metadata - token_id: beasts.token_id, - beast_id: beasts.beast_id, - prefix: beasts.prefix, - suffix: beasts.suffix, - level: beasts.level, - health: beasts.health, - shiny: beasts.shiny, - animated: beasts.animated, - // Beast data (Loot Survivor stats) - adventurers_killed: beast_data.adventurers_killed, - last_death_loot_survivor: beast_data.last_death_timestamp, - last_killed_by: beast_data.last_killed_by, - entity_hash: beast_data.entity_hash, - // Beast stats (Summit game state) - current_health: beast_stats.current_health, - bonus_health: beast_stats.bonus_health, - bonus_xp: beast_stats.bonus_xp, - attack_streak: beast_stats.attack_streak, - last_death_summit: beast_stats.last_death_timestamp, - revival_count: beast_stats.revival_count, - extra_lives: beast_stats.extra_lives, - captured_summit: beast_stats.captured_summit, - used_revival_potion: beast_stats.used_revival_potion, - used_attack_potion: beast_stats.used_attack_potion, - max_attack_streak: beast_stats.max_attack_streak, - summit_held_seconds: beast_stats.summit_held_seconds, - spirit: beast_stats.spirit, - luck: beast_stats.luck, - specials: beast_stats.specials, - wisdom: beast_stats.wisdom, - diplomacy: beast_stats.diplomacy, - rewards_earned: beast_stats.rewards_earned, - rewards_claimed: beast_stats.rewards_claimed, - // Skulls claimed (one row per beast) - skulls: skulls_claimed.skulls, - // Quest rewards claimed - quest_rewards_amount: quest_rewards_claimed.amount, - }) - .from(beast_owners) - .innerJoin(beasts, eq(beasts.token_id, beast_owners.token_id)) - .leftJoin(beast_data, eq(beast_data.token_id, beast_owners.token_id)) - .leftJoin(beast_stats, eq(beast_stats.token_id, beast_owners.token_id)) - .leftJoin(skulls_claimed, eq(skulls_claimed.beast_token_id, beast_owners.token_id)) - .leftJoin(quest_rewards_claimed, eq(quest_rewards_claimed.beast_token_id, beast_owners.token_id)) - .where(eq(beast_owners.owner, owner)); - - // Transform to Beast interface format - return c.json( - results.map((r) => { + return respondWithCachedJson(c, CACHE_POLICIES.beastsByOwner, async () => { + // Get beast data with all joins including skulls + const results = await db + .select({ + // Beast NFT metadata + token_id: beasts.token_id, + beast_id: beasts.beast_id, + prefix: beasts.prefix, + suffix: beasts.suffix, + level: beasts.level, + health: beasts.health, + shiny: beasts.shiny, + animated: beasts.animated, + // Beast data (Loot Survivor stats) + adventurers_killed: beast_data.adventurers_killed, + last_death_loot_survivor: beast_data.last_death_timestamp, + last_killed_by: beast_data.last_killed_by, + entity_hash: beast_data.entity_hash, + // Beast stats (Summit game state) + current_health: beast_stats.current_health, + bonus_health: beast_stats.bonus_health, + bonus_xp: beast_stats.bonus_xp, + attack_streak: beast_stats.attack_streak, + last_death_summit: beast_stats.last_death_timestamp, + revival_count: beast_stats.revival_count, + extra_lives: beast_stats.extra_lives, + captured_summit: beast_stats.captured_summit, + used_revival_potion: beast_stats.used_revival_potion, + used_attack_potion: beast_stats.used_attack_potion, + max_attack_streak: beast_stats.max_attack_streak, + summit_held_seconds: beast_stats.summit_held_seconds, + spirit: beast_stats.spirit, + luck: beast_stats.luck, + specials: beast_stats.specials, + wisdom: beast_stats.wisdom, + diplomacy: beast_stats.diplomacy, + rewards_earned: beast_stats.rewards_earned, + rewards_claimed: beast_stats.rewards_claimed, + // Skulls claimed (one row per beast) + skulls: skulls_claimed.skulls, + // Quest rewards claimed + quest_rewards_amount: quest_rewards_claimed.amount, + }) + .from(beast_owners) + .innerJoin(beasts, eq(beasts.token_id, beast_owners.token_id)) + .leftJoin(beast_data, eq(beast_data.token_id, beast_owners.token_id)) + .leftJoin(beast_stats, eq(beast_stats.token_id, beast_owners.token_id)) + .leftJoin(skulls_claimed, eq(skulls_claimed.beast_token_id, beast_owners.token_id)) + .leftJoin(quest_rewards_claimed, eq(quest_rewards_claimed.beast_token_id, beast_owners.token_id)) + .where(eq(beast_owners.owner, owner)); + + // Transform to Beast interface format + return results.map((r) => { const beastId = r.beast_id; const prefixId = r.prefix; const suffixId = r.suffix; @@ -422,8 +513,8 @@ app.get("/beasts/:owner", async (c) => { // Hash from beast_data (if linked) entity_hash: r.entity_hash ?? undefined, }; - }) - ); + }); + }); }); /** @@ -435,6 +526,7 @@ app.get("/beasts/:owner", async (c) => { * - category: Filter by category (optional, comma-separated for multiple) * - sub_category: Filter by sub_category (optional, comma-separated for multiple) * - player: Filter by player address (optional) + * - include_total: Set to false to skip count(*) and return pagination.total=null */ app.get("/logs", async (c) => { const limit = Math.min(parseInt(c.req.query("limit") || "50", 10), 100); @@ -442,10 +534,11 @@ app.get("/logs", async (c) => { const categoryParam = c.req.query("category"); const subCategoryParam = c.req.query("sub_category"); const player = c.req.query("player"); + const includeTotal = parseIncludeTotal(c.req.query("include_total")); // Parse comma-separated values into arrays - const categories = categoryParam ? categoryParam.split(',').filter(Boolean) : []; - const subCategories = subCategoryParam ? subCategoryParam.split(',').filter(Boolean) : []; + const categories = categoryParam ? categoryParam.split(",").filter(Boolean) : []; + const subCategories = subCategoryParam ? subCategoryParam.split(",").filter(Boolean) : []; // Build where conditions const conditions = []; @@ -456,22 +549,28 @@ app.get("/logs", async (c) => { const whereClause = conditions.length > 0 ? and(...conditions) : undefined; return respondWithCachedJson(c, CACHE_POLICIES.logs, async () => { + const rowsLimit = includeTotal ? limit : limit + 1; const results = await db .select() .from(summit_log) .where(whereClause) .orderBy(desc(summit_log.block_number), desc(summit_log.event_index)) - .limit(limit) + .limit(rowsLimit) .offset(offset); - const countResult = await db - .select({ count: sql`count(*)` }) - .from(summit_log) - .where(whereClause); - const total = Number(countResult[0]?.count ?? 0); + const pageRows = results.slice(0, limit); + const hasMoreWithoutTotal = !includeTotal && results.length > limit; + let total: number | null = null; + if (includeTotal) { + const countResult = await db + .select({ count: sql`count(*)` }) + .from(summit_log) + .where(whereClause); + total = Number(countResult[0]?.count ?? 0); + } return { - data: results.map((r) => ({ + data: pageRows.map((r) => ({ id: r.id, block_number: r.block_number.toString(), event_index: r.event_index, @@ -487,7 +586,7 @@ app.get("/logs", async (c) => { limit, offset, total, - has_more: offset + results.length < total, + has_more: includeTotal ? offset + pageRows.length < (total ?? 0) : hasMoreWithoutTotal, }, }; }); @@ -524,14 +623,17 @@ app.get("/beasts/stats/counts", async (c) => { * Query params: * - limit: Number of results (default: 25, max: 100) * - offset: Pagination offset (default: 0) + * - include_total: Set to false to skip count(*) and return pagination.total=null * * Returns beasts with full metadata and pagination info including total count */ app.get("/beasts/stats/top", async (c) => { const limit = Math.min(parseInt(c.req.query("limit") || "25", 10), 100); const offset = parseInt(c.req.query("offset") || "0", 10); + const includeTotal = parseIncludeTotal(c.req.query("include_total")); return respondWithCachedJson(c, CACHE_POLICIES.beastsStatsTop, async () => { + const rowsLimit = includeTotal ? limit : limit + 1; const results = await db .select({ token_id: beast_stats.token_id, @@ -552,17 +654,22 @@ app.get("/beasts/stats/top", async (c) => { desc(beast_stats.bonus_xp), desc(beast_stats.last_death_timestamp) ) - .limit(limit) + .limit(rowsLimit) .offset(offset); - const countResult = await db - .select({ count: sql`count(*)` }) - .from(beast_stats) - .where(sql`${beast_stats.summit_held_seconds} > 0`); - const total = Number(countResult[0]?.count ?? 0); + const pageRows = results.slice(0, limit); + const hasMoreWithoutTotal = !includeTotal && results.length > limit; + let total: number | null = null; + if (includeTotal) { + const countResult = await db + .select({ count: sql`count(*)` }) + .from(beast_stats) + .where(sql`${beast_stats.summit_held_seconds} > 0`); + total = Number(countResult[0]?.count ?? 0); + } return { - data: results.map((r) => { + data: pageRows.map((r) => { const beastName = BEAST_NAMES[r.beast_id] ?? "Unknown"; const prefix = ITEM_NAME_PREFIXES[r.prefix] ?? ""; const suffix = ITEM_NAME_SUFFIXES[r.suffix] ?? ""; @@ -584,7 +691,7 @@ app.get("/beasts/stats/top", async (c) => { limit, offset, total, - has_more: offset + results.length < total, + has_more: includeTotal ? offset + pageRows.length < (total ?? 0) : hasMoreWithoutTotal, }, }; }); diff --git a/indexer/.gitignore b/indexer/.gitignore index cd65474d..6dbf5684 100644 --- a/indexer/.gitignore +++ b/indexer/.gitignore @@ -4,6 +4,7 @@ node_modules/ # Build outputs dist/ .apibara/ +.cache/ # Environment files .env diff --git a/indexer/migrations/0004_api_perf_indexes.sql b/indexer/migrations/0004_api_perf_indexes.sql new file mode 100644 index 00000000..a2721c2d --- /dev/null +++ b/indexer/migrations/0004_api_perf_indexes.sql @@ -0,0 +1,17 @@ +CREATE INDEX IF NOT EXISTS "beast_stats_top_order_idx" ON "beast_stats" USING btree ( + "summit_held_seconds" DESC NULLS LAST, + "bonus_xp" DESC NULLS LAST, + "last_death_timestamp" DESC NULLS LAST, + "token_id" +);--> statement-breakpoint + +CREATE INDEX IF NOT EXISTS "beast_owners_owner_token_idx" ON "beast_owners" USING btree ( + "owner", + "token_id" +);--> statement-breakpoint + +CREATE INDEX IF NOT EXISTS "beasts_prefix_suffix_token_idx" ON "beasts" USING btree ( + "prefix", + "suffix", + "token_id" +); diff --git a/indexer/migrations/meta/_journal.json b/indexer/migrations/meta/_journal.json index 50638ad7..01007261 100644 --- a/indexer/migrations/meta/_journal.json +++ b/indexer/migrations/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1770990939403, "tag": "0003_consumables", "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1771012800000, + "tag": "0004_api_perf_indexes", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/indexer/src/lib/schema.ts b/indexer/src/lib/schema.ts index cf105986..33a7a528 100644 --- a/indexer/src/lib/schema.ts +++ b/indexer/src/lib/schema.ts @@ -78,6 +78,12 @@ export const beast_stats = pgTable( (table) => [ index("beast_stats_current_health_idx").on(table.current_health), index("beast_stats_summit_held_seconds_idx").on(table.summit_held_seconds.desc()), + index("beast_stats_top_order_idx").on( + table.summit_held_seconds.desc(), + table.bonus_xp.desc(), + table.last_death_timestamp.desc(), + table.token_id + ), index("beast_stats_updated_at_idx").on(table.updated_at.desc()), // Partial index for beasts with diplomacy upgrade index("beast_stats_diplomacy_token_idx").on(table.token_id).where(sql`diplomacy`), @@ -294,6 +300,7 @@ export const beast_owners = pgTable( }, (table) => [ index("beast_owners_owner_idx").on(table.owner), + index("beast_owners_owner_token_idx").on(table.owner, table.token_id), index("beast_owners_token_id_idx").on(table.token_id), ] ); @@ -327,6 +334,7 @@ export const beasts = pgTable( index("beasts_beast_id_idx").on(table.beast_id), index("beasts_prefix_idx").on(table.prefix), index("beasts_suffix_idx").on(table.suffix), + index("beasts_prefix_suffix_token_idx").on(table.prefix, table.suffix, table.token_id), index("beasts_level_idx").on(table.level.desc()), ] ); From fb11e7464ef01dca73fd0322c27fc6cbc301fab5 Mon Sep 17 00:00:00 2001 From: loothero Date: Fri, 6 Mar 2026 05:54:05 -0800 Subject: [PATCH 10/39] chore(api): add stress test runner script --- api/package.json | 3 +- api/scripts/stress-test.ts | 464 +++++++++++++++++++++++++++++++++++++ 2 files changed, 466 insertions(+), 1 deletion(-) create mode 100644 api/scripts/stress-test.ts diff --git a/api/package.json b/api/package.json index e8c5faf5..2b38a2a2 100644 --- a/api/package.json +++ b/api/package.json @@ -9,7 +9,8 @@ "lint:ci": "eslint . --max-warnings=0 --report-unused-inline-configs error", "start": "node dist/index.js", "test": "vitest run", - "test:coverage": "vitest run --coverage" + "test:coverage": "vitest run --coverage", + "stress:test": "tsx scripts/stress-test.ts" }, "description": "Summit API server with REST endpoints and WebSocket subscriptions", "dependencies": { diff --git a/api/scripts/stress-test.ts b/api/scripts/stress-test.ts new file mode 100644 index 00000000..bae0e8e6 --- /dev/null +++ b/api/scripts/stress-test.ts @@ -0,0 +1,464 @@ +import WebSocket from "ws"; + +type CounterMap = Record; + +interface Config { + baseUrl: string; + wsUrl: string; + durationSec: number; + httpConcurrency: number; + wsConnections: number; + requestTimeoutMs: number; + httpPauseMs: number; + reportEverySec: number; + wsPingIntervalMs: number; + weightedEndpoints: WeightedEndpoint[]; +} + +interface HttpMetrics { + total: number; + success: number; + failed: number; + timedOut: number; + latencyTotalMs: number; + latencyMinMs: number; + latencyMaxMs: number; + statuses: CounterMap; +} + +interface WsMetrics { + opened: number; + closed: number; + active: number; + errors: number; + messages: number; + reconnects: number; +} + +interface WeightedEndpoint { + endpoint: string; + weight: number; +} + +const DEFAULT_ENDPOINTS = [ + "/health", + "/beasts/stats/counts", + "/beasts/stats/top?limit=25&offset=0", + "/leaderboard", + "/logs?limit=50&offset=0", + "/quest-rewards/total", +]; + +const LOCAL_HOST_RE = /^(localhost|127\.0\.0\.1|\[::1\])(?::\d+)?$/i; + +function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (const arg of argv) { + if (!arg.startsWith("--")) continue; + const raw = arg.slice(2); + if (!raw) continue; + const eq = raw.indexOf("="); + if (eq === -1) { + out[raw] = "true"; + continue; + } + out[raw.slice(0, eq)] = raw.slice(eq + 1); + } + return out; +} + +function parseNumber(value: string | undefined, fallback: number): number { + if (!value) return fallback; + const n = Number(value); + if (!Number.isFinite(n)) return fallback; + return n; +} + +function splitList(raw: string): string[] { + const delimiter = raw.includes(";") ? ";" : ","; + return raw.split(delimiter).map((x) => x.trim()).filter(Boolean); +} + +function hasScheme(url: string): boolean { + return /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(url); +} + +function normalizeBaseUrl(raw: string): string { + const trimmed = raw.trim().replace(/\/+$/, ""); + if (!trimmed) return "http://localhost:3001"; + if (hasScheme(trimmed)) return trimmed; + + if (LOCAL_HOST_RE.test(trimmed)) { + return `http://${trimmed}`; + } + return `https://${trimmed}`; +} + +function normalizeWsUrl(raw: string): string { + const trimmed = raw.trim().replace(/\/+$/, ""); + if (!trimmed) return "ws://localhost:3001/ws"; + if (hasScheme(trimmed)) return trimmed; + + return `${LOCAL_HOST_RE.test(trimmed) ? "ws" : "wss"}://${trimmed}`; +} + +function parseWeightedEndpoints(raw: string): WeightedEndpoint[] { + const items = splitList(raw); + const parsed: WeightedEndpoint[] = []; + + for (const item of items) { + const sepIdx = item.lastIndexOf("::"); + if (sepIdx <= 0) { + throw new Error( + `Invalid weighted endpoint entry "${item}". Use "::" and separate entries with ";" or ",".` + ); + } + + const endpoint = item.slice(0, sepIdx).trim(); + const weightStr = item.slice(sepIdx + 2).trim(); + const weight = Math.floor(Number(weightStr)); + + if (!endpoint) { + throw new Error(`Invalid weighted endpoint entry "${item}": endpoint is empty.`); + } + if (!Number.isFinite(weight) || weight <= 0) { + throw new Error(`Invalid weighted endpoint entry "${item}": weight must be a positive integer.`); + } + + parsed.push({ endpoint, weight }); + } + + if (parsed.length === 0) { + throw new Error("No valid weighted endpoints provided."); + } + + return parsed; +} + +function parseConfig(): Config { + const args = parseArgs(process.argv.slice(2)); + const baseUrlRaw = args["base-url"] || process.env.STRESS_BASE_URL || "http://localhost:3001"; + const baseUrl = normalizeBaseUrl(baseUrlRaw); + + const wsUrlRaw = args["ws-url"] || process.env.STRESS_WS_URL || toWsUrl(baseUrl); + const wsUrl = normalizeWsUrl(wsUrlRaw); + + const endpointWeightsRaw = args["endpoint-weights"] || process.env.STRESS_ENDPOINT_WEIGHTS; + const endpointsRaw = args.endpoints || process.env.STRESS_ENDPOINTS || DEFAULT_ENDPOINTS.join(","); + const endpoints = splitList(endpointsRaw); + if (endpoints.length === 0 && !endpointWeightsRaw) { + throw new Error("No endpoints configured. Use --endpoints or --endpoint-weights."); + } + + const weightedEndpoints = endpointWeightsRaw + ? parseWeightedEndpoints(endpointWeightsRaw) + : endpoints.map((endpoint) => ({ endpoint, weight: 1 })); + + return { + baseUrl, + wsUrl, + durationSec: Math.max(5, parseNumber(args["duration-sec"] || process.env.STRESS_DURATION_SEC, 60)), + httpConcurrency: Math.max( + 1, + Math.floor(parseNumber(args["http-concurrency"] || process.env.STRESS_HTTP_CONCURRENCY, 10)) + ), + wsConnections: Math.max( + 0, + Math.floor(parseNumber(args["ws-connections"] || process.env.STRESS_WS_CONNECTIONS, 25)) + ), + requestTimeoutMs: Math.max( + 250, + Math.floor(parseNumber(args["request-timeout-ms"] || process.env.STRESS_REQUEST_TIMEOUT_MS, 4000)) + ), + httpPauseMs: Math.max( + 0, + Math.floor(parseNumber(args["http-pause-ms"] || process.env.STRESS_HTTP_PAUSE_MS, 0)) + ), + reportEverySec: Math.max( + 1, + Math.floor(parseNumber(args["report-every-sec"] || process.env.STRESS_REPORT_EVERY_SEC, 5)) + ), + wsPingIntervalMs: Math.max( + 1000, + Math.floor(parseNumber(args["ws-ping-interval-ms"] || process.env.STRESS_WS_PING_INTERVAL_MS, 10000)) + ), + weightedEndpoints, + }; +} + +function toWsUrl(baseUrl: string): string { + if (baseUrl.startsWith("https://")) return `${baseUrl.replace(/^https:\/\//, "wss://")}/ws`; + if (baseUrl.startsWith("http://")) return `${baseUrl.replace(/^http:\/\//, "ws://")}/ws`; + return `${baseUrl}/ws`; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function pickWeightedEndpoint(items: WeightedEndpoint[]): string { + const totalWeight = items.reduce((acc, item) => acc + item.weight, 0); + let cursor = Math.random() * totalWeight; + + for (const item of items) { + cursor -= item.weight; + if (cursor < 0) { + return item.endpoint; + } + } + + return items[items.length - 1].endpoint; +} + +function toHttpUrl(baseUrl: string, endpoint: string): string { + if (hasScheme(endpoint)) return endpoint; + const normalizedPath = endpoint.startsWith("/") ? endpoint : `/${endpoint}`; + return new URL(normalizedPath, `${baseUrl}/`).toString(); +} + +function recordHttpResult( + metrics: HttpMetrics, + elapsedMs: number, + statusKey: string, + ok: boolean, + timedOut = false +): void { + metrics.total += 1; + metrics.latencyTotalMs += elapsedMs; + metrics.latencyMinMs = Math.min(metrics.latencyMinMs, elapsedMs); + metrics.latencyMaxMs = Math.max(metrics.latencyMaxMs, elapsedMs); + metrics.statuses[statusKey] = (metrics.statuses[statusKey] || 0) + 1; + + if (ok) { + metrics.success += 1; + return; + } + + metrics.failed += 1; + if (timedOut) { + metrics.timedOut += 1; + } +} + +function printUsage(): void { + console.log("Usage:"); + console.log(" npm run stress:test -- --base-url=https://your-api --duration-sec=60"); + console.log(""); + console.log("Options:"); + console.log(" --base-url=http://localhost:3001"); + console.log(" --ws-url=ws://localhost:3001/ws"); + console.log(" --duration-sec=60"); + console.log(" --http-concurrency=10"); + console.log(" --ws-connections=25"); + console.log(" --request-timeout-ms=4000"); + console.log(" --http-pause-ms=0"); + console.log(" --report-every-sec=5"); + console.log(" --ws-ping-interval-ms=10000"); + console.log(" --endpoints=/health,/leaderboard,/logs?limit=20&offset=0"); + console.log(" --endpoint-weights=/leaderboard::6;/diplomacy?prefix=64&suffix=18::4;/beasts/stats/counts::2"); +} + +async function run(): Promise { + if (process.argv.includes("--help") || process.argv.includes("-h")) { + printUsage(); + process.exit(0); + } + + const config = parseConfig(); + const startedAt = Date.now(); + const endsAt = startedAt + config.durationSec * 1000; + const isRunning = (): boolean => !stopping && Date.now() < endsAt; + + const http: HttpMetrics = { + total: 0, + success: 0, + failed: 0, + timedOut: 0, + latencyTotalMs: 0, + latencyMinMs: Number.POSITIVE_INFINITY, + latencyMaxMs: 0, + statuses: {}, + }; + + const ws: WsMetrics = { + opened: 0, + closed: 0, + active: 0, + errors: 0, + messages: 0, + reconnects: 0, + }; + + let stopping = false; + const stop = () => { + stopping = true; + }; + + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + + const socketState = new Map< + number, + { + socket: WebSocket; + pingTimer?: ReturnType; + reconnectTimer?: ReturnType; + } + >(); + + const reporter = setInterval(() => { + const elapsedSec = (Date.now() - startedAt) / 1000; + const reqRate = elapsedSec > 0 ? http.total / elapsedSec : 0; + const avgLatency = http.total > 0 ? http.latencyTotalMs / http.total : 0; + + console.log( + `[report ${elapsedSec.toFixed(1)}s] http total=${http.total} ok=${http.success} fail=${http.failed}` + + ` timeout=${http.timedOut} rps=${reqRate.toFixed(1)} lat(avg/min/max)=${avgLatency.toFixed(1)}` + + `/${Number.isFinite(http.latencyMinMs) ? http.latencyMinMs.toFixed(1) : "0.0"}/${http.latencyMaxMs.toFixed(1)}ms` + + ` ws active=${ws.active} opened=${ws.opened} closed=${ws.closed} msg=${ws.messages} err=${ws.errors}` + ); + }, config.reportEverySec * 1000); + + const openWs = (id: number): void => { + if (!isRunning()) return; + + const socket = new WebSocket(config.wsUrl); + const state: { + socket: WebSocket; + pingTimer?: ReturnType; + reconnectTimer?: ReturnType; + } = { socket }; + socketState.set(id, state); + + socket.on("open", () => { + ws.opened += 1; + ws.active += 1; + + socket.send( + JSON.stringify({ + type: "subscribe", + channels: ["summit", "event"], + }) + ); + + const timer = setInterval(() => { + if (socket.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: "ping" })); + } + }, config.wsPingIntervalMs); + + state.pingTimer = timer; + }); + + socket.on("message", () => { + ws.messages += 1; + }); + + socket.on("error", () => { + ws.errors += 1; + }); + + socket.on("close", () => { + ws.closed += 1; + ws.active = Math.max(0, ws.active - 1); + if (state.pingTimer) { + clearInterval(state.pingTimer); + state.pingTimer = undefined; + } + + if (isRunning()) { + ws.reconnects += 1; + state.reconnectTimer = setTimeout(() => openWs(id), 500); + } + }); + }; + + for (let i = 0; i < config.wsConnections; i++) { + openWs(i); + } + + async function httpWorker(): Promise { + while (isRunning()) { + const endpoint = pickWeightedEndpoint(config.weightedEndpoints); + const url = toHttpUrl(config.baseUrl, endpoint); + const requestStarted = Date.now(); + + try { + const response = await fetch(url, { + method: "GET", + signal: AbortSignal.timeout(config.requestTimeoutMs), + headers: { + "accept": "application/json", + }, + }); + + // Consume body so connections are released promptly. + await response.arrayBuffer(); + const elapsed = Date.now() - requestStarted; + recordHttpResult(http, elapsed, String(response.status), response.ok); + } catch (error) { + const elapsed = Date.now() - requestStarted; + const name = error instanceof Error ? error.name : "UnknownError"; + const isTimeout = name === "AbortError" || name === "TimeoutError"; + recordHttpResult(http, elapsed, isTimeout ? "timeout" : name, false, isTimeout); + } + + if (config.httpPauseMs > 0) { + await sleep(config.httpPauseMs); + } + } + } + + console.log("Starting API stress test with config:"); + console.log(JSON.stringify(config, null, 2)); + + const workers = Array.from({ length: config.httpConcurrency }, () => httpWorker()); + await Promise.all(workers); + + stopping = true; + clearInterval(reporter); + process.off("SIGINT", stop); + process.off("SIGTERM", stop); + + for (const state of socketState.values()) { + if (state.pingTimer) { + clearInterval(state.pingTimer); + } + if (state.reconnectTimer) { + clearTimeout(state.reconnectTimer); + } + try { + state.socket.close(); + } catch { + // Ignore close errors. + } + } + socketState.clear(); + + const elapsedSec = (Date.now() - startedAt) / 1000; + const avgLatency = http.total > 0 ? http.latencyTotalMs / http.total : 0; + const reqRate = elapsedSec > 0 ? http.total / elapsedSec : 0; + + console.log("\n--- Stress Test Summary ---"); + console.log(`Elapsed: ${elapsedSec.toFixed(2)}s`); + console.log(`HTTP requests: ${http.total}`); + console.log(`HTTP success: ${http.success}`); + console.log(`HTTP failed: ${http.failed}`); + console.log(`HTTP timeouts: ${http.timedOut}`); + console.log(`HTTP throughput: ${reqRate.toFixed(2)} req/s`); + console.log( + `HTTP latency avg/min/max: ${avgLatency.toFixed(2)}/${Number.isFinite(http.latencyMinMs) ? http.latencyMinMs.toFixed(2) : "0.00"}/${http.latencyMaxMs.toFixed(2)} ms` + ); + console.log(`WS opened: ${ws.opened}`); + console.log(`WS closed: ${ws.closed}`); + console.log(`WS active (final): ${ws.active}`); + console.log(`WS reconnects: ${ws.reconnects}`); + console.log(`WS messages: ${ws.messages}`); + console.log(`WS errors: ${ws.errors}`); + console.log(`HTTP status/error counts: ${JSON.stringify(http.statuses)}`); +} + +run().catch((error) => { + console.error("Stress test crashed:", error); + process.exit(1); +}); From 440c8e565872d8185ab19e7762db0a85ea13a09a Mon Sep 17 00:00:00 2001 From: loothero Date: Fri, 6 Mar 2026 05:55:40 -0800 Subject: [PATCH 11/39] docs(indexer): add stress-test benchmarking guidance --- indexer/AGENTS.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/indexer/AGENTS.md b/indexer/AGENTS.md index de255d32..6c9beb0d 100644 --- a/indexer/AGENTS.md +++ b/indexer/AGENTS.md @@ -98,6 +98,17 @@ Critical semantic note: - Parity: `pnpm test:parity` - DB tooling: `pnpm db:generate`, `pnpm db:migrate`, `pnpm db:studio` +## Performance Benchmarking +- Use the API stress harness at `../api/scripts/stress-test.ts` for repeatable load testing. +- Preferred entrypoint from repo root: + - `cd api && npm run stress:test -- --base-url=https:// --duration-sec=300 --http-concurrency=500 --ws-connections=0 --report-every-sec=10` +- For app-like traffic, prefer weighted endpoints via `--endpoint-weights` instead of uniform random endpoints. +- During benchmarks, monitor both Railway services (`summit-api`, `summit-indexer`) and capture: + - HTTP throughput/latency/timeout rate from stress output + - API resource metrics (CPU, memory, DB pool waiting, cache hit/bypass) + - indexer warnings/errors and restart/crash signals +- Treat mixed HTTP+WS and HTTP-only runs as separate profiles and compare like-for-like. + ## CI for Indexer - Triggered by `indexer/**` and `contracts/src/models/beast.cairo`. - Job sequence: `pnpm exec tsc --noEmit` -> build -> parity -> coverage -> Codecov. From 947e7c8c028ddb33c70db251ab10973845146db2 Mon Sep 17 00:00:00 2001 From: loothero Date: Fri, 6 Mar 2026 05:56:07 -0800 Subject: [PATCH 12/39] chore(client): point API host to summit-api-production-ca43 --- client/src/utils/networkConfig.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/utils/networkConfig.ts b/client/src/utils/networkConfig.ts index ea763237..a62adbe5 100644 --- a/client/src/utils/networkConfig.ts +++ b/client/src/utils/networkConfig.ts @@ -57,8 +57,8 @@ export const NETWORKS = { slot: "pg-summit-2", rpcUrl: "https://api.cartridge.gg/x/starknet/mainnet/rpc/v0_9", torii: "https://api.cartridge.gg/x/pg-mainnet-10/torii", - apiUrl: "https://summit-production-69ed.up.railway.app", - wsUrl: "wss://summit-production-69ed.up.railway.app/ws", + apiUrl: "https://summit-api-production-ca43.up.railway.app", + wsUrl: "wss://summit-api-production-ca43.up.railway.app/ws", tokens: { erc20: [ { From e0db11300f3680cbbf487250b5fff479940ecfe7 Mon Sep 17 00:00:00 2001 From: loothero Date: Fri, 6 Mar 2026 06:03:19 -0800 Subject: [PATCH 13/39] fix(client): guard null owners in leaderboard address handling --- client/src/components/DiplomacyPopover.tsx | 16 +++++++- client/src/components/Leaderboard.jsx | 32 +++++++++++++--- client/src/components/Leaderboard.test.tsx | 25 ++++++++++++ .../components/dialogs/LeaderboardModal.tsx | 2 +- client/src/utils/addressNameCache.ts | 38 +++++++++++++++---- 5 files changed, 97 insertions(+), 16 deletions(-) diff --git a/client/src/components/DiplomacyPopover.tsx b/client/src/components/DiplomacyPopover.tsx index 07a0debc..b3304017 100644 --- a/client/src/components/DiplomacyPopover.tsx +++ b/client/src/components/DiplomacyPopover.tsx @@ -10,10 +10,22 @@ interface DiplomacyPopoverProps { onClose: () => void; diplomacy: Diplomacy; summitBeast: Beast; - leaderboard: { owner: string; amount: number }[]; + leaderboard: { owner: string | null; amount: number }[]; addressNames: Record; } +const sameAddress = (left: string | null | undefined, right: string | null | undefined): boolean => { + if (!left || !right) { + return false; + } + + try { + return addAddressPadding(left) === addAddressPadding(right); + } catch { + return false; + } +}; + export function DiplomacyPopover({ anchorEl, onClose, @@ -53,7 +65,7 @@ export function DiplomacyPopover({ .map((beast) => { const ownerRank = beast.owner ? leaderboard.findIndex(p => - addAddressPadding(p.owner) === addAddressPadding(beast.owner!) + sameAddress(p.owner, beast.owner) ) + 1 : 0; return ( diff --git a/client/src/components/Leaderboard.jsx b/client/src/components/Leaderboard.jsx index 6fdc7e8e..532e8b65 100644 --- a/client/src/components/Leaderboard.jsx +++ b/client/src/components/Leaderboard.jsx @@ -17,6 +17,23 @@ import { addAddressPadding } from 'starknet'; import { DiplomacyPopover } from './DiplomacyPopover'; import RewardsRemainingBar from './RewardsRemainingBar'; +const normalizeAddress = (address) => + typeof address === 'string' && address.length > 0 + ? address.replace(/^0x0+/, '0x').toLowerCase() + : null; + +const sameAddress = (left, right) => { + if (typeof left !== 'string' || typeof right !== 'string' || !left || !right) { + return false; + } + + try { + return addAddressPadding(left) === addAddressPadding(right); + } catch { + return false; + } +}; + function Leaderboard() { const { beastsRegistered, beastsAlive, consumablesSupply, fetchStats } = useStatistics() const { summit, leaderboard, setLeaderboard } = useGameStore() @@ -62,7 +79,9 @@ function Leaderboard() { // Add top 5 leaderboard addresses data.slice(0, 5).forEach(player => { - addressesToLookup.push(player.owner); + if (typeof player.owner === 'string' && player.owner.length > 0) { + addressesToLookup.push(player.owner); + } }); // Add summit owner if exists @@ -84,7 +103,8 @@ function Leaderboard() { const names = {}; // Map all names using original addresses as keys addressesToLookup.forEach(address => { - const normalized = address.replace(/^0x0+/, "0x").toLowerCase(); + const normalized = normalizeAddress(address); + if (!normalized) return; names[address] = addressMap.get(normalized) || null; }); @@ -114,7 +134,7 @@ function Leaderboard() { const diplomacyRewards = diplomacyRewardPerSecond * secondsHeld * diplomacyCount; // Find summit owner in leaderboard - const player = leaderboard.find(player => addAddressPadding(player.owner) === addAddressPadding(summitOwner)) + const player = leaderboard.find(player => sameAddress(player.owner, summitOwner)) const gainedSince = (secondsHeld * SUMMIT_REWARDS_PER_SECOND) - diplomacyRewards; const score = (player?.amount || 0) + gainedSince; @@ -145,7 +165,7 @@ function Leaderboard() { const displayName = cartridgeName || 'Warlock' return ( - + {index + 1}. {displayName} @@ -191,10 +211,10 @@ function Leaderboard() { {leaderboard.slice(0, 5).map((player, index) => ( ))} diff --git a/client/src/components/Leaderboard.test.tsx b/client/src/components/Leaderboard.test.tsx index b9127b09..5a40ea61 100644 --- a/client/src/components/Leaderboard.test.tsx +++ b/client/src/components/Leaderboard.test.tsx @@ -150,4 +150,29 @@ describe("Leaderboard", () => { expect(renderedText).toContain("SUMMIT"); expect(renderedText).toContain("Diplomacy"); }); + + it("ignores null owners in leaderboard data without crashing", async () => { + hoisted.getLeaderboardMock.mockResolvedValue([ + { owner: null, amount: 999 } as unknown as { owner: string; amount: number }, + ...apiLeaderboard, + ]); + + let renderer: ReturnType; + + await act(async () => { + renderer = create(); + }); + + await act(async () => { + await Promise.resolve(); + }); + + expect(hoisted.getLeaderboardMock).toHaveBeenCalledTimes(1); + + const lookupArgs = hoisted.lookupAddressNamesMock.mock.calls[0]?.[0] ?? []; + expect(lookupArgs).not.toContain(null); + + const renderedText = JSON.stringify(renderer!.toJSON()); + expect(renderedText).toContain("THE BIG FIVE"); + }); }); diff --git a/client/src/components/dialogs/LeaderboardModal.tsx b/client/src/components/dialogs/LeaderboardModal.tsx index 46fd18de..cdaf5e6c 100644 --- a/client/src/components/dialogs/LeaderboardModal.tsx +++ b/client/src/components/dialogs/LeaderboardModal.tsx @@ -56,7 +56,7 @@ export default function LeaderboardModal({ open, onClose }: LeaderboardModalProp const addressesToLookup = playerPagedItems .map((p) => p.owner) - .filter((addr) => addressNames[addr] === undefined); + .filter((addr): addr is string => typeof addr === 'string' && addr.length > 0 && addressNames[addr] === undefined); if (addressesToLookup.length === 0) return; diff --git a/client/src/utils/addressNameCache.ts b/client/src/utils/addressNameCache.ts index 6a0664aa..232f13bf 100644 --- a/client/src/utils/addressNameCache.ts +++ b/client/src/utils/addressNameCache.ts @@ -9,11 +9,22 @@ interface AddressNameCache { [normalizedAddress: string]: string | null; } +type MaybeAddress = string | null | undefined; + /** * Normalizes an address to ensure consistent cache keys */ -function normalizeAddress(address: string): string { - return address.replace(/^0x0+/, "0x").toLowerCase(); +function normalizeAddress(address: MaybeAddress): string | null { + if (typeof address !== "string") { + return null; + } + + const trimmed = address.trim(); + if (!trimmed) { + return null; + } + + return trimmed.replace(/^0x0+/, "0x").toLowerCase(); } /** @@ -53,8 +64,12 @@ function saveCache(cache: AddressNameCache): void { /** * Looks up a single address name, checking cache first */ -export async function lookupAddressName(address: string): Promise { +export async function lookupAddressName(address: MaybeAddress): Promise { const normalized = normalizeAddress(address); + if (!normalized) { + return null; + } + const cache = getCache(); // Check cache first @@ -83,9 +98,15 @@ export async function lookupAddressName(address: string): Promise * Returns a Map of normalized address to name */ export async function lookupAddressNames( - addresses: string[] + addresses: MaybeAddress[] ): Promise> { - const normalized = addresses.map(normalizeAddress); + const normalized = Array.from( + new Set( + addresses + .map(normalizeAddress) + .filter((address): address is string => address !== null) + ) + ); const cache = getCache(); const result = new Map(); const uncachedAddresses: string[] = []; @@ -129,8 +150,12 @@ export async function lookupAddressNames( /** * Manually adds or updates a name in the cache */ -export function cacheAddressName(address: string, name: string | null): void { +export function cacheAddressName(address: MaybeAddress, name: string | null): void { const normalized = normalizeAddress(address); + if (!normalized) { + return; + } + const cache = getCache(); cache[normalized] = name; @@ -161,4 +186,3 @@ export function getCacheStats(): { size: Object.keys(cache).length, }; } - From f9898fbe679705784ba14feedf3feb609a3612ce Mon Sep 17 00:00:00 2001 From: loothero Date: Fri, 6 Mar 2026 08:15:17 -0800 Subject: [PATCH 14/39] fix(client): prevent ws event crashes on null player addresses --- client/src/contexts/GameDirector.test.tsx | 26 ++++++++++++++++++++++- client/src/contexts/GameDirector.tsx | 25 +++++++++++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/client/src/contexts/GameDirector.test.tsx b/client/src/contexts/GameDirector.test.tsx index 77220920..cd52e59f 100644 --- a/client/src/contexts/GameDirector.test.tsx +++ b/client/src/contexts/GameDirector.test.tsx @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { GameAction, selection } from "@/types/game"; const hoisted = vi.hoisted(() => ({ + useWebSocketMock: vi.fn(), getSummitDataMock: vi.fn(async () => null), getDiplomacyMock: vi.fn(async () => []), executeActionMock: vi.fn(async () => []), @@ -49,7 +50,7 @@ vi.mock("./starknet", () => ({ })); vi.mock("@/hooks/useWebSocket", () => ({ - useWebSocket: vi.fn(), + useWebSocket: hoisted.useWebSocketMock, })); vi.mock("@/api/starknet", () => ({ @@ -142,6 +143,7 @@ async function renderProvider() { describe("GameDirector executeGameAction", () => { beforeEach(() => { vi.clearAllMocks(); + hoisted.useWebSocketMock.mockImplementation(() => undefined); hoisted.getSummitDataMock.mockResolvedValue(null); hoisted.attackMock.mockReturnValue([]); }); @@ -169,4 +171,26 @@ describe("GameDirector executeGameAction", () => { expect(hoisted.executeActionMock).not.toHaveBeenCalled(); expect(capturedDirector.pauseUpdates).toBe(false); }); + + it("handles websocket events when player is null and account is disconnected", async () => { + await renderProvider(); + + const wsOptions = hoisted.useWebSocketMock.mock.calls[0]?.[0]; + expect(wsOptions).toBeDefined(); + + expect(() => + wsOptions.onEvent({ + id: "evt-1", + block_number: "1", + event_index: 0, + category: "Unknown", + sub_category: "Unknown", + data: {}, + player: null, + token_id: null, + transaction_hash: "0x1", + created_at: new Date().toISOString(), + }) + ).not.toThrow(); + }); }); diff --git a/client/src/contexts/GameDirector.tsx b/client/src/contexts/GameDirector.tsx index 80a4f346..6d16ef41 100644 --- a/client/src/contexts/GameDirector.tsx +++ b/client/src/contexts/GameDirector.tsx @@ -24,7 +24,7 @@ import { getBeastRevivalTime, } from "@/utils/beasts"; import { useAccount } from "@starknet-react/core"; -import { addAddressPadding, type Call } from "starknet"; +import { type Call } from "starknet"; import type { PropsWithChildren } from "react"; @@ -70,6 +70,25 @@ const isSummitEvent = ( event: TranslatedGameEvent ): event is SummitEventTranslation => event.componentName === "Summit"; +function normalizeAddress(address: string | null | undefined): string | null { + if (typeof address !== "string") { + return null; + } + + const trimmed = address.trim(); + if (!trimmed) { + return null; + } + + return trimmed.replace(/^0x0+/, "0x").toLowerCase(); +} + +function addressesEqual(left: string | null | undefined, right: string | null | undefined): boolean { + const normalizedLeft = normalizeAddress(left); + const normalizedRight = normalizeAddress(right); + return normalizedLeft !== null && normalizedRight !== null && normalizedLeft === normalizedRight; +} + export const GameDirector = ({ children }: PropsWithChildren) => { const { account } = useAccount(); const { currentNetworkConfig } = useDynamicConnector(); @@ -160,7 +179,7 @@ export const GameDirector = ({ children }: PropsWithChildren) => { addLiveEvent(data); const { category, sub_category, data: eventData } = data; - const isOwnEvent = data.player === addAddressPadding(account?.address ?? ""); + const isOwnEvent = addressesEqual(data.player, account?.address); // Helper to get beast info from event data const getBeastInfo = () => { @@ -470,7 +489,7 @@ export const GameDirector = ({ children }: PropsWithChildren) => { } // Fetch diplomacy if not already set - if (!summit.diplomacy) { + if (!summit.diplomacy && summit.beast.prefix && summit.beast.suffix) { const fetchDiplomacy = async () => { try { const beasts = await getDiplomacy( From c9b7dd5356fd7e1529350f0559e6c5ddf31901f4 Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 09:54:16 -0800 Subject: [PATCH 15/39] fix(indexer): patch apibara drizzle reorg trigger churn --- indexer/package.json | 5 + ...pibara__plugin-drizzle@2.1.0-beta.55.patch | 345 ++++++++++++++++++ indexer/pnpm-lock.yaml | 9 +- 3 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch diff --git a/indexer/package.json b/indexer/package.json index 69ec3b17..27f9fb3d 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -51,5 +51,10 @@ "typescript": "^5.7.0", "typescript-eslint": "^8.55.0", "vitest": "^3.1.1" + }, + "pnpm": { + "patchedDependencies": { + "@apibara/plugin-drizzle@2.1.0-beta.55": "patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch" + } } } diff --git a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch new file mode 100644 index 00000000..ba095499 --- /dev/null +++ b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch @@ -0,0 +1,345 @@ +diff --git a/dist/index.cjs b/dist/index.cjs +index 4926b17a6418da810dff724625b8c4a846944b47..d3aea075fa68339c78c9b221c570e5fd523ad3c0 100644 +--- a/dist/index.cjs ++++ b/dist/index.cjs +@@ -410,11 +410,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { + DECLARE + table_name TEXT := TG_ARGV[0]::TEXT; + id_col TEXT := TG_ARGV[1]::TEXT; +- order_key INTEGER := TG_ARGV[2]::INTEGER; +- indexer_id TEXT := TG_ARGV[3]::TEXT; ++ order_key_text TEXT := current_setting('${constants.SCHEMA_NAME}.reorg_order_key', true); ++ order_key INTEGER; ++ indexer_id TEXT := TG_ARGV[2]::TEXT; + new_id_value TEXT := row_to_json(NEW.*)->>id_col; + old_id_value TEXT := row_to_json(OLD.*)->>id_col; + BEGIN ++ IF order_key_text IS NULL THEN ++ RETURN NULL; ++ END IF; ++ ++ order_key := order_key_text::INTEGER; ++ + IF (TG_OP = 'DELETE') THEN + INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; +@@ -439,7 +446,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { + ); + } + } +-async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { ++async function registerTriggers(tx, tables, idColumnMap, indexerId) { + try { + for (const table of tables) { + const tableIdColumn = getIdColumnForTable(table, idColumnMap); +@@ -453,7 +460,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { + CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} + AFTER INSERT OR UPDATE OR DELETE ON ${table} + DEFERRABLE INITIALLY DEFERRED +- FOR EACH ROW EXECUTE FUNCTION ${constants.SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', ${Number(endCursor.orderKey)}, '${indexerId}'); ++ FOR EACH ROW EXECUTE FUNCTION ${constants.SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', '${indexerId}'); + `) + ); + } +@@ -463,6 +470,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { + }); + } + } ++async function setReorgOrderKey(tx, endCursor) { ++ try { ++ await tx.execute( ++ drizzleOrm.sql.raw( ++ `SELECT set_config('${constants.SCHEMA_NAME}.reorg_order_key', '${Number(endCursor.orderKey)}', true);` ++ ) ++ ); ++ } catch (error) { ++ throw new DrizzleStorageError("Failed to set reorg order key", { ++ cause: error ++ }); ++ } ++} + async function removeTriggers(db, tables, indexerId) { + try { + for (const table of tables) { +@@ -653,6 +673,7 @@ function drizzleStorage({ + let indexerId = ""; + const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; + let prevFinality; ++ let reorgTriggersRegistered = false; + const schema = _schema ?? db._.schema ?? {}; + const idColumnMap = { + "*": typeof idColumn === "string" ? idColumn : "id", +@@ -823,13 +844,16 @@ function drizzleStorage({ + await invalidate(tx, cursor, idColumnMap, indexerId); + } + if (finality !== "finalized") { +- await registerTriggers( +- tx, +- tableNames, +- endCursor, +- idColumnMap, +- indexerId +- ); ++ if (!reorgTriggersRegistered) { ++ await registerTriggers( ++ tx, ++ tableNames, ++ idColumnMap, ++ indexerId ++ ); ++ reorgTriggersRegistered = true; ++ } ++ await setReorgOrderKey(tx, endCursor); + } + await next(); + delete context[constants.DRIZZLE_PROPERTY]; +@@ -842,11 +866,7 @@ function drizzleStorage({ + } + prevFinality = finality; + }); +- if (finality !== "finalized") { +- await removeTriggers(db, tableNames, indexerId); +- } + } catch (error) { +- await removeTriggers(db, tableNames, indexerId); + throw error; + } + }); +diff --git a/dist/index.mjs b/dist/index.mjs +index 1bdf312081394c65988b248952fef093ec89e812..e6c089008903f68058f175ccf6a8a8167f7022b7 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -408,11 +408,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { + DECLARE + table_name TEXT := TG_ARGV[0]::TEXT; + id_col TEXT := TG_ARGV[1]::TEXT; +- order_key INTEGER := TG_ARGV[2]::INTEGER; +- indexer_id TEXT := TG_ARGV[3]::TEXT; ++ order_key_text TEXT := current_setting('${SCHEMA_NAME}.reorg_order_key', true); ++ order_key INTEGER; ++ indexer_id TEXT := TG_ARGV[2]::TEXT; + new_id_value TEXT := row_to_json(NEW.*)->>id_col; + old_id_value TEXT := row_to_json(OLD.*)->>id_col; + BEGIN ++ IF order_key_text IS NULL THEN ++ RETURN NULL; ++ END IF; ++ ++ order_key := order_key_text::INTEGER; ++ + IF (TG_OP = 'DELETE') THEN + INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; +@@ -437,7 +444,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { + ); + } + } +-async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { ++async function registerTriggers(tx, tables, idColumnMap, indexerId) { + try { + for (const table of tables) { + const tableIdColumn = getIdColumnForTable(table, idColumnMap); +@@ -451,7 +458,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { + CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} + AFTER INSERT OR UPDATE OR DELETE ON ${table} + DEFERRABLE INITIALLY DEFERRED +- FOR EACH ROW EXECUTE FUNCTION ${SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', ${Number(endCursor.orderKey)}, '${indexerId}'); ++ FOR EACH ROW EXECUTE FUNCTION ${SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', '${indexerId}'); + `) + ); + } +@@ -461,6 +468,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { + }); + } + } ++async function setReorgOrderKey(tx, endCursor) { ++ try { ++ await tx.execute( ++ sql.raw( ++ `SELECT set_config('${SCHEMA_NAME}.reorg_order_key', '${Number(endCursor.orderKey)}', true);` ++ ) ++ ); ++ } catch (error) { ++ throw new DrizzleStorageError("Failed to set reorg order key", { ++ cause: error ++ }); ++ } ++} + async function removeTriggers(db, tables, indexerId) { + try { + for (const table of tables) { +@@ -651,6 +671,7 @@ function drizzleStorage({ + let indexerId = ""; + const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; + let prevFinality; ++ let reorgTriggersRegistered = false; + const schema = _schema ?? db._.schema ?? {}; + const idColumnMap = { + "*": typeof idColumn === "string" ? idColumn : "id", +@@ -821,13 +842,16 @@ function drizzleStorage({ + await invalidate(tx, cursor, idColumnMap, indexerId); + } + if (finality !== "finalized") { +- await registerTriggers( +- tx, +- tableNames, +- endCursor, +- idColumnMap, +- indexerId +- ); ++ if (!reorgTriggersRegistered) { ++ await registerTriggers( ++ tx, ++ tableNames, ++ idColumnMap, ++ indexerId ++ ); ++ reorgTriggersRegistered = true; ++ } ++ await setReorgOrderKey(tx, endCursor); + } + await next(); + delete context[DRIZZLE_PROPERTY]; +@@ -840,11 +864,7 @@ function drizzleStorage({ + } + prevFinality = finality; + }); +- if (finality !== "finalized") { +- await removeTriggers(db, tableNames, indexerId); +- } + } catch (error) { +- await removeTriggers(db, tableNames, indexerId); + throw error; + } + }); +diff --git a/src/index.ts b/src/index.ts +index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..d389b9d25d635be500551ca5cebb81c1e636a53c 100644 +--- a/src/index.ts ++++ b/src/index.ts +@@ -31,7 +31,7 @@ import { + initializeReorgRollbackTable, + invalidate, + registerTriggers, +- removeTriggers, ++ setReorgOrderKey, + } from "./storage"; + import { + DrizzleStorageError, +@@ -186,6 +186,7 @@ export function drizzleStorage< + let indexerId = ""; + const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; + let prevFinality: DataFinality | undefined; ++ let reorgTriggersRegistered = false; + const schema: TSchema = (_schema as TSchema) ?? db._.schema ?? {}; + const idColumnMap: IdColumnMap = { + "*": typeof idColumn === "string" ? idColumn : "id", +@@ -425,13 +426,16 @@ export function drizzleStorage< + } + + if (finality !== "finalized") { +- await registerTriggers( +- tx, +- tableNames, +- endCursor, +- idColumnMap, +- indexerId, +- ); ++ if (!reorgTriggersRegistered) { ++ await registerTriggers( ++ tx, ++ tableNames, ++ idColumnMap, ++ indexerId, ++ ); ++ reorgTriggersRegistered = true; ++ } ++ await setReorgOrderKey(tx, endCursor); + } + + await next(); +@@ -447,14 +451,7 @@ export function drizzleStorage< + + prevFinality = finality; + }); +- +- if (finality !== "finalized") { +- // remove trigger outside of the transaction or it won't be triggered. +- await removeTriggers(db, tableNames, indexerId); +- } + } catch (error) { +- await removeTriggers(db, tableNames, indexerId); +- + throw error; + } + }); +diff --git a/src/storage.ts b/src/storage.ts +index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32ebcf9d3afe 100644 +--- a/src/storage.ts ++++ b/src/storage.ts +@@ -91,11 +91,18 @@ export async function initializeReorgRollbackTable< + DECLARE + table_name TEXT := TG_ARGV[0]::TEXT; + id_col TEXT := TG_ARGV[1]::TEXT; +- order_key INTEGER := TG_ARGV[2]::INTEGER; +- indexer_id TEXT := TG_ARGV[3]::TEXT; ++ order_key_text TEXT := current_setting('${SCHEMA_NAME}.reorg_order_key', true); ++ order_key INTEGER; ++ indexer_id TEXT := TG_ARGV[2]::TEXT; + new_id_value TEXT := row_to_json(NEW.*)->>id_col; + old_id_value TEXT := row_to_json(OLD.*)->>id_col; + BEGIN ++ IF order_key_text IS NULL THEN ++ RETURN NULL; ++ END IF; ++ ++ order_key := order_key_text::INTEGER; ++ + IF (TG_OP = 'DELETE') THEN + INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; +@@ -129,7 +136,6 @@ export async function registerTriggers< + >( + tx: PgTransaction, + tables: string[], +- endCursor: Cursor, + idColumnMap: IdColumnMap, + indexerId: string, + ) { +@@ -148,7 +154,7 @@ export async function registerTriggers< + CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} + AFTER INSERT OR UPDATE OR DELETE ON ${table} + DEFERRABLE INITIALLY DEFERRED +- FOR EACH ROW EXECUTE FUNCTION ${SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', ${Number(endCursor.orderKey)}, '${indexerId}'); ++ FOR EACH ROW EXECUTE FUNCTION ${SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', '${indexerId}'); + `), + ); + } +@@ -159,6 +165,28 @@ export async function registerTriggers< + } + } + ++export async function setReorgOrderKey< ++ TQueryResult extends PgQueryResultHKT, ++ TFullSchema extends Record = Record, ++ TSchema extends ++ TablesRelationalConfig = ExtractTablesWithRelations, ++>( ++ tx: PgTransaction, ++ endCursor: Cursor, ++) { ++ try { ++ await tx.execute( ++ sql.raw( ++ `SELECT set_config('${SCHEMA_NAME}.reorg_order_key', '${Number(endCursor.orderKey)}', true);`, ++ ), ++ ); ++ } catch (error) { ++ throw new DrizzleStorageError("Failed to set reorg order key", { ++ cause: error, ++ }); ++ } ++} ++ + export async function removeTriggers< + TQueryResult extends PgQueryResultHKT, + TFullSchema extends Record = Record, diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 694eab63..261667d4 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -4,6 +4,11 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +patchedDependencies: + '@apibara/plugin-drizzle@2.1.0-beta.55': + hash: 0c8314eafcc5681c417ea10956f55967774a0c7ab45febfba9a223fdcfa1182c + path: patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch + importers: .: @@ -13,7 +18,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(patch_hash=0c8314eafcc5681c417ea10956f55967774a0c7ab45febfba9a223fdcfa1182c)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -2502,7 +2507,7 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=0c8314eafcc5681c417ea10956f55967774a0c7ab45febfba9a223fdcfa1182c)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) From 85f71892ccfa26677c4e053b96bed702eb33f87d Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 09:56:41 -0800 Subject: [PATCH 16/39] fix(indexer): include pnpm patches in docker build --- indexer/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/indexer/Dockerfile b/indexer/Dockerfile index 23c78bf4..4c7e9d21 100644 --- a/indexer/Dockerfile +++ b/indexer/Dockerfile @@ -9,6 +9,7 @@ WORKDIR /app # Install dependencies COPY package.json pnpm-lock.yaml ./ +COPY patches ./patches RUN corepack enable && \ corepack prepare pnpm@10.0.0 --activate && \ pnpm install --frozen-lockfile @@ -30,6 +31,7 @@ RUN addgroup -g 1001 -S nodejs && \ # Install production dependencies + tsx for status check script COPY package.json pnpm-lock.yaml ./ +COPY patches ./patches RUN corepack enable && \ corepack prepare pnpm@10.0.0 --activate && \ pnpm install --prod --frozen-lockfile && \ From 4671556e24a140b1b0e387053b7fb35db7881247 Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 10:24:38 -0800 Subject: [PATCH 17/39] fix(indexer): mark reorg triggers registered after commit --- ...pibara__plugin-drizzle@2.1.0-beta.55.patch | 64 ++++++++++++++----- indexer/pnpm-lock.yaml | 23 ++++++- 2 files changed, 67 insertions(+), 20 deletions(-) diff --git a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch index ba095499..0b802714 100644 --- a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch +++ b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch @@ -1,5 +1,5 @@ diff --git a/dist/index.cjs b/dist/index.cjs -index 4926b17a6418da810dff724625b8c4a846944b47..d3aea075fa68339c78c9b221c570e5fd523ad3c0 100644 +index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97e96c300b 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -410,11 +410,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { @@ -69,7 +69,15 @@ index 4926b17a6418da810dff724625b8c4a846944b47..d3aea075fa68339c78c9b221c570e5fd const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -823,13 +844,16 @@ function drizzleStorage({ +@@ -813,6 +834,7 @@ function drizzleStorage({ + indexer$1.hooks.hook("handler:middleware", async ({ use }) => { + use(async (context, next) => { + try { ++ let registeredTriggersInTxn = false; + const { endCursor, finality, cursor } = context; + if (!endCursor) { + throw new DrizzleStorageError("End Cursor is undefined"); +@@ -823,13 +845,16 @@ function drizzleStorage({ await invalidate(tx, cursor, idColumnMap, indexerId); } if (finality !== "finalized") { @@ -87,26 +95,28 @@ index 4926b17a6418da810dff724625b8c4a846944b47..d3aea075fa68339c78c9b221c570e5fd + idColumnMap, + indexerId + ); -+ reorgTriggersRegistered = true; ++ registeredTriggersInTxn = true; + } + await setReorgOrderKey(tx, endCursor); } await next(); delete context[constants.DRIZZLE_PROPERTY]; -@@ -842,11 +866,7 @@ function drizzleStorage({ +@@ -842,11 +867,10 @@ function drizzleStorage({ } prevFinality = finality; }); - if (finality !== "finalized") { - await removeTriggers(db, tableNames, indexerId); -- } ++ if (registeredTriggersInTxn) { ++ reorgTriggersRegistered = true; + } } catch (error) { - await removeTriggers(db, tableNames, indexerId); throw error; } }); diff --git a/dist/index.mjs b/dist/index.mjs -index 1bdf312081394c65988b248952fef093ec89e812..e6c089008903f68058f175ccf6a8a8167f7022b7 100644 +index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070ebdb18c4ab 100644 --- a/dist/index.mjs +++ b/dist/index.mjs @@ -408,11 +408,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { @@ -176,7 +186,15 @@ index 1bdf312081394c65988b248952fef093ec89e812..e6c089008903f68058f175ccf6a8a816 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -821,13 +842,16 @@ function drizzleStorage({ +@@ -811,6 +832,7 @@ function drizzleStorage({ + indexer.hooks.hook("handler:middleware", async ({ use }) => { + use(async (context, next) => { + try { ++ let registeredTriggersInTxn = false; + const { endCursor, finality, cursor } = context; + if (!endCursor) { + throw new DrizzleStorageError("End Cursor is undefined"); +@@ -821,13 +843,16 @@ function drizzleStorage({ await invalidate(tx, cursor, idColumnMap, indexerId); } if (finality !== "finalized") { @@ -194,26 +212,28 @@ index 1bdf312081394c65988b248952fef093ec89e812..e6c089008903f68058f175ccf6a8a816 + idColumnMap, + indexerId + ); -+ reorgTriggersRegistered = true; ++ registeredTriggersInTxn = true; + } + await setReorgOrderKey(tx, endCursor); } await next(); delete context[DRIZZLE_PROPERTY]; -@@ -840,11 +864,7 @@ function drizzleStorage({ +@@ -840,11 +865,10 @@ function drizzleStorage({ } prevFinality = finality; }); - if (finality !== "finalized") { - await removeTriggers(db, tableNames, indexerId); -- } ++ if (registeredTriggersInTxn) { ++ reorgTriggersRegistered = true; + } } catch (error) { - await removeTriggers(db, tableNames, indexerId); throw error; } }); diff --git a/src/index.ts b/src/index.ts -index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..d389b9d25d635be500551ca5cebb81c1e636a53c 100644 +index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..6762c2cdb9ad114f74b0a37e7e2d317ae9d3b5cc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,7 +31,7 @@ import { @@ -233,7 +253,15 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..d389b9d25d635be500551ca5cebb81c1 const schema: TSchema = (_schema as TSchema) ?? db._.schema ?? {}; const idColumnMap: IdColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -425,13 +426,16 @@ export function drizzleStorage< +@@ -402,6 +403,7 @@ export function drizzleStorage< + indexer.hooks.hook("handler:middleware", async ({ use }) => { + use(async (context, next) => { + try { ++ let registeredTriggersInTxn = false; + const { endCursor, finality, cursor } = context as { + cursor: Cursor; + endCursor: Cursor; +@@ -425,13 +427,16 @@ export function drizzleStorage< } if (finality !== "finalized") { @@ -251,21 +279,23 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..d389b9d25d635be500551ca5cebb81c1 + idColumnMap, + indexerId, + ); -+ reorgTriggersRegistered = true; ++ registeredTriggersInTxn = true; + } + await setReorgOrderKey(tx, endCursor); } await next(); -@@ -447,14 +451,7 @@ export function drizzleStorage< - +@@ -448,13 +453,11 @@ export function drizzleStorage< prevFinality = finality; }); -- + - if (finality !== "finalized") { - // remove trigger outside of the transaction or it won't be triggered. - await removeTriggers(db, tableNames, indexerId); -- } ++ if (registeredTriggersInTxn) { ++ // Mark registration only after the transaction commits successfully. ++ reorgTriggersRegistered = true; + } } catch (error) { - await removeTriggers(db, tableNames, indexerId); - diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 261667d4..6d0bba70 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: patchedDependencies: '@apibara/plugin-drizzle@2.1.0-beta.55': - hash: 0c8314eafcc5681c417ea10956f55967774a0c7ab45febfba9a223fdcfa1182c + hash: b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7 path: patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch importers: @@ -18,7 +18,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(patch_hash=0c8314eafcc5681c417ea10956f55967774a0c7ab45febfba9a223fdcfa1182c)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -825,21 +825,25 @@ packages: resolution: {integrity: sha512-bYyZLXzJ2boZ7CdUuCSAaTcWkVKcBUOL+B86zv+tRyrtk4BIpHF+L+vOg5uPD/PHwrIglxAno5MN4NnpkUj5fQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.0.0-beta.3': resolution: {integrity: sha512-t/jaaFrCSvwX2075jRfa2bwAcsuTtY1/sIT4XqsDg2MVxWQtaUyBx5Mi0pqZKTjdOPnL+f/zoUC9dxT2lUpNmw==} cpu: [arm64] os: [linux] + libc: [musl] '@rolldown/binding-linux-x64-gnu@1.0.0-beta.3': resolution: {integrity: sha512-EeDNLPU0Xw8ByRWxNLO30AF0fKYkdb/6rH5G073NFBDkj7ggYR/CvsNBjtDeCJ7+I6JG4xUjete2+VeV+GQjiA==} cpu: [x64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-musl@1.0.0-beta.3': resolution: {integrity: sha512-iTcAj8FKac3nyQhvFuqKt6Xqu9YNDbe1ew6US2OSN4g3zwfujgylaRCitEG+Uzd7AZfSVVLAfqrxKMa36Sj9Mg==} cpu: [x64] os: [linux] + libc: [musl] '@rolldown/binding-wasm32-wasi@1.0.0-beta.3': resolution: {integrity: sha512-sYgbsbyspvVZ2zplqsTxjf2N3e8UQGQnSsN5u4bMX461gY5vAsjUiA4nf1/ztDBMHWT79lF2QNx4csjnjSxMlA==} @@ -922,66 +926,79 @@ packages: resolution: {integrity: sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.57.1': resolution: {integrity: sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.57.1': resolution: {integrity: sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.57.1': resolution: {integrity: sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.57.1': resolution: {integrity: sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.57.1': resolution: {integrity: sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.57.1': resolution: {integrity: sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.57.1': resolution: {integrity: sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.57.1': resolution: {integrity: sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.57.1': resolution: {integrity: sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.57.1': resolution: {integrity: sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.57.1': resolution: {integrity: sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.57.1': resolution: {integrity: sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.57.1': resolution: {integrity: sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==} @@ -2507,7 +2524,7 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=0c8314eafcc5681c417ea10956f55967774a0c7ab45febfba9a223fdcfa1182c)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) From bbd27a5d796dc1f948b79c7ac036113cbf2093b1 Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 10:49:14 -0800 Subject: [PATCH 18/39] Set Apibara finality to pending --- indexer/indexers/summit.indexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/indexers/summit.indexer.ts b/indexer/indexers/summit.indexer.ts index 98685ecf..f2b79278 100644 --- a/indexer/indexers/summit.indexer.ts +++ b/indexer/indexers/summit.indexer.ts @@ -990,7 +990,7 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { return defineIndexer(StarknetStream)({ streamUrl, - finality: "accepted", + finality: "pending", startingBlock, filter: { events: [ From 3a0910edf074d3850c65ff410a8966edc9342514 Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 11:21:46 -0800 Subject: [PATCH 19/39] perf(indexer): avoid eager id extraction in reorg trigger --- ...pibara__plugin-drizzle@2.1.0-beta.55.patch | 97 +++++++++++++------ indexer/pnpm-lock.yaml | 6 +- 2 files changed, 71 insertions(+), 32 deletions(-) diff --git a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch index 0b802714..3a936319 100644 --- a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch +++ b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch @@ -1,18 +1,20 @@ diff --git a/dist/index.cjs b/dist/index.cjs -index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97e96c300b 100644 +index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b3b23937d 100644 --- a/dist/index.cjs +++ b/dist/index.cjs -@@ -410,11 +410,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { +@@ -410,18 +410,29 @@ async function initializeReorgRollbackTable(tx, indexerId) { DECLARE table_name TEXT := TG_ARGV[0]::TEXT; id_col TEXT := TG_ARGV[1]::TEXT; - order_key INTEGER := TG_ARGV[2]::INTEGER; - indexer_id TEXT := TG_ARGV[3]::TEXT; +- new_id_value TEXT := row_to_json(NEW.*)->>id_col; +- old_id_value TEXT := row_to_json(OLD.*)->>id_col; + order_key_text TEXT := current_setting('${constants.SCHEMA_NAME}.reorg_order_key', true); + order_key INTEGER; + indexer_id TEXT := TG_ARGV[2]::TEXT; - new_id_value TEXT := row_to_json(NEW.*)->>id_col; - old_id_value TEXT := row_to_json(OLD.*)->>id_col; ++ new_id_value TEXT; ++ old_id_value TEXT; BEGIN + IF order_key_text IS NULL THEN + RETURN NULL; @@ -21,9 +23,20 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 + order_key := order_key_text::INTEGER; + IF (TG_OP = 'DELETE') THEN ++ old_id_value := row_to_json(OLD.*)->>id_col; INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; -@@ -439,7 +446,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { + ELSIF (TG_OP = 'UPDATE') THEN ++ new_id_value := row_to_json(NEW.*)->>id_col; ++ old_id_value := row_to_json(OLD.*)->>id_col; + INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id; + ELSIF (TG_OP = 'INSERT') THEN ++ new_id_value := row_to_json(NEW.*)->>id_col; + INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'I', table_name, order_key, new_id_value, null, indexer_id; + END IF; +@@ -439,7 +450,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { ); } } @@ -32,7 +45,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -453,7 +460,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -453,7 +464,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -41,7 +54,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 `) ); } -@@ -463,6 +470,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -463,6 +474,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { }); } } @@ -61,7 +74,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 async function removeTriggers(db, tables, indexerId) { try { for (const table of tables) { -@@ -653,6 +673,7 @@ function drizzleStorage({ +@@ -653,6 +677,7 @@ function drizzleStorage({ let indexerId = ""; const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; let prevFinality; @@ -69,7 +82,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -813,6 +834,7 @@ function drizzleStorage({ +@@ -813,6 +838,7 @@ function drizzleStorage({ indexer$1.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -77,7 +90,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 const { endCursor, finality, cursor } = context; if (!endCursor) { throw new DrizzleStorageError("End Cursor is undefined"); -@@ -823,13 +845,16 @@ function drizzleStorage({ +@@ -823,13 +849,16 @@ function drizzleStorage({ await invalidate(tx, cursor, idColumnMap, indexerId); } if (finality !== "finalized") { @@ -101,7 +114,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 } await next(); delete context[constants.DRIZZLE_PROPERTY]; -@@ -842,11 +867,10 @@ function drizzleStorage({ +@@ -842,11 +871,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -116,20 +129,22 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 } }); diff --git a/dist/index.mjs b/dist/index.mjs -index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070ebdb18c4ab 100644 +index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00feb1d081 100644 --- a/dist/index.mjs +++ b/dist/index.mjs -@@ -408,11 +408,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { +@@ -408,18 +408,29 @@ async function initializeReorgRollbackTable(tx, indexerId) { DECLARE table_name TEXT := TG_ARGV[0]::TEXT; id_col TEXT := TG_ARGV[1]::TEXT; - order_key INTEGER := TG_ARGV[2]::INTEGER; - indexer_id TEXT := TG_ARGV[3]::TEXT; +- new_id_value TEXT := row_to_json(NEW.*)->>id_col; +- old_id_value TEXT := row_to_json(OLD.*)->>id_col; + order_key_text TEXT := current_setting('${SCHEMA_NAME}.reorg_order_key', true); + order_key INTEGER; + indexer_id TEXT := TG_ARGV[2]::TEXT; - new_id_value TEXT := row_to_json(NEW.*)->>id_col; - old_id_value TEXT := row_to_json(OLD.*)->>id_col; ++ new_id_value TEXT; ++ old_id_value TEXT; BEGIN + IF order_key_text IS NULL THEN + RETURN NULL; @@ -138,9 +153,20 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb + order_key := order_key_text::INTEGER; + IF (TG_OP = 'DELETE') THEN ++ old_id_value := row_to_json(OLD.*)->>id_col; INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; -@@ -437,7 +444,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { + ELSIF (TG_OP = 'UPDATE') THEN ++ new_id_value := row_to_json(NEW.*)->>id_col; ++ old_id_value := row_to_json(OLD.*)->>id_col; + INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id; + ELSIF (TG_OP = 'INSERT') THEN ++ new_id_value := row_to_json(NEW.*)->>id_col; + INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'I', table_name, order_key, new_id_value, null, indexer_id; + END IF; +@@ -437,7 +448,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { ); } } @@ -149,7 +175,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -451,7 +458,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -451,7 +462,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -158,7 +184,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb `) ); } -@@ -461,6 +468,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -461,6 +472,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { }); } } @@ -178,7 +204,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb async function removeTriggers(db, tables, indexerId) { try { for (const table of tables) { -@@ -651,6 +671,7 @@ function drizzleStorage({ +@@ -651,6 +675,7 @@ function drizzleStorage({ let indexerId = ""; const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; let prevFinality; @@ -186,7 +212,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -811,6 +832,7 @@ function drizzleStorage({ +@@ -811,6 +836,7 @@ function drizzleStorage({ indexer.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -194,7 +220,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb const { endCursor, finality, cursor } = context; if (!endCursor) { throw new DrizzleStorageError("End Cursor is undefined"); -@@ -821,13 +843,16 @@ function drizzleStorage({ +@@ -821,13 +847,16 @@ function drizzleStorage({ await invalidate(tx, cursor, idColumnMap, indexerId); } if (finality !== "finalized") { @@ -218,7 +244,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb } await next(); delete context[DRIZZLE_PROPERTY]; -@@ -840,11 +865,10 @@ function drizzleStorage({ +@@ -840,11 +869,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -303,20 +329,22 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..6762c2cdb9ad114f74b0a37e7e2d317a } }); diff --git a/src/storage.ts b/src/storage.ts -index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32ebcf9d3afe 100644 +index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..cfb5636f39a6d085190af2bad8784e7d76ea847a 100644 --- a/src/storage.ts +++ b/src/storage.ts -@@ -91,11 +91,18 @@ export async function initializeReorgRollbackTable< +@@ -91,18 +91,29 @@ export async function initializeReorgRollbackTable< DECLARE table_name TEXT := TG_ARGV[0]::TEXT; id_col TEXT := TG_ARGV[1]::TEXT; - order_key INTEGER := TG_ARGV[2]::INTEGER; - indexer_id TEXT := TG_ARGV[3]::TEXT; +- new_id_value TEXT := row_to_json(NEW.*)->>id_col; +- old_id_value TEXT := row_to_json(OLD.*)->>id_col; + order_key_text TEXT := current_setting('${SCHEMA_NAME}.reorg_order_key', true); + order_key INTEGER; + indexer_id TEXT := TG_ARGV[2]::TEXT; - new_id_value TEXT := row_to_json(NEW.*)->>id_col; - old_id_value TEXT := row_to_json(OLD.*)->>id_col; ++ new_id_value TEXT; ++ old_id_value TEXT; BEGIN + IF order_key_text IS NULL THEN + RETURN NULL; @@ -325,9 +353,20 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb + order_key := order_key_text::INTEGER; + IF (TG_OP = 'DELETE') THEN ++ old_id_value := row_to_json(OLD.*)->>id_col; INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; -@@ -129,7 +136,6 @@ export async function registerTriggers< + ELSIF (TG_OP = 'UPDATE') THEN ++ new_id_value := row_to_json(NEW.*)->>id_col; ++ old_id_value := row_to_json(OLD.*)->>id_col; + INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id; + ELSIF (TG_OP = 'INSERT') THEN ++ new_id_value := row_to_json(NEW.*)->>id_col; + INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) + SELECT 'I', table_name, order_key, new_id_value, null, indexer_id; + END IF; +@@ -129,7 +140,6 @@ export async function registerTriggers< >( tx: PgTransaction, tables: string[], @@ -335,7 +374,7 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb idColumnMap: IdColumnMap, indexerId: string, ) { -@@ -148,7 +154,7 @@ export async function registerTriggers< +@@ -148,7 +158,7 @@ export async function registerTriggers< CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -344,7 +383,7 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb `), ); } -@@ -159,6 +165,28 @@ export async function registerTriggers< +@@ -159,6 +169,28 @@ export async function registerTriggers< } } diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 6d0bba70..3b335102 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: patchedDependencies: '@apibara/plugin-drizzle@2.1.0-beta.55': - hash: b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7 + hash: 937897f0f5f45faa6c66c29eb7ac04ad3bffafd32c1bd5d0ab452f6809d05146 path: patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch importers: @@ -18,7 +18,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(patch_hash=937897f0f5f45faa6c66c29eb7ac04ad3bffafd32c1bd5d0ab452f6809d05146)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -2524,7 +2524,7 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=937897f0f5f45faa6c66c29eb7ac04ad3bffafd32c1bd5d0ab452f6809d05146)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) From 4c384ab1ef0eec820f684a2e05994bff278d865f Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 11:52:21 -0800 Subject: [PATCH 20/39] Revert "perf(indexer): avoid eager id extraction in reorg trigger" This reverts commit 3a0910edf074d3850c65ff410a8966edc9342514. --- ...pibara__plugin-drizzle@2.1.0-beta.55.patch | 97 ++++++------------- indexer/pnpm-lock.yaml | 6 +- 2 files changed, 32 insertions(+), 71 deletions(-) diff --git a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch index 3a936319..0b802714 100644 --- a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch +++ b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch @@ -1,20 +1,18 @@ diff --git a/dist/index.cjs b/dist/index.cjs -index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b3b23937d 100644 +index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97e96c300b 100644 --- a/dist/index.cjs +++ b/dist/index.cjs -@@ -410,18 +410,29 @@ async function initializeReorgRollbackTable(tx, indexerId) { +@@ -410,11 +410,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { DECLARE table_name TEXT := TG_ARGV[0]::TEXT; id_col TEXT := TG_ARGV[1]::TEXT; - order_key INTEGER := TG_ARGV[2]::INTEGER; - indexer_id TEXT := TG_ARGV[3]::TEXT; -- new_id_value TEXT := row_to_json(NEW.*)->>id_col; -- old_id_value TEXT := row_to_json(OLD.*)->>id_col; + order_key_text TEXT := current_setting('${constants.SCHEMA_NAME}.reorg_order_key', true); + order_key INTEGER; + indexer_id TEXT := TG_ARGV[2]::TEXT; -+ new_id_value TEXT; -+ old_id_value TEXT; + new_id_value TEXT := row_to_json(NEW.*)->>id_col; + old_id_value TEXT := row_to_json(OLD.*)->>id_col; BEGIN + IF order_key_text IS NULL THEN + RETURN NULL; @@ -23,20 +21,9 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b + order_key := order_key_text::INTEGER; + IF (TG_OP = 'DELETE') THEN -+ old_id_value := row_to_json(OLD.*)->>id_col; INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; - ELSIF (TG_OP = 'UPDATE') THEN -+ new_id_value := row_to_json(NEW.*)->>id_col; -+ old_id_value := row_to_json(OLD.*)->>id_col; - INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) - SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id; - ELSIF (TG_OP = 'INSERT') THEN -+ new_id_value := row_to_json(NEW.*)->>id_col; - INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) - SELECT 'I', table_name, order_key, new_id_value, null, indexer_id; - END IF; -@@ -439,7 +450,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { +@@ -439,7 +446,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { ); } } @@ -45,7 +32,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -453,7 +464,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -453,7 +460,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -54,7 +41,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b `) ); } -@@ -463,6 +474,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -463,6 +470,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { }); } } @@ -74,7 +61,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b async function removeTriggers(db, tables, indexerId) { try { for (const table of tables) { -@@ -653,6 +677,7 @@ function drizzleStorage({ +@@ -653,6 +673,7 @@ function drizzleStorage({ let indexerId = ""; const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; let prevFinality; @@ -82,7 +69,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -813,6 +838,7 @@ function drizzleStorage({ +@@ -813,6 +834,7 @@ function drizzleStorage({ indexer$1.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -90,7 +77,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b const { endCursor, finality, cursor } = context; if (!endCursor) { throw new DrizzleStorageError("End Cursor is undefined"); -@@ -823,13 +849,16 @@ function drizzleStorage({ +@@ -823,13 +845,16 @@ function drizzleStorage({ await invalidate(tx, cursor, idColumnMap, indexerId); } if (finality !== "finalized") { @@ -114,7 +101,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b } await next(); delete context[constants.DRIZZLE_PROPERTY]; -@@ -842,11 +871,10 @@ function drizzleStorage({ +@@ -842,11 +867,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -129,22 +116,20 @@ index 4926b17a6418da810dff724625b8c4a846944b47..8dec3d40c96ee9f2da8d25f04284648b } }); diff --git a/dist/index.mjs b/dist/index.mjs -index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00feb1d081 100644 +index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070ebdb18c4ab 100644 --- a/dist/index.mjs +++ b/dist/index.mjs -@@ -408,18 +408,29 @@ async function initializeReorgRollbackTable(tx, indexerId) { +@@ -408,11 +408,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { DECLARE table_name TEXT := TG_ARGV[0]::TEXT; id_col TEXT := TG_ARGV[1]::TEXT; - order_key INTEGER := TG_ARGV[2]::INTEGER; - indexer_id TEXT := TG_ARGV[3]::TEXT; -- new_id_value TEXT := row_to_json(NEW.*)->>id_col; -- old_id_value TEXT := row_to_json(OLD.*)->>id_col; + order_key_text TEXT := current_setting('${SCHEMA_NAME}.reorg_order_key', true); + order_key INTEGER; + indexer_id TEXT := TG_ARGV[2]::TEXT; -+ new_id_value TEXT; -+ old_id_value TEXT; + new_id_value TEXT := row_to_json(NEW.*)->>id_col; + old_id_value TEXT := row_to_json(OLD.*)->>id_col; BEGIN + IF order_key_text IS NULL THEN + RETURN NULL; @@ -153,20 +138,9 @@ index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00 + order_key := order_key_text::INTEGER; + IF (TG_OP = 'DELETE') THEN -+ old_id_value := row_to_json(OLD.*)->>id_col; INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; - ELSIF (TG_OP = 'UPDATE') THEN -+ new_id_value := row_to_json(NEW.*)->>id_col; -+ old_id_value := row_to_json(OLD.*)->>id_col; - INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) - SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id; - ELSIF (TG_OP = 'INSERT') THEN -+ new_id_value := row_to_json(NEW.*)->>id_col; - INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) - SELECT 'I', table_name, order_key, new_id_value, null, indexer_id; - END IF; -@@ -437,7 +448,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { +@@ -437,7 +444,7 @@ async function initializeReorgRollbackTable(tx, indexerId) { ); } } @@ -175,7 +149,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00 try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -451,7 +462,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -451,7 +458,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -184,7 +158,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00 `) ); } -@@ -461,6 +472,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -461,6 +468,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { }); } } @@ -204,7 +178,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00 async function removeTriggers(db, tables, indexerId) { try { for (const table of tables) { -@@ -651,6 +675,7 @@ function drizzleStorage({ +@@ -651,6 +671,7 @@ function drizzleStorage({ let indexerId = ""; const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; let prevFinality; @@ -212,7 +186,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -811,6 +836,7 @@ function drizzleStorage({ +@@ -811,6 +832,7 @@ function drizzleStorage({ indexer.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -220,7 +194,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00 const { endCursor, finality, cursor } = context; if (!endCursor) { throw new DrizzleStorageError("End Cursor is undefined"); -@@ -821,13 +847,16 @@ function drizzleStorage({ +@@ -821,13 +843,16 @@ function drizzleStorage({ await invalidate(tx, cursor, idColumnMap, indexerId); } if (finality !== "finalized") { @@ -244,7 +218,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..969fd87818e4d6e44e2308d3ee5a2a00 } await next(); delete context[DRIZZLE_PROPERTY]; -@@ -840,11 +869,10 @@ function drizzleStorage({ +@@ -840,11 +865,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -329,22 +303,20 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..6762c2cdb9ad114f74b0a37e7e2d317a } }); diff --git a/src/storage.ts b/src/storage.ts -index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..cfb5636f39a6d085190af2bad8784e7d76ea847a 100644 +index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32ebcf9d3afe 100644 --- a/src/storage.ts +++ b/src/storage.ts -@@ -91,18 +91,29 @@ export async function initializeReorgRollbackTable< +@@ -91,11 +91,18 @@ export async function initializeReorgRollbackTable< DECLARE table_name TEXT := TG_ARGV[0]::TEXT; id_col TEXT := TG_ARGV[1]::TEXT; - order_key INTEGER := TG_ARGV[2]::INTEGER; - indexer_id TEXT := TG_ARGV[3]::TEXT; -- new_id_value TEXT := row_to_json(NEW.*)->>id_col; -- old_id_value TEXT := row_to_json(OLD.*)->>id_col; + order_key_text TEXT := current_setting('${SCHEMA_NAME}.reorg_order_key', true); + order_key INTEGER; + indexer_id TEXT := TG_ARGV[2]::TEXT; -+ new_id_value TEXT; -+ old_id_value TEXT; + new_id_value TEXT := row_to_json(NEW.*)->>id_col; + old_id_value TEXT := row_to_json(OLD.*)->>id_col; BEGIN + IF order_key_text IS NULL THEN + RETURN NULL; @@ -353,20 +325,9 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..cfb5636f39a6d085190af2bad8784e7d + order_key := order_key_text::INTEGER; + IF (TG_OP = 'DELETE') THEN -+ old_id_value := row_to_json(OLD.*)->>id_col; INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id; - ELSIF (TG_OP = 'UPDATE') THEN -+ new_id_value := row_to_json(NEW.*)->>id_col; -+ old_id_value := row_to_json(OLD.*)->>id_col; - INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) - SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id; - ELSIF (TG_OP = 'INSERT') THEN -+ new_id_value := row_to_json(NEW.*)->>id_col; - INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id) - SELECT 'I', table_name, order_key, new_id_value, null, indexer_id; - END IF; -@@ -129,7 +140,6 @@ export async function registerTriggers< +@@ -129,7 +136,6 @@ export async function registerTriggers< >( tx: PgTransaction, tables: string[], @@ -374,7 +335,7 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..cfb5636f39a6d085190af2bad8784e7d idColumnMap: IdColumnMap, indexerId: string, ) { -@@ -148,7 +158,7 @@ export async function registerTriggers< +@@ -148,7 +154,7 @@ export async function registerTriggers< CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -383,7 +344,7 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..cfb5636f39a6d085190af2bad8784e7d `), ); } -@@ -159,6 +169,28 @@ export async function registerTriggers< +@@ -159,6 +165,28 @@ export async function registerTriggers< } } diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 3b335102..6d0bba70 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: patchedDependencies: '@apibara/plugin-drizzle@2.1.0-beta.55': - hash: 937897f0f5f45faa6c66c29eb7ac04ad3bffafd32c1bd5d0ab452f6809d05146 + hash: b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7 path: patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch importers: @@ -18,7 +18,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(patch_hash=937897f0f5f45faa6c66c29eb7ac04ad3bffafd32c1bd5d0ab452f6809d05146)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -2524,7 +2524,7 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=937897f0f5f45faa6c66c29eb7ac04ad3bffafd32c1bd5d0ab452f6809d05146)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) From c8204ac4aadef2fcc9c454ec88d67d264767c829 Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 11:57:51 -0800 Subject: [PATCH 21/39] perf(indexer): add covering index on rewards_earned(owner, amount) The /leaderboard endpoint aggregates SUM(amount) GROUP BY owner across the entire rewards_earned table. The existing owner-only index forces a sequential scan to fetch amount from the heap (163k seq scans observed). A composite (owner, amount) index enables an index-only scan for this query, eliminating the full table scan on every leaderboard request. Co-Authored-By: Claude Opus 4.6 --- api/src/db/schema.ts | 1 + .../0005_rewards_earned_leaderboard_index.sql | 1 + indexer/migrations/meta/0005_snapshot.json | 2007 +++++++++++++++++ indexer/migrations/meta/_journal.json | 9 +- indexer/src/lib/schema.ts | 1 + 5 files changed, 2018 insertions(+), 1 deletion(-) create mode 100644 indexer/migrations/0005_rewards_earned_leaderboard_index.sql create mode 100644 indexer/migrations/meta/0005_snapshot.json diff --git a/api/src/db/schema.ts b/api/src/db/schema.ts index aa91e5ca..f0f2bd3e 100644 --- a/api/src/db/schema.ts +++ b/api/src/db/schema.ts @@ -197,6 +197,7 @@ export const rewards_earned = pgTable( table.event_index ), index("rewards_earned_owner_idx").on(table.owner), + index("rewards_earned_owner_amount_idx").on(table.owner, table.amount), index("rewards_earned_beast_token_id_idx").on(table.beast_token_id), index("rewards_earned_created_at_idx").on(table.created_at.desc()), ] diff --git a/indexer/migrations/0005_rewards_earned_leaderboard_index.sql b/indexer/migrations/0005_rewards_earned_leaderboard_index.sql new file mode 100644 index 00000000..595603ac --- /dev/null +++ b/indexer/migrations/0005_rewards_earned_leaderboard_index.sql @@ -0,0 +1 @@ +CREATE INDEX "rewards_earned_owner_amount_idx" ON "rewards_earned" USING btree ("owner","amount"); \ No newline at end of file diff --git a/indexer/migrations/meta/0005_snapshot.json b/indexer/migrations/meta/0005_snapshot.json new file mode 100644 index 00000000..2b36a5ba --- /dev/null +++ b/indexer/migrations/meta/0005_snapshot.json @@ -0,0 +1,2007 @@ +{ + "id": "1f76ea3e-cf02-4917-a6d8-42a8f4f56ff6", + "prevId": "915c1fe2-8986-43a9-a313-2051039abfbe", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.battles": { + "name": "battles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "attacking_beast_token_id": { + "name": "attacking_beast_token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "attacking_player": { + "name": "attacking_player", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attack_index": { + "name": "attack_index", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "defending_beast_token_id": { + "name": "defending_beast_token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "attack_count": { + "name": "attack_count", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "attack_damage": { + "name": "attack_damage", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "critical_attack_count": { + "name": "critical_attack_count", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "critical_attack_damage": { + "name": "critical_attack_damage", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "counter_attack_count": { + "name": "counter_attack_count", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "counter_attack_damage": { + "name": "counter_attack_damage", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "critical_counter_attack_count": { + "name": "critical_counter_attack_count", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "critical_counter_attack_damage": { + "name": "critical_counter_attack_damage", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "attack_potions": { + "name": "attack_potions", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "revive_potions": { + "name": "revive_potions", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "xp_gained": { + "name": "xp_gained", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "transaction_hash": { + "name": "transaction_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "battles_block_tx_event_idx": { + "name": "battles_block_tx_event_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "transaction_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "battles_attacking_beast_idx": { + "name": "battles_attacking_beast_idx", + "columns": [ + { + "expression": "attacking_beast_token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "battles_attacking_player_idx": { + "name": "battles_attacking_player_idx", + "columns": [ + { + "expression": "attacking_player", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "battles_defending_beast_idx": { + "name": "battles_defending_beast_idx", + "columns": [ + { + "expression": "defending_beast_token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "battles_created_at_idx": { + "name": "battles_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "battles_block_number_idx": { + "name": "battles_block_number_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.beast_data": { + "name": "beast_data", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "entity_hash": { + "name": "entity_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "adventurers_killed": { + "name": "adventurers_killed", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_death_timestamp": { + "name": "last_death_timestamp", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_killed_by": { + "name": "last_killed_by", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "token_id": { + "name": "token_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "beast_data_token_id_idx": { + "name": "beast_data_token_id_idx", + "columns": [ + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_data_adventurers_killed_idx": { + "name": "beast_data_adventurers_killed_idx", + "columns": [ + { + "expression": "adventurers_killed", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_data_updated_at_idx": { + "name": "beast_data_updated_at_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "beast_data_entity_hash_unique": { + "name": "beast_data_entity_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "entity_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.beast_owners": { + "name": "beast_owners", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "token_id": { + "name": "token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "beast_owners_owner_idx": { + "name": "beast_owners_owner_idx", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_owners_owner_token_idx": { + "name": "beast_owners_owner_token_idx", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_owners_token_id_idx": { + "name": "beast_owners_token_id_idx", + "columns": [ + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "beast_owners_token_id_unique": { + "name": "beast_owners_token_id_unique", + "nullsNotDistinct": false, + "columns": [ + "token_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.beast_stats": { + "name": "beast_stats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "token_id": { + "name": "token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "current_health": { + "name": "current_health", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "bonus_health": { + "name": "bonus_health", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "bonus_xp": { + "name": "bonus_xp", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "attack_streak": { + "name": "attack_streak", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "last_death_timestamp": { + "name": "last_death_timestamp", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "revival_count": { + "name": "revival_count", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "extra_lives": { + "name": "extra_lives", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "captured_summit": { + "name": "captured_summit", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "used_revival_potion": { + "name": "used_revival_potion", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "used_attack_potion": { + "name": "used_attack_potion", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "max_attack_streak": { + "name": "max_attack_streak", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "summit_held_seconds": { + "name": "summit_held_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "spirit": { + "name": "spirit", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "luck": { + "name": "luck", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "specials": { + "name": "specials", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "wisdom": { + "name": "wisdom", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "diplomacy": { + "name": "diplomacy", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "rewards_earned": { + "name": "rewards_earned", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "rewards_claimed": { + "name": "rewards_claimed", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "transaction_hash": { + "name": "transaction_hash", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "beast_stats_current_health_idx": { + "name": "beast_stats_current_health_idx", + "columns": [ + { + "expression": "current_health", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_stats_summit_held_seconds_idx": { + "name": "beast_stats_summit_held_seconds_idx", + "columns": [ + { + "expression": "summit_held_seconds", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_stats_top_order_idx": { + "name": "beast_stats_top_order_idx", + "columns": [ + { + "expression": "summit_held_seconds", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "bonus_xp", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "last_death_timestamp", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_stats_updated_at_idx": { + "name": "beast_stats_updated_at_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beast_stats_diplomacy_token_idx": { + "name": "beast_stats_diplomacy_token_idx", + "columns": [ + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "diplomacy", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "beast_stats_token_id_unique": { + "name": "beast_stats_token_id_unique", + "nullsNotDistinct": false, + "columns": [ + "token_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.beasts": { + "name": "beasts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "token_id": { + "name": "token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "beast_id": { + "name": "beast_id", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "prefix": { + "name": "prefix", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "suffix": { + "name": "suffix", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "health": { + "name": "health", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "shiny": { + "name": "shiny", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "animated": { + "name": "animated", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "beasts_token_id_idx": { + "name": "beasts_token_id_idx", + "columns": [ + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beasts_beast_id_idx": { + "name": "beasts_beast_id_idx", + "columns": [ + { + "expression": "beast_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beasts_prefix_idx": { + "name": "beasts_prefix_idx", + "columns": [ + { + "expression": "prefix", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beasts_suffix_idx": { + "name": "beasts_suffix_idx", + "columns": [ + { + "expression": "suffix", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beasts_prefix_suffix_token_idx": { + "name": "beasts_prefix_suffix_token_idx", + "columns": [ + { + "expression": "prefix", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "suffix", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "beasts_level_idx": { + "name": "beasts_level_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "beasts_token_id_unique": { + "name": "beasts_token_id_unique", + "nullsNotDistinct": false, + "columns": [ + "token_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.consumables": { + "name": "consumables", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "xlife_count": { + "name": "xlife_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attack_count": { + "name": "attack_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "revive_count": { + "name": "revive_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "poison_count": { + "name": "poison_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "consumables_owner_unique": { + "name": "consumables_owner_unique", + "nullsNotDistinct": false, + "columns": [ + "owner" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.corpse_events": { + "name": "corpse_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "adventurer_id": { + "name": "adventurer_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "player": { + "name": "player", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "transaction_hash": { + "name": "transaction_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "corpse_events_block_tx_event_adv_idx": { + "name": "corpse_events_block_tx_event_adv_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "transaction_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "adventurer_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "corpse_events_adventurer_id_idx": { + "name": "corpse_events_adventurer_id_idx", + "columns": [ + { + "expression": "adventurer_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "corpse_events_player_idx": { + "name": "corpse_events_player_idx", + "columns": [ + { + "expression": "player", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "corpse_events_created_at_idx": { + "name": "corpse_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.poison_events": { + "name": "poison_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "beast_token_id": { + "name": "beast_token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "block_timestamp": { + "name": "block_timestamp", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "player": { + "name": "player", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "transaction_hash": { + "name": "transaction_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "poison_events_block_tx_event_idx": { + "name": "poison_events_block_tx_event_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "transaction_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "poison_events_beast_token_id_idx": { + "name": "poison_events_beast_token_id_idx", + "columns": [ + { + "expression": "beast_token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "poison_events_player_idx": { + "name": "poison_events_player_idx", + "columns": [ + { + "expression": "player", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "poison_events_created_at_idx": { + "name": "poison_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.quest_rewards_claimed": { + "name": "quest_rewards_claimed", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "beast_token_id": { + "name": "beast_token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "quest_rewards_claimed_beast_token_id_idx": { + "name": "quest_rewards_claimed_beast_token_id_idx", + "columns": [ + { + "expression": "beast_token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "quest_rewards_claimed_beast_token_id_unique": { + "name": "quest_rewards_claimed_beast_token_id_unique", + "nullsNotDistinct": false, + "columns": [ + "beast_token_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rewards_claimed": { + "name": "rewards_claimed", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "player": { + "name": "player", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "beast_token_ids": { + "name": "beast_token_ids", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "transaction_hash": { + "name": "transaction_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "rewards_claimed_block_tx_event_idx": { + "name": "rewards_claimed_block_tx_event_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "transaction_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rewards_claimed_player_idx": { + "name": "rewards_claimed_player_idx", + "columns": [ + { + "expression": "player", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rewards_claimed_created_at_idx": { + "name": "rewards_claimed_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rewards_earned": { + "name": "rewards_earned", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "beast_token_id": { + "name": "beast_token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "transaction_hash": { + "name": "transaction_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "rewards_earned_block_tx_event_idx": { + "name": "rewards_earned_block_tx_event_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "transaction_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rewards_earned_owner_idx": { + "name": "rewards_earned_owner_idx", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rewards_earned_owner_amount_idx": { + "name": "rewards_earned_owner_amount_idx", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "amount", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rewards_earned_beast_token_id_idx": { + "name": "rewards_earned_beast_token_id_idx", + "columns": [ + { + "expression": "beast_token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rewards_earned_created_at_idx": { + "name": "rewards_earned_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skulls_claimed": { + "name": "skulls_claimed", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "beast_token_id": { + "name": "beast_token_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "skulls": { + "name": "skulls", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "skulls_claimed_skulls_idx": { + "name": "skulls_claimed_skulls_idx", + "columns": [ + { + "expression": "skulls", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "skulls_claimed_beast_token_id_unique": { + "name": "skulls_claimed_beast_token_id_unique", + "nullsNotDistinct": false, + "columns": [ + "beast_token_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.summit_log": { + "name": "summit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_category": { + "name": "sub_category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "player": { + "name": "player", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_id": { + "name": "token_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "transaction_hash": { + "name": "transaction_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "inserted_at": { + "name": "inserted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "summit_log_block_tx_event_idx": { + "name": "summit_log_block_tx_event_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "transaction_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "summit_log_order_idx": { + "name": "summit_log_order_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "summit_log_category_idx": { + "name": "summit_log_category_idx", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "summit_log_sub_category_idx": { + "name": "summit_log_sub_category_idx", + "columns": [ + { + "expression": "sub_category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "summit_log_player_idx": { + "name": "summit_log_player_idx", + "columns": [ + { + "expression": "player", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "summit_log_token_id_idx": { + "name": "summit_log_token_id_idx", + "columns": [ + { + "expression": "token_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "summit_log_category_order_idx": { + "name": "summit_log_category_order_idx", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_number", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "summit_log_player_order_idx": { + "name": "summit_log_player_order_idx", + "columns": [ + { + "expression": "player", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_number", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/indexer/migrations/meta/_journal.json b/indexer/migrations/meta/_journal.json index 01007261..e9b62413 100644 --- a/indexer/migrations/meta/_journal.json +++ b/indexer/migrations/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1771012800000, "tag": "0004_api_perf_indexes", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1772913600000, + "tag": "0005_rewards_earned_leaderboard_index", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/indexer/src/lib/schema.ts b/indexer/src/lib/schema.ts index 33a7a528..cd9865c2 100644 --- a/indexer/src/lib/schema.ts +++ b/indexer/src/lib/schema.ts @@ -157,6 +157,7 @@ export const rewards_earned = pgTable( // Unique constraint for idempotent re-indexing uniqueIndex("rewards_earned_block_tx_event_idx").on(table.block_number, table.transaction_hash, table.event_index), index("rewards_earned_owner_idx").on(table.owner), + index("rewards_earned_owner_amount_idx").on(table.owner, table.amount), index("rewards_earned_beast_token_id_idx").on(table.beast_token_id), index("rewards_earned_created_at_idx").on(table.created_at.desc()), ] From 091a156ec991720a5255782da00409c6e5a991ca Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 12:21:59 -0800 Subject: [PATCH 22/39] fix(indexer): use IF NOT EXISTS in leaderboard index migration The index was already applied to the live DB, causing the migration to fail on indexer restart with "relation already exists". Co-Authored-By: Claude Opus 4.6 --- indexer/migrations/0005_rewards_earned_leaderboard_index.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/migrations/0005_rewards_earned_leaderboard_index.sql b/indexer/migrations/0005_rewards_earned_leaderboard_index.sql index 595603ac..50866b6a 100644 --- a/indexer/migrations/0005_rewards_earned_leaderboard_index.sql +++ b/indexer/migrations/0005_rewards_earned_leaderboard_index.sql @@ -1 +1 @@ -CREATE INDEX "rewards_earned_owner_amount_idx" ON "rewards_earned" USING btree ("owner","amount"); \ No newline at end of file +CREATE INDEX IF NOT EXISTS "rewards_earned_owner_amount_idx" ON "rewards_earned" USING btree ("owner","amount"); \ No newline at end of file From 3d496f28fbe26701d8e240db50e5ea1b6c50a64b Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 12:34:51 -0800 Subject: [PATCH 23/39] fix(indexer): set Railway restart policy to ALWAYS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apibara v2 no longer auto-reconnects on error — when the DB connection drops (e.g. during a DB restart), the indexer exits cleanly with code 0. With the default ON_FAILURE policy, Railway considers this a successful completion and does not restart. Setting ALWAYS ensures the indexer auto-recovers from transient DB outages. Co-Authored-By: Claude Opus 4.6 --- indexer/railway.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 indexer/railway.json diff --git a/indexer/railway.json b/indexer/railway.json new file mode 100644 index 00000000..5cedce74 --- /dev/null +++ b/indexer/railway.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://railway.com/railway.schema.json", + "deploy": { + "restartPolicyType": "ALWAYS", + "restartPolicyMaxRetries": 10 + } +} From c38d4a74695c11eec5295b2ffc91f5aecd567706 Mon Sep 17 00:00:00 2001 From: loothero Date: Sat, 7 Mar 2026 15:38:32 -0800 Subject: [PATCH 24/39] fix(indexer): isolate invalidate from block reorg order key txn --- ...pibara__plugin-drizzle@2.1.0-beta.55.patch | 77 ++++++++++++++----- indexer/pnpm-lock.yaml | 6 +- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch index 0b802714..4a56da80 100644 --- a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch +++ b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch @@ -1,5 +1,5 @@ diff --git a/dist/index.cjs b/dist/index.cjs -index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97e96c300b 100644 +index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a934360361 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -410,11 +410,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { @@ -69,7 +69,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -813,6 +834,7 @@ function drizzleStorage({ +@@ -813,23 +834,29 @@ function drizzleStorage({ indexer$1.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -77,9 +77,17 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 const { endCursor, finality, cursor } = context; if (!endCursor) { throw new DrizzleStorageError("End Cursor is undefined"); -@@ -823,13 +845,16 @@ function drizzleStorage({ - await invalidate(tx, cursor, idColumnMap, indexerId); - } + } ++ if (prevFinality === "pending") { ++ await withTransaction(db, async (tx) => { ++ await invalidate(tx, cursor, idColumnMap, indexerId); ++ }); ++ } + await withTransaction(db, async (tx) => { + context[constants.DRIZZLE_PROPERTY] = { db: tx }; +- if (prevFinality === "pending") { +- await invalidate(tx, cursor, idColumnMap, indexerId); +- } if (finality !== "finalized") { - await registerTriggers( - tx, @@ -101,7 +109,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 } await next(); delete context[constants.DRIZZLE_PROPERTY]; -@@ -842,11 +867,10 @@ function drizzleStorage({ +@@ -842,11 +869,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -116,7 +124,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..739a4a2974712d6737f16b6d1db4ce97 } }); diff --git a/dist/index.mjs b/dist/index.mjs -index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070ebdb18c4ab 100644 +index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288c32c4989 100644 --- a/dist/index.mjs +++ b/dist/index.mjs @@ -408,11 +408,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { @@ -186,7 +194,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -811,6 +832,7 @@ function drizzleStorage({ +@@ -811,23 +832,29 @@ function drizzleStorage({ indexer.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -194,9 +202,17 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb const { endCursor, finality, cursor } = context; if (!endCursor) { throw new DrizzleStorageError("End Cursor is undefined"); -@@ -821,13 +843,16 @@ function drizzleStorage({ - await invalidate(tx, cursor, idColumnMap, indexerId); - } + } ++ if (prevFinality === "pending") { ++ await withTransaction(db, async (tx) => { ++ await invalidate(tx, cursor, idColumnMap, indexerId); ++ }); ++ } + await withTransaction(db, async (tx) => { + context[DRIZZLE_PROPERTY] = { db: tx }; +- if (prevFinality === "pending") { +- await invalidate(tx, cursor, idColumnMap, indexerId); +- } if (finality !== "finalized") { - await registerTriggers( - tx, @@ -218,7 +234,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb } await next(); delete context[DRIZZLE_PROPERTY]; -@@ -840,11 +865,10 @@ function drizzleStorage({ +@@ -840,11 +867,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -233,7 +249,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..230cdc077e5be2dcd155b52d972070eb } }); diff --git a/src/index.ts b/src/index.ts -index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..6762c2cdb9ad114f74b0a37e7e2d317ae9d3b5cc 100644 +index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..9c3276659d86986db1fb20daa65bdd5e968bf8ca 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,7 +31,7 @@ import { @@ -261,9 +277,30 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..6762c2cdb9ad114f74b0a37e7e2d317a const { endCursor, finality, cursor } = context as { cursor: Cursor; endCursor: Cursor; -@@ -425,13 +427,16 @@ export function drizzleStorage< - } - +@@ -412,6 +414,14 @@ export function drizzleStorage< + throw new DrizzleStorageError("End Cursor is undefined"); + } + ++ if (prevFinality === "pending") { ++ await withTransaction(db, async (tx) => { ++ // Invalidate in a dedicated transaction so rollback writes are ++ // never captured with the current block's reorg_order_key. ++ await invalidate(tx, cursor, idColumnMap, indexerId); ++ }); ++ } ++ + await withTransaction(db, async (tx) => { + context[DRIZZLE_PROPERTY] = { db: tx } as DrizzleStorage< + TQueryResult, +@@ -419,19 +429,17 @@ export function drizzleStorage< + TSchema + >; + +- if (prevFinality === "pending") { +- // invalidate if previous block's finality was "pending" +- await invalidate(tx, cursor, idColumnMap, indexerId); +- } +- if (finality !== "finalized") { - await registerTriggers( - tx, @@ -283,12 +320,12 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..6762c2cdb9ad114f74b0a37e7e2d317a + } + await setReorgOrderKey(tx, endCursor); } - + await next(); -@@ -448,13 +453,11 @@ export function drizzleStorage< +@@ -448,13 +456,11 @@ export function drizzleStorage< prevFinality = finality; }); - + - if (finality !== "finalized") { - // remove trigger outside of the transaction or it won't be triggered. - await removeTriggers(db, tableNames, indexerId); @@ -347,7 +384,7 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb @@ -159,6 +165,28 @@ export async function registerTriggers< } } - + +export async function setReorgOrderKey< + TQueryResult extends PgQueryResultHKT, + TFullSchema extends Record = Record, diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 6d0bba70..f5e3dd07 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: patchedDependencies: '@apibara/plugin-drizzle@2.1.0-beta.55': - hash: b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7 + hash: a9303b3dca5e07d6cecbceeb8fcefe9afdc4e614fbf1600188f68ae3b94c1c7a path: patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch importers: @@ -18,7 +18,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(patch_hash=a9303b3dca5e07d6cecbceeb8fcefe9afdc4e614fbf1600188f68ae3b94c1c7a)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -2524,7 +2524,7 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=b2aaf410780d4d72d35c46bbecbc23e794647d6e1c00b03d5831d9ef0f0528f7)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=a9303b3dca5e07d6cecbceeb8fcefe9afdc4e614fbf1600188f68ae3b94c1c7a)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) From dd8fe53e266f7d9fbac616550a56fe499a1d6162 Mon Sep 17 00:00:00 2001 From: loothero Date: Sun, 8 Mar 2026 14:12:29 -0700 Subject: [PATCH 25/39] chore: add Railway watch patterns for monorepo PR environments Each service now has a railway.json with watchPatterns so Railway's Focused PR Environments can determine which services are affected by file changes in a PR. Co-Authored-By: Claude Opus 4.6 --- api/railway.json | 6 ++++++ client/railway.json | 6 ++++++ indexer/railway.json | 3 +++ 3 files changed, 15 insertions(+) create mode 100644 api/railway.json create mode 100644 client/railway.json diff --git a/api/railway.json b/api/railway.json new file mode 100644 index 00000000..8765fe3f --- /dev/null +++ b/api/railway.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://railway.com/railway.schema.json", + "build": { + "watchPatterns": ["api/**"] + } +} diff --git a/client/railway.json b/client/railway.json new file mode 100644 index 00000000..d0504000 --- /dev/null +++ b/client/railway.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://railway.com/railway.schema.json", + "build": { + "watchPatterns": ["client/**"] + } +} diff --git a/indexer/railway.json b/indexer/railway.json index 5cedce74..4b373a21 100644 --- a/indexer/railway.json +++ b/indexer/railway.json @@ -1,5 +1,8 @@ { "$schema": "https://railway.com/railway.schema.json", + "build": { + "watchPatterns": ["indexer/**"] + }, "deploy": { "restartPolicyType": "ALWAYS", "restartPolicyMaxRetries": 10 From 347e49e4af129923ded72b1327f07762daf20e89 Mon Sep 17 00:00:00 2001 From: loothero Date: Sun, 8 Mar 2026 17:24:17 -0700 Subject: [PATCH 26/39] fix(api): keep /beasts/all total accurate on empty pages --- api/src/index.ts | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/api/src/index.ts b/api/src/index.ts index cdeee7b4..d17106e7 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -233,6 +233,20 @@ app.get("/beasts/all", async (c) => { data: Array>; pagination: { limit: number; offset: number; total: number | null; has_more: boolean }; }> => { + const loadTotalCount = async (): Promise => { + const countResult = owner + ? await db + .select({ count: sql`count(*)` }) + .from(beasts) + .innerJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) + .where(whereClause) + : await db + .select({ count: sql`count(*)` }) + .from(beasts) + .where(whereClause); + return Number(countResult[0]?.count ?? 0); + }; + const tokenRowsLimit = includeTotal ? limit : limit + 1; const tokenRows = sort === "level" ? await ( @@ -268,13 +282,14 @@ app.get("/beasts/all", async (c) => { const pageTokenIds = tokenRows.slice(0, limit).map((row) => row.token_id); if (pageTokenIds.length === 0) { + const total = includeTotal ? await loadTotalCount() : null; return { data: [], pagination: { limit, offset, - total: includeTotal ? 0 : null, - has_more: false, + total, + has_more: includeTotal ? offset < (total ?? 0) : hasMoreWithoutTotal, }, }; } @@ -312,17 +327,7 @@ app.get("/beasts/all", async (c) => { let total: number | null = null; if (includeTotal) { - const countResult = owner - ? await db - .select({ count: sql`count(*)` }) - .from(beasts) - .innerJoin(beast_owners, eq(beast_owners.token_id, beasts.token_id)) - .where(whereClause) - : await db - .select({ count: sql`count(*)` }) - .from(beasts) - .where(whereClause); - total = Number(countResult[0]?.count ?? 0); + total = await loadTotalCount(); } return { From d0f541d958e06f5337bd83f71033026bfb01ffbb Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 10:36:37 -0700 Subject: [PATCH 27/39] refactor: consolidate address utils, extract API constants, remove phantom debug endpoints - Create shared addressUtils.ts with normalizeAddress() and addressesEqual() - Replace 4 independent implementations across client with shared imports - Add REWARD_AMOUNT_SCALE and QUEST_REWARD_SCALE constants in API - Remove unimplemented debug endpoint discovery entries Co-Authored-By: Claude Opus 4.6 --- api/src/index.ts | 16 ++--- client/src/components/DiplomacyPopover.tsx | 14 +---- client/src/components/Leaderboard.jsx | 19 +----- client/src/contexts/GameDirector.tsx | 20 +------ client/src/utils/addressNameCache.ts | 19 +----- client/src/utils/addressUtils.test.ts | 70 ++++++++++++++++++++++ client/src/utils/addressUtils.ts | 34 +++++++++++ 7 files changed, 114 insertions(+), 78 deletions(-) create mode 100644 client/src/utils/addressUtils.test.ts create mode 100644 client/src/utils/addressUtils.ts diff --git a/api/src/index.ts b/api/src/index.ts index d17106e7..1b4572bf 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -44,7 +44,10 @@ import { shouldBypassCache, } from "./lib/cache.js"; -const isDevelopment = process.env.NODE_ENV !== "production"; +/** Reward amounts are stored as integers scaled by 1e5; divide to get display values */ +const REWARD_AMOUNT_SCALE = 100_000; +/** Quest reward amounts are stored as integers scaled by 1e2 */ +const QUEST_REWARD_SCALE = 100; const apiCache = new ApiResponseCache({ enabled: parseCacheEnabled(), maxEntries: parseMaxEntries(process.env.API_CACHE_MAX_ENTRIES), @@ -814,7 +817,7 @@ app.get("/leaderboard", async (c) => { return results.map((r) => ({ owner: r.owner, - amount: Number(r.amount) / 100000, + amount: Number(r.amount) / REWARD_AMOUNT_SCALE, })); }); }); @@ -828,7 +831,7 @@ app.get("/quest-rewards/total", async (c) => { .select({ total: sql`coalesce(sum(${quest_rewards_claimed.amount}), 0)` }) .from(quest_rewards_claimed); - return { total: Number(result[0]?.total ?? 0) / 100 }; + return { total: Number(result[0]?.total ?? 0) / QUEST_REWARD_SCALE }; }); }); @@ -906,13 +909,6 @@ app.get("/", (c) => { }, }; - if (isDevelopment) { - endpoints.debug = { - test_summit_update: "POST /debug/test-summit-update", - test_summit_log: "POST /debug/test-summit-log", - }; - } - return c.json({ name: "Summit API", version: "1.0.0", diff --git a/client/src/components/DiplomacyPopover.tsx b/client/src/components/DiplomacyPopover.tsx index b3304017..378352c4 100644 --- a/client/src/components/DiplomacyPopover.tsx +++ b/client/src/components/DiplomacyPopover.tsx @@ -3,7 +3,7 @@ import type { Beast, Diplomacy } from '@/types/game'; import { gameColors } from '@/utils/themes'; import HandshakeIcon from '@mui/icons-material/Handshake'; import { Box, Popover, Typography } from '@mui/material'; -import { addAddressPadding } from 'starknet'; +import { addressesEqual as sameAddress } from '@/utils/addressUtils'; interface DiplomacyPopoverProps { anchorEl: HTMLElement | null; @@ -14,18 +14,6 @@ interface DiplomacyPopoverProps { addressNames: Record; } -const sameAddress = (left: string | null | undefined, right: string | null | undefined): boolean => { - if (!left || !right) { - return false; - } - - try { - return addAddressPadding(left) === addAddressPadding(right); - } catch { - return false; - } -}; - export function DiplomacyPopover({ anchorEl, onClose, diff --git a/client/src/components/Leaderboard.jsx b/client/src/components/Leaderboard.jsx index 532e8b65..4db79691 100644 --- a/client/src/components/Leaderboard.jsx +++ b/client/src/components/Leaderboard.jsx @@ -13,26 +13,9 @@ import HandshakeIcon from '@mui/icons-material/Handshake'; import RefreshIcon from '@mui/icons-material/Refresh'; import { Box, IconButton, Typography } from '@mui/material'; import { useEffect, useRef, useState } from 'react'; -import { addAddressPadding } from 'starknet'; import { DiplomacyPopover } from './DiplomacyPopover'; import RewardsRemainingBar from './RewardsRemainingBar'; - -const normalizeAddress = (address) => - typeof address === 'string' && address.length > 0 - ? address.replace(/^0x0+/, '0x').toLowerCase() - : null; - -const sameAddress = (left, right) => { - if (typeof left !== 'string' || typeof right !== 'string' || !left || !right) { - return false; - } - - try { - return addAddressPadding(left) === addAddressPadding(right); - } catch { - return false; - } -}; +import { normalizeAddress, addressesEqual as sameAddress } from '@/utils/addressUtils'; function Leaderboard() { const { beastsRegistered, beastsAlive, consumablesSupply, fetchStats } = useStatistics() diff --git a/client/src/contexts/GameDirector.tsx b/client/src/contexts/GameDirector.tsx index 6d16ef41..8f85227e 100644 --- a/client/src/contexts/GameDirector.tsx +++ b/client/src/contexts/GameDirector.tsx @@ -11,6 +11,7 @@ import type { BattleEvent, Beast, GameAction, SpectatorBattleEvent, Summit } fro import { BEAST_NAMES, ITEM_NAME_PREFIXES, ITEM_NAME_SUFFIXES } from "@/utils/BeastData"; import { fetchBeastImage } from "@/utils/beasts"; import { lookupAddressName } from "@/utils/addressNameCache"; +import { normalizeAddress, addressesEqual } from "@/utils/addressUtils"; import type { BattleEventTranslation, LiveBeastStatsEventTranslation, @@ -70,25 +71,6 @@ const isSummitEvent = ( event: TranslatedGameEvent ): event is SummitEventTranslation => event.componentName === "Summit"; -function normalizeAddress(address: string | null | undefined): string | null { - if (typeof address !== "string") { - return null; - } - - const trimmed = address.trim(); - if (!trimmed) { - return null; - } - - return trimmed.replace(/^0x0+/, "0x").toLowerCase(); -} - -function addressesEqual(left: string | null | undefined, right: string | null | undefined): boolean { - const normalizedLeft = normalizeAddress(left); - const normalizedRight = normalizeAddress(right); - return normalizedLeft !== null && normalizedRight !== null && normalizedLeft === normalizedRight; -} - export const GameDirector = ({ children }: PropsWithChildren) => { const { account } = useAccount(); const { currentNetworkConfig } = useDynamicConnector(); diff --git a/client/src/utils/addressNameCache.ts b/client/src/utils/addressNameCache.ts index 232f13bf..3b780a62 100644 --- a/client/src/utils/addressNameCache.ts +++ b/client/src/utils/addressNameCache.ts @@ -1,4 +1,5 @@ import { lookupAddresses } from '@cartridge/controller'; +import { normalizeAddress, type MaybeAddress } from './addressUtils'; const CACHE_KEY = 'addressNameCache'; @@ -9,24 +10,6 @@ interface AddressNameCache { [normalizedAddress: string]: string | null; } -type MaybeAddress = string | null | undefined; - -/** - * Normalizes an address to ensure consistent cache keys - */ -function normalizeAddress(address: MaybeAddress): string | null { - if (typeof address !== "string") { - return null; - } - - const trimmed = address.trim(); - if (!trimmed) { - return null; - } - - return trimmed.replace(/^0x0+/, "0x").toLowerCase(); -} - /** * Gets the cache from localStorage */ diff --git a/client/src/utils/addressUtils.test.ts b/client/src/utils/addressUtils.test.ts new file mode 100644 index 00000000..7cc857e7 --- /dev/null +++ b/client/src/utils/addressUtils.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; +import { normalizeAddress, addressesEqual } from './addressUtils'; + +describe('normalizeAddress', () => { + it('strips leading zeros', () => { + expect(normalizeAddress('0x00abc')).toBe('0xabc'); + }); + + it('lowercases hex', () => { + expect(normalizeAddress('0x00ABC')).toBe('0xabc'); + }); + + it('preserves 0x prefix with single zero stripped', () => { + expect(normalizeAddress('0x0')).toBe('0x'); + }); + + it('returns null for null/undefined/empty', () => { + expect(normalizeAddress(null)).toBeNull(); + expect(normalizeAddress(undefined)).toBeNull(); + expect(normalizeAddress('')).toBeNull(); + expect(normalizeAddress(' ')).toBeNull(); + }); + + it('trims whitespace', () => { + expect(normalizeAddress(' 0x00abc ')).toBe('0xabc'); + }); + + it('handles already-normalized addresses', () => { + expect(normalizeAddress('0xabc')).toBe('0xabc'); + }); + + it('handles full 66-char padded address', () => { + const padded = '0x0000000000000000000000000000000000000000000000000000000000000abc'; + expect(normalizeAddress(padded)).toBe('0xabc'); + }); +}); + +describe('addressesEqual', () => { + it('matches equivalent addresses with different padding', () => { + expect(addressesEqual('0xabc', '0x0000000000000000000000000000000000000000000000000000000000000abc')).toBe(true); + }); + + it('matches identical addresses', () => { + expect(addressesEqual('0xabc', '0xabc')).toBe(true); + }); + + it('is case-insensitive via padding', () => { + expect(addressesEqual('0xABC', '0xabc')).toBe(true); + }); + + it('returns false for null/undefined', () => { + expect(addressesEqual(null, '0xabc')).toBe(false); + expect(addressesEqual('0xabc', null)).toBe(false); + expect(addressesEqual(null, null)).toBe(false); + expect(addressesEqual(undefined, undefined)).toBe(false); + }); + + it('returns false for empty strings', () => { + expect(addressesEqual('', '0xabc')).toBe(false); + expect(addressesEqual('0xabc', '')).toBe(false); + }); + + it('returns false for different addresses', () => { + expect(addressesEqual('0xabc', '0xdef')).toBe(false); + }); + + it('handles malformed input gracefully', () => { + expect(addressesEqual('not-an-address', '0xabc')).toBe(false); + }); +}); diff --git a/client/src/utils/addressUtils.ts b/client/src/utils/addressUtils.ts new file mode 100644 index 00000000..40cc3336 --- /dev/null +++ b/client/src/utils/addressUtils.ts @@ -0,0 +1,34 @@ +import { addAddressPadding } from 'starknet'; + +export type MaybeAddress = string | null | undefined; + +/** + * Normalizes an address by stripping leading zeros (for cache keys and display). + */ +export function normalizeAddress(address: MaybeAddress): string | null { + if (typeof address !== 'string') { + return null; + } + + const trimmed = address.trim(); + if (!trimmed) { + return null; + } + + return trimmed.replace(/^0x0+/, '0x').toLowerCase(); +} + +/** + * Compares two addresses for equality using zero-padded form. + */ +export function addressesEqual(left: MaybeAddress, right: MaybeAddress): boolean { + if (!left || !right) { + return false; + } + + try { + return addAddressPadding(left) === addAddressPadding(right); + } catch { + return false; + } +} From 93f5dc34319bd77f4e55bd18e79a47e46e85043f Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 10:36:42 -0700 Subject: [PATCH 28/39] fix(client): add missing mock fields in ActionBar and Leaderboard tests ActionBar mock was missing ignoredPlayers, maxBeastsPerAttack, and skipSharedDiplomacy. Leaderboard mock was missing consumablesSupply and had stale fetchBeastCounts instead of fetchStats. Co-Authored-By: Claude Opus 4.6 --- client/src/components/ActionBar.test.tsx | 3 +++ client/src/components/Leaderboard.test.tsx | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/components/ActionBar.test.tsx b/client/src/components/ActionBar.test.tsx index b968c026..75739c13 100644 --- a/client/src/components/ActionBar.test.tsx +++ b/client/src/components/ActionBar.test.tsx @@ -56,6 +56,9 @@ const mockAutopilotState = { poisonConservativeExtraLivesTrigger: 0, poisonConservativeAmount: 0, poisonAggressiveAmount: 0, + maxBeastsPerAttack: 10, + skipSharedDiplomacy: false, + ignoredPlayers: [] as Array<{ address: string }>, }; vi.mock("@/contexts/controller", () => ({ diff --git a/client/src/components/Leaderboard.test.tsx b/client/src/components/Leaderboard.test.tsx index 5a40ea61..e62d7558 100644 --- a/client/src/components/Leaderboard.test.tsx +++ b/client/src/components/Leaderboard.test.tsx @@ -17,7 +17,8 @@ vi.mock("@/contexts/Statistics", () => ({ useStatistics: () => ({ beastsRegistered: 10, beastsAlive: 3, - fetchBeastCounts: hoisted.fetchBeastCountsMock, + consumablesSupply: { attack: 0, revive: 0, xlife: 0, poison: 0 }, + fetchStats: hoisted.fetchBeastCountsMock, }), })); From 0b2b148994884621fa9958696193f4590a7573bd Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 11:27:26 -0700 Subject: [PATCH 29/39] fix(indexer): include .mjs scripts in ESLint node globals config The railway-metrics-summary.mjs script was failing lint:ci because the ESLint config only covered scripts/**/*.ts, missing .mjs files. Co-Authored-By: Claude Opus 4.6 --- indexer/eslint.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/indexer/eslint.config.js b/indexer/eslint.config.js index d210bc87..964a00e2 100644 --- a/indexer/eslint.config.js +++ b/indexer/eslint.config.js @@ -69,6 +69,7 @@ export default tseslint.config( "drizzle.config.ts", "vitest.config.ts", "scripts/**/*.ts", + "scripts/**/*.mjs", ], languageOptions: { globals: { From be016156295c25fa988c6722dd02dbc402d9e7f9 Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 11:27:41 -0700 Subject: [PATCH 30/39] fix(client): remove unused normalizeAddress import in GameDirector Co-Authored-By: Claude Opus 4.6 --- client/src/contexts/GameDirector.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/contexts/GameDirector.tsx b/client/src/contexts/GameDirector.tsx index 8f85227e..d7ee9e27 100644 --- a/client/src/contexts/GameDirector.tsx +++ b/client/src/contexts/GameDirector.tsx @@ -11,7 +11,7 @@ import type { BattleEvent, Beast, GameAction, SpectatorBattleEvent, Summit } fro import { BEAST_NAMES, ITEM_NAME_PREFIXES, ITEM_NAME_SUFFIXES } from "@/utils/BeastData"; import { fetchBeastImage } from "@/utils/beasts"; import { lookupAddressName } from "@/utils/addressNameCache"; -import { normalizeAddress, addressesEqual } from "@/utils/addressUtils"; +import { addressesEqual } from "@/utils/addressUtils"; import type { BattleEventTranslation, LiveBeastStatsEventTranslation, From 5be347fe8acbc27a7495df54d01f9aad2eabb21c Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 12:44:57 -0700 Subject: [PATCH 31/39] fix(telemetry): harden metrics intervals, empty-block tracking, and WS error sampling - Validate METRICS_INTERVAL_MS and DB_METRICS_INTERVAL_MS with safeInterval() to prevent NaN/non-positive values creating a tight-loop timer (api + indexer) - Update perfState on empty-block early-return path so metrics report actual indexer progress during event-free stretches - Fix WS send-error sampling predicate (=== 1 -> === 0) so WS_SEND_ERROR_SAMPLE_EVERY=1 correctly logs every error Co-Authored-By: Claude Opus 4.6 --- api/src/lib/metrics.ts | 9 +++++++-- api/src/ws/subscriptions.ts | 2 +- indexer/indexers/summit.indexer.ts | 2 ++ indexer/src/lib/metrics.ts | 9 +++++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/api/src/lib/metrics.ts b/api/src/lib/metrics.ts index ce83ae79..8558fb34 100644 --- a/api/src/lib/metrics.ts +++ b/api/src/lib/metrics.ts @@ -33,9 +33,14 @@ export function isMetricsEnabled(): boolean { return !(raw === "0" || raw === "false" || raw === "off" || raw === "no"); } +function safeInterval(envValue: string | undefined, optionValue: number | undefined, fallback: number): number { + const raw = Number(envValue || optionValue || fallback); + return Number.isFinite(raw) && raw > 0 ? raw : fallback; +} + export function startResourceMetrics(options: ResourceMetricsOptions): { stop: () => void } { - const intervalMs = Number(process.env.METRICS_INTERVAL_MS || options.intervalMs || 30_000); - const dbProbeIntervalMs = Number(process.env.DB_METRICS_INTERVAL_MS || options.dbProbeIntervalMs || 60_000); + const intervalMs = safeInterval(process.env.METRICS_INTERVAL_MS, options.intervalMs, 30_000); + const dbProbeIntervalMs = safeInterval(process.env.DB_METRICS_INTERVAL_MS, options.dbProbeIntervalMs, 60_000); const log = options.log ?? console.log; let inFlight = false; diff --git a/api/src/ws/subscriptions.ts b/api/src/ws/subscriptions.ts index 20c681f9..4d539c56 100644 --- a/api/src/ws/subscriptions.ts +++ b/api/src/ws/subscriptions.ts @@ -311,7 +311,7 @@ export class SubscriptionHub { return true; } catch (error) { this.bumpCounter("sendErrors"); - if (this.windowCounters.sendErrors % this.sendErrorSampleEvery === 1) { + if (this.windowCounters.sendErrors % this.sendErrorSampleEvery === 0) { log.warn("ws_send_failed_sampled", { error, window_send_errors: this.windowCounters.sendErrors, diff --git a/indexer/indexers/summit.indexer.ts b/indexer/indexers/summit.indexer.ts index 438c83e5..1e2ec7b1 100644 --- a/indexer/indexers/summit.indexer.ts +++ b/indexer/indexers/summit.indexer.ts @@ -1109,6 +1109,8 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { lastProgressLog = now; blocksWithoutEvents = 0; } + perfState.last_block_number = block_number.toString(); + perfState.blocks_without_events = blocksWithoutEvents; return; // Skip processing for empty blocks } diff --git a/indexer/src/lib/metrics.ts b/indexer/src/lib/metrics.ts index ce83ae79..8558fb34 100644 --- a/indexer/src/lib/metrics.ts +++ b/indexer/src/lib/metrics.ts @@ -33,9 +33,14 @@ export function isMetricsEnabled(): boolean { return !(raw === "0" || raw === "false" || raw === "off" || raw === "no"); } +function safeInterval(envValue: string | undefined, optionValue: number | undefined, fallback: number): number { + const raw = Number(envValue || optionValue || fallback); + return Number.isFinite(raw) && raw > 0 ? raw : fallback; +} + export function startResourceMetrics(options: ResourceMetricsOptions): { stop: () => void } { - const intervalMs = Number(process.env.METRICS_INTERVAL_MS || options.intervalMs || 30_000); - const dbProbeIntervalMs = Number(process.env.DB_METRICS_INTERVAL_MS || options.dbProbeIntervalMs || 60_000); + const intervalMs = safeInterval(process.env.METRICS_INTERVAL_MS, options.intervalMs, 30_000); + const dbProbeIntervalMs = safeInterval(process.env.DB_METRICS_INTERVAL_MS, options.dbProbeIntervalMs, 60_000); const log = options.log ?? console.log; let inFlight = false; From 45e7f1ae841a8cc37c02e84321ed59ca48a9e65d Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 12:45:37 -0700 Subject: [PATCH 32/39] fix(client): normalize Leaderboard type and handle null owner addresses - Use shared Leaderboard type in summitApi, DiplomacyPopover, and LeaderboardModal - Make Leaderboard.owner nullable to match API responses for zero addresses - Return null from normalizeAddress for zero addresses (0x0, 0x000...) - Guard LeaderboardModal against null owner in name lookup, key, and formatAddress Co-Authored-By: Claude Opus 4.6 --- client/src/api/summitApi.ts | 4 ++-- client/src/components/DiplomacyPopover.tsx | 4 ++-- client/src/components/dialogs/LeaderboardModal.tsx | 6 +++--- client/src/types/game.ts | 2 +- client/src/utils/addressUtils.test.ts | 6 ++++-- client/src/utils/addressUtils.ts | 3 ++- client/tsconfig.tsbuildinfo | 2 +- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/client/src/api/summitApi.ts b/client/src/api/summitApi.ts index 6aa5f450..02d7dd12 100644 --- a/client/src/api/summitApi.ts +++ b/client/src/api/summitApi.ts @@ -3,7 +3,7 @@ */ import { useDynamicConnector } from "@/contexts/starknet"; -import type { Beast, DiplomacyBeast } from "@/types/game"; +import type { Beast, DiplomacyBeast, Leaderboard } from "@/types/game"; import { BEAST_NAMES, BEAST_TIERS, ITEM_NAME_PREFIXES, ITEM_NAME_SUFFIXES } from "@/utils/BeastData"; // Reverse lookup: name -> id @@ -195,7 +195,7 @@ export const useSummitApi = () => { /** * Get rewards leaderboard */ - const getLeaderboard = async (): Promise<{ owner: string; amount: number }[]> => { + const getLeaderboard = async (): Promise => { const response = await fetch(`${currentNetworkConfig.apiUrl}/leaderboard`); if (!response.ok) { throw new Error(`Failed to fetch leaderboard: ${response.status}`); diff --git a/client/src/components/DiplomacyPopover.tsx b/client/src/components/DiplomacyPopover.tsx index 378352c4..fbbb9794 100644 --- a/client/src/components/DiplomacyPopover.tsx +++ b/client/src/components/DiplomacyPopover.tsx @@ -1,5 +1,5 @@ import { DIPLOMACY_REWARDS_PER_SECOND } from '@/contexts/GameDirector'; -import type { Beast, Diplomacy } from '@/types/game'; +import type { Beast, Diplomacy, Leaderboard } from '@/types/game'; import { gameColors } from '@/utils/themes'; import HandshakeIcon from '@mui/icons-material/Handshake'; import { Box, Popover, Typography } from '@mui/material'; @@ -10,7 +10,7 @@ interface DiplomacyPopoverProps { onClose: () => void; diplomacy: Diplomacy; summitBeast: Beast; - leaderboard: { owner: string | null; amount: number }[]; + leaderboard: Leaderboard[]; addressNames: Record; } diff --git a/client/src/components/dialogs/LeaderboardModal.tsx b/client/src/components/dialogs/LeaderboardModal.tsx index cdaf5e6c..197d1223 100644 --- a/client/src/components/dialogs/LeaderboardModal.tsx +++ b/client/src/components/dialogs/LeaderboardModal.tsx @@ -201,9 +201,9 @@ export default function LeaderboardModal({ open, onClose }: LeaderboardModalProp ) : ( playerPagedItems.map((player, index) => { const globalRank = (playersPage - 1) * PAGE_SIZE + index + 1; - const name = addressNames[player.owner]; + const name = player.owner ? addressNames[player.owner] : null; return ( - + {globalRank}. {name ? ( @@ -220,7 +220,7 @@ export default function LeaderboardModal({ open, onClose }: LeaderboardModalProp ) : ( - {formatAddress(player.owner)} + {formatAddress(player.owner ?? '')} )} diff --git a/client/src/types/game.ts b/client/src/types/game.ts index 4f3a1418..b2e07e85 100644 --- a/client/src/types/game.ts +++ b/client/src/types/game.ts @@ -25,7 +25,7 @@ export interface Diplomacy { } export interface Leaderboard { - owner: string; + owner: string | null; amount: number; } diff --git a/client/src/utils/addressUtils.test.ts b/client/src/utils/addressUtils.test.ts index 7cc857e7..767248fe 100644 --- a/client/src/utils/addressUtils.test.ts +++ b/client/src/utils/addressUtils.test.ts @@ -10,8 +10,10 @@ describe('normalizeAddress', () => { expect(normalizeAddress('0x00ABC')).toBe('0xabc'); }); - it('preserves 0x prefix with single zero stripped', () => { - expect(normalizeAddress('0x0')).toBe('0x'); + it('returns null for zero addresses', () => { + expect(normalizeAddress('0x0')).toBeNull(); + expect(normalizeAddress('0x00')).toBeNull(); + expect(normalizeAddress('0x000000')).toBeNull(); }); it('returns null for null/undefined/empty', () => { diff --git a/client/src/utils/addressUtils.ts b/client/src/utils/addressUtils.ts index 40cc3336..f2500ac9 100644 --- a/client/src/utils/addressUtils.ts +++ b/client/src/utils/addressUtils.ts @@ -15,7 +15,8 @@ export function normalizeAddress(address: MaybeAddress): string | null { return null; } - return trimmed.replace(/^0x0+/, '0x').toLowerCase(); + const result = trimmed.replace(/^0x0+/, '0x').toLowerCase(); + return result === '0x' ? null : result; } /** diff --git a/client/tsconfig.tsbuildinfo b/client/tsconfig.tsbuildinfo index c9ae3037..63629e7b 100644 --- a/client/tsconfig.tsbuildinfo +++ b/client/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/App.tsx","./src/main.tsx","./src/patchCoverage.imports.test.ts","./src/vite-env.d.ts","./src/api/ekubo.test.ts","./src/api/ekubo.ts","./src/api/starknet.test.ts","./src/api/starknet.ts","./src/api/summitApi.test.ts","./src/api/summitApi.ts","./src/components/ActionBar.test.tsx","./src/components/ActionBar.tsx","./src/components/AttackingBeasts.tsx","./src/components/BeastCard.tsx","./src/components/BeastCollection.tsx","./src/components/BeastProfile.tsx","./src/components/BurgerMenu.tsx","./src/components/ClaimRewardsButton.tsx","./src/components/Countdown.tsx","./src/components/DiplomacyPopover.tsx","./src/components/EventHistoryButton.tsx","./src/components/FinalShowdown.tsx","./src/components/GameNotificationFeed.tsx","./src/components/Icons.tsx","./src/components/KilledByAdventurers.tsx","./src/components/Leaderboard.jsx","./src/components/Leaderboard.test.tsx","./src/components/LeaderboardButton.tsx","./src/components/Migrating.tsx","./src/components/ProfileCard.tsx","./src/components/QuestBoard.tsx","./src/components/QuestRewardsRemainingBar.tsx","./src/components/RewardsRemainingBar.tsx","./src/components/Summit.tsx","./src/components/TermsOfServiceModal.tsx","./src/components/dialogs/AutopilotConfigModal.tsx","./src/components/dialogs/BeastDexModal.tsx","./src/components/dialogs/BeastUpgradeModal.tsx","./src/components/dialogs/ConnectWallet.tsx","./src/components/dialogs/DCATab.tsx","./src/components/dialogs/EventHistoryModal.tsx","./src/components/dialogs/LeaderboardModal.tsx","./src/components/dialogs/MarketplaceModal.tsx","./src/components/dialogs/QuestsModal.tsx","./src/components/dialogs/SummitGiftModal.tsx","./src/components/dialogs/TopUpStrkModal.tsx","./src/contexts/GameDirector.test.tsx","./src/contexts/GameDirector.tsx","./src/contexts/QuestGuide.tsx","./src/contexts/Statistics.test.tsx","./src/contexts/Statistics.tsx","./src/contexts/controller.test.tsx","./src/contexts/controller.tsx","./src/contexts/sound.tsx","./src/contexts/starknet.tsx","./src/dojo/useGameTokens.test.tsx","./src/dojo/useGameTokens.ts","./src/dojo/useSystemCalls.ts","./src/hooks/useWebSocket.ts","./src/pages/MainPage.tsx","./src/stores/autopilotStore.ts","./src/stores/gameStore.ts","./src/types/game.ts","./src/utils/BeastData.ts","./src/utils/addressNameCache.ts","./src/utils/analytics.ts","./src/utils/beasts.test.ts","./src/utils/beasts.ts","./src/utils/events.test.ts","./src/utils/events.ts","./src/utils/networkConfig.ts","./src/utils/styles.ts","./src/utils/summitRewards.ts","./src/utils/themes.ts","./src/utils/translation.test.ts","./src/utils/translation.ts","./src/utils/utils.test.ts","./src/utils/utils.ts","./src/utils/variants.test.ts","./src/utils/variants.ts"],"errors":true,"version":"5.9.3"} \ No newline at end of file +{"root":["./src/App.tsx","./src/main.tsx","./src/patchCoverage.imports.test.ts","./src/vite-env.d.ts","./src/api/ekubo.test.ts","./src/api/ekubo.ts","./src/api/starknet.test.ts","./src/api/starknet.ts","./src/api/summitApi.test.ts","./src/api/summitApi.ts","./src/components/ActionBar.test.tsx","./src/components/ActionBar.tsx","./src/components/AttackingBeasts.tsx","./src/components/BeastCard.tsx","./src/components/BeastCollection.tsx","./src/components/BeastProfile.tsx","./src/components/BurgerMenu.tsx","./src/components/ClaimRewardsButton.tsx","./src/components/Countdown.tsx","./src/components/DiplomacyPopover.tsx","./src/components/EventHistoryButton.tsx","./src/components/FinalShowdown.tsx","./src/components/GameNotificationFeed.tsx","./src/components/Icons.tsx","./src/components/KilledByAdventurers.tsx","./src/components/Leaderboard.jsx","./src/components/Leaderboard.test.tsx","./src/components/LeaderboardButton.tsx","./src/components/Migrating.tsx","./src/components/ProfileCard.tsx","./src/components/QuestBoard.tsx","./src/components/QuestRewardsRemainingBar.tsx","./src/components/RewardsRemainingBar.tsx","./src/components/Summit.tsx","./src/components/TermsOfServiceModal.tsx","./src/components/dialogs/AutopilotConfigModal.tsx","./src/components/dialogs/BeastDexModal.tsx","./src/components/dialogs/BeastUpgradeModal.tsx","./src/components/dialogs/ConnectWallet.tsx","./src/components/dialogs/DCATab.tsx","./src/components/dialogs/EventHistoryModal.tsx","./src/components/dialogs/LeaderboardModal.tsx","./src/components/dialogs/MarketplaceModal.tsx","./src/components/dialogs/QuestsModal.tsx","./src/components/dialogs/SummitGiftModal.tsx","./src/components/dialogs/TopUpStrkModal.tsx","./src/contexts/GameDirector.test.tsx","./src/contexts/GameDirector.tsx","./src/contexts/QuestGuide.tsx","./src/contexts/Statistics.test.tsx","./src/contexts/Statistics.tsx","./src/contexts/controller.test.tsx","./src/contexts/controller.tsx","./src/contexts/sound.tsx","./src/contexts/starknet.tsx","./src/dojo/useGameTokens.test.tsx","./src/dojo/useGameTokens.ts","./src/dojo/useSystemCalls.ts","./src/hooks/useWebSocket.ts","./src/pages/MainPage.tsx","./src/stores/autopilotStore.ts","./src/stores/gameStore.ts","./src/types/game.ts","./src/utils/BeastData.ts","./src/utils/addressNameCache.ts","./src/utils/addressUtils.test.ts","./src/utils/addressUtils.ts","./src/utils/analytics.ts","./src/utils/beasts.test.ts","./src/utils/beasts.ts","./src/utils/events.test.ts","./src/utils/events.ts","./src/utils/networkConfig.ts","./src/utils/styles.ts","./src/utils/summitRewards.ts","./src/utils/themes.ts","./src/utils/translation.test.ts","./src/utils/translation.ts","./src/utils/utils.test.ts","./src/utils/utils.ts","./src/utils/variants.test.ts","./src/utils/variants.ts"],"version":"5.9.3"} \ No newline at end of file From 526ffa132e02c97176cf944b49c90b624d63dacb Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 13:27:55 -0700 Subject: [PATCH 33/39] fix(indexer): make reorg trigger registration idempotent across restarts Add DROP TRIGGER IF EXISTS before CREATE CONSTRAINT TRIGGER in the drizzle plugin patch so that triggers from a previous process run don't cause duplicate-trigger errors on restart/redeploy. Co-Authored-By: Claude Opus 4.6 --- ...pibara__plugin-drizzle@2.1.0-beta.55.patch | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch index 4a56da80..0cfd2f85 100644 --- a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch +++ b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch @@ -32,7 +32,8 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -453,7 +460,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -453,7 +460,8 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { ++ DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table}; CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -41,7 +42,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 `) ); } -@@ -463,6 +470,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -463,6 +471,19 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { }); } } @@ -61,7 +62,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 async function removeTriggers(db, tables, indexerId) { try { for (const table of tables) { -@@ -653,6 +673,7 @@ function drizzleStorage({ +@@ -653,6 +674,7 @@ function drizzleStorage({ let indexerId = ""; const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; let prevFinality; @@ -69,7 +70,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -813,23 +834,29 @@ function drizzleStorage({ +@@ -813,23 +835,29 @@ function drizzleStorage({ indexer$1.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -109,7 +110,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 } await next(); delete context[constants.DRIZZLE_PROPERTY]; -@@ -842,11 +869,10 @@ function drizzleStorage({ +@@ -842,11 +870,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -157,7 +158,8 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -451,7 +458,7 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -451,7 +458,8 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { ++ DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table}; CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -166,7 +168,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 `) ); } -@@ -461,6 +468,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { +@@ -461,6 +469,19 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { }); } } @@ -186,7 +188,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 async function removeTriggers(db, tables, indexerId) { try { for (const table of tables) { -@@ -651,6 +671,7 @@ function drizzleStorage({ +@@ -651,6 +672,7 @@ function drizzleStorage({ let indexerId = ""; const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true"; let prevFinality; @@ -194,7 +196,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -811,23 +832,29 @@ function drizzleStorage({ +@@ -811,23 +833,29 @@ function drizzleStorage({ indexer.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -234,7 +236,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 } await next(); delete context[DRIZZLE_PROPERTY]; -@@ -840,11 +867,10 @@ function drizzleStorage({ +@@ -840,11 +868,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -372,7 +374,8 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb idColumnMap: IdColumnMap, indexerId: string, ) { -@@ -148,7 +154,7 @@ export async function registerTriggers< +@@ -148,7 +154,8 @@ export async function registerTriggers< ++ DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table}; CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} DEFERRABLE INITIALLY DEFERRED @@ -381,7 +384,7 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb `), ); } -@@ -159,6 +165,28 @@ export async function registerTriggers< +@@ -159,6 +166,28 @@ export async function registerTriggers< } } From 13ce183a1c3af420388bd2dd813b60a632e2743a Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 13:51:00 -0700 Subject: [PATCH 34/39] fix(api): exclude /beasts/:owner from response cache The endpoint derives current_health from Date.now() for revival window checks, so caching serves stale alive/dead state across the boundary. Also regenerate indexer lockfile to match updated patch hash. Co-Authored-By: Claude Opus 4.6 --- api/src/index.ts | 11 ++++++----- indexer/pnpm-lock.yaml | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/src/index.ts b/api/src/index.ts index 1b4572bf..54f41bda 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -55,7 +55,6 @@ const apiCache = new ApiResponseCache({ const CACHE_POLICIES: Record< | "beastsAll" - | "beastsByOwner" | "logs" | "beastsStatsCounts" | "beastsStatsTop" @@ -67,7 +66,6 @@ const CACHE_POLICIES: Record< CachePolicy > = { beastsAll: { freshTtlMs: 2_000, staleTtlMs: 8_000 }, - beastsByOwner: { freshTtlMs: 3_000, staleTtlMs: 12_000 }, logs: { freshTtlMs: 2_000, staleTtlMs: 8_000 }, beastsStatsCounts: { freshTtlMs: 5_000, staleTtlMs: 20_000 }, beastsStatsTop: { freshTtlMs: 3_000, staleTtlMs: 12_000 }, @@ -391,7 +389,9 @@ app.get("/beasts/all", async (c) => { app.get("/beasts/:owner", async (c) => { const owner = normalizeAddress(c.req.param("owner")); - return respondWithCachedJson(c, CACHE_POLICIES.beastsByOwner, async () => { + // Not cached: current_health is derived from Date.now() (revival window), + // so caching would serve stale alive/dead state across the boundary. + { // Get beast data with all joins including skulls const results = await db .select({ @@ -443,7 +443,7 @@ app.get("/beasts/:owner", async (c) => { .where(eq(beast_owners.owner, owner)); // Transform to Beast interface format - return results.map((r) => { + const result = results.map((r) => { const beastId = r.beast_id; const prefixId = r.prefix; const suffixId = r.suffix; @@ -522,7 +522,8 @@ app.get("/beasts/:owner", async (c) => { entity_hash: r.entity_hash ?? undefined, }; }); - }); + return c.json(result); + } }); /** diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 029c8f29..8d7af51f 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -11,7 +11,7 @@ overrides: patchedDependencies: '@apibara/plugin-drizzle@2.1.0-beta.55': - hash: a9303b3dca5e07d6cecbceeb8fcefe9afdc4e614fbf1600188f68ae3b94c1c7a + hash: 8463299f597bed8be89eeb2ae585fe38b310ee323309ad2c4d76959aa6255556 path: patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch importers: @@ -23,7 +23,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(patch_hash=a9303b3dca5e07d6cecbceeb8fcefe9afdc4e614fbf1600188f68ae3b94c1c7a)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(patch_hash=8463299f597bed8be89eeb2ae585fe38b310ee323309ad2c4d76959aa6255556)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -2516,7 +2516,7 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=a9303b3dca5e07d6cecbceeb8fcefe9afdc4e614fbf1600188f68ae3b94c1c7a)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=8463299f597bed8be89eeb2ae585fe38b310ee323309ad2c4d76959aa6255556)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) From 1dc8969261598d7690f7c3a0f05893f7d72abe85 Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 15:28:22 -0700 Subject: [PATCH 35/39] docs(api): fix README to match runtime behavior - Remove /beasts/:owner from cached-endpoints list (now uncached) - Update DATABASE_SSL docs: defaults to enabled with warning, not fail-fast Co-Authored-By: Claude Opus 4.6 --- api/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/README.md b/api/README.md index 2a502b96..cbce0fce 100644 --- a/api/README.md +++ b/api/README.md @@ -24,7 +24,7 @@ For AI-oriented coding guidance and deeper architecture notes, read `AGENTS.md` ## Environment - `DATABASE_URL` (required) -- `DATABASE_SSL` (`"true"` or `"false"`; required in production) +- `DATABASE_SSL` (`"true"` or `"false"`; defaults to `"true"` in production with a warning when unset) - `DB_POOL_MAX` (default `15`) - `PORT` (default `3001`) - `NODE_ENV` (`production` hides debug entries from `/` discovery payload) @@ -32,7 +32,7 @@ For AI-oriented coding guidance and deeper architecture notes, read `AGENTS.md` - `API_CACHE_MAX_ENTRIES` (optional; default `500`) Production note: -- API startup fails fast when `NODE_ENV=production` and `DATABASE_SSL` is unset. +- When `NODE_ENV=production` and `DATABASE_SSL` is unset, SSL defaults to enabled and a warning is logged. ## Quick Start @@ -164,7 +164,6 @@ Realtime pipeline: - API is public read-only (no auth layer). - A thin in-memory SWR cache is applied to high-traffic read endpoints: - `/beasts/all` (common public list patterns) - - `/beasts/:owner` - `/logs` - `/beasts/stats/counts` - `/beasts/stats/top` From 2ff2c978d68637c5dd3db0c5f32b2003fec806a5 Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 18:31:16 -0700 Subject: [PATCH 36/39] fix(api): make /beasts/all ordering deterministic with NULLS LAST - Add NULLS LAST to summit_held_seconds DESC so beasts without stats don't outrank real holders on the default sort - Add token_id tie-breaker to both sort branches to prevent duplicate/skipped rows across OFFSET pages Co-Authored-By: Claude Opus 4.6 --- api/src/index.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/src/index.ts b/api/src/index.ts index 54f41bda..ffc40f08 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -259,7 +259,7 @@ app.get("/beasts/all", async (c) => { : db.select({ token_id: beasts.token_id }).from(beasts) ) .where(whereClause) - .orderBy(desc(beasts.level)) + .orderBy(desc(beasts.level), desc(beasts.token_id)) .limit(tokenRowsLimit) .offset(offset) : await ( @@ -275,7 +275,10 @@ app.get("/beasts/all", async (c) => { .leftJoin(beast_stats, eq(beast_stats.token_id, beasts.token_id)) ) .where(whereClause) - .orderBy(desc(beast_stats.summit_held_seconds)) + .orderBy( + sql`${beast_stats.summit_held_seconds} DESC NULLS LAST`, + desc(beasts.token_id) + ) .limit(tokenRowsLimit) .offset(offset); From 0651feb379ad72fdd6c088d252eed57da013ee2a Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 18:41:58 -0700 Subject: [PATCH 37/39] fix(indexer): remove SIGINT/SIGTERM metric handlers that block graceful shutdown The signal listeners replaced Node's default termination path but never called process.exit(), causing the indexer to ignore shutdown signals on Railway. The beforeExit handler is sufficient since metric timers are unref'd and won't keep the process alive. Co-Authored-By: Claude Opus 4.6 --- indexer/indexers/summit.indexer.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/indexer/indexers/summit.indexer.ts b/indexer/indexers/summit.indexer.ts index 1e2ec7b1..cb60c836 100644 --- a/indexer/indexers/summit.indexer.ts +++ b/indexer/indexers/summit.indexer.ts @@ -933,8 +933,6 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { } }; process.once("beforeExit", stopMetricEmitters); - process.once("SIGINT", stopMetricEmitters); - process.once("SIGTERM", stopMetricEmitters); // getBeast selector: starknet_keccak("getBeast") const GET_BEAST_SELECTOR = "0x0385b69551f247794fe651459651cdabc76b6cdf4abacafb5b28ceb3b1ac2e98"; From ad7c9cf0ebce27203758c1cc5eaa02e0f1723fc2 Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 19:01:24 -0700 Subject: [PATCH 38/39] docs(indexer): document split-transaction tradeoff in drizzle patch The invalidation and replacement run in separate transactions to prevent trigger-captured rollback entries from corrupting the reorg table. If the replacement transaction fails, readers see stale state until process restart. Document this tradeoff in all 3 patch targets. Co-Authored-By: Claude Opus 4.6 --- ...pibara__plugin-drizzle@2.1.0-beta.55.patch | 67 +++++++++++++------ indexer/pnpm-lock.yaml | 6 +- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch index 0cfd2f85..660d1a15 100644 --- a/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch +++ b/indexer/patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch @@ -1,5 +1,5 @@ diff --git a/dist/index.cjs b/dist/index.cjs -index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a934360361 100644 +index 4926b17a6418da810dff724625b8c4a846944b47..7ff74d6813f8ce058c6bb207a97bf0b6ec45da15 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -410,11 +410,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { @@ -32,7 +32,10 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -453,7 +460,8 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { +@@ -450,10 +457,11 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { + ); + await tx.execute( + drizzleOrm.sql.raw(` + DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table}; CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} @@ -42,7 +45,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 `) ); } -@@ -463,6 +471,19 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { +@@ -463,6 +471,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { }); } } @@ -70,7 +73,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -813,23 +835,29 @@ function drizzleStorage({ +@@ -813,23 +835,35 @@ function drizzleStorage({ indexer$1.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -81,6 +84,12 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 } + if (prevFinality === "pending") { + await withTransaction(db, async (tx) => { ++ // Invalidate in a dedicated transaction so rollback writes are ++ // never captured with the current block's reorg_order_key. ++ // TRADEOFF: If the replacement transaction (below) fails after this ++ // commits, readers see state with the tip removed until process restart. ++ // Apibara does not retry handler errors, so recovery depends on Railway's ++ // ALWAYS restart policy re-indexing from the last checkpoint. + await invalidate(tx, cursor, idColumnMap, indexerId); + }); + } @@ -110,7 +119,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 } await next(); delete context[constants.DRIZZLE_PROPERTY]; -@@ -842,11 +870,10 @@ function drizzleStorage({ +@@ -842,11 +876,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -125,7 +134,7 @@ index 4926b17a6418da810dff724625b8c4a846944b47..81bf8e7c2c80d2ba56318ce0c7b3f4a9 } }); diff --git a/dist/index.mjs b/dist/index.mjs -index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288c32c4989 100644 +index 1bdf312081394c65988b248952fef093ec89e812..e26e503c70b7a9f87dd9620d27dacaf686bf69d4 100644 --- a/dist/index.mjs +++ b/dist/index.mjs @@ -408,11 +408,18 @@ async function initializeReorgRollbackTable(tx, indexerId) { @@ -158,7 +167,10 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 try { for (const table of tables) { const tableIdColumn = getIdColumnForTable(table, idColumnMap); -@@ -451,7 +458,8 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { +@@ -448,10 +455,11 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { + ); + await tx.execute( + sql.raw(` + DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table}; CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} @@ -168,7 +180,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 `) ); } -@@ -461,6 +469,19 @@ async function registerTriggers(tx, tables, idColumnMap, indexerId) { +@@ -461,6 +469,19 @@ async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) { }); } } @@ -196,7 +208,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 const schema = _schema ?? db._.schema ?? {}; const idColumnMap = { "*": typeof idColumn === "string" ? idColumn : "id", -@@ -811,23 +833,29 @@ function drizzleStorage({ +@@ -811,23 +833,35 @@ function drizzleStorage({ indexer.hooks.hook("handler:middleware", async ({ use }) => { use(async (context, next) => { try { @@ -207,6 +219,12 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 } + if (prevFinality === "pending") { + await withTransaction(db, async (tx) => { ++ // Invalidate in a dedicated transaction so rollback writes are ++ // never captured with the current block's reorg_order_key. ++ // TRADEOFF: If the replacement transaction (below) fails after this ++ // commits, readers see state with the tip removed until process restart. ++ // Apibara does not retry handler errors, so recovery depends on Railway's ++ // ALWAYS restart policy re-indexing from the last checkpoint. + await invalidate(tx, cursor, idColumnMap, indexerId); + }); + } @@ -236,7 +254,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 } await next(); delete context[DRIZZLE_PROPERTY]; -@@ -840,11 +868,10 @@ function drizzleStorage({ +@@ -840,11 +874,10 @@ function drizzleStorage({ } prevFinality = finality; }); @@ -251,7 +269,7 @@ index 1bdf312081394c65988b248952fef093ec89e812..641f18ef7df9a4d1055d7a74bcb99288 } }); diff --git a/src/index.ts b/src/index.ts -index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..9c3276659d86986db1fb20daa65bdd5e968bf8ca 100644 +index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..f79d046e77ee4bb42e08685cab95f15a37acf1f6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,7 +31,7 @@ import { @@ -279,14 +297,18 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..9c3276659d86986db1fb20daa65bdd5e const { endCursor, finality, cursor } = context as { cursor: Cursor; endCursor: Cursor; -@@ -412,6 +414,14 @@ export function drizzleStorage< +@@ -412,6 +414,18 @@ export function drizzleStorage< throw new DrizzleStorageError("End Cursor is undefined"); } - + + if (prevFinality === "pending") { + await withTransaction(db, async (tx) => { + // Invalidate in a dedicated transaction so rollback writes are + // never captured with the current block's reorg_order_key. ++ // TRADEOFF: If the replacement transaction (below) fails after this ++ // commits, readers see state with the tip removed until process restart. ++ // Apibara does not retry handler errors, so recovery depends on Railway's ++ // ALWAYS restart policy re-indexing from the last checkpoint. + await invalidate(tx, cursor, idColumnMap, indexerId); + }); + } @@ -294,10 +316,10 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..9c3276659d86986db1fb20daa65bdd5e await withTransaction(db, async (tx) => { context[DRIZZLE_PROPERTY] = { db: tx } as DrizzleStorage< TQueryResult, -@@ -419,19 +429,17 @@ export function drizzleStorage< +@@ -419,19 +433,17 @@ export function drizzleStorage< TSchema >; - + - if (prevFinality === "pending") { - // invalidate if previous block's finality was "pending" - await invalidate(tx, cursor, idColumnMap, indexerId); @@ -322,12 +344,12 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..9c3276659d86986db1fb20daa65bdd5e + } + await setReorgOrderKey(tx, endCursor); } - + await next(); -@@ -448,13 +456,11 @@ export function drizzleStorage< +@@ -448,13 +460,11 @@ export function drizzleStorage< prevFinality = finality; }); - + - if (finality !== "finalized") { - // remove trigger outside of the transaction or it won't be triggered. - await removeTriggers(db, tableNames, indexerId); @@ -342,7 +364,7 @@ index 3761cf45d3ce34a37ebfb2a012804e161d4589d6..9c3276659d86986db1fb20daa65bdd5e } }); diff --git a/src/storage.ts b/src/storage.ts -index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32ebcf9d3afe 100644 +index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..bfe1a7d137d444bc2a7812a9fd41739ea8bcaf40 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -91,11 +91,18 @@ export async function initializeReorgRollbackTable< @@ -374,7 +396,10 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb idColumnMap: IdColumnMap, indexerId: string, ) { -@@ -148,7 +154,8 @@ export async function registerTriggers< +@@ -145,10 +151,11 @@ export async function registerTriggers< + ); + await tx.execute( + sql.raw(` + DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table}; CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)} AFTER INSERT OR UPDATE OR DELETE ON ${table} @@ -387,7 +412,7 @@ index 1d6e951c860ee24afcd0511fa032cb5d9c99a0e3..e8ed33a6c75e110878b40c9002aa32eb @@ -159,6 +166,28 @@ export async function registerTriggers< } } - + +export async function setReorgOrderKey< + TQueryResult extends PgQueryResultHKT, + TFullSchema extends Record = Record, diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 8d7af51f..85f7cd66 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -11,7 +11,7 @@ overrides: patchedDependencies: '@apibara/plugin-drizzle@2.1.0-beta.55': - hash: 8463299f597bed8be89eeb2ae585fe38b310ee323309ad2c4d76959aa6255556 + hash: 75ca36241d9ec34fefe74972b6217bd01a2770b9bf07d0ac0fa79af614d10965 path: patches/@apibara__plugin-drizzle@2.1.0-beta.55.patch importers: @@ -23,7 +23,7 @@ importers: version: 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/plugin-drizzle': specifier: 2.1.0-beta.55 - version: 2.1.0-beta.55(patch_hash=8463299f597bed8be89eeb2ae585fe38b310ee323309ad2c4d76959aa6255556)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) + version: 2.1.0-beta.55(patch_hash=75ca36241d9ec34fefe74972b6217bd01a2770b9bf07d0ac0fa79af614d10965)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': specifier: 2.1.0-beta.56 version: 2.1.0-beta.56(typescript@5.9.3) @@ -2516,7 +2516,7 @@ snapshots: - utf-8-validate - zod - '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=8463299f597bed8be89eeb2ae585fe38b310ee323309ad2c4d76959aa6255556)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': + '@apibara/plugin-drizzle@2.1.0-beta.55(patch_hash=75ca36241d9ec34fefe74972b6217bd01a2770b9bf07d0ac0fa79af614d10965)(drizzle-orm@0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.18.0)(gel@2.2.0)(pg@8.20.0))(pg@8.20.0)(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0))': dependencies: '@apibara/indexer': 2.1.0-beta.56(typescript@5.9.3)(vitest@3.2.4(@types/node@22.19.11)(jiti@2.6.1)(tsx@4.21.0)) '@apibara/protocol': 2.1.0-beta.56(typescript@5.9.3) From 0d13099e0dc7c928a18dfdfa7f2a593419d483c9 Mon Sep 17 00:00:00 2001 From: loothero Date: Mon, 9 Mar 2026 19:27:25 -0700 Subject: [PATCH 39/39] fix(api): close HTTP server before draining DB pool on shutdown server.close() stops accepting new requests so in-flight handlers finish before the subscription hub and pool are torn down. Prevents "Cannot use a pool after calling end" errors during deploy rollouts. Co-Authored-By: Claude Opus 4.6 --- api/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/index.ts b/api/src/index.ts index ffc40f08..2ef112bd 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -990,9 +990,10 @@ if (isMetricsEnabled()) { ); } -// Graceful shutdown +// Graceful shutdown: stop accepting requests before tearing down resources async function shutdown() { log.info("api_shutdown_started"); + await new Promise((resolve) => server.close(() => resolve())); for (const emitter of metricEmitters) { emitter.stop(); }