From 82cb61482bff5d2297d24e1eca560d6b19beeba6 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Fri, 17 Oct 2025 18:38:53 +0545 Subject: [PATCH 1/2] Replace INSERT OR REPLACE with ON CONFLICT for timestamp-based account updates --- src/Data/Collector.ts | 8 +++++--- src/dbstore/accounts.ts | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index cdfdf22..2c2e8f3 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -686,7 +686,7 @@ export async function checkIfValidOverwrite(receipt: any, txId: string): Promise ) if (nestedCountersInstance) nestedCountersInstance.countEvent('duplicate-receipts', `txId : ${txId}`) - // Liberdus App Receipt + // Liberdus App Receipt const existingStatus = existingReceipt.appReceiptData.success if (existingStatus === true) { return false // you cannot override a successful receipt (status 1) with any new receipt @@ -938,7 +938,8 @@ export const storeReceiptData = async ( const accountExist = await Account.queryAccountByAccountId(account.accountId) if (accountExist) { - if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj) + // if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj) + combineAccounts.push(accObj) } else { // await Account.insertAccount(accObj) combineAccounts.push(accObj) @@ -999,7 +1000,8 @@ export const storeReceiptData = async ( const accountExist = await Account.queryAccountByAccountId(account.accountId) if (accountExist) { - if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj) + // if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj) + combineAccounts.push(accObj) } else { combineAccounts.push(accObj) } diff --git a/src/dbstore/accounts.ts b/src/dbstore/accounts.ts index cce5d03..6aaa159 100644 --- a/src/dbstore/accounts.ts +++ b/src/dbstore/accounts.ts @@ -25,7 +25,14 @@ export async function insertAccount(account: AccountsCopy): Promise { // Construct the SQL query with placeholders const placeholders = `(${columns.map(() => '?').join(', ')})` - const sql = `INSERT OR REPLACE INTO accounts (${columns.join(', ')}) VALUES ${placeholders}` + const sql = `INSERT INTO accounts (${columns.join(', ')}) VALUES ${placeholders} + ON CONFLICT(accountId) DO UPDATE SET + cycleNumber = excluded.cycleNumber, + timestamp = excluded.timestamp, + data = excluded.data, + hash = excluded.hash, + isGlobal = excluded.isGlobal + WHERE excluded.timestamp > accounts.timestamp` // Map the `account` object to match the columns const values = columns.map((column) => @@ -66,7 +73,14 @@ export async function bulkInsertAccounts(accounts: AccountsCopy[]): Promise `(${columns.map(() => '?').join(', ')})`).join(', ') - const sql = `INSERT OR REPLACE INTO accounts (${columns.join(', ')}) VALUES ${placeholders}` + const sql = `INSERT INTO accounts (${columns.join(', ')}) VALUES ${placeholders} + ON CONFLICT(accountId) DO UPDATE SET + cycleNumber = excluded.cycleNumber, + timestamp = excluded.timestamp, + data = excluded.data, + hash = excluded.hash, + isGlobal = excluded.isGlobal + WHERE excluded.timestamp > accounts.timestamp` // Flatten the batch into a single list of values const values = batch.flatMap((account) => From 888dc185a87372eb9280e4b7b9e7906889845f9c Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 20 Oct 2025 14:56:06 +0800 Subject: [PATCH 2/2] Optimize account update path and tune SQLite for better performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add queryAccountTimestamp(accountId) to fetch only the timestamp - Update Collector to enqueue account updates only when newer than existing timestamp, avoiding redundant writes - Tune SQLite PRAGMAs: synchronous=NORMAL, temp_store=MEMORY, cache_size≈64MB, wal_autocheckpoint=1000 (retain WAL mode) --- src/Data/Collector.ts | 14 ++++++-------- src/dbstore/accounts.ts | 12 ++++++++++++ src/dbstore/sqlite3storage.ts | 4 ++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 2c2e8f3..4f1f2d8 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -936,10 +936,9 @@ export const storeReceiptData = async ( if (account.hash !== account.data['hash']) Logger.mainLogger.error('Mismatched account hash', txId, account.accountId) - const accountExist = await Account.queryAccountByAccountId(account.accountId) - if (accountExist) { - // if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj) - combineAccounts.push(accObj) + const existingAccountTimestamp = await Account.queryAccountTimestamp(account.accountId) + if (existingAccountTimestamp) { + if (accObj.timestamp > existingAccountTimestamp) combineAccounts.push(accObj) } else { // await Account.insertAccount(accObj) combineAccounts.push(accObj) @@ -998,10 +997,9 @@ export const storeReceiptData = async ( Logger.mainLogger.error('Mismatched account hash', txId, account.accountId) } - const accountExist = await Account.queryAccountByAccountId(account.accountId) - if (accountExist) { - // if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj) - combineAccounts.push(accObj) + const existingAccountTimestamp = await Account.queryAccountByAccountId(account.accountId) + if (existingAccountTimestamp) { + if (accObj.timestamp > existingAccountTimestamp.timestamp) combineAccounts.push(accObj) } else { combineAccounts.push(accObj) } diff --git a/src/dbstore/accounts.ts b/src/dbstore/accounts.ts index 6aaa159..01d7cf2 100644 --- a/src/dbstore/accounts.ts +++ b/src/dbstore/accounts.ts @@ -167,6 +167,18 @@ export async function queryAccountByAccountId(accountId: string): Promise { + try { + const sql = `SELECT timestamp FROM accounts WHERE accountId=?` + const dbAccount = (await db.get(accountDatabase, sql, [accountId])) as DbAccountCopy + if (dbAccount) return dbAccount.timestamp + return null + } catch (e) { + Logger.mainLogger.error(e) + return null + } +} + export async function queryLatestAccounts(count: number): Promise { if (!Number.isInteger(count)) { Logger.mainLogger.error('queryLatestAccounts - Invalid count value') diff --git a/src/dbstore/sqlite3storage.ts b/src/dbstore/sqlite3storage.ts index 0e15828..388bc27 100644 --- a/src/dbstore/sqlite3storage.ts +++ b/src/dbstore/sqlite3storage.ts @@ -10,6 +10,10 @@ export const createDB = async (dbPath: string, dbName: string): Promise { if (time > 500 && time < 1000) { console.log('SLOW QUERY', process.pid, sql, time)