Tuning db perf

src/Data/Collector.ts

@@ -686,7 +686,7 @@ export async function checkIfValidOverwrite(receipt: any, txId: string): Promise
       )
     if (nestedCountersInstance) nestedCountersInstance.countEvent('duplicate-receipts', `txId : ${txId}`)
 
-    // Liberdus App Receipt 
+    // Liberdus App Receipt
     const existingStatus = existingReceipt.appReceiptData.success
     if (existingStatus === true) {
       return false // you cannot override a successful receipt (status 1) with any new receipt
@@ -936,9 +936,9 @@ export const storeReceiptData = async (
               if (account.hash !== account.data['hash'])
                 Logger.mainLogger.error('Mismatched account hash', txId, account.accountId)
 
-              const accountExist = await Account.queryAccountByAccountId(account.accountId)
-              if (accountExist) {
-                if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj)
+              const existingAccountTimestamp = await Account.queryAccountTimestamp(account.accountId)
+              if (existingAccountTimestamp) {
+                if (accObj.timestamp > existingAccountTimestamp) combineAccounts.push(accObj)
               } else {
                 // await Account.insertAccount(accObj)
                 combineAccounts.push(accObj)
@@ -997,9 +997,9 @@ export const storeReceiptData = async (
               Logger.mainLogger.error('Mismatched account hash', txId, account.accountId)
             }
 
-            const accountExist = await Account.queryAccountByAccountId(account.accountId)
-            if (accountExist) {
-              if (accObj.timestamp > accountExist.timestamp) await Account.updateAccount(accObj)
+            const existingAccountTimestamp = await Account.queryAccountByAccountId(account.accountId)
+            if (existingAccountTimestamp) {
+              if (accObj.timestamp > existingAccountTimestamp.timestamp) combineAccounts.push(accObj)
             } else {
               combineAccounts.push(accObj)
             }

src/dbstore/accounts.ts

@@ -25,7 +25,14 @@ export async function insertAccount(account: AccountsCopy): Promise<void> {
 
     // Construct the SQL query with placeholders
     const placeholders = `(${columns.map(() => '?').join(', ')})`
-    const sql = `INSERT OR REPLACE INTO accounts (${columns.join(', ')}) VALUES ${placeholders}`
+    const sql = `INSERT INTO accounts (${columns.join(', ')}) VALUES ${placeholders}
+      ON CONFLICT(accountId) DO UPDATE SET
+        cycleNumber = excluded.cycleNumber,
+        timestamp = excluded.timestamp,
+        data = excluded.data,
+        hash = excluded.hash,
+        isGlobal = excluded.isGlobal
+      WHERE excluded.timestamp > accounts.timestamp`
 
     // Map the `account` object to match the columns
     const values = columns.map((column) =>
@@ -66,7 +73,14 @@ export async function bulkInsertAccounts(accounts: AccountsCopy[]): Promise<void
         try {
           // Construct the SQL query for this batch
           const placeholders = batch.map(() => `(${columns.map(() => '?').join(', ')})`).join(', ')
-          const sql = `INSERT OR REPLACE INTO accounts (${columns.join(', ')}) VALUES ${placeholders}`
+          const sql = `INSERT INTO accounts (${columns.join(', ')}) VALUES ${placeholders}
+            ON CONFLICT(accountId) DO UPDATE SET
+              cycleNumber = excluded.cycleNumber,
+              timestamp = excluded.timestamp,
+              data = excluded.data,
+              hash = excluded.hash,
+              isGlobal = excluded.isGlobal
+            WHERE excluded.timestamp > accounts.timestamp`
 
           // Flatten the batch into a single list of values
           const values = batch.flatMap((account) =>
@@ -153,6 +167,18 @@ export async function queryAccountByAccountId(accountId: string): Promise<Accoun
   }
 }
 
+export async function queryAccountTimestamp(accountId: string): Promise<number | null> {
+  try {
+    const sql = `SELECT timestamp FROM accounts WHERE accountId=?`
+    const dbAccount = (await db.get(accountDatabase, sql, [accountId])) as DbAccountCopy
+    if (dbAccount) return dbAccount.timestamp
+    return null
+  } catch (e) {
+    Logger.mainLogger.error(e)
+    return null
+  }
+}
+
 export async function queryLatestAccounts(count: number): Promise<AccountsCopy[] | null> {
   if (!Number.isInteger(count)) {
     Logger.mainLogger.error('queryLatestAccounts - Invalid count value')

src/dbstore/sqlite3storage.ts

@@ -10,6 +10,10 @@ export const createDB = async (dbPath: string, dbName: string): Promise<Database
     }
   })
   await run(db, 'PRAGMA journal_mode=WAL')
+  await run(db, 'PRAGMA synchronous = NORMAL;');
+  await run(db, 'PRAGMA temp_store = MEMORY;');
+  await run(db, 'PRAGMA cache_size = -64000;');       // ~64MB cache
+  await run(db, 'PRAGMA wal_autocheckpoint = 1000;'); // Checkpoint every 1000 ( default value ) pages
   db.on('profile', (sql, time) => {
     if (time > 500 && time < 1000) {
       console.log('SLOW QUERY', process.pid, sql, time)