32415 - Merge Multiple Dumps from corp_processing table

data-tool/restore_extract.sh

@@ -24,6 +24,7 @@ RESTORE=(corp_processing colin_tracking mig_group mig_batch mig_corp_batch mig_c
 
 # -- Runtime options -----------------------------------------------------------
 DUMP="${DUMP}"
+DELTA_MODE="${DELTA_MODE:-false}"  
 
 ##############################################################################
 # INTERNAL HELPERS
@@ -51,20 +52,59 @@ printf "🔄  Re‑importing Postgres from Oracle …\n"
 # -- EMPTY the tables but keep their structure                               #
 ##############################################################################
 printf "🧹  Truncating existing rows …\n"
-psql $(pg_conn_opts) -v ON_ERROR_STOP=1 -q <<SQL
-  TRUNCATE TABLE $(IFS=,; echo "${RESTORE[*]}") RESTART IDENTITY;
+if [ "$DELTA_MODE" = "true" ]; then
+  printf "⚠️  DELTA_MODE is true: skipping truncation of preserved tables.\n"
+else
+   printf "DELTA_MODE is false: truncating preserved tables.\n"
+   psql $(pg_conn_opts) -v ON_ERROR_STOP=1 -q <<SQL
+   TRUNCATE TABLE $(IFS=,; echo "${RESTORE[*]}") RESTART IDENTITY;
 SQL
+fi
 #  - RESTART IDENTITY zeros the sequences; we'll set them correctly later.
 #  - No CASCADE → we don’t wipe child tables that reference these rows.
 
 ##############################################################################
 # ── RESTORE DATA FOR THE PRESERVED TABLES                                   #
 ##############################################################################
-
-printf "🚚  Copying preserved rows (constraints temporarily disabled) …\n"
-pg_restore $(pg_conn_opts) --section=data --data-only \
-          --disable-triggers \
-          $(as_table_opts "${RESTORE[@]}") "$DUMP"
+if [ "$DELTA_MODE" = "true" ]; then
+  printf "⚠️  DELTA_MODE is true: skipping restore of preserved tables, processing table only "corp_processing".\n"
+  psql $(pg_conn_opts) -v ON_ERROR_STOP=1 <<  SQL
+  ALTER TABLE IF EXISTS corp_processing RENAME TO corp_processing_old;
+  CREATE TABLE corp_processing (LIKE corp_processing_old INCLUDING ALL);
+SQL
+  pg_restore $(pg_conn_opts) --section=data --data-only \
+            --disable-triggers \
+            --table="corp_processing" \
+            "$DUMP" 2>&1 | grep -v "ERROR" || true
+  psql $(pg_conn_opts) -v ON_ERROR_STOP=0 <<'MERGE'
+INSERT INTO corp_processing_old
+SELECT * FROM corp_processing
+ON CONFLICT (corp_num, flow_name, environment) DO UPDATE
+SET
+  corp_type_cd = EXCLUDED.corp_type_cd,
+  corp_name = EXCLUDED.corp_name,
+  filings_count = EXCLUDED.filings_count,
+  processed_status = EXCLUDED.processed_status,
+  failed_event_file_type = EXCLUDED.failed_event_file_type,
+  last_processed_event_id = EXCLUDED.last_processed_event_id,
+  failed_event_id = EXCLUDED.failed_event_id,
+  last_error = EXCLUDED.last_error,
+  claimed_at = EXCLUDED.claimed_at,
+  flow_run_id = EXCLUDED.flow_run_id,
+  mig_batch_id = EXCLUDED.mig_batch_id,
+  account_ids = EXCLUDED.account_ids,
+  last_modified = EXCLUDED.last_modified;
+
+DROP TABLE corp_processing;
+ALTER TABLE corp_processing_old RENAME TO corp_processing;
+MERGE
+
+else
+  printf "🚚  Copying preserved rows (constraints temporarily disabled) …\n"
+  pg_restore $(pg_conn_opts) --section=data --data-only \
+            --disable-triggers \
+            $(as_table_opts "${RESTORE[@]}") "$DUMP"
+fi
 
 ##############################################################################
 # ── FIX ANY SEQUENCES                                                       #