diff --git a/data_management/main.py b/data_management/main.py index 9f47c8e..c637dcd 100644 --- a/data_management/main.py +++ b/data_management/main.py @@ -43,6 +43,9 @@ def upsert_new_spectrack_specimens(self): def load_biopsy_tracking(self): return self.tableau.load_biopsy_tracking() + def load_biopsy_tracking_long(self): + return self.tableau.load_biopsy_tracking_long() + def load_data_manager_data(self): return self.tableau.load_data_manager_data() @@ -88,6 +91,7 @@ def update_biomarker_tracking_redcap_ids(self): if args.action == "insert" or args.action == "update": records_modified = main.load_biopsy_tracking() records_modified = records_modified + main.load_data_manager_data() + records_modified = records_modified + main.load_biopsy_tracking_long() if "records_modified" in locals(): logger.info(f"{records_modified} records modified") diff --git a/data_management/services/dlu_management.py b/data_management/services/dlu_management.py index 8b52a16..1bb0dc0 100644 --- a/data_management/services/dlu_management.py +++ b/data_management/services/dlu_management.py @@ -191,6 +191,12 @@ def get_biopsy_tracking(self): ) return result + def get_biopsy_tracking_long(self): + result = self.db.get_data( + "select * from biopsy_tracking_long_v" + ) + return result + def get_data_manager_data(self): result = self.db.get_data( """ diff --git a/data_management/services/tableau.py b/data_management/services/tableau.py index ec05b17..92f10cb 100644 --- a/data_management/services/tableau.py +++ b/data_management/services/tableau.py @@ -27,7 +27,17 @@ def load_biopsy_tracking(self): query = "INSERT INTO biopsy_tracking (redcap_record_id, `Whole Slide Images`, `Single-nucleus RNA-Seq Status`, `Single-nucleus RNA-Seq Specimen ID`, `ATAC RNA-seq Status`, `ATAC RNA-seq Specimen ID`, `Single-cell RNA-Seq Status`, `Single-cell RNA-Seq Specimen ID`, `Regional Transcriptomics Status`, `Regional Transcriptomics Specimen ID`, `Bulk total/mRNA Experiment Status`, `Bulk total/mRNA Experiment Specimen ID`, `3D Tissue Imaging and Cytometry Experiment Status`, `3D Tissue Imaging and Cytometry Experiment Specimen ID`, `Regional Proteomics Experiment Status`, `Regional Proteomics Specimen ID`, `Spatial Metabolomics Experiment Status`, `Spatial Metabolomics Specimen ID`, `Spatial Lipidomics Experiment Status`, `Spatial Lipidomics Specimen ID`, `Spatial N-glycomics Experiment Status`, `Spatial N-glycomics Specimen ID`, `Spatial Transcriptomics Experiment Status`, `Spatial Transcriptomics Specimen ID`, `CODEX (IU) Experiment Status`, `CODEX (IU) Specimen ID`, `CODEX (UCSF) Experiment Status`, `CODEX (UCSF) Specimen ID`, `IMC Experiment Status`, `IMC Specimen ID`, `DNA Methyl-seq Experiment Status`, `DNA Methyl-seq Specimen ID`, `CUT & RUN Experiment Status`, `CUT & RUN Specimen ID`, `Metabolon Timed Urine - UHPLC MS-MS Experiment Status`, `Metabolon Timed Urine - UHPLC MS-MS Specimen ID`, `Metabolon Plasma EDTA - UHPLC MS-MS Experiment Status`, `Metabolon Plasma EDTA - UHPLC MS-MS Specimen ID`, `MSDQ120 Spot Urine Biomarker Status`, `MSDQ120 Spot Urine Biomarker Specimen ID`, `MSDQ120 Plasma EDTA Biomarker Status`, `MSDQ120 Plasma EDTA Biomarker Specimen ID`, `Litholink Timed Urine - BCAU680 - Status`, `Litholink Timed Urine - BCAU680 - Specimen ID`, `Stool Microbiome - Qaigen NextEra Status`, `Stool Microbiome - Qaigen NextEra Specimen ID`, `Clinical Chemistry Spot/Timed Urine - BCAU5812 Status`, `Clinical Chemistry Spot/Timed Urine - BCAU5812 Specimen ID`, `Clinical Chemistry Serum - BCAU5812 Status`, `Clinical Chemistry Serum - BCAU5812 Specimen ID`, `SomaScan Plasma EDTA - Status`, `SomaScan Plasma EDTA - Specimen ID`, `SomaScan Spot Urine - Status`, `SomaScan Spot Urine - Specimen ID`, `Descriptor Scoring (TIV)`, `Segmentation/Features Data - Status`, `fMRI - Status`, `Retinal - Status`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" records_modified = 0 for result in bt_results: - insert_result = self.db_tableau.insert_data(query, tuple(result.values())) + self.db_tableau.insert_data(query, tuple(result.values())) + records_modified = records_modified + 1 + return records_modified + + def load_biopsy_tracking_long(self): + self.truncate_biopsy_tracking_long() + bt_results = self.dlu_management.get_biopsy_tracking_long() + query = "INSERT INTO biopsy_tracking_long(redcap_id, specimen_id, dlu_packageType, status) VALUES(%s, %s, %s, %s)" + records_modified = 0 + for result in bt_results: + self.db_tableau.insert_data(query, tuple(result.values())) records_modified = records_modified + 1 return records_modified @@ -39,7 +49,7 @@ def load_data_manager_data(self): records_modified = 0 for result in results: result["dlu_created"] = result["dlu_created"].strftime('%Y-%m-%d %H:%M:%S') - insert_result = self.db_tableau.insert_data(query, tuple(result.values())) + self.db_tableau.insert_data(query, tuple(result.values())) records_modified = records_modified + 1 return records_modified diff --git a/data_management/sql/biopsy_tracking_long.sql b/data_management/sql/biopsy_tracking_long.sql new file mode 100644 index 0000000..3b86ee2 --- /dev/null +++ b/data_management/sql/biopsy_tracking_long.sql @@ -0,0 +1,9 @@ +CREATE TABLE biopsy_tracking_long +( + redcap_id varchar(100) NULL, + specimen_id varchar(100) NULL, + dlu_packageType varchar(100) NULL, + status varchar(100) NULL +) ENGINE=InnoDB +DEFAULT CHARSET=latin1 +COLLATE=latin1_swedish_ci; \ No newline at end of file