Bump numpy from 1.16.1 to 1.22.0 in /scripts/GoogleCloud/BQ

.gitignore

@@ -1,2 +1,16 @@
 *.pyc
-LocalEnv.py
+LocalEnv.py
+.DS_Store
+scripts/LabTestAnalysis/output*
+scripts/LabTestAnalysis/machine_learning/data*
+starr/data*
+stride/data*
+scripts/LabCulturePrediction/data*
+scripts/LabCulturePrediction/*.tab
+scripts/LabCulturePrediction/DecisionMakingClass/*.csv
+scripts/ER_Infection/data*
+scripts/ER_Infection/notebooks/mit_data_analysis/data*
+.idea*
+venv*
+*token.pickle
+.Rproj.user

LocalEnv.py.template

@@ -2,7 +2,7 @@
 
 # This file serves as documentation to help users understand LocalEnv.py.
 #
-# LocalEnv.py is created dynamically by setup.sh, but if you don't need
+# LocalEnv.py can be created dynamically by setup.sh, but if you don't need
 # to do any of the other steps in setup.sh (e.g. installing libraries,
 # initializing DB) it may be easier to just make a copy of this file
 # and name it LocalEnv.py before editing the actual values.
@@ -12,15 +12,44 @@
 #
 # Other variables which are likely to vary between dev environments should
 # be added to this template file, then be given default values via setup.sh.
+import os
 
+BOX_CLIENT_ID = "BOX_API_CLIENT_ID"
+BOX_CLIENT_SECRET = "BOX_API_CLIENT_SECRET"
+BOX_ACCESS_TOKEN = "BOX_API_ACCESS_TOKEN"
+BOX_STRIDE_FOLDER_ID = "BOX_STRIDE_FOLDER_ID"
+
+# Database connection type
+DATASET_SOURCE_NAME = 'STRIDE'
+DATABASE_CONNECTOR_NAME = "psycopg2"
+SQL_PLACEHOLDER = "%s"
+
+# Connection parameters to primary working database of interest
 LOCAL_PROD_DB_PARAM = {}
-LOCAL_PROD_DB_PARAM["HOST"] = 'localhost'
-LOCAL_PROD_DB_PARAM["DSN"] = 'prod'
-LOCAL_PROD_DB_PARAM["UID"] = 'produid'
-LOCAL_PROD_DB_PARAM["PWD"] = 'prodpwd'
+LOCAL_PROD_DB_PARAM["HOST"] = 'localhost'   # Database host. Localhost if running on your local computer. For GCP, name of the project. For AWS RDS instances, look for the "Endpoint" hostname, e.g. YourDatabaseIdentifier.cwyfvxgvic6c.us-east-1.rds.amazonaws.com
+LOCAL_PROD_DB_PARAM["DSN"] = 'databaseName'    # Specific database/dataset name hosted by the database server (e.g., stride_inpatient_2008_2017)
+LOCAL_PROD_DB_PARAM["UID"] = 'databaseUserId'
+LOCAL_PROD_DB_PARAM["PWD"] = 'databaseUserPassword'
 
+# Connection parameters for a test database to create and manipulate during unit tests
+#	The DSN (database name) MUST NOT BE the same as any real working database.
+#	Many of the Python application unit tests' first and last steps
+#	will be to delete any database identified by these test identifiers.
+#	The database user (UID) and password (PWD) will need to be a valid 
+#	database user with permissions to create and delete such databases.
 LOCAL_TEST_DB_PARAM = {}
 LOCAL_TEST_DB_PARAM["HOST"] = 'localhost'
-LOCAL_TEST_DB_PARAM["DSN"] = 'test'
+LOCAL_TEST_DB_PARAM["DSN"] = 'unittest_db'
 LOCAL_TEST_DB_PARAM["UID"] = 'testuid'
 LOCAL_TEST_DB_PARAM["PWD"] = 'testpwd'
+
+PATH_TO_CDSS = "/foo/bar/CDSS"; # Directory where this file is contained in
+# Look for environment variable on authenticating to Google cloud server
+# If env var not already there, then manually set here.
+# See https://github.com/HealthRex/CDSS/wiki/Google-BigQuery-Access for more info
+#if "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ:
+#    os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = "/foo/bar/google-cloud-platform-token.json";
+if "GOOGLE_CLOUD_PROJECT" not in os.environ:
+	os.environ["GOOGLE_CLOUD_PROJECT"] = LOCAL_PROD_DB_PARAM["HOST"];
+
+TEST_RUNNER_VERBOSITY = 2

README.md

@@ -1,100 +1,28 @@
 # HealthRex Laboratory at Stanford University
 ## PI: Jonathan Chen (http://web.stanford.edu/~jonc101)
 
-Shared workspace in development
+Review the Wiki (https://github.com/HealthRex/CDSS/wiki) for Starter Notes on using some of the common data sources and codebase as well as general lab/group infrastructure.
 
-General Guidelines for Code Repo:
-* Avoid any large data files, so the repo stays lightweight for new devs to quickly download/clone.
-* For one-off or very project specific files and scripts, basically do whatever you want in the workspace areas under the scripts directory (but again, avoid big data files).
-* Try to promote reusable components to the medinfo core application modules.
-
-
-
-====================================================
-
-Starter Notes:
-?This needs to be organized more, but a sketch of notes I usually email people initially.
-?Perhaps move this to Wiki
-?Similarly integrate CDSS/scripts/CDSS/setupNotes.txt into a high-level Readme starter
+Code is made freely available for academic use. For commercial use or other questions, email stanford [dot] healthrex [at] gmail [dot] com.
 
+See our group's publication list for relevant citations to reference. For example: Chen, J. H., Podchiyska, T. & Altman, R. B. Journal of the American Medical Informatics Association ocv091 (2016). doi:10.1093/jamia/ocv091
 
-General Data / Code Access
-	• Training / Agreements
-		○ Register on CITIProgram.org
-			§ Complete the module on "Biomedical Responsible Conduct of Research"
-			§ Send me a copy of the Completion Certificate
-
-(2B) HIPAA Training
-	1. 2B) HIPAA Training (required for almost all datasets. Because current CITI training does not offer an excellent data security module, we are requiring HIPAA training for datasets with or derived from health information or containing PII)
-		a. Go to your personal Axess page.
-		b. Log in with your SUNet and password.
-		c. Click on the STARS link (top).
-		d. Click on “Training Needs Assessment”
-		e. Only email your supervisor if you want to.
-		f. Indicate that you work on research projects.
-		g. Click on All Learning.
-		h. If needed, use the search tool and search for  HIPAA. Enroll in HIPAA/Protecting Patient Privacy (PRIV-0010).
-		i. Take the course.
-		j. Once the course(s) is complete, you should receive an email confirming your completion. It should also show up as completed if you click on the “Training History” bar.
-		k. Please save the screen shot as a PDF and name the file: firstinitiallastname_HIPAA_DDMMMYY.  So if Isabella Chu completed her classes July 3, 2016, ichu_HIPAA_03Jul16.
-
-
-		○ Complete the HCUP Data Use Agreement
-			• View Accessible HCUP DUA Training Course (Text Only) 
-				· At the bottom of this page, fill in your name to print out a completion certificate and send me a copy
-			• https://www.hcup-us.ahrq.gov/DUA/dua_508/DUA508version.jsp
-				· Fill out and complete the HCUP Data Use Agreement, then send a copy to me and the HCUPDistributor@AHRQ.gov
-		○ Health Data Protection / Hard Drive Encryption
-			• Any computers you use to work on health data should be encrypted. This is usually a transparent option that is easy to turn on in the background and is generally a good idea anyway in case your computer gets lost or stolen.
-			• Web link below should help you figure out if you’ve setup encryption properly. If you install Stanford’s BigFix program, it walks through most of the steps.
-			• https://med.stanford.edu/datasecurity/amie/
-			• I'd recommend working on a (laptop) computer with at least 16gb RAM and 512gb hard drive. I can help you get this if you do not already have it.
-
-	• Code:
-		○ Use Git or an SVN client to download the codebase
-			§ > git clone https://github.com/HealthRex/CDSS.git
-				or 
-			§ > svn checkout https://github.com/HealthRex/CDSS.git
-			§ If that works, it should download a bunch of Python 2.7 code to your computer
-
-	• Dev Environment:
-		○ Install Python PIP to facilitate other installations
-			§ If you installed Python 2.7.8, it might already be installed, otherwise there’s a get_pip.py bootstrapper installer on the internet)
-			§ Install some Python modules (e.g., scipy, psycopg2)
-				· > python -m pip install psycopg2
-		○ Install a local PostgreSQL database server instance
-		○ Create a blank test database with the schema definition in cpoeStats.sql.
-			§ > psql -f medinfo/db/definition/cpoeStats.sql -U <YourDBUsername> testdb
-		○ Create another blank application database, but restore it with sample data:
-			§ Box Data Directory: STRIDE-Inpatient-2008-2014/dump2009-2014-5year-time
-			§ Restore Script Driver: restoreCPOETables.sh
-				· This can take hours (overnight) to all finish patient_item and clinical_item_association tables. Make sure you start with a blank database and that the Indexes and Constraints don't yet exist, otherwise it will drag down the insert/population process
-		○ Point the application code to the databases:
-			§ Edit medinfo/db/Env.py file’s DB_PARAM to match the application database
-			§ Edit TEST_DB_PARAM settings to point to that blank database copy.
-		○ Run a basic application database unit test
-			§ > python medinfo/db/test/TestDBUtil.py
-			§  If that runs, you’re in good shape to have the Python application code interact with the SQL database
-
-	• Data Access
-		○ Box - Share data directories (Stanford Email)
-		○ Google - Work files share
-		○ OneNote - Notebook section share
-		○ XP-Dev - SVN Code
-
-
+General Guidelines for Code Repo:
+* Avoid any large data files, so the repo stays lightweight for new devs to quickly download/clone.
+* For one-off or very project specific files and scripts, basically do whatever you want in the workspace areas under the /scripts directory (but again, avoid big data files and also avoid any private / patient information, including analysis results that include individual patient items, as this repo will publicly accessible).
+* Avoid creating new / separate repos for your work, unless meant to be a completely independent package. Instead create sub-folders within this common repository.
+* Promote reusable components and packages to the medinfo core application modules.
+
+
+Broad description of core application directories
+* medinfo/scripts/DevWorkshop - Assorted tutorials and screening programs to learn and practice core technical skills.
+* medinfo/analysis - General purpose analysis and data manipulation modules, not specific to any type of project. For example, serially calculating t-tests, list rank similarity measures, ROC plots, precision-recall curves, SQL-like manipulation functions for CSV / TSV files.
+* medinfo/common - General purpose computing utilities, such as calculating different 2x2 contingency stats, adding progress trackers to long processes.
+* medinfo/cpoe - More project specific applications related to Computerized Physician Order Entry projects, implementing different approaches to clinical order recommendations and evaluating/analyzing them with different experiments on historical data. Application code for clinical case simulations for users to interact with.
+* medinfo/dataconversion - General and project specific utilities to pre-process data sources. Given a dump of hospital data, conversion scripts to unify into a simplified / pre-processed clinical_item transaction series. FeatureMatrixFactory to extract out clinical data into simple "feature matrix" / dataframe form to feed into assorted learning algorithms. Subdirecties with additional supporting mapping data (e.g., ICD9 codes to Charlson comorbidity categories).
+* medinfo/db - Utilities to connect between Python code and SQL databases, with a relatively plain JSON-like model of tables represented by lists of dictionaries (name-value pairs of each row of data). ResultsFormatter has several convenience functions to interconvert between SQL data tables, CSV/TSV plain text files, Pandas dataframes, and JSON-like lists of Python dictionaries. Several project specific application database schemas in the definition subdirectory. Support subdirectory with "dump" and "restore" convenience scripts to move database content between systems.
+* medinfo/geography - Not much here yet. One example of how to generate data labeled geographic maps of the US.
+* medinfo/textanalysis - Not much here yet. One example of a project specific parsing script that translates a stream of text documents into an interactive HTML file that attempts to auto-annotate features of the documents based on Python coded annotator classes.
+* medinfo/web - View and Controller layer for web interface to application functions.
 
-	• Population Health Sciences Access
-
-	Here are detailed instructions for accessing PHS data. If your staff have completed their trainings and IRB, it should take about 10 minutes to complete these forms: https://tinyurl.com/PHS-Data-Access
-
-	In addition, all PHS Data Users are required to take the PHS Data Core Training and Quiz. This should take you 15 - 20 minutes (including quiz). Completion with 100% on the quiz will waive CITI and HIPAA training for most datasets. CITI and HIPAA take about 4 - 6 hours so we hope you view this as a good deal.
-
-	PHS Data Security Training: http://tinyurl.com/PHS-Data-Security-Training
-
-	For Macs open with Quicktime. For PCs, just click on the image and click the play arrow. It should start.
-
-	If you are already a PHS member and you do not have your personalized link, please request it from Isabella Chu at itaylor@stanford.edu.
-
-	If you need help, my office hours are Thursdays at SIEPR 334 and Fridays online (by appointment) from 10 AM to 11 AM or by appointment. You can sign up for office hours clicking on the PHS Office Hours Sign Up near the top of this page: https://med.stanford.edu/phs/phs-data-center.html or emailing me at itaylor@stanford.edu.
 

TestCDSS.py

@@ -1,24 +1,24 @@
-"""Look through directory tree for unit test cases to run.
+"""
+Look through directory tree for unit test cases to run.
 Comparable to command-line functionality:
 
 python -m unittest discover -v -s <directoryName> -p "Test*.py"
-
 """
 
 import unittest
 import logging
 
-# Set logging level.
-log = logging.getLogger("CDSS")
+import os
 
-# TODO(sbala): Fix TestDataManager.test_compositeRelated
-# TODO(sbala): Fix TestSTRIDEOrderMedConversion.test_dataConversion_denormalized
+if __name__ == "__main__":
+    # Set logging level.
+    log = logging.getLogger("CDSS")
+    log.setLevel(logging.CRITICAL)
 
-# Load test suite.
-loader = unittest.TestLoader()
-suite = loader.discover('.', pattern="Test*.py")
-log.setLevel(logging.ERROR)
+    # Load test suite.
+    loader = unittest.TestLoader()
+    suite = loader.discover('.', pattern="Test*.py")
 
-# Run test suite.
-testRunner = unittest.runner.TextTestRunner(verbosity=1)
-testRunner.run(suite)
+    # Run test suite.
+    testRunner = unittest.runner.TextTestRunner(verbosity=1)
+    testRunner.run(suite)

medinfo/analysis/AccuracyPerTopItems.py

@@ -10,18 +10,18 @@
 import time;
 import json;
 from optparse import OptionParser
-from cStringIO import StringIO;
+from io import StringIO;
 import pylab;
 from medinfo.db.Model import columnFromModelList;
 from medinfo.common.Const import COMMENT_TAG, VALUE_DELIM;
 from medinfo.common.StatsUtil import ContingencyStats;
 from medinfo.common.Util import stdOpen, ProgressDots;
 from medinfo.db.ResultsFormatter import TextResultsFormatter, TabDictReader;
 from medinfo.db.Model import RowItemModel;
-from Util import log;
-from Const import OUTCOME_PRESENT, OUTCOME_ABSENT;
+from .Util import log;
+from .Const import OUTCOME_PRESENT, OUTCOME_ABSENT;
 
-from BaseAnalysis import BaseAnalysis;
+from .BaseAnalysis import BaseAnalysis;
 
 AXIS_DELIM = ":";
 
@@ -30,22 +30,22 @@ def __init__(self):
         BaseAnalysis.__init__(self);
 
     def __call__(self, inputFile, colOutcome, metricsByScoreCol, maxItems):
-        scoreCols = metricsByScoreCol.keys();
+        scoreCols = list(metricsByScoreCol.keys());
         scoreModels = self.parseScoreModelsFromFile(inputFile, colOutcome, scoreCols);
 
         # Count up total number of items and positive outcomes
         nPositive = sum([scoreModel[colOutcome] for scoreModel in scoreModels]);    # Assumes outcome labels are 0 and 1 for negative and positive, respectively
         nItems = len(scoreModels);
 
         # Prepare result dictionaries to populate
-        resultDicts = [{"ItemsConsidered": i+1} for i in xrange(maxItems)];
+        resultDicts = [{"ItemsConsidered": i+1} for i in range(maxItems)];
 
         for colScore in scoreCols:
             # Sort by each score column in descending order
             scoreModels.sort(key=lambda model: model[colScore], reverse=True);
 
             nPositiveFound = 0;
-            for i in xrange(maxItems):
+            for i in range(maxItems):
                 nConsidered = i+1;
                 scoreModel = scoreModels[i];
                 if scoreModel[colOutcome] == OUTCOME_PRESENT:
@@ -192,9 +192,9 @@ def main(self, argv):
             outputFile = stdOpen(outputFilename,"w");
 
             # Print comment line with arguments to allow for deconstruction later as well as extra results
-            print >> outputFile, COMMENT_TAG, json.dumps(summaryData);
+            print(COMMENT_TAG, json.dumps(summaryData), file=outputFile);
             # Insert a header row
-            resultDicts.insert(0, RowItemModel(resultDicts[0].keys(),resultDicts[0].keys()) );
+            resultDicts.insert(0, RowItemModel(list(resultDicts[0].keys()),list(resultDicts[0].keys())) );
 
             formatter = TextResultsFormatter( outputFile );
             formatter.formatResultDicts( resultDicts );

medinfo/analysis/BaseAnalysis.py

@@ -6,7 +6,7 @@
 import sys, os
 import time;
 from optparse import OptionParser
-from cStringIO import StringIO;
+from io import StringIO;
 from math import sqrt;
 from datetime import timedelta;
 
@@ -16,10 +16,10 @@
 from medinfo.db import DBUtil;
 from medinfo.db.Model import SQLQuery, RowItemModel;
 from medinfo.db.Model import modelListFromTable, modelDictFromList;
-from Util import log;
+from .Util import log;
 
-from Const import OUTCOME_ABSENT, OUTCOME_PRESENT, OUTCOME_IN_QUERY;
-from Const import NEGATIVE_OUTCOME_STRS;
+from .Const import OUTCOME_ABSENT, OUTCOME_PRESENT, OUTCOME_IN_QUERY;
+from .Const import NEGATIVE_OUTCOME_STRS;
 
 
 class BaseAnalysis: