Modified code to parse response only if it was successful.

.gitignore

@@ -1,3 +1,4 @@
+.env
 *.pyc
 LocalEnv.py
 .DS_Store

scripts/DevWorkshop/llm-api/agnostic_evaluator_models.py

@@ -1,6 +1,7 @@
 from functools import partial
 import requests
 import json
+import time
 import os 
 
 class API_text_to_text:
@@ -12,6 +13,11 @@ def __init__(self, model_init, model_call):
     def gen_txt_to_txt(self, input_txt):
         return self.model_call(input_txt, **self.model_init_dict)
 
+def error_handling(model_name, i, max_calls, sleep_time, e):
+            print(f"Failed with {model_name} (SHC) call {i}/{max_calls}, waited {i*sleep_time} seconds. Error: {e}.")
+            time.sleep(sleep_time)
+            return f"Failed to get a response from {model_name} (SHC) after {i*sleep_time} seconds. Error: {e}."
+
 # Gemini via Vertex AI
 def gemini_init(model_name, credentials_path):
     # For HIPAA compliance, everything remains in our Google Cloud Project
@@ -20,36 +26,61 @@ def gemini_init(model_name, credentials_path):
     from vertexai.preview.generative_models import GenerativeModel
     return {"ready_model": GenerativeModel(model_name), "model_name": model_name}
 
-def gemini_call(input_txt, **kwargs):
-    import time
+def gemini_call(input_txt, max_calls = 10, sleep_time = 5, **kwargs):
     ready_model = kwargs["ready_model"]
     model_name = kwargs["model_name"]
 
-    max_calls = 10; sleep_time = 5
     for i in range (max_calls):
         try:
             response = ready_model.generate_content([input_txt])
             full_response = response.candidates[0].content.parts[0].text
             break
         except Exception as e:
-            print(f"Failed with {model_name} call {i}/{max_calls}, waited {i*sleep_time} seconds. Error: {e}.")
-            time.sleep(sleep_time)
-            full_response = f"Failed to get a response from {model_name} after {i*sleep_time} seconds."
+            full_response = error_handling(model_name, i, max_calls, sleep_time, e)
+    return full_response
+
+# Gemini 2.5 Pro via SHC
+def gemini_shc_init(model_name, my_key):
+    headers = {'Ocp-Apim-Subscription-Key': my_key, 'Content-Type': 'application/json'}
+    url = 'https://apim.stanfordhealthcare.org/gemini-25-pro/gemini-25-pro'
+    return {"model_name": model_name, "url": url, "headers": headers}
+
+def gemini_shc_call(input_txt, max_calls=10, sleep_time=5, **kwargs):
+    model_name = kwargs["model_name"]
+    url = kwargs["url"]
+    headers = kwargs["headers"]
+    payload = json.dumps({"contents": [{"role": "user", "parts": [{ "text": input_txt }]}]})
+
+    for i in range(max_calls):
+        try:
+            response = requests.request("POST", url, headers=headers, data=payload)
+            full_response = ''.join([i['candidates'][0]['content']['parts'][0]['text'] for i in json.loads(response.text)])
+            break
+        except Exception as e:
+            full_response = error_handling(model_name, i, max_calls, sleep_time, e)
     return full_response
 
 # Open AI models via SHC
 def openai_init(model_name, my_key):
     headers = {'Ocp-Apim-Subscription-Key': my_key, 'Content-Type': 'application/json'}
-    url = f"https://apim.stanfordhealthcare.org/openai-eastus2/deployments/{model_name}/chat/completions?api-version=2025-01-01-preview" 
+    url = f"https://apim.stanfordhealthcare.org/openai-eastus2/deployments/{model_name}/chat/completions?api-version=2025-01-01-preview"
+    if model_name == "gpt-4o":
+        url = "https://apim.stanfordhealthcare.org/openai20/deployments/gpt-4o/chat/completions?api-version=2023-05-15" 
     return {"model_name": model_name, "url": url, "headers": headers}
 
-def openai_call(input_txt, **kwargs):
+def openai_call(input_txt, max_calls = 10, sleep_time = 5, **kwargs):
     model_name = kwargs["model_name"]
     url = kwargs["url"]
     headers = kwargs["headers"]
     payload = json.dumps({"model": model_name, "messages": [{"role": "user", "content": input_txt}]})
-    response = requests.request("POST", url, headers=headers, data=payload)
-    return json.loads(response.text)['choices'][0]['message']['content']
+    for i in range(max_calls):
+        try:
+            response = requests.request("POST", url, headers=headers, data=payload)
+            full_response = json.loads(response.text)['choices'][0]['message']['content']
+            break
+        except Exception as e:
+           full_response = error_handling(model_name, i, max_calls, sleep_time, e)
+    return full_response
 
 # Deepseek-R1 via SHC
 def deepseek_init(model_name, my_key, view_thinking=False):
@@ -58,18 +89,24 @@ def deepseek_init(model_name, my_key, view_thinking=False):
     url = "https://apim.stanfordhealthcare.org/deepseekr1/v1/chat/completions"
     return {"url": url, "headers": headers, "view_thinking": view_thinking}
 
-def deepseek_call(input_txt, **kwargs):
+def deepseek_call(input_txt, max_calls = 10, sleep_time = 5, **kwargs):
     url = kwargs["url"]
     headers = kwargs["headers"]
     payload = json.dumps({"model": "deepseek-chat", "messages": [{"role": "user", "content": input_txt}], "temperature": 0.8, "max_tokens": 4096, "top_p": 1, "stream": False})
-    response = requests.request("POST", url, headers=headers, data=payload)
-    full_response = json.loads(response.text)['choices'][0]['message']['content']
-    if kwargs["view_thinking"]:
-        return full_response
-    def _extract_after_think(text):
-        parts = text.split("</think>")
-        return parts[1].strip() if len(parts) > 1 else text
-    return _extract_after_think(full_response)
+    for i in range(max_calls):
+        try:
+            response = requests.request("POST", url, headers=headers, data=payload)
+            full_response = json.loads(response.text)['choices'][0]['message']['content']
+            if kwargs["view_thinking"]:
+                break
+            def _extract_after_think(text):
+                parts = text.split("</think>")
+                return parts[1].strip() if len(parts) > 1 else text
+            full_response = _extract_after_think(full_response)
+            break
+        except Exception as e:
+            full_response = error_handling("deepseek-r1", i, max_calls, sleep_time, e)
+    return full_response
 
 # Microsoft model via SHC
 def microsoft_init(model_name, my_key):
@@ -78,12 +115,18 @@ def microsoft_init(model_name, my_key):
     url = "https://apim.stanfordhealthcare.org/phi35mi/v1/chat/completions"
     return {"url": url, "headers": headers}
 
-def microsoft_call(input_txt, **kwargs):
+def microsoft_call(input_txt, max_calls = 10, sleep_time = 5, **kwargs):
     url = kwargs["url"]
     headers = kwargs["headers"]
     payload = json.dumps({"messages": [{"role": "user", "content": input_txt}], "max_tokens": 2048, "temperature": 0.8, "top_p": 0.1, "presence_penalty": 0, "frequency_penalty": 0, "model": "Phi-3.5-mini-instruct"})
-    response = requests.request("POST", url, headers=headers, data=payload)
-    return json.loads(response.text)["choices"][0]["message"]["content"]
+    for i in range(max_calls):
+        try:
+            response = requests.request("POST", url, headers=headers, data=payload)
+            full_response = json.loads(response.text)["choices"][0]["message"]["content"]
+            break
+        except Exception as e:
+            full_response = error_handling("phi-3.5-mini-instruct", i, max_calls, sleep_time, e)
+    return full_response
 
 # Anthropic model via SHC
 def anthropic_init(model_name, my_key):
@@ -96,13 +139,19 @@ def anthropic_init(model_name, my_key):
     headers = {'Ocp-Apim-Subscription-Key': my_key, 'Content-Type': 'application/json'}
     return {"model_id": model_id, "url": url, "headers": headers}
 
-def anthropic_call(input_txt, **kwargs):
+def anthropic_call(input_txt, max_calls = 10, sleep_time = 5, **kwargs):
     model_id = kwargs["model_id"]
     url = kwargs["url"]
     headers = kwargs["headers"]
     payload = json.dumps({"model_id": model_id, "prompt_text": input_txt})
-    response = requests.request("POST", url, headers=headers, data=payload) 
-    return json.loads(response.text)['content'][0]['text']
+    for i in range(max_calls):
+        try:
+            response = requests.request("POST", url, headers=headers, data=payload) 
+            full_response = json.loads(response.text)['content'][0]['text']
+            break
+        except Exception as e:
+            full_response = error_handling(model_id, i, max_calls, sleep_time, e)
+    return full_response
 
 # Meta model via SHC
 def meta_init(model_name, my_key):
@@ -114,20 +163,35 @@ def meta_init(model_name, my_key):
     url = f"https://apim.stanfordhealthcare.org/{model_name}/v1/chat/completions"
     return {"full_model_name": full_model_name, "url": url, "headers": headers}
 
-def meta_call(input_txt, **kwargs):
+def meta_call(input_txt, max_calls = 10, sleep_time = 5, **kwargs):
     full_model_name = kwargs["full_model_name"]
     url = kwargs["url"]
     headers = kwargs["headers"]
     payload = json.dumps({"model": full_model_name, "messages": [{"role": "user", "content": input_txt}]})
-    response = requests.request("POST", url, headers=headers, data=payload)
-    return json.loads(response.text)['choices'][0]['message']['content']
+    for i in range(max_calls):
+        try:
+            response = requests.request("POST", url, headers=headers, data=payload)
+            full_response = json.loads(response.text)['choices'][0]['message']['content']
+            break
+        except Exception as e:
+            full_response = error_handling(full_model_name, i, max_calls, sleep_time, e)
+    return full_response
 
 if __name__ == "main":
+    from dotenv import load_dotenv
+    load_dotenv("../../../.env")
+
     my_question = """First, state what LLM you are based on. Please answer with the precise version of the model.
     Next, answer the following hard physics question.
     What is the difference between the cosmological constant and the vacuum energy?"""
 
-    lab_key = "enter the lab key here"
+    lab_key = os.getenv("LAB_KEY") # enter the lab key here
+
+    # Using Gemini 2.5 pro via SHC
+    gemini_shc_init_partial = partial(gemini_shc_init, "gemini-2.5-pro-preview-05-06", lab_key)
+    gemini_shc_instance = API_text_to_text(gemini_shc_init_partial, gemini_shc_call)
+    res = gemini_shc_instance.gen_txt_to_txt(my_question)
+    print(res)
 
     # Using Meta via SHC
     llama_init = partial(meta_init, "llama4-maverick", lab_key)

scripts/DevWorkshop/llm-api/phi-llm-api-python.md

@@ -4,7 +4,7 @@ This guide demonstrates how to use Python to interact with various Large Languag
 
 *Created by François Grolleau on 02/19/2025 
 Contributors: Yixing Jiang 
-Last update April 29, 2025.*
+Last update June 16, 2025.*
 
 ## Table of Contents
 - [Prerequisites](#prerequisites)
@@ -18,8 +18,8 @@ Last update April 29, 2025.*
   - [Claude 3.7 Sonnet](#claude-37-sonnet-api-call)
   - [Claude 3.5 Sonnet v2](#claude-35-sonnet-v2-api-call)
 - [Google Models](#google-models)
+  - [Gemini 2.5 Pro](#gemini-25-pro-api-call)
   - [Gemini 2.0 Flash](#gemini-20-flash-api-call)
-  - [Gemini 1.5 Pro](#gemini-15-pro-api-call)
 - [Meta Models](#meta-models)
   - [Llama 3.3 70B](#llama-33-70b-instruct-api-call)
   - [Llama 4 Maverick](#llama-4-maverick-api-call)
@@ -141,25 +141,22 @@ print(response.text)
 ---
 
 ## Google Models
-### Gemini 2.0 Flash API Call
+### Gemini 2.5 Pro API Call
 ```python
-url = "https://apim.stanfordhealthcare.org/gcp-gem20flash-fa/apim-gcp-gem20flash-fa" 
+url = "https://apim.stanfordhealthcare.org/gemini-25-pro/gemini-25-pro"
 payload = json.dumps({
-    "contents": {"role": "user", "parts": {"text": my_question}}, 
-    "safety_settings": {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"}, 
-    "generation_config": {"temperature": 0.2, "topP": 0.8, "topK": 40}
+    "contents": [{"role": "user", "parts": [{"text": my_question}]}]
 })
 response = requests.request("POST", url, headers=headers, data=payload) 
 print(response.text)
 ```
 
-
-### Gemini 1.5 Pro API Call
+### Gemini 2.0 Flash API Call
 ```python
-url = "https://apim.stanfordhealthcare.org/gcpgemini/apim-gcp-oauth-fa" 
+url = "https://apim.stanfordhealthcare.org/gcp-gem20flash-fa/apim-gcp-gem20flash-fa" 
 payload = json.dumps({
     "contents": {"role": "user", "parts": {"text": my_question}}, 
-    "safety_settings": {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_LOW_AND_ABOVE"}, 
+    "safety_settings": {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"}, 
     "generation_config": {"temperature": 0.2, "topP": 0.8, "topK": 40}
 })
 response = requests.request("POST", url, headers=headers, data=payload) 
@@ -296,7 +293,7 @@ def gen_txt_to_txt(input_txt):
 Use the `API_text_to_text` class from `agnostic_evaluator_models.py` to seamlessly switch between different LLM providers:
 
 ```python
-from agnostic_evaluator_models import API_text_to_text, meta_init, meta_call, gemini_init, gemini_call
+from agnostic_evaluator_models import API_text_to_text, meta_init, meta_call, gemini_init, gemini_call, gemini_shc_init, gemini_shc_call
 from functools import partial
 
 lab_key = "enter the lab's key here!"
@@ -309,6 +306,12 @@ llama_instance = API_text_to_text(llama_init, meta_call)
 res = llama_instance.gen_txt_to_txt(my_question)
 print(res)
 
+# Using Gemini 2.5 Pro via SHC
+gemini_shc_init_partial = partial(gemini_shc_init, "gemini-2.5-pro-preview-05-06", lab_key)
+gemini_shc_instance = API_text_to_text(gemini_shc_init_partial, gemini_shc_call)
+res = gemini_shc_instance.gen_txt_to_txt(my_question)
+print(res)
+
 # Using Gemini 2.5 Pro Preview via Vertex AI
 credentials_path = "path/to/your/google_application_default_credentials.json"
 
@@ -323,4 +326,4 @@ More examples are provided in the `agnostic_evaluator_models.py` under `if __nam
 
 > **Note:** If you haven't already set up Google Cloud authentication, you'll need to create an `application_default_credentials.json` file. You can follow the step-by-step instructions in our [Google Cloud setup guide](https://github.com/HealthRex/CDSS/blob/master/scripts/DevWorkshop/ReadMe.GoogleCloud-BigQuery-VPC.txt) to generate and configure these credentials.
 
-*Final fun note: As of April 2025, no model seems to get the physics question right. According to cosmologist Sean Carroll "The vacuum energy and the cosmological constant are two different labels for exactly the same thing; don't let anyone tell you differently" (Quanta and Fields, Dutton, 2024. Chapter 6 p. 146). Looks like latest models continue bullshitting very confidently when facts are rarely in their training data.* 😄
+*Final fun note: As of June 2025, no model seems to get the physics question right. According to cosmologist Sean Carroll "The vacuum energy and the cosmological constant are two different labels for exactly the same thing; don't let anyone tell you differently" (Quanta and Fields, Dutton, 2024. Chapter 6 p. 146). Looks like latest models continue bullshitting very confidently when facts are rarely in their training data.* 😄

scripts/antibiotic-susceptibility/sql/queries/aim_4/AIM4/.gitignore

@@ -1 +1,5 @@
-csv_folder/*
+csv_folder/*
+Embedding_Pilot_Exp/data/*
+Embedding_Pilot_Exp/error_checking/*
+Embedding_Pilot_Exp/strictest/*
+Embedding_Pilot_Exp/.env