diff --git a/.gitignore b/.gitignore index b6ae2a7..19300fd 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ Thumbs.db # SQLite / local DBs *.db +*.db-shm +*.db-wal *.sqlite # Environment variables / secrets (add if created) diff --git a/docs/api_endpoints.xlsx b/docs/api_endpoints.xlsx index 903fce8..f49a332 100644 Binary files a/docs/api_endpoints.xlsx and b/docs/api_endpoints.xlsx differ diff --git a/normalize_soa.py b/normalize_soa.py index e1ad818..11f7431 100644 --- a/normalize_soa.py +++ b/normalize_soa.py @@ -37,7 +37,6 @@ import csv import os import re -import sys from dataclasses import asdict, dataclass from typing import Any, Dict, List, Optional @@ -259,7 +258,6 @@ def build_visit_activities( vas: List[VisitActivity] = [] next_id = 1 for a_idx, r in enumerate(rows, start=1): - activity_name = r[0] for v_idx, visit in enumerate(visits, start=1): if v_idx >= len(r): continue diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index 705bbc6..9c473a7 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -65,8 +65,9 @@ from .routers import rollback as rollback_router from .routers import visits as visits_router from .routers.arms import create_arm # re-export for backward compatibility -from .routers.arms import delete_arm, update_arm -from .schemas import ArmCreate, ArmUpdate, SOACreate, SOAMetadataUpdate +from .routers.arms import delete_arm +from .schemas import ArmCreate, SOACreate, SOAMetadataUpdate +from .utils import get_next_code_uid as _get_next_code_uid load_dotenv() # must come BEFORE reading env-based configuration so values are populated DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") @@ -78,6 +79,12 @@ # SDTM dataset specializations cache (similar TTL) _sdtm_specializations_cache = {"data": None, "fetched_at": 0} _SDTM_SPECIALIZATIONS_CACHE_TTL = 60 * 60 +# Category-specific biomedical concepts cache (per category key) +_category_concepts_cache: dict[str, dict] = {} +_CATEGORY_CONCEPTS_CACHE_TTL = 60 * 60 # 1 hour +# Biomedical concept categories cache (whole list) +_bc_categories_cache = {"data": None, "fetched_at": 0} +_BC_CATEGORIES_CACHE_TTL = 60 * 60 # 1 hour app = FastAPI(title="SoA Builder API", version="0.1.0") logger = logging.getLogger("soa_builder.concepts") if not logger.handlers: @@ -875,7 +882,7 @@ def _fetch_arms_for_edit(soa_id: int) -> list[dict]: conn = _connect() cur = conn.cursor() cur.execute( - "SELECT id,name,label,description,order_index FROM arm WHERE soa_id=? ORDER BY order_index", + "SELECT id,name,label,description,order_index,COALESCE(type,''),COALESCE(data_origin_type,'') FROM arm WHERE soa_id=? ORDER BY order_index", (soa_id,), ) rows = [ @@ -885,6 +892,8 @@ def _fetch_arms_for_edit(soa_id: int) -> list[dict]: "label": r[2], "description": r[3], "order_index": r[4], + "type": r[5] or None, + "data_origin_type": r[6] or None, } for r in cur.fetchall() ] @@ -1000,7 +1009,7 @@ def _fetch_matrix(soa_id: int): return visits, activities, cells -def fetch_biomedical_concept_categories() -> list[dict]: +def fetch_biomedical_concept_categories(force: bool = False) -> list[dict]: """Return list of Biomedical Concept Categories from CDISC Library. Normalized shape: @@ -1027,6 +1036,15 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: return base_prefix + h return base_prefix + "/" + h + # Cache lookup + now = time.time() + if ( + not force + and _bc_categories_cache.get("data") + and now - _bc_categories_cache.get("fetched_at", 0) < _BC_CATEGORIES_CACHE_TTL + ): + return _bc_categories_cache.get("data") or [] + try: resp = requests.get(url, headers=headers, timeout=15) if resp.status_code != 200: @@ -1069,13 +1087,15 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: ) categories.sort(key=lambda c: (c["title"] or "").lower()) logger.info("Fetched %d BC categories from remote API", len(categories)) + _bc_categories_cache["data"] = categories + _bc_categories_cache["fetched_at"] = now return categories except Exception as e: # pragma: no cover logger.error("BC categories fetch error: %s", e) return [] -def fetch_biomedical_concepts_by_category(name: str) -> list[dict]: +def fetch_biomedical_concepts_by_category(name: str, force: bool = False) -> list[dict]: """Return biomedical concepts for a given category name. Uses category-specific endpoint: /mdr/bc/biomedicalconcepts?category= @@ -1108,6 +1128,14 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: return base_prefix + h return base_prefix + "/" + h + # Cache lookup + now = time.time() + ckey = category.lower() + if not force: + cached = _category_concepts_cache.get(ckey) + if cached and now - cached.get("fetched_at", 0) < _CATEGORY_CONCEPTS_CACHE_TTL: + return cached.get("data", []) or [] + concepts: list[dict] = [] try: resp = requests.get(url, headers=headers, timeout=20) @@ -1205,6 +1233,8 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: logger.info( "Fetched %d biomedical concepts for category '%s'", len(concepts), category ) + # Populate cache + _category_concepts_cache[ckey] = {"data": concepts, "fetched_at": now} return concepts except Exception as e: # pragma: no cover logger.error("BC concepts by category fetch error for '%s': %s", category, e) @@ -1659,6 +1689,7 @@ async def lifespan(app: FastAPI): # pragma: no cover @app.post("/ui/soa/{soa_id}/concepts_refresh") def ui_refresh_concepts(request: Request, soa_id: int): + """Fetch Biomedical Concepts; refresh cache""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") fetch_biomedical_concepts(force=True) @@ -1713,6 +1744,7 @@ def export_reorder_audit_csv(soa_id: int): @app.get("/concepts/status") def concepts_status(): + """Return diagnostics for Biomedical Concepts fetch/cache.""" return { "count": len(_concept_cache.get("data") or []), "fetched_at": _concept_cache.get("fetched_at"), @@ -1826,6 +1858,7 @@ def _matrix_arrays(soa_id: int): @app.post("/soa") def create_soa(payload: SOACreate): + """Create new Schedule of Activities""" conn = _connect() cur = conn.cursor() # Enforce unique study_id if provided @@ -1858,6 +1891,7 @@ def create_soa(payload: SOACreate): @app.get("/soa/{soa_id}") def get_soa(soa_id: int): + """Return SoA by ID""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -1909,6 +1943,7 @@ def get_soa(soa_id: int): @app.post("/soa/{soa_id}/metadata") def update_soa_metadata(soa_id: int, payload: SOAMetadataUpdate): + """Update metadata for SoA/Study.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -1971,6 +2006,7 @@ def update_soa_metadata(soa_id: int, payload: SOAMetadataUpdate): @app.post("/soa/{soa_id}/activities/{activity_id}/concepts") def set_activity_concepts(soa_id: int, activity_id: int, payload: ConceptsUpdate): + """Update Biomedical Concept assigned to an Activity.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2018,6 +2054,7 @@ def _get_activity_concepts(activity_id: int): def ui_add_activity_concept( request: Request, soa_id: int, activity_id: int, concept_code: str = Form(...) ): + """Add Biomedical Concept to an Activity.""" if not activity_id: raise HTTPException(400, "Missing activity_id") if not _soa_exists(soa_id): @@ -2065,6 +2102,7 @@ def ui_add_activity_concept( def ui_remove_activity_concept( request: Request, soa_id: int, activity_id: int, concept_code: str = Form(...) ): + """Remove Biomedical Concept from Activity.""" if not activity_id: raise HTTPException(400, "Missing activity_id") if not _soa_exists(soa_id): @@ -2099,6 +2137,7 @@ def ui_remove_activity_concept( @app.post("/soa/{soa_id}/cells") def set_cell(soa_id: int, payload: CellCreate): + """Set 'X' in SoA Matrix cell.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2135,6 +2174,7 @@ def set_cell(soa_id: int, payload: CellCreate): @app.get("/soa/{soa_id}/matrix") def get_matrix(soa_id: int): + """Return SoA Matrix for Visits, Activities and assigned Matrix Cells.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -2143,6 +2183,7 @@ def get_matrix(soa_id: int): @app.get("/soa/{soa_id}/export/xlsx") def export_xlsx(soa_id: int, left: Optional[int] = None, right: Optional[int] = None): + """Export SoA Matrix to XLSX.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -2210,23 +2251,23 @@ def export_xlsx(soa_id: int, left: Optional[int] = None, right: Optional[int] = conn_info = _connect() cur_info = conn_info.cursor() cur_info.execute( - "SELECT name, created_at, study_id, study_label, study_description FROM soa WHERE id=?", + "SELECT name, study_id, study_label, study_description, created_at FROM soa WHERE id=?", (soa_id,), ) - info_row = cur_info.fetchone() - conn_info.close() - if info_row: - soa_name_val, created_at_val, study_id_val, study_label_val, study_desc_val = ( - info_row + row_info = cur_info.fetchone() + if row_info: + soa_name_val, study_id_val, study_label_val, study_desc_val, created_at_val = ( + row_info ) else: - soa_name_val, created_at_val, study_id_val, study_label_val, study_desc_val = ( + soa_name_val, study_id_val, study_label_val, study_desc_val, created_at_val = ( f"SOA {soa_id}", None, None, None, None, ) + conn_info.close() freezes = _list_freezes(soa_id) last_freeze_label = freezes[0]["version_label"] if freezes else None last_freeze_time = freezes[0]["created_at"] if freezes else None @@ -2357,7 +2398,7 @@ def export_xlsx(soa_id: int, left: Optional[int] = None, right: Optional[int] = @app.get("/soa/{soa_id}/export/pdf") def export_pdf(soa_id: int): - """Generate a lightweight PDF summary of the SOA (arms, visits, activities, concept mappings). + """Export lightweight PDF summary of the SOA (arms, visits, activities, concept mappings). The PDF is intentionally simple and produced without external dependencies to avoid introducing new packages. It uses a single page with monospaced layout style commands. @@ -2544,6 +2585,7 @@ def get_normalized(soa_id: int): @app.post("/soa/{soa_id}/matrix/import") def import_matrix(soa_id: int, payload: MatrixImport): + """Import SoA Matrix.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not payload.visits: @@ -2642,6 +2684,7 @@ def _reindex(table: str, soa_id: int): @app.delete("/soa/{soa_id}/visits/{visit_id}") def delete_visit(soa_id: int, visit_id: int): + """Delete Visit from an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2679,6 +2722,7 @@ def delete_visit(soa_id: int, visit_id: int): @app.delete("/soa/{soa_id}/activities/{activity_id}") def delete_activity(soa_id: int, activity_id: int): + """Delete Activity from an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2709,6 +2753,7 @@ def delete_activity(soa_id: int, activity_id: int): @app.delete("/soa/{soa_id}/epochs/{epoch_id}") def delete_epoch(soa_id: int, epoch_id: int): + """Delete an Epoch from an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2745,6 +2790,7 @@ def delete_epoch(soa_id: int, epoch_id: int): @app.get("/", response_class=HTMLResponse) def ui_index(request: Request): + """Render home page for the SoA Workbench.""" conn = _connect() cur = conn.cursor() cur.execute( @@ -2773,6 +2819,7 @@ def ui_index(request: Request): @app.post("/ui/soa/{soa_id}/add_activity", response_class=HTMLResponse) def ui_add_activity(request: Request, soa_id: int, name: str = Form(...)): + """Add an Activity to an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") nm = (name or "").strip() @@ -2812,6 +2859,7 @@ def ui_create_soa( study_label: Optional[str] = Form(None), study_description: Optional[str] = Form(None), ): + """Create a new SoA.""" conn = _connect() cur = conn.cursor() # Uniqueness check @@ -2846,6 +2894,7 @@ def ui_update_meta( study_label: Optional[str] = Form(None), study_description: Optional[str] = Form(None), ): + """Update the metadata for an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2890,6 +2939,7 @@ def ui_update_meta( @app.get("/ui/soa/{soa_id}/edit", response_class=HTMLResponse) def ui_edit(request: Request, soa_id: int): + """Render edit HTML page for an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -2980,15 +3030,128 @@ def ui_edit(request: Request, soa_id: int): conn_meta = _connect() cur_meta = conn_meta.cursor() cur_meta.execute( - "SELECT study_id, study_label, study_description FROM soa WHERE id=?", (soa_id,) + "SELECT study_id, study_label, study_description, name, created_at FROM soa WHERE id=?", + (soa_id,), ) meta_row = cur_meta.fetchone() conn_meta.close() + if meta_row: + study_id_val, study_label_val, study_desc_val, soa_name_val, created_at_val = ( + meta_row + ) + else: + study_id_val, study_label_val, study_desc_val, soa_name_val, created_at_val = ( + None, + None, + None, + f"SOA {soa_id}", + None, + ) study_meta = { - "study_id": meta_row[0] if meta_row else None, - "study_label": meta_row[1] if meta_row else None, - "study_description": meta_row[2] if meta_row else None, + "study_id": study_id_val, + "study_label": study_label_val, + "study_description": study_desc_val, + "soa_name": soa_name_val, + "created_at": created_at_val, + } + # Compute next Code_N using a fresh cursor + conn_codes = _connect() + cur_codes = conn_codes.cursor() + # Precompute next Code_N if needed for UI defaults (currently not displayed) + _ = _get_next_code_uid(cur_codes, soa_id) + conn_codes.close() + # Load Protocol Terminology (C174222) options + conn_pt = _connect() + cur_pt = conn_pt.cursor() + cur_pt.execute( + "SELECT cdisc_submission_value FROM protocol_terminology WHERE codelist_code='C174222' ORDER BY cdisc_submission_value" + ) + protocol_terminology_C174222 = [ + {"cdisc_submission_value": r[0] or ""} for r in cur_pt.fetchall() + ] + conn_pt.close() + # Build mapping code_uid -> submission value (Arm Type C174222) + conn_map = _connect() + cur_map = conn_map.cursor() + cur_map.execute( + "SELECT c.code_uid, pt.cdisc_submission_value " + "FROM code c JOIN protocol_terminology pt ON pt.code = c.code " + "WHERE c.soa_id=? AND c.codelist_code='C174222'", + (soa_id,), + ) + code_to_submission = {row[0]: row[1] for row in cur_map.fetchall()} + conn_map.close() + submission_values = { + opt.get("cdisc_submission_value") or "" for opt in protocol_terminology_C174222 } + + # DDF Terminology options for Arm type (C188727) + conn_ddft = _connect() + cur_ddft = conn_ddft.cursor() + cur_ddft.execute( + "SELECT cdisc_submission_value FROM ddf_terminology WHERE codelist_code = 'C188727' ORDER BY cdisc_submission_value" + ) + ddf_terminology_C188727 = [ + {"cdisc_submission_value": r[0] or ""} for r in cur_ddft.fetchall() + ] + conn_ddft.close() + # Build mapping code_uid -> submission value (Arm dataOriginType C188727) + conn_ddf_map = _connect() + cur_ddf_map = conn_ddf_map.cursor() + cur_ddf_map.execute( + "SELECT c.code_uid, dt.cdisc_submission_value " + "FROM code c JOIN ddf_terminology dt ON dt.code = c.code " + "WHERE c.soa_id=? AND c.codelist_code='C188727'", + (soa_id,), + ) + ddf_code_to_submission = {row[0]: row[1] for row in cur_ddf_map.fetchall()} + conn_ddf_map.close() + ddf_submission_values = { + ddf_opt.get("cdisc_submission_value") or "" + for ddf_opt in ddf_terminology_C188727 + } + + base_arms = _fetch_arms_for_edit(soa_id) + arms_enriched = [] + for a in base_arms: + arm_type = a.get("type") + type_display = code_to_submission.get(arm_type) + data_origin_type = a.get("data_origin_type") + data_origin_type_display = ddf_code_to_submission.get(data_origin_type) + if type_display is None and arm_type: + type_display = arm_type if arm_type in submission_values else None + if data_origin_type_display is None and data_origin_type: + data_origin_type_display = ( + data_origin_type if data_origin_type in ddf_submission_values else None + ) + arms_enriched.append( + { + **a, + "type_display": type_display, + "data_origin_type_display": data_origin_type_display, + } + ) + + # Admin audit view: recent arm audits for this SOA + conn_audit = _connect() + cur_audit = conn_audit.cursor() + cur_audit.execute( + "SELECT id, arm_id, action, before_json, after_json, performed_at FROM arm_audit WHERE soa_id=? ORDER BY id DESC LIMIT 50", + (soa_id,), + ) + arm_audits = [ + { + "id": r[0], + "arm_id": r[1], + "action": r[2], + "before_json": r[3], + "after_json": r[4], + "performed_at": r[5], + } + for r in cur_audit.fetchall() + ] + conn_audit.close() + return templates.TemplateResponse( request, "edit.html", @@ -2998,7 +3161,7 @@ def ui_edit(request: Request, soa_id: int): "visits": visits, "activities": activities_page, "elements": elements, - "arms": _fetch_arms_for_edit(soa_id), + "arms": arms_enriched, "cell_map": cell_map, "concepts": concepts, "activity_concepts": activity_concepts, @@ -3010,6 +3173,9 @@ def ui_edit(request: Request, soa_id: int): "freeze_count": len(freeze_list), "last_frozen_at": last_frozen_at, **study_meta, + "protocol_terminology_C174222": protocol_terminology_C174222, + "ddf_terminology_C188727": ddf_terminology_C188727, + "arm_audits": arm_audits, }, ) @@ -3041,9 +3207,9 @@ def ui_concepts_list(request: Request): @app.get("/ui/concept_categories", response_class=HTMLResponse) -def ui_categories_list(request: Request): +def ui_categories_list(request: Request, force: bool = False): """Render table listing biomedical concept categories (name + title + href).""" - categories = fetch_biomedical_concept_categories() or [] + categories = fetch_biomedical_concept_categories(force=force) or [] rows = [ { "name": c.get("name"), @@ -3057,6 +3223,7 @@ def ui_categories_list(request: Request): request, "concept_categories.html", { + "force": force, "rows": rows, "count": len(rows), "missing_key": subscription_key is None, @@ -3065,7 +3232,7 @@ def ui_categories_list(request: Request): @app.get("/ui/concept_categories/view", response_class=HTMLResponse) -def ui_category_detail(request: Request, name: str = ""): +def ui_category_detail(request: Request, name: str = "", force: bool = False): """Render list of biomedical concepts within a given category name. Query params: @@ -3076,7 +3243,7 @@ def ui_category_detail(request: Request, name: str = ""): return HTMLResponse( "

Category name required.

Back

" ) - concepts = fetch_biomedical_concepts_by_category(category_name) or [] + concepts = fetch_biomedical_concepts_by_category(category_name, force=force) or [] rows = [ { "code": c.get("code"), @@ -3090,6 +3257,7 @@ def ui_category_detail(request: Request, name: str = ""): "concept_category_detail.html", { "category": category_name, + "force": force, "rows": rows, "count": len(rows), }, @@ -3345,7 +3513,7 @@ def ui_add_visit( @app.post("/ui/soa/{soa_id}/add_arm", response_class=HTMLResponse) -def ui_add_arm( +async def ui_add_arm( request: Request, soa_id: int, name: str = Form(...), @@ -3359,12 +3527,153 @@ def ui_add_arm( # Accept blank/empty element selection gracefully. The form may submit "" which would 422 with Optional[int]. eid = int(element_id) if element_id and element_id.strip().isdigit() else None payload = ArmCreate(name=name, label=label, description=description, element_id=eid) - create_arm(soa_id, payload) + # Create base arm (function may not return id; fetch if needed) + created = create_arm(soa_id, payload) + # routers.arms.create_arm returns a row dict; extract id + new_arm_id = None + try: + if isinstance(created, dict): + new_arm_id = created.get("id") + elif isinstance(created, int): + new_arm_id = created + except Exception: + new_arm_id = None + if not new_arm_id: + try: + conn_tmp = _connect() + cur_tmp = conn_tmp.cursor() + cur_tmp.execute( + "SELECT id FROM arm WHERE soa_id=? ORDER BY id DESC LIMIT 1", + (soa_id,), + ) + rtmp = cur_tmp.fetchone() + new_arm_id = rtmp[0] if rtmp else None + conn_tmp.close() + except Exception: + new_arm_id = None + if not new_arm_id: + return HTMLResponse( + f"", + status_code=500, + ) + # Read optional type fields with hyphenated names + try: + form_data = await request.form() + arm_type_submission = (form_data.get("arm-type") or "").strip() + data_origin_type_submission = (form_data.get("data-origin-type") or "").strip() + except Exception: + arm_type_submission = "" + data_origin_type_submission = "" + + # If type selections provided, resolve to terminology codes and persist via junction table + if arm_type_submission or data_origin_type_submission: + conn = _connect() + cur = conn.cursor() + logger.info( + "ui_add_arm: received type selections arm-type='%s', data-origin-type='%s' for soa_id=%s arm_id=%s", + arm_type_submission, + data_origin_type_submission, + soa_id, + new_arm_id, + ) + new_type_uid: Optional[str] = None + new_data_origin_uid: Optional[str] = None + if arm_type_submission: + cur.execute( + "SELECT code FROM protocol_terminology WHERE codelist_code='C174222' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (arm_type_submission, arm_type_submission), + ) + r = cur.fetchone() + resolved_code = r[0] if r else None + if resolved_code is None: + logger.warning( + "ui_add_arm: unknown arm type submission '%s' for soa_id=%s", + arm_type_submission, + soa_id, + ) + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + # Create Code_N + new_type_uid = _get_next_code_uid(cur, soa_id) + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_type_uid, + "protocol_terminology", + "C174222", + resolved_code, + ), + ) + logger.info( + "ui_add_arm: created code junction %s -> table=%s list=%s code=%s", + new_type_uid, + "protocol_terminology", + "C174222", + resolved_code, + ) + if data_origin_type_submission: + cur.execute( + "SELECT code FROM ddf_terminology WHERE codelist_code='C188727' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (data_origin_type_submission, data_origin_type_submission), + ) + r2 = cur.fetchone() + resolved_ddf_code = r2[0] if r2 else None + if resolved_ddf_code is None: + logger.warning( + "ui_add_arm: unknown data origin type submission '%s' for soa_id=%s", + data_origin_type_submission, + soa_id, + ) + conn.close() + # Properly escape the value for safety in HTML/JS context + escaped_selection = json.dumps(data_origin_type_submission) + return HTMLResponse( + f"", + status_code=400, + ) + # Create Code_N (continue numbering) + new_data_origin_uid = _get_next_code_uid(cur, soa_id) + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ), + ) + logger.info( + "ui_add_arm: created code junction %s -> table=%s list=%s code=%s", + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ) + # Update arm row with new code_uids + if new_type_uid or new_data_origin_uid: + cur.execute( + "UPDATE arm SET type=COALESCE(?, type), data_origin_type=COALESCE(?, data_origin_type) WHERE id=? AND soa_id=?", + (new_type_uid, new_data_origin_uid, new_arm_id, soa_id), + ) + logger.info( + "ui_add_arm: updated arm id=%s set type=%s data_origin_type=%s", + new_arm_id, + new_type_uid, + new_data_origin_uid, + ) + conn.commit() + # routers.arms.create_arm already records a create audit; avoid duplicating here + conn.close() return HTMLResponse(f"") @app.post("/ui/soa/{soa_id}/update_arm", response_class=HTMLResponse) -def ui_update_arm( +async def ui_update_arm( request: Request, soa_id: int, arm_id: int = Form(...), @@ -3373,12 +3682,259 @@ def ui_update_arm( description: Optional[str] = Form(None), element_id: Optional[str] = Form(None), ): + """Form handler to update an existing Arm.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") - # Coerce possible blank element selection to None; avoid 422 validation error from string "" into Optional[int]. - eid = int(element_id) if element_id and element_id.strip().isdigit() else None - payload = ArmUpdate(name=name, label=label, description=description, element_id=eid) - update_arm(soa_id, arm_id, payload) + + # Read raw form to capture field names with hyphens: 'arm-type' and 'data-origin-type' + try: + form_data = await request.form() + arm_type_submission = (form_data.get("arm-type") or "").strip() + data_origin_type_submission = (form_data.get("data-origin-type") or "").strip() + except Exception: + arm_type_submission = "" + data_origin_type_submission = "" + logger.info( + "ui_update_arm: arm_id=%s soa_id=%s incoming arm-type='%s' data-origin-type='%s'", + arm_id, + soa_id, + arm_type_submission, + data_origin_type_submission, + ) + + # Fetch current arm (including existing type code_uid if any) + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id, name, label, description, COALESCE(type,''), COALESCE(data_origin_type,'') FROM arm WHERE id=? AND soa_id=?", + (arm_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Arm not found") + current_code_uid = row[4] or None + current_data_origin_uid = row[5] or None + # Capture prior code values for audits when code mapping changes without uid change + prior_arm_type_code_value: Optional[str] = None + prior_data_origin_code_value: Optional[str] = None + if current_code_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, current_code_uid), + ) + rcv = cur.fetchone() + prior_arm_type_code_value = rcv[0] if rcv else None + if current_data_origin_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, current_data_origin_uid), + ) + rdv = cur.fetchone() + prior_data_origin_code_value = rdv[0] if rdv else None + before_state = { + "id": row[0], + "name": row[1], + "label": row[2], + "description": row[3], + "type": current_code_uid, + "data_origin_type": current_data_origin_uid, + } + + # Resolve submission value to protocol terminology code (C174222) + resolved_code: Optional[str] = None + if arm_type_submission: + cur.execute( + "SELECT code FROM protocol_terminology WHERE codelist_code='C174222' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (arm_type_submission, arm_type_submission), + ) + r = cur.fetchone() + resolved_code = r[0] if r else None + if resolved_code is None: + logger.warning( + "ui_update_arm: unknown arm type submission '%s' for soa_id=%s arm_id=%s", + arm_type_submission, + soa_id, + arm_id, + ) + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + + # Maintain code table row with immutable code_uid (Code_N unique per SoA) + new_code_uid = current_code_uid + if resolved_code is not None: + if current_code_uid: + # Update existing junction row for this code_uid + cur.execute( + "UPDATE code SET code=?, codelist_code='C174222', codelist_table='protocol_terminology' WHERE soa_id=? AND code_uid=?", + (resolved_code, soa_id, current_code_uid), + ) + logger.info( + "ui_update_arm: updated junction code_uid=%s -> table=%s list=%s code=%s", + current_code_uid, + "protocol_terminology", + "C174222", + resolved_code, + ) + else: + # Create new Code_N within this SoA + new_code_uid = _get_next_code_uid(cur, soa_id) + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_code_uid, + "protocol_terminology", + "C174222", + resolved_code, + ), + ) + logger.info( + "ui_update_arm: created junction code_uid=%s -> table=%s list=%s code=%s", + new_code_uid, + "protocol_terminology", + "C174222", + resolved_code, + ) + + # Resolve Data Origin Type submission value to DDF terminology code (C188727) + resolved_ddf_code: Optional[str] = None + new_data_origin_uid = current_data_origin_uid + if data_origin_type_submission: + cur.execute( + "SELECT code FROM ddf_terminology WHERE codelist_code='C188727' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (data_origin_type_submission, data_origin_type_submission), + ) + r2 = cur.fetchone() + resolved_ddf_code = r2[0] if r2 else None + if resolved_ddf_code is None: + logger.warning( + "ui_update_arm: unknown data origin type submission '%s' for soa_id=%s arm_id=%s", + data_origin_type_submission, + soa_id, + arm_id, + ) + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + # Maintain/Upsert immutable Code_N for DDF mapping + if current_data_origin_uid: + cur.execute( + "UPDATE code SET code=?, codelist_code='C188727', codelist_table='ddf_terminology' WHERE soa_id=? AND code_uid=?", + (resolved_ddf_code, soa_id, current_data_origin_uid), + ) + new_data_origin_uid = current_data_origin_uid + logger.info( + "ui_update_arm: updated junction code_uid=%s -> table=%s list=%s code=%s", + current_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ) + else: + # Create new Code_N, ensuring unique across this SoA + new_data_origin_uid = _get_next_code_uid(cur, soa_id) + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ), + ) + logger.info( + "ui_update_arm: created junction code_uid=%s -> table=%s list=%s code=%s", + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ) + + # Apply arm field updates (including setting type to code_uid if resolved) + new_name = name if name is not None else row[1] + new_label = label if label is not None else row[2] + new_desc = description if description is not None else row[3] + cur.execute( + "UPDATE arm SET name=?, label=?, description=?, type=?, data_origin_type=? WHERE id=? AND soa_id=?", + ( + new_name, + new_label, + new_desc, + new_code_uid, + new_data_origin_uid, + arm_id, + soa_id, + ), + ) + logger.info( + "ui_update_arm: applied UPDATE arm id=%s set name='%s' label='%s' type=%s data_origin_type=%s", + arm_id, + new_name, + new_label, + new_code_uid, + new_data_origin_uid, + ) + conn.commit() + # Capture post-update code values + post_arm_type_code_value: Optional[str] = None + post_data_origin_code_value: Optional[str] = None + if new_code_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, new_code_uid), + ) + rav = cur.fetchone() + post_arm_type_code_value = rav[0] if rav else None + if new_data_origin_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, new_data_origin_uid), + ) + rdv2 = cur.fetchone() + post_data_origin_code_value = rdv2[0] if rdv2 else None + after_state = { + "id": arm_id, + "name": new_name, + "label": new_label, + "description": new_desc, + "type": new_code_uid, + "data_origin_type": new_data_origin_uid, + "type_code": post_arm_type_code_value, + "data_origin_type_code": post_data_origin_code_value, + } + # Record audit if any relevant fields or underlying code mappings changed + if ( + before_state["type"] != after_state["type"] + or before_state["data_origin_type"] != after_state["data_origin_type"] + or prior_arm_type_code_value != post_arm_type_code_value + or prior_data_origin_code_value != post_data_origin_code_value + or before_state["name"] != after_state["name"] + or before_state["label"] != after_state["label"] + or before_state["description"] != after_state["description"] + ): + try: + _record_arm_audit( + soa_id, + "update", + arm_id=arm_id, + before=before_state, + after=after_state, + ) + except Exception: + pass + else: + logger.info( + "ui_update_arm: no-op update detected for arm_id=%s (no field or code changes)", + arm_id, + ) + conn.close() return HTMLResponse(f"") @@ -3390,6 +3946,7 @@ def ui_delete_arm(request: Request, soa_id: int, arm_id: int = Form(...)): @app.post("/ui/soa/{soa_id}/reorder_arms", response_class=HTMLResponse) def ui_reorder_arms(request: Request, soa_id: int, order: str = Form("")): + """Form handler to reorder existing Arms.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] @@ -3429,6 +3986,7 @@ def ui_add_element( testrl: Optional[str] = Form(None), teenrl: Optional[str] = Form(None), ): + """Form handler to add an element.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") name = (name or "").strip() @@ -3523,6 +4081,7 @@ def ui_update_element( testrl: Optional[str] = Form(None), teenrl: Optional[str] = Form(None), ): + """Form handler to update an existing Element.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3594,6 +4153,7 @@ def ui_update_element( @app.post("/ui/soa/{soa_id}/delete_element", response_class=HTMLResponse) def ui_delete_element(request: Request, soa_id: int, element_id: int = Form(...)): + """Form handler to delete an existing Element.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3615,6 +4175,7 @@ def ui_add_epoch( epoch_label: Optional[str] = Form(None), epoch_description: Optional[str] = Form(None), ): + """Form handler to add an Epoch.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3664,6 +4225,7 @@ def ui_update_epoch( epoch_label: Optional[str] = Form(None), epoch_description: Optional[str] = Form(None), ): + """Form handler to update an existing Epoch.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3745,6 +4307,7 @@ def ui_set_activity_concepts( activity_id: int, concept_codes: List[str] = Form([]), ): + """Form handler to set Biomedical Concepts related to an Activity.""" payload = ConceptsUpdate(concept_codes=list(dict.fromkeys(concept_codes))) set_activity_concepts(soa_id, activity_id, payload) # HTMX inline update support @@ -3777,6 +4340,7 @@ def ui_set_activity_concepts( def ui_activity_concepts_cell( request: Request, soa_id: int, activity_id: int, edit: int = 0 ): + """Form handler to return Biomedical Concepts related to an Activity""" # Defensive guard: if activity_id is somehow falsy (should not happen for valid int path param) # surface a clear 400 error rather than proceeding and causing confusing downstream behavior. if not activity_id: @@ -3813,6 +4377,7 @@ def ui_set_cell( activity_id: int = Form(...), status: str = Form("X"), ): + """Form handler to set 'X' in SoA Matrix Cell.""" result = set_cell( soa_id, CellCreate(visit_id=visit_id, activity_id=activity_id, status=status) ) @@ -3867,6 +4432,7 @@ def ui_toggle_cell( @app.post("/ui/soa/{soa_id}/delete_visit", response_class=HTMLResponse) def ui_delete_visit(request: Request, soa_id: int, visit_id: int = Form(...)): + """Form handler to delete a visit.""" # Use API logic to delete and log try: delete_visit(soa_id, visit_id) @@ -3891,6 +4457,7 @@ def ui_set_visit_epoch( epoch_id_raw: str = Form(""), # new field name (blank means clear) epoch_id: str = Form(""), # legacy field name used by template select ): + """Form handler to associate an Epoch with a Visit/Encounter.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Determine provided raw value (prefer epoch_id_raw if non-blank) @@ -3930,12 +4497,14 @@ def ui_set_visit_epoch( @app.post("/ui/soa/{soa_id}/delete_activity", response_class=HTMLResponse) def ui_delete_activity(request: Request, soa_id: int, activity_id: int = Form(...)): + """Form handler to delete an Activity""" delete_activity(soa_id, activity_id) return HTMLResponse(f"") @app.post("/ui/soa/{soa_id}/delete_epoch", response_class=HTMLResponse) def ui_delete_epoch(request: Request, soa_id: int, epoch_id: int = Form(...)): + """Form handler to delete an Epoch.""" delete_epoch(soa_id, epoch_id) return HTMLResponse(f"") @@ -3998,7 +4567,7 @@ def ui_reorder_activities(request: Request, soa_id: int, order: str = Form("")): @app.post("/ui/soa/{soa_id}/reorder_epochs", response_class=HTMLResponse) def ui_reorder_epochs(request: Request, soa_id: int, order: str = Form("")): - """Persist new epoch ordering.""" + """Form handler to persist new epoch ordering.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] @@ -4333,6 +4902,7 @@ def ui_ddf_terminology( uploaded: Optional[str] = None, error: Optional[str] = None, ): + """Detail page to display loaded DDF terminology from the SQLite table""" data = get_ddf_terminology( search=search, code=code, @@ -4498,6 +5068,7 @@ def _get_ddf_sources() -> List[str]: def get_ddf_audit( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Return audit report of DDF Terminology loads.""" conn = _connect() cur = conn.cursor() cur.execute( @@ -4556,6 +5127,7 @@ def _valid_date(d: str) -> bool: def export_ddf_audit_csv( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export DDF terminology audit report in CSV format.""" rows = get_ddf_audit(source=source, start=start, end=end) import csv import io @@ -4603,6 +5175,7 @@ def export_ddf_audit_csv( def export_ddf_audit_json( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export DDF terminology audit report in JSON format.""" return get_ddf_audit(source=source, start=start, end=end) @@ -4613,6 +5186,7 @@ def ui_ddf_audit( start: Optional[str] = None, end: Optional[str] = None, ): + """Display audit report of DDF Terminology loads""" rows = get_ddf_audit(source=source, start=start, end=end) sources = _get_ddf_sources() return templates.TemplateResponse( @@ -4756,6 +5330,7 @@ def load_protocol_terminology( def admin_load_protocol( file_path: Optional[str] = None, sheet_name: str = "Protocol Terminology 2025-09-26" ): + """Load new Protocol Terminology XLS.""" project_root = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(__file__))) ) @@ -4802,6 +5377,7 @@ def get_protocol_terminology( limit: int = 50, offset: int = 0, ): + """Return latest Protocol Terminology loaded into SQLite database.""" limit = max(1, min(limit, 200)) offset = max(0, offset) conn = _connect() @@ -4893,6 +5469,7 @@ def ui_protocol_terminology( uploaded: Optional[str] = None, error: Optional[str] = None, ): + """Form handler to display the latest loaded Protocol Terminology from the SQLite database.""" data = get_protocol_terminology( search=search, code=code, @@ -4922,6 +5499,7 @@ def ui_protocol_upload( sheet_name: str = Form("Protocol Terminology 2025-09-26"), file: UploadFile = File(...), ): + """Form handler for the upload of Protocol Terminology XLS.""" filename = file.filename or "uploaded.xls" if not (filename.lower().endswith(".xls") or filename.lower().endswith(".xlsx")): return HTMLResponse( @@ -5051,6 +5629,7 @@ def _get_protocol_sources() -> List[str]: def get_protocol_audit( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Return the Protocol Terminology audit report.""" conn = _connect() cur = conn.cursor() cur.execute( @@ -5108,6 +5687,7 @@ def _valid_date(d: str) -> bool: def export_protocol_audit_csv( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export the latest Protocol Terminology from the SQLite database in CSV format.""" rows = get_protocol_audit(source=source, start=start, end=end) import csv import io @@ -5155,6 +5735,7 @@ def export_protocol_audit_csv( def export_protocol_audit_json( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export the latest Protocol Terminology from the SQLite database in JSON format.""" return get_protocol_audit(source=source, start=start, end=end) @@ -5165,6 +5746,7 @@ def ui_protocol_audit( start: Optional[str] = None, end: Optional[str] = None, ): + """Form handler for display of the Protocol Terminology audit report.""" rows = get_protocol_audit(source=source, start=start, end=end) sources = _get_protocol_sources() return templates.TemplateResponse( diff --git a/src/soa_builder/web/db.py b/src/soa_builder/web/db.py index 2b3e336..681a2b5 100644 --- a/src/soa_builder/web/db.py +++ b/src/soa_builder/web/db.py @@ -8,4 +8,12 @@ def _connect(): - return sqlite3.connect(DB_PATH) + conn = sqlite3.connect(DB_PATH, timeout=5.0, check_same_thread=False) + try: + # Improve concurrency and reduce lock errors + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA busy_timeout=3000") + except Exception: + pass + return conn diff --git a/src/soa_builder/web/initialize_database.py b/src/soa_builder/web/initialize_database.py index fde39eb..8ccd3ac 100644 --- a/src/soa_builder/web/initialize_database.py +++ b/src/soa_builder/web/initialize_database.py @@ -146,5 +146,19 @@ def _init_db(): performed_at TEXT NOT NULL )""" ) + + # create the code table to store unique Code_uid values associated with study objects + cur.execute( + """CREATE TABLE IF NOT EXISTS code ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + code_uid TEXT, -- immutable Code_N identifier unique within an SOA + codelist_table TEXT, + codelist_code TEXT NOT NULL, + code TEXT NOT NULL, + UNIQUE(soa_id, code_uid) + )""" + ) + conn.commit() conn.close() diff --git a/src/soa_builder/web/static/style.css b/src/soa_builder/web/static/style.css index 04ae631..6c3d410 100644 --- a/src/soa_builder/web/static/style.css +++ b/src/soa_builder/web/static/style.css @@ -1,4 +1,4 @@ -body { font-family: system-ui, Arial, sans-serif; margin: 1.5rem; } +body { font-family: Arial, Helvetica, sans-serif; margin: 1.5rem; } header h1 { margin: 0; } nav a { margin-right: 1rem; } .table, table { border-collapse: collapse; } diff --git a/src/soa_builder/web/templates/concept_categories.html b/src/soa_builder/web/templates/concept_categories.html index b59342f..83852ba 100644 --- a/src/soa_builder/web/templates/concept_categories.html +++ b/src/soa_builder/web/templates/concept_categories.html @@ -5,6 +5,10 @@

Biomedical Concept Categories ({{ count }}) +

+ Refresh Categories (bypass cache) + {% if force %}Cache refreshed{% endif %} +

@@ -14,7 +18,7 @@

Biomedical Concept Categories ({{ count }}) - + Concepts Name Title @@ -25,11 +29,12 @@

Biomedical Concept Categories ({{ count }}) {% if r.name %} - {{ r.name }} + View Concepts {% else %} n/a {% endif %} + {{ r.name }} {{ r.title }} {% endfor %} diff --git a/src/soa_builder/web/templates/concept_category_detail.html b/src/soa_builder/web/templates/concept_category_detail.html index 16f63b5..df02857 100644 --- a/src/soa_builder/web/templates/concept_category_detail.html +++ b/src/soa_builder/web/templates/concept_category_detail.html @@ -1,6 +1,11 @@ {% extends 'base.html' %} {% block content %}

Biomedical Concepts in Category: {{ category }}

+

+ Refresh (bypass cache) + {% if force %}Cache bypassed{% endif %} + | Back to Categories +

Total concepts: {{ count }}

{% if rows %} diff --git a/src/soa_builder/web/templates/edit.html b/src/soa_builder/web/templates/edit.html index d670aa0..31ec5eb 100644 --- a/src/soa_builder/web/templates/edit.html +++ b/src/soa_builder/web/templates/edit.html @@ -82,34 +82,6 @@

Editing SoA {{ soa_id }}

-
- Epochs ({{ epochs|length }}) (drag to reorder) -
    - {% for e in epochs %} -
  • - {{ e.order_index }}. E{{ e.epoch_seq }} {{ e.name }} - {% if e.epoch_label %}[{{ e.epoch_label }}]{% endif %} -
    - - -
    -
    - - - - - -
    -
    • - {% endfor %} -
-
- - - - -
-
Visits ({{ visits|length }}) (drag to reorder)
    @@ -145,8 +117,6 @@

    Editing SoA {{ soa_id }}

-
-
Activities ({{ activities|length }}) (drag to reorder)
    @@ -164,7 +134,37 @@

    Editing SoA {{ soa_id }}

-
+
+
+
+ Epochs ({{ epochs|length }}) (drag to reorder) +
    + {% for e in epochs %} +
  • + {{ e.order_index }}. E{{ e.epoch_seq }} {{ e.name }} + {% if e.epoch_label %}[{{ e.epoch_label }}]{% endif %} +
    + + +
    +
    + + + + + +
    +
    • + {% endfor %} +
+
+ + + + +
+
+
Elements ({{ elements|length }}) (drag to reorder)
    {% for el in elements %} @@ -197,7 +197,7 @@

    Editing SoA {{ soa_id }}

-
+
Arms ({{ arms|length }}) (drag to reorder)
    {% for arm in arms %} @@ -213,6 +213,28 @@

    Editing SoA {{ soa_id }}

    + {# Type selection from protocol terminology C174222 #} + {% if protocol_terminology_C174222 %} + + {% endif %} + {# Type selection from ddf terminology C188727 #} + {% if ddf_terminology_C188727 %} + + {% endif %} @@ -222,9 +244,57 @@

    Editing SoA {{ soa_id }}

    + {% if protocol_terminology_C174222 %} + + {% endif %} + {% if ddf_terminology_C188727 %} + + {% endif %}
+ +
+
+
+ Arm Audit (latest {{ arm_audits|length }}) + {% if arm_audits %} + + + + + + + + + + {% for au in arm_audits %} + + + + + + + + + {% endfor %} +
IDArmActionPerformedBeforeAfter
{{ au.id }}{{ au.arm_id }}{{ au.action }}{{ au.performed_at }}{{ au.before_json or '' }}{{ au.after_json or '' }}
+ {% else %} +
No arm audit entries yet.
+ {% endif %} +
diff --git a/src/soa_builder/web/utils.py b/src/soa_builder/web/utils.py new file mode 100644 index 0000000..87e37f7 --- /dev/null +++ b/src/soa_builder/web/utils.py @@ -0,0 +1,20 @@ +from typing import Any + + +def get_next_code_uid(cur: Any, soa_id: int) -> str: + """Compute next unique Code_N for the given SOA. + + Assumes `cur` is a sqlite cursor within an open transaction. + """ + cur.execute( + "SELECT code_uid FROM code WHERE soa_id=? AND code_uid LIKE 'Code_%'", + (soa_id,), + ) + existing = [x[0] for x in cur.fetchall() if x[0]] + n = 1 + if existing: + try: + n = max(int(x.split("_")[1]) for x in existing) + 1 + except Exception: + n = len(existing) + 1 + return f"Code_{n}" diff --git a/tests/test_categories_cache.py b/tests/test_categories_cache.py new file mode 100644 index 0000000..c5864ff --- /dev/null +++ b/tests/test_categories_cache.py @@ -0,0 +1,118 @@ +from types import SimpleNamespace + +import pytest + +from soa_builder.web.app import ( + fetch_biomedical_concept_categories, + _bc_categories_cache, +) + + +class DummyResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +@pytest.fixture(autouse=True) +def clear_categories_cache(): + _bc_categories_cache.clear() + + +def test_categories_cache_hit(monkeypatch): + call_count = SimpleNamespace(n=0) + + payload = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A", "title": "TitleA"} + }, + }, + { + "name": "CategoryB", + "_links": { + "self": {"href": "/mdr/bc/categories/B", "title": "TitleB"} + }, + }, + ] + } + } + + def fake_get(url, headers=None, timeout=None): + call_count.n += 1 + return DummyResponse(payload) + + monkeypatch.setattr("requests.get", fake_get) + + # First call populates cache + cats1 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 1 + assert len(cats1) == 2 + + # Second call within TTL should hit cache, no new HTTP call + cats2 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 1 + assert cats2 == cats1 + + +def test_categories_cache_force_bypass(monkeypatch): + call_count = SimpleNamespace(n=0) + + payload1 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A", "title": "TitleA"} + }, + } + ] + } + } + payload2 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A2", "title": "TitleA2"} + }, + }, + { + "name": "CategoryC", + "_links": { + "self": {"href": "/mdr/bc/categories/C", "title": "TitleC"} + }, + }, + ] + } + } + + def fake_get(url, headers=None, timeout=None): + call_count.n += 1 + # Return payload1 first, payload2 thereafter + return DummyResponse(payload1 if call_count.n == 1 else payload2) + + monkeypatch.setattr("requests.get", fake_get) + + # Populate cache + cats1 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 1 + assert [c["name"] for c in cats1] == ["CategoryA"] + + # Force bypass should trigger a new HTTP call and new content + cats2 = fetch_biomedical_concept_categories(force=True) + assert call_count.n == 2 + assert [c["name"] for c in cats2] == ["CategoryA", "CategoryC"] + + # Regular call after force should use cached latest content (no extra HTTP) + cats3 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 2 + assert cats3 == cats2 diff --git a/tests/test_categories_ui_force.py b/tests/test_categories_ui_force.py new file mode 100644 index 0000000..194cfb1 --- /dev/null +++ b/tests/test_categories_ui_force.py @@ -0,0 +1,89 @@ +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from soa_builder.web.app import app, _bc_categories_cache + +client = TestClient(app) + + +class DummyResponse: + def __init__(self, payload, status_code=200, text=""): + self._payload = payload + self.status_code = status_code + self.text = text or "" + + def json(self): + return self._payload + + +@pytest.fixture(autouse=True) +def clear_categories_cache(): + _bc_categories_cache.clear() + + +def test_ui_categories_force_bypass(monkeypatch): + call_count = SimpleNamespace(n=0) + + payload1 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A", "title": "TitleA"} + }, + }, + ] + } + } + payload2 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A2", "title": "TitleA2"} + }, + }, + { + "name": "CategoryC", + "_links": { + "self": {"href": "/mdr/bc/categories/C", "title": "TitleC"} + }, + }, + ] + } + } + + def fake_get(url, headers=None, timeout=None): + call_count.n += 1 + return DummyResponse(payload1 if call_count.n == 1 else payload2, text="ok") + + monkeypatch.setattr("requests.get", fake_get) + + # Initial request populates cache, shows TitleA only + r1 = client.get("/ui/concept_categories") + assert r1.status_code == 200 + html1 = r1.text + assert "TitleA" in html1 + assert "TitleA2" not in html1 + assert "TitleC" not in html1 + assert call_count.n == 1 + + # Force bypass should fetch again and render updated titles + r2 = client.get("/ui/concept_categories?force=1") + assert r2.status_code == 200 + html2 = r2.text + assert "TitleA2" in html2 + assert "TitleC" in html2 + assert call_count.n == 2 + + # Subsequent non-force call should serve cached updated content (no new HTTP) + r3 = client.get("/ui/concept_categories") + assert r3.status_code == 200 + html3 = r3.text + assert "TitleA2" in html3 + assert "TitleC" in html3 + assert call_count.n == 2 diff --git a/tests/test_code_uid_generation.py b/tests/test_code_uid_generation.py new file mode 100644 index 0000000..bfe7437 --- /dev/null +++ b/tests/test_code_uid_generation.py @@ -0,0 +1,80 @@ +from soa_builder.web.db import _connect +from soa_builder.web.initialize_database import _init_db +from soa_builder.web.app import _get_next_code_uid + + +def setup_module(module): + # Ensure test DB is initialized + _init_db() + + +def test_get_next_code_uid_empty_soa(): + conn = _connect() + cur = conn.cursor() + # create a dummy SOA + cur.execute( + "INSERT INTO soa (name, created_at) VALUES (?, datetime('now'))", ("TestStudy",) + ) + soa_id = cur.lastrowid + conn.commit() + # No existing codes -> should return Code_1 + code_uid = _get_next_code_uid(cur, soa_id) + assert code_uid == "Code_1" + conn.close() + + +def test_get_next_code_uid_mixed_existing(): + conn = _connect() + cur = conn.cursor() + # create a new SOA + cur.execute( + "INSERT INTO soa (name, created_at) VALUES (?, datetime('now'))", + ("TestStudy2",), + ) + soa_id = cur.lastrowid + conn.commit() + # Insert mixed existing code_uids + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + (soa_id, "Code_1", "protocol_terminology", "C174222", "X"), + ) + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + (soa_id, "Code_3", "ddf_terminology", "C188727", "Y"), + ) + # Malformed tail should be ignored in max() and trigger fallback only if parsing fails for all + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + (soa_id, "Code_X", "protocol_terminology", "C174222", "Z"), + ) + conn.commit() + # Expect next to be max numeric + 1 -> Code_4 + next_uid = _get_next_code_uid(cur, soa_id) + assert next_uid == "Code_4" + conn.close() + + +def test_get_next_code_uid_all_invalid_tails(): + conn = _connect() + cur = conn.cursor() + # create a new SOA + cur.execute( + "INSERT INTO soa (name, created_at) VALUES (?, datetime('now'))", + ("TestStudy3",), + ) + soa_id = cur.lastrowid + conn.commit() + # Insert only invalid tails that cannot be parsed as integers + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + (soa_id, "Code_A", "protocol_terminology", "C174222", "X"), + ) + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + (soa_id, "Code_B", "ddf_terminology", "C188727", "Y"), + ) + conn.commit() + # Fallback should use len(existing)+1 -> 3 + next_uid = _get_next_code_uid(cur, soa_id) + assert next_uid == "Code_3" + conn.close() diff --git a/tests/test_concept_categories.py b/tests/test_concept_categories.py index 4c07769..13b90a7 100644 --- a/tests/test_concept_categories.py +++ b/tests/test_concept_categories.py @@ -42,7 +42,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, {"items": []}) monkeypatch.setattr("requests.get", fake_get) - fetch_biomedical_concepts_by_category(raw_category) + fetch_biomedical_concepts_by_category(raw_category, force=True) assert len(captured_urls) == 1 assert expected_url_fragment in captured_urls[0] @@ -68,7 +68,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, payload) monkeypatch.setattr("requests.get", fake_get) - concepts = fetch_biomedical_concepts_by_category("Liver Findings") + concepts = fetch_biomedical_concepts_by_category("Liver Findings", force=True) assert {c["code"] for c in concepts} == {"ALT", "AST"} assert all( c["href"].startswith( @@ -100,7 +100,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, payload) monkeypatch.setattr("requests.get", fake_get) - concepts = fetch_biomedical_concepts_by_category("Liver Findings") + concepts = fetch_biomedical_concepts_by_category("Liver Findings", force=True) codes = {c["code"] for c in concepts} assert codes == {"ALT", "AST"} # Titles preserved @@ -114,7 +114,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(404, {}, text="Not Found") monkeypatch.setattr("requests.get", fake_get) - concepts = fetch_biomedical_concepts_by_category("Liver Findings") + concepts = fetch_biomedical_concepts_by_category("Liver Findings", force=True) assert concepts == [] @@ -139,7 +139,9 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, payload) monkeypatch.setattr("requests.get", fake_get) - resp = client.get("/ui/concept_categories/view", params={"name": "Liver Findings"}) + resp = client.get( + "/ui/concept_categories/view", params={"name": "Liver Findings", "force": True} + ) assert resp.status_code == 200 text = resp.text # Internal links to concept detail page @@ -156,6 +158,8 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, {"items": []}) monkeypatch.setattr("requests.get", fake_get) - resp = client.get("/ui/concept_categories/view", params={"name": "Liver Findings"}) + resp = client.get( + "/ui/concept_categories/view", params={"name": "Liver Findings", "force": True} + ) assert resp.status_code == 200 assert "No concepts found for this category." in resp.text diff --git a/tests/test_concept_category_force_refresh.py b/tests/test_concept_category_force_refresh.py new file mode 100644 index 0000000..c0c512f --- /dev/null +++ b/tests/test_concept_category_force_refresh.py @@ -0,0 +1,66 @@ +import json +from typing import List +from fastapi.testclient import TestClient + +from soa_builder.web.app import app, _category_concepts_cache + +client = TestClient(app) + + +class DummyResp: + def __init__(self, status_code: int, json_data=None, text: str = ""): + self.status_code = status_code + self._json = json_data + self.text = text or json.dumps(json_data or {}) + + def json(self): + if self._json is None: + raise ValueError("No JSON") + return self._json + + +def test_ui_category_force_refresh(monkeypatch): + _category_concepts_cache.clear() + calls: List[str] = [] + payload1 = { + "items": [ + { + "code": "ALT", + "title": "Alanine", + "href": "/mdr/bc/biomedicalconcepts/ALT", + } + ] + } + payload2 = { + "items": [ + { + "code": "AST", + "title": "Aspartate", + "href": "/mdr/bc/biomedicalconcepts/AST", + } + ] + } + + def fake_get(url, headers=None, timeout=0): + # First call returns ALT, subsequent calls return AST + if not calls: + calls.append(url) + return DummyResp(200, payload1) + calls.append(url) + return DummyResp(200, payload2) + + monkeypatch.setattr("requests.get", fake_get) + # Initial request populates cache with ALT + r1 = client.get("/ui/concept_categories/view", params={"name": "Force Test"}) + assert r1.status_code == 200 + assert "/ui/concepts/ALT" in r1.text + # Second request without force still uses cache (ALT) + r2 = client.get("/ui/concept_categories/view", params={"name": "Force Test"}) + assert r2.status_code == 200 + assert "/ui/concepts/ALT" in r2.text + # Third request with force bypasses cache and shows AST + r3 = client.get( + "/ui/concept_categories/view", params={"name": "Force Test", "force": True} + ) + assert r3.status_code == 200 + assert "/ui/concepts/AST" in r3.text diff --git a/tests/test_concepts_by_category_ui_force.py b/tests/test_concepts_by_category_ui_force.py new file mode 100644 index 0000000..c036973 --- /dev/null +++ b/tests/test_concepts_by_category_ui_force.py @@ -0,0 +1,88 @@ +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from soa_builder.web.app import app, _category_concepts_cache + +client = TestClient(app) + + +class DummyResponse: + def __init__(self, payload, status_code=200, text=""): + self._payload = payload + self.status_code = status_code + self.text = text or "" + + def json(self): + return self._payload + + +@pytest.fixture(autouse=True) +def clear_concepts_cache(): + _category_concepts_cache.clear() + + +def test_ui_concepts_by_category_force_bypass(monkeypatch): + call_count = SimpleNamespace(n=0) + + # The app queries: /mdr/bc/biomedicalconcepts?category=CategoryA (encoded) + # Provide payloads representing concepts in direct 'items' list form. + payload1 = { + "items": [ + { + "code": "C100", + "title": "Alpha", + "href": "/mdr/bc/biomedicalconcepts/C100", + }, + ] + } + payload2 = { + "items": [ + { + "code": "C100", + "title": "Alpha v2", + "href": "/mdr/bc/biomedicalconcepts/C100", + }, + { + "code": "C200", + "title": "Beta", + "href": "/mdr/bc/biomedicalconcepts/C200", + }, + ] + } + + def fake_get(url, headers=None, timeout=None): + # Ensure we are mocking the concepts-by-category endpoint + assert "/mdr/bc/biomedicalconcepts?category=" in url + call_count.n += 1 + return DummyResponse(payload1 if call_count.n == 1 else payload2, text="ok") + + monkeypatch.setattr("requests.get", fake_get) + + # Initial request populates cache, shows Alpha only + r1 = client.get("/ui/concept_categories/view", params={"name": "CategoryA"}) + assert r1.status_code == 200 + html1 = r1.text + assert "Alpha" in html1 + assert "Alpha v2" not in html1 + assert "Beta" not in html1 + assert call_count.n == 1 + + # Force bypass should fetch again and render updated concept titles + r2 = client.get( + "/ui/concept_categories/view", params={"name": "CategoryA", "force": 1} + ) + assert r2.status_code == 200 + html2 = r2.text + assert "Alpha v2" in html2 + assert "Beta" in html2 + assert call_count.n == 2 + + # Subsequent non-force call should serve cached updated content (no new HTTP) + r3 = client.get("/ui/concept_categories/view", params={"name": "CategoryA"}) + assert r3.status_code == 200 + html3 = r3.text + assert "Alpha v2" in html3 + assert "Beta" in html3 + assert call_count.n == 2