diff --git a/.gitignore b/.gitignore index b6ae2a7..19300fd 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ Thumbs.db # SQLite / local DBs *.db +*.db-shm +*.db-wal *.sqlite # Environment variables / secrets (add if created) diff --git a/docs/api_endpoints.xlsx b/docs/api_endpoints.xlsx index 903fce8..f49a332 100644 Binary files a/docs/api_endpoints.xlsx and b/docs/api_endpoints.xlsx differ diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index 705bbc6..2721b6a 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -65,8 +65,8 @@ from .routers import rollback as rollback_router from .routers import visits as visits_router from .routers.arms import create_arm # re-export for backward compatibility -from .routers.arms import delete_arm, update_arm -from .schemas import ArmCreate, ArmUpdate, SOACreate, SOAMetadataUpdate +from .routers.arms import delete_arm +from .schemas import ArmCreate, SOACreate, SOAMetadataUpdate load_dotenv() # must come BEFORE reading env-based configuration so values are populated DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") @@ -78,6 +78,12 @@ # SDTM dataset specializations cache (similar TTL) _sdtm_specializations_cache = {"data": None, "fetched_at": 0} _SDTM_SPECIALIZATIONS_CACHE_TTL = 60 * 60 +# Category-specific biomedical concepts cache (per category key) +_category_concepts_cache: dict[str, dict] = {} +_CATEGORY_CONCEPTS_CACHE_TTL = 60 * 60 # 1 hour +# Biomedical concept categories cache (whole list) +_bc_categories_cache = {"data": None, "fetched_at": 0} +_BC_CATEGORIES_CACHE_TTL = 60 * 60 # 1 hour app = FastAPI(title="SoA Builder API", version="0.1.0") logger = logging.getLogger("soa_builder.concepts") if not logger.handlers: @@ -875,7 +881,7 @@ def _fetch_arms_for_edit(soa_id: int) -> list[dict]: conn = _connect() cur = conn.cursor() cur.execute( - "SELECT id,name,label,description,order_index FROM arm WHERE soa_id=? ORDER BY order_index", + "SELECT id,name,label,description,order_index,COALESCE(type,''),COALESCE(data_origin_type,'') FROM arm WHERE soa_id=? ORDER BY order_index", (soa_id,), ) rows = [ @@ -885,6 +891,8 @@ def _fetch_arms_for_edit(soa_id: int) -> list[dict]: "label": r[2], "description": r[3], "order_index": r[4], + "type": r[5] or None, + "data_origin_type": r[6] or None, } for r in cur.fetchall() ] @@ -1000,7 +1008,7 @@ def _fetch_matrix(soa_id: int): return visits, activities, cells -def fetch_biomedical_concept_categories() -> list[dict]: +def fetch_biomedical_concept_categories(force: bool = False) -> list[dict]: """Return list of Biomedical Concept Categories from CDISC Library. Normalized shape: @@ -1027,6 +1035,15 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: return base_prefix + h return base_prefix + "/" + h + # Cache lookup + now = time.time() + if ( + not force + and _bc_categories_cache.get("data") + and now - _bc_categories_cache.get("fetched_at", 0) < _BC_CATEGORIES_CACHE_TTL + ): + return _bc_categories_cache.get("data") or [] + try: resp = requests.get(url, headers=headers, timeout=15) if resp.status_code != 200: @@ -1069,13 +1086,15 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: ) categories.sort(key=lambda c: (c["title"] or "").lower()) logger.info("Fetched %d BC categories from remote API", len(categories)) + _bc_categories_cache["data"] = categories + _bc_categories_cache["fetched_at"] = now return categories except Exception as e: # pragma: no cover logger.error("BC categories fetch error: %s", e) return [] -def fetch_biomedical_concepts_by_category(name: str) -> list[dict]: +def fetch_biomedical_concepts_by_category(name: str, force: bool = False) -> list[dict]: """Return biomedical concepts for a given category name. Uses category-specific endpoint: /mdr/bc/biomedicalconcepts?category= @@ -1108,6 +1127,14 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: return base_prefix + h return base_prefix + "/" + h + # Cache lookup + now = time.time() + ckey = category.lower() + if not force: + cached = _category_concepts_cache.get(ckey) + if cached and now - cached.get("fetched_at", 0) < _CATEGORY_CONCEPTS_CACHE_TTL: + return cached.get("data", []) or [] + concepts: list[dict] = [] try: resp = requests.get(url, headers=headers, timeout=20) @@ -1205,6 +1232,8 @@ def _normalize_href(h: Optional[str]) -> Optional[str]: logger.info( "Fetched %d biomedical concepts for category '%s'", len(concepts), category ) + # Populate cache + _category_concepts_cache[ckey] = {"data": concepts, "fetched_at": now} return concepts except Exception as e: # pragma: no cover logger.error("BC concepts by category fetch error for '%s': %s", category, e) @@ -1659,6 +1688,7 @@ async def lifespan(app: FastAPI): # pragma: no cover @app.post("/ui/soa/{soa_id}/concepts_refresh") def ui_refresh_concepts(request: Request, soa_id: int): + """Fetch Biomedical Concepts; refresh cache""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") fetch_biomedical_concepts(force=True) @@ -1713,6 +1743,7 @@ def export_reorder_audit_csv(soa_id: int): @app.get("/concepts/status") def concepts_status(): + """Return diagnostics for Biomedical Concepts fetch/cache.""" return { "count": len(_concept_cache.get("data") or []), "fetched_at": _concept_cache.get("fetched_at"), @@ -1826,6 +1857,7 @@ def _matrix_arrays(soa_id: int): @app.post("/soa") def create_soa(payload: SOACreate): + """Create new Schedule of Activities""" conn = _connect() cur = conn.cursor() # Enforce unique study_id if provided @@ -1858,6 +1890,7 @@ def create_soa(payload: SOACreate): @app.get("/soa/{soa_id}") def get_soa(soa_id: int): + """Return SoA by ID""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -1909,6 +1942,7 @@ def get_soa(soa_id: int): @app.post("/soa/{soa_id}/metadata") def update_soa_metadata(soa_id: int, payload: SOAMetadataUpdate): + """Update metadata for SoA/Study.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -1971,6 +2005,7 @@ def update_soa_metadata(soa_id: int, payload: SOAMetadataUpdate): @app.post("/soa/{soa_id}/activities/{activity_id}/concepts") def set_activity_concepts(soa_id: int, activity_id: int, payload: ConceptsUpdate): + """Update Biomedical Concept assigned to an Activity.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2018,6 +2053,7 @@ def _get_activity_concepts(activity_id: int): def ui_add_activity_concept( request: Request, soa_id: int, activity_id: int, concept_code: str = Form(...) ): + """Add Biomedical Concept to an Activity.""" if not activity_id: raise HTTPException(400, "Missing activity_id") if not _soa_exists(soa_id): @@ -2065,6 +2101,7 @@ def ui_add_activity_concept( def ui_remove_activity_concept( request: Request, soa_id: int, activity_id: int, concept_code: str = Form(...) ): + """Remove Biomedical Concept from Activity.""" if not activity_id: raise HTTPException(400, "Missing activity_id") if not _soa_exists(soa_id): @@ -2099,6 +2136,7 @@ def ui_remove_activity_concept( @app.post("/soa/{soa_id}/cells") def set_cell(soa_id: int, payload: CellCreate): + """Set 'X' in SoA Matrix cell.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2135,6 +2173,7 @@ def set_cell(soa_id: int, payload: CellCreate): @app.get("/soa/{soa_id}/matrix") def get_matrix(soa_id: int): + """Return SoA Matrix for Visits, Activities and assigned Matrix Cells.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -2143,6 +2182,7 @@ def get_matrix(soa_id: int): @app.get("/soa/{soa_id}/export/xlsx") def export_xlsx(soa_id: int, left: Optional[int] = None, right: Optional[int] = None): + """Export SoA Matrix to XLSX.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -2357,7 +2397,7 @@ def export_xlsx(soa_id: int, left: Optional[int] = None, right: Optional[int] = @app.get("/soa/{soa_id}/export/pdf") def export_pdf(soa_id: int): - """Generate a lightweight PDF summary of the SOA (arms, visits, activities, concept mappings). + """Export lightweight PDF summary of the SOA (arms, visits, activities, concept mappings). The PDF is intentionally simple and produced without external dependencies to avoid introducing new packages. It uses a single page with monospaced layout style commands. @@ -2544,6 +2584,7 @@ def get_normalized(soa_id: int): @app.post("/soa/{soa_id}/matrix/import") def import_matrix(soa_id: int, payload: MatrixImport): + """Import SoA Matrix.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not payload.visits: @@ -2642,6 +2683,7 @@ def _reindex(table: str, soa_id: int): @app.delete("/soa/{soa_id}/visits/{visit_id}") def delete_visit(soa_id: int, visit_id: int): + """Delete Visit from an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2679,6 +2721,7 @@ def delete_visit(soa_id: int, visit_id: int): @app.delete("/soa/{soa_id}/activities/{activity_id}") def delete_activity(soa_id: int, activity_id: int): + """Delete Activity from an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2709,6 +2752,7 @@ def delete_activity(soa_id: int, activity_id: int): @app.delete("/soa/{soa_id}/epochs/{epoch_id}") def delete_epoch(soa_id: int, epoch_id: int): + """Delete an Epoch from an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2745,6 +2789,7 @@ def delete_epoch(soa_id: int, epoch_id: int): @app.get("/", response_class=HTMLResponse) def ui_index(request: Request): + """Render home page for the SoA Workbench.""" conn = _connect() cur = conn.cursor() cur.execute( @@ -2773,6 +2818,7 @@ def ui_index(request: Request): @app.post("/ui/soa/{soa_id}/add_activity", response_class=HTMLResponse) def ui_add_activity(request: Request, soa_id: int, name: str = Form(...)): + """Add an Activity to an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") nm = (name or "").strip() @@ -2812,6 +2858,7 @@ def ui_create_soa( study_label: Optional[str] = Form(None), study_description: Optional[str] = Form(None), ): + """Create a new SoA.""" conn = _connect() cur = conn.cursor() # Uniqueness check @@ -2846,6 +2893,7 @@ def ui_update_meta( study_label: Optional[str] = Form(None), study_description: Optional[str] = Form(None), ): + """Update the metadata for an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -2890,6 +2938,7 @@ def ui_update_meta( @app.get("/ui/soa/{soa_id}/edit", response_class=HTMLResponse) def ui_edit(request: Request, soa_id: int): + """Render edit HTML page for an SoA.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) @@ -2989,6 +3038,98 @@ def ui_edit(request: Request, soa_id: int): "study_label": meta_row[1] if meta_row else None, "study_description": meta_row[2] if meta_row else None, } + # Protocol terminology options for Arm Type (C174222) + conn_pt = _connect() + cur_pt = conn_pt.cursor() + cur_pt.execute( + "SELECT cdisc_submission_value FROM protocol_terminology WHERE codelist_code='C174222' ORDER BY cdisc_submission_value" + ) + protocol_terminology_C174222 = [ + {"cdisc_submission_value": r[0] or ""} for r in cur_pt.fetchall() + ] + conn_pt.close() + # Build mapping code_uid -> submission value (Arm Type C174222) + conn_map = _connect() + cur_map = conn_map.cursor() + cur_map.execute( + "SELECT c.code_uid, pt.cdisc_submission_value " + "FROM code c JOIN protocol_terminology pt ON pt.code = c.code " + "WHERE c.soa_id=? AND c.codelist_code='C174222'", + (soa_id,), + ) + code_to_submission = {row[0]: row[1] for row in cur_map.fetchall()} + conn_map.close() + submission_values = { + opt.get("cdisc_submission_value") or "" for opt in protocol_terminology_C174222 + } + + # DDF Terminology options for Arm type (C188727) + conn_ddft = _connect() + cur_ddft = conn_ddft.cursor() + cur_ddft.execute( + "SELECT cdisc_submission_value FROM ddf_terminology WHERE codelist_code = 'C188727' ORDER BY cdisc_submission_value" + ) + ddf_terminology_C188727 = [ + {"cdisc_submission_value": r[0] or ""} for r in cur_ddft.fetchall() + ] + conn_ddft.close() + # Build mapping code_uid -> submission value (Arm dataOriginType C188727) + conn_ddf_map = _connect() + cur_ddf_map = conn_ddf_map.cursor() + cur_ddf_map.execute( + "SELECT c.code_uid, dt.cdisc_submission_value " + "FROM code c JOIN ddf_terminology dt ON dt.code = c.code " + "WHERE c.soa_id=? AND c.codelist_code='C188727'", + (soa_id,), + ) + ddf_code_to_submission = {row[0]: row[1] for row in cur_ddf_map.fetchall()} + conn_ddf_map.close() + ddf_submission_values = { + ddf_opt.get("cdisc_submission_value") or "" + for ddf_opt in ddf_terminology_C188727 + } + + base_arms = _fetch_arms_for_edit(soa_id) + arms_enriched = [] + for a in base_arms: + type = a.get("type") + type_display = code_to_submission.get(type) + data_origin_type = a.get("data_origin_type") + data_origin_type_display = ddf_code_to_submission.get(data_origin_type) + if type_display is None and type: + type_display = type if type in submission_values else None + if data_origin_type_display is None and data_origin_type: + data_origin_type_display = ( + data_origin_type if data_origin_type in ddf_submission_values else None + ) + arms_enriched.append( + { + **a, + "type_display": type_display, + "data_origin_type_display": data_origin_type_display, + } + ) + + # Admin audit view: recent arm audits for this SOA + conn_audit = _connect() + cur_audit = conn_audit.cursor() + cur_audit.execute( + "SELECT id, arm_id, action, before_json, after_json, performed_at FROM arm_audit WHERE soa_id=? ORDER BY id DESC LIMIT 50", + (soa_id,), + ) + arm_audits = [ + { + "id": r[0], + "arm_id": r[1], + "action": r[2], + "before_json": r[3], + "after_json": r[4], + "performed_at": r[5], + } + for r in cur_audit.fetchall() + ] + conn_audit.close() + return templates.TemplateResponse( request, "edit.html", @@ -2998,7 +3139,7 @@ def ui_edit(request: Request, soa_id: int): "visits": visits, "activities": activities_page, "elements": elements, - "arms": _fetch_arms_for_edit(soa_id), + "arms": arms_enriched, "cell_map": cell_map, "concepts": concepts, "activity_concepts": activity_concepts, @@ -3010,6 +3151,9 @@ def ui_edit(request: Request, soa_id: int): "freeze_count": len(freeze_list), "last_frozen_at": last_frozen_at, **study_meta, + "protocol_terminology_C174222": protocol_terminology_C174222, + "ddf_terminology_C188727": ddf_terminology_C188727, + "arm_audits": arm_audits, }, ) @@ -3041,9 +3185,9 @@ def ui_concepts_list(request: Request): @app.get("/ui/concept_categories", response_class=HTMLResponse) -def ui_categories_list(request: Request): +def ui_categories_list(request: Request, force: bool = False): """Render table listing biomedical concept categories (name + title + href).""" - categories = fetch_biomedical_concept_categories() or [] + categories = fetch_biomedical_concept_categories(force=force) or [] rows = [ { "name": c.get("name"), @@ -3057,6 +3201,7 @@ def ui_categories_list(request: Request): request, "concept_categories.html", { + "force": force, "rows": rows, "count": len(rows), "missing_key": subscription_key is None, @@ -3065,7 +3210,7 @@ def ui_categories_list(request: Request): @app.get("/ui/concept_categories/view", response_class=HTMLResponse) -def ui_category_detail(request: Request, name: str = ""): +def ui_category_detail(request: Request, name: str = "", force: bool = False): """Render list of biomedical concepts within a given category name. Query params: @@ -3076,7 +3221,7 @@ def ui_category_detail(request: Request, name: str = ""): return HTMLResponse( "

Category name required.

Back

" ) - concepts = fetch_biomedical_concepts_by_category(category_name) or [] + concepts = fetch_biomedical_concepts_by_category(category_name, force=force) or [] rows = [ { "code": c.get("code"), @@ -3090,6 +3235,7 @@ def ui_category_detail(request: Request, name: str = ""): "concept_category_detail.html", { "category": category_name, + "force": force, "rows": rows, "count": len(rows), }, @@ -3345,7 +3491,7 @@ def ui_add_visit( @app.post("/ui/soa/{soa_id}/add_arm", response_class=HTMLResponse) -def ui_add_arm( +async def ui_add_arm( request: Request, soa_id: int, name: str = Form(...), @@ -3359,12 +3505,173 @@ def ui_add_arm( # Accept blank/empty element selection gracefully. The form may submit "" which would 422 with Optional[int]. eid = int(element_id) if element_id and element_id.strip().isdigit() else None payload = ArmCreate(name=name, label=label, description=description, element_id=eid) - create_arm(soa_id, payload) + # Create base arm (function may not return id; fetch if needed) + created = create_arm(soa_id, payload) + # routers.arms.create_arm returns a row dict; extract id + new_arm_id = None + try: + if isinstance(created, dict): + new_arm_id = created.get("id") + elif isinstance(created, int): + new_arm_id = created + except Exception: + new_arm_id = None + if not new_arm_id: + try: + conn_tmp = _connect() + cur_tmp = conn_tmp.cursor() + cur_tmp.execute( + "SELECT id FROM arm WHERE soa_id=? ORDER BY id DESC LIMIT 1", + (soa_id,), + ) + rtmp = cur_tmp.fetchone() + new_arm_id = rtmp[0] if rtmp else None + conn_tmp.close() + except Exception: + new_arm_id = None + if not new_arm_id: + return HTMLResponse( + f"", + status_code=500, + ) + # Read optional type fields with hyphenated names + try: + form_data = await request.form() + arm_type_submission = (form_data.get("arm-type") or "").strip() + data_origin_type_submission = (form_data.get("data-origin-type") or "").strip() + except Exception: + arm_type_submission = "" + data_origin_type_submission = "" + + # If type selections provided, resolve to terminology codes and persist via junction table + if arm_type_submission or data_origin_type_submission: + conn = _connect() + cur = conn.cursor() + logger.info( + "ui_add_arm: received type selections arm-type='%s', data-origin-type='%s' for soa_id=%s arm_id=%s", + arm_type_submission, + data_origin_type_submission, + soa_id, + new_arm_id, + ) + new_type_uid: Optional[str] = None + new_data_origin_uid: Optional[str] = None + if arm_type_submission: + cur.execute( + "SELECT code FROM protocol_terminology WHERE codelist_code='C174222' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (arm_type_submission, arm_type_submission), + ) + r = cur.fetchone() + resolved_code = r[0] if r else None + if resolved_code is None: + logger.warning( + "ui_add_arm: unknown arm type submission '%s' for soa_id=%s", + arm_type_submission, + soa_id, + ) + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + # Create Code_N + cur.execute( + "SELECT code_uid FROM code WHERE soa_id=? AND code_uid LIKE 'Code_%'", + (soa_id,), + ) + existing = [x[0] for x in cur.fetchall() if x[0]] + n = 1 + if existing: + try: + n = max(int(x.split("_")[1]) for x in existing) + 1 + except Exception: + n = len(existing) + 1 + new_type_uid = f"Code_{n}" + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_type_uid, + "protocol_terminology", + "C174222", + resolved_code, + ), + ) + logger.info( + "ui_add_arm: created code junction %s -> table=%s list=%s code=%s", + new_type_uid, + "protocol_terminology", + "C174222", + resolved_code, + ) + if data_origin_type_submission: + cur.execute( + "SELECT code FROM ddf_terminology WHERE codelist_code='C188727' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (data_origin_type_submission, data_origin_type_submission), + ) + r2 = cur.fetchone() + resolved_ddf_code = r2[0] if r2 else None + if resolved_ddf_code is None: + logger.warning( + "ui_add_arm: unknown data origin type submission '%s' for soa_id=%s", + data_origin_type_submission, + soa_id, + ) + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + # Create Code_N (continue numbering) + cur.execute( + "SELECT code_uid FROM code WHERE soa_id=? AND code_uid LIKE 'Code_%'", + (soa_id,), + ) + existing = [x[0] for x in cur.fetchall() if x[0]] + n = 1 + if existing: + try: + n = max(int(x.split("_")[1]) for x in existing) + 1 + except Exception: + n = len(existing) + 1 + new_data_origin_uid = f"Code_{n}" + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ), + ) + logger.info( + "ui_add_arm: created code junction %s -> table=%s list=%s code=%s", + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ) + # Update arm row with new code_uids + if new_type_uid or new_data_origin_uid: + cur.execute( + "UPDATE arm SET type=COALESCE(?, type), data_origin_type=COALESCE(?, data_origin_type) WHERE id=? AND soa_id=?", + (new_type_uid, new_data_origin_uid, new_arm_id, soa_id), + ) + logger.info( + "ui_add_arm: updated arm id=%s set type=%s data_origin_type=%s", + new_arm_id, + new_type_uid, + new_data_origin_uid, + ) + conn.commit() + # routers.arms.create_arm already records a create audit; avoid duplicating here + conn.close() return HTMLResponse(f"") @app.post("/ui/soa/{soa_id}/update_arm", response_class=HTMLResponse) -def ui_update_arm( +async def ui_update_arm( request: Request, soa_id: int, arm_id: int = Form(...), @@ -3373,12 +3680,281 @@ def ui_update_arm( description: Optional[str] = Form(None), element_id: Optional[str] = Form(None), ): + """Form handler to update an existing Arm.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") - # Coerce possible blank element selection to None; avoid 422 validation error from string "" into Optional[int]. - eid = int(element_id) if element_id and element_id.strip().isdigit() else None - payload = ArmUpdate(name=name, label=label, description=description, element_id=eid) - update_arm(soa_id, arm_id, payload) + + # Read raw form to capture field names with hyphens: 'arm-type' and 'data-origin-type' + try: + form_data = await request.form() + arm_type_submission = (form_data.get("arm-type") or "").strip() + data_origin_type_submission = (form_data.get("data-origin-type") or "").strip() + except Exception: + arm_type_submission = "" + data_origin_type_submission = "" + logger.info( + "ui_update_arm: arm_id=%s soa_id=%s incoming arm-type='%s' data-origin-type='%s'", + arm_id, + soa_id, + arm_type_submission, + data_origin_type_submission, + ) + + # Fetch current arm (including existing type code_uid if any) + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id, name, label, description, COALESCE(type,''), COALESCE(data_origin_type,'') FROM arm WHERE id=? AND soa_id=?", + (arm_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Arm not found") + current_code_uid = row[4] or None + current_data_origin_uid = row[5] or None + # Capture prior code values for audits when code mapping changes without uid change + prior_arm_type_code_value: Optional[str] = None + prior_data_origin_code_value: Optional[str] = None + if current_code_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, current_code_uid), + ) + rcv = cur.fetchone() + prior_arm_type_code_value = rcv[0] if rcv else None + if current_data_origin_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, current_data_origin_uid), + ) + rdv = cur.fetchone() + prior_data_origin_code_value = rdv[0] if rdv else None + before_state = { + "id": row[0], + "name": row[1], + "label": row[2], + "description": row[3], + "type": current_code_uid, + "data_origin_type": current_data_origin_uid, + } + + # Resolve submission value to protocol terminology code (C174222) + resolved_code: Optional[str] = None + if arm_type_submission: + cur.execute( + "SELECT code FROM protocol_terminology WHERE codelist_code='C174222' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (arm_type_submission, arm_type_submission), + ) + r = cur.fetchone() + resolved_code = r[0] if r else None + if resolved_code is None: + logger.warning( + "ui_update_arm: unknown arm type submission '%s' for soa_id=%s arm_id=%s", + arm_type_submission, + soa_id, + arm_id, + ) + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + + # Maintain code table row with immutable code_uid (Code_N unique per SoA) + new_code_uid = current_code_uid + if resolved_code is not None: + if current_code_uid: + # Update existing junction row for this code_uid + cur.execute( + "UPDATE code SET code=?, codelist_code='C174222', codelist_table='protocol_terminology' WHERE soa_id=? AND code_uid=?", + (resolved_code, soa_id, current_code_uid), + ) + logger.info( + "ui_update_arm: updated junction code_uid=%s -> table=%s list=%s code=%s", + current_code_uid, + "protocol_terminology", + "C174222", + resolved_code, + ) + else: + # Create new Code_N within this SoA + cur.execute( + "SELECT code_uid FROM code WHERE soa_id=? AND code_uid LIKE 'Code_%'", + (soa_id,), + ) + existing = [x[0] for x in cur.fetchall() if x[0]] + n = 1 + if existing: + try: + n = max(int(x.split("_")[1]) for x in existing) + 1 + except Exception: + n = len(existing) + 1 + new_code_uid = f"Code_{n}" + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_code_uid, + "protocol_terminology", + "C174222", + resolved_code, + ), + ) + logger.info( + "ui_update_arm: created junction code_uid=%s -> table=%s list=%s code=%s", + new_code_uid, + "protocol_terminology", + "C174222", + resolved_code, + ) + + # Resolve Data Origin Type submission value to DDF terminology code (C188727) + resolved_ddf_code: Optional[str] = None + new_data_origin_uid = current_data_origin_uid + if data_origin_type_submission: + cur.execute( + "SELECT code FROM ddf_terminology WHERE codelist_code='C188727' AND (cdisc_submission_value=? OR LOWER(TRIM(cdisc_submission_value))=LOWER(TRIM(?)))", + (data_origin_type_submission, data_origin_type_submission), + ) + r2 = cur.fetchone() + resolved_ddf_code = r2[0] if r2 else None + if resolved_ddf_code is None: + logger.warning( + "ui_update_arm: unknown data origin type submission '%s' for soa_id=%s arm_id=%s", + data_origin_type_submission, + soa_id, + arm_id, + ) + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + # Maintain/Upsert immutable Code_N for DDF mapping + if current_data_origin_uid: + cur.execute( + "UPDATE code SET code=?, codelist_code='C188727', codelist_table='ddf_terminology' WHERE soa_id=? AND code_uid=?", + (resolved_ddf_code, soa_id, current_data_origin_uid), + ) + new_data_origin_uid = current_data_origin_uid + logger.info( + "ui_update_arm: updated junction code_uid=%s -> table=%s list=%s code=%s", + current_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ) + else: + # Create new Code_N, ensuring unique across this SoA + cur.execute( + "SELECT code_uid FROM code WHERE soa_id=? AND code_uid LIKE 'Code_%'", + (soa_id,), + ) + existing = [x[0] for x in cur.fetchall() if x[0]] + n = 1 + if existing: + try: + n = max(int(x.split("_")[1]) for x in existing) + 1 + except Exception: + n = len(existing) + 1 + new_data_origin_uid = f"Code_{n}" + cur.execute( + "INSERT INTO code (soa_id, code_uid, codelist_table, codelist_code, code) VALUES (?,?,?,?,?)", + ( + soa_id, + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ), + ) + logger.info( + "ui_update_arm: created junction code_uid=%s -> table=%s list=%s code=%s", + new_data_origin_uid, + "ddf_terminology", + "C188727", + resolved_ddf_code, + ) + + # Apply arm field updates (including setting type to code_uid if resolved) + new_name = name if name is not None else row[1] + new_label = label if label is not None else row[2] + new_desc = description if description is not None else row[3] + cur.execute( + "UPDATE arm SET name=?, label=?, description=?, type=?, data_origin_type=? WHERE id=? AND soa_id=?", + ( + new_name, + new_label, + new_desc, + new_code_uid, + new_data_origin_uid, + arm_id, + soa_id, + ), + ) + logger.info( + "ui_update_arm: applied UPDATE arm id=%s set name='%s' label='%s' type=%s data_origin_type=%s", + arm_id, + new_name, + new_label, + new_code_uid, + new_data_origin_uid, + ) + conn.commit() + # Capture post-update code values + post_arm_type_code_value: Optional[str] = None + post_data_origin_code_value: Optional[str] = None + if new_code_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, new_code_uid), + ) + rav = cur.fetchone() + post_arm_type_code_value = rav[0] if rav else None + if new_data_origin_uid: + cur.execute( + "SELECT code FROM code WHERE soa_id=? AND code_uid=?", + (soa_id, new_data_origin_uid), + ) + rdv2 = cur.fetchone() + post_data_origin_code_value = rdv2[0] if rdv2 else None + after_state = { + "id": arm_id, + "name": new_name, + "label": new_label, + "description": new_desc, + "type": new_code_uid, + "data_origin_type": new_data_origin_uid, + "type_code": post_arm_type_code_value, + "data_origin_type_code": post_data_origin_code_value, + } + # Record audit if any relevant fields or underlying code mappings changed + if ( + before_state["type"] != after_state["type"] + or before_state["data_origin_type"] != after_state["data_origin_type"] + or prior_arm_type_code_value != post_arm_type_code_value + or prior_data_origin_code_value != post_data_origin_code_value + or before_state["name"] != after_state["name"] + or before_state["label"] != after_state["label"] + or before_state["description"] != after_state["description"] + ): + try: + _record_arm_audit( + soa_id, + "update", + arm_id=arm_id, + before=before_state, + after=after_state, + ) + except Exception: + pass + else: + logger.info( + "ui_update_arm: no-op update detected for arm_id=%s (no field or code changes)", + arm_id, + ) + conn.close() return HTMLResponse(f"") @@ -3390,6 +3966,7 @@ def ui_delete_arm(request: Request, soa_id: int, arm_id: int = Form(...)): @app.post("/ui/soa/{soa_id}/reorder_arms", response_class=HTMLResponse) def ui_reorder_arms(request: Request, soa_id: int, order: str = Form("")): + """Form handler to reorder existing Arms.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] @@ -3429,6 +4006,7 @@ def ui_add_element( testrl: Optional[str] = Form(None), teenrl: Optional[str] = Form(None), ): + """Form handler to add an element.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") name = (name or "").strip() @@ -3523,6 +4101,7 @@ def ui_update_element( testrl: Optional[str] = Form(None), teenrl: Optional[str] = Form(None), ): + """Form handler to update an existing Element.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3594,6 +4173,7 @@ def ui_update_element( @app.post("/ui/soa/{soa_id}/delete_element", response_class=HTMLResponse) def ui_delete_element(request: Request, soa_id: int, element_id: int = Form(...)): + """Form handler to delete an existing Element.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3615,6 +4195,7 @@ def ui_add_epoch( epoch_label: Optional[str] = Form(None), epoch_description: Optional[str] = Form(None), ): + """Form handler to add an Epoch.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3664,6 +4245,7 @@ def ui_update_epoch( epoch_label: Optional[str] = Form(None), epoch_description: Optional[str] = Form(None), ): + """Form handler to update an existing Epoch.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() @@ -3745,6 +4327,7 @@ def ui_set_activity_concepts( activity_id: int, concept_codes: List[str] = Form([]), ): + """Form handler to set Biomedical Concepts related to an Activity.""" payload = ConceptsUpdate(concept_codes=list(dict.fromkeys(concept_codes))) set_activity_concepts(soa_id, activity_id, payload) # HTMX inline update support @@ -3777,6 +4360,7 @@ def ui_set_activity_concepts( def ui_activity_concepts_cell( request: Request, soa_id: int, activity_id: int, edit: int = 0 ): + """Form handler to return Biomedical Concepts related to an Activity""" # Defensive guard: if activity_id is somehow falsy (should not happen for valid int path param) # surface a clear 400 error rather than proceeding and causing confusing downstream behavior. if not activity_id: @@ -3813,6 +4397,7 @@ def ui_set_cell( activity_id: int = Form(...), status: str = Form("X"), ): + """Form handler to set 'X' in SoA Matrix Cell.""" result = set_cell( soa_id, CellCreate(visit_id=visit_id, activity_id=activity_id, status=status) ) @@ -3867,6 +4452,7 @@ def ui_toggle_cell( @app.post("/ui/soa/{soa_id}/delete_visit", response_class=HTMLResponse) def ui_delete_visit(request: Request, soa_id: int, visit_id: int = Form(...)): + """Form handler to delete a visit.""" # Use API logic to delete and log try: delete_visit(soa_id, visit_id) @@ -3891,6 +4477,7 @@ def ui_set_visit_epoch( epoch_id_raw: str = Form(""), # new field name (blank means clear) epoch_id: str = Form(""), # legacy field name used by template select ): + """Form handler to associate an Epoch with a Visit/Encounter.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Determine provided raw value (prefer epoch_id_raw if non-blank) @@ -3930,12 +4517,14 @@ def ui_set_visit_epoch( @app.post("/ui/soa/{soa_id}/delete_activity", response_class=HTMLResponse) def ui_delete_activity(request: Request, soa_id: int, activity_id: int = Form(...)): + """Form handler to delete an Activity""" delete_activity(soa_id, activity_id) return HTMLResponse(f"") @app.post("/ui/soa/{soa_id}/delete_epoch", response_class=HTMLResponse) def ui_delete_epoch(request: Request, soa_id: int, epoch_id: int = Form(...)): + """Form handler to delete an Epoch.""" delete_epoch(soa_id, epoch_id) return HTMLResponse(f"") @@ -3998,7 +4587,7 @@ def ui_reorder_activities(request: Request, soa_id: int, order: str = Form("")): @app.post("/ui/soa/{soa_id}/reorder_epochs", response_class=HTMLResponse) def ui_reorder_epochs(request: Request, soa_id: int, order: str = Form("")): - """Persist new epoch ordering.""" + """Form handler to persist new epoch ordering.""" if not _soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] @@ -4333,6 +4922,7 @@ def ui_ddf_terminology( uploaded: Optional[str] = None, error: Optional[str] = None, ): + """Detail page to display loaded DDF terminology from the SQLite table""" data = get_ddf_terminology( search=search, code=code, @@ -4498,6 +5088,7 @@ def _get_ddf_sources() -> List[str]: def get_ddf_audit( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Return audit report of DDF Terminology loads.""" conn = _connect() cur = conn.cursor() cur.execute( @@ -4556,6 +5147,7 @@ def _valid_date(d: str) -> bool: def export_ddf_audit_csv( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export DDF terminology audit report in CSV format.""" rows = get_ddf_audit(source=source, start=start, end=end) import csv import io @@ -4603,6 +5195,7 @@ def export_ddf_audit_csv( def export_ddf_audit_json( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export DDF terminology audit report in JSON format.""" return get_ddf_audit(source=source, start=start, end=end) @@ -4613,6 +5206,7 @@ def ui_ddf_audit( start: Optional[str] = None, end: Optional[str] = None, ): + """Display audit report of DDF Terminology loads""" rows = get_ddf_audit(source=source, start=start, end=end) sources = _get_ddf_sources() return templates.TemplateResponse( @@ -4756,6 +5350,7 @@ def load_protocol_terminology( def admin_load_protocol( file_path: Optional[str] = None, sheet_name: str = "Protocol Terminology 2025-09-26" ): + """Load new Protocol Terminology XLS.""" project_root = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(__file__))) ) @@ -4802,6 +5397,7 @@ def get_protocol_terminology( limit: int = 50, offset: int = 0, ): + """Return latest Protocol Terminology loaded into SQLite database.""" limit = max(1, min(limit, 200)) offset = max(0, offset) conn = _connect() @@ -4893,6 +5489,7 @@ def ui_protocol_terminology( uploaded: Optional[str] = None, error: Optional[str] = None, ): + """Form handler to display the latest loaded Protocol Terminology from the SQLite database.""" data = get_protocol_terminology( search=search, code=code, @@ -4922,6 +5519,7 @@ def ui_protocol_upload( sheet_name: str = Form("Protocol Terminology 2025-09-26"), file: UploadFile = File(...), ): + """Form handler for the upload of Protocol Terminology XLS.""" filename = file.filename or "uploaded.xls" if not (filename.lower().endswith(".xls") or filename.lower().endswith(".xlsx")): return HTMLResponse( @@ -5051,6 +5649,7 @@ def _get_protocol_sources() -> List[str]: def get_protocol_audit( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Return the Protocol Terminology audit report.""" conn = _connect() cur = conn.cursor() cur.execute( @@ -5108,6 +5707,7 @@ def _valid_date(d: str) -> bool: def export_protocol_audit_csv( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export the latest Protocol Terminology from the SQLite database in CSV format.""" rows = get_protocol_audit(source=source, start=start, end=end) import csv import io @@ -5155,6 +5755,7 @@ def export_protocol_audit_csv( def export_protocol_audit_json( source: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None ): + """Export the latest Protocol Terminology from the SQLite database in JSON format.""" return get_protocol_audit(source=source, start=start, end=end) @@ -5165,6 +5766,7 @@ def ui_protocol_audit( start: Optional[str] = None, end: Optional[str] = None, ): + """Form handler for display of the Protocol Terminology audit report.""" rows = get_protocol_audit(source=source, start=start, end=end) sources = _get_protocol_sources() return templates.TemplateResponse( diff --git a/src/soa_builder/web/initialize_database.py b/src/soa_builder/web/initialize_database.py index fde39eb..8ccd3ac 100644 --- a/src/soa_builder/web/initialize_database.py +++ b/src/soa_builder/web/initialize_database.py @@ -146,5 +146,19 @@ def _init_db(): performed_at TEXT NOT NULL )""" ) + + # create the code table to store unique Code_uid values associated with study objects + cur.execute( + """CREATE TABLE IF NOT EXISTS code ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + code_uid TEXT, -- immutable Code_N identifier unique within an SOA + codelist_table TEXT, + codelist_code TEXT NOT NULL, + code TEXT NOT NULL, + UNIQUE(soa_id, code_uid) + )""" + ) + conn.commit() conn.close() diff --git a/src/soa_builder/web/static/style.css b/src/soa_builder/web/static/style.css index 04ae631..6c3d410 100644 --- a/src/soa_builder/web/static/style.css +++ b/src/soa_builder/web/static/style.css @@ -1,4 +1,4 @@ -body { font-family: system-ui, Arial, sans-serif; margin: 1.5rem; } +body { font-family: Arial, Helvetica, sans-serif; margin: 1.5rem; } header h1 { margin: 0; } nav a { margin-right: 1rem; } .table, table { border-collapse: collapse; } diff --git a/src/soa_builder/web/templates/concept_categories.html b/src/soa_builder/web/templates/concept_categories.html index b59342f..83852ba 100644 --- a/src/soa_builder/web/templates/concept_categories.html +++ b/src/soa_builder/web/templates/concept_categories.html @@ -5,6 +5,10 @@

Biomedical Concept Categories ({{ count }}) +

+ Refresh Categories (bypass cache) + {% if force %}Cache refreshed{% endif %} +

@@ -14,7 +18,7 @@

Biomedical Concept Categories ({{ count }}) - + Concepts Name Title @@ -25,11 +29,12 @@

Biomedical Concept Categories ({{ count }}) {% if r.name %} - {{ r.name }} + View Concepts {% else %} n/a {% endif %} + {{ r.name }} {{ r.title }} {% endfor %} diff --git a/src/soa_builder/web/templates/concept_category_detail.html b/src/soa_builder/web/templates/concept_category_detail.html index 16f63b5..df02857 100644 --- a/src/soa_builder/web/templates/concept_category_detail.html +++ b/src/soa_builder/web/templates/concept_category_detail.html @@ -1,6 +1,11 @@ {% extends 'base.html' %} {% block content %}

Biomedical Concepts in Category: {{ category }}

+

+ Refresh (bypass cache) + {% if force %}Cache bypassed{% endif %} + | Back to Categories +

Total concepts: {{ count }}

{% if rows %} diff --git a/src/soa_builder/web/templates/edit.html b/src/soa_builder/web/templates/edit.html index d670aa0..31ec5eb 100644 --- a/src/soa_builder/web/templates/edit.html +++ b/src/soa_builder/web/templates/edit.html @@ -82,34 +82,6 @@

Editing SoA {{ soa_id }}

-
- Epochs ({{ epochs|length }}) (drag to reorder) -
    - {% for e in epochs %} -
  • - {{ e.order_index }}. E{{ e.epoch_seq }} {{ e.name }} - {% if e.epoch_label %}[{{ e.epoch_label }}]{% endif %} -
    - - -
    -
    - - - - - -
    -
    • - {% endfor %} -
-
- - - - -
-
Visits ({{ visits|length }}) (drag to reorder)
    @@ -145,8 +117,6 @@

    Editing SoA {{ soa_id }}

-
-
Activities ({{ activities|length }}) (drag to reorder)
    @@ -164,7 +134,37 @@

    Editing SoA {{ soa_id }}

-
+
+
+
+ Epochs ({{ epochs|length }}) (drag to reorder) +
    + {% for e in epochs %} +
  • + {{ e.order_index }}. E{{ e.epoch_seq }} {{ e.name }} + {% if e.epoch_label %}[{{ e.epoch_label }}]{% endif %} +
    + + +
    +
    + + + + + +
    +
    • + {% endfor %} +
+
+ + + + +
+
+
Elements ({{ elements|length }}) (drag to reorder)
    {% for el in elements %} @@ -197,7 +197,7 @@

    Editing SoA {{ soa_id }}

-
+
Arms ({{ arms|length }}) (drag to reorder)
    {% for arm in arms %} @@ -213,6 +213,28 @@

    Editing SoA {{ soa_id }}

    + {# Type selection from protocol terminology C174222 #} + {% if protocol_terminology_C174222 %} + + {% endif %} + {# Type selection from ddf terminology C188727 #} + {% if ddf_terminology_C188727 %} + + {% endif %} @@ -222,9 +244,57 @@

    Editing SoA {{ soa_id }}

    + {% if protocol_terminology_C174222 %} + + {% endif %} + {% if ddf_terminology_C188727 %} + + {% endif %}
+ +
+
+
+ Arm Audit (latest {{ arm_audits|length }}) + {% if arm_audits %} + + + + + + + + + + {% for au in arm_audits %} + + + + + + + + + {% endfor %} +
IDArmActionPerformedBeforeAfter
{{ au.id }}{{ au.arm_id }}{{ au.action }}{{ au.performed_at }}{{ au.before_json or '' }}{{ au.after_json or '' }}
+ {% else %} +
No arm audit entries yet.
+ {% endif %} +
diff --git a/tests/test_categories_cache.py b/tests/test_categories_cache.py new file mode 100644 index 0000000..c5864ff --- /dev/null +++ b/tests/test_categories_cache.py @@ -0,0 +1,118 @@ +from types import SimpleNamespace + +import pytest + +from soa_builder.web.app import ( + fetch_biomedical_concept_categories, + _bc_categories_cache, +) + + +class DummyResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +@pytest.fixture(autouse=True) +def clear_categories_cache(): + _bc_categories_cache.clear() + + +def test_categories_cache_hit(monkeypatch): + call_count = SimpleNamespace(n=0) + + payload = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A", "title": "TitleA"} + }, + }, + { + "name": "CategoryB", + "_links": { + "self": {"href": "/mdr/bc/categories/B", "title": "TitleB"} + }, + }, + ] + } + } + + def fake_get(url, headers=None, timeout=None): + call_count.n += 1 + return DummyResponse(payload) + + monkeypatch.setattr("requests.get", fake_get) + + # First call populates cache + cats1 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 1 + assert len(cats1) == 2 + + # Second call within TTL should hit cache, no new HTTP call + cats2 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 1 + assert cats2 == cats1 + + +def test_categories_cache_force_bypass(monkeypatch): + call_count = SimpleNamespace(n=0) + + payload1 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A", "title": "TitleA"} + }, + } + ] + } + } + payload2 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A2", "title": "TitleA2"} + }, + }, + { + "name": "CategoryC", + "_links": { + "self": {"href": "/mdr/bc/categories/C", "title": "TitleC"} + }, + }, + ] + } + } + + def fake_get(url, headers=None, timeout=None): + call_count.n += 1 + # Return payload1 first, payload2 thereafter + return DummyResponse(payload1 if call_count.n == 1 else payload2) + + monkeypatch.setattr("requests.get", fake_get) + + # Populate cache + cats1 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 1 + assert [c["name"] for c in cats1] == ["CategoryA"] + + # Force bypass should trigger a new HTTP call and new content + cats2 = fetch_biomedical_concept_categories(force=True) + assert call_count.n == 2 + assert [c["name"] for c in cats2] == ["CategoryA", "CategoryC"] + + # Regular call after force should use cached latest content (no extra HTTP) + cats3 = fetch_biomedical_concept_categories(force=False) + assert call_count.n == 2 + assert cats3 == cats2 diff --git a/tests/test_categories_ui_force.py b/tests/test_categories_ui_force.py new file mode 100644 index 0000000..194cfb1 --- /dev/null +++ b/tests/test_categories_ui_force.py @@ -0,0 +1,89 @@ +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from soa_builder.web.app import app, _bc_categories_cache + +client = TestClient(app) + + +class DummyResponse: + def __init__(self, payload, status_code=200, text=""): + self._payload = payload + self.status_code = status_code + self.text = text or "" + + def json(self): + return self._payload + + +@pytest.fixture(autouse=True) +def clear_categories_cache(): + _bc_categories_cache.clear() + + +def test_ui_categories_force_bypass(monkeypatch): + call_count = SimpleNamespace(n=0) + + payload1 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A", "title": "TitleA"} + }, + }, + ] + } + } + payload2 = { + "_links": { + "categories": [ + { + "name": "CategoryA", + "_links": { + "self": {"href": "/mdr/bc/categories/A2", "title": "TitleA2"} + }, + }, + { + "name": "CategoryC", + "_links": { + "self": {"href": "/mdr/bc/categories/C", "title": "TitleC"} + }, + }, + ] + } + } + + def fake_get(url, headers=None, timeout=None): + call_count.n += 1 + return DummyResponse(payload1 if call_count.n == 1 else payload2, text="ok") + + monkeypatch.setattr("requests.get", fake_get) + + # Initial request populates cache, shows TitleA only + r1 = client.get("/ui/concept_categories") + assert r1.status_code == 200 + html1 = r1.text + assert "TitleA" in html1 + assert "TitleA2" not in html1 + assert "TitleC" not in html1 + assert call_count.n == 1 + + # Force bypass should fetch again and render updated titles + r2 = client.get("/ui/concept_categories?force=1") + assert r2.status_code == 200 + html2 = r2.text + assert "TitleA2" in html2 + assert "TitleC" in html2 + assert call_count.n == 2 + + # Subsequent non-force call should serve cached updated content (no new HTTP) + r3 = client.get("/ui/concept_categories") + assert r3.status_code == 200 + html3 = r3.text + assert "TitleA2" in html3 + assert "TitleC" in html3 + assert call_count.n == 2 diff --git a/tests/test_concept_categories.py b/tests/test_concept_categories.py index 4c07769..13b90a7 100644 --- a/tests/test_concept_categories.py +++ b/tests/test_concept_categories.py @@ -42,7 +42,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, {"items": []}) monkeypatch.setattr("requests.get", fake_get) - fetch_biomedical_concepts_by_category(raw_category) + fetch_biomedical_concepts_by_category(raw_category, force=True) assert len(captured_urls) == 1 assert expected_url_fragment in captured_urls[0] @@ -68,7 +68,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, payload) monkeypatch.setattr("requests.get", fake_get) - concepts = fetch_biomedical_concepts_by_category("Liver Findings") + concepts = fetch_biomedical_concepts_by_category("Liver Findings", force=True) assert {c["code"] for c in concepts} == {"ALT", "AST"} assert all( c["href"].startswith( @@ -100,7 +100,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, payload) monkeypatch.setattr("requests.get", fake_get) - concepts = fetch_biomedical_concepts_by_category("Liver Findings") + concepts = fetch_biomedical_concepts_by_category("Liver Findings", force=True) codes = {c["code"] for c in concepts} assert codes == {"ALT", "AST"} # Titles preserved @@ -114,7 +114,7 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(404, {}, text="Not Found") monkeypatch.setattr("requests.get", fake_get) - concepts = fetch_biomedical_concepts_by_category("Liver Findings") + concepts = fetch_biomedical_concepts_by_category("Liver Findings", force=True) assert concepts == [] @@ -139,7 +139,9 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, payload) monkeypatch.setattr("requests.get", fake_get) - resp = client.get("/ui/concept_categories/view", params={"name": "Liver Findings"}) + resp = client.get( + "/ui/concept_categories/view", params={"name": "Liver Findings", "force": True} + ) assert resp.status_code == 200 text = resp.text # Internal links to concept detail page @@ -156,6 +158,8 @@ def fake_get(url, headers=None, timeout=0): return DummyResp(200, {"items": []}) monkeypatch.setattr("requests.get", fake_get) - resp = client.get("/ui/concept_categories/view", params={"name": "Liver Findings"}) + resp = client.get( + "/ui/concept_categories/view", params={"name": "Liver Findings", "force": True} + ) assert resp.status_code == 200 assert "No concepts found for this category." in resp.text diff --git a/tests/test_concept_category_force_refresh.py b/tests/test_concept_category_force_refresh.py new file mode 100644 index 0000000..c0c512f --- /dev/null +++ b/tests/test_concept_category_force_refresh.py @@ -0,0 +1,66 @@ +import json +from typing import List +from fastapi.testclient import TestClient + +from soa_builder.web.app import app, _category_concepts_cache + +client = TestClient(app) + + +class DummyResp: + def __init__(self, status_code: int, json_data=None, text: str = ""): + self.status_code = status_code + self._json = json_data + self.text = text or json.dumps(json_data or {}) + + def json(self): + if self._json is None: + raise ValueError("No JSON") + return self._json + + +def test_ui_category_force_refresh(monkeypatch): + _category_concepts_cache.clear() + calls: List[str] = [] + payload1 = { + "items": [ + { + "code": "ALT", + "title": "Alanine", + "href": "/mdr/bc/biomedicalconcepts/ALT", + } + ] + } + payload2 = { + "items": [ + { + "code": "AST", + "title": "Aspartate", + "href": "/mdr/bc/biomedicalconcepts/AST", + } + ] + } + + def fake_get(url, headers=None, timeout=0): + # First call returns ALT, subsequent calls return AST + if not calls: + calls.append(url) + return DummyResp(200, payload1) + calls.append(url) + return DummyResp(200, payload2) + + monkeypatch.setattr("requests.get", fake_get) + # Initial request populates cache with ALT + r1 = client.get("/ui/concept_categories/view", params={"name": "Force Test"}) + assert r1.status_code == 200 + assert "/ui/concepts/ALT" in r1.text + # Second request without force still uses cache (ALT) + r2 = client.get("/ui/concept_categories/view", params={"name": "Force Test"}) + assert r2.status_code == 200 + assert "/ui/concepts/ALT" in r2.text + # Third request with force bypasses cache and shows AST + r3 = client.get( + "/ui/concept_categories/view", params={"name": "Force Test", "force": True} + ) + assert r3.status_code == 200 + assert "/ui/concepts/AST" in r3.text diff --git a/tests/test_concepts_by_category_ui_force.py b/tests/test_concepts_by_category_ui_force.py new file mode 100644 index 0000000..c036973 --- /dev/null +++ b/tests/test_concepts_by_category_ui_force.py @@ -0,0 +1,88 @@ +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from soa_builder.web.app import app, _category_concepts_cache + +client = TestClient(app) + + +class DummyResponse: + def __init__(self, payload, status_code=200, text=""): + self._payload = payload + self.status_code = status_code + self.text = text or "" + + def json(self): + return self._payload + + +@pytest.fixture(autouse=True) +def clear_concepts_cache(): + _category_concepts_cache.clear() + + +def test_ui_concepts_by_category_force_bypass(monkeypatch): + call_count = SimpleNamespace(n=0) + + # The app queries: /mdr/bc/biomedicalconcepts?category=CategoryA (encoded) + # Provide payloads representing concepts in direct 'items' list form. + payload1 = { + "items": [ + { + "code": "C100", + "title": "Alpha", + "href": "/mdr/bc/biomedicalconcepts/C100", + }, + ] + } + payload2 = { + "items": [ + { + "code": "C100", + "title": "Alpha v2", + "href": "/mdr/bc/biomedicalconcepts/C100", + }, + { + "code": "C200", + "title": "Beta", + "href": "/mdr/bc/biomedicalconcepts/C200", + }, + ] + } + + def fake_get(url, headers=None, timeout=None): + # Ensure we are mocking the concepts-by-category endpoint + assert "/mdr/bc/biomedicalconcepts?category=" in url + call_count.n += 1 + return DummyResponse(payload1 if call_count.n == 1 else payload2, text="ok") + + monkeypatch.setattr("requests.get", fake_get) + + # Initial request populates cache, shows Alpha only + r1 = client.get("/ui/concept_categories/view", params={"name": "CategoryA"}) + assert r1.status_code == 200 + html1 = r1.text + assert "Alpha" in html1 + assert "Alpha v2" not in html1 + assert "Beta" not in html1 + assert call_count.n == 1 + + # Force bypass should fetch again and render updated concept titles + r2 = client.get( + "/ui/concept_categories/view", params={"name": "CategoryA", "force": 1} + ) + assert r2.status_code == 200 + html2 = r2.text + assert "Alpha v2" in html2 + assert "Beta" in html2 + assert call_count.n == 2 + + # Subsequent non-force call should serve cached updated content (no new HTTP) + r3 = client.get("/ui/concept_categories/view", params={"name": "CategoryA"}) + assert r3.status_code == 200 + html3 = r3.text + assert "Alpha v2" in html3 + assert "Beta" in html3 + assert call_count.n == 2