diff --git a/docs/SoA-Workbench-UAT-Testing.docx b/docs/SoA-Workbench-UAT-Testing.docx new file mode 100644 index 0000000..dbbc16a Binary files /dev/null and b/docs/SoA-Workbench-UAT-Testing.docx differ diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index ed8318c..9573e16 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -49,6 +49,7 @@ _migrate_add_epoch_seq, _migrate_add_study_fields, _migrate_arm_add_type_fields, + _migrate_element_audit_columns, _migrate_copy_cell_data, _migrate_create_code_junction, _migrate_drop_arm_element_link, @@ -61,6 +62,7 @@ from .routers import activities as activities_router from .routers import arms as arms_router from .routers import elements as elements_router +from .routers.elements import _next_element_identifier from .routers import epochs as epochs_router from .routers import freezes as freezes_router from .routers import rollback as rollback_router @@ -68,7 +70,29 @@ from .routers.arms import create_arm # re-export for backward compatibility from .routers.arms import delete_arm from .schemas import ArmCreate, SOACreate, SOAMetadataUpdate -from .utils import get_next_code_uid as _get_next_code_uid, load_epoch_type_options +from .utils import ( + get_next_code_uid as _get_next_code_uid, + load_epoch_type_options, + soa_exists, +) + +# Audit functions +from .audit import _record_element_audit + + +def _configure_logging(): + level = logging.INFO + if os.environ.get("SOA_BUILDER_DEBUG") == "1": + level = logging.DEBUG + logging.basicConfig( + level=level, format="%(asctime)s %(name)s %(levelname)s: %(message)s" + ) + # Quiet noisy libraries if present + logging.getLogger("sqlalchemy").setLevel(logging.WARNING) + return logging.getLogger("soa_builder") + + +logger = _configure_logging() load_dotenv() # must come BEFORE reading env-based configuration so values are populated DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") @@ -122,6 +146,7 @@ _migrate_rollback_add_elements_restored() _migrate_activity_add_uid() _migrate_arm_add_type_fields() +_migrate_element_audit_columns() _backfill_dataset_date("ddf_terminology", "ddf_terminology_audit") _backfill_dataset_date("protocol_terminology", "protocol_terminology_audit") @@ -141,11 +166,10 @@ def _get_concepts_override(): return os.environ.get("CDISC_CONCEPTS_JSON") -# Audit functions -def _record_element_audit( +def _record_transition_rule_audit( soa_id: int, action: str, - element_id: Optional[int], + transition_rule_id: Optional[int], before: Optional[dict] = None, after: Optional[dict] = None, ): @@ -153,10 +177,11 @@ def _record_element_audit( conn = _connect() cur = conn.cursor() cur.execute( - "INSERT INTO element_audit (soa_id, element_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + "INSERT INTO transition_rule_audit (soa_id, transition_rule_id, action, before_json, after_json, performed_at) " + "VALUES (?,?,?,?,?,?)", ( soa_id, - element_id, + transition_rule_id, action, json.dumps(before) if before else None, json.dumps(after) if after else None, @@ -165,8 +190,8 @@ def _record_element_audit( ) conn.commit() conn.close() - except Exception as e: # pragma: no cover - logger.warning("Failed recording element audit: %s", e) + except Exception as e: + logger.warning("Failed recording transition rule audit: %s", e) def _record_visit_audit( @@ -293,7 +318,7 @@ def _record_arm_audit( @app.post("/soa/{soa_id}/visits/reorder", response_class=JSONResponse) def reorder_visits_api(soa_id: int, order: List[int]): """JSON reorder endpoint for visits (parity with elements). Body is array of visit IDs in desired order.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: raise HTTPException(400, "Order list required") @@ -317,7 +342,7 @@ def reorder_visits_api(soa_id: int, order: List[int]): @app.post("/soa/{soa_id}/activities/reorder", response_class=JSONResponse) def reorder_activities_api(soa_id: int, order: List[int]): """JSON reorder endpoint for activities.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: raise HTTPException(400, "Order list required") @@ -423,7 +448,7 @@ def _get_freeze(soa_id: int, freeze_id: int): def _create_freeze(soa_id: int, version_label: Optional[str]): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Auto version label if not provided conn = _connect() @@ -884,7 +909,7 @@ def _fetch_arms_for_edit(soa_id: int) -> list[dict]: conn = _connect() cur = conn.cursor() cur.execute( - "SELECT id,name,label,description,order_index,COALESCE(type,''),COALESCE(data_origin_type,'') FROM arm WHERE soa_id=? ORDER BY order_index", + "SELECT id,name,label,description,order_index,arm_uid,COALESCE(type,''),COALESCE(data_origin_type,'') FROM arm WHERE soa_id=? ORDER BY order_index", (soa_id,), ) rows = [ @@ -894,8 +919,9 @@ def _fetch_arms_for_edit(soa_id: int) -> list[dict]: "label": r[2], "description": r[3], "order_index": r[4], - "type": r[5] or None, - "data_origin_type": r[6] or None, + "arm_uid": r[5], + "type": r[6] or None, + "data_origin_type": r[7] or None, } for r in cur.fetchall() ] @@ -976,13 +1002,7 @@ class MatrixImport(BaseModel): # --------------------- Helpers --------------------- -def _soa_exists(soa_id: int) -> bool: - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - row = cur.fetchone() - conn.close() - return row is not None +# Use shared utils.soa_exists instead of local helper def _fetch_matrix(soa_id: int): @@ -1011,6 +1031,40 @@ def _fetch_matrix(soa_id: int): return visits, activities, cells +def _list_study_cells(soa_id: int) -> list[dict]: + """List study_cell rows, including element and arm names filtered by soa_id. + + Returns: [{id, study_cell_uid, arm_uid, epoch_uid, element_uid, element_name, arm_name, epoch_name}] + """ + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT sc.id, sc.study_cell_uid, sc.arm_uid, sc.epoch_uid, sc.element_uid, " + " e.name AS element_name, a.name AS arm_name, ep.name AS epoch_name " + "FROM study_cell sc " + "LEFT JOIN element e ON e.element_id = sc.element_uid AND e.soa_id = sc.soa_id " + "LEFT JOIN arm a ON a.arm_uid = sc.arm_uid AND a.soa_id = sc.soa_id " + "LEFT JOIN epoch ep ON ep.epoch_uid = sc.epoch_uid AND ep.soa_id = sc.soa_id " + "WHERE sc.soa_id=? ORDER BY sc.id", + (soa_id,), + ) + rows = [ + { + "id": r[0], + "study_cell_uid": r[1], + "arm_uid": r[2], + "epoch_uid": r[3], + "element_uid": r[4], + "element_name": r[5], + "arm_name": r[6], + "epoch_name": r[7], + } + for r in cur.fetchall() + ] + conn.close() + return rows + + def fetch_biomedical_concept_categories(force: bool = False) -> list[dict]: """Return list of Biomedical Concept Categories from CDISC Library. @@ -1692,7 +1746,7 @@ async def lifespan(app: FastAPI): # pragma: no cover @app.post("/ui/soa/{soa_id}/concepts_refresh") def ui_refresh_concepts(request: Request, soa_id: int): """Fetch Biomedical Concepts; refresh cache""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") fetch_biomedical_concepts(force=True) # If HTMX request, use HX-Redirect header for clean redirect without injecting script @@ -1710,7 +1764,7 @@ def ui_refresh_concepts(request: Request, soa_id: int): @app.get("/soa/{soa_id}/reorder_audit/export/csv") def export_reorder_audit_csv(soa_id: int): """Export reorder audit history to CSV.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") rows = _list_reorder_audit(soa_id) # Prepare CSV lines @@ -1896,7 +1950,7 @@ def create_soa(payload: SOACreate): @app.get("/soa/{soa_id}") def get_soa(soa_id: int): """Return SoA by ID""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) # Fetch epochs @@ -1948,7 +2002,7 @@ def get_soa(soa_id: int): @app.post("/soa/{soa_id}/metadata") def update_soa_metadata(soa_id: int, payload: SOAMetadataUpdate): """Update metadata for SoA/Study.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2011,7 +2065,7 @@ def update_soa_metadata(soa_id: int, payload: SOAMetadataUpdate): @app.post("/soa/{soa_id}/activities/{activity_id}/concepts") def set_activity_concepts(soa_id: int, activity_id: int, payload: ConceptsUpdate): """Update Biomedical Concept assigned to an Activity.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2061,7 +2115,7 @@ def ui_add_activity_concept( """Add Biomedical Concept to an Activity.""" if not activity_id: raise HTTPException(400, "Missing activity_id") - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") code = concept_code.strip() if not code: @@ -2109,7 +2163,7 @@ def ui_remove_activity_concept( """Remove Biomedical Concept from Activity.""" if not activity_id: raise HTTPException(400, "Missing activity_id") - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") code = concept_code.strip() if not code: @@ -2142,7 +2196,7 @@ def ui_remove_activity_concept( @app.post("/soa/{soa_id}/cells") def set_cell(soa_id: int, payload: CellCreate): """Set 'X' in SoA Matrix cell.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2179,7 +2233,7 @@ def set_cell(soa_id: int, payload: CellCreate): @app.get("/soa/{soa_id}/matrix") def get_matrix(soa_id: int): """Return SoA Matrix for Visits, Activities and assigned Matrix Cells.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) return {"visits": visits, "activities": activities, "cells": cells} @@ -2188,7 +2242,7 @@ def get_matrix(soa_id: int): @app.get("/soa/{soa_id}/export/xlsx") def export_xlsx(soa_id: int, left: Optional[int] = None, right: Optional[int] = None): """Export SoA Matrix to XLSX.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) if not visits or not activities: @@ -2407,7 +2461,7 @@ def export_pdf(soa_id: int): The PDF is intentionally simple and produced without external dependencies to avoid introducing new packages. It uses a single page with monospaced layout style commands. """ - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2576,7 +2630,7 @@ def add(line: str): @app.get("/soa/{soa_id}/normalized") def get_normalized(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") csv_path = _generate_wide_csv(soa_id) out_dir = os.path.join(NORMALIZED_ROOT, f"soa_{soa_id}") @@ -2590,7 +2644,7 @@ def get_normalized(soa_id: int): @app.post("/soa/{soa_id}/matrix/import") def import_matrix(soa_id: int, payload: MatrixImport): """Import SoA Matrix.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not payload.visits: raise HTTPException(400, "visits list empty") @@ -2689,7 +2743,7 @@ def _reindex(table: str, soa_id: int): @app.delete("/soa/{soa_id}/visits/{visit_id}") def delete_visit(soa_id: int, visit_id: int): """Delete Visit from an SoA.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2727,7 +2781,7 @@ def delete_visit(soa_id: int, visit_id: int): @app.delete("/soa/{soa_id}/activities/{activity_id}") def delete_activity(soa_id: int, activity_id: int): """Delete Activity from an SoA.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2758,7 +2812,7 @@ def delete_activity(soa_id: int, activity_id: int): @app.delete("/soa/{soa_id}/epochs/{epoch_id}") def delete_epoch(soa_id: int, epoch_id: int): """Delete an Epoch from an SoA.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2841,7 +2895,7 @@ def ui_index(request: Request): @app.post("/ui/soa/{soa_id}/add_activity", response_class=HTMLResponse) def ui_add_activity(request: Request, soa_id: int, name: str = Form(...)): """Add an Activity to an SoA.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") nm = (name or "").strip() if not nm: @@ -2869,6 +2923,9 @@ def ui_add_activity(request: Request, soa_id: int, name: str = Form(...)): "activity_uid": f"Activity_{order_index}", }, ) + # If HTMX, redirect back to edit page; otherwise script redirect + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": f"/ui/soa/{soa_id}/edit"}) return HTMLResponse( f"" ) @@ -2918,7 +2975,7 @@ def ui_update_meta( study_description: Optional[str] = Form(None), ): """Update the metadata for an SoA.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -2957,15 +3014,47 @@ def ui_update_meta( ) conn.commit() conn.close() + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": f"/ui/soa/{soa_id}/edit"}) return HTMLResponse( f"" ) +# Helper to fetch element audit rows with legacy-safe columns +def _fetch_element_audits(soa_id: int): + conn_ea = _connect() + cur_ea = conn_ea.cursor() + cur_ea.execute("PRAGMA table_info(element_audit)") + cols = [row[1] for row in cur_ea.fetchall()] + want = [ + "id", + "element_id", + "action", + "before_json", + "after_json", + "performed_at", + ] + available = [c for c in want if c in cols] + element_audits = [] + if available: + select_sql = f"SELECT {', '.join(available)} FROM element_audit WHERE soa_id=? ORDER BY id DESC" + cur_ea.execute(select_sql, (soa_id,)) + for r in cur_ea.fetchall(): + item = {} + for i, c in enumerate(available): + item[c] = r[i] + for k in want: + item.setdefault(k, None) + element_audits.append(item) + conn_ea.close() + return element_audits + + @app.get("/ui/soa/{soa_id}/edit", response_class=HTMLResponse) def ui_edit(request: Request, soa_id: int): """Render edit HTML page for an SoA.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") visits, activities, cells = _fetch_matrix(soa_id) # Epochs list @@ -2991,7 +3080,7 @@ def ui_edit(request: Request, soa_id: int): conn_el = _connect() cur_el = conn_el.cursor() cur_el.execute( - "SELECT id,name,label,description,testrl,teenrl,order_index,created_at FROM element WHERE soa_id=? ORDER BY order_index", + "SELECT id,name,label,description,testrl,teenrl,order_index,created_at,element_id FROM element WHERE soa_id=? ORDER BY order_index", (soa_id,), ) elements = [ @@ -3004,6 +3093,7 @@ def ui_edit(request: Request, soa_id: int): teenrl=r[5], order_index=r[6], created_at=r[7], + element_id=r[8], ) for r in cur_el.fetchall() ] @@ -3217,6 +3307,26 @@ def ui_edit(request: Request, soa_id: int): ] conn_epoch_audit.close() + # Admin audit view: recent study cell audits for this SoA + conn_sc_audit = _connect() + cur_sc_audit = conn_sc_audit.cursor() + cur_sc_audit.execute( + "SELECT id, study_cell_id, action, before_json, after_json, performed_at FROM study_cell_audit WHERE soa_id=? ORDER BY id DESC LIMIT 20", + (soa_id,), + ) + study_cell_audits = [ + { + "id": r[0], + "study_cell_id": r[1], + "action": r[2], + "before_json": r[3], + "after_json": r[4], + "performed_at": r[5], + } + for r in cur_sc_audit.fetchall() + ] + conn_sc_audit.close() + # Enrich epochs using API-only map: code -> submissionValue # Resolve stored epoch.type (code_uid) to terminology code via code table, then map to submissionValue. code_map: dict[int, str] = {} @@ -3266,6 +3376,32 @@ def ui_edit(request: Request, soa_id: int): except Exception: pass + study_cells = _list_study_cells(soa_id) + + # Transition Rules list + conn_tr = _connect() + cur_tr = conn_tr.cursor() + cur_tr.execute( + "SELECT transition_rule_uid,name,label,description,text,order_index,created_at FROM transition_rule WHERE soa_id=? ORDER BY order_index", + (soa_id,), + ) + transition_rules = [ + dict( + transition_rule_uid=r[0], + name=r[1], + label=r[2], + description=r[3], + text=r[4], + order_index=r[5], + created_at=r[6], + ) + for r in cur_tr.fetchall() + ] + conn_tr.close() + + # Element audit list + element_audits = _fetch_element_audits(soa_id) + return templates.TemplateResponse( request, "edit.html", @@ -3292,8 +3428,13 @@ def ui_edit(request: Request, soa_id: int): "arm_audits": arm_audits, "epoch_audits": epoch_audits, "activity_audits": activity_audits, + "study_cell_audits": study_cell_audits, + "element_audits": element_audits, # Epoch Type options (C99079) "epoch_type_options": epoch_type_options, + # Study Cells + "study_cells": study_cells, + "transition_rules": transition_rules, }, ) @@ -3575,7 +3716,7 @@ def ui_add_visit( Accepts either form field name `epoch_id_raw` (new) or `epoch_id` (legacy). Blank selection is treated as None without triggering 422 validation. """ - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Determine which raw epoch string was provided provided = (epoch_id_raw or "").strip() or (epoch_id or "").strip() @@ -3642,7 +3783,7 @@ async def ui_add_arm( element_id: Optional[str] = Form(None), ): """Form handler to create a new Arm.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Accept blank/empty element selection gracefully. The form may submit "" which would 422 with Optional[int]. eid = int(element_id) if element_id and element_id.strip().isdigit() else None @@ -3803,7 +3944,7 @@ async def ui_update_arm( element_id: Optional[str] = Form(None), ): """Form handler to update an existing Arm.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Read raw form to capture field names with hyphens: 'arm-type' and 'data-origin-type' @@ -4071,7 +4212,7 @@ def ui_delete_arm(request: Request, soa_id: int, arm_id: int = Form(...)): @app.post("/ui/soa/{soa_id}/reorder_arms", response_class=HTMLResponse) def ui_reorder_arms(request: Request, soa_id: int, order: str = Form("")): """Form handler to reorder existing Arms.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] if not ids: @@ -4111,7 +4252,7 @@ def ui_add_element( teenrl: Optional[str] = Form(None), ): """Form handler to add an element.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") name = (name or "").strip() if not name: @@ -4129,19 +4270,8 @@ def ui_add_element( element_cols = {r[1] for r in cur.fetchall()} element_identifier: Optional[str] = None if "element_id" in element_cols: - # Generate StudyElement_ where n is next unused integer for this SOA - cur.execute("SELECT element_id FROM element WHERE soa_id=?", (soa_id,)) - existing_raw = [r[0] for r in cur.fetchall() if r[0]] - used_nums = set() - for val in existing_raw: - if val.startswith("StudyElement_"): - tail = val.split("StudyElement_")[-1] - if tail.isdigit(): - used_nums.add(int(tail)) - next_n = 1 - while next_n in used_nums: - next_n += 1 - element_identifier = f"StudyElement_{next_n}" + # Generate StudyElement_ monotonically increasing for this SOA + element_identifier = _next_element_identifier(soa_id) cur.execute( """INSERT INTO element (soa_id,name,label,description,testrl,teenrl,order_index,created_at,element_id) VALUES (?,?,?,?,?,?,?,?,?)""", @@ -4175,10 +4305,11 @@ def ui_add_element( eid = cur.lastrowid conn.commit() conn.close() + # Audit should store the logical StudyElement_N in element_audit.element_id, not the row id _record_element_audit( soa_id, "create", - eid, + element_identifier, before=None, after={ "id": eid, @@ -4208,7 +4339,7 @@ def ui_update_element( teenrl: Optional[str] = Form(None), ): """Form handler to update an existing Element.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -4267,10 +4398,27 @@ def ui_update_element( updated_fields = [ f for f in mutable_fields if before and before.get(f) != after.get(f) ] + # Fetch element.element_id for audit key + try: + conn_k = _connect() + cur_k = conn_k.cursor() + cur_k.execute("PRAGMA table_info(element)") + cols_k = {r[1] for r in cur_k.fetchall()} + element_uid_for_audit = None + if "element_id" in cols_k: + cur_k.execute( + "SELECT element_id FROM element WHERE id=? AND soa_id=?", + (element_id, soa_id), + ) + row_k = cur_k.fetchone() + element_uid_for_audit = row_k[0] if row_k else None + conn_k.close() + except Exception: + element_uid_for_audit = None _record_element_audit( soa_id, "update", - element_id, + element_uid_for_audit, before=before, after={**after, "updated_fields": updated_fields}, ) @@ -4282,16 +4430,284 @@ def ui_update_element( @app.post("/ui/soa/{soa_id}/delete_element", response_class=HTMLResponse) def ui_delete_element(request: Request, soa_id: int, element_id: int = Form(...)): """Form handler to delete an existing Element.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() + # Capture before snapshot including logical element_id (StudyElement_N) if present + cur.execute("PRAGMA table_info(element)") + cols = {r[1] for r in cur.fetchall()} + has_uid = "element_id" in cols + if has_uid: + cur.execute( + "SELECT id, name, label, description, testrl, teenrl, order_index, element_id FROM element WHERE id=? AND soa_id=?", + (element_id, soa_id), + ) + else: + cur.execute( + "SELECT id, name, label, description, testrl, teenrl, order_index FROM element WHERE id=? AND soa_id=?", + (element_id, soa_id), + ) + row_b = cur.fetchone() + before = None + if row_b: + before = { + "id": row_b[0], + "name": row_b[1], + "label": row_b[2], + "description": row_b[3], + "testrl": row_b[4], + "teenrl": row_b[5], + "order_index": row_b[6], + "element_id": (row_b[7] if has_uid else None), + } + # Perform delete cur.execute("DELETE FROM element WHERE id=? AND soa_id=?", (element_id, soa_id)) conn.commit() conn.close() + # Use logical element_id (StudyElement_N) for audit key if available + element_uid_for_audit = ( + before.get("element_id") if isinstance(before, dict) else None + ) _record_element_audit( - soa_id, "delete", element_id, before={"id": element_id}, after=None + soa_id, + "delete", + element_uid_for_audit, + before=before or {"id": element_id}, + after=None, + ) + return HTMLResponse( + f"" + ) + + +# --------------------- Study Cell UI Endpoints --------------------- +def _next_study_cell_uid(cur, soa_id: int) -> str: + """Compute next StudyCell_N unique within an SoA.""" + cur.execute("SELECT study_cell_uid FROM study_cell WHERE soa_id=?", (soa_id,)) + max_n = 0 + for (uid,) in cur.fetchall(): + if isinstance(uid, str) and uid.startswith("StudyCell_"): + try: + n = int(uid.split("_")[-1]) + if n > max_n: + max_n = n + except Exception: + pass + return f"StudyCell_{max_n + 1}" + + +# _next_element_identifier is defined in routers/elements.py; imported above + + +@app.post("/ui/soa/{soa_id}/add_study_cell", response_class=HTMLResponse) +def ui_add_study_cell( + request: Request, + soa_id: int, + arm_uid: str = Form(...), + epoch_uid: str = Form(...), + element_uids: List[str] = Form(...), +): + """Add one or more Study Cell rows for Arm×Epoch×Elements. + + Duplicate prevention enforced on (soa_id, arm_uid, epoch_uid, element_uid). + """ + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + arm_uid = (arm_uid or "").strip() + epoch_uid = (epoch_uid or "").strip() + element_ids: list[str] = [ + str(e).strip() for e in (element_uids or []) if str(e).strip() + ] + if not arm_uid or not epoch_uid or not element_ids: + return HTMLResponse( + f"", + status_code=400, + ) + conn = _connect() + cur = conn.cursor() + # basic existence checks (optional) + cur.execute("SELECT 1 FROM arm WHERE soa_id=? AND arm_uid=?", (soa_id, arm_uid)) + if not cur.fetchone(): + conn.close() + return HTMLResponse( + f"", + status_code=404, + ) + cur.execute( + "SELECT 1 FROM epoch WHERE soa_id=? AND epoch_uid=?", (soa_id, epoch_uid) + ) + if not cur.fetchone(): + conn.close() + return HTMLResponse( + f"", + status_code=404, + ) + inserted = 0 + for el_uid in element_ids: + # ensure element exists if element_id column present + cur.execute("PRAGMA table_info(element)") + cols = {r[1] for r in cur.fetchall()} + if "element_id" in cols: + cur.execute( + "SELECT 1 FROM element WHERE soa_id=? AND element_id=?", + (soa_id, el_uid), + ) + if not cur.fetchone(): + # skip silently; or alert once (keeping UX simple) + continue + # duplicate prevention + cur.execute( + "SELECT id FROM study_cell WHERE soa_id=? AND arm_uid=? AND epoch_uid=? AND element_uid=?", + (soa_id, arm_uid, epoch_uid, el_uid), + ) + if cur.fetchone(): + continue + sc_uid = _next_study_cell_uid(cur, soa_id) + cur.execute( + "INSERT INTO study_cell (soa_id, study_cell_uid, arm_uid, epoch_uid, element_uid) VALUES (?,?,?,?,?)", + (soa_id, sc_uid, arm_uid, epoch_uid, el_uid), + ) + sc_id = cur.lastrowid + # Inline audit write for reliability + cur.execute( + "INSERT INTO study_cell_audit (soa_id, study_cell_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + ( + soa_id, + sc_id, + "create", + None, + json.dumps( + { + "study_cell_uid": sc_uid, + "arm_uid": arm_uid, + "epoch_uid": epoch_uid, + "element_uid": el_uid, + } + ), + datetime.now(timezone.utc).isoformat(), + ), + ) + inserted += 1 + conn.commit() + conn.close() + return HTMLResponse( + f"" + ) + + +@app.post("/ui/soa/{soa_id}/update_study_cell", response_class=HTMLResponse) +def ui_update_study_cell( + request: Request, + soa_id: int, + study_cell_id: int = Form(...), + arm_uid: Optional[str] = Form(None), + epoch_uid: Optional[str] = Form(None), + element_uid: Optional[str] = Form(None), +): + """Update a Study Cell's Arm/Epoch/Element values. + + Duplicate prevention enforced; if update causes a duplicate, no change is applied. + """ + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id, arm_uid, epoch_uid, element_uid FROM study_cell WHERE id=? AND soa_id=?", + (study_cell_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Study Cell not found") + _, curr_arm, curr_epoch, curr_el = row + new_arm = (arm_uid or curr_arm or "").strip() or curr_arm + new_epoch = (epoch_uid or curr_epoch or "").strip() or curr_epoch + new_el = (element_uid or curr_el or "").strip() or curr_el + # duplicate check + cur.execute( + "SELECT id FROM study_cell WHERE soa_id=? AND arm_uid=? AND epoch_uid=? AND element_uid=? AND id<>?", + (soa_id, new_arm, new_epoch, new_el, study_cell_id), + ) + if cur.fetchone(): + conn.close() + return HTMLResponse( + f"", + status_code=400, + ) + before = { + "arm_uid": curr_arm, + "epoch_uid": curr_epoch, + "element_uid": curr_el, + } + cur.execute( + "UPDATE study_cell SET arm_uid=?, epoch_uid=?, element_uid=? WHERE id=? AND soa_id=?", + (new_arm, new_epoch, new_el, study_cell_id, soa_id), + ) + # Inline audit write for reliability + cur.execute( + "INSERT INTO study_cell_audit (soa_id, study_cell_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + ( + soa_id, + study_cell_id, + "update", + json.dumps(before), + json.dumps( + { + "arm_uid": new_arm, + "epoch_uid": new_epoch, + "element_uid": new_el, + } + ), + datetime.now(timezone.utc).isoformat(), + ), + ) + conn.commit() + conn.close() + return HTMLResponse( + f"" + ) + + +@app.post("/ui/soa/{soa_id}/delete_study_cell", response_class=HTMLResponse) +def ui_delete_study_cell(request: Request, soa_id: int, study_cell_id: int = Form(...)): + """Delete a Study Cell by id.""" + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + # capture before state for audit + cur.execute( + "SELECT study_cell_uid, arm_uid, epoch_uid, element_uid FROM study_cell WHERE id=? AND soa_id=?", + (study_cell_id, soa_id), + ) + row = cur.fetchone() + before = None + if row: + before = { + "study_cell_uid": row[0], + "arm_uid": row[1], + "epoch_uid": row[2], + "element_uid": row[3], + } + # Inline audit write for reliability + cur.execute( + "INSERT INTO study_cell_audit (soa_id, study_cell_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + ( + soa_id, + study_cell_id, + "delete", + json.dumps(before) if before else None, + None, + datetime.now(timezone.utc).isoformat(), + ), ) + cur.execute( + "DELETE FROM study_cell WHERE id=? AND soa_id=?", (study_cell_id, soa_id) + ) + conn.commit() + conn.close() return HTMLResponse( f"" ) @@ -4307,7 +4723,7 @@ def ui_add_epoch( epoch_type_submission_value: Optional[str] = Form(None), ): """Form handler to add an Epoch.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -4396,7 +4812,7 @@ def ui_update_epoch( epoch_type_submission_value: Optional[str] = Form(None), ): """Form handler to update an existing Epoch.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -4543,6 +4959,299 @@ def ui_update_epoch( ) +# --------------------------- Transition Rules ----------------------# +def _next_transition_rule_uid(soa_id: int) -> str: + """Compute next monotonically increasing TransitionRule_N for an SoA. + Considers existing transition_rule rows and any prior UIDs found in transition_rule_audit. + This guarantees we never reuse a lower N even after deletes. + """ + conn = _connect() + cur = conn.cursor() + max_n = 0 + try: + cur.execute( + "SELECT transition_rule_uid FROM transition_rule WHERE soa_id=?", + (soa_id,), + ) + for (uid,) in cur.fetchall(): + if isinstance(uid, str) and uid.startswith("TransitionRule_"): + tail = uid.split("TransitionRule_")[-1] + if tail.isdigit(): + max_n = max(max_n, int(tail)) + except Exception: + pass + try: + cur.execute( + "SELECT before_json, after_json FROM transition_rule_audit WHERE soa_id=?", + (soa_id,), + ) + for bjson, ajson in cur.fetchall(): + for js in (bjson, ajson): + if not js: + continue + try: + obj = json.loads(js) + except Exception: + obj = None + if isinstance(obj, dict): + uid = obj.get("transition_rule_uid") or obj.get( + "transition_rule_id" + ) + if isinstance(uid, str) and uid.startswith("TransitionRule_"): + tail = uid.split("TransitionRule_")[-1] + if tail.isdigit(): + max_n = max(max_n, int(tail)) + except Exception: + pass + conn.close() + return f"TransitionRule_{max_n + 1}" + + +@app.post("/ui/soa/{soa_id}/add_transition_rule", response_class=HTMLResponse) +def ui_add_transition_rule( + request: Request, + soa_id: int, + name: str = Form(...), + label: Optional[str] = Form(None), + description: Optional[str] = Form(None), + text: str = Form(...), +): + """Form handler to add a Transition Rule.""" + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + name = (name or "").strip() + if not name: + raise HTTPException(400, "Name required") + text = (text or "").strip() + if not text: + raise HTTPException(400, "Text required") + conn = _connect() + cur = conn.cursor() + # Determine next order index + cur.execute( + "SELECT COALESCE(MAX(order_index), 0) FROM transition_rule WHERE soa_id=?", + (soa_id,), + ) + next_ord = (cur.fetchone() or [0])[0] + 1 + now = datetime.now(timezone.utc).isoformat() + # Generate TransitionRule_ monotonically increasing for this SOA + transition_rule_identifier = _next_transition_rule_uid(soa_id) + cur.execute( + """INSERT INTO transition_rule (soa_id,transition_rule_uid,name,label,description,text,order_index,created_at) VALUES (?,?,?,?,?,?,?,?)""", + ( + soa_id, + transition_rule_identifier, + name, + (label or "").strip() or None, + (description or "").strip() or None, + text, + next_ord, + now, + ), + ) + eid = cur.lastrowid + conn.commit() + conn.close() + _record_transition_rule_audit( + soa_id, + "create", + eid, + before=None, + after={ + "id": eid, + "transition_rule_uid": transition_rule_identifier, + "name": name, + "label": (label or "").strip() or None, + "description": (description or "").strip() or None, + "text": text, + "order_index": next_ord, + }, + ) + return HTMLResponse( + f"" + ) + + +@app.post("/ui/soa/{soa_id}/update_transition_rule", response_class=HTMLResponse) +def ui_transition_rule_update( + request: Request, + soa_id: int, + transition_rule_uid: str = Form(...), + name: Optional[str] = Form(None), + label: Optional[str] = Form(None), + description: Optional[str] = Form(None), + text: Optional[str] = Form(None), +): + """Form handler to update an existing Transition Rule.""" + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + # Verify exists and get id + cur.execute( + "SELECT id,transition_rule_uid,name,label,description,text,order_index,created_at FROM transition_rule WHERE soa_id=? AND transition_rule_uid=?", + (soa_id, transition_rule_uid), + ) + b = cur.fetchone() + if not b: + conn.close() + raise HTTPException(404, "Transition Rule not found") + before = { + "id": b[0], + "transition_rule_uid": b[1], + "name": b[2], + "label": b[3], + "description": b[4], + "text": b[5], + "order_index": b[6], + "created_at": b[7], + } + sets = [] + vals: list[Any] = [] + if name is not None: + sets.append("name=?") + vals.append((name or "").strip() or None) + if label is not None: + sets.append("label=?") + vals.append((label or "").strip() or None) + if description is not None: + sets.append("description=?") + vals.append((description or "").strip() or None) + if text is not None: + sets.append("text=?") + vals.append((text or "").strip() or None) + if sets: + vals.append(before["id"]) + cur.execute(f"UPDATE transition_rule SET {', '.join(sets)} WHERE id=?", vals) + conn.commit() + # Fetch after + cur.execute( + "SELECT id,name,label,description,text,order_index,created_at FROM transition_rule WHERE id=?", + (before["id"],), + ) + a = cur.fetchone() + conn.close() + after = { + "id": a[0], + "name": a[1], + "label": a[2], + "description": a[3], + "text": a[4], + "order_index": a[5], + "created_at": a[6], + } + mutable_fields = ["name", "label", "description", "text"] + updated_fields = [ + f for f in mutable_fields if before and before.get(f) != after.get(f) + ] + _record_transition_rule_audit( + soa_id, + "update", + before["id"], + before=before, + after={**after, "updated_fields": updated_fields}, + ) + # HTMX inline update: return refreshed list markup when requested + if request.headers.get("HX-Request") == "true": + conn_tr = _connect() + cur_tr = conn_tr.cursor() + cur_tr.execute( + "SELECT transition_rule_uid,name,label,description,text,order_index,created_at FROM transition_rule WHERE soa_id=? ORDER BY order_index", + (soa_id,), + ) + transition_rules = [ + dict( + transition_rule_uid=r[0], + name=r[1], + label=r[2], + description=r[3], + text=r[4], + order_index=r[5], + created_at=r[6], + ) + for r in cur_tr.fetchall() + ] + conn_tr.close() + html = templates.get_template("transition_rules_list.html").render( + transition_rules=transition_rules, soa_id=soa_id + ) + return HTMLResponse(html) + return HTMLResponse( + f"" + ) + + +@app.post("/ui/soa/{soa_id}/delete_transition_rule") +def ui_delete_transition_rule( + request: Request, soa_id: int, transition_rule_uid: str = Form(...) +): + """Form handler to delete a transition rule""" + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + # Capture before for audit + cur.execute( + "SELECT id,transition_rule_uid,name,label,description,text,order_index,created_at FROM transition_rule WHERE soa_id=? AND transition_rule_uid=?", + (soa_id, transition_rule_uid), + ) + b = cur.fetchone() + before = None + if b: + before = { + "id": b[0], + "transition_rule_uid": b[1], + "name": b[2], + "label": b[3], + "description": b[4], + "text": b[5], + "order_index": b[6], + "created_at": b[7], + } + cur.execute( + "DELETE FROM transition_rule WHERE transition_rule_uid=? AND soa_id=?", + (transition_rule_uid, soa_id), + ) + conn.commit() + conn.close() + _record_transition_rule_audit( + soa_id, + "delete", + before["id"] if before else None, + before=before, + after=None, + ) + # HTMX inline update: return refreshed list markup when requested + if request.headers.get("HX-Request") == "true": + conn_tr = _connect() + cur_tr = conn_tr.cursor() + cur_tr.execute( + "SELECT transition_rule_uid,name,label,description,text,order_index,created_at FROM transition_rule WHERE soa_id=? ORDER BY order_index", + (soa_id,), + ) + transition_rules = [ + dict( + transition_rule_uid=r[0], + name=r[1], + label=r[2], + description=r[3], + text=r[4], + order_index=r[5], + created_at=r[6], + ) + for r in cur_tr.fetchall() + ] + conn_tr.close() + html = templates.get_template("transition_rules_list.html").render( + transition_rules=transition_rules, soa_id=soa_id + ) + return HTMLResponse(html) + return HTMLResponse( + f"" + ) + + +# -------------------------- Biomedical Concepts --------------------# @app.post( "/ui/soa/{soa_id}/activity/{activity_id}/concepts", response_class=HTMLResponse ) @@ -4592,7 +5301,7 @@ def ui_activity_concepts_cell( # surface a clear 400 error rather than proceeding and causing confusing downstream behavior. if not activity_id: raise HTTPException(status_code=400, detail="Missing activity_id") - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") concepts = fetch_biomedical_concepts() conn = _connect() @@ -4640,7 +5349,7 @@ def ui_toggle_cell( ): """Toggle logic: blank -> X, X -> blank (delete row). Returns updated snippet with next action encoded. This avoids stale hx-vals attributes after a partial swap.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Determine current status conn = _connect() @@ -4707,7 +5416,7 @@ def ui_set_visit_epoch( epoch_id: str = Form(""), # legacy field name used by template select ): """Form handler to associate an Epoch with a Visit/Encounter.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") # Determine provided raw value (prefer epoch_id_raw if non-blank) raw_val = (epoch_id_raw or "").strip() or (epoch_id or "").strip() @@ -4767,7 +5476,7 @@ def ui_delete_epoch(request: Request, soa_id: int, epoch_id: int = Form(...)): @app.post("/ui/soa/{soa_id}/reorder_visits", response_class=HTMLResponse) def ui_reorder_visits(request: Request, soa_id: int, order: str = Form("")): """Persist new visit ordering. 'order' is a comma-separated list of visit IDs in desired order.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] if not ids: @@ -4795,7 +5504,7 @@ def ui_reorder_visits(request: Request, soa_id: int, order: str = Form("")): @app.post("/ui/soa/{soa_id}/reorder_activities", response_class=HTMLResponse) def ui_reorder_activities(request: Request, soa_id: int, order: str = Form("")): """Persist new activity ordering. 'order' is a comma-separated list of activity IDs in desired order.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] if not ids: @@ -4823,7 +5532,7 @@ def ui_reorder_activities(request: Request, soa_id: int, order: str = Form("")): @app.post("/ui/soa/{soa_id}/reorder_epochs", response_class=HTMLResponse) def ui_reorder_epochs(request: Request, soa_id: int, order: str = Form("")): """Form handler to persist new epoch ordering.""" - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") ids = [int(x) for x in order.split(",") if x.strip().isdigit()] if not ids: diff --git a/src/soa_builder/web/audit.py b/src/soa_builder/web/audit.py index 6c2b685..b2847ef 100644 --- a/src/soa_builder/web/audit.py +++ b/src/soa_builder/web/audit.py @@ -45,6 +45,18 @@ def _record_element_audit( try: conn = _connect() cur = conn.cursor() + # Ensure table exists (defensive for migrated databases) + cur.execute( + """CREATE TABLE IF NOT EXISTS element_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + element_id INTEGER, + action TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) cur.execute( "INSERT INTO element_audit (soa_id, element_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", ( @@ -139,3 +151,42 @@ def _record_activity_audit( conn.close() except Exception as e: logger.warning("Failed recording activity audit: %s", e) + + +def _record_study_cell_audit( + soa_id: int, + action: str, + study_cell_id: int | None, + before: Optional[Dict[str, Any]] = None, + after: Optional[Dict[str, Any]] = None, +): + try: + conn = _connect() + cur = conn.cursor() + # Ensure table exists (defensive for migrated databases) + cur.execute( + """CREATE TABLE IF NOT EXISTS study_cell_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + study_cell_id INTEGER, + action TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) + cur.execute( + "INSERT INTO study_cell_audit (soa_id, study_cell_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + ( + soa_id, + study_cell_id, + action, + json.dumps(before) if before else None, + json.dumps(after) if after else None, + datetime.now(timezone.utc).isoformat(), + ), + ) + conn.commit() + conn.close() + except Exception as e: + logger.warning("Failed recording study_cell audit: %s", e) diff --git a/src/soa_builder/web/initialize_database.py b/src/soa_builder/web/initialize_database.py index 8ccd3ac..c4c06d4 100644 --- a/src/soa_builder/web/initialize_database.py +++ b/src/soa_builder/web/initialize_database.py @@ -101,6 +101,30 @@ def _init_db(): performed_at TEXT NOT NULL )""" ) + # Study Cell audit table + cur.execute( + """CREATE TABLE IF NOT EXISTS study_cell_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + study_cell_id INTEGER, + action TEXT NOT NULL, -- create|update|delete + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) + # Transition rule audit table + cur.execute( + """CREATE TABLE IF NOT EXISTS transition_rule_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + transition_rule_id INTEGER, + action TEXT NOT NULL, -- create|update|delete + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) # Epochs: high-level study phase grouping (optional). Behaves like visits/activities list ordering. cur.execute( """CREATE TABLE IF NOT EXISTS epoch (id INTEGER PRIMARY KEY AUTOINCREMENT, soa_id INTEGER, name TEXT, order_index INTEGER)""" @@ -160,5 +184,33 @@ def _init_db(): )""" ) + # create the study_cell table to store the relationship between Epoch, Arm and related elements + cur.execute( + """CREATE TABLE IF NOT EXISTS study_cell ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + study_cell_uid TEXT NOT NULL, --immutable StudyCell_N identifier unique within SOA + arm_uid TEXT NOT NULL, + epoch_uid TEXT NOT NULL, + element_uid TEXT NOT NULL + )""" + ) + + # create the transition_rule table to store the transition rules for elements, encounters + cur.execute( + """CREATE TABLE IF NOT EXISTS transition_rule ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + transition_rule_uid TEXT NOT NULL, --immutable TransitionRule_N identifier unique within SOA + name TEXT NOT NULL, + label TEXT, + description TEXT, + text TEXT NOT NULL, + order_index INTEGER, + created_at TEXT, + UNIQUE(soa_id, transition_rule_uid) + )""" + ) + conn.commit() conn.close() diff --git a/src/soa_builder/web/migrate_database.py b/src/soa_builder/web/migrate_database.py index 16fc06e..03b8ef0 100644 --- a/src/soa_builder/web/migrate_database.py +++ b/src/soa_builder/web/migrate_database.py @@ -655,6 +655,62 @@ def _migrate_arm_add_type_fields(): logger.warning("Arm type/data_origin_type migration failed: %s", e) +# Migration: Ensure element_audit has before_json/after_json columns +def _migrate_element_audit_columns(): + """Add missing columns before_json and after_json to element_audit. + + Handles legacy schemas that only had id, soa_id, element_id, action, performed_at. + Safe to run multiple times; idempotent via schema inspection. + """ + try: + conn = _connect() + cur = conn.cursor() + # Ensure table exists; if not present, create with full schema + cur.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='element_audit'" + ) + exists = cur.fetchone() is not None + if not exists: + cur.execute( + """CREATE TABLE IF NOT EXISTS element_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + element_id INTEGER, + action TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) + conn.commit() + logger.info("Created element_audit table with full schema") + conn.close() + return + # Add columns if missing + cur.execute("PRAGMA table_info(element_audit)") + cols = {r[1] for r in cur.fetchall()} + alters = [] + if "before_json" not in cols: + alters.append("ALTER TABLE element_audit ADD COLUMN before_json TEXT") + if "after_json" not in cols: + alters.append("ALTER TABLE element_audit ADD COLUMN after_json TEXT") + for stmt in alters: + try: + cur.execute(stmt) + except Exception as e: # pragma: no cover + logger.warning( + "Failed element_audit column migration '%s': %s", stmt, e + ) + if alters: + conn.commit() + logger.info( + "Applied element_audit column migrations: %s", ", ".join(alters) + ) + conn.close() + except Exception as e: # pragma: no cover + logger.warning("element_audit column migration failed: %s", e) + + # Backfill dataset_date for existing terminology tables def _backfill_dataset_date(table: str, audit_table: str): """If terminology table exists and has dataset_date (or sheet_dataset_date) column with blank values, diff --git a/src/soa_builder/web/routers/activities.py b/src/soa_builder/web/routers/activities.py index b29b2e6..afd795f 100644 --- a/src/soa_builder/web/routers/activities.py +++ b/src/soa_builder/web/routers/activities.py @@ -1,4 +1,5 @@ import json +import logging # Lightweight concept fetcher to avoid circular import with app.py import os @@ -11,10 +12,14 @@ from ..audit import _record_activity_audit, _record_reorder_audit from ..db import _connect from ..schemas import ActivityCreate, ActivityUpdate, BulkActivities +from ..utils import soa_exists _ACT_CONCEPT_CACHE = {"data": None, "fetched_at": 0} _ACT_CONCEPT_TTL = 60 * 60 +router = APIRouter(prefix="/soa/{soa_id}") +logger = logging.getLogger("soa_builder.web.routers.activities") + def fetch_biomedical_concepts(force: bool = False): override_json = os.environ.get("CDISC_CONCEPTS_JSON") @@ -53,7 +58,8 @@ def fetch_biomedical_concepts(force: bool = False): if code: concepts.append({"code": code, "title": title}) return concepts - except Exception: + except Exception as e: + logger.debug("fetch_biomedical_concepts override JSON parse failed: %s", e) return [] now = time.time() if ( @@ -68,21 +74,12 @@ def fetch_biomedical_concepts(force: bool = False): return [] -router = APIRouter(prefix="/soa/{soa_id}") - - -def _soa_exists(soa_id: int) -> bool: - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - ok = cur.fetchone() is not None - conn.close() - return ok +# Removed local _soa_exists; using shared utils.soa_exists @router.get("/activities", response_class=JSONResponse) def list_activities(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -100,7 +97,7 @@ def list_activities(soa_id: int): @router.get("/activities/{activity_id}", response_class=JSONResponse) def get_activity(soa_id: int, activity_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -123,7 +120,7 @@ def get_activity(soa_id: int, activity_id: int): @router.post("/activities", response_class=JSONResponse) def add_activity(soa_id: int, payload: ActivityCreate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -152,7 +149,7 @@ def add_activity(soa_id: int, payload: ActivityCreate): @router.patch("/activities/{activity_id}", response_class=JSONResponse) def update_activity(soa_id: int, activity_id: int, payload: ActivityUpdate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -196,7 +193,7 @@ def update_activity(soa_id: int, activity_id: int, payload: ActivityUpdate): @router.post("/activities/reorder", response_class=JSONResponse) def reorder_activities_api(soa_id: int, order: List[int]): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: raise HTTPException(400, "Order list required") @@ -256,7 +253,7 @@ def reorder_activities_api(soa_id: int, order: List[int]): @router.post("/activities/bulk", response_class=JSONResponse) def add_activities_bulk(soa_id: int, payload: BulkActivities): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") names = [n.strip() for n in payload.names if n and n.strip()] if not names: @@ -293,7 +290,7 @@ def add_activities_bulk(soa_id: int, payload: BulkActivities): @router.post("/activities/{activity_id}/concepts", response_class=JSONResponse) def set_activity_concepts(soa_id: int, activity_id: int, concept_codes: List[str]): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() diff --git a/src/soa_builder/web/routers/arms.py b/src/soa_builder/web/routers/arms.py index b3eccbc..53f2a7c 100644 --- a/src/soa_builder/web/routers/arms.py +++ b/src/soa_builder/web/routers/arms.py @@ -7,22 +7,18 @@ from ..audit import _record_arm_audit, _record_reorder_audit from ..db import _connect from ..schemas import ArmCreate, ArmUpdate +from ..utils import soa_exists router = APIRouter(prefix="/soa/{soa_id}") +logger = logging.getLogger("soa_builder.web.routers.arms") -def _soa_exists(soa_id: int) -> bool: - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - ok = cur.fetchone() is not None - conn.close() - return ok +# Removed local _soa_exists; using shared utils.soa_exists @router.get("/arms", response_class=JSONResponse) def list_arms(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -49,7 +45,7 @@ def list_arms(soa_id: int): @router.post("/arms", response_class=JSONResponse, status_code=201) def create_arm(soa_id: int, payload: ArmCreate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") name = (payload.name or "").strip() if not name: @@ -72,7 +68,7 @@ def create_arm(soa_id: int, payload: ArmCreate): if tail.isdigit(): used_nums.add(int(tail)) else: - logging.getLogger("soa_builder.concepts").warning( + logger.warning( "Invalid arm_uid format encountered (ignored for numbering): %s", uid, ) @@ -113,7 +109,7 @@ def create_arm(soa_id: int, payload: ArmCreate): @router.patch("/arms/{arm_id}", response_class=JSONResponse) def update_arm(soa_id: int, arm_id: int, payload: ArmUpdate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -190,7 +186,7 @@ def update_arm(soa_id: int, arm_id: int, payload: ArmUpdate): @router.delete("/arms/{arm_id}", response_class=JSONResponse) def delete_arm(soa_id: int, arm_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -221,7 +217,7 @@ def delete_arm(soa_id: int, arm_id: int): @router.post("/arms/reorder", response_class=JSONResponse) def reorder_arms_api(soa_id: int, order: List[int]): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: raise HTTPException(400, "Order list required") diff --git a/src/soa_builder/web/routers/elements.py b/src/soa_builder/web/routers/elements.py index 245b623..2aecfcc 100644 --- a/src/soa_builder/web/routers/elements.py +++ b/src/soa_builder/web/routers/elements.py @@ -1,34 +1,109 @@ import json +import logging from datetime import datetime, timezone -from typing import List +from typing import List, Optional from fastapi import APIRouter, HTTPException from fastapi.responses import JSONResponse from ..audit import _record_element_audit from ..db import _connect +from ..utils import soa_exists from ..schemas import ElementCreate, ElementUpdate router = APIRouter(prefix="/soa/{soa_id}") +logger = logging.getLogger("soa_builder.web.routers.elements") -def _soa_exists(soa_id: int) -> bool: +"""Shared SOA existence check imported from utils.soa_exists""" + + +def _next_element_identifier(soa_id: int) -> str: + """Compute next monotonically increasing StudyElement_N for an SoA. + Scans current element rows and element_audit snapshots to avoid reusing numbers after deletes. + """ conn = _connect() cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - ok = cur.fetchone() is not None + max_n = 0 + try: + cur.execute("SELECT element_id FROM element WHERE soa_id=?", (soa_id,)) + for (eid,) in cur.fetchall(): + if isinstance(eid, str) and eid.startswith("StudyElement_"): + tail = eid.split("StudyElement_")[-1] + if tail.isdigit(): + max_n = max(max_n, int(tail)) + except Exception as e: + logger.exception( + "_next_element_identifier scan elements failed for soa_id=%s: %s", + soa_id, + e, + ) + try: + cur.execute( + "SELECT before_json, after_json FROM element_audit WHERE soa_id=?", + (soa_id,), + ) + for bjson, ajson in cur.fetchall(): + for js in (bjson, ajson): + if not js: + continue + try: + obj = json.loads(js) + except Exception as e: + logger.debug( + "_next_element_identifier JSON parse failed soa_id=%s: %s", + soa_id, + e, + ) + obj = None + if isinstance(obj, dict): + val = obj.get("element_id") + if isinstance(val, str) and val.startswith("StudyElement_"): + tail = val.split("StudyElement_")[-1] + if tail.isdigit(): + max_n = max(max_n, int(tail)) + except Exception as e: + logger.exception( + "_next_element_identifier scan element_audit failed for soa_id=%s: %s", + soa_id, + e, + ) conn.close() - return ok + return f"StudyElement_{max_n + 1}" + + +def _get_element_uid(soa_id: int, row_id: int) -> Optional[str]: + """Return element.element_id (StudyElement_N) for row id if column exists, else None.""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute("PRAGMA table_info(element)") + cols = {r[1] for r in cur.fetchall()} + if "element_id" not in cols: + conn.close() + return None + cur.execute( + "SELECT element_id FROM element WHERE id=? AND soa_id=?", + (row_id, soa_id), + ) + r = cur.fetchone() + conn.close() + return r[0] if r else None + except Exception as e: + logger.exception( + "_get_element_uid failed for soa_id=%s row_id=%s: %s", soa_id, row_id, e + ) + return None @router.get("/elements", response_class=JSONResponse) def list_elements(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() cur.execute( - "SELECT id,name,label,description,testrl,teenrl,order_index,created_at FROM element WHERE soa_id=? ORDER BY order_index", + "SELECT id,name,label,description,testrl,teenrl,order_index,created_at,element_id FROM element WHERE soa_id=? ORDER BY order_index", (soa_id,), ) rows = [ @@ -41,6 +116,7 @@ def list_elements(soa_id: int): "teenrl": r[5], "order_index": r[6], "created_at": r[7], + "element_id": r[8] if len(r) > 8 else None, } for r in cur.fetchall() ] @@ -50,12 +126,12 @@ def list_elements(soa_id: int): @router.get("/elements/{element_id}", response_class=JSONResponse) def get_element(soa_id: int, element_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() cur.execute( - "SELECT id,name,label,description,testrl,teenrl,order_index,created_at FROM element WHERE id=? AND soa_id=?", + "SELECT id,name,label,description,testrl,teenrl,order_index,created_at,element_id FROM element WHERE id=? AND soa_id=?", (element_id, soa_id), ) r = cur.fetchone() @@ -72,12 +148,13 @@ def get_element(soa_id: int, element_id: int): "teenrl": r[5], "order_index": r[6], "created_at": r[7], + "element_id": r[8] if len(r) > 8 else None, } @router.get("/element_audit", response_class=JSONResponse) def list_element_audit(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -89,11 +166,17 @@ def list_element_audit(soa_id: int): for r in cur.fetchall(): try: before = json.loads(r[3]) if r[3] else None - except Exception: + except Exception as e: + logger.debug( + "list_element_audit before JSON parse failed soa_id=%s: %s", soa_id, e + ) before = None try: after = json.loads(r[4]) if r[4] else None - except Exception: + except Exception as e: + logger.debug( + "list_element_audit after JSON parse failed soa_id=%s: %s", soa_id, e + ) after = None rows.append( { @@ -111,7 +194,7 @@ def list_element_audit(soa_id: int): @router.post("/elements", response_class=JSONResponse, status_code=201) def create_element(soa_id: int, payload: ElementCreate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") name = (payload.name or "").strip() if not name: @@ -123,20 +206,42 @@ def create_element(soa_id: int, payload: ElementCreate): ) next_ord = (cur.fetchone() or [0])[0] + 1 now = datetime.now(timezone.utc).isoformat() - cur.execute( - """INSERT INTO element (soa_id,name,label,description,testrl,teenrl,order_index,created_at) - VALUES (?,?,?,?,?,?,?,?)""", - ( - soa_id, - name, - (payload.label or "").strip() or None, - (payload.description or "").strip() or None, - (payload.testrl or "").strip() or None, - (payload.teenrl or "").strip() or None, - next_ord, - now, - ), - ) + # Insert, setting element_id if column exists + cur.execute("PRAGMA table_info(element)") + element_cols = {r[1] for r in cur.fetchall()} + element_identifier: Optional[str] = None + if "element_id" in element_cols: + element_identifier = _next_element_identifier(soa_id) + cur.execute( + """INSERT INTO element (soa_id,name,label,description,testrl,teenrl,order_index,created_at,element_id) + VALUES (?,?,?,?,?,?,?,?,?)""", + ( + soa_id, + name, + (payload.label or "").strip() or None, + (payload.description or "").strip() or None, + (payload.testrl or "").strip() or None, + (payload.teenrl or "").strip() or None, + next_ord, + now, + element_identifier, + ), + ) + else: + cur.execute( + """INSERT INTO element (soa_id,name,label,description,testrl,teenrl,order_index,created_at) + VALUES (?,?,?,?,?,?,?,?)""", + ( + soa_id, + name, + (payload.label or "").strip() or None, + (payload.description or "").strip() or None, + (payload.testrl or "").strip() or None, + (payload.teenrl or "").strip() or None, + next_ord, + now, + ), + ) eid = cur.lastrowid conn.commit() conn.close() @@ -149,19 +254,21 @@ def create_element(soa_id: int, payload: ElementCreate): "teenrl": (payload.teenrl or "").strip() or None, "order_index": next_ord, "created_at": now, + "element_id": element_identifier, } - _record_element_audit(soa_id, "create", eid, before=None, after=el) + # Audit with logical StudyElement_N when available + _record_element_audit(soa_id, "create", element_identifier, before=None, after=el) return el @router.patch("/elements/{element_id}", response_class=JSONResponse) def update_element(soa_id: int, element_id: int, payload: ElementUpdate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() cur.execute( - "SELECT id,name,label,description,testrl,teenrl,order_index,created_at FROM element WHERE id=? AND soa_id=?", + "SELECT id,name,label,description,testrl,teenrl,order_index,created_at,element_id FROM element WHERE id=? AND soa_id=?", (element_id, soa_id), ) row = cur.fetchone() @@ -177,6 +284,7 @@ def update_element(soa_id: int, element_id: int, payload: ElementUpdate): "teenrl": row[5], "order_index": row[6], "created_at": row[7], + "element_id": row[8], } new_name = (payload.name if payload.name is not None else before["name"]) or "" cur.execute( @@ -196,7 +304,7 @@ def update_element(soa_id: int, element_id: int, payload: ElementUpdate): ) conn.commit() cur.execute( - "SELECT id,name,label,description,testrl,teenrl,order_index,created_at FROM element WHERE id=?", + "SELECT id,name,label,description,testrl,teenrl,order_index,created_at,element_id FROM element WHERE id=?", (element_id,), ) r = cur.fetchone() @@ -210,13 +318,18 @@ def update_element(soa_id: int, element_id: int, payload: ElementUpdate): "teenrl": r[5], "order_index": r[6], "created_at": r[7], + "element_id": r[8], } mutable_fields = ["name", "label", "description", "testrl", "teenrl"] updated_fields = [f for f in mutable_fields if before.get(f) != after.get(f)] + # Audit with logical StudyElement_N key + element_uid_for_audit = after.get("element_id") or _get_element_uid( + soa_id, element_id + ) _record_element_audit( soa_id, "update", - element_id, + element_uid_for_audit, before=before, after={**after, "updated_fields": updated_fields}, ) @@ -225,12 +338,12 @@ def update_element(soa_id: int, element_id: int, payload: ElementUpdate): @router.delete("/elements/{element_id}", response_class=JSONResponse) def delete_element(soa_id: int, element_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() cur.execute( - "SELECT id,name,label,description,testrl,teenrl,order_index,created_at FROM element WHERE id=? AND soa_id=?", + "SELECT id,name,label,description,testrl,teenrl,order_index,created_at,element_id FROM element WHERE id=? AND soa_id=?", (element_id, soa_id), ) row = cur.fetchone() @@ -246,17 +359,24 @@ def delete_element(soa_id: int, element_id: int): "teenrl": row[5], "order_index": row[6], "created_at": row[7], + "element_id": row[8], } cur.execute("DELETE FROM element WHERE id=?", (element_id,)) conn.commit() conn.close() - _record_element_audit(soa_id, "delete", element_id, before=before, after=None) + _record_element_audit( + soa_id, + "delete", + before.get("element_id"), + before=before, + after=None, + ) return JSONResponse({"deleted": True, "id": element_id}) @router.post("/elements/reorder", response_class=JSONResponse) def reorder_elements_api(soa_id: int, order: List[int]): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: raise HTTPException(400, "Order list required") diff --git a/src/soa_builder/web/routers/epochs.py b/src/soa_builder/web/routers/epochs.py index 3e00146..f77c562 100644 --- a/src/soa_builder/web/routers/epochs.py +++ b/src/soa_builder/web/routers/epochs.py @@ -1,4 +1,5 @@ import json +import logging import os import sqlite3 from datetime import datetime, timezone @@ -8,25 +9,18 @@ from fastapi.responses import JSONResponse from ..schemas import EpochCreate, EpochUpdate +from ..utils import soa_exists DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") router = APIRouter() +logger = logging.getLogger("soa_builder.web.routers.epochs") def _connect(): return sqlite3.connect(DB_PATH) -def _soa_exists(soa_id: int) -> bool: - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - row = cur.fetchone() - conn.close() - return row is not None - - def _record_epoch_audit( soa_id: int, action: str, @@ -50,13 +44,19 @@ def _record_epoch_audit( ) conn.commit() conn.close() - except Exception: - pass + except Exception as e: + logger.exception( + "_record_epoch_audit failed soa_id=%s epoch_id=%s action=%s: %s", + soa_id, + epoch_id, + action, + e, + ) @router.post("/soa/{soa_id}/epochs") def add_epoch(soa_id: int, payload: EpochCreate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -99,7 +99,7 @@ def add_epoch(soa_id: int, payload: EpochCreate): @router.get("/soa/{soa_id}/epochs") def list_epochs(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -124,7 +124,7 @@ def list_epochs(soa_id: int): @router.get("/soa/{soa_id}/epochs/{epoch_id}") def get_epoch(soa_id: int, epoch_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -149,7 +149,7 @@ def get_epoch(soa_id: int, epoch_id: int): @router.post("/soa/{soa_id}/epochs/{epoch_id}/metadata") def update_epoch_metadata(soa_id: int, epoch_id: int, payload: EpochUpdate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -178,8 +178,13 @@ def update_epoch_metadata(soa_id: int, epoch_id: int, payload: EpochUpdate): tr = cur.fetchone() if before is not None: before["type"] = tr[0] if tr else None - except Exception: - pass + except Exception as e: + logger.debug( + "update_epoch_metadata type fetch failed soa_id=%s epoch_id=%s: %s", + soa_id, + epoch_id, + e, + ) sets = [] vals = [] if payload.name is not None: @@ -223,7 +228,7 @@ def update_epoch_metadata(soa_id: int, epoch_id: int, payload: EpochUpdate): @router.post("/soa/{soa_id}/epochs/reorder", response_class=JSONResponse) def reorder_epochs_api(soa_id: int, order: List[int]): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: raise HTTPException(400, "Order list required") diff --git a/src/soa_builder/web/routers/freezes.py b/src/soa_builder/web/routers/freezes.py index eba1c90..839738e 100644 --- a/src/soa_builder/web/routers/freezes.py +++ b/src/soa_builder/web/routers/freezes.py @@ -1,29 +1,26 @@ import json +import logging import os import sqlite3 from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, JSONResponse from fastapi.templating import Jinja2Templates +from ..utils import soa_exists DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "templates") templates = Jinja2Templates(directory=TEMPLATES_DIR) router = APIRouter() +logger = logging.getLogger("soa_builder.web.routers.freezes") def _connect(): return sqlite3.connect(DB_PATH) -def _soa_exists(soa_id: int) -> bool: - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - r = cur.fetchone() - conn.close() - return r is not None +# Removed local _soa_exists; using shared utils.soa_exists # Dynamic helper imports inside endpoint bodies avoid circular import at module load. @@ -31,7 +28,7 @@ def _soa_exists(soa_id: int) -> bool: @router.post("/ui/soa/{soa_id}/freeze", response_class=HTMLResponse) def ui_freeze_soa(request: Request, soa_id: int, version_label: str = Form("")): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") try: from ..app import _create_freeze # type: ignore @@ -52,7 +49,7 @@ def ui_freeze_soa(request: Request, soa_id: int, version_label: str = Form("")): @router.get("/soa/{soa_id}/freeze/{freeze_id}") def get_freeze(soa_id: int, freeze_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -66,7 +63,13 @@ def get_freeze(soa_id: int, freeze_id: int): raise HTTPException(404, "Freeze not found") try: data = json.loads(row[0]) - except Exception: + except Exception as e: + logger.exception( + "get_freeze JSON decode failed soa_id=%s freeze_id=%s: %s", + soa_id, + freeze_id, + e, + ) data = {"error": "Corrupt snapshot"} return JSONResponse(data) diff --git a/src/soa_builder/web/routers/rollback.py b/src/soa_builder/web/routers/rollback.py index b32d02f..01a3d50 100644 --- a/src/soa_builder/web/routers/rollback.py +++ b/src/soa_builder/web/routers/rollback.py @@ -1,32 +1,28 @@ import io import os +import logging import pandas as pd from fastapi import APIRouter, HTTPException, Request from fastapi.responses import HTMLResponse, StreamingResponse from fastapi.templating import Jinja2Templates -from ..db import _connect +from ..utils import soa_exists DB_PATH = os.environ.get("SOA_BUILDER_DB", "soa_builder_web.db") TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "templates") templates = Jinja2Templates(directory=TEMPLATES_DIR) router = APIRouter() +logger = logging.getLogger("soa_builder.web.routers.rollback") -def _soa_exists(soa_id: int) -> bool: - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - r = cur.fetchone() - conn.close() - return r is not None +# Removed local _soa_exists; using shared utils.soa_exists @router.get("/soa/{soa_id}/rollback_audit") def get_rollback_audit_json(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") from ..app import _list_rollback_audit # type: ignore @@ -35,7 +31,7 @@ def get_rollback_audit_json(soa_id: int): @router.get("/soa/{soa_id}/reorder_audit") def get_reorder_audit_json(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") from ..app import _list_reorder_audit # type: ignore @@ -44,7 +40,7 @@ def get_reorder_audit_json(soa_id: int): @router.get("/ui/soa/{soa_id}/rollback_audit", response_class=HTMLResponse) def ui_rollback_audit(request: Request, soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") from ..app import _list_rollback_audit # type: ignore @@ -57,7 +53,7 @@ def ui_rollback_audit(request: Request, soa_id: int): @router.get("/ui/soa/{soa_id}/reorder_audit", response_class=HTMLResponse) def ui_reorder_audit(request: Request, soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") from ..app import _list_reorder_audit # type: ignore @@ -70,7 +66,7 @@ def ui_reorder_audit(request: Request, soa_id: int): @router.get("/soa/{soa_id}/rollback_audit/export/xlsx") def export_rollback_audit_xlsx(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") from ..app import _list_rollback_audit # type: ignore @@ -103,30 +99,38 @@ def export_rollback_audit_xlsx(soa_id: int): @router.get("/soa/{soa_id}/reorder_audit/export/xlsx") def export_reorder_audit_xlsx(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") from ..app import _list_reorder_audit # type: ignore rows = _list_reorder_audit(soa_id) flat = [] for r in rows: - moves = [] - old_pos = {vid: idx + 1 for idx, vid in enumerate(r.get("old_order", []))} - new_order = r.get("new_order", []) - for idx, vid in enumerate(new_order, start=1): - op = old_pos.get(vid) - if op and op != idx: - moves.append(f"{vid}:{op}->{idx}") - flat.append( - { - "id": r.get("id"), - "entity_type": r.get("entity_type"), - "performed_at": r.get("performed_at"), - "old_order": ",".join(map(str, r.get("old_order", []))), - "new_order": ",".join(map(str, new_order)), - "moves": "; ".join(moves) if moves else "", - } - ) + try: + moves = [] + old_pos = {vid: idx + 1 for idx, vid in enumerate(r.get("old_order", []))} + new_order = r.get("new_order", []) + for idx, vid in enumerate(new_order, start=1): + op = old_pos.get(vid) + if op and op != idx: + moves.append(f"{vid}:{op}->{idx}") + flat.append( + { + "id": r.get("id"), + "entity_type": r.get("entity_type"), + "performed_at": r.get("performed_at"), + "old_order": ",".join(map(str, r.get("old_order", []))), + "new_order": ",".join(map(str, new_order)), + "moves": "; ".join(moves) if moves else "", + } + ) + except Exception as e: + logger.debug( + "export_reorder_audit_xlsx flatten failure soa_id=%s row_id=%s: %s", + soa_id, + r.get("id"), + e, + ) df = pd.DataFrame(flat) if df.empty: df = pd.DataFrame( diff --git a/src/soa_builder/web/routers/visits.py b/src/soa_builder/web/routers/visits.py index 87296c5..dcfdf9d 100644 --- a/src/soa_builder/web/routers/visits.py +++ b/src/soa_builder/web/routers/visits.py @@ -1,27 +1,24 @@ from typing import List from fastapi import APIRouter, HTTPException +import logging from fastapi.responses import JSONResponse from ..audit import _record_reorder_audit, _record_visit_audit from ..db import _connect +from ..utils import soa_exists from ..schemas import VisitCreate, VisitUpdate router = APIRouter(prefix="/soa/{soa_id}") +logger = logging.getLogger("soa_builder.web.routers.visits") -def _soa_exists(soa_id: int) -> bool: - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT 1 FROM soa WHERE id=?", (soa_id,)) - ok = cur.fetchone() is not None - conn.close() - return ok +# Removed local _soa_exists; using shared utils.soa_exists @router.get("/visits", response_class=JSONResponse) def list_visits(soa_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -45,7 +42,7 @@ def list_visits(soa_id: int): @router.get("/visits/{visit_id}", response_class=JSONResponse) def get_visit(soa_id: int, visit_id: int): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -69,7 +66,7 @@ def get_visit(soa_id: int, visit_id: int): @router.post("/visits", response_class=JSONResponse) def add_visit(soa_id: int, payload: VisitCreate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -108,7 +105,7 @@ def add_visit(soa_id: int, payload: VisitCreate): @router.patch("/visits/{visit_id}", response_class=JSONResponse) def update_visit(soa_id: int, visit_id: int, payload: VisitUpdate): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") conn = _connect() cur = conn.cursor() @@ -178,7 +175,7 @@ def update_visit(soa_id: int, visit_id: int, payload: VisitUpdate): @router.post("/visits/reorder", response_class=JSONResponse) def reorder_visits_api(soa_id: int, order: List[int]): - if not _soa_exists(soa_id): + if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") if not order: raise HTTPException(400, "Order list required") diff --git a/src/soa_builder/web/templates/ddf_terminology.html b/src/soa_builder/web/templates/ddf_terminology.html index 4a6bfd5..65b3277 100644 --- a/src/soa_builder/web/templates/ddf_terminology.html +++ b/src/soa_builder/web/templates/ddf_terminology.html @@ -4,10 +4,12 @@

DDF Terminology

{% if uploaded %}
Upload successful: table reloaded.
{% endif %} {% if error %}
Error: {{ error }}
{% endif %}
- Upload new terminology file:
- - - +
+ Upload Excel Workbook (.xls/.xlsx) + + + +
diff --git a/src/soa_builder/web/templates/edit.html b/src/soa_builder/web/templates/edit.html index 2e6e134..1c251fd 100644 --- a/src/soa_builder/web/templates/edit.html +++ b/src/soa_builder/web/templates/edit.html @@ -184,11 +184,11 @@

Editing SoA {{ soa_id }}

{{ el.order_index }}. {{ el.name }} {% if el.label %}[{{ el.label }}]{% endif %} {% if el.testrl or el.teenrl %}(rules){% endif %} - +
-
+ @@ -200,7 +200,7 @@

Editing SoA {{ soa_id }}

{% endfor %} - + @@ -277,6 +277,85 @@

Editing SoA {{ soa_id }}

+
+ Study Cells ({{ study_cells|length }}) +
+
+ + +
+
+ + +
+
+ + + Select one or more elements (Cmd/Ctrl+Click). +
+
+ +
+
+
+ + + + + + + + + {% for sc in study_cells %} + + + + + + + + {% else %} + + {% endfor %} +
UIDArmEpochElementActions
{{ sc.study_cell_uid }} + {% set arm_match = arms | selectattr('arm_uid', 'equalto', sc.arm_uid) | list %} + {% if arm_match and arm_match[0] and arm_match[0].name %} + {{ arm_match[0].name }} + {% else %} + {{ sc.arm_uid }} + {% endif %} + {{ sc.epoch_name or sc.epoch_uid }}{{ sc.element_name or sc.element_uid }} +
+ + +
+
No study cells yet.
+
+
+
+ Transition Rules ({{ transition_rules|length }}) + {% include 'transition_rules_list.html' %} +
+ + + + + +
+
@@ -360,6 +439,34 @@

Editing SoA {{ soa_id }}

No epoch audit entries yet.
{% endif %}
+
+ Study Cell Audit (latest {{ study_cell_audits|length }}) + {% if study_cell_audits %} + + + + + + + + + + {% for au in study_cell_audits %} + + + + + + + + + {% endfor %} +
IDStudy CellActionPerformedBeforeAfter
{{ au.id }}{{ au.study_cell_id }}{{ au.action }}{{ au.performed_at }}{{ au.before_json or '' }}{{ au.after_json or '' }}
+ {% else %} +
No study cell audit entries yet.
+ {% endif %} +
+ {% include 'element_audit_section.html' %}

@@ -370,8 +477,8 @@

Matrix

Concepts {% for v in visits %} -
{{ v.name }}
-
{{ v.raw_header or v.name }}
+
{{ v.raw_header or v.name }}
+
Encounter: {{ v.name }}
{% if v.epoch_id %} {% set ep = (epochs | selectattr('id','equalto', v.epoch_id) | list) %} {% if ep and ep[0] %}
Epoch: {{ ep[0].name }}
{% endif %} @@ -398,7 +505,6 @@

Matrix

Generate Normalized Summary (JSON)