diff --git a/.gitignore b/.gitignore index e52d566..e580969 100644 --- a/.gitignore +++ b/.gitignore @@ -97,5 +97,8 @@ docs/~* files/~* output/* SOA Workbench Wishlist.docx +NCT01750580_limited.json +CLAUDE.md +edit-column-collapse.html # End of file diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index 667a1ff..56bf8a1 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -56,6 +56,7 @@ _migrate_timing_add_member_of_timeline, _migrate_instances_add_member_of_timeline, _migrate_matrix_cells_add_instance_id, + _migrate_activity_concept_add_href, ) from .routers import activities as activities_router from .routers import arms as arms_router @@ -150,6 +151,7 @@ def _configure_logging(): # Database migration steps +_migrate_activity_concept_add_href() _migrate_matrix_cells_add_instance_id() _migrate_instances_add_member_of_timeline() _migrate_timing_add_member_of_timeline() @@ -1961,6 +1963,118 @@ def _matrix_arrays(soa_id: int): return instance_headers, rows +def _fetch_enriched_instances(soa_id: int): + """Return enriched instance data with all header information for XLSX export.""" + conn = _connect() + cur = conn.cursor() + cur.execute( + """ + SELECT i.id,i.name,i.instance_uid,i.label,i.member_of_timeline, + v.name AS encounter_name,v.label AS encounter_label, + e.name AS epoch_name,e.epoch_label as epoch_label, + tm.window_label,tm.label AS timing_label,tm.name AS timing_name,tm.value AS study_day + FROM instances i + LEFT JOIN visit v ON v.encounter_uid = i.encounter_uid AND v.soa_id = i.soa_id + LEFT JOIN epoch e ON e.epoch_uid = i.epoch_uid AND e.soa_id = i.soa_id + LEFT JOIN timing tm ON tm.id = v.scheduledAtId AND tm.soa_id = v.soa_id + WHERE i.soa_id=? + ORDER BY COALESCE(i.member_of_timeline, 'zzz'), LENGTH(i.instance_uid), i.instance_uid + """, + (soa_id,), + ) + instances = [ + { + "id": r[0], + "name": r[1], + "instance_uid": r[2], + "label": r[3], + "member_of_timeline": r[4], + "encounter_name": r[5], + "encounter_label": r[6], + "epoch_name": r[7], + "epoch_label": r[8], + "window_label": r[9], + "timing_label": r[10], + "timing_name": r[11], + "study_day": r[12], + } + for r in cur.fetchall() + ] + conn.close() + return instances + + +def _add_header_rows_to_worksheet(worksheet, enriched_instances): + """Add header rows to a worksheet with instance metadata.""" + # Insert 6 rows at the top for header rows + worksheet.insert_rows(1, 6) + + # Build header rows + # Row 1: Epoch (with merged cells for consecutive same values) + worksheet.cell(1, 1, "") + worksheet.cell(1, 2, "Epoch:") + col_idx = 3 + epoch_groups = [] # Track (value, start_col, end_col) for merging + prev_epoch = None + start_col = 3 + for i, inst in enumerate(enriched_instances): + epoch_val = inst.get("epoch_label") or inst.get("epoch_name") or "" + if prev_epoch is None: + prev_epoch = epoch_val + start_col = col_idx + elif prev_epoch != epoch_val: + epoch_groups.append((prev_epoch, start_col, col_idx - 1)) + prev_epoch = epoch_val + start_col = col_idx + col_idx += 1 + # Add last group + if prev_epoch is not None: + epoch_groups.append((prev_epoch, start_col, col_idx - 1)) + + # Write and merge epoch cells + for epoch_val, start, end in epoch_groups: + worksheet.cell(1, start, epoch_val) + if start != end: + worksheet.merge_cells( + start_row=1, start_column=start, end_row=1, end_column=end + ) + + # Row 2: Encounter + worksheet.cell(2, 1, "") + worksheet.cell(2, 2, "Encounter:") + for i, inst in enumerate(enriched_instances): + encounter_val = inst.get("encounter_label") or inst.get("encounter_name") or "" + worksheet.cell(2, i + 3, encounter_val) + + # Row 3: Instance (ScheduledActivityInstance) + worksheet.cell(3, 1, "") + worksheet.cell(3, 2, "Instance:") + for i, inst in enumerate(enriched_instances): + instance_val = inst.get("label") or inst.get("name") or "" + worksheet.cell(3, i + 3, instance_val) + + # Row 4: Study Day + worksheet.cell(4, 1, "") + worksheet.cell(4, 2, "Study Day:") + for i, inst in enumerate(enriched_instances): + study_day_val = inst.get("study_day") or "" + worksheet.cell(4, i + 3, study_day_val) + + # Row 5: Timing + worksheet.cell(5, 1, "") + worksheet.cell(5, 2, "Timing:") + for i, inst in enumerate(enriched_instances): + timing_val = inst.get("timing_label") or inst.get("timing_name") or "" + worksheet.cell(5, i + 3, timing_val) + + # Row 6: Visit Window + worksheet.cell(6, 1, "") + worksheet.cell(6, 2, "Visit Window:") + for i, inst in enumerate(enriched_instances): + window_val = inst.get("window_label") or "" + worksheet.cell(6, i + 3, window_val) + + # API endpoint for creating new Study/SOA @app.post("/soa") def create_soa(payload: SOACreate): @@ -2722,13 +2836,108 @@ def export_xlsx(soa_id: int, left: Optional[int] = None, right: Optional[int] = except Exception as e: # Provide an error sheet to highlight issue rather than failing entire export concept_diff_df = pd.DataFrame([[str(e)]], columns=["ConceptDiffError"]) + # Fetch enriched instances for header rows + enriched_instances = _fetch_enriched_instances(soa_id) + + # Fetch timelines + conn_tl = _connect() + cur_tl = conn_tl.cursor() + cur_tl.execute( + """ + SELECT schedule_timeline_uid,name,main_timeline + FROM schedule_timelines + WHERE soa_id=? + ORDER BY main_timeline DESC, name + """, + (soa_id,), + ) + timelines = [ + { + "schedule_timeline_uid": r[0], + "name": r[1], + "main_timeline": bool(r[2]), + } + for r in cur_tl.fetchall() + ] + conn_tl.close() + + # Group enriched instances by timeline + instances_by_timeline = {} + for inst in enriched_instances: + timeline_key = inst.get("member_of_timeline") or "unassigned" + if timeline_key not in instances_by_timeline: + instances_by_timeline[timeline_key] = [] + instances_by_timeline[timeline_key].append(inst) + with pd.ExcelWriter(bio, engine="openpyxl") as writer: study_df.to_excel(writer, index=False, sheet_name="Study") - df.to_excel(writer, index=False, sheet_name="SoA") mapping_df.to_excel(writer, index=False, sheet_name="ConceptMappings") audit_df.to_excel(writer, index=False, sheet_name="RollbackAudit") if concept_diff_df is not None: concept_diff_df.to_excel(writer, index=False, sheet_name="ConceptDiff") + + # Create a worksheet for each timeline + if timelines: + for timeline in timelines: + timeline_uid = timeline["schedule_timeline_uid"] + timeline_name = timeline["name"] + timeline_instances = instances_by_timeline.get(timeline_uid, []) + + if not timeline_instances: + continue + + # Build matrix data for this timeline + cell_lookup = { + (c["instance_id"], c["activity_id"]): c.get("status", "") + for c in cells + if c.get("instance_id") is not None + and c.get("activity_id") is not None + } + + # Build instance headers for this timeline + instance_headers_tl = [inst["name"] for inst in timeline_instances] + + # Build rows for this timeline + rows_tl = [] + for a in activities: + row = [a["name"]] + for inst in timeline_instances: + row.append(cell_lookup.get((inst["id"], a["id"]), "")) + rows_tl.append(row) + + # Create DataFrame for this timeline + df_tl = pd.DataFrame( + rows_tl, columns=["Activity"] + instance_headers_tl + ) + + # Add concepts columns + if len(concepts_strings) == len(df_tl): + df_tl.insert(1, "Concepts", concepts_strings) + df_tl["Concept UIDs"] = concept_titles_strings + + # Sanitize sheet name (max 31 chars, no special chars) + sheet_name = f"SoA - {timeline_name}"[:31] + sheet_name = ( + sheet_name.replace("/", "-") + .replace("\\", "-") + .replace("*", "-") + .replace("?", "-") + .replace(":", "-") + .replace("[", "-") + .replace("]", "-") + ) + + # Write to Excel + df_tl.to_excel(writer, index=False, sheet_name=sheet_name) + + # Add header rows + worksheet_tl = writer.sheets[sheet_name] + _add_header_rows_to_worksheet(worksheet_tl, timeline_instances) + else: + # No timelines, create single SoA sheet as before + df.to_excel(writer, index=False, sheet_name="SoA") + worksheet = writer.sheets["SoA"] + _add_header_rows_to_worksheet(worksheet, enriched_instances) bio.seek(0) # Dynamic filename pattern: studyid_version.xlsx # Determine study_id and version context @@ -3116,9 +3325,6 @@ def delete_activity(soa_id: int, activity_id: int): return {"deleted_activity_id": activity_id} -# API endpoint for deleting an Epoch <- moved to routers/epochs.py - - @app.get("/", response_class=HTMLResponse) def ui_index(request: Request): """Render home page for the SoA Workbench.""" @@ -3280,38 +3486,6 @@ def ui_update_meta( ) -# Helper to fetch element audit rows with legacy-safe columns -> Deprecated (Moved to audits.py, audits.html) -""" -def _fetch_element_audits(soa_id: int): - conn_ea = _connect() - cur_ea = conn_ea.cursor() - cur_ea.execute("PRAGMA table_info(element_audit)") - cols = [row[1] for row in cur_ea.fetchall()] - want = [ - "id", - "element_id", - "action", - "before_json", - "after_json", - "performed_at", - ] - available = [c for c in want if c in cols] - element_audits = [] - if available: - select_sql = f"SELECT {', '.join(available)} FROM element_audit WHERE soa_id=? ORDER BY id DESC" - cur_ea.execute(select_sql, (soa_id,)) - for r in cur_ea.fetchall(): - item = {} - for i, c in enumerate(available): - item[c] = r[i] - for k in want: - item.setdefault(k, None) - element_audits.append(item) - conn_ea.close() - return element_audits -""" - - # UI endpoint for rendering SOA edit page @app.get("/ui/soa/{soa_id}/edit", response_class=HTMLResponse) def ui_edit(request: Request, soa_id: int): @@ -3582,51 +3756,16 @@ def ui_edit(request: Request, soa_id: int): cur_inst = conn_inst.cursor() cur_inst.execute( """ - SELECT i.id, - i.name, - i.instance_uid, - i.label, - i.member_of_timeline, - (SELECT t.name - FROM schedule_timelines t - WHERE t.schedule_timeline_uid = i.member_of_timeline - AND t.soa_id = i.soa_id) AS timeline_name, - (SELECT v.name - FROM visit v - WHERE v.encounter_uid = i.encounter_uid - AND v.soa_id = i.soa_id) AS encounter_name, - (SELECT e.name - FROM epoch e - WHERE e.epoch_uid = i.epoch_uid - AND e.soa_id = i.soa_id) AS epoch_name, - (SELECT tm.window_label - FROM visit v - JOIN timing tm - ON tm.id = v.scheduledAtId - AND tm.soa_id = v.soa_id - WHERE v.encounter_uid = i.encounter_uid - AND v.soa_id = i.soa_id - LIMIT 1) AS window_label, - (SELECT tm.label - FROM visit v - JOIN timing tm - ON tm.id = v.scheduledAtId - AND tm.soa_id = v.soa_id - WHERE v.encounter_uid = i.encounter_uid - AND v.soa_id = i.soa_id - LIMIT 1) AS timing_label, - (SELECT tm.value - FROM visit v - JOIN timing tm - ON tm.id = v.scheduledAtId - AND tm.soa_id = v.soa_id - WHERE v.encounter_uid = i.encounter_uid - AND v.soa_id = i.soa_id - LIMIT 1) AS study_day + SELECT i.id,i.name,i.instance_uid,i.label,i.member_of_timeline,st.name AS timeline_name,st.label AS timeline_label, + v.name AS encounter_name,v.label AS encounter_label,e.name AS epoch_name,e.epoch_label as epoch_label,tm.window_label,tm.label AS timing_label,tm.name AS timing_name,tm.value AS study_day FROM instances i - WHERE soa_id=? - ORDER BY member_of_timeline, length(instance_uid), instance_uid - """, + LEFT JOIN schedule_timelines st ON st.schedule_timeline_uid = i.member_of_timeline AND st.soa_id = i.soa_id + LEFT JOIN visit v ON v.encounter_uid = i.encounter_uid AND v.soa_id = i.soa_id + LEFT JOIN epoch e ON e.epoch_uid = i.epoch_uid AND e.soa_id = i.soa_id + LEFT JOIN timing tm ON tm.id = v.scheduledAtId AND tm.soa_id = v.soa_id + WHERE i.soa_id=? + ORDER BY COALESCE(i.member_of_timeline, 'zzz'), LENGTH(i.instance_uid), i.instance_uid + """, (soa_id,), ) instances = [ @@ -3637,11 +3776,15 @@ def ui_edit(request: Request, soa_id: int): "label": r[3], "member_of_timeline": r[4], "timeline_name": r[5], - "encounter_name": r[6], - "epoch_name": r[7], - "window_label": r[8], - "timing_label": r[9], - "study_day": iso_duration_to_days(r[10]), + "timeline_label": r[6], + "encounter_name": r[7], + "encounter_label": r[8], + "epoch_name": r[9], + "epoch_label": r[10], + "window_label": r[11], + "timing_label": r[12], + "timing_name": r[13], + "study_day": iso_duration_to_days(r[14]), } for r in cur_inst.fetchall() ] diff --git a/src/soa_builder/web/initialize_database.py b/src/soa_builder/web/initialize_database.py index aa178f0..2e7e08f 100644 --- a/src/soa_builder/web/initialize_database.py +++ b/src/soa_builder/web/initialize_database.py @@ -31,7 +31,8 @@ def _init_db(): concept_title TEXT, concept_uid TEXT, -- immutable BiomedicalConcept_N identifier unique within an SOA activity_uid TEXT, -- joins to the activity table using this uid unique within an SOA - soa_id INT + soa_id INT, + href TEXT -- stores the API address where the BC exists; codeSystem & codeSystemVersion )""" ) diff --git a/src/soa_builder/web/migrate_database.py b/src/soa_builder/web/migrate_database.py index 6bf4195..ae7eaa9 100644 --- a/src/soa_builder/web/migrate_database.py +++ b/src/soa_builder/web/migrate_database.py @@ -979,3 +979,19 @@ def _migrate_matrix_cells_add_instance_id(): conn.close() except Exception as e: logger.warning("matrix_cells instance_id migration failed: %s", e) + + +def _migrate_activity_concept_add_href(): + """Add href column to store the API URI from which codeSystem and codeSystemVersion USDM properties can be derived""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute("PRAGMA table_info(activity_concept)") + cols = {r[1] for r in cur.fetchall()} + if "href" not in cols: + cur.execute("ALTER TABLE activity_concept ADD COLUMN href TEXT") + conn.commit() + logger.info("Added href column to the activity_concept table") + conn.close() + except Exception as e: + logger.warning("activity_concept href migration failed: %s", e) diff --git a/src/soa_builder/web/routers/instances.py b/src/soa_builder/web/routers/instances.py index fcb63af..712575d 100644 --- a/src/soa_builder/web/routers/instances.py +++ b/src/soa_builder/web/routers/instances.py @@ -1,8 +1,8 @@ import logging import os -from typing import Optional +from typing import Optional, List -from fastapi import APIRouter, HTTPException, Request, Form +from fastapi import APIRouter, HTTPException, Request, Form, Body from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates @@ -420,3 +420,47 @@ def delete_instance(soa_id: int, instance_id: int): def ui_del_instance(request: Request, soa_id: int, instance_id: int): delete_instance(soa_id, instance_id) return RedirectResponse(url=f"/ui/soa/{int(soa_id)}/instances", status_code=303) + + +# API endpoint to reorder instances +@router.post("/soa/{soa_id}/instances/reorder", response_class=JSONResponse) +def reorder_instances_api( + soa_id: int, + order: List[int] = Body(..., embed=True), # JSON body: {"order":[...]} +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + if not order: + raise HTTPException(400, "Order list required") + + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,name FROM instances WHERE soa_id=? ORDER BY order_index", (soa_id,) + ) + rows = cur.fetchall() + old_order = [r[0] for r in rows] # IDs for API response + id_to_name = {r[0]: r[1] for r in rows} + old_order_names = [r[1] for r in rows] # Names for audit + + cur.execute("SELECT id,name FROM instances WHERE soa_id=?", (soa_id,)) + existing = {r[0] for r in cur.fetchall()} + if set(order) - existing: + conn.close() + raise HTTPException(400, "Order contains invalid instance id") + + for idx, instance_id in enumerate(order, start=1): + cur.execute("UPDATE instances SET order_index=? WHERE id=?", (idx, instance_id)) + conn.commit() + conn.close() + + new_order_names = [id_to_name.get(iid, str(iid)) for iid in order] + + _record_instance_audit( + soa_id, + "reorder", + instance_id=None, + before={"old_order": old_order_names}, + after={"new_order": new_order_names}, + ) + return JSONResponse({"ok": True, "old_order": old_order, "new_order": order}) diff --git a/src/soa_builder/web/templates/arms.html b/src/soa_builder/web/templates/arms.html index a963c64..a15dddf 100644 --- a/src/soa_builder/web/templates/arms.html +++ b/src/soa_builder/web/templates/arms.html @@ -2,6 +2,12 @@ {% block content %}