diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..b3d417b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "verification", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/src/openutm_verification/server/router.py b/src/openutm_verification/server/router.py index d966c40..22f40d5 100644 --- a/src/openutm_verification/server/router.py +++ b/src/openutm_verification/server/router.py @@ -42,6 +42,19 @@ async def list_scenarios(): return [f.stem for f in path.glob("*.yaml")] +@scenario_router.get("/api/suites") +async def list_suites(runner: Any = Depends(get_runner)): + """Return suite-to-scenario mapping from the loaded configuration.""" + config = runner.config + result: dict[str, list[str]] = {} + for suite_name, suite_config in config.suites.items(): + if suite_config.scenarios: + result[suite_name] = [s.name for s in suite_config.scenarios] + else: + result[suite_name] = [] + return result + + @scenario_router.get("/api/scenarios/{scenario}") async def get_scenario(scenario: str): """Get the content of a specific scenario.""" diff --git a/web-editor/src/components/ScenarioEditor/CustomNode.tsx b/web-editor/src/components/ScenarioEditor/CustomNode.tsx index 53c85e7..f3e8d3a 100644 --- a/web-editor/src/components/ScenarioEditor/CustomNode.tsx +++ b/web-editor/src/components/ScenarioEditor/CustomNode.tsx @@ -80,7 +80,7 @@ export const CustomNode = ({ data, selected }: NodeProps>) => { )} {data.status && ( -
+
{data.status === 'success' && ( )} diff --git a/web-editor/src/components/ScenarioEditor/ScenarioList.tsx b/web-editor/src/components/ScenarioEditor/ScenarioList.tsx index e9d6c62..3a325c7 100644 --- a/web-editor/src/components/ScenarioEditor/ScenarioList.tsx +++ b/web-editor/src/components/ScenarioEditor/ScenarioList.tsx @@ -1,5 +1,5 @@ -import { useState, useEffect } from 'react'; -import { FileText, MessageCircleQuestionMark } from 'lucide-react'; +import { useState, useEffect, useMemo } from 'react'; +import { FileText, MessageCircleQuestionMark, ChevronDown, ChevronRight, FolderOpen } from 'lucide-react'; import styles from '../../styles/Toolbox.module.css'; import type { Operation, ScenarioDefinition, NodeData, ScenarioConfig } from '../../types/scenario'; import type { Node, Edge } from '@xyflow/react'; @@ -13,17 +13,82 @@ interface ScenarioListProps { refreshKey?: number; } +type SuiteMap = Record; + export const ScenarioList = ({ onLoadScenario, operations, currentScenarioName, onSelectScenario, refreshKey = 0 }: ScenarioListProps) => { const [scenarios, setScenarios] = useState([]); + const [suites, setSuites] = useState({}); const [loading, setLoading] = useState(false); + const [collapsedSuites, setCollapsedSuites] = useState>(new Set()); useEffect(() => { - fetch('/api/scenarios') - .then(res => res.json()) - .then((data: string[]) => setScenarios(data.sort())) - .catch(err => console.error('Failed to load scenarios:', err)); + const fetchScenarios = async (): Promise => { + const res = await fetch('/api/scenarios'); + if (!res.ok) return []; + const data: unknown = await res.json(); + return Array.isArray(data) && data.every(item => typeof item === 'string') ? data : []; + }; + + const fetchSuites = async (): Promise => { + const res = await fetch('/api/suites'); + if (!res.ok) return {}; + const data: unknown = await res.json(); + if (typeof data !== 'object' || data === null || Array.isArray(data)) return {}; + const result: SuiteMap = {}; + for (const [key, value] of Object.entries(data as Record)) { + if (Array.isArray(value) && value.every(item => typeof item === 'string')) { + result[key] = value; + } + } + return result; + }; + + Promise.all([ + fetchScenarios(), + fetchSuites().catch(() => ({} as SuiteMap)), + ]).then(([scenarioList, suiteMap]) => { + setScenarios(scenarioList.sort()); + setSuites(suiteMap); + }).catch(err => console.error('Failed to load scenarios:', err)); }, [refreshKey]); + const hasSuites = Object.keys(suites).length > 0; + + const groupedScenarios = useMemo(() => { + const suiteNames = Object.keys(suites).sort((a, b) => a.localeCompare(b)); + const scenarioSet = new Set(scenarios); + const assigned = new Set(suiteNames.flatMap(s => suites[s])); + const ungrouped = scenarios.filter(s => !assigned.has(s)); + + const groups: { suite: string; label: string; items: string[] }[] = []; + for (const suite of suiteNames) { + const items = suites[suite] + .filter(name => scenarioSet.has(name)) + .slice() + .sort((a, b) => a.localeCompare(b)); + if (items.length > 0) { + groups.push({ + suite, + label: suite.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase()), + items, + }); + } + } + if (ungrouped.length > 0) { + groups.push({ suite: '__ungrouped__', label: 'Other Scenarios', items: ungrouped.sort() }); + } + return groups; + }, [scenarios, suites]); + + const toggleSuite = (suite: string) => { + setCollapsedSuites(prev => { + const next = new Set(prev); + if (next.has(suite)) next.delete(suite); + else next.add(suite); + return next; + }); + }; + const handleLoad = async (filename: string) => { if (loading) return; setLoading(true); @@ -44,34 +109,65 @@ export const ScenarioList = ({ onLoadScenario, operations, currentScenarioName, } }; + const renderScenarioItem = (name: string) => ( +
handleLoad(name)} + role="button" + tabIndex={0} + title={name} + style={{ + cursor: 'pointer', + opacity: loading ? 0.5 : 1, + borderColor: name === currentScenarioName ? 'var(--accent-primary)' : 'var(--border-color)', + backgroundColor: name === currentScenarioName ? 'var(--bg-secondary)' : 'var(--bg-primary)' + }} + > + + {name.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase())} +
+ ); + return (
-
-
- Pre-built scenarios you can load to get started + {hasSuites ? 'Pre-built scenarios grouped by test suite' : 'Pre-built scenarios'}
- {scenarios.map(name => ( -
handleLoad(name)} - role="button" - tabIndex={0} - title={name} - style={{ - cursor: 'pointer', - opacity: loading ? 0.5 : 1, - borderColor: name === currentScenarioName ? 'var(--accent-primary)' : 'var(--border-color)', - backgroundColor: name === currentScenarioName ? 'var(--bg-secondary)' : 'var(--bg-primary)' - }} - > - - {name.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase())} -
- ))} + + {hasSuites ? ( + groupedScenarios.map(({ suite, label, items }) => { + const isCollapsed = collapsedSuites.has(suite); + return ( +
+ + {!isCollapsed && ( +
+ {items.map(renderScenarioItem)} +
+ )} +
+ ); + }) + ) : ( + scenarios.map(renderScenarioItem) + )} + {scenarios.length === 0 && (
No scenarios found diff --git a/web-editor/src/components/ScenarioEditor/__tests__/CustomNode.test.tsx b/web-editor/src/components/ScenarioEditor/__tests__/CustomNode.test.tsx index 5c7e86b..2b1907b 100644 --- a/web-editor/src/components/ScenarioEditor/__tests__/CustomNode.test.tsx +++ b/web-editor/src/components/ScenarioEditor/__tests__/CustomNode.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, fireEvent } from '@testing-library/react'; +import { render, screen } from '@testing-library/react'; import { describe, it, expect, vi } from 'vitest'; import { CustomNode } from '../CustomNode'; import { ReactFlowProvider } from '@xyflow/react'; @@ -43,27 +43,16 @@ describe('CustomNode', () => { it('renders success status icon', () => { render(, { wrapper }); - // Check for CheckCircle icon (or button containing it) - const button = screen.getByTitle('Success'); - expect(button).toBeInTheDocument(); + expect(screen.getByTestId('status-success')).toBeInTheDocument(); }); - it('calls onShowResult when status icon is clicked', () => { - render(, { wrapper }); - const button = screen.getByTitle('Success'); - fireEvent.click(button); - expect(mockData.onShowResult).toHaveBeenCalledWith(mockData.result); - }); - - it('renders failure status', () => { + it('renders failure status icon', () => { const failureProps = { ...defaultProps, data: { ...mockData, status: 'failure' as const } }; render(, { wrapper }); - const button = screen.getByTitle('Failure'); - expect(button).toBeInTheDocument(); - // We could check for color or specific icon if we could query by icon + expect(screen.getByTestId('status-failure')).toBeInTheDocument(); }); it('applies selected style', () => { diff --git a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx index f2530e1..0778de7 100644 --- a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx +++ b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx @@ -59,23 +59,11 @@ describe('PropertiesPanel', () => { expect(defaultProps.onUpdateParameter).toHaveBeenCalledWith('1', 'arg1', 'newValue'); }); - it('resolves group step references correctly', () => { - const groupContainerNode: Node = { - id: 'group_container_1', + it('resolves step references correctly', () => { + const fetchNode: Node = { + id: 'fetch_node', type: 'custom', position: { x: 0, y: 0 }, - data: { - label: '📦 group_1', - isGroupContainer: true, - parameters: [] - } - }; - - const groupStep1: Node = { - id: 'group_container_1_step_0', - type: 'custom', - parentId: 'group_container_1', - position: { x: 0, y: 0 }, data: { label: 'Fetch Data', stepId: 'fetch', @@ -83,10 +71,9 @@ describe('PropertiesPanel', () => { } }; - const groupStep2: Node = { - id: 'group_container_1_step_1', + const submitNode: Node = { + id: 'submit_node', type: 'custom', - parentId: 'group_container_1', position: { x: 0, y: 100 }, data: { label: 'Submit Data', @@ -95,7 +82,7 @@ describe('PropertiesPanel', () => { { name: 'data', type: 'object', - default: '${{ group.fetch.result }}' + default: { $ref: 'steps.fetch.result' } } ] } @@ -103,8 +90,9 @@ describe('PropertiesPanel', () => { const props = { ...defaultProps, - selectedNode: groupStep2, - allNodes: [groupContainerNode, groupStep1, groupStep2] + selectedNode: submitNode, + connectedNodes: [fetchNode], + allNodes: [fetchNode, submitNode] }; render(); diff --git a/web-editor/src/components/ScenarioEditor/__tests__/ScenarioList.test.tsx b/web-editor/src/components/ScenarioEditor/__tests__/ScenarioList.test.tsx new file mode 100644 index 0000000..d831c10 --- /dev/null +++ b/web-editor/src/components/ScenarioEditor/__tests__/ScenarioList.test.tsx @@ -0,0 +1,170 @@ +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { ScenarioList } from '../ScenarioList'; + +// Mock convertYamlToGraph so we don't need real operations +vi.mock('../../../utils/scenarioConversion', () => ({ + convertYamlToGraph: vi.fn(() => ({ nodes: [], edges: [], config: undefined })), +})); + +const mockOnLoadScenario = vi.fn(); +const mockOnSelectScenario = vi.fn(); + +const defaultProps = { + onLoadScenario: mockOnLoadScenario, + operations: [], + currentScenarioName: null, + onSelectScenario: mockOnSelectScenario, +}; + +function mockFetchResponses(scenarios: string[], suites: Record | null) { + vi.spyOn(globalThis, 'fetch').mockImplementation((url) => { + const urlStr = typeof url === 'string' ? url : url.toString(); + if (urlStr === '/api/scenarios') { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(scenarios), + } as Response); + } + if (urlStr === '/api/suites') { + if (suites === null) { + return Promise.reject(new Error('Network error')); + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve(suites), + } as Response); + } + // For individual scenario loads + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ name: 'test', steps: [] }), + } as Response); + }); +} + +describe('ScenarioList', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('renders grouped suites from a suite map', async () => { + mockFetchResponses( + ['scenario_a', 'scenario_b', 'scenario_c'], + { + alpha_suite: ['scenario_a', 'scenario_b'], + beta_suite: ['scenario_c'], + }, + ); + + render(); + + await waitFor(() => { + expect(screen.getByText('Alpha Suite')).toBeInTheDocument(); + expect(screen.getByText('Beta Suite')).toBeInTheDocument(); + }); + + // Scenarios should be visible under their groups + expect(screen.getByText('Scenario A')).toBeInTheDocument(); + expect(screen.getByText('Scenario B')).toBeInTheDocument(); + expect(screen.getByText('Scenario C')).toBeInTheDocument(); + }); + + it('renders ungrouped scenarios under "Other Scenarios"', async () => { + mockFetchResponses( + ['grouped_one', 'orphan_scenario'], + { + my_suite: ['grouped_one'], + }, + ); + + render(); + + await waitFor(() => { + expect(screen.getByText('My Suite')).toBeInTheDocument(); + expect(screen.getByText('Other Scenarios')).toBeInTheDocument(); + }); + + expect(screen.getByText('Grouped One')).toBeInTheDocument(); + expect(screen.getByText('Orphan Scenario')).toBeInTheDocument(); + }); + + it('falls back to flat list when /api/suites fails', async () => { + mockFetchResponses(['scenario_x', 'scenario_y'], null); + + render(); + + await waitFor(() => { + expect(screen.getByText('Scenario X')).toBeInTheDocument(); + expect(screen.getByText('Scenario Y')).toBeInTheDocument(); + }); + + // No suite group headers should appear + expect(screen.queryByRole('button', { name: /suite/i })).not.toBeInTheDocument(); + }); + + it('toggles collapse/expand on suite header click', async () => { + mockFetchResponses( + ['scenario_a', 'scenario_b'], + { test_suite: ['scenario_a', 'scenario_b'] }, + ); + + render(); + + await waitFor(() => { + expect(screen.getByText('Test Suite')).toBeInTheDocument(); + }); + + // Initially expanded + expect(screen.getByText('Scenario A')).toBeVisible(); + const header = screen.getByText('Test Suite').closest('button')!; + expect(header).toHaveAttribute('aria-expanded', 'true'); + + // Click to collapse + fireEvent.click(header); + expect(screen.queryByText('Scenario A')).not.toBeInTheDocument(); + expect(header).toHaveAttribute('aria-expanded', 'false'); + + // Click to expand again + fireEvent.click(header); + expect(screen.getByText('Scenario A')).toBeVisible(); + expect(header).toHaveAttribute('aria-expanded', 'true'); + }); + + it('sorts suite names deterministically', async () => { + mockFetchResponses( + ['s1', 's2', 's3'], + { + zulu_suite: ['s1'], + alpha_suite: ['s2'], + mike_suite: ['s3'], + }, + ); + + render(); + + await waitFor(() => { + expect(screen.getByText('Alpha Suite')).toBeInTheDocument(); + }); + + const headers = screen.getAllByRole('button').filter( + (btn) => btn.classList.contains('groupHeader') || btn.getAttribute('aria-expanded') !== null, + ); + const labels = headers.map((h) => h.textContent?.replace(/\d+$/, '').trim()); + expect(labels).toEqual(['Alpha Suite', 'Mike Suite', 'Zulu Suite']); + }); + + it('shows "No scenarios found" when there are none', async () => { + mockFetchResponses([], {}); + + render(); + + await waitFor(() => { + expect(screen.getByText('No scenarios found')).toBeInTheDocument(); + }); + }); +}); diff --git a/web-editor/src/components/ScenarioEditor/__tests__/Toolbox.test.tsx b/web-editor/src/components/ScenarioEditor/__tests__/Toolbox.test.tsx index b2fdae2..0a4fcb8 100644 --- a/web-editor/src/components/ScenarioEditor/__tests__/Toolbox.test.tsx +++ b/web-editor/src/components/ScenarioEditor/__tests__/Toolbox.test.tsx @@ -28,14 +28,20 @@ const mockOperations: Operation[] = [ ]; describe('Toolbox', () => { + const switchToToolboxTab = () => { + fireEvent.click(screen.getByText('Toolbox')); + }; + it('renders correctly', () => { render(); + switchToToolboxTab(); expect(screen.getByText('ClassA')).toBeInTheDocument(); expect(screen.getByText('ClassB')).toBeInTheDocument(); }); it('displays operations under groups', () => { render(); + switchToToolboxTab(); expect(screen.getByText('Operation 1')).toBeInTheDocument(); expect(screen.getByText('Operation 2')).toBeInTheDocument(); expect(screen.getByText('Operation 3')).toBeInTheDocument(); @@ -43,6 +49,7 @@ describe('Toolbox', () => { it('collapses and expands groups', () => { render(); + switchToToolboxTab(); const groupHeader = screen.getByText('ClassA'); // Initially expanded @@ -59,6 +66,7 @@ describe('Toolbox', () => { it('sets data transfer on drag start', () => { render(); + switchToToolboxTab(); const operationItem = screen.getByText('Operation 1'); const dataTransfer = { diff --git a/web-editor/src/hooks/__tests__/useScenarioRunner.test.ts b/web-editor/src/hooks/__tests__/useScenarioRunner.test.ts index 98795a7..eacf834 100644 --- a/web-editor/src/hooks/__tests__/useScenarioRunner.test.ts +++ b/web-editor/src/hooks/__tests__/useScenarioRunner.test.ts @@ -53,6 +53,11 @@ describe('useScenarioRunner', () => { .mockResolvedValueOnce({ ok: true, json: async () => ({ run_id: '1' }) + }) + // generate-report call + .mockResolvedValueOnce({ + ok: true, + text: async () => 'ok' }); const mockEventSource = { @@ -62,14 +67,20 @@ describe('useScenarioRunner', () => { close: vi.fn(), }; - (globalThis.EventSource as unknown as Mock).mockImplementation(() => { + (globalThis.EventSource as unknown as Mock).mockImplementation(function (this: typeof mockEventSource) { + Object.assign(this, mockEventSource); setTimeout(() => { - const doneHandler = mockEventSource.addEventListener.mock.calls.find(call => call[0] === 'done')?.[1]; + // Fire onmessage with step result data first + this.onmessage?.({ + data: JSON.stringify({ id: '1', status: 'success', result: { success: true } }) + } as MessageEvent); + // Then fire the done event + const doneHandler = mockEventSource.addEventListener.mock.calls.find((call: unknown[]) => call[0] === 'done')?.[1]; if (doneHandler) { doneHandler({ data: JSON.stringify({ status: 'completed' }) } as MessageEvent); } }, 0); - return mockEventSource; + return this; }); const { result } = renderHook(() => useScenarioRunner()); @@ -89,7 +100,8 @@ describe('useScenarioRunner', () => { id: '1', status: 'success', result: { success: true }, - error: undefined + error: undefined, + logs: [] }], status: 'completed', duration: 0 @@ -102,6 +114,24 @@ describe('useScenarioRunner', () => { }); it('runs scenario with config', async () => { + const mockEventSource = { + onmessage: null as ((event: MessageEvent) => void) | null, + onerror: null as ((event: Event) => void) | null, + addEventListener: vi.fn(), + close: vi.fn(), + }; + + (globalThis.EventSource as unknown as Mock).mockImplementation(function (this: typeof mockEventSource) { + Object.assign(this, mockEventSource); + setTimeout(() => { + const doneHandler = mockEventSource.addEventListener.mock.calls.find((call: unknown[]) => call[0] === 'done')?.[1]; + if (doneHandler) { + doneHandler({ data: JSON.stringify({ status: 'completed' }) } as MessageEvent); + } + }, 0); + return this; + }); + (globalThis.fetch as Mock) .mockResolvedValueOnce({ ok: true, @@ -110,6 +140,10 @@ describe('useScenarioRunner', () => { .mockResolvedValueOnce({ ok: true, json: async () => ({ run_id: '1' }) + }) + .mockResolvedValueOnce({ + ok: true, + text: async () => 'ok' }); const { result } = renderHook(() => useScenarioRunner());