From c24e6506b79fb10d2eb542d8578058509d7f731f Mon Sep 17 00:00:00 2001 From: Travis Vu Date: Wed, 21 Jan 2026 17:28:11 -0800 Subject: [PATCH 1/4] feat(data-mapper-v2): Make XSLT the source of truth for map definitions This change fundamentally shifts how Data Mapper v2 stores and loads maps: **Core Changes:** - XSLT is now the authoritative source for map definitions - Added XsltParser to extract map definitions from XSLT text value templates - Added XsltMetadataSerializer for v3 metadata format (UI layout only) - Removed LML embedding in XSLT comments (v2 format deprecated) **XPath to LML Conversion:** - Convert single-quoted string literals to double quotes for LML compatibility - Convert XPath infix math operators (+, -, *, div, mod) to function calls - Remove namespace prefixes (math:, xs:, etc.) from function names - Flatten associative operators (+, *) to multi-argument function calls **Bug Fixes:** - Fix delete button not working for custom values in unbounded inputs - Fix node position changes not triggering dirty state (Save button) **VS Code Extension:** - Update file service to use XSLT content directly - Add commands for loading/saving XSLT content --- Localize/lang/strings.json | 4 +- apps/vs-code-designer/package.json | 2 + .../app/commands/dataMapper/DataMapperExt.ts | 154 +++++ .../commands/dataMapper/DataMapperPanel.ts | 168 ++++- .../src/app/commands/dataMapper/dataMapper.ts | 240 +++++-- .../commands/dataMapper/extensionConfig.ts | 17 + apps/vs-code-designer/src/main.ts | 11 +- apps/vs-code-designer/src/package.json | 4 +- .../src/app/dataMapper/appV2.tsx | 2 + .../services/dataMapperFileService.ts | 28 +- apps/vs-code-react/src/run-service/types.ts | 11 + .../vs-code-react/src/state/DataMapSliceV2.ts | 13 + .../src/webviewCommunication.tsx | 28 +- ...2026-01-21-lml-to-xslt-migration-design.md | 177 +++++ .../components/codeView/CodeViewPanelBody.tsx | 15 +- .../commandBar/EditorCommandBar.tsx | 39 +- .../src/components/test/TestPanel.tsx | 125 ++-- .../src/core/DataMapDataProvider.tsx | 37 + .../dataMapperFileService.ts | 28 +- .../src/core/state/DataMapSlice.ts | 23 +- .../src/mapHandling/XsltMetadataSerializer.ts | 228 +++++++ .../src/mapHandling/XsltParser.ts | 641 ++++++++++++++++++ .../__test__/XsltMetadataSerializer.spec.ts | 384 +++++++++++ .../mapHandling/__test__/XsltParser.spec.ts | 446 ++++++++++++ libs/data-mapper-v2/src/utils/index.ts | 2 + .../src/lib/models/dataMapper.ts | 19 +- .../src/lib/models/extensioncommand.ts | 2 + pnpm-lock.yaml | 64 +- turbo.json | 1 + 29 files changed, 2724 insertions(+), 189 deletions(-) create mode 100644 docs/plans/2026-01-21-lml-to-xslt-migration-design.md create mode 100644 libs/data-mapper-v2/src/mapHandling/XsltMetadataSerializer.ts create mode 100644 libs/data-mapper-v2/src/mapHandling/XsltParser.ts create mode 100644 libs/data-mapper-v2/src/mapHandling/__test__/XsltMetadataSerializer.spec.ts create mode 100644 libs/data-mapper-v2/src/mapHandling/__test__/XsltParser.spec.ts diff --git a/Localize/lang/strings.json b/Localize/lang/strings.json index 746ec9e6fb9..a61d600d571 100644 --- a/Localize/lang/strings.json +++ b/Localize/lang/strings.json @@ -3810,6 +3810,7 @@ "_qJpnIL.comment": "Label for description of custom endsWith Function", "_qKVOwV.comment": "Placeholder text for the MCP server name field", "_qMFpNH.comment": "Loading dynamic data", + "_qNy7WP.comment": "Message to display when XSLT content hasn't been generated yet", "_qSejoi.comment": "Label for description of custom lessOrEquals Function", "_qSt0Sb.comment": "Accessibility prefix for the input label", "_qUWBUX.comment": "A duration of time shown in days", @@ -4097,7 +4098,7 @@ "_wQcEXt.comment": "Required parameters for the custom Replace Function", "_wQsEwc.comment": "Required length parameter to obtain substring", "_wT/gMB.comment": "Description for featured connectors field", - "_wTaSTp.comment": "Tooltip for disabled test button", + "_wTaSTp.comment": "Message when XSLT content is not available", "_wV3Lmd.comment": "Label to delete dictionary item", "_wWVQuK.comment": "Label for description of custom if Function", "_wWuzqz.comment": "Ok button text", @@ -4919,6 +4920,7 @@ "qJpnIL": "Checks if the string ends with a value (case-insensitive, invariant culture)", "qKVOwV": "Enter a name for the MCP server", "qMFpNH": "Loading dynamic data", + "qNy7WP": "XSLT not yet generated. Save the map to generate XSLT output.", "qSejoi": "Returns true if the first argument is less than or equal to the second", "qSt0Sb": "Required", "qUWBUX": "{days, plural, one {# day} other {# days}}", diff --git a/apps/vs-code-designer/package.json b/apps/vs-code-designer/package.json index b24284c592f..812560c0d69 100644 --- a/apps/vs-code-designer/package.json +++ b/apps/vs-code-designer/package.json @@ -42,12 +42,14 @@ "process-tree": "^1.0.3", "ps-tree": "^1.2.0", "recursive-copy": "^2.0.14", + "saxon-js": "^2.6.0", "semver": "^6.3.1", "tslib": "2.4.0", "uuid": "^10.0.0", "vscode-extension-tester": "^8.17.0", "vscode-nls": "^5.2.0", "xml2js": "0.6.2", + "xslt3": "^2.7.0", "yaml": "^2.7.0", "yaml-types": "^0.4.0", "yargs-parser": "21.1.1", diff --git a/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperExt.ts b/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperExt.ts index 96884d5490c..3188c710986 100644 --- a/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperExt.ts +++ b/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperExt.ts @@ -15,6 +15,63 @@ import { parse } from 'yaml'; import { localize } from '../../../localize'; import { assetsFolderName, dataMapNameValidation } from '../../../constants'; +/** + * UI metadata for function positions and canvas state. + */ +interface XsltUiMetadata { + functionNodes: Array<{ + reactFlowGuid: string; + functionKey: string; + position: { x: number; y: number }; + connections: Array<{ name: string; inputOrder: number }>; + connectionShorthand: string; + }>; + canvasRect?: { x: number; y: number; width: number; height: number }; +} + +/** + * Metadata embedded in XSLT files for Data Mapper v2. + * Contains mapDefinition in the metadata (legacy format). + * @deprecated Use XsltMapMetadataV3 which doesn't embed mapDefinition + */ +interface XsltMapMetadataV2 { + version: '2.0'; + sourceSchema: string; + targetSchema: string; + mapDefinition: MapDefinitionEntry; + metadata: XsltUiMetadata; +} + +/** + * Metadata embedded in XSLT files for Data Mapper v3. + * Only embeds layout metadata - mapping logic is derived from XSLT content. + */ +interface XsltMapMetadataV3 { + version: '3.0'; + sourceSchema: string; + targetSchema: string; + metadata: XsltUiMetadata; +} + +/** + * Union type supporting both v2 and v3 metadata formats. + */ +type XsltMapMetadata = XsltMapMetadataV2 | XsltMapMetadataV3; + +/** + * Type guard to check if metadata is v2 format (with mapDefinition). + */ +const isV2Metadata = (metadata: XsltMapMetadata): metadata is XsltMapMetadataV2 => { + return metadata.version === '2.0' && 'mapDefinition' in metadata; +}; + +/** + * Type guard to check if metadata is v3 format (without mapDefinition). + */ +const isV3Metadata = (metadata: XsltMapMetadata): metadata is XsltMapMetadataV3 => { + return metadata.version === '3.0'; +}; + export default class DataMapperExt { public static async openDataMapperPanel( context: IActionContext, @@ -119,4 +176,101 @@ export default class DataMapperExt { } } } + + /** + * Regex to extract metadata JSON from XSLT comment. + */ + private static readonly METADATA_REGEX = //; + + /** + * Checks if an XSLT string has embedded metadata. + */ + public static hasEmbeddedMetadata(xslt: string): boolean { + return DataMapperExt.METADATA_REGEX.test(xslt); + } + + /** + * Extracts metadata from an XSLT string if present. + * Supports both v2 (with mapDefinition) and v3 (without mapDefinition) formats. + */ + public static extractMetadataFromXslt(xslt: string): XsltMapMetadata | null { + const match = xslt.match(DataMapperExt.METADATA_REGEX); + + if (!match || !match[1]) { + return null; + } + + try { + const metadata = JSON.parse(match[1]) as XsltMapMetadata; + + // Validate required fields (common to both v2 and v3) + if (!metadata.version || !metadata.sourceSchema || !metadata.targetSchema) { + console.warn('XSLT metadata missing required fields'); + return null; + } + + // v2 requires mapDefinition, v3 does not + if (metadata.version === '2.0' && !('mapDefinition' in metadata)) { + console.warn('XSLT metadata v2.0 missing mapDefinition'); + return null; + } + + return metadata; + } catch (error) { + console.error('Failed to parse XSLT metadata JSON:', error); + return null; + } + } + + /** + * Return type for loadMapFromXslt + */ + public static loadMapFromXslt( + xsltContent: string, + _extInstance: typeof ext + ): { + mapDefinition: MapDefinitionEntry; + sourceSchemaFileName: string; + targetSchemaFileName: string; + metadata?: XsltUiMetadata; + xsltContent?: string; + isV3Format?: boolean; + } | null { + const metadata = DataMapperExt.extractMetadataFromXslt(xsltContent); + + if (!metadata) { + return null; + } + + // Handle v2 format (with embedded mapDefinition) + if (isV2Metadata(metadata)) { + // Fix custom values in the map definition (same processing as LML) + DataMapperExt.fixMapDefinitionCustomValues(metadata.mapDefinition); + + return { + mapDefinition: metadata.mapDefinition, + sourceSchemaFileName: metadata.sourceSchema, + targetSchemaFileName: metadata.targetSchema, + metadata: metadata.metadata, + isV3Format: false, + }; + } + + // Handle v3 format (without embedded mapDefinition) + // For v3, we pass the raw XSLT content so the webview can parse it + // to derive the mapping connections from the actual XSLT + if (isV3Metadata(metadata)) { + return { + // Empty mapDefinition - webview will derive from XSLT + mapDefinition: {}, + sourceSchemaFileName: metadata.sourceSchema, + targetSchemaFileName: metadata.targetSchema, + metadata: metadata.metadata, + xsltContent: xsltContent, + isV3Format: true, + }; + } + + return null; + } } diff --git a/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperPanel.ts b/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperPanel.ts index 0cdf393ff11..b25c879e9f3 100644 --- a/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperPanel.ts +++ b/apps/vs-code-designer/src/app/commands/dataMapper/DataMapperPanel.ts @@ -10,6 +10,7 @@ import { dataMapDefinitionsPath, dataMapsPath, draftMapDefinitionSuffix, + draftXsltExtension, mapDefinitionExtension, mapXsltExtension, schemasPath, @@ -23,7 +24,7 @@ import type { SchemaType, MapMetadata, IFileSysTreeItem } from '@microsoft/logic import type { IActionContext } from '@microsoft/vscode-azext-utils'; import { callWithTelemetryAndErrorHandlingSync } from '@microsoft/vscode-azext-utils'; import type { MapDefinitionData, MessageToVsix, MessageToWebview } from '@microsoft/vscode-extension-logic-apps'; -import { ExtensionCommand, Platform, ProjectName } from '@microsoft/vscode-extension-logic-apps'; +import { ExtensionCommand, ProjectName } from '@microsoft/vscode-extension-logic-apps'; import { copyFileSync, existsSync as fileExistsSync, @@ -33,13 +34,17 @@ import { readdirSync, readFileSync, mkdirSync, + writeFileSync, } from 'fs'; import * as path from 'path'; +import * as os from 'os'; +import { execSync } from 'child_process'; import type { WebviewPanel } from 'vscode'; import { RelativePattern, window, workspace } from 'vscode'; import * as vscode from 'vscode'; import { copyOverImportedSchemas } from './DataMapperPanelUtils'; import { switchToDataMapperV2 } from '../setDataMapperVersion'; +import SaxonJS from 'saxon-js'; export default class DataMapperPanel { public panel: WebviewPanel; @@ -210,16 +215,145 @@ export default class DataMapperPanel { }); break; } + case ExtensionCommand.testXsltTransform: { + this.handleTestXsltTransform(msg.data.xsltContent, msg.data.inputXml); + break; + } } } public isTestDisabledForOS() { + // Local XSLT transformation is now available on all platforms via SaxonJS + // So we no longer need to disable testing on macOS this.sendMsgToWebview({ command: ExtensionCommand.isTestDisabledForOS, - data: process.platform === Platform.mac, + data: false, }); } + /** + * Handles XSLT 3.0 transformation locally using SaxonJS with xslt3 CLI compilation. + * This allows testing data maps on all platforms without requiring a .NET backend. + * + * The approach: + * 1. Write XSLT to a temp file + * 2. Use xslt3 CLI to compile XSLT to SEF (Stylesheet Export Format) + * 3. Parse SEF and use with SaxonJS.transform() + * 4. Return result and clean up temp files + */ + public async handleTestXsltTransform(xsltContent: string, inputXml: string) { + const tempDir = os.tmpdir(); + const uniqueId = Date.now().toString(); + const xsltPath = path.join(tempDir, `datamap-${uniqueId}.xslt`); + const sefPath = path.join(tempDir, `datamap-${uniqueId}.sef.json`); + + console.log('[DataMapper Test] Starting XSLT transformation test'); + console.log('[DataMapper Test] Extension path:', ext.context.extensionPath); + + try { + // Step 1: Write XSLT to temp file + console.log('[DataMapper Test] Writing XSLT to:', xsltPath); + writeFileSync(xsltPath, xsltContent, 'utf8'); + + // Step 2: Find xslt3 CLI path - use the .js file directly for cross-platform compatibility + const xslt3Path = path.join(ext.context.extensionPath, 'node_modules', 'xslt3', 'xslt3.js'); + console.log('[DataMapper Test] xslt3 path:', xslt3Path); + console.log('[DataMapper Test] xslt3 exists:', fileExistsSync(xslt3Path)); + + if (!fileExistsSync(xslt3Path)) { + throw new Error(`xslt3 CLI not found at: ${xslt3Path}`); + } + + // Step 3: Compile XSLT to SEF using xslt3 CLI via node + const compileCmd = `node "${xslt3Path}" -xsl:"${xsltPath}" -export:"${sefPath}" -nogo`; + console.log('[DataMapper Test] Compile command:', compileCmd); + + try { + const compileResult = execSync(compileCmd, { + encoding: 'utf8', + timeout: 30000, // 30 second timeout + maxBuffer: 10 * 1024 * 1024, // 10MB buffer + }); + console.log('[DataMapper Test] Compile output:', compileResult || '(empty)'); + } catch (execError: unknown) { + const err = execError as { stderr?: string; stdout?: string; message?: string }; + console.error('[DataMapper Test] Compile failed:', err.stderr || err.message); + throw new Error(`XSLT compilation failed: ${err.stderr || err.message}`); + } + + // Step 4: Check if SEF was created + console.log('[DataMapper Test] Checking SEF at:', sefPath); + console.log('[DataMapper Test] SEF exists:', fileExistsSync(sefPath)); + + if (!fileExistsSync(sefPath)) { + throw new Error('Failed to compile XSLT: SEF file was not created'); + } + + // Step 5: Read and parse SEF + const sefContent = readFileSync(sefPath, 'utf8'); + console.log('[DataMapper Test] SEF size:', sefContent.length, 'bytes'); + const sef = JSON.parse(sefContent); + + // Step 6: Execute transformation using SaxonJS + console.log('[DataMapper Test] Starting SaxonJS transform'); + const result = await SaxonJS.transform( + { + stylesheetInternal: sef, + sourceText: inputXml, + destination: 'serialized', + }, + 'async' + ); + console.log('[DataMapper Test] Transform complete, result length:', result.principalResult?.length ?? 0); + + // Send successful result back to webview + console.log('[DataMapper Test] Sending success result to webview'); + this.sendMsgToWebview({ + command: ExtensionCommand.testXsltTransformResult, + data: { + success: true, + outputXml: result.principalResult ?? '', + statusCode: 200, + statusText: 'OK', + }, + }); + } catch (error) { + // Send error result back to webview + console.error('[DataMapper Test] Error:', error); + let errorMessage = 'Unknown error during XSLT transformation'; + if (error instanceof Error) { + errorMessage = error.message; + } else if (typeof error === 'object' && error !== null) { + const execError = error as { stderr?: string; message?: string }; + errorMessage = execError.stderr || execError.message || errorMessage; + } + + console.log('[DataMapper Test] Sending error result to webview:', errorMessage); + this.sendMsgToWebview({ + command: ExtensionCommand.testXsltTransformResult, + data: { + success: false, + error: errorMessage, + statusCode: 500, + statusText: 'Transformation Error', + }, + }); + } finally { + // Clean up temp files + try { + if (fileExistsSync(xsltPath)) { + removeFileSync(xsltPath); + } + if (fileExistsSync(sefPath)) { + removeFileSync(sefPath); + } + console.log('[DataMapper Test] Cleaned up temp files'); + } catch { + // Ignore cleanup errors + } + } + } + public updateWebviewPanelTitle() { this.panel.title = `${this.dataMapName ?? 'Untitled'} ${this.dataMapStateIsDirty ? '●' : ''}`; } @@ -236,7 +370,12 @@ export default class DataMapperPanel { public handleLoadMapDefinitionIfAny() { if (this.mapDefinitionData) { - const mapMetadata = this.readMapMetadataFile(); + // Use embedded metadata if available, otherwise try to read from separate file (legacy) + let mapMetadata = this.mapDefinitionData.metadata; + if (!mapMetadata) { + mapMetadata = this.readMapMetadataFile(); + } + this.sendMsgToWebview({ command: ExtensionCommand.loadDataMap, data: { @@ -491,16 +630,20 @@ export default class DataMapperPanel { }); } - public saveDraftDataMapDefinition(mapDefFileContents: string) { - const mapDefileName = `${this.dataMapName}${draftMapDefinitionSuffix}${mapDefinitionExtension}`; - const dataMapDefFolderPath = path.join(ext.defaultLogicAppPath, dataMapDefinitionsPath); - const filePath = path.join(dataMapDefFolderPath, mapDefileName); + /** + * Saves a draft XSLT file with embedded metadata. + * The draft file is saved to the Maps folder with a .draft.xslt extension. + */ + public saveDraftDataMapDefinition(draftXsltContent: string) { + const draftFileName = `${this.dataMapName}${draftXsltExtension}`; + const dataMapFolderPath = path.join(ext.defaultLogicAppPath, dataMapsPath); + const filePath = path.join(dataMapFolderPath, draftFileName); // Mkdir as extra insurance that directory exists so file can be written // Harmless if directory already exists - fs.mkdir(dataMapDefFolderPath, { recursive: true }) + fs.mkdir(dataMapFolderPath, { recursive: true }) .then(() => { - fs.writeFile(filePath, mapDefFileContents, 'utf8'); + fs.writeFile(filePath, draftXsltContent, 'utf8'); }) .catch(ext.showError); } @@ -545,6 +688,13 @@ export default class DataMapperPanel { } public deleteDraftDataMapDefinition() { + // Delete new format draft XSLT file + const draftXsltPath = path.join(ext.defaultLogicAppPath, dataMapsPath, `${this.dataMapName}${draftXsltExtension}`); + if (fileExistsSync(draftXsltPath)) { + removeFileSync(draftXsltPath); + } + + // Also delete legacy draft LML file if it exists const draftMapDefinitionPath = path.join( ext.defaultLogicAppPath, dataMapDefinitionsPath, diff --git a/apps/vs-code-designer/src/app/commands/dataMapper/dataMapper.ts b/apps/vs-code-designer/src/app/commands/dataMapper/dataMapper.ts index 5822e0b8fa3..0563cb3f9e5 100644 --- a/apps/vs-code-designer/src/app/commands/dataMapper/dataMapper.ts +++ b/apps/vs-code-designer/src/app/commands/dataMapper/dataMapper.ts @@ -6,8 +6,17 @@ import { extensionCommand } from '../../../constants'; import { ext } from '../../../extensionVariables'; import { localize } from '../../../localize'; import DataMapperExt from './DataMapperExt'; -import { dataMapDefinitionsPath, draftMapDefinitionSuffix, schemasPath, supportedDataMapDefinitionFileExts } from './extensionConfig'; -import { isNullOrUndefined, type MapDefinitionEntry } from '@microsoft/logic-apps-shared'; +import { + dataMapDefinitionsPath, + dataMapsPath, + draftMapDefinitionSuffix, + draftXsltExtension, + mapXsltExtension, + schemasPath, + supportedDataMapDefinitionFileExts, + supportedDataMapFileExts, +} from './extensionConfig'; +import { isNullOrUndefined, type MapDefinitionEntry, type MapMetadataV2 } from '@microsoft/logic-apps-shared'; import type { IActionContext } from '@microsoft/vscode-azext-utils'; import { callWithTelemetryAndErrorHandling } from '@microsoft/vscode-azext-utils'; import { existsSync as fileExistsSync, promises as fs } from 'fs'; @@ -33,8 +42,7 @@ export async function createNewDataMapCmd(context: IActionContext): Promise { - let mapDefinitionPath: string | undefined = uri?.fsPath; - let draftFileIsFoundAndShouldBeUsed = false; + let mapFilePath: string | undefined = uri?.fsPath; if (isNullOrUndefined(ext.defaultLogicAppPath)) { const workspaceFolder = await getWorkspaceFolder( context, @@ -48,21 +56,23 @@ export async function loadDataMapFileCmd(context: IActionContext, uri: Uri): Pro ext.defaultLogicAppPath = projectPath; } - // Handle if Uri isn't provided/defined (cmd pallette or btn) - if (!mapDefinitionPath) { + // Handle if Uri isn't provided/defined (cmd palette or btn) + if (!mapFilePath) { + // Show file picker supporting both XSLT (new format) and LML (legacy format) const fileUris = await window.showOpenDialog({ - title: 'Select a data map definition to load', - defaultUri: Uri.file(path.join(ext.defaultLogicAppPath, dataMapDefinitionsPath)), + title: 'Select a data map to load', + defaultUri: Uri.file(path.join(ext.defaultLogicAppPath, dataMapsPath)), canSelectMany: false, canSelectFiles: true, canSelectFolders: false, filters: { - 'Data Map Definition': supportedDataMapDefinitionFileExts.map((ext) => ext.replace('.', '')), + 'Data Map (XSLT)': supportedDataMapFileExts.map((ext) => ext.replace('.', '')), + 'Data Map Definition (Legacy)': supportedDataMapDefinitionFileExts.map((ext) => ext.replace('.', '')), }, }); if (fileUris && fileUris.length > 0) { - mapDefinitionPath = fileUris[0].fsPath; + mapFilePath = fileUris[0].fsPath; } else { context.telemetry.properties.result = 'Canceled'; context.telemetry.properties.wasUsingFilePicker = 'true'; @@ -70,31 +80,130 @@ export async function loadDataMapFileCmd(context: IActionContext, uri: Uri): Pro } } - // Check if there's a draft version of the map (more up-to-date version) definition first, and load that if so - const mapDefinitionFileName = path.basename(mapDefinitionPath); + const fileExt = path.extname(mapFilePath).toLowerCase(); + const isXsltFile = supportedDataMapFileExts.includes(fileExt); + const isLmlFile = supportedDataMapDefinitionFileExts.includes(fileExt); + + // Load based on file type + if (isXsltFile) { + await loadFromXsltFile(context, mapFilePath); + } else if (isLmlFile) { + await loadFromLmlFile(context, mapFilePath); + } else { + ext.showError(localize('UnsupportedFileType', 'Unsupported file type: {0}', fileExt)); + } +} + +/** + * Load a data map from an XSLT file with embedded metadata (new format). + */ +async function loadFromXsltFile(context: IActionContext, xsltPath: string): Promise { + const mapFileName = path.basename(xsltPath); + const isDraftFile = mapFileName.includes('.draft.'); + + // Check for draft version if not already loading a draft + let fileToLoad = xsltPath; + if (!isDraftFile) { + const draftPath = xsltPath.replace(mapXsltExtension, draftXsltExtension); + if (fileExistsSync(draftPath)) { + fileToLoad = draftPath; + context.telemetry.properties.loadingDraftFile = 'true'; + } + } + + const fileContents = await fs.readFile(fileToLoad, 'utf-8'); + + // Try to extract embedded metadata + const loadedData = DataMapperExt.loadMapFromXslt(fileContents, ext); + + if (!loadedData) { + // XSLT doesn't have embedded metadata - try to find and migrate from LML + context.telemetry.properties.xsltHasEmbeddedMetadata = 'false'; + + const dataMapName = path.basename(xsltPath, mapXsltExtension).replace('.draft', ''); + const lmlPath = path.join(ext.defaultLogicAppPath, dataMapDefinitionsPath, `${dataMapName}.lml`); + + if (fileExistsSync(lmlPath)) { + ext.showWarning( + localize( + 'MigratingFromLml', + 'XSLT file does not contain embedded metadata. Loading from legacy LML file and migrating. Please save to update the XSLT.' + ) + ); + await loadFromLmlFile(context, lmlPath); + } else { + ext.showError(localize('NoMetadataFound', 'XSLT file does not contain embedded metadata and no legacy LML file was found.')); + } + return; + } + + context.telemetry.properties.xsltHasEmbeddedMetadata = 'true'; + + // Verify schema files exist + const schemasFolder = path.join(ext.defaultLogicAppPath, schemasPath); + const srcSchemaPath = path.join(schemasFolder, loadedData.sourceSchemaFileName); + const tgtSchemaPath = path.join(schemasFolder, loadedData.targetSchemaFileName); + + if (!fileExistsSync(srcSchemaPath)) { + const resolved = await attemptToResolveMissingSchemaFile(context, loadedData.sourceSchemaFileName, srcSchemaPath); + if (!resolved) { + return; + } + } + + if (!fileExistsSync(tgtSchemaPath)) { + const resolved = await attemptToResolveMissingSchemaFile(context, loadedData.targetSchemaFileName, tgtSchemaPath); + if (!resolved) { + return; + } + } + + const dataMapName = path.basename(xsltPath, mapXsltExtension).replace('.draft', ''); + + // Open the panel with map definition and embedded metadata + // For v3 format, pass xsltContent so webview can parse it to derive connections + DataMapperExt.openDataMapperPanel(context, dataMapName, { + mapDefinition: loadedData.mapDefinition, + sourceSchemaFileName: loadedData.sourceSchemaFileName, + targetSchemaFileName: loadedData.targetSchemaFileName, + metadata: loadedData.metadata as MapMetadataV2 | undefined, + xsltContent: loadedData.xsltContent, + isV3Format: loadedData.isV3Format, + }); +} + +/** + * Load a data map from an LML file (legacy format). + * @deprecated Use loadFromXsltFile for new maps. This is for backward compatibility. + */ +async function loadFromLmlFile(context: IActionContext, lmlPath: string): Promise { + const mapDefinitionFileName = path.basename(lmlPath); const mapDefFileExt = path.extname(mapDefinitionFileName); + let draftFileIsFoundAndShouldBeUsed = false; + + // Check for draft version const draftMapDefinitionPath = path.join( - path.dirname(mapDefinitionPath), + path.dirname(lmlPath), mapDefinitionFileName.replace(mapDefFileExt, `${draftMapDefinitionSuffix}${mapDefFileExt}`) ); if (!mapDefinitionFileName.includes(draftMapDefinitionSuffix)) { - // The file we're loading isn't a draft file itself, so now it makes sense to check for a draft version if (fileExistsSync(draftMapDefinitionPath)) { draftFileIsFoundAndShouldBeUsed = true; } } let mapDefinition: MapDefinitionEntry = {}; + // Try to load the draft file first if (draftFileIsFoundAndShouldBeUsed) { const fileContents = await fs.readFile(draftMapDefinitionPath, 'utf-8'); mapDefinition = DataMapperExt.loadMapDefinition(fileContents, ext); } - //If there is no draft file, or the draft file fails to deserialize, fall back to the base file + // If there is no draft file, or the draft file fails to deserialize, fall back to the base file if (Object.keys(mapDefinition).length === 0) { - const fileContents = await fs.readFile(mapDefinitionPath, 'utf-8'); + const fileContents = await fs.readFile(lmlPath, 'utf-8'); mapDefinition = DataMapperExt.loadMapDefinition(fileContents, ext); } @@ -105,80 +214,32 @@ export async function loadDataMapFileCmd(context: IActionContext, uri: Uri): Pro typeof mapDefinition.$targetSchema !== 'string' ) { if (Object.keys(mapDefinition).length !== 0) { - context.telemetry.properties.eventDescription = 'Attempted to load invalid map, missing schema definitions'; // only show error if schemas are missing but object exists + context.telemetry.properties.eventDescription = 'Attempted to load invalid map, missing schema definitions'; ext.showError(localize('MissingSourceTargetSchema', 'Invalid map definition: $sourceSchema and $targetSchema must be defined.')); } return; } - // Attempt to load schema files if specified + // Verify schema files exist const schemasFolder = path.join(ext.defaultLogicAppPath, schemasPath); const srcSchemaPath = path.join(schemasFolder, mapDefinition.$sourceSchema); const tgtSchemaPath = path.join(schemasFolder, mapDefinition.$targetSchema); - const attemptToResolveMissingSchemaFile = async (schemaName: string, schemaPath: string): Promise => { - return !!(await callWithTelemetryAndErrorHandling( - extensionCommand.dataMapAttemptToResolveMissingSchemaFile, - async (_context: IActionContext) => { - const findSchemaFileButton = 'Find schema file'; - const clickedButton = await window.showErrorMessage( - `Error loading map definition: ${schemaName} was not found in the Schemas folder!`, - findSchemaFileButton - ); - - if (clickedButton && clickedButton === findSchemaFileButton) { - const fileUris = await window.showOpenDialog({ - title: 'Select the missing schema file', - canSelectMany: false, - canSelectFiles: true, - canSelectFolders: false, - filters: { 'XML Schema': ['xsd'], 'JSON Schema': ['json'] }, - }); - - if (fileUris && fileUris.length > 0) { - // Copy the schema file they selected to the Schemas folder (can safely continue map definition loading) - await fs.copyFile(fileUris[0].fsPath, schemaPath); - context.telemetry.properties.result = 'Succeeded'; - - return true; - } - } - - // If user doesn't select a file, or doesn't click the above action, just return (cancel loading the MapDef) - context.telemetry.properties.result = 'Canceled'; - context.telemetry.properties.wasResolvingMissingSchemaFile = 'true'; - - return false; - } - )); - }; - - // If schema file doesn't exist, prompt to find/select it if (!fileExistsSync(srcSchemaPath)) { - const successfullyFoundAndCopiedSchemaFile = await attemptToResolveMissingSchemaFile(mapDefinition.$sourceSchema, srcSchemaPath); - - if (!successfullyFoundAndCopiedSchemaFile) { - context.telemetry.properties.result = 'Canceled'; - context.telemetry.properties.missingSourceSchema = 'true'; - - ext.showError(localize('MissingSourceSchema', 'No source schema file was selected. Aborting load...')); + const resolved = await attemptToResolveMissingSchemaFile(context, mapDefinition.$sourceSchema, srcSchemaPath); + if (!resolved) { return; } } if (!fileExistsSync(tgtSchemaPath)) { - const successfullyFoundAndCopiedSchemaFile = await attemptToResolveMissingSchemaFile(mapDefinition.$targetSchema, tgtSchemaPath); - - if (!successfullyFoundAndCopiedSchemaFile) { - context.telemetry.properties.result = 'Canceled'; - context.telemetry.properties.missingTargetSchema = 'true'; - - ext.showError(localize('MissingTargetSchema', 'No target schema file was selected. Aborting load...')); + const resolved = await attemptToResolveMissingSchemaFile(context, mapDefinition.$targetSchema, tgtSchemaPath); + if (!resolved) { return; } } - const dataMapName = path.basename(mapDefinitionPath, path.extname(mapDefinitionPath)).replace(draftMapDefinitionSuffix, ''); // Gets filename w/o ext (and w/o draft suffix) + const dataMapName = path.basename(lmlPath, path.extname(lmlPath)).replace(draftMapDefinitionSuffix, ''); // Set map definition data to be loaded once webview sends webviewLoaded msg DataMapperExt.openDataMapperPanel(context, dataMapName, { @@ -187,3 +248,40 @@ export async function loadDataMapFileCmd(context: IActionContext, uri: Uri): Pro targetSchemaFileName: path.basename(tgtSchemaPath), }); } + +/** + * Prompts user to find a missing schema file. + */ +async function attemptToResolveMissingSchemaFile(context: IActionContext, schemaName: string, schemaPath: string): Promise { + return !!(await callWithTelemetryAndErrorHandling( + extensionCommand.dataMapAttemptToResolveMissingSchemaFile, + async (_context: IActionContext) => { + const findSchemaFileButton = 'Find schema file'; + const clickedButton = await window.showErrorMessage( + `Error loading map definition: ${schemaName} was not found in the Schemas folder!`, + findSchemaFileButton + ); + + if (clickedButton && clickedButton === findSchemaFileButton) { + const fileUris = await window.showOpenDialog({ + title: 'Select the missing schema file', + canSelectMany: false, + canSelectFiles: true, + canSelectFolders: false, + filters: { 'XML Schema': ['xsd'], 'JSON Schema': ['json'] }, + }); + + if (fileUris && fileUris.length > 0) { + await fs.copyFile(fileUris[0].fsPath, schemaPath); + context.telemetry.properties.result = 'Succeeded'; + return true; + } + } + + context.telemetry.properties.result = 'Canceled'; + context.telemetry.properties.wasResolvingMissingSchemaFile = 'true'; + ext.showError(localize('MissingSchema', 'No schema file was selected. Aborting load...')); + return false; + } + )); +} diff --git a/apps/vs-code-designer/src/app/commands/dataMapper/extensionConfig.ts b/apps/vs-code-designer/src/app/commands/dataMapper/extensionConfig.ts index 2979b98336f..e14057f1213 100644 --- a/apps/vs-code-designer/src/app/commands/dataMapper/extensionConfig.ts +++ b/apps/vs-code-designer/src/app/commands/dataMapper/extensionConfig.ts @@ -4,7 +4,14 @@ *--------------------------------------------------------------------------------------------*/ export const webviewType = 'dataMapperWebview'; +/** + * @deprecated LML files are being phased out. Use supportedDataMapFileExts instead. + */ export const supportedDataMapDefinitionFileExts = ['.lml', '.yml']; +/** + * Primary data map file extensions (XSLT with embedded metadata). + */ +export const supportedDataMapFileExts = ['.xslt']; export const supportedSchemaFileExts = ['.xsd', '.json']; export const supportedCustomXsltFileExts = ['.xslt', '.xml']; export const supportedDataMapperFolders = ['Maps', 'MapDefinitions', 'Schemas']; @@ -14,11 +21,21 @@ export const schemasPath = `${artifactsPath}/Schemas`; export const customXsltPath = 'Artifacts/DataMapper/Extensions/InlineXslt'; export const customFunctionsPath = 'Artifacts/DataMapper/Extensions/Functions'; export const dataMapsPath = `${artifactsPath}/Maps`; +/** + * @deprecated MapDefinitions folder is being phased out. Maps are now stored directly in dataMapsPath as XSLT. + */ export const dataMapDefinitionsPath = `${artifactsPath}/MapDefinitions`; export const defaultDataMapFilename = 'default'; export const draftMapDefinitionSuffix = '.draft'; +/** + * @deprecated LML extension is being phased out. Use mapXsltExtension instead. + */ export const mapDefinitionExtension = '.lml'; export const mapXsltExtension = '.xslt'; +/** + * Draft file extension for XSLT files. + */ +export const draftXsltExtension = '.draft.xslt'; export const backendRuntimeBaseUrl = 'http://localhost:'; diff --git a/apps/vs-code-designer/src/main.ts b/apps/vs-code-designer/src/main.ts index 7d3422954b1..26be2c0a941 100644 --- a/apps/vs-code-designer/src/main.ts +++ b/apps/vs-code-designer/src/main.ts @@ -3,6 +3,7 @@ import { runPostWorkflowCreateStepsFromCache } from './app/commands/createWorkfl import { runPostExtractStepsFromCache } from './app/utils/cloudToLocalUtils'; import { supportedDataMapDefinitionFileExts, + supportedDataMapFileExts, supportedDataMapperFolders, supportedSchemaFileExts, } from './app/commands/dataMapper/extensionConfig'; @@ -54,14 +55,12 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push(workspaceWatcher); // Data Mapper context - vscode.commands.executeCommand( - 'setContext', - extensionCommand.dataMapSetSupportedDataMapDefinitionFileExts, - supportedDataMapDefinitionFileExts - ); + // Include both XSLT (new) and LML/YAML (legacy) file extensions for opening Data Mapper + const allSupportedDataMapExts = [...supportedDataMapFileExts, ...supportedDataMapDefinitionFileExts]; + vscode.commands.executeCommand('setContext', extensionCommand.dataMapSetSupportedDataMapDefinitionFileExts, allSupportedDataMapExts); vscode.commands.executeCommand('setContext', extensionCommand.dataMapSetSupportedSchemaFileExts, supportedSchemaFileExts); vscode.commands.executeCommand('setContext', extensionCommand.dataMapSetSupportedFileExts, [ - ...supportedDataMapDefinitionFileExts, + ...allSupportedDataMapExts, ...supportedSchemaFileExts, ]); vscode.commands.executeCommand('setContext', extensionCommand.dataMapSetDmFolders, supportedDataMapperFolders); diff --git a/apps/vs-code-designer/src/package.json b/apps/vs-code-designer/src/package.json index e3106862b2b..91e14674d85 100644 --- a/apps/vs-code-designer/src/package.json +++ b/apps/vs-code-designer/src/package.json @@ -1057,6 +1057,8 @@ "publisher": "ms-azuretools", "icon": "assets/logicapp.png", "dependencies": { - "tslib": "2.4.0" + "saxon-js": "^2.6.0", + "tslib": "2.4.0", + "xslt3": "^2.7.0" } } diff --git a/apps/vs-code-react/src/app/dataMapper/appV2.tsx b/apps/vs-code-react/src/app/dataMapper/appV2.tsx index 57f15d89223..6d7d320a9cf 100644 --- a/apps/vs-code-react/src/app/dataMapper/appV2.tsx +++ b/apps/vs-code-react/src/app/dataMapper/appV2.tsx @@ -40,6 +40,7 @@ export const DataMapperAppV2 = () => { const fetchedFunctions = useSelector((state: RootState) => state.dataMap.fetchedFunctions); const dataMapperVersion = useSelector((state: RootState) => state.project.dataMapperVersion); const isTestDisabledForOS = useSelector((state: RootState) => state.dataMap.isTestDisabledForOS); + const testXsltTransformResult = useSelector((state: RootState) => state.dataMap.testXsltTransformResult); const runtimePort = useSelector((state: RootState) => state.dataMap.runtimePort); @@ -186,6 +187,7 @@ export const DataMapperAppV2 = () => { // Passed in here too so it can be managed in the Redux store so components can track the current theme theme={theme} isTestDisabledForOS={isTestDisabledForOS} + testXsltTransformResult={testXsltTransformResult} >
{ this.sendMsgToVsix({ command: ExtensionCommand.saveDataMapDefinition, @@ -34,10 +37,21 @@ export class DataMapperFileService implements IDataMapperFileService { }); }; - public saveDraftStateCall(dataMapDefinition: string): void { + /** + * Saves the XSLT with embedded metadata to the filesystem. + * This is the primary save method. + */ + public saveMapXsltCall = (xsltWithMetadata: string) => { + this.sendMsgToVsix({ + command: ExtensionCommand.saveDataMapXslt, + data: xsltWithMetadata, + }); + }; + + public saveDraftStateCall(xsltWithMetadata: string): void { this.sendMsgToVsix({ command: ExtensionCommand.saveDraftDataMapDefinition, - data: dataMapDefinition, + data: xsltWithMetadata, }); } @@ -47,6 +61,9 @@ export class DataMapperFileService implements IDataMapperFileService { }); }; + /** + * @deprecated Use saveMapXsltCall instead. Kept for backward compatibility. + */ public saveXsltCall = (xslt: string) => { this.sendMsgToVsix({ command: ExtensionCommand.saveDataMapXslt, @@ -76,4 +93,11 @@ export class DataMapperFileService implements IDataMapperFileService { data: { title, text, level }, }); } + + public testXsltTransform = (xsltContent: string, inputXml: string) => { + this.sendMsgToVsix({ + command: ExtensionCommand.testXsltTransform, + data: { xsltContent, inputXml }, + }); + }; } diff --git a/apps/vs-code-react/src/run-service/types.ts b/apps/vs-code-react/src/run-service/types.ts index 3db975219df..10ca85d2592 100644 --- a/apps/vs-code-react/src/run-service/types.ts +++ b/apps/vs-code-react/src/run-service/types.ts @@ -437,3 +437,14 @@ export interface GetTestFeatureEnablementStatus { command: typeof ExtensionCommand.isTestDisabledForOS; data: boolean; } + +export interface TestXsltTransformResultMessage { + command: typeof ExtensionCommand.testXsltTransformResult; + data: { + success: boolean; + outputXml?: string; + error?: string; + statusCode: number; + statusText: string; + }; +} diff --git a/apps/vs-code-react/src/state/DataMapSliceV2.ts b/apps/vs-code-react/src/state/DataMapSliceV2.ts index 4e2aa48796f..387386697b3 100644 --- a/apps/vs-code-react/src/state/DataMapSliceV2.ts +++ b/apps/vs-code-react/src/state/DataMapSliceV2.ts @@ -3,6 +3,14 @@ import type { DataMapSchema, MapDefinitionEntry, MapMetadataV2, IFileSysTreeItem import type { PayloadAction } from '@reduxjs/toolkit'; import { createSlice } from '@reduxjs/toolkit'; +export interface TestXsltTransformResult { + success: boolean; + outputXml?: string; + error?: string; + statusCode: number; + statusText: string; +} + export interface DataMapState { runtimePort?: string; armToken?: string; @@ -22,6 +30,7 @@ export interface DataMapState { fetchedFunctions?: FunctionData[]; useExpandedFunctionCards: boolean; isTestDisabledForOS?: boolean; + testXsltTransformResult?: TestXsltTransformResult; } const initialState: DataMapState = { @@ -86,6 +95,9 @@ export const dataMapSlice = createSlice({ changeIsTestDisabledForOS: (state, action: PayloadAction) => { state.isTestDisabledForOS = action.payload; }, + setTestXsltTransformResult: (state, action: PayloadAction) => { + state.testXsltTransformResult = action.payload; + }, }, }); @@ -107,6 +119,7 @@ export const { changeCustomXsltPathList, changeFetchedFunctions, changeUseExpandedFunctionCards, + setTestXsltTransformResult, } = dataMapSlice.actions; export default dataMapSlice.reducer; diff --git a/apps/vs-code-react/src/webviewCommunication.tsx b/apps/vs-code-react/src/webviewCommunication.tsx index a5b7fc072d6..f61d228680b 100644 --- a/apps/vs-code-react/src/webviewCommunication.tsx +++ b/apps/vs-code-react/src/webviewCommunication.tsx @@ -26,6 +26,7 @@ import type { PackageExistenceResultMessage, UpdateRuntimeBaseUrlMessage, UpdateCallbackInfoMessage, + TestXsltTransformResultMessage, } from './run-service'; import { changeCustomXsltPathList, @@ -52,7 +53,9 @@ import { changeUseExpandedFunctionCards as changeUseExpandedFunctionCardsV2, changeXsltContent as changeXsltContentV2, changeXsltFilename as changeXsltFilenameV2, + setTestXsltTransformResult, } from './state/DataMapSliceV2'; +import { xsltToMapDefinition } from '@microsoft/logic-apps-data-mapper-v2'; import { initializeDesigner, updateCallbackUrl, @@ -109,7 +112,8 @@ type DataMapperMessageType = | SetRuntimePortMessage | GetConfigurationSettingMessage | GetDataMapperVersionMessage - | GetTestFeatureEnablementStatus; + | GetTestFeatureEnablementStatus + | TestXsltTransformResultMessage; type WorkflowMessageType = | UpdateCallbackInfoMessage | UpdateRuntimeBaseUrlMessage @@ -213,7 +217,23 @@ export const WebViewCommunication: React.FC<{ children: ReactNode }> = ({ childr // NOTE: DataMapDataProvider ensures the functions and schemas are loaded before loading the mapDefinition connections dispatch(changeSourceSchemaFilenameV2(message.data.sourceSchemaFileName)); dispatch(changeTargetSchemaFilenameV2(message.data.targetSchemaFileName)); - dispatch(changeMapDefinitionV2(message.data.mapDefinition)); + + // For v3 format, parse the XSLT to derive the mapDefinition + // This makes XSLT the source of truth for mapping logic + if (message.data.isV3Format && message.data.xsltContent) { + const derivedMapDefinition = xsltToMapDefinition( + message.data.xsltContent, + message.data.sourceSchemaFileName, + message.data.targetSchemaFileName + ); + dispatch(changeMapDefinitionV2(derivedMapDefinition ?? {})); + // Also store the raw XSLT content for display in code view + dispatch(changeXsltContentV2(message.data.xsltContent)); + } else { + // v2 format: use embedded mapDefinition directly + dispatch(changeMapDefinitionV2(message.data.mapDefinition)); + } + dispatch(changeDataMapFilename(message.data.mapDefinitionName)); dispatch(changeDataMapMetadataV2(message.data.metadata)); break; @@ -243,6 +263,10 @@ export const WebViewCommunication: React.FC<{ children: ReactNode }> = ({ childr dispatch(changeIsTestDisabledForOS(message.data)); break; } + case ExtensionCommand.testXsltTransformResult: { + dispatch(setTestXsltTransformResult(message.data)); + break; + } default: throw new Error('Unknown post message received'); } diff --git a/docs/plans/2026-01-21-lml-to-xslt-migration-design.md b/docs/plans/2026-01-21-lml-to-xslt-migration-design.md new file mode 100644 index 00000000000..e8a351ebef0 --- /dev/null +++ b/docs/plans/2026-01-21-lml-to-xslt-migration-design.md @@ -0,0 +1,177 @@ +# LML to XSLT Migration Design + +## Overview + +Remove LML (Logic Mapper Language) files from the Data Mapper v2 workflow. Instead of maintaining separate files for map definitions (LML), XSLT transformations, and metadata, we will embed all mapping information directly into the XSLT file as a structured comment. + +## Goals + +1. **Single file format** - One XSLT file contains everything needed to load and save a data map +2. **Simplified user experience** - Users only manage one file per map +3. **Backward compatibility** - Existing LML files auto-migrate on open + +## File Format + +### New XSLT Structure + +```xml + + + + + +``` + +### Metadata JSON Schema + +```typescript +interface XsltMapMetadata { + version: "2.0"; + sourceSchema: string; + targetSchema: string; + mapDefinition: MapDefinitionEntry; // Reuses existing LML structure + metadata: { + functionNodes: FunctionMetadata[]; + canvasRect: Rect; + }; +} +``` + +### File Locations + +| File Type | Old Location | New Location | +|-----------|--------------|--------------| +| Map definition | `DataMaps/{name}.yml` | Embedded in XSLT | +| XSLT | `maps/{name}.xslt` | `maps/{name}.xslt` (unchanged) | +| Metadata | `.vscode/{name}DataMapMetadata-v2.json` | Embedded in XSLT | +| Draft | `DataMaps/{name}.draft.yml` | `maps/{name}.draft.xslt` | + +## Implementation + +### 1. XSLT Metadata Utilities + +Create new file: `libs/data-mapper-v2/src/mapHandling/XsltMetadataSerializer.ts` + +```typescript +// Embed metadata into XSLT +export function embedMetadataInXslt(xslt: string, metadata: XsltMapMetadata): string + +// Extract metadata from XSLT +export function extractMetadataFromXslt(xslt: string): XsltMapMetadata | null + +// Check if XSLT has embedded metadata +export function hasEmbeddedMetadata(xslt: string): boolean +``` + +### 2. Save Workflow Changes + +**EditorCommandBar.tsx** - Update `onSaveClick`: +1. Serialize connections to map definition (existing) +2. Serialize UI metadata (existing) +3. Call backend to generate XSLT (existing) +4. Embed metadata JSON into XSLT comment +5. Save single XSLT file via `saveXsltCall` +6. Delete legacy files (LML, metadata JSON) + +### 3. Load Workflow Changes + +**DataMapperPanel.ts** - Update load flow: +1. Check for XSLT file in `maps/` folder +2. If XSLT exists with embedded metadata: + - Extract metadata from comment + - Parse map definition and UI metadata + - Load into designer +3. If old LML file exists (migration): + - Load LML file + - Load metadata JSON if present + - Generate XSLT with embedded metadata + - Save new format + - Delete old files + +### 4. Draft File Handling + +**DataMapperPanel.ts**: +- Change `saveDraftDataMapDefinition` to save `maps/{name}.draft.xslt` +- Draft files contain full XSLT with embedded metadata +- On load, check for draft XSLT before main XSLT + +### 5. File Service Interface Updates + +**IDataMapperFileService**: +- Modify `saveMapDefinitionCall` to `saveMapCall(xsltWithMetadata: string)` +- Remove separate `saveMapMetadata` call +- Update `saveDraftStateCall` to save draft XSLT + +### 6. Extension Config Updates + +**extensionConfig.ts**: +- Remove `mapDefinitionExtension = '.lml'` +- Update `supportedDataMapDefinitionFileExts` to `['.xslt']` +- Remove `dataMapDefinitionsPath` (no longer needed) + +## Migration Strategy + +### Auto-Migration on Open + +When opening a map: +1. First check for `maps/{name}.xslt` with embedded metadata +2. If not found, check for `DataMaps/{name}.yml` (legacy LML) +3. If LML found: + - Load LML content + - Load metadata from `.vscode/` if present + - Generate XSLT via backend + - Embed metadata + - Save to `maps/{name}.xslt` + - Delete legacy files: + - `DataMaps/{name}.yml` + - `DataMaps/{name}.draft.yml` + - `.vscode/{name}DataMapMetadata-v2.json` + +### Files to Delete After Migration + +- `DataMaps/{name}.yml` +- `DataMaps/{name}.draft.yml` +- `.vscode/{name}DataMapMetadata-v2.json` + +## Testing Strategy + +1. Unit tests for metadata serialization/deserialization +2. Unit tests for XSLT embedding/extraction +3. Integration tests for save workflow +4. Integration tests for load workflow +5. Migration tests for LML → XSLT conversion +6. E2E tests in VS Code extension + +## Rollout + +1. Implement metadata utilities +2. Update save workflow +3. Update load workflow with migration +4. Update draft handling +5. Clean up old code paths +6. Update documentation + +## Risks and Mitigations + +| Risk | Mitigation | +|------|------------| +| Data loss during migration | Keep backup of LML before deletion | +| Invalid XSLT after embedding | Validate XML structure after embedding | +| Large metadata bloating XSLT | JSON is compact; LML reuse keeps it small | +| XSLT comment stripped by tools | Document that comment must be preserved | diff --git a/libs/data-mapper-v2/src/components/codeView/CodeViewPanelBody.tsx b/libs/data-mapper-v2/src/components/codeView/CodeViewPanelBody.tsx index ac7a5821cd7..4c952fdc351 100644 --- a/libs/data-mapper-v2/src/components/codeView/CodeViewPanelBody.tsx +++ b/libs/data-mapper-v2/src/components/codeView/CodeViewPanelBody.tsx @@ -14,13 +14,14 @@ export const CodeViewPanelBody = (_props: CodeViewPanelBodyProps) => { const editorRef = useRef(null); const styles = useStyles(); - const dataMapDefinition = useSelector((state: RootState) => state.dataMap.present.curDataMapOperation.dataMapLML); + // Show XSLT content (the actual saved format) instead of LML + const xsltContent = useSelector((state: RootState) => state.dataMap.present.curDataMapOperation.xsltContent); const resources = useMemo( () => ({ - EMPTY_MAP_DEFINITION: intl.formatMessage({ - defaultMessage: 'Unable to generate data map definition', - id: 'sv+IcU', - description: `Message to display when the data map definition can't be generated`, + EMPTY_XSLT_CONTENT: intl.formatMessage({ + defaultMessage: 'XSLT not yet generated. Save the map to generate XSLT output.', + id: '7V737/', + description: `Message to display when XSLT content hasn't been generated yet`, }), }), [intl] @@ -37,8 +38,8 @@ export const CodeViewPanelBody = (_props: CodeViewPanelBodyProps) => { return (
{ const isCodeViewOpen = useSelector((state: RootState) => state.panel.codeViewPanel.isOpen); const { sourceSchema, targetSchema } = useSelector((state: RootState) => state.dataMap.present.curDataMapOperation); - const xsltFilename = useSelector((state: RootState) => state.dataMap.present.curDataMapOperation.xsltFilename); + const xsltContent = useSelector((state: RootState) => state.dataMap.present.curDataMapOperation.xsltContent); const toasterId = useId('toaster'); @@ -90,10 +91,19 @@ export const EditorCommandBar = (_props: EditorCommandBarProps) => { throw new Error('Canvas bounds are not defined, cannot save map metadata.'); } - const mapMetadata = JSON.stringify(generateMapMetadata(functions, currentConnections, canvasRect)); + if (!sourceSchema || !targetSchema) { + DataMapperFileService().sendNotification(failedXsltMessage, 'Source or target schema is not defined.', LogEntryLevel.Error); + return; + } if (dataMapDefinition.isSuccess) { - DataMapperFileService().saveMapDefinitionCall(dataMapDefinition.definition, mapMetadata); + // Generate UI metadata (function positions, canvas state) + // Note: We no longer embed mapDefinition - XSLT is the source of truth for mappings + const uiMetadata = generateMapMetadata(functions, currentConnections, canvasRect); + + // Create v3 metadata object (only UI layout, no mapDefinition) + // The actual mapping logic is derived from the XSLT content when loading + const xsltMetadata = createXsltMapMetadataV3(sourceSchema.name, targetSchema.name, uiMetadata); dispatch( saveDataMap({ @@ -102,19 +112,28 @@ export const EditorCommandBar = (_props: EditorCommandBarProps) => { }) ); + // Generate XSLT from the backend, then embed metadata and save generateDataMapXslt(dataMapDefinition.definition) .then((xsltStr) => { - DataMapperFileService().saveXsltCall(xsltStr); + // Embed metadata into the XSLT as a comment + const xsltWithMetadata = embedMetadataInXslt(xsltStr, xsltMetadata); + + // Update XSLT content in state for code view display + dispatch(setXsltContent(xsltWithMetadata)); + + // Save the combined XSLT file (this is now the only file we need) + DataMapperFileService().saveMapXsltCall(xsltWithMetadata); + LoggerService().log({ level: LogEntryLevel.Verbose, - area: `${LogCategory.DataMapperDesigner}/onGenerateClick`, - message: 'Successfully generated xslt', + area: `${LogCategory.DataMapperDesigner}/onSaveClick`, + message: 'Successfully saved map with embedded metadata', }); }) .catch((error: Error) => { LoggerService().log({ level: LogEntryLevel.Error, - area: `${LogCategory.DataMapperDesigner}/onGenerateClick`, + area: `${LogCategory.DataMapperDesigner}/onSaveClick`, message: JSON.stringify(error.message), }); DataMapperFileService().sendNotification(failedXsltMessage, error.message, LogEntryLevel.Error); @@ -211,11 +230,11 @@ export const EditorCommandBar = (_props: EditorCommandBarProps) => { save: !isDirty || sourceInEditState || targetInEditState, undo: undoStack.length === 0, discard: !isDirty, - test: sourceInEditState || targetInEditState || !xsltFilename || isTestDisabledForOS, + test: sourceInEditState || targetInEditState || !xsltContent || isTestDisabledForOS, codeView: sourceInEditState || targetInEditState, mapChecker: sourceInEditState || targetInEditState, }), - [isDirty, undoStack.length, sourceInEditState, targetInEditState, xsltFilename, isTestDisabledForOS] + [isDirty, undoStack.length, sourceInEditState, targetInEditState, xsltContent, isTestDisabledForOS] ); const mapCheckerIcon = useMemo(() => { diff --git a/libs/data-mapper-v2/src/components/test/TestPanel.tsx b/libs/data-mapper-v2/src/components/test/TestPanel.tsx index 2a082602eb7..58bf551cb3c 100644 --- a/libs/data-mapper-v2/src/components/test/TestPanel.tsx +++ b/libs/data-mapper-v2/src/components/test/TestPanel.tsx @@ -1,4 +1,4 @@ -import { useCallback, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useIntl } from 'react-intl'; import { Button, Spinner } from '@fluentui/react-components'; import { useSelector, useDispatch } from 'react-redux'; @@ -7,7 +7,7 @@ import { Panel } from '../common/panel/Panel'; import { toggleTestPanel, updateTestOutput } from '../../core/state/PanelSlice'; import { useStyles } from './styles'; import { TestPanelBody } from './TestPanelBody'; -import { testDataMap } from '../../core/queries/datamap'; +import { DataMapperFileService } from '../../core/services/dataMapperFileService/dataMapperFileService'; import { LogCategory } from '../../utils/Logging.Utils'; import { guid, LogEntryLevel, LoggerService } from '@microsoft/logic-apps-shared'; import { PanelXButton } from '../common/panel/PanelXButton'; @@ -18,8 +18,11 @@ export const TestPanel = (_props: TestPanelProps) => { const styles = useStyles(); const dispatch = useDispatch(); const [loading, setLoading] = useState(false); - const { testMapInput, isOpen } = useSelector((state: RootState) => state.panel.testPanel); - const xsltFilename = useSelector((state: RootState) => state.dataMap.present.curDataMapOperation.xsltFilename); + const { testMapInput, testMapOutput, testMapOutputError, isOpen } = useSelector((state: RootState) => state.panel.testPanel); + const xsltContent = useSelector((state: RootState) => state.dataMap.present.curDataMapOperation.xsltContent); + + // Track if we're waiting for a test result + const waitingForResultRef = useRef(false); const onCloseClick = useCallback(() => { dispatch(toggleTestPanel()); @@ -47,62 +50,83 @@ export const TestPanel = (_props: TestPanelProps) => { id: 'ldn/IC', description: 'Test button', }), + SAVE_BEFORE_TEST: intl.formatMessage({ + defaultMessage: 'Please save the map before testing', + id: 'aulGxd', + description: 'Message when XSLT content is not available', + }), }), [intl] ); + // Watch for test results to stop loading + useEffect(() => { + if (waitingForResultRef.current && (testMapOutput || testMapOutputError)) { + waitingForResultRef.current = false; + setLoading(false); + + if (testMapOutput) { + LoggerService().log({ + level: LogEntryLevel.Verbose, + area: `${LogCategory.TestMapPanel}/testDataMap`, + message: 'Successfully tested data map', + args: [ + { + statusCode: testMapOutput.statusCode, + statusText: testMapOutput.statusText, + }, + ], + }); + } else if (testMapOutputError) { + LoggerService().log({ + level: LogEntryLevel.Error, + area: `${LogCategory.TestMapPanel}/testDataMap`, + message: testMapOutputError.message, + }); + } + } + }, [testMapOutput, testMapOutputError]); + const onTestClick = useCallback(() => { - if (!!xsltFilename && !!testMapInput) { + if (!!xsltContent && !!testMapInput) { const attempt = guid(); setLoading(true); - // Clear out test output - dispatch(updateTestOutput({})); + waitingForResultRef.current = true; - testDataMap(xsltFilename, testMapInput) - .then((response) => { - dispatch( - updateTestOutput({ - response: response, - }) - ); + // Clear out previous test output + dispatch(updateTestOutput({})); - LoggerService().log({ - level: LogEntryLevel.Verbose, - area: `${LogCategory.TestMapPanel}/testDataMap`, - message: 'Successfully tested data map', - args: [ - { - attempt, - statusCode: response.statusCode, - statusText: response.statusText, - }, - ], - }); + LoggerService().log({ + level: LogEntryLevel.Verbose, + area: `${LogCategory.TestMapPanel}/testDataMap`, + message: 'Starting local XSLT transformation test', + args: [{ attempt }], + }); - setLoading(false); - }) - .catch((error: Error) => { - LoggerService().log({ - level: LogEntryLevel.Error, - area: `${LogCategory.TestMapPanel}/testDataMap`, - message: error.message, - args: [ - { - attempt, - }, - ], - }); + // Use local XSLT transformation via the file service + // The result will come back through the DataMapDataProvider -> Redux flow + try { + DataMapperFileService().testXsltTransform(xsltContent, testMapInput); + } catch (error) { + // Handle synchronous errors (e.g., service not initialized) + LoggerService().log({ + level: LogEntryLevel.Error, + area: `${LogCategory.TestMapPanel}/testDataMap`, + message: error instanceof Error ? error.message : 'Failed to start transformation', + args: [{ attempt }], + }); - dispatch( - updateTestOutput({ - error: error, - }) - ); + dispatch( + updateTestOutput({ + error: error instanceof Error ? error : new Error('Failed to start transformation'), + }) + ); - setLoading(false); - }); + waitingForResultRef.current = false; + setLoading(false); + } } - }, [testMapInput, xsltFilename, dispatch]); + }, [testMapInput, xsltContent, dispatch]); return ( { body={} footer={
-