Add value plugin for inline JSON and YAML data in markdown fence blocks, schema definitions for inline data elements, and compiler support

.github/copilot-instructions.md

@@ -29,6 +29,12 @@ Chartifact consists of several interoperating modules:
 ### Testing
 - Currently we don't have much test coverage. We can add this later.
 
+### Version Control
+- **Do NOT check in UMD build artifacts**: Files matching `*.umd.js` in `docs/dist/v1/` are build outputs and should not be committed in PRs
+- **Do NOT edit `docs/schema/idoc_v1.d.ts` directly**: This file is generated from `packages/schema-doc/src/`. Edit the source files in the schema-doc package instead
+- **Do NOT check in built schema files**: Files `docs/schema/idoc_v1.d.ts` and `docs/schema/idoc_v1.json` are generated from `packages/schema-doc/src/` during the build process and should not be committed in PRs
+- These files are generated during the build process and committing them creates unnecessary merge conflicts
+
 ## Project-Specific Conventions
 
 1. **File Formats**:

packages/compiler/src/validate/common.ts

@@ -5,7 +5,7 @@ import { validateTransforms } from './transforms.js';
 
 const illegalChars = '/|\\\'"`,.;:~-=+?!@#$%^&*()[]{}<>';
 
-export const ignoredSignals = ['width', 'height', 'padding', 'autosize', 'background', 'style', 'parent', 'datum', 'item', 'event', 'cursor', 'origins'];
+export const ignoredSignals = ['width', 'height', 'padding', 'autosize', 'background', 'style', 'parent', 'datum', 'item', 'event', 'cursor', 'value'];
 
 // Utility functions for property validation
 export function validateRequiredString(value: any, propertyName: string, elementType: string): string[] {

packages/markdown/src/factory.ts

@@ -152,31 +152,41 @@ export function create() {
             if (directPlugin) {
                 return directPlugin;
             }
+
+            // Split info into words for further processing
+            const infoWords = info.split(/\s+/);
+
             // Third priority: Check for plugin names with additional parameters (like "csv variableId")
-            else {
-                const infoWords = info.split(/\s+/);
-                if (infoWords.length > 0) {
-                    const pluginPrefix = findPluginByPrefix(infoWords[0]);
-                    if (pluginPrefix) {
-                        return pluginPrefix;
-                    }
+            if (infoWords.length > 0) {
+                const pluginPrefix = findPluginByPrefix(infoWords[0]);
+                if (pluginPrefix) {
+                    return pluginPrefix;
                 }
             }
+
             // Fourth priority: Check if it starts with "json " and extract the plugin name
-            if (info.startsWith('json ')) {
-                const jsonPluginName = info.slice(5).trim();
-                const jsonPlugin = findPlugin(jsonPluginName);
+            if (infoWords[0] === 'json' && infoWords.length > 1) {
+                const jsonPlugin = findPlugin(infoWords[1]);
                 if (jsonPlugin) {
                     return jsonPlugin;
                 }
+                // Default to 'value' plugin for json data (e.g., "json products")
+                const valuePlugin = findPlugin('value');
+                if (valuePlugin) {
+                    return valuePlugin;
+                }
             }
             // Fifth priority: Check if it starts with "yaml " and extract the plugin name
-            else if (info.startsWith('yaml ')) {
-                const yamlPluginName = info.slice(5).trim();
-                const yamlPlugin = findPlugin(yamlPluginName);
+            else if (info.startsWith('yaml ') && infoWords.length > 1) {
+                const yamlPlugin = findPlugin(infoWords[1]);
                 if (yamlPlugin) {
                     return yamlPlugin;
                 }
+                // Default to 'value' plugin for yaml data (e.g., "yaml products")
+                const valuePlugin = findPlugin('value');
+                if (valuePlugin) {
+                    return valuePlugin;
+                }
             }
         }
 

packages/markdown/src/plugins/config.ts

@@ -9,6 +9,131 @@ import { getJsonScriptTag } from "./util.js";
 import { SpecReview } from 'common';
 import * as yaml from 'js-yaml';
 
+/**
+ * Parse fence info and extract all metadata.
+ * @param info The fence info string
+ * @returns Object with format, pluginName, params (including variableId), and wasDefaultId flag
+ */
+export function parseFenceInfo(info: string): {
+    format: 'json' | 'yaml';
+    pluginName: string;
+    params: Map<string, string>;
+    wasDefaultId: boolean;
+} {
+    const parts = info.trim().split(/\s+/);
+
+    // Determine format (json is default)
+    let format: 'json' | 'yaml' = 'json';
+    let startIndex = 0;
+
+    if (parts[0] === 'json' || parts[0] === 'yaml') {
+        format = parts[0];
+        startIndex = 1;
+    }
+
+    // The next part could be the plugin name OR a parameter
+    let pluginName = '';
+    let pluginNameIndex = startIndex;
+
+    if (startIndex < parts.length && !parts[startIndex].includes(':')) {
+        // It's a plugin name (doesn't have a colon)
+        pluginName = parts[startIndex];
+        pluginNameIndex = startIndex + 1;
+    }
+
+    const params = new Map<string, string>();
+    let variableId: string | null = null;
+
+    // Parse remaining parts starting after plugin name (if found)
+    for (let i = pluginNameIndex; i < parts.length; i++) {
+        const part = parts[i];
+        const colonIndex = part.indexOf(':');
+
+        if (colonIndex > 0) {
+            // Parameter with colon
+            const key = part.slice(0, colonIndex);
+            const value = part.slice(colonIndex + 1);
+
+            if (value) {
+                // Format: "key:value"
+                params.set(key, value);
+            } else if (i + 1 < parts.length) {
+                // Format: "key: value" (value in next part)
+                params.set(key, parts[++i]);
+            }
+        } else if (!variableId) {
+            // First non-parameter value becomes variableId
+            variableId = part;
+        }
+    }
+
+    // If variableId param exists, use it; otherwise use the direct value
+    const explicitVariableId = params.get('variableId');
+    const finalVariableId = explicitVariableId || variableId;
+    const wasDefaultId = !finalVariableId;
+
+    if (finalVariableId) {
+        params.set('variableId', finalVariableId);
+    }
+
+    return { format, pluginName, params, wasDefaultId };
+}
+
+/*
+//Tests for parseFenceInfo
+const tests: [string, { format: 'json' | 'yaml'; pluginName: string; variableId: string | undefined; wasDefaultId: boolean }][] = [
+    //Direct format cases
+    ["dsv products", { format: "json", pluginName: "dsv", variableId: "products", wasDefaultId: false }],
+    ["csv officeSupplies", { format: "json", pluginName: "csv", variableId: "officeSupplies", wasDefaultId: false }],
+
+    //Explicit format cases
+    ["json inventory", { format: "json", pluginName: "inventory", variableId: undefined, wasDefaultId: true }],
+    ["yaml products", { format: "yaml", pluginName: "products", variableId: undefined, wasDefaultId: true }],
+
+    //Explicit plugin name
+    ["json value inventory", { format: "json", pluginName: "value", variableId: "inventory", wasDefaultId: false }],
+    ["yaml value products", { format: "yaml", pluginName: "value", variableId: "products", wasDefaultId: false }],
+
+    //Direct plugin without format
+    ["value inventory", { format: "json", pluginName: "value", variableId: "inventory", wasDefaultId: false }],
+
+    //Explicit variableId parameter (no space)
+    ["json variableId:inventory", { format: "json", pluginName: "", variableId: "inventory", wasDefaultId: false }],
+    ["dsv variableId:products", { format: "json", pluginName: "dsv", variableId: "products", wasDefaultId: false }],
+    ["json value variableId:inventory", { format: "json", pluginName: "value", variableId: "inventory", wasDefaultId: false }],
+    ["yaml value variableId:products", { format: "yaml", pluginName: "value", variableId: "products", wasDefaultId: false }],
+
+    //Explicit variableId parameter (with space)
+    ["json variableId: inventory", { format: "json", pluginName: "", variableId: "inventory", wasDefaultId: false }],
+    ["dsv variableId: products", { format: "json", pluginName: "dsv", variableId: "products", wasDefaultId: false }],
+    ["json value variableId: inventory", { format: "json", pluginName: "value", variableId: "inventory", wasDefaultId: false }],
+    ["yaml value variableId: products", { format: "yaml", pluginName: "value", variableId: "products", wasDefaultId: false }],
+
+    //Multiple parameters
+    ["dsv products delimiter:|", { format: "json", pluginName: "dsv", variableId: "products", wasDefaultId: false }],
+    ["dsv delimiter:| variableId:products", { format: "json", pluginName: "dsv", variableId: "products", wasDefaultId: false }],
+    ["dsv delimiter: | variableId: products", { format: "json", pluginName: "dsv", variableId: "products", wasDefaultId: false }],
+
+    //No variableId
+    ["json value", { format: "json", pluginName: "value", variableId: undefined, wasDefaultId: true }],
+    ["dsv", { format: "json", pluginName: "dsv", variableId: undefined, wasDefaultId: true }],
+];
+
+tests.forEach(([input, expected], i) => {
+    const result = parseFenceInfo(input);
+    const variableId = result.params.get('variableId');
+    const pass = 
+        result.format === expected.format &&
+        result.pluginName === expected.pluginName &&
+        variableId === expected.variableId &&
+        result.wasDefaultId === expected.wasDefaultId;
+
+    console.log(
+        `${pass ? '✅' : '❌'} Test ${i + 1}: ${pass ? 'PASS' : `FAIL\n  Input: "${input}"\n  Got: ${JSON.stringify({format: result.format, pluginName: result.pluginName, variableId, wasDefaultId: result.wasDefaultId})}\n  Expected: ${JSON.stringify(expected)}`}`
+    );
+});
+*/
+
 /**
  * Creates a plugin that can parse both JSON and YAML formats
  */

packages/markdown/src/plugins/dsv.ts

@@ -9,6 +9,7 @@ import { sanitizedHTML, sanitizeHtmlComment } from '../sanitize.js';
 import { pluginClassName } from './util.js';
 import { PluginNames } from './interfaces.js';
 import { SpecReview } from 'common';
+import { parseFenceInfo } from './config.js';
 
 interface DsvInstance {
     id: string;
@@ -24,65 +25,58 @@ export interface DsvSpec {
 }
 
 /**
- * Utility function to parse variable ID from fence info.
- * Supports both "pluginName variableId" and "pluginName variableId:name" formats.
- * @param info The fence info string (e.g., "csv myData" or "csv variableId:myData")
+ * Utility function to parse DSV fence info.
+ * Supports formats like:
+ * - "dsv products delimiter:|"
+ * - "dsv delimiter:| variableId:products"
+ * - "dsv delimiter: | variableId: products"
+ * @param info The fence info string
  * @param pluginName The plugin name (csv, tsv, dsv)
  * @param index The fence index for default naming
- * @returns Object with variableId and wasDefaultId flag
+ * @returns Object with variableId, delimiter, and flags
  */
-export function parseVariableId(info: string, pluginName: string, index: number): { variableId: string; wasDefaultId: boolean } {
-    const parts = info.trim().split(/\s+/);
+export function parseDsvInfo(info: string, pluginName: string, index: number): {
+    variableId: string;
+    delimiter: string;
+    wasDefaultId: boolean;
+    wasDefaultDelimiter: boolean;
+} {
+    const { params, wasDefaultId } = parseFenceInfo(info);
 
-    // Check for explicit variableId: parameter
-    for (const part of parts) {
-        if (part.startsWith('variableId:')) {
-            return { 
-                variableId: part.slice(11).trim(), // Remove 'variableId:' prefix and trim spaces
-                wasDefaultId: false 
-            };
-        }
-    }
+    // Get variableId (already handled by parseFenceInfo)
+    const variableId = params.get('variableId') || `${pluginName}Data${index}`;
 
-    // Check for direct format (second parameter that's not a special parameter)
-    if (parts.length >= 2) {
-        const secondPart = parts[1];
-        if (!secondPart.startsWith('delimiter:') && !secondPart.startsWith('variableId:')) {
-            return { 
-                variableId: secondPart, 
-                wasDefaultId: false 
-            };
-        }
-    }
+    // Get delimiter from parameter or use default
+    let delimiter = params.get('delimiter') || ',';
+    const wasDefaultDelimiter = !params.has('delimiter');
+
+    // Handle special delimiter characters
+    if (delimiter === '\\t') delimiter = '\t';
+    if (delimiter === '\\n') delimiter = '\n';
+    if (delimiter === '\\r') delimiter = '\r';
 
-    // Default variable ID
-    return { 
-        variableId: `${pluginName}Data${index}`, 
-        wasDefaultId: true 
+    return {
+        variableId,
+        delimiter,
+        wasDefaultId,
+        wasDefaultDelimiter
     };
 }
 
 /**
- * Utility function to parse delimiter from fence info.
- * @param info The fence info string (e.g., "dsv delimiter:| variableId:myData")
- * @returns Object with delimiter and wasDefaultDelimiter flag
+ * Utility function to parse variable ID from fence info.
+ * Used by CSV and TSV plugins.
+ * @param info The fence info string (e.g., "csv myData" or "csv variableId:myData")
+ * @param pluginName The plugin name (csv, tsv, dsv)
+ * @param index The fence index for default naming
+ * @returns Object with variableId and wasDefaultId flag
  */
-export function parseDelimiter(info: string): { delimiter: string; wasDefaultDelimiter: boolean } {
-    const parts = info.trim().split(/\s+/);
-
-    for (const part of parts) {
-        if (part.startsWith('delimiter:')) {
-            let delimiter = part.slice(10).trim(); // Remove 'delimiter:' prefix and trim spaces
-            // Handle special cases
-            if (delimiter === '\\t') delimiter = '\t';
-            if (delimiter === '\\n') delimiter = '\n';
-            if (delimiter === '\\r') delimiter = '\r';
-            return { delimiter, wasDefaultDelimiter: false };
-        }
-    }
-
-    // Default to comma
-    return { delimiter: ',', wasDefaultDelimiter: true };
+export function parseVariableId(info: string, pluginName: string, index: number): { variableId: string; wasDefaultId: boolean } {
+    const result = parseDsvInfo(info, pluginName, index);
+    return {
+        variableId: result.variableId,
+        wasDefaultId: result.wasDefaultId
+    };
 }
 
 function inspectDsvSpec(spec: DsvSpec): RawFlaggableSpec<DsvSpec> {
@@ -92,17 +86,13 @@ function inspectDsvSpec(spec: DsvSpec): RawFlaggableSpec<DsvSpec> {
         reasons: []
     };
 
-    // Flag if we had to use defaults
+    // Only flag if we had to use default variable ID
+    // Using default delimiter (comma) is fine and shouldn't be flagged
     if (spec.wasDefaultId) {
         result.hasFlags = true;
         result.reasons.push('No variable ID specified - using default');
     }
 
-    if (spec.wasDefaultDelimiter) {
-        result.hasFlags = true;
-        result.reasons.push('No delimiter specified - using default comma');
-    }
-
     return result;
 }
 
@@ -115,9 +105,8 @@ export const dsvPlugin: Plugin<DsvSpec> = {
         const content = token.content.trim();
         const info = token.info.trim();
 
-        // Use utility functions to parse delimiter and variable ID
-        const { delimiter, wasDefaultDelimiter } = parseDelimiter(info);
-        const { variableId, wasDefaultId } = parseVariableId(info, 'dsv', index);
+        // Parse both delimiter and variable ID in one pass
+        const { variableId, delimiter, wasDefaultId, wasDefaultDelimiter } = parseDsvInfo(info, 'dsv', index);
 
         return sanitizedHTML('pre', { 
             id: `${pluginName}-${index}`, 

packages/markdown/src/plugins/index.ts

@@ -9,6 +9,7 @@ import { checkboxPlugin } from './checkbox.js';
 import { commentPlugin } from './comment.js';
 import { cssPlugin } from './css.js';
 import { csvPlugin } from './csv.js';
+import { valuePlugin } from './value.js';
 import { dsvPlugin } from './dsv.js';
 import { googleFontsPlugin } from './google-fonts.js';
 import { dropdownPlugin } from './dropdown.js';
@@ -30,6 +31,7 @@ export function registerNativePlugins() {
     registerMarkdownPlugin(commentPlugin);
     registerMarkdownPlugin(cssPlugin);
     registerMarkdownPlugin(csvPlugin);
+    registerMarkdownPlugin(valuePlugin);
     registerMarkdownPlugin(dsvPlugin);
     registerMarkdownPlugin(googleFontsPlugin);
     registerMarkdownPlugin(dropdownPlugin);