diff --git a/package-lock.json b/package-lock.json index a2476d53e9f..af85c8e0b54 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46859,6 +46859,7 @@ "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", "@dagrejs/dagre": "^1.1.3", + "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", diff --git a/runtime/drivers/azure/azure.go b/runtime/drivers/azure/azure.go index dbfd51ffd90..9bcd743bef8 100644 --- a/runtime/drivers/azure/azure.go +++ b/runtime/drivers/azure/azure.go @@ -39,12 +39,14 @@ var spec = drivers.Spec{ Secret: true, }, { - Key: "azure_storage_connection_string", - Type: drivers.StringPropertyType, - Secret: true, + Key: "azure_storage_connection_string", + Type: drivers.StringPropertyType, + DisplayName: "Azure Connection String", + Description: "Azure connection string for storage account", + Placeholder: "Paste your Azure connection string here", + Secret: true, }, }, - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ { Key: "path", diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 90e84dfd0ca..005e7ab4868 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -26,14 +26,22 @@ var spec = drivers.Spec{ DocsURL: "https://docs.rilldata.com/build/connectors/data-source/s3", ConfigProperties: []*drivers.PropertySpec{ { - Key: "aws_access_key_id", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_access_key_id", + Type: drivers.StringPropertyType, + DisplayName: "AWS access key ID", + Description: "AWS access key ID for explicit credentials", + Placeholder: "Enter your AWS access key ID", + Secret: true, + Required: true, }, { - Key: "aws_secret_access_key", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_secret_access_key", + Type: drivers.StringPropertyType, + DisplayName: "AWS secret access key", + Description: "AWS secret access key for explicit credentials", + Placeholder: "Enter your AWS secret access key", + Secret: true, + Required: true, }, { Key: "region", diff --git a/web-common/package.json b/web-common/package.json index a34c03c267c..c1dfd35cc60 100644 --- a/web-common/package.json +++ b/web-common/package.json @@ -19,7 +19,6 @@ }, "devDependencies": { "@bufbuild/protobuf": "^1.0.0", - "@dagrejs/dagre": "^1.1.3", "@codemirror/autocomplete": "^6.18.1", "@codemirror/commands": "^6.7.0", "@codemirror/lang-json": "^6.0.1", @@ -34,6 +33,8 @@ "@codemirror/view": "^6.34.1", "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", + "@dagrejs/dagre": "^1.1.3", + "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", @@ -64,6 +65,7 @@ "@types/dompurify": "^3.0.5", "@types/luxon": "^3.4.2", "@types/memoizee": "^0.4.11", + "@xyflow/svelte": "^0.1.39", "autoprefixer": "^10.4.20", "bits-ui": "^0.22.0", "chroma-js": "^3.1.2", @@ -114,7 +116,6 @@ "vega-lite": "^5.23.0", "vega-typings": "^1.3.1", "vitest": "^3.1.1", - "@xyflow/svelte": "^0.1.39", "yaml": "^2.4.5", "yup": "^1.4.0" }, diff --git a/web-common/src/features/entity-management/name-utils.ts b/web-common/src/features/entity-management/name-utils.ts index 01f9567a937..52a6d72dc99 100644 --- a/web-common/src/features/entity-management/name-utils.ts +++ b/web-common/src/features/entity-management/name-utils.ts @@ -21,8 +21,22 @@ export function getName(name: string, others: string[]): string { const set = new Set(others.map((other) => other.toLowerCase())); let result = name; + const incrementableSuffix = /(.+)_([0-9]+)$/; while (set.has(result.toLowerCase())) { + // Special-case for "s3": don't roll over to "s4", append suffix instead. + if (name.toLowerCase() === "s3") { + const match = incrementableSuffix.exec(result); + if (match) { + const base = match[1]; + const number = Number.parseInt(match[2], 10) + 1; + result = `${base}_${number}`; + continue; + } + result = `${name}_1`; + continue; + } + result = INCREMENT.exec(result)?.[1] ? result.replace(INCREMENT, (m) => (+m + 1).toString()) : `${result}_1`; diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte deleted file mode 100644 index 3248cf7487d..00000000000 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ /dev/null @@ -1,506 +0,0 @@ - - -
-
- - {#if connectorType === "rill-managed"} -
- -
- {/if} -
- - {#if connectorType === "self-hosted" || connectorType === "clickhouse-cloud"} - - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {@const isPortField = propertyKey === "port"} - {@const isSSLField = propertyKey === "ssl"} - -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - options={connectorType === "clickhouse-cloud" && isPortField - ? [ - { value: "8443", label: "8443 (HTTPS)" }, - { value: "9440", label: "9440 (Native Secure)" }, - ] - : undefined} - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- - -
- {#each dsnProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- -
- {/each} -
- - - {:else} - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- {/if} -
diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index ee081dc8f34..1ae60780bf2 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -8,24 +8,27 @@ import type { SuperValidated } from "sveltekit-superforms"; import type { AddDataFormType, ConnectorType } from "./types"; - import AddClickHouseForm from "./AddClickHouseForm.svelte"; + import MultiStepConnectorFlow from "./MultiStepConnectorFlow.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; - import { isEmpty } from "./utils"; + import { hasOnlyDsn, isEmpty } from "./utils"; + import JSONSchemaFormRenderer from "../../templates/JSONSchemaFormRenderer.svelte"; import { CONNECTION_TAB_OPTIONS, type ClickHouseConnectorType, } from "./constants"; - import { getInitialFormValuesFromProperties } from "../sourceUtils"; - import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; - import { AddDataFormManager } from "./AddDataFormManager"; - import { hasOnlyDsn } from "./utils"; + import { + AddDataFormManager, + type ClickhouseUiState, + } from "./AddDataFormManager"; + import ClickhouseFormRenderer from "./ClickhouseFormRenderer.svelte"; import AddDataFormSection from "./AddDataFormSection.svelte"; + import { get, type Writable } from "svelte/store"; + import { getConnectorSchema } from "./connector-schemas"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -54,45 +57,31 @@ formType, onParamsUpdate: (e: any) => handleOnUpdate(e), onDsnUpdate: (e: any) => handleOnUpdate(e), + getSelectedAuthMethod: () => + get(connectorStepStore).selectedAuthMethod ?? undefined, }); const isMultiStepConnector = formManager.isMultiStepConnector; const isSourceForm = formManager.isSourceForm; const isConnectorForm = formManager.isConnectorForm; const onlyDsn = hasOnlyDsn(connector, isConnectorForm); - $: stepState = $connectorStepStore; - $: stepProperties = - isMultiStepConnector && stepState.step === "source" - ? (connector.sourceProperties ?? []) - : properties; - $: if ( - isMultiStepConnector && - stepState.step === "source" && - stepState.connectorConfig - ) { - // Initialize form with source properties and default values - const sourceProperties = connector.sourceProperties ?? []; - const initialValues = getInitialFormValuesFromProperties(sourceProperties); - - // Merge with stored connector config - const combinedValues = { ...stepState.connectorConfig, ...initialValues }; - - paramsForm.update(() => combinedValues, { taint: false }); - } + let activeAuthMethod: string | null = null; + let prevAuthMethod: string | null = null; + let stepState = $connectorStepStore; + let multiStepSubmitDisabled = false; + let multiStepButtonLabel = ""; + let multiStepLoadingCopy = ""; + let shouldShowSkipLink = false; + let primaryButtonLabel = ""; + let primaryLoadingCopy = ""; - // Update form when (re)entering step 1: restore defaults for connector properties - $: if (isMultiStepConnector && stepState.step === "connector") { - paramsForm.update( - () => - getInitialFormValuesFromProperties(connector.configProperties ?? []), - { taint: false }, - ); - } + $: stepState = $connectorStepStore; // Form 1: Individual parameters const paramsFormId = formManager.paramsFormId; const properties = formManager.properties; const filteredParamsProperties = formManager.filteredParamsProperties; + let multiStepFormId = paramsFormId; const { form: paramsForm, errors: paramsErrors, @@ -123,15 +112,63 @@ let clickhouseError: string | null = null; let clickhouseErrorDetails: string | undefined = undefined; - let clickhouseFormId: string = ""; - let clickhouseSubmitting: boolean; - let clickhouseIsSubmitDisabled: boolean; let clickhouseConnectorType: ClickHouseConnectorType = "self-hosted"; - let clickhouseParamsForm; - let clickhouseDsnForm; - let clickhouseShowSaveAnyway: boolean = false; + let clickhouseUiState: ClickhouseUiState | null = null; + let clickhouseSaving = false; + let effectiveClickhouseSubmitting = false; + const paramsFormStore = paramsForm as unknown as Writable< + Record + >; + const dsnFormStore = dsnForm as unknown as Writable>; + + const connectorSchema = getConnectorSchema(connector.name ?? ""); + const hasSchema = Boolean(connectorSchema); + + // ClickHouse-specific derived state handled by the manager + $: if (connector.name === "clickhouse") { + clickhouseUiState = formManager.computeClickhouseState({ + connectorType: clickhouseConnectorType, + connectionTab, + paramsFormValues: $paramsForm, + dsnFormValues: $dsnForm, + paramsErrors: $paramsErrors, + dsnErrors: $dsnErrors, + paramsForm, + dsnForm, + paramsSubmitting: $paramsSubmitting, + dsnSubmitting: $dsnSubmitting, + }); + + if ( + clickhouseUiState?.enforcedConnectionTab && + clickhouseUiState.enforcedConnectionTab !== connectionTab + ) { + connectionTab = clickhouseUiState.enforcedConnectionTab; + } + + if (clickhouseUiState?.shouldClearErrors) { + clickhouseError = null; + clickhouseErrorDetails = undefined; + } + } else { + clickhouseUiState = null; + } + + $: effectiveClickhouseSubmitting = + connector.name === "clickhouse" + ? clickhouseSaving || clickhouseUiState?.submitting || false + : submitting; + + // Hide Save Anyway once we advance to the model step in multi-step flows. + $: if (isMultiStepConnector && stepState.step === "source") { + showSaveAnyway = false; + } $: isSubmitDisabled = (() => { + if (isMultiStepConnector) { + return multiStepSubmitDisabled; + } + if (onlyDsn || connectionTab === "dsn") { // DSN form: check required DSN properties for (const property of dsnProperties) { @@ -149,12 +186,7 @@ return false; } else { // Parameters form: check required properties - // Use stepProperties for multi-step connectors, otherwise use properties - const propertiesToCheck = isMultiStepConnector - ? stepProperties - : properties; - - for (const property of propertiesToCheck) { + for (const property of properties) { if (property.required) { const key = String(property.key); const value = $paramsForm[key]; @@ -168,7 +200,9 @@ } })(); - $: formId = formManager.getActiveFormId({ connectionTab, onlyDsn }); + $: formId = isMultiStepConnector + ? multiStepFormId || formManager.getActiveFormId({ connectionTab, onlyDsn }) + : formManager.getActiveFormId({ connectionTab, onlyDsn }); $: submitting = (() => { if (onlyDsn || connectionTab === "dsn") { @@ -178,7 +212,36 @@ } })(); - $: isSubmitting = submitting; + $: primaryButtonLabel = isMultiStepConnector + ? multiStepButtonLabel + : formManager.getPrimaryButtonLabel({ + isConnectorForm, + step: stepState.step, + submitting, + clickhouseConnectorType, + clickhouseSubmitting: effectiveClickhouseSubmitting, + selectedAuthMethod: activeAuthMethod ?? undefined, + }); + + $: primaryLoadingCopy = (() => { + if (connector.name === "clickhouse") return "Connecting..."; + if (isMultiStepConnector) return multiStepLoadingCopy; + return activeAuthMethod === "public" + ? "Continuing..." + : "Testing connection..."; + })(); + + // Clear Save Anyway state whenever auth method changes (any direction). + $: if (activeAuthMethod !== prevAuthMethod) { + prevAuthMethod = activeAuthMethod; + showSaveAnyway = false; + saveAnyway = false; + } + + $: isSubmitting = + connector.name === "clickhouse" + ? effectiveClickhouseSubmitting + : submitting; // Reset errors when form is modified $: (() => { @@ -213,13 +276,13 @@ const values = connector.name === "clickhouse" ? connectionTab === "dsn" - ? $clickhouseDsnForm - : $clickhouseParamsForm + ? $dsnForm + : $paramsForm : onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; if (connector.name === "clickhouse") { - clickhouseSubmitting = true; + clickhouseSaving = true; } const result = await formManager.saveConnectorAnyway({ queryClient, @@ -247,7 +310,7 @@ } saveAnyway = false; if (connector.name === "clickhouse") { - clickhouseSubmitting = false; + clickhouseSaving = false; } } @@ -262,20 +325,20 @@ paramsFormValues: $paramsForm, dsnFormValues: $dsnForm, clickhouseConnectorType, - clickhouseParamsValues: $clickhouseParamsForm, - clickhouseDsnValues: $clickhouseDsnForm, + clickhouseParamsValues: $paramsForm, + clickhouseDsnValues: $dsnForm, }); $: isClickhouse = connector.name === "clickhouse"; - $: shouldShowSaveAnywayButton = - isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway); + $: shouldShowSaveAnywayButton = isConnectorForm && showSaveAnyway; $: saveAnywayLoading = isClickhouse - ? clickhouseSubmitting && saveAnyway + ? effectiveClickhouseSubmitting && saveAnyway : submitting && saveAnyway; handleOnUpdate = formManager.makeOnUpdate({ onClose, queryClient, getConnectionTab: () => connectionTab, + getSelectedAuthMethod: () => activeAuthMethod || undefined, setParamsError: (message: string | null, details?: string) => { paramsError = message; paramsErrorDetails = details; @@ -310,21 +373,21 @@ class="flex flex-col flex-grow {formManager.formHeight} overflow-y-auto p-6" > {#if connector.name === "clickhouse"} - { - clickhouseError = error; - clickhouseErrorDetails = details; - }} - bind:formId={clickhouseFormId} - bind:isSubmitting={clickhouseSubmitting} - bind:isSubmitDisabled={clickhouseIsSubmitDisabled} - bind:connectorType={clickhouseConnectorType} + {:else if hasDsnFormOption} {:else if isMultiStepConnector} - {#if stepState.step === "connector"} - - - - - {:else} - - - - - {/if} + + {:else if hasSchema} + + + {:else} {/if} - {#if isMultiStepConnector && stepState.step === "connector"} - - {/if} - @@ -482,31 +536,45 @@
- {#if dsnError || paramsError || clickhouseError} - + {#if dsnError || paramsError || clickhouseError} + + {/if} + + - {/if} - - + + {#if shouldShowSkipLink} +
+ Already connected? +
+ {/if} +
diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index e2d0f9c4c10..b3b9fd10565 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -1,13 +1,14 @@ import { superForm, defaults } from "sveltekit-superforms"; import type { SuperValidated } from "sveltekit-superforms"; +import * as yupLib from "yup"; import { - yup, + yup as yupAdapter, type Infer as YupInfer, type InferIn as YupInferIn, } from "sveltekit-superforms/adapters"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { AddDataFormType } from "./types"; -import { getValidationSchemaForConnector, dsnSchema } from "./FormValidation"; +import { getValidationSchemaForConnector } from "./FormValidation"; import { getInitialFormValuesFromProperties, inferSourceName, @@ -16,7 +17,11 @@ import { submitAddConnectorForm, submitAddSourceForm, } from "./submitAddDataForm"; -import { normalizeConnectorError } from "./utils"; +import { + normalizeConnectorError, + applyClickHouseCloudRequirements, + isEmpty, +} from "./utils"; import { FORM_HEIGHT_DEFAULT, FORM_HEIGHT_TALL, @@ -27,24 +32,61 @@ import { connectorStepStore, setConnectorConfig, setStep, + type ConnectorStepState, } from "./connectorStepStore"; import { get } from "svelte/store"; import { compileConnectorYAML } from "../../connectors/code-utils"; import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; -import { applyClickHouseCloudRequirements } from "./utils"; import type { ActionResult } from "@sveltejs/kit"; +import { getConnectorSchema } from "./connector-schemas"; +import { findRadioEnumKey } from "../../templates/schema-utils"; + +const dsnSchema = yupLib.object({ + dsn: yupLib.string().required("DSN is required"), +}); + +export type ClickhouseUiState = { + properties: ConnectorDriverProperty[]; + filteredProperties: ConnectorDriverProperty[]; + dsnProperties: ConnectorDriverProperty[]; + isSubmitDisabled: boolean; + formId: string; + submitting: boolean; + enforcedConnectionTab?: "parameters" | "dsn"; + shouldClearErrors?: boolean; +}; + +type SuperFormStore = { + update: ( + updater: (value: Record) => Record, + options?: any, + ) => void; +}; + +type ClickhouseStateArgs = { + connectorType: ClickHouseConnectorType; + connectionTab: "parameters" | "dsn"; + paramsFormValues: Record; + dsnFormValues: Record; + paramsErrors: Record; + dsnErrors: Record; + paramsForm: SuperFormStore; + dsnForm: SuperFormStore; + paramsSubmitting: boolean; + dsnSubmitting: boolean; +}; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { form: SuperValidated, any, Record>; }; -// Shape of the step store for multi-step connectors -type ConnectorStepState = { - step: "connector" | "source"; - connectorConfig: Record | null; +const BUTTON_LABELS = { + public: { idle: "Continue", submitting: "Continuing..." }, + connector: { idle: "Test and Connect", submitting: "Testing connection..." }, + source: { idle: "Import Data", submitting: "Importing data..." }, }; export class AddDataFormManager { @@ -63,21 +105,33 @@ export class AddDataFormManager { dsn: ReturnType; private connector: V1ConnectorDriver; private formType: AddDataFormType; + private clickhouseInitialValues: Record; + private clickhousePrevConnectorType?: ClickHouseConnectorType; // Centralized error normalization for this manager private normalizeError(e: unknown): { message: string; details?: string } { return normalizeConnectorError(this.connector.name ?? "", e); } + private getSelectedAuthMethod?: () => string | undefined; + constructor(args: { connector: V1ConnectorDriver; formType: AddDataFormType; onParamsUpdate: (event: SuperFormUpdateEvent) => void; onDsnUpdate: (event: SuperFormUpdateEvent) => void; + getSelectedAuthMethod?: () => string | undefined; }) { - const { connector, formType, onParamsUpdate, onDsnUpdate } = args; + const { + connector, + formType, + onParamsUpdate, + onDsnUpdate, + getSelectedAuthMethod, + } = args; this.connector = connector; this.formType = formType; + this.getSelectedAuthMethod = getSelectedAuthMethod; // Layout height this.formHeight = TALL_FORM_CONNECTORS.has(connector.name ?? "") @@ -125,12 +179,12 @@ export class AddDataFormManager { ); // Superforms: params - const paramsSchemaDef = getValidationSchemaForConnector( + const paramsAdapter = getValidationSchemaForConnector( connector.name as string, + formType, ); - const paramsAdapter = yup(paramsSchemaDef); - type ParamsOut = YupInfer; - type ParamsIn = YupInferIn; + type ParamsOut = Record; + type ParamsIn = Record; const initialFormValues = getInitialFormValuesFromProperties( this.properties, ); @@ -143,10 +197,11 @@ export class AddDataFormManager { validators: paramsAdapter, onUpdate: onParamsUpdate, resetForm: false, + validationMethod: "onsubmit", }); // Superforms: dsn - const dsnAdapter = yup(dsnSchema); + const dsnAdapter = yupAdapter(dsnSchema); type DsnOut = YupInfer; type DsnIn = YupInferIn; this.dsn = superForm(defaults(dsnAdapter), { @@ -154,7 +209,18 @@ export class AddDataFormManager { validators: dsnAdapter, onUpdate: onDsnUpdate, resetForm: false, + validationMethod: "onsubmit", }); + + // ClickHouse-specific defaults + this.clickhouseInitialValues = + connector.name === "clickhouse" + ? getInitialFormValuesFromProperties(connector.configProperties ?? []) + : {}; + this.clickhousePrevConnectorType = + connector.name === "clickhouse" + ? ("self-hosted" as ClickHouseConnectorType) + : undefined; } get isSourceForm(): boolean { @@ -169,6 +235,37 @@ export class AddDataFormManager { return MULTI_STEP_CONNECTORS.includes(this.connector.name ?? ""); } + /** + * Determines whether the "Save Anyway" button should be shown for the current submission. + */ + private shouldShowSaveAnywayButton(args: { + isConnectorForm: boolean; + event?: + | { + result?: Extract; + } + | undefined; + stepState: ConnectorStepState | undefined; + selectedAuthMethod?: string; + }): boolean { + const { isConnectorForm, event, stepState, selectedAuthMethod } = args; + + // Only show for connector forms (not sources) + if (!isConnectorForm) return false; + + // Need a submission result to show the button + if (!event?.result) return false; + + // Multi-step connectors: don't show on source step (final step) + if (stepState?.step === "source") return false; + + // Public auth bypasses connection test, so no "Save Anyway" needed + if (stepState?.step === "connector" && selectedAuthMethod === "public") + return false; + + return true; + } + getActiveFormId(args: { connectionTab: "parameters" | "dsn"; onlyDsn: boolean; @@ -182,7 +279,7 @@ export class AddDataFormManager { handleSkip(): void { const stepState = get(connectorStepStore) as ConnectorStepState; if (!this.isMultiStepConnector || stepState.step !== "connector") return; - setConnectorConfig(get(this.params.form) as Record); + setConnectorConfig({}); setStep("source"); } @@ -201,6 +298,7 @@ export class AddDataFormManager { submitting: boolean; clickhouseConnectorType?: ClickHouseConnectorType; clickhouseSubmitting?: boolean; + selectedAuthMethod?: string; }): string { const { isConnectorForm, @@ -208,6 +306,7 @@ export class AddDataFormManager { submitting, clickhouseConnectorType, clickhouseSubmitting, + selectedAuthMethod, } = args; const isClickhouse = this.connector.name === "clickhouse"; @@ -222,21 +321,172 @@ export class AddDataFormManager { if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { - return submitting ? "Testing connection..." : "Test and Connect"; + if (selectedAuthMethod === "public") { + return submitting + ? BUTTON_LABELS.public.submitting + : BUTTON_LABELS.public.idle; + } + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } if (this.isMultiStepConnector && step === "source") { - return submitting ? "Creating model..." : "Test and Add data"; + return submitting + ? BUTTON_LABELS.source.submitting + : BUTTON_LABELS.source.idle; } - return submitting ? "Testing connection..." : "Test and Connect"; + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } return "Test and Add data"; } + computeClickhouseState(args: ClickhouseStateArgs): ClickhouseUiState | null { + if (this.connector.name !== "clickhouse") return null; + const { + connectorType, + connectionTab, + paramsFormValues, + dsnFormValues, + paramsErrors, + dsnErrors, + paramsForm, + paramsSubmitting, + dsnSubmitting, + } = args; + + // Keep connector_type in sync on the params form + paramsForm.update( + ($form: any) => ({ + ...$form, + connector_type: connectorType, + }), + { taint: false } as any, + ); + + // Apply defaults when the ClickHouse connector type changes + if ( + connectorType === "rill-managed" && + Object.keys(paramsFormValues ?? {}).length > 1 + ) { + paramsForm.update( + () => ({ managed: true, connector_type: "rill-managed" }), + { taint: false } as any, + ); + } else if ( + this.clickhousePrevConnectorType === "rill-managed" && + connectorType === "self-hosted" + ) { + paramsForm.update( + () => ({ ...this.clickhouseInitialValues, managed: false }), + { taint: false } as any, + ); + } else if ( + this.clickhousePrevConnectorType !== "clickhouse-cloud" && + connectorType === "clickhouse-cloud" + ) { + paramsForm.update( + () => ({ + ...this.clickhouseInitialValues, + managed: false, + port: "8443", + ssl: true, + }), + { taint: false } as any, + ); + } else if ( + this.clickhousePrevConnectorType === "clickhouse-cloud" && + connectorType === "self-hosted" + ) { + paramsForm.update( + () => ({ ...this.clickhouseInitialValues, managed: false }), + { taint: false } as any, + ); + } + this.clickhousePrevConnectorType = connectorType; + + const enforcedConnectionTab = + connectorType === "rill-managed" ? ("parameters" as const) : undefined; + const activeConnectionTab = enforcedConnectionTab ?? connectionTab; + + const properties = + connectorType === "rill-managed" + ? (this.connector.sourceProperties ?? []) + : (this.connector.configProperties ?? []); + + const filteredProperties = properties.filter( + (property) => + !property.noPrompt && + property.key !== "managed" && + (activeConnectionTab !== "dsn" ? property.key !== "dsn" : true), + ); + + const dsnProperties = + this.connector.configProperties?.filter( + (property) => property.key === "dsn", + ) ?? []; + + const isSubmitDisabled = (() => { + if (connectorType === "rill-managed") { + for (const property of filteredProperties) { + if (property.required) { + const key = String(property.key); + const value = paramsFormValues?.[key]; + if (isEmpty(value) || (paramsErrors?.[key] as any)?.length) + return true; + } + } + return false; + } + if (activeConnectionTab === "dsn") { + for (const property of dsnProperties) { + if (property.required) { + const key = String(property.key); + const value = dsnFormValues?.[key]; + if (isEmpty(value) || (dsnErrors?.[key] as any)?.length) + return true; + } + } + return false; + } + + for (const property of filteredProperties) { + if (property.required && property.key !== "managed") { + const key = String(property.key); + const value = paramsFormValues?.[key]; + if (isEmpty(value) || (paramsErrors?.[key] as any)?.length) + return true; + } + } + if (connectorType === "clickhouse-cloud" && !paramsFormValues?.ssl) + return true; + return false; + })(); + + const submitting = + activeConnectionTab === "dsn" ? dsnSubmitting : paramsSubmitting; + const formId = + activeConnectionTab === "dsn" ? this.dsnFormId : this.paramsFormId; + + return { + properties, + filteredProperties, + dsnProperties, + isSubmitDisabled, + formId, + submitting, + enforcedConnectionTab, + shouldClearErrors: connectorType === "rill-managed", + }; + } + makeOnUpdate(args: { onClose: () => void; queryClient: any; getConnectionTab: () => "parameters" | "dsn"; + getSelectedAuthMethod?: () => string | undefined; setParamsError: (message: string | null, details?: string) => void; setDsnError: (message: string | null, details?: string) => void; setShowSaveAnyway?: (value: boolean) => void; @@ -245,6 +495,7 @@ export class AddDataFormManager { onClose, queryClient, getConnectionTab, + getSelectedAuthMethod, setParamsError, setDsnError, setShowSaveAnyway, @@ -262,27 +513,76 @@ export class AddDataFormManager { Record >; result?: Extract; + cancel?: () => void; }) => { - // For non-ClickHouse connectors, expose Save Anyway when a submission starts + const values = event.form.data; + const schema = getConnectorSchema(this.connector.name ?? ""); + const authKey = schema ? findRadioEnumKey(schema) : null; + const selectedAuthMethod = + (authKey && values && values[authKey] != null + ? String(values[authKey]) + : undefined) || + getSelectedAuthMethod?.() || + ""; + const stepState = get(connectorStepStore) as ConnectorStepState; + + // Fast-path: public auth skips validation/test and goes straight to source step. if ( - isConnectorForm && - connector.name !== "clickhouse" && - typeof setShowSaveAnyway === "function" && - event?.result + isMultiStepConnector && + stepState.step === "connector" && + selectedAuthMethod === "public" ) { - setShowSaveAnyway(true); + setConnectorConfig(values); + setStep("source"); + return; } - if (!event.form.valid) return; + if (isMultiStepConnector && stepState.step === "source") { + const sourceValidator = getValidationSchemaForConnector( + connector.name as string, + "source", + ); + const result = await sourceValidator.validate(values); + if (!result.success) { + const fieldErrors: Record = {}; + for (const issue of result.issues ?? []) { + const key = + issue.path?.[0] != null ? String(issue.path[0]) : "_errors"; + if (!fieldErrors[key]) fieldErrors[key] = []; + fieldErrors[key].push(issue.message); + } + (this.params.errors as any).set(fieldErrors); + event.cancel?.(); + return; + } + (this.params.errors as any).set({}); + } else if (!event.form.valid) { + return; + } - const values = event.form.data; + if ( + typeof setShowSaveAnyway === "function" && + this.shouldShowSaveAnywayButton({ + isConnectorForm, + event, + stepState, + selectedAuthMethod, + }) + ) { + setShowSaveAnyway(true); + } try { - const stepState = get(connectorStepStore) as ConnectorStepState; if (isMultiStepConnector && stepState.step === "source") { await submitAddSourceForm(queryClient, connector, values); onClose(); } else if (isMultiStepConnector && stepState.step === "connector") { + // For public auth, skip Test & Connect and go straight to the next step. + if (selectedAuthMethod === "public") { + setConnectorConfig(values); + setStep("source"); + return; + } await submitAddConnectorForm(queryClient, connector, values, false); setConnectorConfig(values); setStep("source"); @@ -392,16 +692,22 @@ export class AddDataFormManager { clickhouseDsnValues, } = ctx; + const connectorPropertiesForPreview = + isMultiStepConnector && stepState?.step === "connector" + ? (connector.configProperties ?? []) + : filteredParamsProperties; + const getConnectorYamlPreview = (values: Record) => { + const orderedProperties = + onlyDsn || connectionTab === "dsn" + ? filteredDsnProperties + : connectorPropertiesForPreview; return compileConnectorYAML(connector, values, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; return !property.noPrompt; }, - orderedProperties: - onlyDsn || connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, + orderedProperties, }); }; diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 42c8289d1b1..e2dd69ade1e 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -71,6 +71,9 @@ }); function goToConnectorForm(connector: V1ConnectorDriver) { + // Reset multi-step state (auth selection, connector config) when switching connectors. + resetConnectorStep(); + const state = { step: 2, selectedConnector: connector, @@ -121,7 +124,7 @@ // FIXME: excluding salesforce until we implement the table discovery APIs $: isConnectorType = - selectedConnector?.name === "gcs" || + selectedConnector?.implementsObjectStore || selectedConnector?.implementsOlap || selectedConnector?.implementsSqlStore || (selectedConnector?.implementsWarehouse && diff --git a/web-common/src/features/sources/modal/ClickhouseFormRenderer.svelte b/web-common/src/features/sources/modal/ClickhouseFormRenderer.svelte new file mode 100644 index 00000000000..e78df906d5c --- /dev/null +++ b/web-common/src/features/sources/modal/ClickhouseFormRenderer.svelte @@ -0,0 +1,176 @@ + + +
+
+ + {#if clickhouseConnectorType === "rill-managed"} +
+ +
+ {/if} +
+ + {#if clickhouseConnectorType === "self-hosted" || clickhouseConnectorType === "clickhouse-cloud"} + + + + {#each clickhouseUiState?.filteredProperties ?? [] as property (property.key)} + {@const propertyKey = property.key ?? ""} + {@const isPortField = propertyKey === "port"} + {@const isSSLField = propertyKey === "ssl"} + +
+ {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} + onStringInputChange(e)} + alwaysShowError + options={clickhouseConnectorType === "clickhouse-cloud" && + isPortField + ? [ + { value: "8443", label: "8443 (HTTPS)" }, + { value: "9440", label: "9440 (Native Secure)" }, + ] + : undefined} + /> + {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} + + {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} + + {/if} +
+ {/each} + + + + + {#each clickhouseUiState?.dsnProperties ?? [] as property (property.key)} + {@const propertyKey = property.key ?? ""} +
+ +
+ {/each} + + + + {:else} + + {#each clickhouseUiState?.filteredProperties ?? [] as property (property.key)} + {@const propertyKey = property.key ?? ""} +
+ {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} + onStringInputChange(e)} + alwaysShowError + /> + {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} + + {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} + + {/if} +
+ {/each} + + {/if} +
diff --git a/web-common/src/features/sources/modal/FormRenderer.svelte b/web-common/src/features/sources/modal/FormRenderer.svelte index a114345627f..dc33afc5fff 100644 --- a/web-common/src/features/sources/modal/FormRenderer.svelte +++ b/web-common/src/features/sources/modal/FormRenderer.svelte @@ -7,7 +7,7 @@ ConnectorDriverPropertyType, type ConnectorDriverProperty, } from "@rilldata/web-common/runtime-client"; - import { normalizeErrors } from "./utils"; + import { normalizeErrors } from "../../templates/error-utils"; export let properties: Array = []; export let form: any; // expect a store from parent diff --git a/web-common/src/features/sources/modal/FormValidation.test.ts b/web-common/src/features/sources/modal/FormValidation.test.ts new file mode 100644 index 00000000000..722052755c4 --- /dev/null +++ b/web-common/src/features/sources/modal/FormValidation.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from "vitest"; + +import { getValidationSchemaForConnector } from "./FormValidation"; + +describe("getValidationSchemaForConnector (multi-step auth)", () => { + it("enforces required fields for access key auth", async () => { + const schema = getValidationSchemaForConnector("s3", "connector"); + + const result = await schema.validate({}); + expect(result.success).toBe(false); + if (result.success) throw new Error("expected validation to fail"); + expect(result.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: ["aws_access_key_id"] }), + expect.objectContaining({ path: ["aws_secret_access_key"] }), + ]), + ); + }); + + it("allows public auth without credentials", async () => { + const schema = getValidationSchemaForConnector("s3", "connector"); + + const result = await schema.validate({ auth_method: "public" }); + expect(result.success).toBe(true); + }); + + it("requires source fields from JSON schema for multi-step connectors", async () => { + const schema = getValidationSchemaForConnector("s3", "source"); + + const result = await schema.validate({}); + expect(result.success).toBe(false); + if (result.success) throw new Error("expected validation to fail"); + expect(result.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: ["path"] }), + expect.objectContaining({ path: ["name"] }), + ]), + ); + }); + + it("rejects invalid s3 path on source step", async () => { + const schema = getValidationSchemaForConnector("s3", "source"); + + const result = await schema.validate({ + path: "s3:/bucket", + name: "valid_name", + }); + expect(result.success).toBe(false); + if (result.success) throw new Error("expected validation to fail"); + expect(result.issues).toEqual( + expect.arrayContaining([expect.objectContaining({ path: ["path"] })]), + ); + }); + + it("accepts valid s3 path on source step", async () => { + const schema = getValidationSchemaForConnector("s3", "source"); + + const result = await schema.validate({ + path: "s3://bucket/prefix", + name: "valid_name", + }); + expect(result.success).toBe(true); + }); +}); diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 92c0f87c338..d1ed1a07bf1 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,16 @@ -import { dsnSchema, getYupSchema } from "./yupSchemas"; +import type { ValidationAdapter } from "sveltekit-superforms/adapters"; -export { dsnSchema }; +import { createSchemasafeValidator } from "./jsonSchemaValidator"; +import { getConnectorSchema } from "./connector-schemas"; +import type { AddDataFormType } from "./types"; -export function getValidationSchemaForConnector(name: string) { - return getYupSchema[name as keyof typeof getYupSchema]; +export function getValidationSchemaForConnector( + name: string, + formType: AddDataFormType, +): ValidationAdapter> { + const jsonSchema = getConnectorSchema(name); + const step = formType === "source" ? "source" : "connector"; + + if (jsonSchema) return createSchemasafeValidator(jsonSchema, step); + throw new Error(`No validation schema found for connector: ${name}`); } diff --git a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte deleted file mode 100644 index e007489218a..00000000000 --- a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte +++ /dev/null @@ -1,129 +0,0 @@ - - - -
-
-
Authentication method
- - - {#if option.value === "credentials"} - - {:else if option.value === "hmac"} -
- - -
- {/if} - - -
- - - {#each filteredParamsProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {#if propertyKey !== "path" && propertyKey !== "google_application_credentials" && propertyKey !== "key_id" && propertyKey !== "secret"} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/if} - {/each} -
diff --git a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte new file mode 100644 index 00000000000..1eda032d84d --- /dev/null +++ b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte @@ -0,0 +1,174 @@ + + + + + diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts new file mode 100644 index 00000000000..0e2346fbbd5 --- /dev/null +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -0,0 +1,49 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; +import { athenaSchema } from "../../templates/schemas/athena"; +import { azureSchema } from "../../templates/schemas/azure"; +import { bigquerySchema } from "../../templates/schemas/bigquery"; +import { clickhouseSchema } from "../../templates/schemas/clickhouse"; +import { gcsSchema } from "../../templates/schemas/gcs"; +import { mysqlSchema } from "../../templates/schemas/mysql"; +import { postgresSchema } from "../../templates/schemas/postgres"; +import { redshiftSchema } from "../../templates/schemas/redshift"; +import { salesforceSchema } from "../../templates/schemas/salesforce"; +import { snowflakeSchema } from "../../templates/schemas/snowflake"; +import { sqliteSchema } from "../../templates/schemas/sqlite"; +import { localFileSchema } from "../../templates/schemas/local_file"; +import { duckdbSchema } from "../../templates/schemas/duckdb"; +import { httpsSchema } from "../../templates/schemas/https"; +import { motherduckSchema } from "../../templates/schemas/motherduck"; +import { druidSchema } from "../../templates/schemas/druid"; +import { pinotSchema } from "../../templates/schemas/pinot"; +import { s3Schema } from "../../templates/schemas/s3"; + +export const multiStepFormSchemas: Record = { + athena: athenaSchema, + bigquery: bigquerySchema, + clickhouse: clickhouseSchema, + mysql: mysqlSchema, + postgres: postgresSchema, + redshift: redshiftSchema, + salesforce: salesforceSchema, + snowflake: snowflakeSchema, + sqlite: sqliteSchema, + motherduck: motherduckSchema, + duckdb: duckdbSchema, + druid: druidSchema, + pinot: pinotSchema, + local_file: localFileSchema, + https: httpsSchema, + s3: s3Schema, + gcs: gcsSchema, + azure: azureSchema, +}; + +export function getConnectorSchema( + connectorName: string, +): MultiStepFormSchema | null { + const schema = + multiStepFormSchemas[connectorName as keyof typeof multiStepFormSchemas]; + if (!schema?.properties) return null; + return schema; +} diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 9ce2451f126..1168a65e532 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -2,12 +2,16 @@ import { writable } from "svelte/store"; export type ConnectorStep = "connector" | "source"; -export const connectorStepStore = writable<{ +export type ConnectorStepState = { step: ConnectorStep; connectorConfig: Record | null; -}>({ + selectedAuthMethod: string | null; +}; + +export const connectorStepStore = writable({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); export function setStep(step: ConnectorStep) { @@ -18,9 +22,17 @@ export function setConnectorConfig(config: Record) { connectorStepStore.update((state) => ({ ...state, connectorConfig: config })); } +export function setAuthMethod(method: string | null) { + connectorStepStore.update((state) => ({ + ...state, + selectedAuthMethod: method, + })); +} + export function resetConnectorStep() { connectorStepStore.set({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); } diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 4bffc52825d..8e02f8a7b44 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -17,7 +17,7 @@ export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "dsn", label: "Enter connection string" }, ]; -export type GCSAuthMethod = "credentials" | "hmac"; +export type GCSAuthMethod = "public" | "credentials" | "hmac"; export const GCS_AUTH_OPTIONS: { value: GCSAuthMethod; @@ -37,6 +37,65 @@ export const GCS_AUTH_OPTIONS: { description: "Use HMAC access key and secret for S3-compatible authentication.", }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, +]; + +export type S3AuthMethod = "access_keys" | "public"; + +export const S3_AUTH_OPTIONS: { + value: S3AuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "access_keys", + label: "Access keys", + description: "Use AWS access key ID and secret access key.", + }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, +]; + +export type AzureAuthMethod = + | "account_key" + | "sas_token" + | "connection_string" + | "public"; + +export const AZURE_AUTH_OPTIONS: { + value: AzureAuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "connection_string", + label: "Connection String", + description: "Alternative for cloud deployment", + }, + { + value: "account_key", + label: "Storage Account Key", + description: "Recommended for cloud deployment", + }, + { + value: "sas_token", + label: "Shared Access Signature (SAS) Token", + description: "Most secure, fine-grained control", + }, + { + value: "public", + label: "Public", + description: "Access publicly readable blobs without credentials.", + }, ]; // pre-defined order for sources @@ -67,7 +126,7 @@ export const OLAP_ENGINES = [ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) -export const MULTI_STEP_CONNECTORS = ["gcs"]; +export const MULTI_STEP_CONNECTORS = ["gcs", "s3", "azure"]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; diff --git a/web-common/src/features/sources/modal/jsonSchemaValidator.ts b/web-common/src/features/sources/modal/jsonSchemaValidator.ts new file mode 100644 index 00000000000..1804f0713b4 --- /dev/null +++ b/web-common/src/features/sources/modal/jsonSchemaValidator.ts @@ -0,0 +1,193 @@ +import type { ValidatorOptions, ValidationError } from "@exodus/schemasafe"; +import { validator as compileValidator } from "@exodus/schemasafe"; +import { schemasafe } from "sveltekit-superforms/adapters"; +import type { ValidationAdapter } from "sveltekit-superforms/adapters"; + +import { getFieldLabel, isStepMatch } from "../../templates/schema-utils"; +import type { + JSONSchemaConditional, + JSONSchemaConstraint, + MultiStepFormSchema, +} from "../../templates/schemas/types"; + +const DEFAULT_SCHEMASAFE_OPTIONS: ValidatorOptions = { + includeErrors: true, + allErrors: true, + allowUnusedKeywords: true, + formats: { + uri: (value: string) => { + if (typeof value !== "string") return false; + try { + // Allow custom schemes such as s3:// or gs:// + new URL(value); + return true; + } catch { + return false; + } + }, + // We treat file inputs as strings; superforms handles the upload. + file: () => true, + }, +}; + +type Step = "connector" | "source" | string | undefined; + +export function buildStepSchema( + schema: MultiStepFormSchema, + step: Step, +): MultiStepFormSchema { + const properties = Object.entries(schema.properties ?? {}).reduce< + NonNullable + >((acc, [key, prop]) => { + if (!isStepMatch(schema, key, step)) return acc; + acc[key] = prop; + return acc; + }, {}); + + const required = (schema.required ?? []).filter((key) => + isStepMatch(schema, key, step), + ); + + const filteredAllOf = (schema.allOf ?? []) + .map((conditional) => filterConditional(conditional, schema, step)) + .filter((conditional): conditional is JSONSchemaConditional => + Boolean( + conditional && + ((conditional.then?.required?.length ?? 0) > 0 || + (conditional.else?.required?.length ?? 0) > 0), + ), + ); + + return { + $schema: schema.$schema, + type: "object", + properties, + ...(required.length ? { required } : {}), + ...(filteredAllOf.length ? { allOf: filteredAllOf } : {}), + }; +} + +export function createSchemasafeValidator( + schema: MultiStepFormSchema, + step: Step, + opts?: { config?: ValidatorOptions }, +): ValidationAdapter> { + const stepSchema = buildStepSchema(schema, step); + const validator = compileValidator(stepSchema, { + ...DEFAULT_SCHEMASAFE_OPTIONS, + ...opts?.config, + }); + + const baseAdapter = schemasafe(stepSchema, { + config: { + ...DEFAULT_SCHEMASAFE_OPTIONS, + ...opts?.config, + }, + }); + + return { + ...baseAdapter, + async validate(data: Record = {}) { + const pruned = pruneEmptyFields(data); + const isValid = validator(pruned as any); + if (isValid) { + return { data: pruned, success: true }; + } + + const issues = (validator.errors ?? []).map((error) => + toIssue(error, schema), + ); + return { success: false, issues }; + }, + }; +} + +function pruneEmptyFields( + values: Record, +): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(values ?? {})) { + if (value === "" || value === null || value === undefined) continue; + result[key] = value; + } + return result; +} + +function filterConditional( + conditional: JSONSchemaConditional, + schema: MultiStepFormSchema, + step: Step, +): JSONSchemaConditional | null { + const thenRequired = filterRequired(conditional.then, schema, step); + const elseRequired = filterRequired(conditional.else, schema, step); + + if (!thenRequired.length && !elseRequired.length) return null; + + return { + if: conditional.if, + then: thenRequired.length ? { required: thenRequired } : undefined, + else: elseRequired.length ? { required: elseRequired } : undefined, + }; +} + +function filterRequired( + constraint: JSONSchemaConstraint | undefined, + schema: MultiStepFormSchema, + step: Step, +): string[] { + return (constraint?.required ?? []).filter((key) => + isStepMatch(schema, key, step), + ); +} + +function toIssue(error: ValidationError, schema: MultiStepFormSchema) { + const pathSegments = parseInstanceLocation(error.instanceLocation); + const key = pathSegments[0]; + return { + path: pathSegments, + message: buildMessage(schema, key, error), + }; +} + +function buildMessage( + schema: MultiStepFormSchema, + key: string | undefined, + error: ValidationError, +): string { + if (!key) return "Invalid value"; + const prop = schema.properties?.[key] as any; + const label = getFieldLabel(schema, key); + const keyword = parseKeyword(error.keywordLocation); + + if (keyword === "required") return `${label} is required`; + + if (keyword === "pattern") { + const custom = prop?.errorMessage?.pattern as string | undefined; + return custom || `${label} is invalid`; + } + + if (keyword === "format") { + const custom = prop?.errorMessage?.format as string | undefined; + const format = prop?.format as string | undefined; + if (custom) return custom; + if (format) return `${label} must be a valid ${format}`; + return `${label} is invalid`; + } + + if (keyword === "type" && prop?.type) { + return `${label} must be a ${prop.type}`; + } + + return `${label} is invalid`; +} + +function parseInstanceLocation(location: string): string[] { + if (!location || location === "#") return []; + return location.replace(/^#\//, "").split("/").filter(Boolean); +} + +function parseKeyword(location: string): string { + if (!location) return ""; + const parts = location.split("/"); + return parts[parts.length - 1] || ""; +} diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index c6ec540ad6e..d9bdbd6f332 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -154,88 +154,6 @@ async function getOriginalEnvBlob( } } -export async function submitAddSourceForm( - queryClient: QueryClient, - connector: V1ConnectorDriver, - formValues: AddDataFormValues, -): Promise { - const instanceId = get(runtime).instanceId; - await beforeSubmitForm(instanceId, connector); - - const newSourceName = formValues.name as string; - - const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( - connector, - formValues, - ); - - // Make a new .yaml file - const newSourceFilePath = getFileAPIPathFromNameAndType( - newSourceName, - EntityType.Table, - ); - await runtimeServicePutFile(instanceId, { - path: newSourceFilePath, - blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), - create: true, - createOnly: false, - }); - - const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); - - // Create or update the `.env` file - const newEnvBlob = await updateDotEnvWithSecrets( - queryClient, - rewrittenConnector, - rewrittenFormValues, - "source", - ); - - // Make sure the file has reconciled before testing the connection - await runtimeServicePutFileAndWaitForReconciliation(instanceId, { - path: ".env", - blob: newEnvBlob, - create: true, - createOnly: false, - }); - - // Wait for source resource-level reconciliation - // This must happen after .env reconciliation since sources depend on secrets - try { - await waitForResourceReconciliation( - instanceId, - newSourceName, - ResourceKind.Model, - ); - } catch (error) { - // The source file was already created, so we need to delete it - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - const errorDetails = (error as any).details; - - throw { - message: error.message || "Unable to establish a connection", - details: - errorDetails && errorDetails !== error.message - ? errorDetails - : undefined, - }; - } - - // Check for file errors - // If the model file has errors, rollback the changes - const errorMessage = await fileArtifacts.checkFileErrors( - queryClient, - instanceId, - newSourceFilePath, - ); - if (errorMessage) { - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - throw new Error(errorMessage); - } - - await goto(`/files/${newSourceFilePath}`); -} - async function saveConnectorAnyway( queryClient: QueryClient, connector: V1ConnectorDriver, @@ -489,3 +407,85 @@ export async function submitAddConnectorForm( // Wait for the submission to complete await submissionPromise; } + +export async function submitAddSourceForm( + queryClient: QueryClient, + connector: V1ConnectorDriver, + formValues: AddDataFormValues, +): Promise { + const instanceId = get(runtime).instanceId; + await beforeSubmitForm(instanceId, connector); + + const newSourceName = formValues.name as string; + + const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( + connector, + formValues, + ); + + // Make a new .yaml file + const newSourceFilePath = getFileAPIPathFromNameAndType( + newSourceName, + EntityType.Table, + ); + await runtimeServicePutFile(instanceId, { + path: newSourceFilePath, + blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), + create: true, + createOnly: false, + }); + + const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); + + // Create or update the `.env` file + const newEnvBlob = await updateDotEnvWithSecrets( + queryClient, + rewrittenConnector, + rewrittenFormValues, + "source", + ); + + // Make sure the file has reconciled before testing the connection + await runtimeServicePutFileAndWaitForReconciliation(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); + + // Wait for source resource-level reconciliation + // This must happen after .env reconciliation since sources depend on secrets + try { + await waitForResourceReconciliation( + instanceId, + newSourceName, + ResourceKind.Model, + ); + } catch (error) { + // The source file was already created, so we need to delete it + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + const errorDetails = (error as any).details; + + throw { + message: error.message || "Unable to establish a connection", + details: + errorDetails && errorDetails !== error.message + ? errorDetails + : undefined, + }; + } + + // Check for file errors + // If the model file has errors, rollback the changes + const errorMessage = await fileArtifacts.checkFileErrors( + queryClient, + instanceId, + newSourceFilePath, + ); + if (errorMessage) { + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + throw new Error(errorMessage); + } + + await goto(`/files/${newSourceFilePath}`); +} diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index 5cb0785a5a5..8fd405fe6de 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -1,3 +1,51 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; + +export type { + JSONSchemaCondition, + JSONSchemaConditional, + JSONSchemaConstraint, + JSONSchemaField, + JSONSchemaObject, + MultiStepFormSchema, +} from "../../templates/schemas/types"; + export type AddDataFormType = "source" | "connector"; export type ConnectorType = "parameters" | "dsn"; + +export type AuthOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export type AuthField = + | { + type: "credentials"; + id: string; + hint?: string; + optional?: boolean; + accept?: string; + } + | { + type: "input"; + id: string; + label: string; + placeholder?: string; + optional?: boolean; + secret?: boolean; + hint?: string; + }; + +export type MultiStepFormConfig = { + schema: MultiStepFormSchema; + authMethodKey: string; + authOptions: AuthOption[]; + clearFieldsByMethod: Record; + excludedKeys: string[]; + authFieldGroups: Record; + requiredFieldsByMethod: Record; + fieldLabels: Record; + defaultAuthMethod?: string; +}; diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 5a7128802d2..6bd2b9106f4 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,6 +1,13 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; +import type { MultiStepFormSchema } from "./types"; +import { + findRadioEnumKey, + getRadioEnumOptions, + getRequiredFieldsByEnumValue, + isStepMatch, +} from "../../templates/schema-utils"; /** * Returns true for undefined, null, empty string, or whitespace-only string. @@ -22,16 +29,6 @@ export function isEmpty(val: any) { * - If input resembles a Zod `_errors` array, returns that. * - Otherwise returns undefined. */ -export function normalizeErrors( - err: any, -): string | string[] | null | undefined { - if (!err) return undefined; - if (Array.isArray(err)) return err; - if (typeof err === "string") return err; - if (err._errors && Array.isArray(err._errors)) return err._errors; - return undefined; -} - /** * Converts unknown error inputs into a unified connector error shape. * - Prefers native Error.message when present @@ -83,6 +80,104 @@ export function hasOnlyDsn( return hasDsn && !hasOthers; } +/** + * Returns true when the active multi-step auth method has missing or invalid + * required fields. Falls back to configured default/first auth method. + */ +export function isMultiStepConnectorDisabled( + schema: MultiStepFormSchema | null, + paramsFormValue: Record, + paramsFormErrors: Record, + step?: "connector" | "source" | string, +) { + if (!schema) return true; + + // For source step, gate on required fields from the JSON schema. + const currentStep = step || (paramsFormValue?.__step as string | undefined); + if (currentStep === "source") { + const required = getRequiredFieldsForStep( + schema, + paramsFormValue, + "source", + ); + if (!required.length) return false; + return !required.every((fieldId) => { + if (!isStepMatch(schema, fieldId, "source")) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); + } + + const authInfo = getRadioEnumOptions(schema); + const options = authInfo?.options ?? []; + const authKey = authInfo?.key || findRadioEnumKey(schema); + const methodFromForm = + authKey && paramsFormValue?.[authKey] != null + ? String(paramsFormValue[authKey]) + : undefined; + const hasValidFormSelection = options.some( + (opt) => opt.value === methodFromForm, + ); + const method = + (hasValidFormSelection && methodFromForm) || + authInfo?.defaultValue || + options[0]?.value; + + if (!method) return true; + + // Selecting "public" should always enable the button for multi-step auth flows. + if (method === "public") return false; + + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { + step: "connector", + }); + const requiredFields = requiredByMethod[method] ?? []; + if (!requiredFields.length) return true; + + return !requiredFields.every((fieldId) => { + if (!isStepMatch(schema, fieldId, "connector")) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); +} + +function getRequiredFieldsForStep( + schema: MultiStepFormSchema, + values: Record, + step: "connector" | "source" | string, +) { + const required = new Set(); + (schema.required ?? []).forEach((key) => { + if (isStepMatch(schema, key, step)) required.add(key); + }); + + for (const conditional of schema.allOf ?? []) { + const condition = conditional.if?.properties; + const matches = matchesCondition(condition, values); + const branch = matches ? conditional.then : conditional.else; + branch?.required?.forEach((key) => { + if (isStepMatch(schema, key, step)) required.add(key); + }); + } + + return Array.from(required); +} + +function matchesCondition( + condition: Record | undefined, + values: Record, +) { + if (!condition || !Object.keys(condition).length) return false; + return Object.entries(condition).every(([depKey, def]) => { + if (def.const === undefined || def.const === null) return false; + return String(values?.[depKey]) === String(def.const); + }); +} + /** * Applies ClickHouse Cloud-specific default requirements for connector values. * - For ClickHouse Cloud: enforces `ssl: true` diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts deleted file mode 100644 index b12ec332eae..00000000000 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ /dev/null @@ -1,222 +0,0 @@ -import * as yup from "yup"; -import { - INVALID_NAME_MESSAGE, - VALID_NAME_PATTERN, -} from "../../entity-management/name-utils"; - -export const getYupSchema = { - s3: yup.object().shape({ - path: yup - .string() - .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") - .required("S3 URI is required"), - aws_region: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - gcs: yup.object().shape({ - google_application_credentials: yup.string().optional(), - key_id: yup.string().optional(), - secret: yup.string().optional(), - path: yup - .string() - .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") - .optional(), - }), - - https: yup.object().shape({ - path: yup - .string() - .matches(/^https?:\/\//, 'Path must start with "http(s)://"') - .required("Path is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - duckdb: yup.object().shape({ - path: yup.string().required("path is required"), - attach: yup.string().optional(), - }), - - motherduck: yup.object().shape({ - token: yup.string().required("Token is required"), - path: yup.string().required("Path is required"), - schema_name: yup.string().required("Schema name is required"), - }), - - sqlite: yup.object().shape({ - db: yup.string().required("db is required"), - table: yup.string().required("table is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - bigquery: yup.object().shape({ - project_id: yup.string(), - google_application_credentials: yup - .string() - .required("Google application credentials is required"), - }), - - azure: yup.object().shape({ - path: yup - .string() - .matches( - /^azure:\/\//, - "Must be an Azure URI (e.g. azure://container/path)", - ) - .required("Path is required"), - azure_storage_account: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - postgres: yup.object().shape({ - dsn: yup.string().optional(), - host: yup.string().optional(), - port: yup.string().optional(), - user: yup.string().optional(), - password: yup.string().optional(), - dbname: yup.string().optional(), - sslmode: yup.string().optional(), - }), - - snowflake: yup.object().shape({ - dsn: yup.string().optional(), - account: yup.string().required("Account is required"), - user: yup.string().required("Username is required"), - password: yup.string().required("Password is required"), - database: yup.string().optional(), - schema: yup.string().optional(), - warehouse: yup.string().optional(), - role: yup.string().optional(), - }), - - salesforce: yup.object().shape({ - soql: yup.string().required("soql is required"), - sobject: yup.string().required("sobject is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - - athena: yup.object().shape({ - aws_access_key_id: yup.string().required("AWS access key ID is required"), - aws_secret_access_key: yup - .string() - .required("AWS secret access key is required"), - output_location: yup.string().required("S3 URI is required"), - }), - - redshift: yup.object().shape({ - aws_access_key_id: yup.string().required("AWS access key ID is required"), - aws_secret_access_key: yup - .string() - .required("AWS secret access key is required"), - workgroup: yup.string().optional(), - region: yup.string().optional(), // TODO: add validation - database: yup.string().required("database name is required"), - }), - - mysql: yup.object().shape({ - dsn: yup.string().optional(), - user: yup.string().optional(), - password: yup.string().optional(), - host: yup.string().optional(), - port: yup.string().optional(), - database: yup.string().optional(), - sslmode: yup.string().optional(), - }), - - clickhouse: yup.object().shape({ - dsn: yup.string().optional(), - managed: yup.boolean(), - host: yup.string(), - // .required("Host is required") - // .matches( - // /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - // "Do not prefix the host with `http(s)://`", // It will be added by the runtime - // ), - port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - username: yup.string(), - password: yup.string(), - cluster: yup.string(), - ssl: yup.boolean(), - name: yup.string(), // Required for typing - // User-provided connector names requires a little refactor. Commenting out for now. - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Connector name is required"), - }), - - druid: yup.object().shape({ - host: yup - .string() - .required("Host is required") - .matches( - /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - "Do not prefix the host with `http(s)://`", // It will be added by the runtime - ), - port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - username: yup.string(), - password: yup.string(), - ssl: yup.boolean(), - name: yup.string(), // Required for typing - // User-provided connector names requires a little refactor. Commenting out for now. - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Connector name is required"), - }), - - pinot: yup.object().shape({ - broker_host: yup - .string() - .required("Broker host is required") - .matches( - /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - "Do not prefix the host with `http(s)://`", // It will be added by the runtime - ), - broker_port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - controller_host: yup - .string() - .required("Controller host is required") - .matches( - /^(?!https?:\/\/)[a-zA-Z0-9.-]+$/, - "Do not prefix the host with `http(s)://`", // It will be added by the runtime - ), - controller_port: yup - .string() // Purposefully using a string input, not a numeric input - .matches(/^\d+$/, "Port must be a number"), - username: yup.string(), - password: yup.string(), - ssl: yup.boolean(), - name: yup.string(), // Required for typing - // User-provided connector names requires a little refactor. Commenting out for now. - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Connector name is required"), - }), -}; - -export const dsnSchema = yup.object().shape({ - dsn: yup.string().required("DSN is required"), -}); diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte new file mode 100644 index 00000000000..3d012838118 --- /dev/null +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -0,0 +1,268 @@ + + +{#if schema} + {#each renderOrder as [key, prop]} + {#if isRadioEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + + {#if groupedFields.get(key)} + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} +
+ +
+ {/each} + {/if} + + +
+ {:else} +
+ +
+ {/if} + {/each} +{/if} diff --git a/web-common/src/features/templates/SchemaField.svelte b/web-common/src/features/templates/SchemaField.svelte new file mode 100644 index 00000000000..3bfa6783001 --- /dev/null +++ b/web-common/src/features/templates/SchemaField.svelte @@ -0,0 +1,66 @@ + + +{#if prop["x-display"] === "file" || prop.format === "file"} + +{:else if prop.type === "boolean"} + +{:else if options?.length} + +{:else} + onStringInputChange(e)} + alwaysShowError + /> +{/if} diff --git a/web-common/src/features/templates/error-utils.ts b/web-common/src/features/templates/error-utils.ts new file mode 100644 index 00000000000..1c2c74dcbf4 --- /dev/null +++ b/web-common/src/features/templates/error-utils.ts @@ -0,0 +1,16 @@ +/** + * Normalizes a variety of error shapes into a string, string[], or undefined. + * - If input is an array, returns it as-is. + * - If input is a string, returns it. + * - If input resembles a Zod `_errors` array, returns that. + * - Otherwise returns undefined. + */ +export function normalizeErrors( + err: any, +): string | string[] | null | undefined { + if (!err) return undefined; + if (Array.isArray(err)) return err; + if (typeof err === "string") return err; + if (err._errors && Array.isArray(err._errors)) return err._errors; + return undefined; +} diff --git a/web-common/src/features/templates/schema-utils.ts b/web-common/src/features/templates/schema-utils.ts new file mode 100644 index 00000000000..a4544947297 --- /dev/null +++ b/web-common/src/features/templates/schema-utils.ts @@ -0,0 +1,154 @@ +import type { + JSONSchemaConditional, + MultiStepFormSchema, +} from "./schemas/types"; + +export type RadioEnumOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export function isStepMatch( + schema: MultiStepFormSchema | null, + key: string, + step?: "connector" | "source" | string, +): boolean { + if (!schema?.properties) return false; + const prop = schema.properties[key]; + if (!prop) return false; + if (!step) return true; + const propStep = prop["x-step"]; + if (!propStep) return true; + return propStep === step; +} + +export function isVisibleForValues( + schema: MultiStepFormSchema, + key: string, + values: Record, +): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; + const conditions = prop["x-visible-if"]; + if (!conditions) return true; + + return Object.entries(conditions).every(([depKey, expected]) => { + const actual = values?.[depKey]; + if (Array.isArray(expected)) { + return expected.map(String).includes(String(actual)); + } + return String(actual) === String(expected); + }); +} + +export function getFieldLabel( + schema: MultiStepFormSchema, + key: string, +): string { + return schema.properties?.[key]?.title || key; +} + +export function findRadioEnumKey(schema: MultiStepFormSchema): string | null { + if (!schema.properties) return null; + for (const [key, value] of Object.entries(schema.properties)) { + if (value.enum && value["x-display"] === "radio") { + return key; + } + } + return null; +} + +export function getRadioEnumOptions(schema: MultiStepFormSchema): { + key: string; + options: RadioEnumOption[]; + defaultValue?: string; +} | null { + const enumKey = findRadioEnumKey(schema); + if (!enumKey) return null; + const enumProperty = schema.properties?.[enumKey]; + if (!enumProperty?.enum) return null; + + const labels = enumProperty["x-enum-labels"] ?? []; + const descriptions = enumProperty["x-enum-descriptions"] ?? []; + const options = + enumProperty.enum?.map((value, idx) => ({ + value: String(value), + label: labels[idx] ?? String(value), + description: + descriptions[idx] ?? enumProperty.description ?? "Choose an option", + hint: enumProperty["x-hint"], + })) ?? []; + + const defaultValue = + enumProperty.default !== undefined && enumProperty.default !== null + ? String(enumProperty.default) + : options[0]?.value; + + return { + key: enumKey, + options, + defaultValue: defaultValue || undefined, + }; +} + +export function getRequiredFieldsByEnumValue( + schema: MultiStepFormSchema, + opts?: { step?: "connector" | "source" | string }, +): Record { + const enumInfo = getRadioEnumOptions(schema); + if (!enumInfo) return {}; + + const conditionals = schema.allOf ?? []; + const baseRequired = new Set(schema.required ?? []); + const result: Record = {}; + + const matchesStep = (field: string) => { + if (!opts?.step) return true; + const prop = schema.properties?.[field]; + if (!prop) return false; + const propStep = prop["x-step"]; + if (!propStep) return true; + return propStep === opts.step; + }; + + for (const option of enumInfo.options) { + const required = new Set(); + + baseRequired.forEach((field) => { + if (matchesStep(field)) { + required.add(field); + } + }); + + for (const conditional of conditionals) { + const matches = matchesEnumCondition( + conditional, + enumInfo.key, + option.value, + ); + const target = matches ? conditional.then : conditional.else; + target?.required?.forEach((field) => { + if (matchesStep(field)) { + required.add(field); + } + }); + } + + result[option.value] = Array.from(required); + } + + return result; +} + +function matchesEnumCondition( + conditional: JSONSchemaConditional, + enumKey: string, + value: string, +) { + const conditionProps = conditional.if?.properties; + const constValue = conditionProps?.[enumKey]?.const; + if (constValue === undefined || constValue === null) return false; + return String(constValue) === value; +} diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts new file mode 100644 index 00000000000..cd6dd3719d9 --- /dev/null +++ b/web-common/src/features/templates/schemas/athena.ts @@ -0,0 +1,34 @@ +import type { MultiStepFormSchema } from "./types"; + +export const athenaSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS access key ID", + description: "AWS access key ID used to authenticate to Athena", + "x-placeholder": "your_access_key_id", + "x-secret": true, + }, + aws_secret_access_key: { + type: "string", + title: "AWS secret access key", + description: "AWS secret access key paired with the access key ID", + "x-placeholder": "your_secret_access_key", + "x-secret": true, + }, + output_location: { + type: "string", + title: "S3 output location", + description: + "S3 URI where Athena should write query results (e.g., s3://bucket/path/)", + pattern: "^s3://.+", + errorMessage: { + pattern: "Must be an S3 URI (e.g., s3://bucket/path/)", + }, + "x-placeholder": "s3://bucket-name/path/", + }, + }, + required: ["aws_access_key_id", "aws_secret_access_key", "output_location"], +}; diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts new file mode 100644 index 00000000000..ac3146ca864 --- /dev/null +++ b/web-common/src/features/templates/schemas/azure.ts @@ -0,0 +1,108 @@ +import type { MultiStepFormSchema } from "./types"; + +export const azureSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["connection_string", "account_key", "sas_token", "public"], + default: "connection_string", + description: "Choose how to authenticate to Azure Blob Storage", + "x-display": "radio", + "x-enum-labels": [ + "Connection String", + "Storage Account Key", + "SAS Token", + "Public", + ], + "x-enum-descriptions": [ + "Provide a full Azure Storage connection string.", + "Provide the storage account name and access key.", + "Provide the storage account name and SAS token.", + "Access publicly readable blobs without credentials.", + ], + "x-grouped-fields": { + connection_string: ["azure_storage_connection_string"], + account_key: ["azure_storage_account", "azure_storage_key"], + sas_token: ["azure_storage_account", "azure_storage_sas_token"], + public: [], + }, + "x-step": "connector", + }, + azure_storage_connection_string: { + type: "string", + title: "Connection string", + description: "Paste an Azure Storage connection string", + "x-placeholder": "Enter Azure storage connection string", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + azure_storage_account: { + type: "string", + title: "Storage account", + description: "The name of the Azure storage account", + "x-placeholder": "Enter Azure storage account", + "x-step": "connector", + "x-visible-if": { auth_method: ["account_key", "sas_token"] }, + }, + azure_storage_key: { + type: "string", + title: "Access key", + description: "Primary or secondary access key for the storage account", + "x-placeholder": "Enter Azure storage access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "account_key" }, + }, + azure_storage_sas_token: { + type: "string", + title: "SAS token", + description: + "Shared Access Signature token for the storage account (starting with ?sv=)", + "x-placeholder": "Enter Azure SAS token", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "sas_token" }, + }, + path: { + type: "string", + title: "Blob URI", + description: + "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", + pattern: "^azure://.+", + errorMessage: { + pattern: "Must be an Azure URI (e.g. azure://container/path)", + }, + "x-placeholder": "azure://container/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["azure_storage_connection_string"] }, + }, + { + if: { properties: { auth_method: { const: "account_key" } } }, + then: { required: ["azure_storage_account", "azure_storage_key"] }, + }, + { + if: { properties: { auth_method: { const: "sas_token" } } }, + then: { + required: ["azure_storage_account", "azure_storage_sas_token"], + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts new file mode 100644 index 00000000000..6930a9698d6 --- /dev/null +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -0,0 +1,26 @@ +import type { MultiStepFormSchema } from "./types"; + +export const bigquerySchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + google_application_credentials: { + type: "string", + title: "GCP credentials", + description: "Service account JSON (uploaded or pasted)", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-secret": true, + }, + project_id: { + type: "string", + title: "Project ID", + description: "Google Cloud project ID to use for queries", + "x-placeholder": "my-project", + "x-hint": + "If empty, Rill will use the project ID from your credentials when available.", + }, + }, + required: ["google_application_credentials"], +}; diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts new file mode 100644 index 00000000000..ad5061a9eeb --- /dev/null +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -0,0 +1,130 @@ +import type { MultiStepFormSchema } from "./types"; + +export const clickhouseSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connector_type: { + type: "string", + title: "Connection type", + enum: ["self-hosted", "clickhouse-cloud", "rill-managed"], + default: "self-hosted", + "x-display": "radio", + "x-enum-labels": [ + "Self-hosted ClickHouse", + "ClickHouse Cloud", + "Rill-managed ClickHouse", + ], + "x-step": "connector", + }, + dsn: { + type: "string", + title: "Connection string", + description: + "DSN connection string (use instead of individual host/port/user settings)", + "x-placeholder": + "clickhouse://localhost:9000?username=default&password=password", + "x-step": "connector", + }, + managed: { + type: "boolean", + title: "Managed", + description: + "Use a managed ClickHouse instance (handled automatically by Rill)", + default: false, + "x-step": "connector", + }, + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the ClickHouse server", + "x-placeholder": + "your-instance.clickhouse.cloud or your.clickhouse.server.com", + "x-hint": + "Your ClickHouse hostname (e.g., your-instance.clickhouse.cloud or your-server.com)", + "x-step": "connector", + }, + port: { + type: "string", + title: "Port", + description: "Port number of the ClickHouse server", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + default: "9000", + "x-placeholder": "9000", + "x-step": "connector", + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the ClickHouse server", + default: "default", + "x-placeholder": "default", + "x-step": "connector", + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the ClickHouse server", + "x-placeholder": "Database password", + "x-secret": true, + "x-step": "connector", + }, + database: { + type: "string", + title: "Database", + description: "Name of the ClickHouse database to connect to", + default: "default", + "x-placeholder": "default", + "x-step": "connector", + }, + cluster: { + type: "string", + title: "Cluster", + description: + "Cluster name. If set, models are created as distributed tables.", + "x-placeholder": "Cluster name", + "x-step": "connector", + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL to connect to the ClickHouse server", + default: true, + "x-step": "connector", + }, + }, + required: ["connector_type"], + allOf: [ + { + if: { properties: { connector_type: { const: "rill-managed" } } }, + then: { + required: ["managed"], + properties: { + managed: { const: true }, + }, + }, + }, + { + if: { properties: { connector_type: { const: "self-hosted" } } }, + then: { + required: ["host", "username"], + properties: { + managed: { const: false }, + ssl: { default: true }, + }, + }, + }, + { + if: { properties: { connector_type: { const: "clickhouse-cloud" } } }, + then: { + required: ["host", "username", "ssl"], + properties: { + managed: { const: false }, + port: { default: "8443" }, + ssl: { const: true }, + }, + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts new file mode 100644 index 00000000000..c2011829bc2 --- /dev/null +++ b/web-common/src/features/templates/schemas/druid.ts @@ -0,0 +1,61 @@ +import type { MultiStepFormSchema } from "./types"; + +export const druidSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + dsn: { + type: "string", + title: "Connection string", + description: + "Full Druid SQL/Avatica endpoint, e.g. https://host:8888/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=user&avaticaPassword=pass", + "x-placeholder": + "https://example.com/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=user&avaticaPassword=pass", + "x-secret": true, + }, + host: { + type: "string", + title: "Host", + description: "Druid host or IP", + "x-placeholder": "localhost", + }, + port: { + type: "string", + title: "Port", + description: "Druid port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + "x-placeholder": "8888", + }, + username: { + type: "string", + title: "Username", + description: "Druid username", + "x-placeholder": "default", + }, + password: { + type: "string", + title: "Password", + description: "Druid password", + "x-placeholder": "password", + "x-secret": true, + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL for the connection", + default: true, + }, + }, + required: [], + oneOf: [ + { + title: "Use connection string", + required: ["dsn"], + }, + { + title: "Use individual parameters", + required: ["host", "ssl"], + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts new file mode 100644 index 00000000000..3e893aa79f3 --- /dev/null +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -0,0 +1,15 @@ +import type { MultiStepFormSchema } from "./types"; + +export const duckdbSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "Path to external DuckDB database", + "x-placeholder": "/path/to/main.db", + }, + }, + required: ["path"], +}; diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts new file mode 100644 index 00000000000..4474bff30d7 --- /dev/null +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -0,0 +1,87 @@ +import type { MultiStepFormSchema } from "./types"; + +export const gcsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["credentials", "hmac", "public"], + default: "credentials", + description: "Choose how to authenticate to GCS", + "x-display": "radio", + "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with GCS access.", + "Use HMAC access key and secret for S3-compatible authentication.", + "Access publicly readable buckets without credentials.", + ], + "x-grouped-fields": { + credentials: ["google_application_credentials"], + hmac: ["key_id", "secret"], + public: [], + }, + "x-step": "connector", + }, + google_application_credentials: { + type: "string", + title: "Service account key", + description: + "Upload a JSON key file for a service account with GCS access.", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + "x-visible-if": { auth_method: "credentials" }, + }, + key_id: { + type: "string", + title: "Access Key ID", + description: "HMAC access key ID for S3-compatible authentication", + "x-placeholder": "Enter your HMAC access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + secret: { + type: "string", + title: "Secret Access Key", + description: "HMAC secret access key for S3-compatible authentication", + "x-placeholder": "Enter your HMAC secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://[^/]+(/.*)?$", + errorMessage: { + pattern: "Must be a GS URI (e.g. gs://bucket/path)", + }, + "x-placeholder": "gs://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "credentials" } } }, + then: { required: ["google_application_credentials"] }, + }, + { + if: { properties: { auth_method: { const: "hmac" } } }, + then: { required: ["key_id", "secret"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts new file mode 100644 index 00000000000..157b76b3eff --- /dev/null +++ b/web-common/src/features/templates/schemas/https.ts @@ -0,0 +1,27 @@ +import type { MultiStepFormSchema } from "./types"; + +export const httpsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "HTTP/HTTPS URL to the remote file", + pattern: "^https?://.+", + errorMessage: { + pattern: "Path must start with http:// or https://", + }, + "x-placeholder": "https://example.com/file.csv", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name of the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["path", "name"], +}; diff --git a/web-common/src/features/templates/schemas/local_file.ts b/web-common/src/features/templates/schemas/local_file.ts new file mode 100644 index 00000000000..2fec6b68784 --- /dev/null +++ b/web-common/src/features/templates/schemas/local_file.ts @@ -0,0 +1,23 @@ +import type { MultiStepFormSchema } from "./types"; + +export const localFileSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "Local file path or glob (relative to project root)", + "x-placeholder": "data/*.parquet", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["path", "name"], +}; diff --git a/web-common/src/features/templates/schemas/motherduck.ts b/web-common/src/features/templates/schemas/motherduck.ts new file mode 100644 index 00000000000..7597ae50e4e --- /dev/null +++ b/web-common/src/features/templates/schemas/motherduck.ts @@ -0,0 +1,28 @@ +import type { MultiStepFormSchema } from "./types"; + +export const motherduckSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "MotherDuck database path (prefix with md:)", + "x-placeholder": "md:my_db", + }, + token: { + type: "string", + title: "Token", + description: "MotherDuck token", + "x-placeholder": "your_motherduck_token", + "x-secret": true, + }, + schema_name: { + type: "string", + title: "Schema name", + description: "Default schema to use", + "x-placeholder": "main", + }, + }, + required: ["path", "token", "schema_name"], +}; diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts new file mode 100644 index 00000000000..5b8ce24fc25 --- /dev/null +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -0,0 +1,86 @@ +import type { MultiStepFormSchema } from "./types"; + +export const mysqlSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + dsn: { + type: "string", + title: "MySQL connection string", + description: + "Full DSN, e.g. mysql://user:password@host:3306/database?ssl-mode=REQUIRED", + "x-placeholder": "mysql://user:password@host:3306/database", + "x-secret": true, + "x-hint": + "Use DSN or fill host/user/password/database below (not both at once).", + }, + host: { + type: "string", + title: "Host", + description: "MySQL server hostname or IP", + "x-placeholder": "localhost", + }, + port: { + type: "string", + title: "Port", + description: "MySQL server port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + default: "3306", + "x-placeholder": "3306", + }, + database: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "my_database", + }, + user: { + type: "string", + title: "Username", + description: "MySQL user", + "x-placeholder": "mysql", + }, + password: { + type: "string", + title: "Password", + description: "MySQL password", + "x-placeholder": "your_password", + "x-secret": true, + }, + "ssl-mode": { + type: "string", + title: "SSL mode", + description: "Use DISABLED, PREFERRED, or REQUIRED", + enum: ["DISABLED", "PREFERRED", "REQUIRED"], + "x-placeholder": "PREFERRED", + }, + log_queries: { + type: "boolean", + title: "Log queries", + description: "Enable logging of SQL queries (for debugging)", + default: false, + }, + }, + required: [], + oneOf: [ + { + title: "Use DSN", + required: ["dsn"], + not: { + anyOf: [ + { required: ["host"] }, + { required: ["database"] }, + { required: ["user"] }, + { required: ["password"] }, + { required: ["port"] }, + { required: ["ssl-mode"] }, + ], + }, + }, + { + title: "Use individual parameters", + required: ["host", "database", "user"], + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts new file mode 100644 index 00000000000..e437ed9101e --- /dev/null +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -0,0 +1,75 @@ +import type { MultiStepFormSchema } from "./types"; + +export const pinotSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + dsn: { + type: "string", + title: "Connection string", + description: + "Full Pinot connection string, e.g. http(s)://user:password@broker:8000?controller=host:9000", + "x-placeholder": + "https://username:password@localhost:8000?controller=localhost:9000", + "x-secret": true, + }, + broker_host: { + type: "string", + title: "Broker host", + description: "Pinot broker host", + "x-placeholder": "localhost", + }, + broker_port: { + type: "string", + title: "Broker port", + description: "Pinot broker port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + "x-placeholder": "8000", + }, + controller_host: { + type: "string", + title: "Controller host", + description: "Pinot controller host", + "x-placeholder": "localhost", + }, + controller_port: { + type: "string", + title: "Controller port", + description: "Pinot controller port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + "x-placeholder": "9000", + }, + username: { + type: "string", + title: "Username", + description: "Pinot username", + "x-placeholder": "default", + }, + password: { + type: "string", + title: "Password", + description: "Pinot password", + "x-placeholder": "password", + "x-secret": true, + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL", + default: true, + }, + }, + required: [], + oneOf: [ + { + title: "Use connection string", + required: ["dsn"], + }, + { + title: "Use individual parameters", + required: ["broker_host", "controller_host", "ssl"], + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts new file mode 100644 index 00000000000..8ab2626bb8e --- /dev/null +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -0,0 +1,109 @@ +import type { MultiStepFormSchema } from "./types"; + +export const postgresSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + dsn: { + type: "string", + title: "Postgres connection string", + description: + "Full DSN, e.g. postgresql://user:password@host:5432/dbname?sslmode=require", + "x-placeholder": "postgresql://postgres:postgres@localhost:5432/postgres", + "x-secret": true, + "x-hint": + "Use a DSN or provide host/user/password/dbname below (but not both).", + }, + database_url: { + type: "string", + title: "Database URL", + description: "Alternative DSN field (same as dsn)", + "x-placeholder": "postgresql://postgres:postgres@localhost:5432/postgres", + "x-secret": true, + }, + host: { + type: "string", + title: "Host", + description: "Postgres server hostname or IP", + "x-placeholder": "localhost", + }, + port: { + type: "string", + title: "Port", + description: "Postgres server port", + pattern: "^\\d+$", + errorMessage: { pattern: "Port must be a number" }, + default: "5432", + "x-placeholder": "5432", + }, + user: { + type: "string", + title: "Username", + description: "Postgres user", + "x-placeholder": "postgres", + }, + password: { + type: "string", + title: "Password", + description: "Postgres password", + "x-placeholder": "your_password", + "x-secret": true, + }, + dbname: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "postgres", + }, + sslmode: { + type: "string", + title: "SSL mode", + description: "Use disable, allow, prefer, require", + enum: ["disable", "allow", "prefer", "require"], + "x-placeholder": "require", + }, + log_queries: { + type: "boolean", + title: "Log queries", + description: "Enable logging of SQL queries (for debugging)", + default: false, + }, + }, + required: [], + oneOf: [ + { + title: "Use DSN", + required: ["dsn"], + not: { + anyOf: [ + { required: ["database_url"] }, + { required: ["host"] }, + { required: ["port"] }, + { required: ["user"] }, + { required: ["password"] }, + { required: ["dbname"] }, + { required: ["sslmode"] }, + ], + }, + }, + { + title: "Use Database URL", + required: ["database_url"], + not: { + anyOf: [ + { required: ["dsn"] }, + { required: ["host"] }, + { required: ["port"] }, + { required: ["user"] }, + { required: ["password"] }, + { required: ["dbname"] }, + { required: ["sslmode"] }, + ], + }, + }, + { + title: "Use individual parameters", + required: ["host", "user", "dbname"], + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts new file mode 100644 index 00000000000..f2e1ade5536 --- /dev/null +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -0,0 +1,48 @@ +import type { MultiStepFormSchema } from "./types"; + +export const redshiftSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS access key ID", + description: "AWS access key ID", + "x-placeholder": "your_access_key_id", + "x-secret": true, + }, + aws_secret_access_key: { + type: "string", + title: "AWS secret access key", + description: "AWS secret access key", + "x-placeholder": "your_secret_access_key", + "x-secret": true, + }, + region: { + type: "string", + title: "AWS region", + description: "AWS region (e.g. us-east-1)", + "x-placeholder": "us-east-1", + }, + database: { + type: "string", + title: "Database", + description: "Redshift database name", + "x-placeholder": "dev", + }, + workgroup: { + type: "string", + title: "Workgroup", + description: "Redshift Serverless workgroup name", + "x-placeholder": "default", + }, + cluster_identifier: { + type: "string", + title: "Cluster identifier", + description: + "Redshift cluster identifier (use when not using serverless)", + "x-placeholder": "redshift-cluster-1", + }, + }, + required: ["aws_access_key_id", "aws_secret_access_key", "database"], +}; diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts new file mode 100644 index 00000000000..8e9799cbc64 --- /dev/null +++ b/web-common/src/features/templates/schemas/s3.ts @@ -0,0 +1,105 @@ +import type { MultiStepFormSchema } from "./types"; + +export const s3Schema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + description: "Choose how to authenticate to S3", + enum: ["access_keys", "public"], + default: "access_keys", + "x-display": "radio", + "x-enum-labels": ["Access keys", "Public"], + "x-enum-descriptions": [ + "Use AWS access key ID and secret access key.", + "Access publicly readable buckets without credentials.", + ], + "x-grouped-fields": { + access_keys: [ + "aws_access_key_id", + "aws_secret_access_key", + "region", + "endpoint", + "aws_role_arn", + ], + public: [], + }, + "x-step": "connector", + }, + aws_access_key_id: { + type: "string", + title: "Access Key ID", + description: "AWS access key ID for the bucket", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_secret_access_key: { + type: "string", + title: "Secret Access Key", + description: "AWS secret access key for the bucket", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + region: { + type: "string", + title: "Region", + description: + "Rill uses your default AWS region unless you set it explicitly.", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + endpoint: { + type: "string", + title: "Endpoint", + description: + "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", + "x-placeholder": "https://s3.example.com", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_role_arn: { + type: "string", + title: "AWS Role ARN", + description: "AWS Role ARN to assume", + "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + path: { + type: "string", + title: "S3 URI", + description: "Path to your S3 bucket or prefix", + pattern: "^s3://[^/]+(/.*)?$", + errorMessage: { + pattern: "Must be an S3 URI (e.g. s3://bucket/path)", + }, + "x-placeholder": "s3://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "access_keys" } } }, + then: { + required: ["aws_access_key_id", "aws_secret_access_key"], + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/salesforce.ts b/web-common/src/features/templates/schemas/salesforce.ts new file mode 100644 index 00000000000..c60092f32fe --- /dev/null +++ b/web-common/src/features/templates/schemas/salesforce.ts @@ -0,0 +1,94 @@ +import type { MultiStepFormSchema } from "./types"; + +export const salesforceSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + soql: { + type: "string", + title: "SOQL", + description: "SOQL query to extract data", + "x-display": "textarea", + "x-placeholder": "SELECT Id, Name FROM Opportunity", + "x-step": "source", + }, + sobject: { + type: "string", + title: "SObject", + description: "Salesforce object to query", + "x-placeholder": "Opportunity", + "x-step": "source", + }, + queryAll: { + type: "boolean", + title: "Query all", + description: "Include deleted and archived records", + default: false, + "x-step": "source", + }, + username: { + type: "string", + title: "Username", + description: "Salesforce username (usually an email)", + "x-placeholder": "user@example.com", + }, + password: { + type: "string", + title: "Password", + description: + "Salesforce password, optionally followed by security token if required", + "x-placeholder": "your_password_or_password+token", + "x-secret": true, + }, + key: { + type: "string", + title: "JWT private key", + description: "PEM-formatted private key for JWT auth", + "x-display": "textarea", + "x-secret": true, + }, + client_id: { + type: "string", + title: "Connected App Client ID", + description: "Client ID (consumer key) for JWT auth", + "x-placeholder": "Connected App client ID", + }, + endpoint: { + type: "string", + title: "Login endpoint", + description: + "Salesforce login URL (e.g., login.salesforce.com or test.salesforce.com)", + "x-placeholder": "login.salesforce.com", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["soql", "sobject", "name"], + allOf: [ + { + if: { + properties: { + key: { const: "" }, + }, + }, + then: { + required: ["username", "password", "endpoint"], + }, + }, + { + if: { + properties: { + key: { const: undefined }, + }, + }, + then: { + required: ["username", "password", "endpoint"], + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts new file mode 100644 index 00000000000..18dec9a92b7 --- /dev/null +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -0,0 +1,90 @@ +import type { MultiStepFormSchema } from "./types"; + +export const snowflakeSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + dsn: { + type: "string", + title: "Snowflake connection string", + description: + "Full Snowflake DSN, e.g. @//?warehouse=&role=", + "x-placeholder": + "@//?warehouse=&role=", + "x-secret": true, + "x-hint": + "Use a full DSN or fill the fields below (not both). Include authenticator and privateKey for JWT if needed.", + }, + account: { + type: "string", + title: "Account identifier", + description: + "Snowflake account identifier (from your Snowflake URL, before .snowflakecomputing.com)", + "x-placeholder": "abc12345", + }, + user: { + type: "string", + title: "Username", + description: "Snowflake username", + "x-placeholder": "your_username", + }, + password: { + type: "string", + title: "Password", + description: + "Snowflake password (use JWT private key if password auth is disabled)", + "x-placeholder": "your_password", + "x-secret": true, + }, + privateKey: { + type: "string", + title: "Private key (JWT)", + description: + "URL-safe base64 or PEM private key for SNOWFLAKE_JWT authenticator", + "x-display": "textarea", + "x-secret": true, + }, + authenticator: { + type: "string", + title: "Authenticator", + description: "Override authenticator (e.g., SNOWFLAKE_JWT)", + "x-placeholder": "SNOWFLAKE_JWT", + }, + database: { + type: "string", + title: "Database", + description: "Snowflake database", + "x-placeholder": "your_database", + }, + schema: { + type: "string", + title: "Schema", + description: "Default schema", + "x-placeholder": "public", + }, + warehouse: { + type: "string", + title: "Warehouse", + description: "Compute warehouse", + "x-placeholder": "your_warehouse", + }, + role: { + type: "string", + title: "Role", + description: "Snowflake role", + "x-placeholder": "your_role", + }, + parallel_fetch_limit: { + type: "number", + title: "Parallel fetch limit", + description: "Limit concurrent fetches (leave empty for default)", + }, + log_queries: { + type: "boolean", + title: "Log queries", + description: "Enable SQL query logging (debugging)", + default: false, + }, + }, + required: [], +}; diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts new file mode 100644 index 00000000000..039f8550f83 --- /dev/null +++ b/web-common/src/features/templates/schemas/sqlite.ts @@ -0,0 +1,30 @@ +import type { MultiStepFormSchema } from "./types"; + +export const sqliteSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + db: { + type: "string", + title: "Database file", + description: "Path to SQLite db file", + "x-placeholder": "/path/to/sqlite.db", + "x-step": "source", + }, + table: { + type: "string", + title: "Table", + description: "SQLite table name", + "x-placeholder": "table", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name of the source", + "x-placeholder": "my_new_source", + "x-step": "source", + }, + }, + required: ["db", "table", "name"], +}; diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts new file mode 100644 index 00000000000..89cf0bc0720 --- /dev/null +++ b/web-common/src/features/templates/schemas/types.ts @@ -0,0 +1,68 @@ +export type JSONSchemaVisibleIfValue = + | string + | number + | boolean + | Array; + +export type JSONSchemaField = { + type?: "string" | "number" | "boolean" | "object"; + title?: string; + description?: string; + enum?: Array; + const?: string | number | boolean; + default?: string | number | boolean; + pattern?: string; + format?: string; + errorMessage?: { + pattern?: string; + format?: string; + }; + properties?: Record; + required?: string[]; + "x-display"?: "radio" | "select" | "textarea" | "file"; + "x-step"?: "connector" | "source"; + "x-secret"?: boolean; + "x-visible-if"?: Record; + "x-enum-labels"?: string[]; + "x-enum-descriptions"?: string[]; + "x-placeholder"?: string; + "x-hint"?: string; + "x-accept"?: string; + /** + * Explicit grouping for radio/select options: maps an option value to the + * child field keys that should render beneath that option. + */ + "x-grouped-fields"?: Record; + // Allow custom keywords such as errorMessage or future x-extensions. + [key: string]: unknown; +}; + +export type JSONSchemaCondition = { + properties?: Record; +}; + +export type JSONSchemaConstraint = { + required?: string[]; + properties?: Record; + // Allow custom keywords or overrides in constraints + [key: string]: unknown; +}; + +export type JSONSchemaConditional = { + if?: JSONSchemaCondition; + then?: JSONSchemaConstraint; + else?: JSONSchemaConstraint; +}; + +export type JSONSchemaObject = { + $schema?: string; + type: "object"; + title?: string; + description?: string; + properties?: Record; + required?: string[]; + allOf?: JSONSchemaConditional[]; + oneOf?: JSONSchemaConstraint[]; +}; + +export type MultiStepFormSchema = JSONSchemaObject; diff --git a/web-local/tests/connectors/multi-step-connector.spec.ts b/web-local/tests/connectors/multi-step-connector.spec.ts new file mode 100644 index 00000000000..39abf31954d --- /dev/null +++ b/web-local/tests/connectors/multi-step-connector.spec.ts @@ -0,0 +1,187 @@ +import { expect } from "@playwright/test"; +import { test } from "../setup/base"; + +test.describe("Multi-step connector wrapper", () => { + test.use({ project: "Blank" }); + + test("GCS connector - renders connector step schema via wrapper", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + // Choose a multi-step connector (GCS). + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Connector step should show connector preview and connector CTA. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect( + page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }), + ).toBeVisible(); + + // Auth method controls from the connector schema should render. + const hmacRadio = page.getByRole("radio", { name: "HMAC keys" }); + await expect(hmacRadio).toBeVisible(); + await expect(page.getByRole("radio", { name: "Public" })).toBeVisible(); + + // Select HMAC so its fields are rendered. + await hmacRadio.click(); + + // Connector step fields should be present, while source step fields should not yet render. + await expect( + page.getByRole("textbox", { name: "Access Key ID" }), + ).toBeVisible(); + await expect( + page.getByRole("textbox", { name: "Secret Access Key" }), + ).toBeVisible(); + await expect(page.getByRole("textbox", { name: "GS URI" })).toHaveCount(0); + }); + + test("GCS connector - renders source step schema via wrapper", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + // Choose a multi-step connector (GCS). + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Connector step visible with CTA. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect( + page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }), + ).toBeVisible(); + + // Switch to Public auth (no required fields) and continue via CTA. + await page.getByRole("radio", { name: "Public" }).click(); + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + await connectorCta.click(); + + // Source step should now render with source schema fields and CTA. + await expect(page.getByText("Model preview")).toBeVisible(); + const sourceCta = page.getByRole("button", { + name: /Test and Add data|Importing data|Add data/i, + }); + await expect(sourceCta).toBeVisible(); + + // Source fields should be present; connector-only auth fields should not be required to show. + await expect(page.getByRole("textbox", { name: "GCS URI" })).toBeVisible( + {}, + ); + await expect( + page.getByRole("textbox", { name: "Model name" }), + ).toBeVisible(); + }); + + test("GCS connector - preserves auth selection across steps", async ({ + page, + }) => { + const hmacKey = process.env.RILL_RUNTIME_GCS_TEST_HMAC_KEY; + const hmacSecret = process.env.RILL_RUNTIME_GCS_TEST_HMAC_SECRET; + if (!hmacKey || !hmacSecret) { + test.skip( + true, + "RILL_RUNTIME_GCS_TEST_HMAC_KEY or RILL_RUNTIME_GCS_TEST_HMAC_SECRET is not set", + ); + } + + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Pick HMAC auth and fill required fields. + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page.getByRole("textbox", { name: "Access Key ID" }).fill(hmacKey!); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill(hmacSecret!); + + // Submit connector step via CTA to transition to source step. + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + await expect(connectorCta).toBeEnabled(); + await connectorCta.click(); + await expect(page.getByText("Model preview")).toBeVisible(); + + // Go back to connector step. + await page.getByRole("button", { name: "Back" }).click(); + + // Auth selection and values should persist. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect(page.getByRole("radio", { name: "HMAC keys" })).toBeChecked({ + timeout: 5000, + }); + await expect( + page.getByRole("textbox", { name: "Access Key ID" }), + ).toHaveValue(hmacKey!); + await expect( + page.getByRole("textbox", { name: "Secret Access Key" }), + ).toHaveValue(hmacSecret!); + }); + + test("GCS connector - disables submit until auth requirements met", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + + // Default auth is credentials (file upload); switch to HMAC to check required fields. + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await expect(connectorCta).toBeDisabled(); + + // Fill key only -> still disabled. + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(connectorCta).toBeDisabled(); + + // Fill secret -> enabled. + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(connectorCta).toBeEnabled(); + }); + + test("GCS connector - public auth option keeps submit enabled and allows advancing", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + + // Switch to Public (no required fields) -> CTA should remain enabled and allow advancing. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(connectorCta).toBeEnabled(); + await connectorCta.click(); + + // Should land on source step without needing connector fields. + await expect(page.getByText("Model preview")).toBeVisible(); + await expect( + page.getByRole("button", { name: /Test and Add data|Add data/i }), + ).toBeVisible(); + }); +}); diff --git a/web-local/tests/connectors/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts index e571c62d3bf..1ed49d8276c 100644 --- a/web-local/tests/connectors/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -4,6 +4,73 @@ import { test } from "../setup/base"; test.describe("Test Connection", () => { test.use({ project: "Blank" }); + test("Azure connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#azure").click(); + await page.waitForSelector('form[id*="azure"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Select Storage Account Key (default may be different) -> requires account + key. + await page.getByRole("radio", { name: "Storage Account Key" }).click(); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Storage account" }).fill("acct"); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Access key" }).fill("key"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch to Connection String -> requires connection string, so disabled until filled. + await page.getByRole("radio", { name: "Connection String" }).click(); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Connection string" }) + .fill("DefaultEndpointsProtocol=https;"); + await expect(button).toBeEnabled(); + }); + + test("S3 connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#s3").click(); + await page.waitForSelector('form[id*="s3"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Default method is Access keys -> requires access key id + secret. + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch back to Access keys -> fields cleared, so disabled until refilled. + await page.getByRole("radio", { name: "Access keys" }).click(); + await expect(button).toBeDisabled(); + }); + test("GCS connector - HMAC", async ({ page }) => { // Skip test if environment variables are not set if (