From 8b7f4e2dcca74589c387624d6f87a94856e337c5 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 02:46:38 +0800 Subject: [PATCH 001/120] generic multi step auth form --- .../sources/modal/GCSMultiStepForm.svelte | 177 +++++++----------- .../sources/modal/MultiStepAuthForm.svelte | 101 ++++++++++ 2 files changed, 164 insertions(+), 114 deletions(-) create mode 100644 web-common/src/features/sources/modal/MultiStepAuthForm.svelte diff --git a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte index e007489218a..77cadc5735c 100644 --- a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte +++ b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte @@ -1,11 +1,8 @@ - -
-
-
Authentication method
- - - {#if option.value === "credentials"} - - {:else if option.value === "hmac"} -
- - -
- {/if} - - -
- - - {#each filteredParamsProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {#if propertyKey !== "path" && propertyKey !== "google_application_credentials" && propertyKey !== "key_id" && propertyKey !== "secret"} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} + + + {#if option.value === "credentials"} + + {:else if option.value === "hmac"} +
+ +
{/if} - {/each} -
+ + diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte new file mode 100644 index 00000000000..a88defcfe26 --- /dev/null +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -0,0 +1,101 @@ + + + +
+
Authentication method
+ + + + + +
+ + +{#each properties as property (property.key)} + {@const propertyKey = property.key ?? ""} + {#if !excluded.has(propertyKey)} +
+ {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} + onStringInputChange(e)} + alwaysShowError + /> + {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} + + {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} + + {/if} +
+ {/if} +{/each} From 6117dc327a5c653e5d590c76e4c938c2877bcb36 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 02:57:12 +0800 Subject: [PATCH 002/120] inital s3 and azure --- .../features/sources/modal/AddDataForm.svelte | 46 ++++++-- .../sources/modal/AzureMultiStepForm.svelte | 108 ++++++++++++++++++ .../sources/modal/S3MultiStepForm.svelte | 102 +++++++++++++++++ .../src/features/sources/modal/constants.ts | 47 +++++++- 4 files changed, 293 insertions(+), 10 deletions(-) create mode 100644 web-common/src/features/sources/modal/AzureMultiStepForm.svelte create mode 100644 web-common/src/features/sources/modal/S3MultiStepForm.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index ee081dc8f34..61b4daa1d7b 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -23,6 +23,8 @@ import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; + import S3MultiStepForm from "./S3MultiStepForm.svelte"; + import AzureMultiStepForm from "./AzureMultiStepForm.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; @@ -380,22 +382,48 @@ {:else if isMultiStepConnector} {#if stepState.step === "connector"} - + - + {#if connector.name === "gcs"} + + {:else if connector.name === "s3"} + + {:else if connector.name === "azure"} + + {:else} + + {/if} {:else} - + + import Input from "@rilldata/web-common/components/forms/Input.svelte"; + import MultiStepAuthForm from "./MultiStepAuthForm.svelte"; + import { normalizeErrors } from "./utils"; + import { AZURE_AUTH_OPTIONS, type AzureAuthMethod } from "./constants"; + + export let properties: any[] = []; + export let paramsForm: any; + export let paramsErrors: Record; + export let onStringInputChange: (e: Event) => void; + export let handleFileUpload: (file: File) => Promise; + + const filteredParamsProperties = properties; + + const AZURE_CLEAR_FIELDS: Record = { + account_key: ["azure_storage_connection_string", "azure_storage_sas_token"], + sas_token: ["azure_storage_connection_string", "azure_storage_key"], + connection_string: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + ], + }; + + const AZURE_EXCLUDED_KEYS = [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + ]; + + + + + {#if option.value === "account_key"} +
+ + +
+ {:else if option.value === "sas_token"} +
+ + +
+ {:else if option.value === "connection_string"} + + {/if} + + diff --git a/web-common/src/features/sources/modal/S3MultiStepForm.svelte b/web-common/src/features/sources/modal/S3MultiStepForm.svelte new file mode 100644 index 00000000000..09c9ab73061 --- /dev/null +++ b/web-common/src/features/sources/modal/S3MultiStepForm.svelte @@ -0,0 +1,102 @@ + + + + + {#if option.value === "access_keys"} +
+ + +
+ {:else if option.value === "role"} +
+ + + +
+ {/if} + + diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 4bffc52825d..34b052b2dcb 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -39,6 +39,51 @@ export const GCS_AUTH_OPTIONS: { }, ]; +export type S3AuthMethod = "access_keys" | "role"; + +export const S3_AUTH_OPTIONS: { + value: S3AuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "access_keys", + label: "Access keys", + description: "Use AWS access key ID and secret access key.", + }, + { + value: "role", + label: "Assume role", + description: "Assume an AWS IAM role using your local or provided credentials.", + }, +]; + +export type AzureAuthMethod = "account_key" | "sas_token" | "connection_string"; + +export const AZURE_AUTH_OPTIONS: { + value: AzureAuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "account_key", + label: "Access key", + description: "Authenticate with storage account name and access key.", + }, + { + value: "sas_token", + label: "SAS token", + description: "Authenticate with storage account name and SAS token.", + }, + { + value: "connection_string", + label: "Connection string", + description: "Authenticate with a full Azure storage connection string.", + }, +]; + // pre-defined order for sources export const SOURCES = [ "athena", @@ -67,7 +112,7 @@ export const OLAP_ENGINES = [ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) -export const MULTI_STEP_CONNECTORS = ["gcs"]; +export const MULTI_STEP_CONNECTORS = ["gcs", "s3", "azure"]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; From 2e310ac268e5e1fa8d3e21ee2be2406e1c6de607 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:20:45 +0800 Subject: [PATCH 003/120] multi step form renderer --- .../features/sources/modal/AddDataForm.svelte | 33 ++-- .../modal/MultiStepFormRenderer.svelte | 65 +++++++ .../src/features/sources/modal/constants.ts | 183 +++++++++++++++++- .../src/features/sources/modal/types.ts | 33 ++++ .../src/features/sources/modal/yupSchemas.ts | 16 +- 5 files changed, 300 insertions(+), 30 deletions(-) create mode 100644 web-common/src/features/sources/modal/MultiStepFormRenderer.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 61b4daa1d7b..458a6553595 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -15,6 +15,7 @@ import { isEmpty } from "./utils"; import { CONNECTION_TAB_OPTIONS, + multiStepFormConfigs, type ClickHouseConnectorType, } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; @@ -22,9 +23,8 @@ import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; - import S3MultiStepForm from "./S3MultiStepForm.svelte"; - import AzureMultiStepForm from "./AzureMultiStepForm.svelte"; + import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; + import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; @@ -180,6 +180,12 @@ } })(); + $: activeMultiStepConfig = isMultiStepConnector + ? multiStepFormConfigs[ + connector.name as keyof typeof multiStepFormConfigs + ] || null + : null; + $: isSubmitting = submitting; // Reset errors when form is modified @@ -388,24 +394,9 @@ enhance={paramsEnhance} onSubmit={paramsSubmit} > - {#if connector.name === "gcs"} - - {:else if connector.name === "s3"} - - {:else if connector.name === "azure"} - + import CredentialsInput from "@rilldata/web-common/components/forms/CredentialsInput.svelte"; + import Input from "@rilldata/web-common/components/forms/Input.svelte"; + import MultiStepAuthForm from "./MultiStepAuthForm.svelte"; + import { normalizeErrors } from "./utils"; + import { type MultiStepFormConfig } from "./types"; + + export let config: MultiStepFormConfig | null = null; + export let properties: any[] = []; + export let paramsForm: any; + export let paramsErrors: Record; + export let onStringInputChange: (e: Event) => void; + export let handleFileUpload: (file: File) => Promise; + + +{#if config} + + + {#if config.authFieldGroups?.[option.value]} +
+ {#each config.authFieldGroups[option.value] as field (field.id)} + {#if field.type === "credentials"} + + {:else} + + {/if} + {/each} +
+ {/if} + + +{/if} diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 34b052b2dcb..571f4ce310b 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -1,3 +1,5 @@ +import type { MultiStepFormConfig } from "./types"; + export type ClickHouseConnectorType = | "rill-managed" | "self-hosted" @@ -55,7 +57,8 @@ export const S3_AUTH_OPTIONS: { { value: "role", label: "Assume role", - description: "Assume an AWS IAM role using your local or provided credentials.", + description: + "Assume an AWS IAM role using your local or provided credentials.", }, ]; @@ -121,3 +124,181 @@ export const TALL_FORM_CONNECTORS = new Set([ "snowflake", "salesforce", ]); + +export const multiStepFormConfigs: Record = { + gcs: { + authOptions: GCS_AUTH_OPTIONS, + defaultAuthMethod: "credentials", + clearFieldsByMethod: { + credentials: ["key_id", "secret"], + hmac: ["google_application_credentials"], + }, + excludedKeys: ["google_application_credentials", "key_id", "secret"], + authFieldGroups: { + credentials: [ + { + type: "credentials", + id: "google_application_credentials", + optional: false, + hint: "Upload a JSON key file for a service account with GCS access.", + accept: ".json", + }, + ], + hmac: [ + { + type: "input", + id: "key_id", + label: "Access Key ID", + placeholder: "Enter your HMAC access key ID", + optional: false, + secret: true, + hint: "HMAC access key ID for S3-compatible authentication", + }, + { + type: "input", + id: "secret", + label: "Secret Access Key", + placeholder: "Enter your HMAC secret access key", + optional: false, + secret: true, + hint: "HMAC secret access key for S3-compatible authentication", + }, + ], + }, + }, + s3: { + authOptions: S3_AUTH_OPTIONS, + defaultAuthMethod: "access_keys", + clearFieldsByMethod: { + access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], + role: ["aws_access_key_id", "aws_secret_access_key"], + }, + excludedKeys: [ + "aws_access_key_id", + "aws_secret_access_key", + "aws_role_arn", + "aws_role_session_name", + "aws_external_id", + ], + authFieldGroups: { + access_keys: [ + { + type: "input", + id: "aws_access_key_id", + label: "Access Key ID", + placeholder: "Enter AWS access key ID", + optional: false, + secret: true, + hint: "AWS access key ID for the bucket", + }, + { + type: "input", + id: "aws_secret_access_key", + label: "Secret Access Key", + placeholder: "Enter AWS secret access key", + optional: false, + secret: true, + hint: "AWS secret access key for the bucket", + }, + ], + role: [ + { + type: "input", + id: "aws_role_arn", + label: "Role ARN", + placeholder: "Enter AWS IAM role ARN", + optional: false, + secret: true, + hint: "Role ARN to assume for accessing the bucket", + }, + { + type: "input", + id: "aws_role_session_name", + label: "Role session name", + placeholder: "Optional session name (defaults to rill-session)", + optional: true, + }, + { + type: "input", + id: "aws_external_id", + label: "External ID", + placeholder: "Optional external ID for cross-account access", + optional: true, + secret: true, + }, + ], + }, + }, + azure: { + authOptions: AZURE_AUTH_OPTIONS, + defaultAuthMethod: "account_key", + clearFieldsByMethod: { + account_key: [ + "azure_storage_connection_string", + "azure_storage_sas_token", + ], + sas_token: ["azure_storage_connection_string", "azure_storage_key"], + connection_string: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + ], + }, + excludedKeys: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + ], + authFieldGroups: { + account_key: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + hint: "The name of the Azure storage account", + }, + { + type: "input", + id: "azure_storage_key", + label: "Access key", + placeholder: "Enter Azure storage access key", + optional: false, + secret: true, + hint: "Primary or secondary access key for the storage account", + }, + ], + sas_token: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + }, + { + type: "input", + id: "azure_storage_sas_token", + label: "SAS token", + placeholder: "Enter Azure SAS token", + optional: false, + secret: true, + hint: "Shared Access Signature token for the storage account", + }, + ], + connection_string: [ + { + type: "input", + id: "azure_storage_connection_string", + label: "Connection string", + placeholder: "Enter Azure storage connection string", + optional: false, + secret: true, + hint: "Full connection string for the storage account", + }, + ], + }, + }, +}; diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index 5cb0785a5a5..b9cd39a8b1a 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -1,3 +1,36 @@ export type AddDataFormType = "source" | "connector"; export type ConnectorType = "parameters" | "dsn"; + +export type AuthOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export type AuthField = + | { + type: "credentials"; + id: string; + hint?: string; + optional?: boolean; + accept?: string; + } + | { + type: "input"; + id: string; + label: string; + placeholder?: string; + optional?: boolean; + secret?: boolean; + hint?: string; + }; + +export type MultiStepFormConfig = { + authOptions: AuthOption[]; + clearFieldsByMethod: Record; + excludedKeys: string[]; + authFieldGroups: Record; + defaultAuthMethod?: string; +}; diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index b12ec332eae..45e0ad40974 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -11,10 +11,10 @@ export const getYupSchema = { .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") .required("S3 URI is required"), aws_region: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + // name: yup + // .string() + // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + // .required("Source name is required"), }), gcs: yup.object().shape({ @@ -74,10 +74,10 @@ export const getYupSchema = { ) .required("Path is required"), azure_storage_account: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + // name: yup + // .string() + // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + // .required("Source name is required"), }), postgres: yup.object().shape({ From 77749499908d658c39ca28d85251ca756608e69d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:32:23 +0800 Subject: [PATCH 004/120] exclude name from connector form --- web-common/src/features/sources/modal/constants.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 571f4ce310b..53d6b7574ee 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -133,7 +133,12 @@ export const multiStepFormConfigs: Record = { credentials: ["key_id", "secret"], hmac: ["google_application_credentials"], }, - excludedKeys: ["google_application_credentials", "key_id", "secret"], + excludedKeys: [ + "google_application_credentials", + "key_id", + "secret", + "name", + ], authFieldGroups: { credentials: [ { @@ -179,6 +184,7 @@ export const multiStepFormConfigs: Record = { "aws_role_arn", "aws_role_session_name", "aws_external_id", + "name", ], authFieldGroups: { access_keys: [ @@ -249,6 +255,7 @@ export const multiStepFormConfigs: Record = { "azure_storage_key", "azure_storage_sas_token", "azure_storage_connection_string", + "name", ], authFieldGroups: { account_key: [ From ef9474c4ff52a0277eae5001202a010646edabb6 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:39:05 +0800 Subject: [PATCH 005/120] reset --- .../src/features/sources/modal/yupSchemas.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 45e0ad40974..b12ec332eae 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -11,10 +11,10 @@ export const getYupSchema = { .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") .required("S3 URI is required"), aws_region: yup.string(), - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Source name is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Source name is required"), }), gcs: yup.object().shape({ @@ -74,10 +74,10 @@ export const getYupSchema = { ) .required("Path is required"), azure_storage_account: yup.string(), - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Source name is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Source name is required"), }), postgres: yup.object().shape({ From e267c51f5b90b3afd83a2a63502bb8f810e023ae Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:41:56 +0800 Subject: [PATCH 006/120] bump specs --- runtime/drivers/azure/azure.go | 10 +++++---- runtime/drivers/s3/s3.go | 38 +++++++++++++--------------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/runtime/drivers/azure/azure.go b/runtime/drivers/azure/azure.go index dbfd51ffd90..9bcd743bef8 100644 --- a/runtime/drivers/azure/azure.go +++ b/runtime/drivers/azure/azure.go @@ -39,12 +39,14 @@ var spec = drivers.Spec{ Secret: true, }, { - Key: "azure_storage_connection_string", - Type: drivers.StringPropertyType, - Secret: true, + Key: "azure_storage_connection_string", + Type: drivers.StringPropertyType, + DisplayName: "Azure Connection String", + Description: "Azure connection string for storage account", + Placeholder: "Paste your Azure connection string here", + Secret: true, }, }, - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ { Key: "path", diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 90e84dfd0ca..81bd18aaf56 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -26,14 +26,22 @@ var spec = drivers.Spec{ DocsURL: "https://docs.rilldata.com/build/connectors/data-source/s3", ConfigProperties: []*drivers.PropertySpec{ { - Key: "aws_access_key_id", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_access_key_id", + Type: drivers.StringPropertyType, + DisplayName: "AWS access key ID", + Description: "AWS access key ID for explicit credentials", + Placeholder: "Enter your AWS access key ID", + Secret: true, + Required: true, }, { - Key: "aws_secret_access_key", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_secret_access_key", + Type: drivers.StringPropertyType, + DisplayName: "AWS secret access key", + Description: "AWS secret access key for explicit credentials", + Placeholder: "Enter your AWS secret access key", + Secret: true, + Required: true, }, { Key: "region", @@ -53,24 +61,6 @@ var spec = drivers.Spec{ Required: false, Hint: "Overrides the S3 endpoint to connect to. This should only be used to connect to S3 compatible services, such as Cloudflare R2 or MinIO.", }, - { - Key: "aws_role_arn", - Type: drivers.StringPropertyType, - Secret: true, - Description: "AWS Role ARN to assume", - }, - { - Key: "aws_role_session_name", - Type: drivers.StringPropertyType, - Secret: true, - Description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", - }, - { - Key: "aws_external_id", - Type: drivers.StringPropertyType, - Secret: true, - Description: "Optional external ID to use when assuming an AWS role for cross-account access.", - }, }, SourceProperties: []*drivers.PropertySpec{ { From 436823abe6a2b59fe83cde7c512595170ead2b99 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 04:06:15 +0800 Subject: [PATCH 007/120] gate multi step configs buttno --- .../features/sources/modal/AddDataForm.svelte | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 458a6553595..79b5d4fda08 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -134,6 +134,30 @@ let clickhouseShowSaveAnyway: boolean = false; $: isSubmitDisabled = (() => { + // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) + if (isMultiStepConnector && stepState.step === "connector") { + const config = + multiStepFormConfigs[ + connector.name as keyof typeof multiStepFormConfigs + ]; + if (!config) return true; + const groups = Object.values(config.authFieldGroups || {}); + if (!groups.length) return false; + const hasError = (fieldId: string) => + Boolean(($paramsErrors[fieldId] as any)?.length); + const groupSatisfied = groups.some((fields) => + fields.every((field: any) => { + const required = !(field.optional ?? false); + if (!required) return true; + const value = $paramsForm[field.id]; + if (isEmpty(value)) return false; + if (hasError(field.id)) return false; + return true; + }), + ); + return !groupSatisfied; + } + if (onlyDsn || connectionTab === "dsn") { // DSN form: check required DSN properties for (const property of dsnProperties) { From 61c0c641b176b95426adc2a7f9d71f1fe459659c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 04:09:44 +0800 Subject: [PATCH 008/120] clean up --- .../sources/modal/AddDataFormManager.ts | 2 ++ .../features/sources/modal/FormValidation.ts | 18 +++++++++++++++++- .../src/features/sources/modal/yupSchemas.ts | 14 ++++++++++---- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index e2d0f9c4c10..d183eab6265 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -127,6 +127,8 @@ export class AddDataFormManager { // Superforms: params const paramsSchemaDef = getValidationSchemaForConnector( connector.name as string, + formType, + { isMultiStepConnector: this.isMultiStepConnector }, ); const paramsAdapter = yup(paramsSchemaDef); type ParamsOut = YupInfer; diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 92c0f87c338..a3e784ac27c 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,23 @@ import { dsnSchema, getYupSchema } from "./yupSchemas"; +import type { AddDataFormType } from "./types"; export { dsnSchema }; -export function getValidationSchemaForConnector(name: string) { +export function getValidationSchemaForConnector( + name: string, + formType: AddDataFormType, + opts?: { isMultiStepConnector?: boolean }, +) { + const { isMultiStepConnector } = opts || {}; + + // For multi-step source flows, prefer the connector-specific schema when present + // so step 1 (connector) validation doesn't require source-only fields. + if (isMultiStepConnector && formType === "source") { + const connectorKey = `${name}_connector`; + if (connectorKey in getYupSchema) { + return getYupSchema[connectorKey as keyof typeof getYupSchema]; + } + } + return getYupSchema[name as keyof typeof getYupSchema]; } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index b12ec332eae..86d15fb30bc 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,16 +5,22 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { - s3: yup.object().shape({ + s3_connector: yup.object().shape({ + aws_access_key_id: yup.string().required("AWS access key ID is required"), + aws_secret_access_key: yup + .string() + .required("AWS secret access key is required"), + }), + + s3_source: yup.object().shape({ path: yup .string() .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") - .required("S3 URI is required"), - aws_region: yup.string(), + .required(), name: yup .string() .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + .required(), }), gcs: yup.object().shape({ From 984b2c27b4f3ce8974f09cea3c53e8f86537c4d4 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 04:13:02 +0800 Subject: [PATCH 009/120] remove unused --- .../sources/modal/AzureMultiStepForm.svelte | 108 ------------------ .../sources/modal/GCSMultiStepForm.svelte | 78 ------------- .../sources/modal/S3MultiStepForm.svelte | 102 ----------------- 3 files changed, 288 deletions(-) delete mode 100644 web-common/src/features/sources/modal/AzureMultiStepForm.svelte delete mode 100644 web-common/src/features/sources/modal/GCSMultiStepForm.svelte delete mode 100644 web-common/src/features/sources/modal/S3MultiStepForm.svelte diff --git a/web-common/src/features/sources/modal/AzureMultiStepForm.svelte b/web-common/src/features/sources/modal/AzureMultiStepForm.svelte deleted file mode 100644 index d877a939304..00000000000 --- a/web-common/src/features/sources/modal/AzureMultiStepForm.svelte +++ /dev/null @@ -1,108 +0,0 @@ - - - - - {#if option.value === "account_key"} -
- - -
- {:else if option.value === "sas_token"} -
- - -
- {:else if option.value === "connection_string"} - - {/if} - - diff --git a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte deleted file mode 100644 index 77cadc5735c..00000000000 --- a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte +++ /dev/null @@ -1,78 +0,0 @@ - - - - - {#if option.value === "credentials"} - - {:else if option.value === "hmac"} -
- - -
- {/if} - - diff --git a/web-common/src/features/sources/modal/S3MultiStepForm.svelte b/web-common/src/features/sources/modal/S3MultiStepForm.svelte deleted file mode 100644 index 09c9ab73061..00000000000 --- a/web-common/src/features/sources/modal/S3MultiStepForm.svelte +++ /dev/null @@ -1,102 +0,0 @@ - - - - - {#if option.value === "access_keys"} -
- - -
- {:else if option.value === "role"} -
- - - -
- {/if} - - From 0d130f1c764a38ac60b61106af6597399ca58ac9 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 11:47:00 +0800 Subject: [PATCH 010/120] lint, prettier --- .../features/sources/modal/MultiStepAuthForm.svelte | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte index a88defcfe26..15a70f11a18 100644 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -5,13 +5,7 @@ import InformationalField from "@rilldata/web-common/components/forms/InformationalField.svelte"; import { ConnectorDriverPropertyType } from "@rilldata/web-common/runtime-client"; import { normalizeErrors } from "./utils"; - - export type AuthOption = { - value: string; - label: string; - description: string; - hint?: string; - }; + import type { AuthOption } from "./types"; export let properties: any[] = []; export let paramsForm: any; @@ -56,7 +50,7 @@ name="auth-fields" {option} paramsFormStore={paramsForm} - paramsErrors={paramsErrors} + {paramsErrors} {handleFileUpload} /> From e17289c0547bb1b5b0766748bbf5d28991664e74 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 11:55:44 +0800 Subject: [PATCH 011/120] separate the long constant --- .../features/sources/modal/AddDataForm.svelte | 2 +- .../src/features/sources/modal/constants.ts | 187 ----------------- .../sources/modal/multi-step-auth-configs.ts | 191 ++++++++++++++++++ 3 files changed, 192 insertions(+), 188 deletions(-) create mode 100644 web-common/src/features/sources/modal/multi-step-auth-configs.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 79b5d4fda08..553be8ac27c 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -15,7 +15,6 @@ import { isEmpty } from "./utils"; import { CONNECTION_TAB_OPTIONS, - multiStepFormConfigs, type ClickHouseConnectorType, } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; @@ -28,6 +27,7 @@ import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; + import { multiStepFormConfigs } from "./multi-step-auth-configs"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 53d6b7574ee..384abf7777c 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -1,5 +1,3 @@ -import type { MultiStepFormConfig } from "./types"; - export type ClickHouseConnectorType = | "rill-managed" | "self-hosted" @@ -124,188 +122,3 @@ export const TALL_FORM_CONNECTORS = new Set([ "snowflake", "salesforce", ]); - -export const multiStepFormConfigs: Record = { - gcs: { - authOptions: GCS_AUTH_OPTIONS, - defaultAuthMethod: "credentials", - clearFieldsByMethod: { - credentials: ["key_id", "secret"], - hmac: ["google_application_credentials"], - }, - excludedKeys: [ - "google_application_credentials", - "key_id", - "secret", - "name", - ], - authFieldGroups: { - credentials: [ - { - type: "credentials", - id: "google_application_credentials", - optional: false, - hint: "Upload a JSON key file for a service account with GCS access.", - accept: ".json", - }, - ], - hmac: [ - { - type: "input", - id: "key_id", - label: "Access Key ID", - placeholder: "Enter your HMAC access key ID", - optional: false, - secret: true, - hint: "HMAC access key ID for S3-compatible authentication", - }, - { - type: "input", - id: "secret", - label: "Secret Access Key", - placeholder: "Enter your HMAC secret access key", - optional: false, - secret: true, - hint: "HMAC secret access key for S3-compatible authentication", - }, - ], - }, - }, - s3: { - authOptions: S3_AUTH_OPTIONS, - defaultAuthMethod: "access_keys", - clearFieldsByMethod: { - access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], - role: ["aws_access_key_id", "aws_secret_access_key"], - }, - excludedKeys: [ - "aws_access_key_id", - "aws_secret_access_key", - "aws_role_arn", - "aws_role_session_name", - "aws_external_id", - "name", - ], - authFieldGroups: { - access_keys: [ - { - type: "input", - id: "aws_access_key_id", - label: "Access Key ID", - placeholder: "Enter AWS access key ID", - optional: false, - secret: true, - hint: "AWS access key ID for the bucket", - }, - { - type: "input", - id: "aws_secret_access_key", - label: "Secret Access Key", - placeholder: "Enter AWS secret access key", - optional: false, - secret: true, - hint: "AWS secret access key for the bucket", - }, - ], - role: [ - { - type: "input", - id: "aws_role_arn", - label: "Role ARN", - placeholder: "Enter AWS IAM role ARN", - optional: false, - secret: true, - hint: "Role ARN to assume for accessing the bucket", - }, - { - type: "input", - id: "aws_role_session_name", - label: "Role session name", - placeholder: "Optional session name (defaults to rill-session)", - optional: true, - }, - { - type: "input", - id: "aws_external_id", - label: "External ID", - placeholder: "Optional external ID for cross-account access", - optional: true, - secret: true, - }, - ], - }, - }, - azure: { - authOptions: AZURE_AUTH_OPTIONS, - defaultAuthMethod: "account_key", - clearFieldsByMethod: { - account_key: [ - "azure_storage_connection_string", - "azure_storage_sas_token", - ], - sas_token: ["azure_storage_connection_string", "azure_storage_key"], - connection_string: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - ], - }, - excludedKeys: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - "azure_storage_connection_string", - "name", - ], - authFieldGroups: { - account_key: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, - hint: "The name of the Azure storage account", - }, - { - type: "input", - id: "azure_storage_key", - label: "Access key", - placeholder: "Enter Azure storage access key", - optional: false, - secret: true, - hint: "Primary or secondary access key for the storage account", - }, - ], - sas_token: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, - }, - { - type: "input", - id: "azure_storage_sas_token", - label: "SAS token", - placeholder: "Enter Azure SAS token", - optional: false, - secret: true, - hint: "Shared Access Signature token for the storage account", - }, - ], - connection_string: [ - { - type: "input", - id: "azure_storage_connection_string", - label: "Connection string", - placeholder: "Enter Azure storage connection string", - optional: false, - secret: true, - hint: "Full connection string for the storage account", - }, - ], - }, - }, -}; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts new file mode 100644 index 00000000000..361dcb2f580 --- /dev/null +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -0,0 +1,191 @@ +import { + AZURE_AUTH_OPTIONS, + GCS_AUTH_OPTIONS, + S3_AUTH_OPTIONS, +} from "./constants"; +import type { MultiStepFormConfig } from "./types"; + +export const multiStepFormConfigs: Record = { + gcs: { + authOptions: GCS_AUTH_OPTIONS, + defaultAuthMethod: "credentials", + clearFieldsByMethod: { + credentials: ["key_id", "secret"], + hmac: ["google_application_credentials"], + }, + excludedKeys: [ + "google_application_credentials", + "key_id", + "secret", + "name", + ], + authFieldGroups: { + credentials: [ + { + type: "credentials", + id: "google_application_credentials", + optional: false, + hint: "Upload a JSON key file for a service account with GCS access.", + accept: ".json", + }, + ], + hmac: [ + { + type: "input", + id: "key_id", + label: "Access Key ID", + placeholder: "Enter your HMAC access key ID", + optional: false, + secret: true, + hint: "HMAC access key ID for S3-compatible authentication", + }, + { + type: "input", + id: "secret", + label: "Secret Access Key", + placeholder: "Enter your HMAC secret access key", + optional: false, + secret: true, + hint: "HMAC secret access key for S3-compatible authentication", + }, + ], + }, + }, + s3: { + authOptions: S3_AUTH_OPTIONS, + defaultAuthMethod: "access_keys", + clearFieldsByMethod: { + access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], + role: ["aws_access_key_id", "aws_secret_access_key"], + }, + excludedKeys: [ + "aws_access_key_id", + "aws_secret_access_key", + "aws_role_arn", + "aws_role_session_name", + "aws_external_id", + "name", + ], + authFieldGroups: { + access_keys: [ + { + type: "input", + id: "aws_access_key_id", + label: "Access Key ID", + placeholder: "Enter AWS access key ID", + optional: false, + secret: true, + hint: "AWS access key ID for the bucket", + }, + { + type: "input", + id: "aws_secret_access_key", + label: "Secret Access Key", + placeholder: "Enter AWS secret access key", + optional: false, + secret: true, + hint: "AWS secret access key for the bucket", + }, + ], + role: [ + { + type: "input", + id: "aws_role_arn", + label: "Role ARN", + placeholder: "Enter AWS IAM role ARN", + optional: false, + secret: true, + hint: "Role ARN to assume for accessing the bucket", + }, + { + type: "input", + id: "aws_role_session_name", + label: "Role session name", + placeholder: "Optional session name (defaults to rill-session)", + optional: true, + }, + { + type: "input", + id: "aws_external_id", + label: "External ID", + placeholder: "Optional external ID for cross-account access", + optional: true, + secret: true, + }, + ], + }, + }, + azure: { + authOptions: AZURE_AUTH_OPTIONS, + defaultAuthMethod: "account_key", + clearFieldsByMethod: { + account_key: [ + "azure_storage_connection_string", + "azure_storage_sas_token", + ], + sas_token: ["azure_storage_connection_string", "azure_storage_key"], + connection_string: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + ], + }, + excludedKeys: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + "name", + ], + authFieldGroups: { + connection_string: [ + { + type: "input", + id: "azure_storage_connection_string", + label: "Connection string", + placeholder: "Enter Azure storage connection string", + optional: false, + secret: true, + hint: "Full connection string for the storage account", + }, + ], + account_key: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + hint: "The name of the Azure storage account", + }, + { + type: "input", + id: "azure_storage_key", + label: "Access key", + placeholder: "Enter Azure storage access key", + optional: false, + secret: true, + hint: "Primary or secondary access key for the storage account", + }, + ], + sas_token: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + }, + { + type: "input", + id: "azure_storage_sas_token", + label: "SAS token", + placeholder: "Enter Azure SAS token", + optional: false, + secret: true, + hint: "Shared Access Signature token for the storage account", + }, + ], + }, + }, +}; From 7e57ff308a8b690ff19fd087766be6758f206c6b Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:02:52 +0800 Subject: [PATCH 012/120] copy changes --- .../src/features/sources/modal/constants.ts | 18 +++++++++--------- .../sources/modal/multi-step-auth-configs.ts | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 384abf7777c..1b5e70f1bd9 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -69,19 +69,19 @@ export const AZURE_AUTH_OPTIONS: { hint?: string; }[] = [ { - value: "account_key", - label: "Access key", - description: "Authenticate with storage account name and access key.", + value: "connection_string", + label: "Connection String", + description: "Alternative for cloud deployment", }, { - value: "sas_token", - label: "SAS token", - description: "Authenticate with storage account name and SAS token.", + value: "account_key", + label: "Storage Account Key", + description: "Recommended for cloud deployment", }, { - value: "connection_string", - label: "Connection string", - description: "Authenticate with a full Azure storage connection string.", + value: "sas_token", + label: "Shared Access Signature (SAS) Token", + description: "Most secure, fine-grained control", }, ]; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 361dcb2f580..9ef4b7f8dc5 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -146,7 +146,7 @@ export const multiStepFormConfigs: Record = { placeholder: "Enter Azure storage connection string", optional: false, secret: true, - hint: "Full connection string for the storage account", + hint: "Paste an Azure Storage connection string", }, ], account_key: [ From 34e09e948c5efe558ff5071712263cc18ea67251 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:28:20 +0800 Subject: [PATCH 013/120] clean up s3 connector form --- .../sources/modal/MultiStepAuthForm.svelte | 48 +++++++++++++------ .../src/features/sources/modal/constants.ts | 8 +--- .../sources/modal/multi-step-auth-configs.ts | 38 ++++----------- 3 files changed, 45 insertions(+), 49 deletions(-) diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte index 15a70f11a18..27fd21a34b3 100644 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -19,6 +19,10 @@ // Keep auth method local to this component; default to provided value or first option. let authMethod: string = defaultAuthMethod || authOptions?.[0]?.value || ""; + $: hasSingleAuthOption = authOptions?.length === 1; + $: if (hasSingleAuthOption && authOptions?.[0]?.value) { + authMethod = authOptions[0].value; + } // Reactive clearing of fields not relevant to the selected auth method. $: if (authMethod && clearFieldsByMethod[authMethod]?.length) { @@ -42,20 +46,36 @@ -
-
Authentication method
- - - - - -
+{#if !hasSingleAuthOption} +
+
Authentication method
+ + + + + +
+{:else if authOptions?.[0]} +
+ +
+{/if} {#each properties as property (property.key)} diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 1b5e70f1bd9..ec7af49839e 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -39,7 +39,7 @@ export const GCS_AUTH_OPTIONS: { }, ]; -export type S3AuthMethod = "access_keys" | "role"; +export type S3AuthMethod = "access_keys"; export const S3_AUTH_OPTIONS: { value: S3AuthMethod; @@ -52,12 +52,6 @@ export const S3_AUTH_OPTIONS: { label: "Access keys", description: "Use AWS access key ID and secret access key.", }, - { - value: "role", - label: "Assume role", - description: - "Assume an AWS IAM role using your local or provided credentials.", - }, ]; export type AzureAuthMethod = "account_key" | "sas_token" | "connection_string"; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 9ef4b7f8dc5..ab0e3f66687 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -55,17 +55,9 @@ export const multiStepFormConfigs: Record = { authOptions: S3_AUTH_OPTIONS, defaultAuthMethod: "access_keys", clearFieldsByMethod: { - access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], - role: ["aws_access_key_id", "aws_secret_access_key"], + access_keys: [], }, - excludedKeys: [ - "aws_access_key_id", - "aws_secret_access_key", - "aws_role_arn", - "aws_role_session_name", - "aws_external_id", - "name", - ], + excludedKeys: ["aws_access_key_id", "aws_secret_access_key", "name"], authFieldGroups: { access_keys: [ { @@ -86,31 +78,21 @@ export const multiStepFormConfigs: Record = { secret: true, hint: "AWS secret access key for the bucket", }, - ], - role: [ - { - type: "input", - id: "aws_role_arn", - label: "Role ARN", - placeholder: "Enter AWS IAM role ARN", - optional: false, - secret: true, - hint: "Role ARN to assume for accessing the bucket", - }, { type: "input", - id: "aws_role_session_name", - label: "Role session name", - placeholder: "Optional session name (defaults to rill-session)", + id: "aws_region", + label: "Region", + placeholder: "us-east-1", optional: true, + hint: "AWS region for the bucket", }, { type: "input", - id: "aws_external_id", - label: "External ID", - placeholder: "Optional external ID for cross-account access", + id: "aws_endpoint", + label: "Endpoint", + placeholder: "https://s3.example.com", optional: true, - secret: true, + hint: "AWS endpoint for the bucket", }, ], }, From b31f85b8f2e2900755faa8229810a8410bf9f5f9 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:45:41 +0800 Subject: [PATCH 014/120] fix multi step connector preview --- .../features/sources/modal/AddDataFormManager.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index d183eab6265..005d2e098e3 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -394,16 +394,22 @@ export class AddDataFormManager { clickhouseDsnValues, } = ctx; + const connectorPropertiesForPreview = + isMultiStepConnector && stepState?.step === "connector" + ? (connector.configProperties ?? []) + : filteredParamsProperties; + const getConnectorYamlPreview = (values: Record) => { + const orderedProperties = + onlyDsn || connectionTab === "dsn" + ? filteredDsnProperties + : connectorPropertiesForPreview; return compileConnectorYAML(connector, values, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; return !property.noPrompt; }, - orderedProperties: - onlyDsn || connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, + orderedProperties, }); }; From 65676582fb840f83892f9067d8c637208bb9ff19 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:46:50 +0800 Subject: [PATCH 015/120] fix excluded keys of s3 --- .../sources/modal/multi-step-auth-configs.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index ab0e3f66687..aac8fab3900 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -57,7 +57,13 @@ export const multiStepFormConfigs: Record = { clearFieldsByMethod: { access_keys: [], }, - excludedKeys: ["aws_access_key_id", "aws_secret_access_key", "name"], + excludedKeys: [ + "aws_access_key_id", + "aws_secret_access_key", + "region", + "endpoint", + "name", + ], authFieldGroups: { access_keys: [ { @@ -80,19 +86,19 @@ export const multiStepFormConfigs: Record = { }, { type: "input", - id: "aws_region", + id: "region", label: "Region", placeholder: "us-east-1", optional: true, - hint: "AWS region for the bucket", + hint: "Rill uses your default AWS region unless you set it explicitly.", }, { type: "input", - id: "aws_endpoint", + id: "endpoint", label: "Endpoint", placeholder: "https://s3.example.com", optional: true, - hint: "AWS endpoint for the bucket", + hint: "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", }, ], }, From 98e450250c5bb5d62cee1f86099db5200a1f7826 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 15:07:48 +0800 Subject: [PATCH 016/120] spacing between preview and help --- web-common/src/features/sources/modal/AddDataForm.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 553be8ac27c..42cf07beaa1 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -525,7 +525,7 @@
{#if dsnError || paramsError || clickhouseError} Date: Tue, 9 Dec 2025 15:31:14 +0800 Subject: [PATCH 017/120] reorg --- .../sources/modal/submitAddDataForm.ts | 164 +++++++++--------- 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index c6ec540ad6e..d9bdbd6f332 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -154,88 +154,6 @@ async function getOriginalEnvBlob( } } -export async function submitAddSourceForm( - queryClient: QueryClient, - connector: V1ConnectorDriver, - formValues: AddDataFormValues, -): Promise { - const instanceId = get(runtime).instanceId; - await beforeSubmitForm(instanceId, connector); - - const newSourceName = formValues.name as string; - - const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( - connector, - formValues, - ); - - // Make a new .yaml file - const newSourceFilePath = getFileAPIPathFromNameAndType( - newSourceName, - EntityType.Table, - ); - await runtimeServicePutFile(instanceId, { - path: newSourceFilePath, - blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), - create: true, - createOnly: false, - }); - - const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); - - // Create or update the `.env` file - const newEnvBlob = await updateDotEnvWithSecrets( - queryClient, - rewrittenConnector, - rewrittenFormValues, - "source", - ); - - // Make sure the file has reconciled before testing the connection - await runtimeServicePutFileAndWaitForReconciliation(instanceId, { - path: ".env", - blob: newEnvBlob, - create: true, - createOnly: false, - }); - - // Wait for source resource-level reconciliation - // This must happen after .env reconciliation since sources depend on secrets - try { - await waitForResourceReconciliation( - instanceId, - newSourceName, - ResourceKind.Model, - ); - } catch (error) { - // The source file was already created, so we need to delete it - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - const errorDetails = (error as any).details; - - throw { - message: error.message || "Unable to establish a connection", - details: - errorDetails && errorDetails !== error.message - ? errorDetails - : undefined, - }; - } - - // Check for file errors - // If the model file has errors, rollback the changes - const errorMessage = await fileArtifacts.checkFileErrors( - queryClient, - instanceId, - newSourceFilePath, - ); - if (errorMessage) { - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - throw new Error(errorMessage); - } - - await goto(`/files/${newSourceFilePath}`); -} - async function saveConnectorAnyway( queryClient: QueryClient, connector: V1ConnectorDriver, @@ -489,3 +407,85 @@ export async function submitAddConnectorForm( // Wait for the submission to complete await submissionPromise; } + +export async function submitAddSourceForm( + queryClient: QueryClient, + connector: V1ConnectorDriver, + formValues: AddDataFormValues, +): Promise { + const instanceId = get(runtime).instanceId; + await beforeSubmitForm(instanceId, connector); + + const newSourceName = formValues.name as string; + + const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( + connector, + formValues, + ); + + // Make a new .yaml file + const newSourceFilePath = getFileAPIPathFromNameAndType( + newSourceName, + EntityType.Table, + ); + await runtimeServicePutFile(instanceId, { + path: newSourceFilePath, + blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), + create: true, + createOnly: false, + }); + + const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); + + // Create or update the `.env` file + const newEnvBlob = await updateDotEnvWithSecrets( + queryClient, + rewrittenConnector, + rewrittenFormValues, + "source", + ); + + // Make sure the file has reconciled before testing the connection + await runtimeServicePutFileAndWaitForReconciliation(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); + + // Wait for source resource-level reconciliation + // This must happen after .env reconciliation since sources depend on secrets + try { + await waitForResourceReconciliation( + instanceId, + newSourceName, + ResourceKind.Model, + ); + } catch (error) { + // The source file was already created, so we need to delete it + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + const errorDetails = (error as any).details; + + throw { + message: error.message || "Unable to establish a connection", + details: + errorDetails && errorDetails !== error.message + ? errorDetails + : undefined, + }; + } + + // Check for file errors + // If the model file has errors, rollback the changes + const errorMessage = await fileArtifacts.checkFileErrors( + queryClient, + instanceId, + newSourceFilePath, + ); + if (errorMessage) { + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + throw new Error(errorMessage); + } + + await goto(`/files/${newSourceFilePath}`); +} From 779e390787687656e1763b832a097f639ec536ec Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 15:43:02 +0800 Subject: [PATCH 018/120] fix azure schema --- web-common/src/features/sources/modal/yupSchemas.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 86d15fb30bc..e1e84c7f750 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -71,7 +71,14 @@ export const getYupSchema = { .required("Google application credentials is required"), }), - azure: yup.object().shape({ + azure_connector: yup.object().shape({ + azure_storage_account: yup.string().optional(), + azure_storage_key: yup.string().optional(), + azure_storage_sas_token: yup.string().optional(), + azure_storage_connection_string: yup.string().optional(), + }), + + azure_source: yup.object().shape({ path: yup .string() .matches( @@ -79,7 +86,6 @@ export const getYupSchema = { "Must be an Azure URI (e.g. azure://container/path)", ) .required("Path is required"), - azure_storage_account: yup.string(), name: yup .string() .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) From 61ef3c44cd3153d7cc836e28eb3d063a55b45845 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 16:01:32 +0800 Subject: [PATCH 019/120] use public instead of skip in gcs --- .../features/sources/modal/AddDataForm.svelte | 30 ++++++++++++++----- .../sources/modal/AddDataFormManager.ts | 14 +++++++++ .../sources/modal/MultiStepAuthForm.svelte | 3 +- .../modal/MultiStepFormRenderer.svelte | 10 +++++++ .../src/features/sources/modal/constants.ts | 7 ++++- .../sources/modal/multi-step-auth-configs.ts | 2 ++ 6 files changed, 57 insertions(+), 9 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 42cf07beaa1..e60dec8036e 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -304,10 +304,28 @@ ? clickhouseSubmitting && saveAnyway : submitting && saveAnyway; + // Track selected auth method for multi-step connectors to adjust UI labels. + // Only initialize when config becomes available; do not reset after user selection. + let selectedAuthMethod: string = ""; + $: if ( + activeMultiStepConfig && + !selectedAuthMethod && + activeMultiStepConfig.authOptions?.length + ) { + selectedAuthMethod = + activeMultiStepConfig.defaultAuthMethod || + activeMultiStepConfig.authOptions?.[0]?.value || + ""; + } + $: if (!activeMultiStepConfig) { + selectedAuthMethod = ""; + } + handleOnUpdate = formManager.makeOnUpdate({ onClose, queryClient, getConnectionTab: () => connectionTab, + getSelectedAuthMethod: () => selectedAuthMethod, setParamsError: (message: string | null, details?: string) => { paramsError = message; paramsErrorDetails = details; @@ -426,6 +444,7 @@ paramsErrors={$paramsErrors} {onStringInputChange} {handleFileUpload} + bind:authMethod={selectedAuthMethod} /> {:else} {/if} - {#if isMultiStepConnector && stepState.step === "connector"} - - {/if} -
diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 005d2e098e3..559bacdd7aa 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -203,6 +203,7 @@ export class AddDataFormManager { submitting: boolean; clickhouseConnectorType?: ClickHouseConnectorType; clickhouseSubmitting?: boolean; + selectedAuthMethod?: string; }): string { const { isConnectorForm, @@ -210,6 +211,7 @@ export class AddDataFormManager { submitting, clickhouseConnectorType, clickhouseSubmitting, + selectedAuthMethod, } = args; const isClickhouse = this.connector.name === "clickhouse"; @@ -224,6 +226,9 @@ export class AddDataFormManager { if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { + if (selectedAuthMethod === "public") { + return submitting ? "Continuing..." : "Continue"; + } return submitting ? "Testing connection..." : "Test and Connect"; } if (this.isMultiStepConnector && step === "source") { @@ -239,6 +244,7 @@ export class AddDataFormManager { onClose: () => void; queryClient: any; getConnectionTab: () => "parameters" | "dsn"; + getSelectedAuthMethod?: () => string | undefined; setParamsError: (message: string | null, details?: string) => void; setDsnError: (message: string | null, details?: string) => void; setShowSaveAnyway?: (value: boolean) => void; @@ -247,6 +253,7 @@ export class AddDataFormManager { onClose, queryClient, getConnectionTab, + getSelectedAuthMethod, setParamsError, setDsnError, setShowSaveAnyway, @@ -278,6 +285,7 @@ export class AddDataFormManager { if (!event.form.valid) return; const values = event.form.data; + const selectedAuthMethod = getSelectedAuthMethod?.(); try { const stepState = get(connectorStepStore) as ConnectorStepState; @@ -285,6 +293,12 @@ export class AddDataFormManager { await submitAddSourceForm(queryClient, connector, values); onClose(); } else if (isMultiStepConnector && stepState.step === "connector") { + // For public auth, skip Test & Connect and go straight to the next step. + if (selectedAuthMethod === "public") { + setConnectorConfig(values); + setStep("source"); + return; + } await submitAddConnectorForm(queryClient, connector, values, false); setConnectorConfig(values); setStep("source"); diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte index 27fd21a34b3..e927cdb68cd 100644 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -18,7 +18,8 @@ export let excludedKeys: string[] = []; // Keep auth method local to this component; default to provided value or first option. - let authMethod: string = defaultAuthMethod || authOptions?.[0]?.value || ""; + export let authMethod: string = + defaultAuthMethod || authOptions?.[0]?.value || ""; $: hasSingleAuthOption = authOptions?.length === 1; $: if (hasSingleAuthOption && authOptions?.[0]?.value) { authMethod = authOptions[0].value; diff --git a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte b/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte index 0bf95e63347..76cf8e997f7 100644 --- a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte +++ b/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte @@ -11,6 +11,15 @@ export let paramsErrors: Record; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; + + // Bubble the selected auth method to the parent so it can adjust UI. + export let authMethod: string = + config?.defaultAuthMethod || config?.authOptions?.[0]?.value || ""; + + $: if (config && !authMethod) { + authMethod = + config.defaultAuthMethod || config.authOptions?.[0]?.value || ""; + } {#if config} @@ -20,6 +29,7 @@ {paramsErrors} {onStringInputChange} {handleFileUpload} + bind:authMethod authOptions={config.authOptions} defaultAuthMethod={config.defaultAuthMethod || config.authOptions?.[0]?.value} diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index ec7af49839e..1931c9bdd95 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -17,7 +17,7 @@ export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "dsn", label: "Enter connection string" }, ]; -export type GCSAuthMethod = "credentials" | "hmac"; +export type GCSAuthMethod = "public" | "credentials" | "hmac"; export const GCS_AUTH_OPTIONS: { value: GCSAuthMethod; @@ -25,6 +25,11 @@ export const GCS_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, { value: "credentials", label: "GCP credentials", diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index aac8fab3900..cd29f5bf586 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -10,6 +10,7 @@ export const multiStepFormConfigs: Record = { authOptions: GCS_AUTH_OPTIONS, defaultAuthMethod: "credentials", clearFieldsByMethod: { + public: ["google_application_credentials", "key_id", "secret"], credentials: ["key_id", "secret"], hmac: ["google_application_credentials"], }, @@ -20,6 +21,7 @@ export const multiStepFormConfigs: Record = { "name", ], authFieldGroups: { + public: [], credentials: [ { type: "credentials", From f56eea9ddc36a2c71b86cfbe3aaeafaaa7ccf25f Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:11:45 +0800 Subject: [PATCH 020/120] public option for object storage connectors --- .../features/sources/modal/AddDataModal.svelte | 2 +- .../features/sources/modal/FormValidation.ts | 8 ++++++++ .../src/features/sources/modal/constants.ts | 18 ++++++++++++++++-- .../sources/modal/multi-step-auth-configs.ts | 14 ++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 42c8289d1b1..289cdf82720 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -121,7 +121,7 @@ // FIXME: excluding salesforce until we implement the table discovery APIs $: isConnectorType = - selectedConnector?.name === "gcs" || + selectedConnector?.implementsObjectStore || selectedConnector?.implementsOlap || selectedConnector?.implementsSqlStore || (selectedConnector?.implementsWarehouse && diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index a3e784ac27c..ee4bfa7e89a 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -19,5 +19,13 @@ export function getValidationSchemaForConnector( } } + // For multi-step connector step, prefer connector-specific schema when present. + if (isMultiStepConnector && formType === "connector") { + const connectorKey = `${name}_connector`; + if (connectorKey in getYupSchema) { + return getYupSchema[connectorKey as keyof typeof getYupSchema]; + } + } + return getYupSchema[name as keyof typeof getYupSchema]; } diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 1931c9bdd95..7e78fdfa751 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -44,7 +44,7 @@ export const GCS_AUTH_OPTIONS: { }, ]; -export type S3AuthMethod = "access_keys"; +export type S3AuthMethod = "access_keys" | "public"; export const S3_AUTH_OPTIONS: { value: S3AuthMethod; @@ -52,6 +52,11 @@ export const S3_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, { value: "access_keys", label: "Access keys", @@ -59,7 +64,11 @@ export const S3_AUTH_OPTIONS: { }, ]; -export type AzureAuthMethod = "account_key" | "sas_token" | "connection_string"; +export type AzureAuthMethod = + | "account_key" + | "sas_token" + | "connection_string" + | "public"; export const AZURE_AUTH_OPTIONS: { value: AzureAuthMethod; @@ -67,6 +76,11 @@ export const AZURE_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ + { + value: "public", + label: "Public", + description: "Access publicly readable blobs without credentials.", + }, { value: "connection_string", label: "Connection String", diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index cd29f5bf586..a1e2d567662 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -58,6 +58,12 @@ export const multiStepFormConfigs: Record = { defaultAuthMethod: "access_keys", clearFieldsByMethod: { access_keys: [], + public: [ + "aws_access_key_id", + "aws_secret_access_key", + "region", + "endpoint", + ], }, excludedKeys: [ "aws_access_key_id", @@ -103,6 +109,7 @@ export const multiStepFormConfigs: Record = { hint: "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", }, ], + public: [], }, }, azure: { @@ -119,6 +126,12 @@ export const multiStepFormConfigs: Record = { "azure_storage_key", "azure_storage_sas_token", ], + public: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + ], }, excludedKeys: [ "azure_storage_account", @@ -176,6 +189,7 @@ export const multiStepFormConfigs: Record = { hint: "Shared Access Signature token for the storage account", }, ], + public: [], }, }, }; From 7fde9391eab6532ca9689d56d3910393ba0d3617 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:17:43 +0800 Subject: [PATCH 021/120] unblock public in s3 --- web-common/src/features/sources/modal/yupSchemas.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index e1e84c7f750..8110d84d926 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -6,10 +6,10 @@ import { export const getYupSchema = { s3_connector: yup.object().shape({ - aws_access_key_id: yup.string().required("AWS access key ID is required"), - aws_secret_access_key: yup - .string() - .required("AWS secret access key is required"), + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + region: yup.string().optional(), + endpoint: yup.string().optional(), }), s3_source: yup.object().shape({ From f0b21b56081c160f3074f1ddee2056386650131c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:24:00 +0800 Subject: [PATCH 022/120] gate save anyway on multi step connector submission --- .../sources/modal/AddDataFormManager.ts | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 559bacdd7aa..1bb8e1b31d1 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -272,23 +272,29 @@ export class AddDataFormManager { >; result?: Extract; }) => { - // For non-ClickHouse connectors, expose Save Anyway when a submission starts + if (!event.form.valid) return; + + const values = event.form.data; + const selectedAuthMethod = getSelectedAuthMethod?.(); + const stepState = get(connectorStepStore) as ConnectorStepState; + + // For non-ClickHouse connectors, expose Save Anyway when a submission starts, + // but skip for multi-step public auth where we bypass submission. if ( isConnectorForm && connector.name !== "clickhouse" && typeof setShowSaveAnyway === "function" && - event?.result + event?.result && + !( + isMultiStepConnector && + stepState?.step === "connector" && + selectedAuthMethod === "public" + ) ) { setShowSaveAnyway(true); } - if (!event.form.valid) return; - - const values = event.form.data; - const selectedAuthMethod = getSelectedAuthMethod?.(); - try { - const stepState = get(connectorStepStore) as ConnectorStepState; if (isMultiStepConnector && stepState.step === "source") { await submitAddSourceForm(queryClient, connector, values); onClose(); From 096d30d3e9d00c2370dd8b15ab95caaa6fda74d5 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:32:53 +0800 Subject: [PATCH 023/120] import your data right data panel --- .../features/sources/modal/AddDataForm.svelte | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e60dec8036e..5a125fd5775 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -543,29 +543,43 @@
- {#if dsnError || paramsError || clickhouseError} - + {#if dsnError || paramsError || clickhouseError} + + {/if} + + - {/if} - - + + {#if isMultiStepConnector && $connectorStepStore.step === "connector"} +
+ Already connected? +
+ {/if} +
From 7408619bb97d493d818361f6de177c946edae257 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:40:10 +0800 Subject: [PATCH 024/120] import data copy for model step 2 --- web-common/src/features/sources/modal/AddDataForm.svelte | 8 +++++--- .../src/features/sources/modal/AddDataFormManager.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 5a125fd5775..788cb706138 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -519,9 +519,11 @@ : submitting} loadingCopy={connector.name === "clickhouse" ? "Connecting..." - : selectedAuthMethod === "public" - ? "Continuing..." - : "Testing connection..."} + : isMultiStepConnector && stepState.step === "source" + ? "Importing data..." + : selectedAuthMethod === "public" + ? "Continuing..." + : "Testing connection..."} form={connector.name === "clickhouse" ? clickhouseFormId : formId} submitForm type="primary" diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 1bb8e1b31d1..fbd6581f17f 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -232,7 +232,7 @@ export class AddDataFormManager { return submitting ? "Testing connection..." : "Test and Connect"; } if (this.isMultiStepConnector && step === "source") { - return submitting ? "Creating model..." : "Test and Add data"; + return submitting ? "Importing data..." : "Import Data"; } return submitting ? "Testing connection..." : "Test and Connect"; } From 7ccba16f24c419859053368deeee6fc59cac952d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:46:53 +0800 Subject: [PATCH 025/120] gate save anyway on step 2 when import data --- web-common/src/features/sources/modal/AddDataFormManager.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index fbd6581f17f..7a6a8797738 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -278,6 +278,7 @@ export class AddDataFormManager { const selectedAuthMethod = getSelectedAuthMethod?.(); const stepState = get(connectorStepStore) as ConnectorStepState; + // FIXME: simplify this logic // For non-ClickHouse connectors, expose Save Anyway when a submission starts, // but skip for multi-step public auth where we bypass submission. if ( @@ -289,7 +290,9 @@ export class AddDataFormManager { isMultiStepConnector && stepState?.step === "connector" && selectedAuthMethod === "public" - ) + ) && + // Do not show Save Anyway on the model (source) step of multi-step flows. + stepState?.step !== "source" ) { setShowSaveAnyway(true); } From 070df21c6c5e6af4b5e8e4ce6fef25461cbccb56 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 20:05:29 +0800 Subject: [PATCH 026/120] gcs source in yup schema --- web-common/src/features/sources/modal/yupSchemas.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 8110d84d926..710ac3a5bd2 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -23,7 +23,7 @@ export const getYupSchema = { .required(), }), - gcs: yup.object().shape({ + gcs_connector: yup.object().shape({ google_application_credentials: yup.string().optional(), key_id: yup.string().optional(), secret: yup.string().optional(), @@ -33,6 +33,17 @@ export const getYupSchema = { .optional(), }), + gcs_source: yup.object().shape({ + path: yup + .string() + .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") + .optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required(), + }), + https: yup.object().shape({ path: yup .string() From 244e1bb292d78bf90e9b0752b15a46edab7d89b0 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 20:42:02 +0800 Subject: [PATCH 027/120] dynamic validaton for multi step connectors --- .../features/sources/modal/AddDataForm.svelte | 8 ++- .../sources/modal/AddDataFormManager.ts | 17 ++++- .../features/sources/modal/FormValidation.ts | 62 ++++++++++++++++++- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 788cb706138..9f36024ed4e 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -56,6 +56,7 @@ formType, onParamsUpdate: (e: any) => handleOnUpdate(e), onDsnUpdate: (e: any) => handleOnUpdate(e), + getSelectedAuthMethod: () => selectedAuthMethod, }); const isMultiStepConnector = formManager.isMultiStepConnector; @@ -141,7 +142,12 @@ connector.name as keyof typeof multiStepFormConfigs ]; if (!config) return true; - const groups = Object.values(config.authFieldGroups || {}); + // Only validate the currently selected auth method; fall back to default. + const method = + selectedAuthMethod || + config.defaultAuthMethod || + config.authOptions?.[0]?.value; + const groups = method ? [config.authFieldGroups?.[method] || []] : []; if (!groups.length) return false; const hasError = (fieldId: string) => Boolean(($paramsErrors[fieldId] as any)?.length); diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 7a6a8797738..f2ed43ef1f6 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -69,15 +69,25 @@ export class AddDataFormManager { return normalizeConnectorError(this.connector.name ?? "", e); } + private getSelectedAuthMethod?: () => string | undefined; + constructor(args: { connector: V1ConnectorDriver; formType: AddDataFormType; onParamsUpdate: (event: SuperFormUpdateEvent) => void; onDsnUpdate: (event: SuperFormUpdateEvent) => void; + getSelectedAuthMethod?: () => string | undefined; }) { - const { connector, formType, onParamsUpdate, onDsnUpdate } = args; + const { + connector, + formType, + onParamsUpdate, + onDsnUpdate, + getSelectedAuthMethod, + } = args; this.connector = connector; this.formType = formType; + this.getSelectedAuthMethod = getSelectedAuthMethod; // Layout height this.formHeight = TALL_FORM_CONNECTORS.has(connector.name ?? "") @@ -128,7 +138,10 @@ export class AddDataFormManager { const paramsSchemaDef = getValidationSchemaForConnector( connector.name as string, formType, - { isMultiStepConnector: this.isMultiStepConnector }, + { + isMultiStepConnector: this.isMultiStepConnector, + authMethodGetter: this.getSelectedAuthMethod, + }, ); const paramsAdapter = yup(paramsSchemaDef); type ParamsOut = YupInfer; diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index ee4bfa7e89a..a47a004a780 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,14 +1,19 @@ +import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import type { AddDataFormType } from "./types"; +import { multiStepFormConfigs } from "./multi-step-auth-configs"; +import type { AddDataFormType, AuthField } from "./types"; export { dsnSchema }; export function getValidationSchemaForConnector( name: string, formType: AddDataFormType, - opts?: { isMultiStepConnector?: boolean }, + opts?: { + isMultiStepConnector?: boolean; + authMethodGetter?: () => string | undefined; + }, ) { - const { isMultiStepConnector } = opts || {}; + const { isMultiStepConnector, authMethodGetter } = opts || {}; // For multi-step source flows, prefer the connector-specific schema when present // so step 1 (connector) validation doesn't require source-only fields. @@ -21,6 +26,13 @@ export function getValidationSchemaForConnector( // For multi-step connector step, prefer connector-specific schema when present. if (isMultiStepConnector && formType === "connector") { + // Generic dynamic schema based on auth options, driven by config. + const dynamicSchema = makeAuthOptionValidationSchema( + name, + authMethodGetter, + ); + if (dynamicSchema) return dynamicSchema; + const connectorKey = `${name}_connector`; if (connectorKey in getYupSchema) { return getYupSchema[connectorKey as keyof typeof getYupSchema]; @@ -29,3 +41,47 @@ export function getValidationSchemaForConnector( return getYupSchema[name as keyof typeof getYupSchema]; } + +/** + * Build a yup schema that enforces required fields for the selected auth option + * using the multi-step auth config. This keeps validation in sync with the UI + * definitions in constants/multi-step-auth-configs. + */ +function makeAuthOptionValidationSchema( + connectorName: string, + getAuthMethod?: () => string | undefined, +) { + const config = + multiStepFormConfigs[connectorName as keyof typeof multiStepFormConfigs]; + if (!config) return null; + + // Collect all field definitions across auth methods. + const fieldValidations: Record = {}; + + for (const [method, fields] of Object.entries(config.authFieldGroups || {})) { + for (const field of fields as AuthField[]) { + // Only validate concrete input/credential fields. + const required = !(field.optional ?? false); + if (!required) continue; + const label = field.type === "input" ? field.label || field.id : field.id; + // Only apply requirement when the selected auth method matches. + fieldValidations[field.id] = ( + fieldValidations[field.id] || yup.string() + ).test( + `required-${field.id}-${method}`, + `${label} is required`, + (value) => { + if (!getAuthMethod) return true; + const current = getAuthMethod(); + if (current !== method) return true; + return !!value; + }, + ); + } + } + + // If nothing to validate, skip dynamic schema. + if (!Object.keys(fieldValidations).length) return null; + + return yup.object().shape(fieldValidations); +} From 0d1316501562c9e3b8032c5a0924362f201ab00d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 20:47:57 +0800 Subject: [PATCH 028/120] add comments to the fallback schema --- web-common/src/features/sources/modal/yupSchemas.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 710ac3a5bd2..949cfecbbe6 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,6 +5,8 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. s3_connector: yup.object().shape({ aws_access_key_id: yup.string().optional(), aws_secret_access_key: yup.string().optional(), @@ -23,6 +25,8 @@ export const getYupSchema = { .required(), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. gcs_connector: yup.object().shape({ google_application_credentials: yup.string().optional(), key_id: yup.string().optional(), @@ -82,6 +86,9 @@ export const getYupSchema = { .required("Google application credentials is required"), }), + // Keep these optional here; per-auth required fields are enforced dynamically + // via multi-step auth configs. This schema acts as a safe fallback (e.g. source + // step selection of `${name}_connector`). azure_connector: yup.object().shape({ azure_storage_account: yup.string().optional(), azure_storage_key: yup.string().optional(), From ece3b75bcf9c33bfd0e030a6c45f95b08bee0bae Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:02:52 +0800 Subject: [PATCH 029/120] azure and s3 e2e --- .../tests/connectors/test-connection.spec.ts | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/web-local/tests/connectors/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts index e571c62d3bf..437211c26a2 100644 --- a/web-local/tests/connectors/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -4,6 +4,72 @@ import { test } from "../setup/base"; test.describe("Test Connection", () => { test.use({ project: "Blank" }); + test("Azure connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#azure").click(); + await page.waitForSelector('form[id*="azure"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Default method is Storage Account Key -> requires account + key. + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Storage account" }).fill("acct"); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Access key" }).fill("key"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch to Connection String -> requires connection string, so disabled until filled. + await page.getByRole("radio", { name: "Connection String" }).click(); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Connection string" }) + .fill("DefaultEndpointsProtocol=https;"); + await expect(button).toBeEnabled(); + }); + + test("S3 connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#s3").click(); + await page.waitForSelector('form[id*="s3"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Default method is Access keys -> requires access key id + secret. + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch back to Access keys -> fields cleared, so disabled until refilled. + await page.getByRole("radio", { name: "Access keys" }).click(); + await expect(button).toBeDisabled(); + }); + test("GCS connector - HMAC", async ({ page }) => { // Skip test if environment variables are not set if ( From 8883830217509048f4616df68c22c5ed11973155 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:17:58 +0800 Subject: [PATCH 030/120] update default auth method for azure --- .../src/features/sources/modal/multi-step-auth-configs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index a1e2d567662..9307704f3b3 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -114,7 +114,7 @@ export const multiStepFormConfigs: Record = { }, azure: { authOptions: AZURE_AUTH_OPTIONS, - defaultAuthMethod: "account_key", + defaultAuthMethod: "connection_string", clearFieldsByMethod: { account_key: [ "azure_storage_connection_string", From 4cd672c07fdbeeb9d87fab0a8ef8a0a39a5433be Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:30:55 +0800 Subject: [PATCH 031/120] hide save anyway when advacning to the model step for multi step --- web-common/src/features/sources/modal/AddDataForm.svelte | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 9f36024ed4e..f4cf730797d 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -134,6 +134,11 @@ let clickhouseDsnForm; let clickhouseShowSaveAnyway: boolean = false; + // Hide Save Anyway once we advance to the model step in multi-step flows. + $: if (isMultiStepConnector && stepState.step === "source") { + showSaveAnyway = false; + } + $: isSubmitDisabled = (() => { // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) if (isMultiStepConnector && stepState.step === "connector") { From 9e16ef1e1000c1c20612e77fa2dc1acf4db1ae8a Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:49:59 +0800 Subject: [PATCH 032/120] fix e2e --- web-local/tests/connectors/test-connection.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web-local/tests/connectors/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts index 437211c26a2..1ed49d8276c 100644 --- a/web-local/tests/connectors/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -16,7 +16,8 @@ test.describe("Test Connection", () => { name: /(Test and Connect|Continue)/, }); - // Default method is Storage Account Key -> requires account + key. + // Select Storage Account Key (default may be different) -> requires account + key. + await page.getByRole("radio", { name: "Storage Account Key" }).click(); await expect(button).toBeDisabled(); await page.getByRole("textbox", { name: "Storage account" }).fill("acct"); From 000c266fc6bf16ac93bcca925d8154d2d1e68556 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 22:10:39 +0800 Subject: [PATCH 033/120] centralize multi step auth method state to the store --- .../features/sources/modal/AddDataForm.svelte | 50 +++++++++++-------- .../sources/modal/AddDataFormManager.ts | 7 +-- .../sources/modal/connectorStepStore.ts | 16 +++++- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index f4cf730797d..f3a81c0414b 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -19,7 +19,7 @@ } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; - import { connectorStepStore } from "./connectorStepStore"; + import { connectorStepStore, setAuthMethod } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; @@ -28,6 +28,7 @@ import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; import { multiStepFormConfigs } from "./multi-step-auth-configs"; + import { get } from "svelte/store"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -56,13 +57,23 @@ formType, onParamsUpdate: (e: any) => handleOnUpdate(e), onDsnUpdate: (e: any) => handleOnUpdate(e), - getSelectedAuthMethod: () => selectedAuthMethod, + getSelectedAuthMethod: () => + get(connectorStepStore).selectedAuthMethod ?? undefined, }); const isMultiStepConnector = formManager.isMultiStepConnector; const isSourceForm = formManager.isSourceForm; const isConnectorForm = formManager.isConnectorForm; const onlyDsn = hasOnlyDsn(connector, isConnectorForm); + const selectedAuthMethodStore = { + subscribe: (run: (value: string) => void) => + connectorStepStore.subscribe((state) => + run(state.selectedAuthMethod ?? ""), + ), + set: (method: string) => setAuthMethod(method || null), + }; + let selectedAuthMethod: string = ""; + $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; $: stepProperties = isMultiStepConnector && stepState.step === "source" @@ -221,6 +232,22 @@ ] || null : null; + $: if (isMultiStepConnector && activeMultiStepConfig) { + const options = activeMultiStepConfig.authOptions ?? []; + const fallback = + activeMultiStepConfig.defaultAuthMethod || options[0]?.value || null; + const hasValidSelection = options.some( + (option) => option.value === stepState.selectedAuthMethod, + ); + if (!hasValidSelection) { + if (fallback !== stepState.selectedAuthMethod) { + setAuthMethod(fallback ?? null); + } + } + } else if (stepState.selectedAuthMethod) { + setAuthMethod(null); + } + $: isSubmitting = submitting; // Reset errors when form is modified @@ -315,23 +342,6 @@ ? clickhouseSubmitting && saveAnyway : submitting && saveAnyway; - // Track selected auth method for multi-step connectors to adjust UI labels. - // Only initialize when config becomes available; do not reset after user selection. - let selectedAuthMethod: string = ""; - $: if ( - activeMultiStepConfig && - !selectedAuthMethod && - activeMultiStepConfig.authOptions?.length - ) { - selectedAuthMethod = - activeMultiStepConfig.defaultAuthMethod || - activeMultiStepConfig.authOptions?.[0]?.value || - ""; - } - $: if (!activeMultiStepConfig) { - selectedAuthMethod = ""; - } - handleOnUpdate = formManager.makeOnUpdate({ onClose, queryClient, @@ -455,7 +465,7 @@ paramsErrors={$paramsErrors} {onStringInputChange} {handleFileUpload} - bind:authMethod={selectedAuthMethod} + bind:authMethod={$selectedAuthMethodStore} /> {:else} , any, Record>; }; -// Shape of the step store for multi-step connectors -type ConnectorStepState = { - step: "connector" | "source"; - connectorConfig: Record | null; -}; - export class AddDataFormManager { formHeight: string; paramsFormId: string; diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 9ce2451f126..1168a65e532 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -2,12 +2,16 @@ import { writable } from "svelte/store"; export type ConnectorStep = "connector" | "source"; -export const connectorStepStore = writable<{ +export type ConnectorStepState = { step: ConnectorStep; connectorConfig: Record | null; -}>({ + selectedAuthMethod: string | null; +}; + +export const connectorStepStore = writable({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); export function setStep(step: ConnectorStep) { @@ -18,9 +22,17 @@ export function setConnectorConfig(config: Record) { connectorStepStore.update((state) => ({ ...state, connectorConfig: config })); } +export function setAuthMethod(method: string | null) { + connectorStepStore.update((state) => ({ + ...state, + selectedAuthMethod: method, + })); +} + export function resetConnectorStep() { connectorStepStore.set({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); } From c28278d6912c9fe0e65720f34ee375e06a7ea91f Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 23:14:51 +0800 Subject: [PATCH 034/120] clean up save anyway logic --- .../sources/modal/AddDataFormManager.ts | 53 ++++++++++++++----- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 641e2459c8a..17b8abf258a 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -179,6 +179,40 @@ export class AddDataFormManager { return MULTI_STEP_CONNECTORS.includes(this.connector.name ?? ""); } + /** + * Determines whether the "Save Anyway" button should be shown for the current submission. + */ + private shouldShowSaveAnywayButton(args: { + isConnectorForm: boolean; + event?: + | { + result?: Extract; + } + | undefined; + stepState: ConnectorStepState | undefined; + selectedAuthMethod?: string; + }): boolean { + const { isConnectorForm, event, stepState, selectedAuthMethod } = args; + + // Only show for connector forms (not sources) + if (!isConnectorForm) return false; + + // ClickHouse has its own error handling + if (this.connector.name === "clickhouse") return false; + + // Need a submission result to show the button + if (!event?.result) return false; + + // Multi-step connectors: don't show on source step (final step) + if (stepState?.step === "source") return false; + + // Public auth bypasses connection test, so no "Save Anyway" needed + if (stepState?.step === "connector" && selectedAuthMethod === "public") + return false; + + return true; + } + getActiveFormId(args: { connectionTab: "parameters" | "dsn"; onlyDsn: boolean; @@ -286,21 +320,14 @@ export class AddDataFormManager { const selectedAuthMethod = getSelectedAuthMethod?.(); const stepState = get(connectorStepStore) as ConnectorStepState; - // FIXME: simplify this logic - // For non-ClickHouse connectors, expose Save Anyway when a submission starts, - // but skip for multi-step public auth where we bypass submission. if ( - isConnectorForm && - connector.name !== "clickhouse" && typeof setShowSaveAnyway === "function" && - event?.result && - !( - isMultiStepConnector && - stepState?.step === "connector" && - selectedAuthMethod === "public" - ) && - // Do not show Save Anyway on the model (source) step of multi-step flows. - stepState?.step !== "source" + this.shouldShowSaveAnywayButton({ + isConnectorForm, + event, + stepState, + selectedAuthMethod, + }) ) { setShowSaveAnyway(true); } From bc00f4af097e88f0871127a129a3f59855d5cc00 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 23:22:48 +0800 Subject: [PATCH 035/120] save anyway e2e --- .../tests/connectors/save-anyway.spec.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/web-local/tests/connectors/save-anyway.spec.ts b/web-local/tests/connectors/save-anyway.spec.ts index 4fbb190ef5b..38f89955e37 100644 --- a/web-local/tests/connectors/save-anyway.spec.ts +++ b/web-local/tests/connectors/save-anyway.spec.ts @@ -47,4 +47,39 @@ test.describe("Save Anyway feature", () => { await expect(codeEditor).toContainText("type: connector"); await expect(codeEditor).toContainText("driver: clickhouse"); }); + + test("GCS connector - shows Save Anyway after failed test", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#gcs").click(); + + await page.waitForSelector('form[id*="gcs"]'); + + // Use HMAC auth with invalid credentials to force a failure + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page.getByRole("textbox", { name: "Access Key ID" }).fill("bad-id"); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("bad-secret"); + + const saveAnywayButton = page.getByRole("button", { + name: "Save Anyway", + }); + + // Should not be visible before submission + await expect(saveAnywayButton).toBeHidden(); + + await page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }) + .click(); + + // Error should surface, and Save Anyway should now be offered + await expect(page.locator(".error-container")).toBeVisible({ + timeout: 15000, + }); + await expect(saveAnywayButton).toBeVisible({ timeout: 15000 }); + }); }); From 34ddbbe16ea974bcddde6be99b90113a3b604bf1 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 23:59:25 +0800 Subject: [PATCH 036/120] isMultiStepConnectorDisabled --- .../features/sources/modal/AddDataForm.svelte | 33 ++++------------- .../src/features/sources/modal/utils.ts | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index f3a81c0414b..b41c05b063d 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -12,7 +12,7 @@ import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; - import { isEmpty } from "./utils"; + import { hasOnlyDsn, isEmpty, isMultiStepConnectorDisabled } from "./utils"; import { CONNECTION_TAB_OPTIONS, type ClickHouseConnectorType, @@ -25,7 +25,6 @@ import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; - import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; import { multiStepFormConfigs } from "./multi-step-auth-configs"; import { get } from "svelte/store"; @@ -72,6 +71,7 @@ ), set: (method: string) => setAuthMethod(method || null), }; + let selectedAuthMethod: string = ""; $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; @@ -153,31 +153,12 @@ $: isSubmitDisabled = (() => { // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) if (isMultiStepConnector && stepState.step === "connector") { - const config = - multiStepFormConfigs[ - connector.name as keyof typeof multiStepFormConfigs - ]; - if (!config) return true; - // Only validate the currently selected auth method; fall back to default. - const method = - selectedAuthMethod || - config.defaultAuthMethod || - config.authOptions?.[0]?.value; - const groups = method ? [config.authFieldGroups?.[method] || []] : []; - if (!groups.length) return false; - const hasError = (fieldId: string) => - Boolean(($paramsErrors[fieldId] as any)?.length); - const groupSatisfied = groups.some((fields) => - fields.every((field: any) => { - const required = !(field.optional ?? false); - if (!required) return true; - const value = $paramsForm[field.id]; - if (isEmpty(value)) return false; - if (hasError(field.id)) return false; - return true; - }), + return isMultiStepConnectorDisabled( + activeMultiStepConfig, + selectedAuthMethod, + $paramsForm, + $paramsErrors, ); - return !groupSatisfied; } if (onlyDsn || connectionTab === "dsn") { diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 5a7128802d2..3c460ec2a09 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,6 +1,7 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; +import type { MultiStepFormConfig } from "./types"; /** * Returns true for undefined, null, empty string, or whitespace-only string. @@ -83,6 +84,42 @@ export function hasOnlyDsn( return hasDsn && !hasOthers; } +/** + * Returns true when the active multi-step auth method has missing or invalid + * required fields. Falls back to configured default/first auth method. + */ +export function isMultiStepConnectorDisabled( + config: MultiStepFormConfig | null, + selectedMethod: string, + paramsFormValue: Record, + paramsFormErrors: Record, +) { + if (!config) return true; + + const options = config.authOptions ?? []; + const hasValidSelection = options.some((opt) => opt.value === selectedMethod); + const method = + (hasValidSelection && selectedMethod) || + config.defaultAuthMethod || + options[0]?.value; + + if (!method) return true; + + const fields = config.authFieldGroups?.[method] || []; + // If method isn't known or has no fields, only allow when explicitly public. + if (!fields.length) return method === "public"; + + return !fields.every((field) => { + if (field.optional ?? false) return true; + + const value = paramsFormValue[field.id]; + const errorsForField = paramsFormErrors[field.id] as any; + const hasErrors = Boolean(errorsForField?.length); + + return !isEmpty(value) && !hasErrors; + }); +} + /** * Applies ClickHouse Cloud-specific default requirements for connector values. * - For ClickHouse Cloud: enforces `ssl: true` From 11a9ef7018e6ba8bdfb26026d14d4d914e76d4ee Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 10 Dec 2025 00:05:55 +0800 Subject: [PATCH 037/120] reorg public option --- .../src/features/sources/modal/constants.ts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 7e78fdfa751..8e02f8a7b44 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -25,11 +25,6 @@ export const GCS_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ - { - value: "public", - label: "Public", - description: "Access publicly readable buckets without credentials.", - }, { value: "credentials", label: "GCP credentials", @@ -42,6 +37,11 @@ export const GCS_AUTH_OPTIONS: { description: "Use HMAC access key and secret for S3-compatible authentication.", }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, ]; export type S3AuthMethod = "access_keys" | "public"; @@ -52,16 +52,16 @@ export const S3_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ - { - value: "public", - label: "Public", - description: "Access publicly readable buckets without credentials.", - }, { value: "access_keys", label: "Access keys", description: "Use AWS access key ID and secret access key.", }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, ]; export type AzureAuthMethod = @@ -76,11 +76,6 @@ export const AZURE_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ - { - value: "public", - label: "Public", - description: "Access publicly readable blobs without credentials.", - }, { value: "connection_string", label: "Connection String", @@ -96,6 +91,11 @@ export const AZURE_AUTH_OPTIONS: { label: "Shared Access Signature (SAS) Token", description: "Most secure, fine-grained control", }, + { + value: "public", + label: "Public", + description: "Access publicly readable blobs without credentials.", + }, ]; // pre-defined order for sources From cb5d63f78186157136cde9cfaf3659822593b07d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 10 Dec 2025 00:10:49 +0800 Subject: [PATCH 038/120] colocate button labels --- .../sources/modal/AddDataFormManager.ts | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 17b8abf258a..b915299e919 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -42,6 +42,12 @@ type SuperFormUpdateEvent = { form: SuperValidated, any, Record>; }; +const BUTTON_LABELS = { + public: { idle: "Continue", submitting: "Continuing..." }, + connector: { idle: "Test and Connect", submitting: "Testing connection..." }, + source: { idle: "Import Data", submitting: "Importing data..." }, +}; + export class AddDataFormManager { formHeight: string; paramsFormId: string; @@ -269,14 +275,22 @@ export class AddDataFormManager { if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { if (selectedAuthMethod === "public") { - return submitting ? "Continuing..." : "Continue"; + return submitting + ? BUTTON_LABELS.public.submitting + : BUTTON_LABELS.public.idle; } - return submitting ? "Testing connection..." : "Test and Connect"; + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } if (this.isMultiStepConnector && step === "source") { - return submitting ? "Importing data..." : "Import Data"; + return submitting + ? BUTTON_LABELS.source.submitting + : BUTTON_LABELS.source.idle; } - return submitting ? "Testing connection..." : "Test and Connect"; + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } return "Test and Add data"; From 8a0c12f1f38224fefb3fc26e5ec9e52cfcdec5dc Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 10 Dec 2025 01:57:39 +0800 Subject: [PATCH 039/120] fixes --- .../src/features/sources/modal/utils.ts | 7 ++-- .../tests/connectors/save-anyway.spec.ts | 35 ------------------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 3c460ec2a09..2aa9a8fec5d 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -105,9 +105,12 @@ export function isMultiStepConnectorDisabled( if (!method) return true; + // Selecting "public" should always enable the button for multi-step auth flows. + if (method === "public") return false; + const fields = config.authFieldGroups?.[method] || []; - // If method isn't known or has no fields, only allow when explicitly public. - if (!fields.length) return method === "public"; + // Unknown auth methods or ones without fields stay disabled. + if (!fields.length) return true; return !fields.every((field) => { if (field.optional ?? false) return true; diff --git a/web-local/tests/connectors/save-anyway.spec.ts b/web-local/tests/connectors/save-anyway.spec.ts index 38f89955e37..4fbb190ef5b 100644 --- a/web-local/tests/connectors/save-anyway.spec.ts +++ b/web-local/tests/connectors/save-anyway.spec.ts @@ -47,39 +47,4 @@ test.describe("Save Anyway feature", () => { await expect(codeEditor).toContainText("type: connector"); await expect(codeEditor).toContainText("driver: clickhouse"); }); - - test("GCS connector - shows Save Anyway after failed test", async ({ - page, - }) => { - await page.getByRole("button", { name: "Add Asset" }).click(); - await page.getByRole("menuitem", { name: "Add Data" }).click(); - await page.locator("#gcs").click(); - - await page.waitForSelector('form[id*="gcs"]'); - - // Use HMAC auth with invalid credentials to force a failure - await page.getByRole("radio", { name: "HMAC keys" }).click(); - await page.getByRole("textbox", { name: "Access Key ID" }).fill("bad-id"); - await page - .getByRole("textbox", { name: "Secret Access Key" }) - .fill("bad-secret"); - - const saveAnywayButton = page.getByRole("button", { - name: "Save Anyway", - }); - - // Should not be visible before submission - await expect(saveAnywayButton).toBeHidden(); - - await page - .getByRole("dialog") - .getByRole("button", { name: "Test and Connect" }) - .click(); - - // Error should surface, and Save Anyway should now be offered - await expect(page.locator(".error-container")).toBeVisible({ - timeout: 15000, - }); - await expect(saveAnywayButton).toBeVisible({ timeout: 15000 }); - }); }); From d048efd326150a7dcec45a76cec76b1d0865960c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 12 Dec 2025 00:02:57 +0800 Subject: [PATCH 040/120] use onsubmit to fix validation flicker rerender --- web-common/src/features/sources/modal/AddDataFormManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index b915299e919..101cc1f2d93 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -159,6 +159,7 @@ export class AddDataFormManager { validators: paramsAdapter, onUpdate: onParamsUpdate, resetForm: false, + validationMethod: "onsubmit", }); // Superforms: dsn @@ -170,6 +171,7 @@ export class AddDataFormManager { validators: dsnAdapter, onUpdate: onDsnUpdate, resetForm: false, + validationMethod: "onsubmit", }); } From b5e69d92f2f6caa14d86a9a282042063c5f22159 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 12 Dec 2025 00:47:13 +0800 Subject: [PATCH 041/120] fix submission for already connected? --- .../features/sources/modal/AddDataFormManager.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 101cc1f2d93..3a12ac8b682 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -234,7 +234,7 @@ export class AddDataFormManager { handleSkip(): void { const stepState = get(connectorStepStore) as ConnectorStepState; if (!this.isMultiStepConnector || stepState.step !== "connector") return; - setConnectorConfig(get(this.params.form) as Record); + setConnectorConfig({}); setStep("source"); } @@ -330,12 +330,20 @@ export class AddDataFormManager { >; result?: Extract; }) => { - if (!event.form.valid) return; - const values = event.form.data; const selectedAuthMethod = getSelectedAuthMethod?.(); const stepState = get(connectorStepStore) as ConnectorStepState; + // When in the source step of a multi-step flow, the superform still uses + // the connector schema, so it can appear invalid because connector fields + // were intentionally skipped. Allow submission in that case and rely on + // the UI-level required checks for source fields. + if ( + !event.form.valid && + !(isMultiStepConnector && stepState.step === "source") + ) + return; + if ( typeof setShowSaveAnyway === "function" && this.shouldShowSaveAnywayButton({ From 7098e746d7c92f238c8def8eccaa82fa269089ae Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 12 Dec 2025 11:31:20 +0800 Subject: [PATCH 042/120] s3 naming --- .../src/features/entity-management/name-utils.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/web-common/src/features/entity-management/name-utils.ts b/web-common/src/features/entity-management/name-utils.ts index 01f9567a937..52a6d72dc99 100644 --- a/web-common/src/features/entity-management/name-utils.ts +++ b/web-common/src/features/entity-management/name-utils.ts @@ -21,8 +21,22 @@ export function getName(name: string, others: string[]): string { const set = new Set(others.map((other) => other.toLowerCase())); let result = name; + const incrementableSuffix = /(.+)_([0-9]+)$/; while (set.has(result.toLowerCase())) { + // Special-case for "s3": don't roll over to "s4", append suffix instead. + if (name.toLowerCase() === "s3") { + const match = incrementableSuffix.exec(result); + if (match) { + const base = match[1]; + const number = Number.parseInt(match[2], 10) + 1; + result = `${base}_${number}`; + continue; + } + result = `${name}_1`; + continue; + } + result = INCREMENT.exec(result)?.[1] ? result.replace(INCREMENT, (m) => (+m + 1).toString()) : `${result}_1`; From 0c137e72a0515d0e37047286097dd98d310a49d2 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 15 Dec 2025 23:58:16 +0800 Subject: [PATCH 043/120] json schema config --- .../features/sources/modal/AddDataForm.svelte | 11 +- .../features/sources/modal/FormValidation.ts | 32 +- .../sources/modal/multi-step-auth-configs.ts | 671 +++++++++++++----- .../src/features/sources/modal/types.ts | 58 ++ 4 files changed, 567 insertions(+), 205 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index b41c05b063d..2356c325327 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -26,7 +26,7 @@ import { AddDataFormManager } from "./AddDataFormManager"; import AddDataFormSection from "./AddDataFormSection.svelte"; - import { multiStepFormConfigs } from "./multi-step-auth-configs"; + import { getMultiStepFormConfig } from "./multi-step-auth-configs"; import { get } from "svelte/store"; export let connector: V1ConnectorDriver; @@ -207,11 +207,10 @@ } })(); - $: activeMultiStepConfig = isMultiStepConnector - ? multiStepFormConfigs[ - connector.name as keyof typeof multiStepFormConfigs - ] || null - : null; + $: activeMultiStepConfig = + isMultiStepConnector && connector.name + ? getMultiStepFormConfig(connector.name) || null + : null; $: if (isMultiStepConnector && activeMultiStepConfig) { const options = activeMultiStepConfig.authOptions ?? []; diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index a47a004a780..88bc40ef303 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,7 @@ import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import { multiStepFormConfigs } from "./multi-step-auth-configs"; -import type { AddDataFormType, AuthField } from "./types"; +import { getMultiStepFormConfig } from "./multi-step-auth-configs"; +import type { AddDataFormType } from "./types"; export { dsnSchema }; @@ -51,24 +51,26 @@ function makeAuthOptionValidationSchema( connectorName: string, getAuthMethod?: () => string | undefined, ) { - const config = - multiStepFormConfigs[connectorName as keyof typeof multiStepFormConfigs]; + const config = getMultiStepFormConfig(connectorName); if (!config) return null; - // Collect all field definitions across auth methods. const fieldValidations: Record = {}; - for (const [method, fields] of Object.entries(config.authFieldGroups || {})) { - for (const field of fields as AuthField[]) { - // Only validate concrete input/credential fields. - const required = !(field.optional ?? false); - if (!required) continue; - const label = field.type === "input" ? field.label || field.id : field.id; - // Only apply requirement when the selected auth method matches. - fieldValidations[field.id] = ( - fieldValidations[field.id] || yup.string() + for (const [method, fields] of Object.entries( + config.requiredFieldsByMethod || {}, + )) { + for (const fieldId of fields) { + const authField = config.authFieldGroups[method]?.find( + (f) => f.id === fieldId, + ); + const label = + config.fieldLabels[fieldId] || + (authField?.type === "input" ? authField.label : authField?.id) || + fieldId; + fieldValidations[fieldId] = ( + fieldValidations[fieldId] || yup.string() ).test( - `required-${field.id}-${method}`, + `required-${fieldId}-${method}`, `${label} is required`, (value) => { if (!getAuthMethod) return true; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 9307704f3b3..ce2d39e7b7a 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -1,195 +1,498 @@ -import { - AZURE_AUTH_OPTIONS, - GCS_AUTH_OPTIONS, - S3_AUTH_OPTIONS, -} from "./constants"; -import type { MultiStepFormConfig } from "./types"; - -export const multiStepFormConfigs: Record = { - gcs: { - authOptions: GCS_AUTH_OPTIONS, - defaultAuthMethod: "credentials", - clearFieldsByMethod: { - public: ["google_application_credentials", "key_id", "secret"], - credentials: ["key_id", "secret"], - hmac: ["google_application_credentials"], +import type { + AuthField, + AuthOption, + JSONSchemaConditional, + JSONSchemaField, + MultiStepFormConfig, + MultiStepFormSchema, +} from "./types"; + +type VisibleIf = Record< + string, + string | number | boolean | Array +>; + +export const multiStepFormSchemas: Record = { + s3: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + description: "Choose how to authenticate to S3", + enum: ["access_keys", "public"], + default: "access_keys", + "x-display": "radio", + "x-enum-labels": ["Access keys", "Public"], + "x-enum-descriptions": [ + "Use AWS access key ID and secret access key.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + aws_access_key_id: { + type: "string", + title: "Access Key ID", + description: "AWS access key ID for the bucket", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_secret_access_key: { + type: "string", + title: "Secret Access Key", + description: "AWS secret access key for the bucket", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + region: { + type: "string", + title: "Region", + description: + "Rill uses your default AWS region unless you set it explicitly.", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + endpoint: { + type: "string", + title: "Endpoint", + description: + "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", + "x-placeholder": "https://s3.example.com", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + path: { + type: "string", + title: "S3 URI", + description: "Path to your S3 bucket or prefix", + pattern: "^s3://", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-step": "source", + }, }, - excludedKeys: [ - "google_application_credentials", - "key_id", - "secret", - "name", - ], - authFieldGroups: { - public: [], - credentials: [ - { - type: "credentials", - id: "google_application_credentials", - optional: false, - hint: "Upload a JSON key file for a service account with GCS access.", - accept: ".json", - }, - ], - hmac: [ - { - type: "input", - id: "key_id", - label: "Access Key ID", - placeholder: "Enter your HMAC access key ID", - optional: false, - secret: true, - hint: "HMAC access key ID for S3-compatible authentication", - }, - { - type: "input", - id: "secret", - label: "Secret Access Key", - placeholder: "Enter your HMAC secret access key", - optional: false, - secret: true, - hint: "HMAC secret access key for S3-compatible authentication", + allOf: [ + { + if: { properties: { auth_method: { const: "access_keys" } } }, + then: { + required: ["aws_access_key_id", "aws_secret_access_key"], }, - ], - }, + }, + ], }, - s3: { - authOptions: S3_AUTH_OPTIONS, - defaultAuthMethod: "access_keys", - clearFieldsByMethod: { - access_keys: [], - public: [ - "aws_access_key_id", - "aws_secret_access_key", - "region", - "endpoint", - ], + gcs: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["credentials", "hmac", "public"], + default: "credentials", + description: "Choose how to authenticate to GCS", + "x-display": "radio", + "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with GCS access.", + "Use HMAC access key and secret for S3-compatible authentication.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + google_application_credentials: { + type: "string", + title: "Service account key", + description: + "Upload a JSON key file for a service account with GCS access.", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + "x-visible-if": { auth_method: "credentials" }, + }, + key_id: { + type: "string", + title: "Access Key ID", + description: "HMAC access key ID for S3-compatible authentication", + "x-placeholder": "Enter your HMAC access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + secret: { + type: "string", + title: "Secret Access Key", + description: "HMAC secret access key for S3-compatible authentication", + "x-placeholder": "Enter your HMAC secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-step": "source", + }, }, - excludedKeys: [ - "aws_access_key_id", - "aws_secret_access_key", - "region", - "endpoint", - "name", + allOf: [ + { + if: { properties: { auth_method: { const: "credentials" } } }, + then: { required: ["google_application_credentials"] }, + }, + { + if: { properties: { auth_method: { const: "hmac" } } }, + then: { required: ["key_id", "secret"] }, + }, ], - authFieldGroups: { - access_keys: [ - { - type: "input", - id: "aws_access_key_id", - label: "Access Key ID", - placeholder: "Enter AWS access key ID", - optional: false, - secret: true, - hint: "AWS access key ID for the bucket", - }, - { - type: "input", - id: "aws_secret_access_key", - label: "Secret Access Key", - placeholder: "Enter AWS secret access key", - optional: false, - secret: true, - hint: "AWS secret access key for the bucket", - }, - { - type: "input", - id: "region", - label: "Region", - placeholder: "us-east-1", - optional: true, - hint: "Rill uses your default AWS region unless you set it explicitly.", - }, - { - type: "input", - id: "endpoint", - label: "Endpoint", - placeholder: "https://s3.example.com", - optional: true, - hint: "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", - }, - ], - public: [], - }, }, azure: { - authOptions: AZURE_AUTH_OPTIONS, - defaultAuthMethod: "connection_string", - clearFieldsByMethod: { - account_key: [ - "azure_storage_connection_string", - "azure_storage_sas_token", - ], - sas_token: ["azure_storage_connection_string", "azure_storage_key"], - connection_string: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - ], - public: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - "azure_storage_connection_string", - ], + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["connection_string", "account_key", "sas_token", "public"], + default: "connection_string", + description: "Choose how to authenticate to Azure Blob Storage", + "x-display": "radio", + "x-enum-labels": [ + "Connection String", + "Storage Account Key", + "SAS Token", + "Public", + ], + "x-enum-descriptions": [ + "Provide a full Azure Storage connection string.", + "Provide the storage account name and access key.", + "Provide the storage account name and SAS token.", + "Access publicly readable blobs without credentials.", + ], + "x-step": "connector", + }, + azure_storage_connection_string: { + type: "string", + title: "Connection string", + description: "Paste an Azure Storage connection string", + "x-placeholder": "Enter Azure storage connection string", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + azure_storage_account: { + type: "string", + title: "Storage account", + description: "The name of the Azure storage account", + "x-placeholder": "Enter Azure storage account", + "x-step": "connector", + "x-visible-if": { auth_method: ["account_key", "sas_token"] }, + }, + azure_storage_key: { + type: "string", + title: "Access key", + description: "Primary or secondary access key for the storage account", + "x-placeholder": "Enter Azure storage access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "account_key" }, + }, + azure_storage_sas_token: { + type: "string", + title: "SAS token", + description: + "Shared Access Signature token for the storage account (starting with ?sv=)", + "x-placeholder": "Enter Azure SAS token", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "sas_token" }, + }, + path: { + type: "string", + title: "Blob URI", + description: + "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", + pattern: "^https?://", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-step": "source", + }, }, - excludedKeys: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - "azure_storage_connection_string", - "name", - ], - authFieldGroups: { - connection_string: [ - { - type: "input", - id: "azure_storage_connection_string", - label: "Connection string", - placeholder: "Enter Azure storage connection string", - optional: false, - secret: true, - hint: "Paste an Azure Storage connection string", - }, - ], - account_key: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, - hint: "The name of the Azure storage account", - }, - { - type: "input", - id: "azure_storage_key", - label: "Access key", - placeholder: "Enter Azure storage access key", - optional: false, - secret: true, - hint: "Primary or secondary access key for the storage account", - }, - ], - sas_token: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, + allOf: [ + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["azure_storage_connection_string"] }, + }, + { + if: { properties: { auth_method: { const: "account_key" } } }, + then: { required: ["azure_storage_account", "azure_storage_key"] }, + }, + { + if: { properties: { auth_method: { const: "sas_token" } } }, + then: { + required: ["azure_storage_account", "azure_storage_sas_token"], }, - { - type: "input", - id: "azure_storage_sas_token", - label: "SAS token", - placeholder: "Enter Azure SAS token", - optional: false, - secret: true, - hint: "Shared Access Signature token for the storage account", - }, - ], - public: [], - }, + }, + ], }, }; + +export function getMultiStepFormConfig( + connectorName: string, +): MultiStepFormConfig | null { + const schema = + multiStepFormSchemas[connectorName as keyof typeof multiStepFormSchemas]; + if (!schema?.properties) return null; + + const authMethodKey = findAuthMethodKey(schema); + if (!authMethodKey) return null; + + const authProperty = schema.properties[authMethodKey]; + const authOptions = buildAuthOptions(authProperty); + if (!authOptions.length) return null; + + const defaultAuthMethod = + authProperty.default !== undefined && authProperty.default !== null + ? String(authProperty.default) + : authOptions[0]?.value; + + const requiredByMethod = buildRequiredByMethod( + schema, + authMethodKey, + authOptions.map((o) => o.value), + ); + const authFieldGroups = buildAuthFieldGroups( + schema, + authMethodKey, + authOptions, + requiredByMethod, + ); + const excludedKeys = buildExcludedKeys( + schema, + authMethodKey, + authFieldGroups, + ); + const clearFieldsByMethod = buildClearFieldsByMethod( + schema, + authMethodKey, + authOptions, + ); + + return { + schema, + authMethodKey, + authOptions, + defaultAuthMethod: defaultAuthMethod || undefined, + clearFieldsByMethod, + excludedKeys, + authFieldGroups, + requiredFieldsByMethod: requiredByMethod, + fieldLabels: buildFieldLabels(schema), + }; +} + +function findAuthMethodKey(schema: MultiStepFormSchema): string | null { + if (!schema.properties) return null; + for (const [key, value] of Object.entries(schema.properties)) { + if (value.enum && value["x-display"] === "radio") { + return key; + } + } + return schema.properties.auth_method ? "auth_method" : null; +} + +function buildAuthOptions(authProperty: JSONSchemaField): AuthOption[] { + if (!authProperty.enum) return []; + const labels = authProperty["x-enum-labels"] ?? []; + const descriptions = authProperty["x-enum-descriptions"] ?? []; + return authProperty.enum.map((value, idx) => ({ + value: String(value), + label: labels[idx] ?? String(value), + description: + descriptions[idx] ?? authProperty.description ?? "Choose an option", + hint: authProperty["x-hint"], + })); +} + +function buildRequiredByMethod( + schema: MultiStepFormSchema, + authMethodKey: string, + methods: string[], +): Record { + const conditionals = schema.allOf ?? []; + const baseRequired = new Set(schema.required ?? []); + const result: Record = {}; + + for (const method of methods) { + const required = new Set(baseRequired); + for (const conditional of conditionals) { + if (!matchesAuthMethod(conditional, authMethodKey, method)) { + conditional.else?.required?.forEach((field) => required.add(field)); + continue; + } + conditional.then?.required?.forEach((field) => required.add(field)); + } + result[method] = Array.from(required); + } + + return result; +} + +function matchesAuthMethod( + conditional: JSONSchemaConditional, + authMethodKey: string, + method: string, +) { + const constValue = + conditional.if?.properties?.[authMethodKey as keyof VisibleIf]?.const; + if (constValue === undefined || constValue === null) return false; + return String(constValue) === method; +} + +function buildAuthFieldGroups( + schema: MultiStepFormSchema, + authMethodKey: string, + authOptions: AuthOption[], + requiredByMethod: Record, +): Record { + const groups: Record = {}; + const properties = schema.properties ?? {}; + + for (const option of authOptions) { + const required = new Set(requiredByMethod[option.value] ?? []); + for (const [key, prop] of Object.entries(properties)) { + if (key === authMethodKey) continue; + if (!isConnectorStep(prop)) continue; + if (!isVisibleForMethod(prop, authMethodKey, option.value)) continue; + + const field: AuthField = toAuthField(key, prop, required.has(key)); + groups[option.value] = [...(groups[option.value] ?? []), field]; + } + } + + return groups; +} + +function buildClearFieldsByMethod( + schema: MultiStepFormSchema, + authMethodKey: string, + authOptions: AuthOption[], +): Record { + const properties = schema.properties ?? {}; + const clear: Record = {}; + + for (const option of authOptions) { + const fields: string[] = []; + for (const [key, prop] of Object.entries(properties)) { + if (key === authMethodKey) continue; + if (!isVisibleForMethod(prop, authMethodKey, option.value)) { + fields.push(key); + } + } + clear[option.value] = fields; + } + + return clear; +} + +function buildExcludedKeys( + schema: MultiStepFormSchema, + authMethodKey: string, + authFieldGroups: Record, +): string[] { + const excluded = new Set([authMethodKey]); + const properties = schema.properties ?? {}; + const groupedFieldKeys = new Set( + Object.values(authFieldGroups) + .flat() + .map((field) => field.id), + ); + + for (const [key, prop] of Object.entries(properties)) { + const step = prop["x-step"]; + if (step === "source") excluded.add(key); + if (groupedFieldKeys.has(key)) excluded.add(key); + } + + return Array.from(excluded); +} + +function buildFieldLabels(schema: MultiStepFormSchema): Record { + const labels: Record = {}; + for (const [key, prop] of Object.entries(schema.properties ?? {})) { + if (prop.title) labels[key] = prop.title; + } + return labels; +} + +function isConnectorStep(prop: JSONSchemaField): boolean { + return (prop["x-step"] ?? "connector") === "connector"; +} + +function isVisibleForMethod( + prop: JSONSchemaField, + authMethodKey: string, + method: string, +): boolean { + const conditions = prop["x-visible-if"]; + if (!conditions) return true; + + const authCondition = conditions[authMethodKey]; + if (authCondition === undefined) return true; + if (Array.isArray(authCondition)) { + return authCondition.map(String).includes(method); + } + return String(authCondition) === method; +} + +function toAuthField( + key: string, + prop: JSONSchemaField, + isRequired: boolean, +): AuthField { + const base = { + id: key, + optional: !isRequired, + hint: prop.description ?? prop["x-hint"], + }; + + if (prop["x-display"] === "file" || prop.format === "file") { + return { + type: "credentials", + accept: prop["x-accept"], + ...base, + }; + } + + return { + type: "input", + label: prop.title ?? key, + placeholder: prop["x-placeholder"], + secret: prop["x-secret"], + ...base, + }; +} diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index b9cd39a8b1a..d2f87c9e3e2 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -27,10 +27,68 @@ export type AuthField = hint?: string; }; +type JSONSchemaVisibleIfValue = + | string + | number + | boolean + | Array; + +export type JSONSchemaField = { + type?: "string" | "number" | "boolean" | "object"; + title?: string; + description?: string; + enum?: Array; + const?: string | number | boolean; + default?: string | number | boolean; + pattern?: string; + format?: string; + properties?: Record; + required?: string[]; + "x-display"?: "radio" | "select" | "textarea" | "file"; + "x-step"?: "connector" | "source"; + "x-secret"?: boolean; + "x-visible-if"?: Record; + "x-enum-labels"?: string[]; + "x-enum-descriptions"?: string[]; + "x-placeholder"?: string; + "x-hint"?: string; + "x-accept"?: string; +}; + +export type JSONSchemaCondition = { + properties?: Record; +}; + +export type JSONSchemaConstraint = { + required?: string[]; +}; + +export type JSONSchemaConditional = { + if?: JSONSchemaCondition; + then?: JSONSchemaConstraint; + else?: JSONSchemaConstraint; +}; + +export type JSONSchemaObject = { + $schema?: string; + type: "object"; + title?: string; + description?: string; + properties?: Record; + required?: string[]; + allOf?: JSONSchemaConditional[]; +}; + +export type MultiStepFormSchema = JSONSchemaObject; + export type MultiStepFormConfig = { + schema: MultiStepFormSchema; + authMethodKey: string; authOptions: AuthOption[]; clearFieldsByMethod: Record; excludedKeys: string[]; authFieldGroups: Record; + requiredFieldsByMethod: Record; + fieldLabels: Record; defaultAuthMethod?: string; }; From a42aca578ee862082bc2662c5bb68a092bd4d4b8 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 16 Dec 2025 10:39:49 +0800 Subject: [PATCH 044/120] json schema form renderer --- .../features/sources/modal/AddDataForm.svelte | 62 ++-- .../features/sources/modal/FormValidation.ts | 27 +- .../modal/JSONSchemaFormRenderer.svelte | 246 +++++++++++++++ .../sources/modal/MultiStepAuthForm.svelte | 116 ------- .../modal/MultiStepFormRenderer.svelte | 75 ----- .../sources/modal/multi-step-auth-configs.ts | 283 ++++++------------ .../src/features/sources/modal/utils.ts | 34 ++- 7 files changed, 411 insertions(+), 432 deletions(-) create mode 100644 web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte delete mode 100644 web-common/src/features/sources/modal/MultiStepAuthForm.svelte delete mode 100644 web-common/src/features/sources/modal/MultiStepFormRenderer.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 2356c325327..66adcf494a5 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -22,11 +22,14 @@ import { connectorStepStore, setAuthMethod } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; + import JSONSchemaFormRenderer from "./JSONSchemaFormRenderer.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; import AddDataFormSection from "./AddDataFormSection.svelte"; - import { getMultiStepFormConfig } from "./multi-step-auth-configs"; + import { + getAuthOptionsFromSchema, + getConnectorSchema, + } from "./multi-step-auth-configs"; import { get } from "svelte/store"; export let connector: V1ConnectorDriver; @@ -154,7 +157,7 @@ // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) if (isMultiStepConnector && stepState.step === "connector") { return isMultiStepConnectorDisabled( - activeMultiStepConfig, + activeSchema, selectedAuthMethod, $paramsForm, $paramsErrors, @@ -207,15 +210,18 @@ } })(); - $: activeMultiStepConfig = + $: activeSchema = isMultiStepConnector && connector.name - ? getMultiStepFormConfig(connector.name) || null + ? getConnectorSchema(connector.name) || null : null; - $: if (isMultiStepConnector && activeMultiStepConfig) { - const options = activeMultiStepConfig.authOptions ?? []; - const fallback = - activeMultiStepConfig.defaultAuthMethod || options[0]?.value || null; + $: activeAuthInfo = activeSchema + ? getAuthOptionsFromSchema(activeSchema) + : null; + + $: if (isMultiStepConnector && activeAuthInfo) { + const options = activeAuthInfo.options ?? []; + const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; const hasValidSelection = options.some( (option) => option.value === stepState.selectedAuthMethod, ); @@ -437,12 +443,12 @@ enhance={paramsEnhance} onSubmit={paramsSubmit} > - {#if activeMultiStepConfig} - - + {#if activeSchema} + + {:else} + + {/if} {/if} {:else} diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 88bc40ef303..244710a1d25 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,11 @@ import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import { getMultiStepFormConfig } from "./multi-step-auth-configs"; -import type { AddDataFormType } from "./types"; +import { + getConnectorSchema, + getFieldLabel, + getRequiredFieldsByAuthMethod, +} from "./multi-step-auth-configs"; +import type { AddDataFormType, MultiStepFormSchema } from "./types"; export { dsnSchema }; @@ -51,22 +55,17 @@ function makeAuthOptionValidationSchema( connectorName: string, getAuthMethod?: () => string | undefined, ) { - const config = getMultiStepFormConfig(connectorName); - if (!config) return null; + const schema = getConnectorSchema(connectorName); + if (!schema) return null; const fieldValidations: Record = {}; + const requiredByMethod = getRequiredFieldsByAuthMethod(schema, { + step: "connector", + }); - for (const [method, fields] of Object.entries( - config.requiredFieldsByMethod || {}, - )) { + for (const [method, fields] of Object.entries(requiredByMethod || {})) { for (const fieldId of fields) { - const authField = config.authFieldGroups[method]?.find( - (f) => f.id === fieldId, - ); - const label = - config.fieldLabels[fieldId] || - (authField?.type === "input" ? authField.label : authField?.id) || - fieldId; + const label = getFieldLabel(schema as MultiStepFormSchema, fieldId); fieldValidations[fieldId] = ( fieldValidations[fieldId] || yup.string() ).test( diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte new file mode 100644 index 00000000000..5d87829c611 --- /dev/null +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -0,0 +1,246 @@ + + +{#if schema} + {#if step === "connector" && authInfo} + {#if authInfo.options.length > 1} +
+
Authentication method
+ + +
+ {#each visibleFieldsFor(option.value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} +
+ + +
+ {:else if authInfo.options[0]} +
+ {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} +
+ {/if} + {:else} +
+ {#each visibleFieldsFor(authMethod, step) as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} +
+ {/if} +{/if} diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte deleted file mode 100644 index e927cdb68cd..00000000000 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ /dev/null @@ -1,116 +0,0 @@ - - - -{#if !hasSingleAuthOption} -
-
Authentication method
- - - - - -
-{:else if authOptions?.[0]} -
- -
-{/if} - - -{#each properties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {#if !excluded.has(propertyKey)} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/if} -{/each} diff --git a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte b/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte deleted file mode 100644 index 76cf8e997f7..00000000000 --- a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte +++ /dev/null @@ -1,75 +0,0 @@ - - -{#if config} - - - {#if config.authFieldGroups?.[option.value]} -
- {#each config.authFieldGroups[option.value] as field (field.id)} - {#if field.type === "credentials"} - - {:else} - - {/if} - {/each} -
- {/if} - - -{/if} diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index ce2d39e7b7a..83e54701444 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -1,9 +1,7 @@ import type { - AuthField, AuthOption, JSONSchemaConditional, JSONSchemaField, - MultiStepFormConfig, MultiStepFormSchema, } from "./types"; @@ -261,61 +259,16 @@ export const multiStepFormSchemas: Record = { }, }; -export function getMultiStepFormConfig( +export function getConnectorSchema( connectorName: string, -): MultiStepFormConfig | null { +): MultiStepFormSchema | null { const schema = multiStepFormSchemas[connectorName as keyof typeof multiStepFormSchemas]; if (!schema?.properties) return null; - - const authMethodKey = findAuthMethodKey(schema); - if (!authMethodKey) return null; - - const authProperty = schema.properties[authMethodKey]; - const authOptions = buildAuthOptions(authProperty); - if (!authOptions.length) return null; - - const defaultAuthMethod = - authProperty.default !== undefined && authProperty.default !== null - ? String(authProperty.default) - : authOptions[0]?.value; - - const requiredByMethod = buildRequiredByMethod( - schema, - authMethodKey, - authOptions.map((o) => o.value), - ); - const authFieldGroups = buildAuthFieldGroups( - schema, - authMethodKey, - authOptions, - requiredByMethod, - ); - const excludedKeys = buildExcludedKeys( - schema, - authMethodKey, - authFieldGroups, - ); - const clearFieldsByMethod = buildClearFieldsByMethod( - schema, - authMethodKey, - authOptions, - ); - - return { - schema, - authMethodKey, - authOptions, - defaultAuthMethod: defaultAuthMethod || undefined, - clearFieldsByMethod, - excludedKeys, - authFieldGroups, - requiredFieldsByMethod: requiredByMethod, - fieldLabels: buildFieldLabels(schema), - }; + return schema; } -function findAuthMethodKey(schema: MultiStepFormSchema): string | null { +export function findAuthMethodKey(schema: MultiStepFormSchema): string | null { if (!schema.properties) return null; for (const [key, value] of Object.entries(schema.properties)) { if (value.enum && value["x-display"] === "radio") { @@ -325,174 +278,122 @@ function findAuthMethodKey(schema: MultiStepFormSchema): string | null { return schema.properties.auth_method ? "auth_method" : null; } -function buildAuthOptions(authProperty: JSONSchemaField): AuthOption[] { - if (!authProperty.enum) return []; +export function getAuthOptionsFromSchema( + schema: MultiStepFormSchema, +): { key: string; options: AuthOption[]; defaultMethod?: string } | null { + const authMethodKey = findAuthMethodKey(schema); + if (!authMethodKey) return null; + const authProperty = schema.properties?.[authMethodKey]; + if (!authProperty?.enum) return null; + const labels = authProperty["x-enum-labels"] ?? []; const descriptions = authProperty["x-enum-descriptions"] ?? []; - return authProperty.enum.map((value, idx) => ({ - value: String(value), - label: labels[idx] ?? String(value), - description: - descriptions[idx] ?? authProperty.description ?? "Choose an option", - hint: authProperty["x-hint"], - })); + const options = + authProperty.enum?.map((value, idx) => ({ + value: String(value), + label: labels[idx] ?? String(value), + description: + descriptions[idx] ?? authProperty.description ?? "Choose an option", + hint: authProperty["x-hint"], + })) ?? []; + + const defaultMethod = + authProperty.default !== undefined && authProperty.default !== null + ? String(authProperty.default) + : options[0]?.value; + + return { + key: authMethodKey, + options, + defaultMethod: defaultMethod || undefined, + }; } -function buildRequiredByMethod( +export function getRequiredFieldsByAuthMethod( schema: MultiStepFormSchema, - authMethodKey: string, - methods: string[], + opts?: { step?: "connector" | "source" }, ): Record { + const authInfo = getAuthOptionsFromSchema(schema); + if (!authInfo) return {}; + const conditionals = schema.allOf ?? []; const baseRequired = new Set(schema.required ?? []); const result: Record = {}; - for (const method of methods) { - const required = new Set(baseRequired); - for (const conditional of conditionals) { - if (!matchesAuthMethod(conditional, authMethodKey, method)) { - conditional.else?.required?.forEach((field) => required.add(field)); - continue; + for (const option of authInfo.options) { + const required = new Set(); + + // Start with base required fields. + baseRequired.forEach((field) => { + if (!opts?.step || isStepMatch(schema, field, opts.step)) { + required.add(field); } - conditional.then?.required?.forEach((field) => required.add(field)); + }); + + // Apply conditionals. + for (const conditional of conditionals) { + const matches = matchesAuthMethod( + conditional, + authInfo.key, + option.value, + ); + const target = matches ? conditional.then : conditional.else; + target?.required?.forEach((field) => { + if (!opts?.step || isStepMatch(schema, field, opts.step)) { + required.add(field); + } + }); } - result[method] = Array.from(required); + + result[option.value] = Array.from(required); } return result; } -function matchesAuthMethod( - conditional: JSONSchemaConditional, - authMethodKey: string, - method: string, -) { - const constValue = - conditional.if?.properties?.[authMethodKey as keyof VisibleIf]?.const; - if (constValue === undefined || constValue === null) return false; - return String(constValue) === method; -} - -function buildAuthFieldGroups( +export function getFieldLabel( schema: MultiStepFormSchema, - authMethodKey: string, - authOptions: AuthOption[], - requiredByMethod: Record, -): Record { - const groups: Record = {}; - const properties = schema.properties ?? {}; - - for (const option of authOptions) { - const required = new Set(requiredByMethod[option.value] ?? []); - for (const [key, prop] of Object.entries(properties)) { - if (key === authMethodKey) continue; - if (!isConnectorStep(prop)) continue; - if (!isVisibleForMethod(prop, authMethodKey, option.value)) continue; - - const field: AuthField = toAuthField(key, prop, required.has(key)); - groups[option.value] = [...(groups[option.value] ?? []), field]; - } - } - - return groups; + key: string, +): string { + return schema.properties?.[key]?.title || key; } -function buildClearFieldsByMethod( +export function isStepMatch( schema: MultiStepFormSchema, - authMethodKey: string, - authOptions: AuthOption[], -): Record { - const properties = schema.properties ?? {}; - const clear: Record = {}; - - for (const option of authOptions) { - const fields: string[] = []; - for (const [key, prop] of Object.entries(properties)) { - if (key === authMethodKey) continue; - if (!isVisibleForMethod(prop, authMethodKey, option.value)) { - fields.push(key); - } - } - clear[option.value] = fields; - } - - return clear; + key: string, + step: "connector" | "source", +): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; + return (prop["x-step"] ?? "connector") === step; } -function buildExcludedKeys( +export function isVisibleForValues( schema: MultiStepFormSchema, - authMethodKey: string, - authFieldGroups: Record, -): string[] { - const excluded = new Set([authMethodKey]); - const properties = schema.properties ?? {}; - const groupedFieldKeys = new Set( - Object.values(authFieldGroups) - .flat() - .map((field) => field.id), - ); - - for (const [key, prop] of Object.entries(properties)) { - const step = prop["x-step"]; - if (step === "source") excluded.add(key); - if (groupedFieldKeys.has(key)) excluded.add(key); - } - - return Array.from(excluded); -} - -function buildFieldLabels(schema: MultiStepFormSchema): Record { - const labels: Record = {}; - for (const [key, prop] of Object.entries(schema.properties ?? {})) { - if (prop.title) labels[key] = prop.title; - } - return labels; -} - -function isConnectorStep(prop: JSONSchemaField): boolean { - return (prop["x-step"] ?? "connector") === "connector"; -} - -function isVisibleForMethod( - prop: JSONSchemaField, - authMethodKey: string, - method: string, + key: string, + values: Record, ): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; const conditions = prop["x-visible-if"]; if (!conditions) return true; - const authCondition = conditions[authMethodKey]; - if (authCondition === undefined) return true; - if (Array.isArray(authCondition)) { - return authCondition.map(String).includes(method); - } - return String(authCondition) === method; + return Object.entries(conditions).every(([depKey, expected]) => { + const actual = values?.[depKey]; + if (Array.isArray(expected)) { + return expected.map(String).includes(String(actual)); + } + return String(actual) === String(expected); + }); } -function toAuthField( - key: string, - prop: JSONSchemaField, - isRequired: boolean, -): AuthField { - const base = { - id: key, - optional: !isRequired, - hint: prop.description ?? prop["x-hint"], - }; - - if (prop["x-display"] === "file" || prop.format === "file") { - return { - type: "credentials", - accept: prop["x-accept"], - ...base, - }; - } - - return { - type: "input", - label: prop.title ?? key, - placeholder: prop["x-placeholder"], - secret: prop["x-secret"], - ...base, - }; +function matchesAuthMethod( + conditional: JSONSchemaConditional, + authMethodKey: string, + method: string, +) { + const constValue = + conditional.if?.properties?.[authMethodKey as keyof VisibleIf]?.const; + if (constValue === undefined || constValue === null) return false; + return String(constValue) === method; } diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 2aa9a8fec5d..9f69917ebc9 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,7 +1,12 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; -import type { MultiStepFormConfig } from "./types"; +import type { MultiStepFormSchema } from "./types"; +import { + getAuthOptionsFromSchema, + getRequiredFieldsByAuthMethod, + isStepMatch, +} from "./multi-step-auth-configs"; /** * Returns true for undefined, null, empty string, or whitespace-only string. @@ -89,18 +94,19 @@ export function hasOnlyDsn( * required fields. Falls back to configured default/first auth method. */ export function isMultiStepConnectorDisabled( - config: MultiStepFormConfig | null, + schema: MultiStepFormSchema | null, selectedMethod: string, paramsFormValue: Record, paramsFormErrors: Record, ) { - if (!config) return true; + if (!schema) return true; - const options = config.authOptions ?? []; + const authInfo = getAuthOptionsFromSchema(schema); + const options = authInfo?.options ?? []; const hasValidSelection = options.some((opt) => opt.value === selectedMethod); const method = (hasValidSelection && selectedMethod) || - config.defaultAuthMethod || + authInfo?.defaultMethod || options[0]?.value; if (!method) return true; @@ -108,17 +114,17 @@ export function isMultiStepConnectorDisabled( // Selecting "public" should always enable the button for multi-step auth flows. if (method === "public") return false; - const fields = config.authFieldGroups?.[method] || []; - // Unknown auth methods or ones without fields stay disabled. - if (!fields.length) return true; - - return !fields.every((field) => { - if (field.optional ?? false) return true; + const requiredByMethod = getRequiredFieldsByAuthMethod(schema, { + step: "connector", + }); + const requiredFields = requiredByMethod[method] ?? []; + if (!requiredFields.length) return true; - const value = paramsFormValue[field.id]; - const errorsForField = paramsFormErrors[field.id] as any; + return !requiredFields.every((fieldId) => { + if (!isStepMatch(schema, fieldId, "connector")) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; const hasErrors = Boolean(errorsForField?.length); - return !isEmpty(value) && !hasErrors; }); } From 4998d27b1ce7ae08f7e5959857b1ed36d6e776cd Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 16 Dec 2025 17:19:09 +0800 Subject: [PATCH 045/120] missing placeholder texts --- .../src/features/sources/modal/multi-step-auth-configs.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 83e54701444..11472193ae6 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -70,6 +70,7 @@ export const multiStepFormSchemas: Record = { title: "S3 URI", description: "Path to your S3 bucket or prefix", pattern: "^s3://", + "x-placeholder": "s3://bucket/path", "x-step": "source", }, name: { @@ -77,6 +78,7 @@ export const multiStepFormSchemas: Record = { title: "Model name", description: "Name for the source model", pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", "x-step": "source", }, }, @@ -142,6 +144,7 @@ export const multiStepFormSchemas: Record = { title: "GCS URI", description: "Path to your GCS bucket or prefix", pattern: "^gs://", + "x-placeholder": "gs://bucket/path", "x-step": "source", }, name: { @@ -149,6 +152,7 @@ export const multiStepFormSchemas: Record = { title: "Model name", description: "Name for the source model", pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", "x-step": "source", }, }, @@ -230,6 +234,7 @@ export const multiStepFormSchemas: Record = { description: "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", pattern: "^https?://", + "x-placeholder": "https://account.blob.core.windows.net/container", "x-step": "source", }, name: { @@ -237,6 +242,7 @@ export const multiStepFormSchemas: Record = { title: "Model name", description: "Name for the source model", pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", "x-step": "source", }, }, From 4f42995926f1180cda386a585ed5891366d521e0 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 16 Dec 2025 23:14:00 +0800 Subject: [PATCH 046/120] prettier --- .../src/features/sources/modal/JSONSchemaFormRenderer.svelte | 2 +- .../src/features/sources/modal/multi-step-auth-configs.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 5d87829c611..4cd4d7ad1f4 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -4,7 +4,7 @@ import Radio from "@rilldata/web-common/components/forms/Radio.svelte"; import CredentialsInput from "@rilldata/web-common/components/forms/CredentialsInput.svelte"; import { normalizeErrors } from "./utils"; - import type { JSONSchemaField, MultiStepFormSchema } from "./types"; + import type { MultiStepFormSchema } from "./types"; import { findAuthMethodKey, getAuthOptionsFromSchema, diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 11472193ae6..0bdfad7c71d 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -1,7 +1,6 @@ import type { AuthOption, JSONSchemaConditional, - JSONSchemaField, MultiStepFormSchema, } from "./types"; From 0657988e9e21860c67f5e4b636099c601889599d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 00:31:04 +0800 Subject: [PATCH 047/120] fix field spacing --- .../modal/JSONSchemaFormRenderer.svelte | 192 +++++++++--------- 1 file changed, 93 insertions(+), 99 deletions(-) diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 4cd4d7ad1f4..399411b16f6 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -95,108 +95,56 @@ name="multi-auth-method" > -
- {#each visibleFieldsFor(option.value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} -
+ {#each visibleFieldsFor(option.value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} {:else if authInfo.options[0]} -
- {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} -
- {/if} - {:else} -
- {#each visibleFieldsFor(authMethod, step) as [key, prop]} + {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]}
{#if prop["x-display"] === "file" || prop.format === "file"} {/each} -
+ {/if} + {:else} + {#each visibleFieldsFor(authMethod, step) as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} {/if} {/if} From 2432eda590770279243b6f31a7a42afad5a733b1 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 21:45:35 +0800 Subject: [PATCH 048/120] individual schemas --- .../sources/modal/multi-step-auth-configs.ts | 258 +----------------- .../features/sources/modal/schemas/azure.ts | 98 +++++++ .../src/features/sources/modal/schemas/gcs.ts | 78 ++++++ .../src/features/sources/modal/schemas/s3.ts | 82 ++++++ 4 files changed, 264 insertions(+), 252 deletions(-) create mode 100644 web-common/src/features/sources/modal/schemas/azure.ts create mode 100644 web-common/src/features/sources/modal/schemas/gcs.ts create mode 100644 web-common/src/features/sources/modal/schemas/s3.ts diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 0bdfad7c71d..561cad4d708 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -3,6 +3,9 @@ import type { JSONSchemaConditional, MultiStepFormSchema, } from "./types"; +import { azureSchema } from "./schemas/azure"; +import { gcsSchema } from "./schemas/gcs"; +import { s3Schema } from "./schemas/s3"; type VisibleIf = Record< string, @@ -10,258 +13,9 @@ type VisibleIf = Record< >; export const multiStepFormSchemas: Record = { - s3: { - $schema: "http://json-schema.org/draft-07/schema#", - type: "object", - properties: { - auth_method: { - type: "string", - title: "Authentication method", - description: "Choose how to authenticate to S3", - enum: ["access_keys", "public"], - default: "access_keys", - "x-display": "radio", - "x-enum-labels": ["Access keys", "Public"], - "x-enum-descriptions": [ - "Use AWS access key ID and secret access key.", - "Access publicly readable buckets without credentials.", - ], - "x-step": "connector", - }, - aws_access_key_id: { - type: "string", - title: "Access Key ID", - description: "AWS access key ID for the bucket", - "x-placeholder": "Enter AWS access key ID", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - aws_secret_access_key: { - type: "string", - title: "Secret Access Key", - description: "AWS secret access key for the bucket", - "x-placeholder": "Enter AWS secret access key", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - region: { - type: "string", - title: "Region", - description: - "Rill uses your default AWS region unless you set it explicitly.", - "x-placeholder": "us-east-1", - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - endpoint: { - type: "string", - title: "Endpoint", - description: - "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", - "x-placeholder": "https://s3.example.com", - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - path: { - type: "string", - title: "S3 URI", - description: "Path to your S3 bucket or prefix", - pattern: "^s3://", - "x-placeholder": "s3://bucket/path", - "x-step": "source", - }, - name: { - type: "string", - title: "Model name", - description: "Name for the source model", - pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", - "x-placeholder": "my_model", - "x-step": "source", - }, - }, - allOf: [ - { - if: { properties: { auth_method: { const: "access_keys" } } }, - then: { - required: ["aws_access_key_id", "aws_secret_access_key"], - }, - }, - ], - }, - gcs: { - $schema: "http://json-schema.org/draft-07/schema#", - type: "object", - properties: { - auth_method: { - type: "string", - title: "Authentication method", - enum: ["credentials", "hmac", "public"], - default: "credentials", - description: "Choose how to authenticate to GCS", - "x-display": "radio", - "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], - "x-enum-descriptions": [ - "Upload a JSON key file for a service account with GCS access.", - "Use HMAC access key and secret for S3-compatible authentication.", - "Access publicly readable buckets without credentials.", - ], - "x-step": "connector", - }, - google_application_credentials: { - type: "string", - title: "Service account key", - description: - "Upload a JSON key file for a service account with GCS access.", - format: "file", - "x-display": "file", - "x-accept": ".json", - "x-step": "connector", - "x-visible-if": { auth_method: "credentials" }, - }, - key_id: { - type: "string", - title: "Access Key ID", - description: "HMAC access key ID for S3-compatible authentication", - "x-placeholder": "Enter your HMAC access key ID", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "hmac" }, - }, - secret: { - type: "string", - title: "Secret Access Key", - description: "HMAC secret access key for S3-compatible authentication", - "x-placeholder": "Enter your HMAC secret access key", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "hmac" }, - }, - path: { - type: "string", - title: "GCS URI", - description: "Path to your GCS bucket or prefix", - pattern: "^gs://", - "x-placeholder": "gs://bucket/path", - "x-step": "source", - }, - name: { - type: "string", - title: "Model name", - description: "Name for the source model", - pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", - "x-placeholder": "my_model", - "x-step": "source", - }, - }, - allOf: [ - { - if: { properties: { auth_method: { const: "credentials" } } }, - then: { required: ["google_application_credentials"] }, - }, - { - if: { properties: { auth_method: { const: "hmac" } } }, - then: { required: ["key_id", "secret"] }, - }, - ], - }, - azure: { - $schema: "http://json-schema.org/draft-07/schema#", - type: "object", - properties: { - auth_method: { - type: "string", - title: "Authentication method", - enum: ["connection_string", "account_key", "sas_token", "public"], - default: "connection_string", - description: "Choose how to authenticate to Azure Blob Storage", - "x-display": "radio", - "x-enum-labels": [ - "Connection String", - "Storage Account Key", - "SAS Token", - "Public", - ], - "x-enum-descriptions": [ - "Provide a full Azure Storage connection string.", - "Provide the storage account name and access key.", - "Provide the storage account name and SAS token.", - "Access publicly readable blobs without credentials.", - ], - "x-step": "connector", - }, - azure_storage_connection_string: { - type: "string", - title: "Connection string", - description: "Paste an Azure Storage connection string", - "x-placeholder": "Enter Azure storage connection string", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "connection_string" }, - }, - azure_storage_account: { - type: "string", - title: "Storage account", - description: "The name of the Azure storage account", - "x-placeholder": "Enter Azure storage account", - "x-step": "connector", - "x-visible-if": { auth_method: ["account_key", "sas_token"] }, - }, - azure_storage_key: { - type: "string", - title: "Access key", - description: "Primary or secondary access key for the storage account", - "x-placeholder": "Enter Azure storage access key", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "account_key" }, - }, - azure_storage_sas_token: { - type: "string", - title: "SAS token", - description: - "Shared Access Signature token for the storage account (starting with ?sv=)", - "x-placeholder": "Enter Azure SAS token", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "sas_token" }, - }, - path: { - type: "string", - title: "Blob URI", - description: - "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", - pattern: "^https?://", - "x-placeholder": "https://account.blob.core.windows.net/container", - "x-step": "source", - }, - name: { - type: "string", - title: "Model name", - description: "Name for the source model", - pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", - "x-placeholder": "my_model", - "x-step": "source", - }, - }, - allOf: [ - { - if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["azure_storage_connection_string"] }, - }, - { - if: { properties: { auth_method: { const: "account_key" } } }, - then: { required: ["azure_storage_account", "azure_storage_key"] }, - }, - { - if: { properties: { auth_method: { const: "sas_token" } } }, - then: { - required: ["azure_storage_account", "azure_storage_sas_token"], - }, - }, - ], - }, + s3: s3Schema, + gcs: gcsSchema, + azure: azureSchema, }; export function getConnectorSchema( diff --git a/web-common/src/features/sources/modal/schemas/azure.ts b/web-common/src/features/sources/modal/schemas/azure.ts new file mode 100644 index 00000000000..853ba021e47 --- /dev/null +++ b/web-common/src/features/sources/modal/schemas/azure.ts @@ -0,0 +1,98 @@ +import type { MultiStepFormSchema } from "../types"; + +export const azureSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["connection_string", "account_key", "sas_token", "public"], + default: "connection_string", + description: "Choose how to authenticate to Azure Blob Storage", + "x-display": "radio", + "x-enum-labels": [ + "Connection String", + "Storage Account Key", + "SAS Token", + "Public", + ], + "x-enum-descriptions": [ + "Provide a full Azure Storage connection string.", + "Provide the storage account name and access key.", + "Provide the storage account name and SAS token.", + "Access publicly readable blobs without credentials.", + ], + "x-step": "connector", + }, + azure_storage_connection_string: { + type: "string", + title: "Connection string", + description: "Paste an Azure Storage connection string", + "x-placeholder": "Enter Azure storage connection string", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + azure_storage_account: { + type: "string", + title: "Storage account", + description: "The name of the Azure storage account", + "x-placeholder": "Enter Azure storage account", + "x-step": "connector", + "x-visible-if": { auth_method: ["account_key", "sas_token"] }, + }, + azure_storage_key: { + type: "string", + title: "Access key", + description: "Primary or secondary access key for the storage account", + "x-placeholder": "Enter Azure storage access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "account_key" }, + }, + azure_storage_sas_token: { + type: "string", + title: "SAS token", + description: + "Shared Access Signature token for the storage account (starting with ?sv=)", + "x-placeholder": "Enter Azure SAS token", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "sas_token" }, + }, + path: { + type: "string", + title: "Blob URI", + description: + "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", + pattern: "^https?://", + "x-placeholder": "https://account.blob.core.windows.net/container", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["azure_storage_connection_string"] }, + }, + { + if: { properties: { auth_method: { const: "account_key" } } }, + then: { required: ["azure_storage_account", "azure_storage_key"] }, + }, + { + if: { properties: { auth_method: { const: "sas_token" } } }, + then: { + required: ["azure_storage_account", "azure_storage_sas_token"], + }, + }, + ], +}; diff --git a/web-common/src/features/sources/modal/schemas/gcs.ts b/web-common/src/features/sources/modal/schemas/gcs.ts new file mode 100644 index 00000000000..bb5d7569b3c --- /dev/null +++ b/web-common/src/features/sources/modal/schemas/gcs.ts @@ -0,0 +1,78 @@ +import type { MultiStepFormSchema } from "../types"; + +export const gcsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["credentials", "hmac", "public"], + default: "credentials", + description: "Choose how to authenticate to GCS", + "x-display": "radio", + "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with GCS access.", + "Use HMAC access key and secret for S3-compatible authentication.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + google_application_credentials: { + type: "string", + title: "Service account key", + description: + "Upload a JSON key file for a service account with GCS access.", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + "x-visible-if": { auth_method: "credentials" }, + }, + key_id: { + type: "string", + title: "Access Key ID", + description: "HMAC access key ID for S3-compatible authentication", + "x-placeholder": "Enter your HMAC access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + secret: { + type: "string", + title: "Secret Access Key", + description: "HMAC secret access key for S3-compatible authentication", + "x-placeholder": "Enter your HMAC secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://", + "x-placeholder": "gs://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "credentials" } } }, + then: { required: ["google_application_credentials"] }, + }, + { + if: { properties: { auth_method: { const: "hmac" } } }, + then: { required: ["key_id", "secret"] }, + }, + ], +}; diff --git a/web-common/src/features/sources/modal/schemas/s3.ts b/web-common/src/features/sources/modal/schemas/s3.ts new file mode 100644 index 00000000000..0e29927d306 --- /dev/null +++ b/web-common/src/features/sources/modal/schemas/s3.ts @@ -0,0 +1,82 @@ +import type { MultiStepFormSchema } from "../types"; + +export const s3Schema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + description: "Choose how to authenticate to S3", + enum: ["access_keys", "public"], + default: "access_keys", + "x-display": "radio", + "x-enum-labels": ["Access keys", "Public"], + "x-enum-descriptions": [ + "Use AWS access key ID and secret access key.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + aws_access_key_id: { + type: "string", + title: "Access Key ID", + description: "AWS access key ID for the bucket", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_secret_access_key: { + type: "string", + title: "Secret Access Key", + description: "AWS secret access key for the bucket", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + region: { + type: "string", + title: "Region", + description: + "Rill uses your default AWS region unless you set it explicitly.", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + endpoint: { + type: "string", + title: "Endpoint", + description: + "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", + "x-placeholder": "https://s3.example.com", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + path: { + type: "string", + title: "S3 URI", + description: "Path to your S3 bucket or prefix", + pattern: "^s3://", + "x-placeholder": "s3://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "access_keys" } } }, + then: { + required: ["aws_access_key_id", "aws_secret_access_key"], + }, + }, + ], +}; From fc0fe6ef45b193349b3822ea7c6d210cd279d9ec Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 22:04:04 +0800 Subject: [PATCH 049/120] templates reorg --- .../sources/modal/multi-step-auth-configs.ts | 6 +- .../src/features/sources/modal/types.ts | 65 ++++--------------- .../modal => templates}/schemas/azure.ts | 2 +- .../modal => templates}/schemas/gcs.ts | 2 +- .../modal => templates}/schemas/s3.ts | 2 +- .../src/features/templates/schemas/types.ts | 53 +++++++++++++++ 6 files changed, 70 insertions(+), 60 deletions(-) rename web-common/src/features/{sources/modal => templates}/schemas/azure.ts (98%) rename web-common/src/features/{sources/modal => templates}/schemas/gcs.ts (97%) rename web-common/src/features/{sources/modal => templates}/schemas/s3.ts (97%) create mode 100644 web-common/src/features/templates/schemas/types.ts diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 561cad4d708..88566546e22 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -3,9 +3,9 @@ import type { JSONSchemaConditional, MultiStepFormSchema, } from "./types"; -import { azureSchema } from "./schemas/azure"; -import { gcsSchema } from "./schemas/gcs"; -import { s3Schema } from "./schemas/s3"; +import { azureSchema } from "../../templates/schemas/azure"; +import { gcsSchema } from "../../templates/schemas/gcs"; +import { s3Schema } from "../../templates/schemas/s3"; type VisibleIf = Record< string, diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index d2f87c9e3e2..8fd405fe6de 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -1,3 +1,14 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; + +export type { + JSONSchemaCondition, + JSONSchemaConditional, + JSONSchemaConstraint, + JSONSchemaField, + JSONSchemaObject, + MultiStepFormSchema, +} from "../../templates/schemas/types"; + export type AddDataFormType = "source" | "connector"; export type ConnectorType = "parameters" | "dsn"; @@ -27,60 +38,6 @@ export type AuthField = hint?: string; }; -type JSONSchemaVisibleIfValue = - | string - | number - | boolean - | Array; - -export type JSONSchemaField = { - type?: "string" | "number" | "boolean" | "object"; - title?: string; - description?: string; - enum?: Array; - const?: string | number | boolean; - default?: string | number | boolean; - pattern?: string; - format?: string; - properties?: Record; - required?: string[]; - "x-display"?: "radio" | "select" | "textarea" | "file"; - "x-step"?: "connector" | "source"; - "x-secret"?: boolean; - "x-visible-if"?: Record; - "x-enum-labels"?: string[]; - "x-enum-descriptions"?: string[]; - "x-placeholder"?: string; - "x-hint"?: string; - "x-accept"?: string; -}; - -export type JSONSchemaCondition = { - properties?: Record; -}; - -export type JSONSchemaConstraint = { - required?: string[]; -}; - -export type JSONSchemaConditional = { - if?: JSONSchemaCondition; - then?: JSONSchemaConstraint; - else?: JSONSchemaConstraint; -}; - -export type JSONSchemaObject = { - $schema?: string; - type: "object"; - title?: string; - description?: string; - properties?: Record; - required?: string[]; - allOf?: JSONSchemaConditional[]; -}; - -export type MultiStepFormSchema = JSONSchemaObject; - export type MultiStepFormConfig = { schema: MultiStepFormSchema; authMethodKey: string; diff --git a/web-common/src/features/sources/modal/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts similarity index 98% rename from web-common/src/features/sources/modal/schemas/azure.ts rename to web-common/src/features/templates/schemas/azure.ts index 853ba021e47..c48aec3fc2b 100644 --- a/web-common/src/features/sources/modal/schemas/azure.ts +++ b/web-common/src/features/templates/schemas/azure.ts @@ -1,4 +1,4 @@ -import type { MultiStepFormSchema } from "../types"; +import type { MultiStepFormSchema } from "./types"; export const azureSchema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", diff --git a/web-common/src/features/sources/modal/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts similarity index 97% rename from web-common/src/features/sources/modal/schemas/gcs.ts rename to web-common/src/features/templates/schemas/gcs.ts index bb5d7569b3c..d41614abf47 100644 --- a/web-common/src/features/sources/modal/schemas/gcs.ts +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -1,4 +1,4 @@ -import type { MultiStepFormSchema } from "../types"; +import type { MultiStepFormSchema } from "./types"; export const gcsSchema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", diff --git a/web-common/src/features/sources/modal/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts similarity index 97% rename from web-common/src/features/sources/modal/schemas/s3.ts rename to web-common/src/features/templates/schemas/s3.ts index 0e29927d306..70dd72c0c61 100644 --- a/web-common/src/features/sources/modal/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -1,4 +1,4 @@ -import type { MultiStepFormSchema } from "../types"; +import type { MultiStepFormSchema } from "./types"; export const s3Schema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts new file mode 100644 index 00000000000..3c21dc4d1fa --- /dev/null +++ b/web-common/src/features/templates/schemas/types.ts @@ -0,0 +1,53 @@ +export type JSONSchemaVisibleIfValue = + | string + | number + | boolean + | Array; + +export type JSONSchemaField = { + type?: "string" | "number" | "boolean" | "object"; + title?: string; + description?: string; + enum?: Array; + const?: string | number | boolean; + default?: string | number | boolean; + pattern?: string; + format?: string; + properties?: Record; + required?: string[]; + "x-display"?: "radio" | "select" | "textarea" | "file"; + "x-step"?: "connector" | "source"; + "x-secret"?: boolean; + "x-visible-if"?: Record; + "x-enum-labels"?: string[]; + "x-enum-descriptions"?: string[]; + "x-placeholder"?: string; + "x-hint"?: string; + "x-accept"?: string; +}; + +export type JSONSchemaCondition = { + properties?: Record; +}; + +export type JSONSchemaConstraint = { + required?: string[]; +}; + +export type JSONSchemaConditional = { + if?: JSONSchemaCondition; + then?: JSONSchemaConstraint; + else?: JSONSchemaConstraint; +}; + +export type JSONSchemaObject = { + $schema?: string; + type: "object"; + title?: string; + description?: string; + properties?: Record; + required?: string[]; + allOf?: JSONSchemaConditional[]; +}; + +export type MultiStepFormSchema = JSONSchemaObject; From 5da92f992d0e722e7ea1b36ab873f5157d01a60f Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 22:35:47 +0800 Subject: [PATCH 050/120] schema driven --- .../features/sources/modal/AddDataForm.svelte | 26 +- .../modal/JSONSchemaFormRenderer.svelte | 385 ++++++++---------- .../sources/modal/multi-step-auth-configs.ts | 8 +- .../sources/modal/schema-field-utils.ts | 111 +++++ .../src/features/templates/schemas/types.ts | 3 +- 5 files changed, 318 insertions(+), 215 deletions(-) create mode 100644 web-common/src/features/sources/modal/schema-field-utils.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 66adcf494a5..966b0de61d3 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -220,15 +220,29 @@ : null; $: if (isMultiStepConnector && activeAuthInfo) { + const authKey = activeAuthInfo.key; const options = activeAuthInfo.options ?? []; const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; + const currentValue = ($paramsForm as Record | undefined)?.[ + authKey + ] as string | undefined; const hasValidSelection = options.some( - (option) => option.value === stepState.selectedAuthMethod, + (option) => option.value === currentValue, ); - if (!hasValidSelection) { - if (fallback !== stepState.selectedAuthMethod) { - setAuthMethod(fallback ?? null); - } + const nextValue = (hasValidSelection ? currentValue : fallback) ?? null; + + if (!hasValidSelection && nextValue !== null) { + paramsForm.update( + ($form) => { + if ($form?.[authKey] === nextValue) return $form; + return { ...$form, [authKey]: nextValue }; + }, + { taint: false }, + ); + } + + if (nextValue !== stepState.selectedAuthMethod) { + setAuthMethod(nextValue); } } else if (stepState.selectedAuthMethod) { setAuthMethod(null); @@ -451,7 +465,6 @@ errors={$paramsErrors} {onStringInputChange} {handleFileUpload} - bind:authMethod={$selectedAuthMethodStore} /> {:else} {:else} ; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; - // Bubble the selected auth method to the parent so it can adjust UI. - export let authMethod: string = ""; + $: properties = schema?.properties ?? {}; - $: authInfo = schema ? getAuthOptionsFromSchema(schema) : null; - $: authMethodKey = schema ? authInfo?.key || findAuthMethodKey(schema) : null; - $: requiredByMethodConnector = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "connector" }) - : {}; - $: requiredByMethodSource = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "source" }) - : {}; - - $: if (schema && authInfo && !authMethod) { - authMethod = authInfo.defaultMethod || authInfo.options[0]?.value || ""; + // Apply defaults from the schema into the form when missing. + $: if (schema && form) { + const defaults = schema.properties ?? {}; + form.update( + ($form) => { + let mutated = false; + const next = { ...$form }; + for (const [key, prop] of Object.entries(defaults)) { + if (next[key] === undefined && prop.default !== undefined) { + next[key] = prop.default; + mutated = true; + } + } + return mutated ? next : $form; + }, + { taint: false }, + ); } - // Clear fields that are not visible for the active auth method to avoid - // sending stale values across methods. - $: if (schema && authMethod && step === "connector") { + // Clear fields that are not visible for the current step to avoid + // sending stale values for hidden inputs. + $: if (schema && form) { form.update( ($form) => { - const properties = schema.properties ?? {}; - for (const key of Object.keys(properties)) { - if (key === authMethodKey) continue; - const prop = properties[key]; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== "connector") continue; - const visible = isVisibleForValues(schema, key, { - ...$form, - [authMethodKey ?? "auth_method"]: authMethod, - }); - if (!visible && key in $form) { - $form[key] = ""; + let mutated = false; + const next = { ...$form }; + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, step)) continue; + const visible = isVisibleForValues(schema, key, next); + if (!visible && Object.prototype.hasOwnProperty.call(next, key)) { + next[key] = ""; + mutated = true; } } - return $form; + return mutated ? next : $form; }, { taint: false }, ); } - function visibleFieldsFor( - method: string | undefined, - currentStep: "connector" | "source", - ) { + $: requiredFields = schema + ? computeRequiredFields(schema, { ...$form }, step) + : new Set(); + + function isRequired(key: string) { + return requiredFields.has(key); + } + + function visibleFields(values: Record = { ...$form }) { if (!schema) return []; - const properties = schema.properties ?? {}; - const values = { ...$form, [authMethodKey ?? "auth_method"]: method }; - return Object.entries(properties).filter(([key, prop]) => { - if (authMethodKey && key === authMethodKey) return false; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== currentStep) return false; - return isVisibleForValues(schema, key, values); - }); + return visibleFieldsForValues(schema, values, step); + } + + function isRadioField(prop: JSONSchemaField) { + return Boolean(prop.enum && prop["x-display"] === "radio"); } - function isRequiredFor(method: string | undefined, key: string): boolean { - if (!schema) return false; - const requiredMap = - step === "connector" ? requiredByMethodConnector : requiredByMethodSource; - const requiredSet = requiredMap[method ?? ""] ?? []; - return requiredSet.includes(key); + function radioOptions(prop: JSONSchemaField) { + return ( + prop.enum?.map((value, idx) => ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + })) ?? [] + ); + } + + $: visibleEntries = visibleFields(); + $: radioEntries = visibleEntries.filter(([, prop]) => isRadioField(prop)); + $: radioDependentKeys = schema + ? keysDependingOn( + schema, + radioEntries.map(([key]) => key), + step, + ) + : new Set(); + $: nonRadioEntries = visibleEntries.filter( + ([key, prop]) => !isRadioField(prop) && !radioDependentKeys.has(key), + ); + + function visibleFieldsForRadioOption( + fieldKey: string, + optionValue: string | number | boolean, + ) { + if (!schema) return []; + const values = { ...$form, [fieldKey]: optionValue }; + return visibleFieldsForValues(schema, values, step).filter( + ([key, prop]) => + key !== fieldKey && + (radioDependentKeys.has(key) || isRadioField(prop)) && + matchesStep(prop, step), + ); } {#if schema} - {#if step === "connector" && authInfo} - {#if authInfo.options.length > 1} -
-
Authentication method
+ {#each radioEntries as [key, prop]} +
+
{prop.title ?? key}
+ + + {#each visibleFieldsForRadioOption(key, option.value) as [childKey, childProp]} +
+ {#if childProp["x-display"] === "file" || childProp.format === "file"} + + {:else if childProp.type === "boolean"} + + {:else if isRadioField(childProp)} + + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + + +
+ {/each} + + {#each nonRadioEntries as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if isRadioField(prop)} - - {#each visibleFieldsFor(option.value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - - -
- {:else if authInfo.options[0]} - {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - {/if} - {:else} - {#each visibleFieldsFor(authMethod, step) as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - {/if} + bind:value={$form[key]} + options={radioOptions(prop)} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} {/if} diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 88566546e22..4782b708ef8 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -70,7 +70,7 @@ export function getAuthOptionsFromSchema( export function getRequiredFieldsByAuthMethod( schema: MultiStepFormSchema, - opts?: { step?: "connector" | "source" }, + opts?: { step?: string }, ): Record { const authInfo = getAuthOptionsFromSchema(schema); if (!authInfo) return {}; @@ -120,11 +120,13 @@ export function getFieldLabel( export function isStepMatch( schema: MultiStepFormSchema, key: string, - step: "connector" | "source", + step: string, ): boolean { const prop = schema.properties?.[key]; if (!prop) return false; - return (prop["x-step"] ?? "connector") === step; + const fieldStep = prop["x-step"]; + if (!fieldStep) return true; + return fieldStep === step; } export function isVisibleForValues( diff --git a/web-common/src/features/sources/modal/schema-field-utils.ts b/web-common/src/features/sources/modal/schema-field-utils.ts new file mode 100644 index 00000000000..c335ee3f9e3 --- /dev/null +++ b/web-common/src/features/sources/modal/schema-field-utils.ts @@ -0,0 +1,111 @@ +import type { + JSONSchemaConditional, + JSONSchemaField, + MultiStepFormSchema, +} from "./types"; + +type Step = string | null | undefined; + +export function matchesStep(prop: JSONSchemaField | undefined, step: Step) { + if (!step) return true; + const fieldStep = prop?.["x-step"]; + return fieldStep ? fieldStep === step : true; +} + +export function isVisibleForValues( + schema: MultiStepFormSchema, + key: string, + values: Record, +): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; + const conditions = prop["x-visible-if"]; + if (!conditions) return true; + + return Object.entries(conditions).every(([depKey, expected]) => { + const actual = values?.[depKey]; + if (Array.isArray(expected)) { + return expected.map(String).includes(String(actual)); + } + return String(actual) === String(expected); + }); +} + +export function visibleFieldsForValues( + schema: MultiStepFormSchema, + values: Record, + step?: Step, +): Array<[string, JSONSchemaField]> { + const properties = schema.properties ?? {}; + return Object.entries(properties).filter(([key, prop]) => { + if (!matchesStep(prop, step)) return false; + return isVisibleForValues(schema, key, values); + }); +} + +export function computeRequiredFields( + schema: MultiStepFormSchema, + values: Record, + step?: Step, +): Set { + const required = new Set(); + const properties = schema.properties ?? {}; + + // Base required fields. + for (const field of schema.required ?? []) { + if (!step || matchesStep(properties[field], step)) { + required.add(field); + } + } + + // Conditional required fields driven by `allOf`. + for (const conditional of schema.allOf ?? []) { + const applies = matchesConditional(conditional, values); + const target = applies ? conditional.then : conditional.else; + for (const field of target?.required ?? []) { + if (!step || matchesStep(properties[field], step)) { + required.add(field); + } + } + } + + return required; +} + +export function dependsOnField(prop: JSONSchemaField, dependency: string) { + const conditions = prop["x-visible-if"]; + if (!conditions) return false; + return Object.prototype.hasOwnProperty.call(conditions, dependency); +} + +export function keysDependingOn( + schema: MultiStepFormSchema, + dependencies: string[], + step?: Step, +): Set { + const properties = schema.properties ?? {}; + const result = new Set(); + + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, step)) continue; + if (dependencies.some((dep) => dependsOnField(prop, dep))) { + result.add(key); + } + } + + return result; +} + +function matchesConditional( + conditional: JSONSchemaConditional, + values: Record, +) { + const conditions = conditional.if?.properties; + if (!conditions) return false; + + return Object.entries(conditions).every(([depKey, constraint]) => { + if (!("const" in constraint)) return false; + const actual = values?.[depKey]; + return String(actual) === String(constraint.const); + }); +} diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index 3c21dc4d1fa..fe3442b0adb 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -16,7 +16,8 @@ export type JSONSchemaField = { properties?: Record; required?: string[]; "x-display"?: "radio" | "select" | "textarea" | "file"; - "x-step"?: "connector" | "source"; + // Arbitrary step identifier so renderers can filter without hardcoding. + "x-step"?: string; "x-secret"?: boolean; "x-visible-if"?: Record; "x-enum-labels"?: string[]; From 2afe15f43220187668966494546a3e20f11cb6a8 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 01:18:38 +0800 Subject: [PATCH 051/120] Revert "schema driven" This reverts commit 99f076dd12eaac035473da674e29f5444c2b829c. --- .../features/sources/modal/AddDataForm.svelte | 26 +- .../modal/JSONSchemaFormRenderer.svelte | 385 ++++++++++-------- .../sources/modal/multi-step-auth-configs.ts | 8 +- .../sources/modal/schema-field-utils.ts | 111 ----- .../src/features/templates/schemas/types.ts | 3 +- 5 files changed, 215 insertions(+), 318 deletions(-) delete mode 100644 web-common/src/features/sources/modal/schema-field-utils.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 966b0de61d3..66adcf494a5 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -220,29 +220,15 @@ : null; $: if (isMultiStepConnector && activeAuthInfo) { - const authKey = activeAuthInfo.key; const options = activeAuthInfo.options ?? []; const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; - const currentValue = ($paramsForm as Record | undefined)?.[ - authKey - ] as string | undefined; const hasValidSelection = options.some( - (option) => option.value === currentValue, + (option) => option.value === stepState.selectedAuthMethod, ); - const nextValue = (hasValidSelection ? currentValue : fallback) ?? null; - - if (!hasValidSelection && nextValue !== null) { - paramsForm.update( - ($form) => { - if ($form?.[authKey] === nextValue) return $form; - return { ...$form, [authKey]: nextValue }; - }, - { taint: false }, - ); - } - - if (nextValue !== stepState.selectedAuthMethod) { - setAuthMethod(nextValue); + if (!hasValidSelection) { + if (fallback !== stepState.selectedAuthMethod) { + setAuthMethod(fallback ?? null); + } } } else if (stepState.selectedAuthMethod) { setAuthMethod(null); @@ -465,6 +451,7 @@ errors={$paramsErrors} {onStringInputChange} {handleFileUpload} + bind:authMethod={$selectedAuthMethodStore} /> {:else} {:else} ; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; - $: properties = schema?.properties ?? {}; + // Bubble the selected auth method to the parent so it can adjust UI. + export let authMethod: string = ""; - // Apply defaults from the schema into the form when missing. - $: if (schema && form) { - const defaults = schema.properties ?? {}; - form.update( - ($form) => { - let mutated = false; - const next = { ...$form }; - for (const [key, prop] of Object.entries(defaults)) { - if (next[key] === undefined && prop.default !== undefined) { - next[key] = prop.default; - mutated = true; - } - } - return mutated ? next : $form; - }, - { taint: false }, - ); + $: authInfo = schema ? getAuthOptionsFromSchema(schema) : null; + $: authMethodKey = schema ? authInfo?.key || findAuthMethodKey(schema) : null; + $: requiredByMethodConnector = schema + ? getRequiredFieldsByAuthMethod(schema, { step: "connector" }) + : {}; + $: requiredByMethodSource = schema + ? getRequiredFieldsByAuthMethod(schema, { step: "source" }) + : {}; + + $: if (schema && authInfo && !authMethod) { + authMethod = authInfo.defaultMethod || authInfo.options[0]?.value || ""; } - // Clear fields that are not visible for the current step to avoid - // sending stale values for hidden inputs. - $: if (schema && form) { + // Clear fields that are not visible for the active auth method to avoid + // sending stale values across methods. + $: if (schema && authMethod && step === "connector") { form.update( ($form) => { - let mutated = false; - const next = { ...$form }; - for (const [key, prop] of Object.entries(properties)) { - if (!matchesStep(prop, step)) continue; - const visible = isVisibleForValues(schema, key, next); - if (!visible && Object.prototype.hasOwnProperty.call(next, key)) { - next[key] = ""; - mutated = true; + const properties = schema.properties ?? {}; + for (const key of Object.keys(properties)) { + if (key === authMethodKey) continue; + const prop = properties[key]; + const stepForField = prop["x-step"] ?? "connector"; + if (stepForField !== "connector") continue; + const visible = isVisibleForValues(schema, key, { + ...$form, + [authMethodKey ?? "auth_method"]: authMethod, + }); + if (!visible && key in $form) { + $form[key] = ""; } } - return mutated ? next : $form; + return $form; }, { taint: false }, ); } - $: requiredFields = schema - ? computeRequiredFields(schema, { ...$form }, step) - : new Set(); - - function isRequired(key: string) { - return requiredFields.has(key); - } - - function visibleFields(values: Record = { ...$form }) { + function visibleFieldsFor( + method: string | undefined, + currentStep: "connector" | "source", + ) { if (!schema) return []; - return visibleFieldsForValues(schema, values, step); - } - - function isRadioField(prop: JSONSchemaField) { - return Boolean(prop.enum && prop["x-display"] === "radio"); + const properties = schema.properties ?? {}; + const values = { ...$form, [authMethodKey ?? "auth_method"]: method }; + return Object.entries(properties).filter(([key, prop]) => { + if (authMethodKey && key === authMethodKey) return false; + const stepForField = prop["x-step"] ?? "connector"; + if (stepForField !== currentStep) return false; + return isVisibleForValues(schema, key, values); + }); } - function radioOptions(prop: JSONSchemaField) { - return ( - prop.enum?.map((value, idx) => ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - })) ?? [] - ); - } - - $: visibleEntries = visibleFields(); - $: radioEntries = visibleEntries.filter(([, prop]) => isRadioField(prop)); - $: radioDependentKeys = schema - ? keysDependingOn( - schema, - radioEntries.map(([key]) => key), - step, - ) - : new Set(); - $: nonRadioEntries = visibleEntries.filter( - ([key, prop]) => !isRadioField(prop) && !radioDependentKeys.has(key), - ); - - function visibleFieldsForRadioOption( - fieldKey: string, - optionValue: string | number | boolean, - ) { - if (!schema) return []; - const values = { ...$form, [fieldKey]: optionValue }; - return visibleFieldsForValues(schema, values, step).filter( - ([key, prop]) => - key !== fieldKey && - (radioDependentKeys.has(key) || isRadioField(prop)) && - matchesStep(prop, step), - ); + function isRequiredFor(method: string | undefined, key: string): boolean { + if (!schema) return false; + const requiredMap = + step === "connector" ? requiredByMethodConnector : requiredByMethodSource; + const requiredSet = requiredMap[method ?? ""] ?? []; + return requiredSet.includes(key); } {#if schema} - {#each radioEntries as [key, prop]} -
-
{prop.title ?? key}
- - - {#each visibleFieldsForRadioOption(key, option.value) as [childKey, childProp]} -
- {#if childProp["x-display"] === "file" || childProp.format === "file"} - - {:else if childProp.type === "boolean"} - - {:else if isRadioField(childProp)} - - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - - -
- {/each} - - {#each nonRadioEntries as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if isRadioField(prop)} + {#if step === "connector" && authInfo} + {#if authInfo.options.length > 1} +
+
Authentication method
- {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} + bind:value={authMethod} + options={authInfo.options} + name="multi-auth-method" + > + + {#each visibleFieldsFor(option.value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + + +
+ {:else if authInfo.options[0]} + {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + {/if} + {:else} + {#each visibleFieldsFor(authMethod, step) as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + {/if} {/if} diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 4782b708ef8..88566546e22 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -70,7 +70,7 @@ export function getAuthOptionsFromSchema( export function getRequiredFieldsByAuthMethod( schema: MultiStepFormSchema, - opts?: { step?: string }, + opts?: { step?: "connector" | "source" }, ): Record { const authInfo = getAuthOptionsFromSchema(schema); if (!authInfo) return {}; @@ -120,13 +120,11 @@ export function getFieldLabel( export function isStepMatch( schema: MultiStepFormSchema, key: string, - step: string, + step: "connector" | "source", ): boolean { const prop = schema.properties?.[key]; if (!prop) return false; - const fieldStep = prop["x-step"]; - if (!fieldStep) return true; - return fieldStep === step; + return (prop["x-step"] ?? "connector") === step; } export function isVisibleForValues( diff --git a/web-common/src/features/sources/modal/schema-field-utils.ts b/web-common/src/features/sources/modal/schema-field-utils.ts deleted file mode 100644 index c335ee3f9e3..00000000000 --- a/web-common/src/features/sources/modal/schema-field-utils.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { - JSONSchemaConditional, - JSONSchemaField, - MultiStepFormSchema, -} from "./types"; - -type Step = string | null | undefined; - -export function matchesStep(prop: JSONSchemaField | undefined, step: Step) { - if (!step) return true; - const fieldStep = prop?.["x-step"]; - return fieldStep ? fieldStep === step : true; -} - -export function isVisibleForValues( - schema: MultiStepFormSchema, - key: string, - values: Record, -): boolean { - const prop = schema.properties?.[key]; - if (!prop) return false; - const conditions = prop["x-visible-if"]; - if (!conditions) return true; - - return Object.entries(conditions).every(([depKey, expected]) => { - const actual = values?.[depKey]; - if (Array.isArray(expected)) { - return expected.map(String).includes(String(actual)); - } - return String(actual) === String(expected); - }); -} - -export function visibleFieldsForValues( - schema: MultiStepFormSchema, - values: Record, - step?: Step, -): Array<[string, JSONSchemaField]> { - const properties = schema.properties ?? {}; - return Object.entries(properties).filter(([key, prop]) => { - if (!matchesStep(prop, step)) return false; - return isVisibleForValues(schema, key, values); - }); -} - -export function computeRequiredFields( - schema: MultiStepFormSchema, - values: Record, - step?: Step, -): Set { - const required = new Set(); - const properties = schema.properties ?? {}; - - // Base required fields. - for (const field of schema.required ?? []) { - if (!step || matchesStep(properties[field], step)) { - required.add(field); - } - } - - // Conditional required fields driven by `allOf`. - for (const conditional of schema.allOf ?? []) { - const applies = matchesConditional(conditional, values); - const target = applies ? conditional.then : conditional.else; - for (const field of target?.required ?? []) { - if (!step || matchesStep(properties[field], step)) { - required.add(field); - } - } - } - - return required; -} - -export function dependsOnField(prop: JSONSchemaField, dependency: string) { - const conditions = prop["x-visible-if"]; - if (!conditions) return false; - return Object.prototype.hasOwnProperty.call(conditions, dependency); -} - -export function keysDependingOn( - schema: MultiStepFormSchema, - dependencies: string[], - step?: Step, -): Set { - const properties = schema.properties ?? {}; - const result = new Set(); - - for (const [key, prop] of Object.entries(properties)) { - if (!matchesStep(prop, step)) continue; - if (dependencies.some((dep) => dependsOnField(prop, dep))) { - result.add(key); - } - } - - return result; -} - -function matchesConditional( - conditional: JSONSchemaConditional, - values: Record, -) { - const conditions = conditional.if?.properties; - if (!conditions) return false; - - return Object.entries(conditions).every(([depKey, constraint]) => { - if (!("const" in constraint)) return false; - const actual = values?.[depKey]; - return String(actual) === String(constraint.const); - }); -} diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index fe3442b0adb..3c21dc4d1fa 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -16,8 +16,7 @@ export type JSONSchemaField = { properties?: Record; required?: string[]; "x-display"?: "radio" | "select" | "textarea" | "file"; - // Arbitrary step identifier so renderers can filter without hardcoding. - "x-step"?: string; + "x-step"?: "connector" | "source"; "x-secret"?: boolean; "x-visible-if"?: Record; "x-enum-labels"?: string[]; From f2a903d877526e8971f2452e792885370d7b9796 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 12:21:26 +0800 Subject: [PATCH 052/120] re-add aws_role_arn --- runtime/drivers/s3/s3.go | 18 ++++++++++++++++++ .../src/features/templates/schemas/s3.ts | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 81bd18aaf56..005e7ab4868 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -61,6 +61,24 @@ var spec = drivers.Spec{ Required: false, Hint: "Overrides the S3 endpoint to connect to. This should only be used to connect to S3 compatible services, such as Cloudflare R2 or MinIO.", }, + { + Key: "aws_role_arn", + Type: drivers.StringPropertyType, + Secret: true, + Description: "AWS Role ARN to assume", + }, + { + Key: "aws_role_session_name", + Type: drivers.StringPropertyType, + Secret: true, + Description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + }, + { + Key: "aws_external_id", + Type: drivers.StringPropertyType, + Secret: true, + Description: "Optional external ID to use when assuming an AWS role for cross-account access.", + }, }, SourceProperties: []*drivers.PropertySpec{ { diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index 70dd72c0c61..f5653bf561b 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -54,6 +54,15 @@ export const s3Schema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "access_keys" }, }, + aws_role_arn: { + type: "string", + title: "AWS Role ARN", + description: "AWS Role ARN to assume", + "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, path: { type: "string", title: "S3 URI", From 7a7d73eef728d2fd441f704911ca6afbed204543 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 13:11:26 +0800 Subject: [PATCH 053/120] fix auth method revalidation --- .../features/sources/modal/AddDataForm.svelte | 15 +- .../modal/JSONSchemaFormRenderer.svelte | 396 +++++++++++------- .../src/features/sources/modal/utils.ts | 10 + 3 files changed, 259 insertions(+), 162 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 66adcf494a5..1efeafe261a 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -28,6 +28,7 @@ import AddDataFormSection from "./AddDataFormSection.svelte"; import { getAuthOptionsFromSchema, + findAuthMethodKey, getConnectorSchema, } from "./multi-step-auth-configs"; import { get } from "svelte/store"; @@ -234,6 +235,18 @@ setAuthMethod(null); } + // Keep auth method store aligned with the form's enum value from the schema. + $: if (isMultiStepConnector && activeSchema) { + const authKey = findAuthMethodKey(activeSchema); + if (authKey) { + const currentValue = $paramsForm?.[authKey] as string | undefined; + const normalized = currentValue ? String(currentValue) : null; + if (normalized !== (stepState.selectedAuthMethod ?? null)) { + setAuthMethod(normalized); + } + } + } + $: isSubmitting = submitting; // Reset errors when form is modified @@ -451,7 +464,6 @@ errors={$paramsErrors} {onStringInputChange} {handleFileUpload} - bind:authMethod={$selectedAuthMethodStore} /> {:else} {:else} ; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; - // Bubble the selected auth method to the parent so it can adjust UI. - export let authMethod: string = ""; + const radioDisplay = "radio"; - $: authInfo = schema ? getAuthOptionsFromSchema(schema) : null; - $: authMethodKey = schema ? authInfo?.key || findAuthMethodKey(schema) : null; - $: requiredByMethodConnector = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "connector" }) - : {}; - $: requiredByMethodSource = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "source" }) - : {}; + $: stepFilter = step; + $: dependentMap = schema + ? buildDependentMap(schema, stepFilter) + : new Map>(); + $: dependentKeys = new Set( + Array.from(dependentMap.values()).flatMap((entries) => + entries.map(([key]) => key), + ), + ); + $: visibleEntries = schema + ? computeVisibleEntries(schema, stepFilter, $form) + : []; + $: requiredFields = schema + ? computeRequiredFields(schema, $form, stepFilter) + : new Set(); + $: renderOrder = schema + ? computeRenderOrder(visibleEntries, dependentMap, dependentKeys) + : []; - $: if (schema && authInfo && !authMethod) { - authMethod = authInfo.defaultMethod || authInfo.options[0]?.value || ""; + // Seed defaults once when schema-provided defaults exist. + $: if (schema) { + form.update( + ($form) => { + const properties = schema.properties ?? {}; + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, stepFilter)) continue; + const current = $form[key]; + if ( + (current === undefined || current === null) && + prop.default !== undefined + ) { + $form[key] = prop.default; + } + } + return $form; + }, + { taint: false }, + ); } - // Clear fields that are not visible for the active auth method to avoid - // sending stale values across methods. - $: if (schema && authMethod && step === "connector") { + // Clear hidden fields for the active step to avoid stale submissions. + $: if (schema) { form.update( ($form) => { const properties = schema.properties ?? {}; - for (const key of Object.keys(properties)) { - if (key === authMethodKey) continue; - const prop = properties[key]; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== "connector") continue; - const visible = isVisibleForValues(schema, key, { - ...$form, - [authMethodKey ?? "auth_method"]: authMethod, - }); - if (!visible && key in $form) { + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, stepFilter)) continue; + const visible = isVisibleForValues(schema, key, $form); + if (!visible && key in $form && $form[key] !== "") { $form[key] = ""; } } @@ -60,144 +74,210 @@ ); } - function visibleFieldsFor( - method: string | undefined, - currentStep: "connector" | "source", + function matchesStep(prop: JSONSchemaField | undefined, stepValue?: string) { + if (!stepValue) return true; + const propStep = prop?.["x-step"]; + if (!propStep) return true; + return propStep === stepValue; + } + + function isRadioEnum(prop: JSONSchemaField) { + return Boolean(prop.enum && prop["x-display"] === radioDisplay); + } + + function computeVisibleEntries( + currentSchema: MultiStepFormSchema, + currentStep: string | undefined, + values: Record, ) { - if (!schema) return []; - const properties = schema.properties ?? {}; - const values = { ...$form, [authMethodKey ?? "auth_method"]: method }; + const properties = currentSchema.properties ?? {}; return Object.entries(properties).filter(([key, prop]) => { - if (authMethodKey && key === authMethodKey) return false; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== currentStep) return false; - return isVisibleForValues(schema, key, values); + if (!matchesStep(prop, currentStep)) return false; + return isVisibleForValues(currentSchema, key, values); + }); + } + + function matchesCondition( + condition: + | Record + | undefined, + values: Record, + ) { + if (!condition || !Object.keys(condition).length) return false; + return Object.entries(condition).every(([depKey, def]) => { + if (def.const === undefined || def.const === null) return false; + return String(values?.[depKey]) === String(def.const); }); } - function isRequiredFor(method: string | undefined, key: string): boolean { - if (!schema) return false; - const requiredMap = - step === "connector" ? requiredByMethodConnector : requiredByMethodSource; - const requiredSet = requiredMap[method ?? ""] ?? []; - return requiredSet.includes(key); + function computeRequiredFields( + currentSchema: MultiStepFormSchema, + values: Record, + currentStep: string | undefined, + ) { + const properties = currentSchema.properties ?? {}; + const required = new Set(); + (currentSchema.required ?? []).forEach((key) => { + if (matchesStep(properties[key], currentStep)) required.add(key); + }); + + for (const conditional of currentSchema.allOf ?? []) { + const condition = conditional.if?.properties; + const matches = matchesCondition(condition, values); + const branch = matches ? conditional.then : conditional.else; + branch?.required?.forEach((key) => { + if (matchesStep(properties[key], currentStep)) required.add(key); + }); + } + return required; + } + + function computeRenderOrder( + entries: Array<[string, JSONSchemaField]>, + dependents: Map>, + dependentKeySet: Set, + ) { + const result: Array<[string, JSONSchemaField]> = []; + const rendered = new Set(); + + for (const [key, prop] of entries) { + if (rendered.has(key)) continue; + + if (isRadioEnum(prop)) { + rendered.add(key); + dependents.get(key)?.forEach(([childKey]) => rendered.add(childKey)); + result.push([key, prop]); + continue; + } + + if (dependentKeySet.has(key)) continue; + + rendered.add(key); + result.push([key, prop]); + } + + return result; + } + + function buildDependentMap( + currentSchema: MultiStepFormSchema, + currentStep: string | undefined, + ) { + const properties = currentSchema.properties ?? {}; + const map = new Map>(); + + for (const [key, prop] of Object.entries(properties)) { + const visibleIf = prop["x-visible-if"]; + if (!visibleIf) continue; + + for (const controllerKey of Object.keys(visibleIf)) { + const controller = properties[controllerKey]; + if (!controller) continue; + if (!matchesStep(controller, currentStep)) continue; + if (!matchesStep(prop, currentStep)) continue; + + const entries = map.get(controllerKey) ?? []; + entries.push([key, prop]); + map.set(controllerKey, entries); + } + } + + return map; + } + + function getDependentFieldsForOption( + controllerKey: string, + optionValue: string | number | boolean, + ) { + if (!schema) return []; + const dependents = dependentMap.get(controllerKey) ?? []; + const values = { ...$form, [controllerKey]: optionValue }; + return dependents.filter(([key]) => + isVisibleForValues(schema, key, values), + ); + } + + function radioOptions(prop: JSONSchemaField) { + return ( + prop.enum?.map((value, idx) => ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + })) ?? [] + ); + } + + function isRequired(key: string) { + return requiredFields.has(key); } {#if schema} - {#if step === "connector" && authInfo} - {#if authInfo.options.length > 1} + {#each renderOrder as [key, prop]} + {#if isRadioEnum(prop)}
-
Authentication method
+ {#if prop.title} +
{prop.title}
+ {/if} - {#each visibleFieldsFor(option.value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} + {#if dependentMap.get(key)?.length} + {#each getDependentFieldsForOption(key, option.value) as [childKey, childProp]} +
+ {#if childProp["x-display"] === "file" || childProp.format === "file"} + + {:else if childProp.type === "boolean"} + + {:else if isRadioEnum(childProp)} + + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + {/if}
- {:else if authInfo.options[0]} - {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - {/if} - {:else} - {#each visibleFieldsFor(authMethod, step) as [key, prop]} + {:else}
{#if prop["x-display"] === "file" || prop.format === "file"} - {:else if prop.enum && prop["x-display"] === "radio"} + {:else if isRadioEnum(prop)} ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} + options={radioOptions(prop)} name={`${key}-radio`} /> {:else} @@ -225,7 +301,7 @@ id={key} label={prop.title ?? key} placeholder={prop["x-placeholder"]} - optional={!isRequiredFor(authMethod, key)} + optional={!isRequired(key)} secret={prop["x-secret"]} hint={prop.description ?? prop["x-hint"]} errors={normalizeErrors(errors?.[key])} @@ -235,6 +311,6 @@ /> {/if}
- {/each} - {/if} + {/if} + {/each} {/if} diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 9f69917ebc9..9fc7caa8728 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -3,6 +3,7 @@ import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; import type { MultiStepFormSchema } from "./types"; import { + findAuthMethodKey, getAuthOptionsFromSchema, getRequiredFieldsByAuthMethod, isStepMatch, @@ -103,9 +104,18 @@ export function isMultiStepConnectorDisabled( const authInfo = getAuthOptionsFromSchema(schema); const options = authInfo?.options ?? []; + const authKey = authInfo?.key || findAuthMethodKey(schema); + const methodFromForm = + authKey && paramsFormValue?.[authKey] != null + ? String(paramsFormValue[authKey]) + : undefined; const hasValidSelection = options.some((opt) => opt.value === selectedMethod); + const hasValidFormSelection = options.some( + (opt) => opt.value === methodFromForm, + ); const method = (hasValidSelection && selectedMethod) || + (hasValidFormSelection && methodFromForm) || authInfo?.defaultMethod || options[0]?.value; From 7337a376a013d410b6fadf122af7718a755e206d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 16:47:29 +0800 Subject: [PATCH 054/120] fix e2e --- web-common/src/features/sources/modal/utils.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 9fc7caa8728..e8b045407ba 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -96,7 +96,7 @@ export function hasOnlyDsn( */ export function isMultiStepConnectorDisabled( schema: MultiStepFormSchema | null, - selectedMethod: string, + _selectedMethod: string, paramsFormValue: Record, paramsFormErrors: Record, ) { @@ -109,12 +109,10 @@ export function isMultiStepConnectorDisabled( authKey && paramsFormValue?.[authKey] != null ? String(paramsFormValue[authKey]) : undefined; - const hasValidSelection = options.some((opt) => opt.value === selectedMethod); const hasValidFormSelection = options.some( (opt) => opt.value === methodFromForm, ); const method = - (hasValidSelection && selectedMethod) || (hasValidFormSelection && methodFromForm) || authInfo?.defaultMethod || options[0]?.value; From 8c65e196d7255c35e635c5d76d2cc348b0188787 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 17:06:42 +0800 Subject: [PATCH 055/120] fix selected auth method remnant --- .../features/sources/modal/AddDataForm.svelte | 18 ++++++++++++++---- web-common/src/features/sources/modal/utils.ts | 1 - 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 1efeafe261a..cf15a5fa9e8 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -77,6 +77,7 @@ }; let selectedAuthMethod: string = ""; + let activeAuthMethod: string | null = null; $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; $: stepProperties = @@ -159,7 +160,6 @@ if (isMultiStepConnector && stepState.step === "connector") { return isMultiStepConnectorDisabled( activeSchema, - selectedAuthMethod, $paramsForm, $paramsErrors, ); @@ -247,6 +247,16 @@ } } + // Auth method to use for UI (labels, CTA), derived from form first. + $: activeAuthMethod = (() => { + if (!(isMultiStepConnector && activeSchema)) return selectedAuthMethod; + const authKey = findAuthMethodKey(activeSchema); + if (authKey && $paramsForm?.[authKey] != null) { + return String($paramsForm[authKey]); + } + return selectedAuthMethod; + })(); + $: isSubmitting = submitting; // Reset errors when form is modified @@ -345,7 +355,7 @@ onClose, queryClient, getConnectionTab: () => connectionTab, - getSelectedAuthMethod: () => selectedAuthMethod, + getSelectedAuthMethod: () => activeAuthMethod || undefined, setParamsError: (message: string | null, details?: string) => { paramsError = message; paramsErrorDetails = details; @@ -551,7 +561,7 @@ ? "Connecting..." : isMultiStepConnector && stepState.step === "source" ? "Importing data..." - : selectedAuthMethod === "public" + : activeAuthMethod === "public" ? "Continuing..." : "Testing connection..."} form={connector.name === "clickhouse" ? clickhouseFormId : formId} @@ -564,7 +574,7 @@ submitting, clickhouseConnectorType, clickhouseSubmitting, - selectedAuthMethod, + selectedAuthMethod: activeAuthMethod ?? selectedAuthMethod, })}
diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index e8b045407ba..041d5ab3b58 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -96,7 +96,6 @@ export function hasOnlyDsn( */ export function isMultiStepConnectorDisabled( schema: MultiStepFormSchema | null, - _selectedMethod: string, paramsFormValue: Record, paramsFormErrors: Record, ) { From 9aa1436e2f7e5760e05dd268cd48c75bf0419385 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 18:13:29 +0800 Subject: [PATCH 056/120] fix continue submission --- .../features/sources/modal/AddDataForm.svelte | 16 +++++++++++-- .../sources/modal/AddDataFormManager.ts | 24 ++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index cf15a5fa9e8..4371bf76890 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -102,8 +102,20 @@ // Update form when (re)entering step 1: restore defaults for connector properties $: if (isMultiStepConnector && stepState.step === "connector") { paramsForm.update( - () => - getInitialFormValuesFromProperties(connector.configProperties ?? []), + ($current) => { + const base = getInitialFormValuesFromProperties( + connector.configProperties ?? [], + ); + // Preserve previously selected auth method when returning to connector step. + if (activeSchema) { + const authKey = findAuthMethodKey(activeSchema); + const persisted = stepState.selectedAuthMethod; + if (authKey && persisted) { + base[authKey] = persisted; + } + } + return { ...base, ...$current }; + }, { taint: false }, ); } diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 3a12ac8b682..f72c1668e19 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -36,6 +36,10 @@ import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-clien import type { ClickHouseConnectorType } from "./constants"; import { applyClickHouseCloudRequirements } from "./utils"; import type { ActionResult } from "@sveltejs/kit"; +import { + findAuthMethodKey, + getConnectorSchema, +} from "./multi-step-auth-configs"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { @@ -331,9 +335,27 @@ export class AddDataFormManager { result?: Extract; }) => { const values = event.form.data; - const selectedAuthMethod = getSelectedAuthMethod?.(); + const schema = getConnectorSchema(this.connector.name ?? ""); + const authKey = schema ? findAuthMethodKey(schema) : null; + const selectedAuthMethod = + (authKey && values && values[authKey] != null + ? String(values[authKey]) + : undefined) || + getSelectedAuthMethod?.() || + ""; const stepState = get(connectorStepStore) as ConnectorStepState; + // Fast-path: public auth skips validation/test and goes straight to source step. + if ( + isMultiStepConnector && + stepState.step === "connector" && + selectedAuthMethod === "public" + ) { + setConnectorConfig(values); + setStep("source"); + return; + } + // When in the source step of a multi-step flow, the superform still uses // the connector schema, so it can appear invalid because connector fields // were intentionally skipped. Allow submission in that case and rely on From 43cb449cd71ef02c7bf6e049d4a048cdda18899a Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 18:25:02 +0800 Subject: [PATCH 057/120] fix lingering save anyway after submission for public option --- web-common/src/features/sources/modal/AddDataForm.svelte | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 4371bf76890..e2b0920393b 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -78,6 +78,7 @@ let selectedAuthMethod: string = ""; let activeAuthMethod: string | null = null; + let prevAuthMethod: string | null = null; $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; $: stepProperties = @@ -269,6 +270,13 @@ return selectedAuthMethod; })(); + // Clear Save Anyway state whenever auth method changes (any direction). + $: if (activeAuthMethod !== prevAuthMethod) { + prevAuthMethod = activeAuthMethod; + showSaveAnyway = false; + saveAnyway = false; + } + $: isSubmitting = submitting; // Reset errors when form is modified From bf489bb4de8d74cfd941dc5f738fe6fd1d8ffd8a Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 18:51:30 +0800 Subject: [PATCH 058/120] clear input fields on auth method change --- .../modal/JSONSchemaFormRenderer.svelte | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 6683905fa6e..91a20f76641 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -57,21 +57,32 @@ } // Clear hidden fields for the active step to avoid stale submissions. + // Depend on `$form` so this runs when the auth method (or other values) change. $: if (schema) { - form.update( - ($form) => { - const properties = schema.properties ?? {}; - for (const [key, prop] of Object.entries(properties)) { - if (!matchesStep(prop, stepFilter)) continue; - const visible = isVisibleForValues(schema, key, $form); - if (!visible && key in $form && $form[key] !== "") { - $form[key] = ""; + const currentValues = $form; + const properties = schema.properties ?? {}; + + const shouldClear = Object.entries(properties).some(([key, prop]) => { + if (!matchesStep(prop, stepFilter)) return false; + const visible = isVisibleForValues(schema, key, currentValues); + return !visible && key in currentValues && currentValues[key] !== ""; + }); + + if (shouldClear) { + form.update( + ($form) => { + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, stepFilter)) continue; + const visible = isVisibleForValues(schema, key, $form); + if (!visible && key in $form && $form[key] !== "") { + $form[key] = ""; + } } - } - return $form; - }, - { taint: false }, - ); + return $form; + }, + { taint: false }, + ); + } } function matchesStep(prop: JSONSchemaField | undefined, stepValue?: string) { From 9517d70ba42464aa4f383bed1235cfe0e9baf3f9 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 19 Dec 2025 15:29:05 +0800 Subject: [PATCH 059/120] explicit group fields --- .../features/sources/modal/AddDataForm.svelte | 8 ++ .../modal/JSONSchemaFormRenderer.svelte | 98 +++++++++---------- .../src/features/templates/schemas/azure.ts | 6 ++ .../src/features/templates/schemas/gcs.ts | 5 + .../src/features/templates/schemas/s3.ts | 10 ++ .../src/features/templates/schemas/types.ts | 5 + 6 files changed, 83 insertions(+), 49 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e2b0920393b..f21e3c54f95 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -236,12 +236,20 @@ $: if (isMultiStepConnector && activeAuthInfo) { const options = activeAuthInfo.options ?? []; const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; + const authKey = + activeAuthInfo.key || (activeSchema && findAuthMethodKey(activeSchema)); const hasValidSelection = options.some( (option) => option.value === stepState.selectedAuthMethod, ); if (!hasValidSelection) { if (fallback !== stepState.selectedAuthMethod) { setAuthMethod(fallback ?? null); + if (fallback && authKey) { + paramsForm.update(($form) => { + if ($form[authKey] !== fallback) $form[authKey] = fallback; + return $form; + }); + } } } } else if (stepState.selectedAuthMethod) { diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 91a20f76641..c3ab54c1f9b 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -17,12 +17,12 @@ const radioDisplay = "radio"; $: stepFilter = step; - $: dependentMap = schema - ? buildDependentMap(schema, stepFilter) - : new Map>(); - $: dependentKeys = new Set( - Array.from(dependentMap.values()).flatMap((entries) => - entries.map(([key]) => key), + $: groupedFields = schema + ? buildGroupedFields(schema, stepFilter) + : new Map>(); + $: groupedChildKeys = new Set( + Array.from(groupedFields.values()).flatMap((group) => + Object.values(group).flat(), ), ); $: visibleEntries = schema @@ -32,10 +32,11 @@ ? computeRequiredFields(schema, $form, stepFilter) : new Set(); $: renderOrder = schema - ? computeRenderOrder(visibleEntries, dependentMap, dependentKeys) + ? computeRenderOrder(visibleEntries, groupedChildKeys) : []; - // Seed defaults once when schema-provided defaults exist. + // Seed defaults for initial render: use explicit defaults, and for radio enums + // fall back to first option when no value is set. $: if (schema) { form.update( ($form) => { @@ -43,11 +44,13 @@ for (const [key, prop] of Object.entries(properties)) { if (!matchesStep(prop, stepFilter)) continue; const current = $form[key]; - if ( - (current === undefined || current === null) && - prop.default !== undefined - ) { + const isUnset = + current === undefined || current === null || current === ""; + + if (isUnset && prop.default !== undefined) { $form[key] = prop.default; + } else if (isUnset && isRadioEnum(prop) && prop.enum?.length) { + $form[key] = String(prop.enum[0]); } } return $form; @@ -145,67 +148,64 @@ function computeRenderOrder( entries: Array<[string, JSONSchemaField]>, - dependents: Map>, - dependentKeySet: Set, + groupedChildKeySet: Set, ) { const result: Array<[string, JSONSchemaField]> = []; - const rendered = new Set(); - for (const [key, prop] of entries) { - if (rendered.has(key)) continue; - - if (isRadioEnum(prop)) { - rendered.add(key); - dependents.get(key)?.forEach(([childKey]) => rendered.add(childKey)); - result.push([key, prop]); - continue; - } - - if (dependentKeySet.has(key)) continue; - - rendered.add(key); + if (groupedChildKeySet.has(key)) continue; result.push([key, prop]); } return result; } - function buildDependentMap( + function buildGroupedFields( currentSchema: MultiStepFormSchema, currentStep: string | undefined, - ) { + ): Map> { const properties = currentSchema.properties ?? {}; - const map = new Map>(); + const map = new Map>(); for (const [key, prop] of Object.entries(properties)) { - const visibleIf = prop["x-visible-if"]; - if (!visibleIf) continue; - - for (const controllerKey of Object.keys(visibleIf)) { - const controller = properties[controllerKey]; - if (!controller) continue; - if (!matchesStep(controller, currentStep)) continue; - if (!matchesStep(prop, currentStep)) continue; + const grouped = prop["x-grouped-fields"]; + if (!grouped) continue; + if (!matchesStep(prop, currentStep)) continue; - const entries = map.get(controllerKey) ?? []; - entries.push([key, prop]); - map.set(controllerKey, entries); + const filteredOptions: Record = {}; + const groupedEntries = Object.entries(grouped) as Array< + [string, string[]] + >; + for (const [optionValue, childKeys] of groupedEntries) { + filteredOptions[optionValue] = childKeys.filter((childKey) => { + const childProp = properties[childKey]; + if (!childProp) return false; + return matchesStep(childProp, currentStep); + }); } + map.set(key, filteredOptions); } return map; } - function getDependentFieldsForOption( + function getGroupedFieldsForOption( controllerKey: string, optionValue: string | number | boolean, ) { if (!schema) return []; - const dependents = dependentMap.get(controllerKey) ?? []; + const properties = schema.properties ?? {}; + const childKeys = + groupedFields.get(controllerKey)?.[String(optionValue)] ?? []; const values = { ...$form, [controllerKey]: optionValue }; - return dependents.filter(([key]) => - isVisibleForValues(schema, key, values), - ); + + return childKeys + .map< + [string, JSONSchemaField | undefined] + >((childKey) => [childKey, properties[childKey]]) + .filter( + (entry): entry is [string, JSONSchemaField] => + Boolean(entry[1]) && isVisibleForValues(schema, entry[0], values), + ); } function radioOptions(prop: JSONSchemaField) { @@ -236,8 +236,8 @@ name={`${key}-radio`} > - {#if dependentMap.get(key)?.length} - {#each getDependentFieldsForOption(key, option.value) as [childKey, childProp]} + {#if groupedFields.get(key)} + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]}
{#if childProp["x-display"] === "file" || childProp.format === "file"} ; }; export type JSONSchemaCondition = { From 3250f2a5b4071df9bb090e1dd0ef8b515dc8e363 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 19 Dec 2025 16:44:17 +0800 Subject: [PATCH 060/120] reduce markup dup in the renderer --- .../modal/JSONSchemaFieldControl.svelte | 55 +++++++++ .../modal/JSONSchemaFormRenderer.svelte | 105 +++++------------- 2 files changed, 82 insertions(+), 78 deletions(-) create mode 100644 web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte diff --git a/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte new file mode 100644 index 00000000000..a7c47942c04 --- /dev/null +++ b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte @@ -0,0 +1,55 @@ + + +{#if prop["x-display"] === "file" || prop.format === "file"} + +{:else if prop.type === "boolean"} + +{:else if options?.length} + +{:else} + onStringInputChange(e)} + alwaysShowError + /> +{/if} diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index c3ab54c1f9b..6cc9b8c8652 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -1,9 +1,6 @@ + +{#if stepState.step === "connector"} + + {#if activeSchema} + + {:else} + + {/if} + +{:else} + + {#if activeSchema} + + {:else} + + {/if} + +{/if} From c81a8295f9ff5c6af5fc98f321f6ad4c3ab0969c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 22 Dec 2025 17:26:45 +0800 Subject: [PATCH 065/120] initial multi step connector --- .../connectors/multi-step-connector.spec.ts | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 web-local/tests/connectors/multi-step-connector.spec.ts diff --git a/web-local/tests/connectors/multi-step-connector.spec.ts b/web-local/tests/connectors/multi-step-connector.spec.ts new file mode 100644 index 00000000000..64ea1fcb756 --- /dev/null +++ b/web-local/tests/connectors/multi-step-connector.spec.ts @@ -0,0 +1,42 @@ +import { expect } from "@playwright/test"; +import { test } from "../setup/base"; + +test.describe("Multi-step connector wrapper", () => { + test.use({ project: "Blank" }); + + test("GCS connector - renders connector step schema via wrapper", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + // Choose a multi-step connector (GCS). + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Connector step should show connector preview and connector CTA. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect( + page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }), + ).toBeVisible(); + + // Auth method controls from the connector schema should render. + const hmacRadio = page.getByRole("radio", { name: "HMAC keys" }); + await expect(hmacRadio).toBeVisible(); + await expect(page.getByRole("radio", { name: "Public" })).toBeVisible(); + + // Select HMAC so its fields are rendered. + await hmacRadio.click(); + + // Connector step fields should be present, while source step fields should not yet render. + await expect( + page.getByRole("textbox", { name: "Access Key ID" }), + ).toBeVisible(); + await expect( + page.getByRole("textbox", { name: "Secret Access Key" }), + ).toBeVisible(); + await expect(page.getByRole("textbox", { name: "GS URI" })).toHaveCount(0); + }); +}); From 99ef2aa8cab10e2335eb929b5d641efe6a5c6783 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 22 Dec 2025 17:56:26 +0800 Subject: [PATCH 066/120] gcs e2e render source step --- .../connectors/multi-step-connector.spec.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/web-local/tests/connectors/multi-step-connector.spec.ts b/web-local/tests/connectors/multi-step-connector.spec.ts index 64ea1fcb756..4462004e9f4 100644 --- a/web-local/tests/connectors/multi-step-connector.spec.ts +++ b/web-local/tests/connectors/multi-step-connector.spec.ts @@ -39,4 +39,45 @@ test.describe("Multi-step connector wrapper", () => { ).toBeVisible(); await expect(page.getByRole("textbox", { name: "GS URI" })).toHaveCount(0); }); + + test("GCS connector - renders source step schema via wrapper", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + // Choose a multi-step connector (GCS). + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Connector step visible with CTA. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect( + page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }), + ).toBeVisible(); + + // Switch to Public auth (no required fields) and continue via CTA. + await page.getByRole("radio", { name: "Public" }).click(); + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + await connectorCta.click(); + + // Source step should now render with source schema fields and CTA. + await expect(page.getByText("Model preview")).toBeVisible(); + const sourceCta = page.getByRole("button", { + name: /Test and Add data|Importing data|Add data/i, + }); + await expect(sourceCta).toBeVisible(); + + // Source fields should be present; connector-only auth fields should not be required to show. + await expect(page.getByRole("textbox", { name: "GCS URI" })).toBeVisible( + {}, + ); + await expect( + page.getByRole("textbox", { name: "Model name" }), + ).toBeVisible(); + }); }); From e3f3a4b1beeafcba9d6620118d39acbaf5ddc1dd Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 22 Dec 2025 18:10:32 +0800 Subject: [PATCH 067/120] preserves auth selection e2e --- .../connectors/multi-step-connector.spec.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/web-local/tests/connectors/multi-step-connector.spec.ts b/web-local/tests/connectors/multi-step-connector.spec.ts index 4462004e9f4..778f44fd9b3 100644 --- a/web-local/tests/connectors/multi-step-connector.spec.ts +++ b/web-local/tests/connectors/multi-step-connector.spec.ts @@ -80,4 +80,53 @@ test.describe("Multi-step connector wrapper", () => { page.getByRole("textbox", { name: "Model name" }), ).toBeVisible(); }); + + test("GCS connector - preserves auth selection across steps", async ({ + page, + }) => { + const hmacKey = process.env.RILL_RUNTIME_GCS_TEST_HMAC_KEY; + const hmacSecret = process.env.RILL_RUNTIME_GCS_TEST_HMAC_SECRET; + if (!hmacKey || !hmacSecret) { + test.skip( + true, + "RILL_RUNTIME_GCS_TEST_HMAC_KEY or RILL_RUNTIME_GCS_TEST_HMAC_SECRET is not set", + ); + } + + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + // Pick HMAC auth and fill required fields. + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page.getByRole("textbox", { name: "Access Key ID" }).fill(hmacKey!); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill(hmacSecret!); + + // Submit connector step via CTA to transition to source step. + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + await expect(connectorCta).toBeEnabled(); + await connectorCta.click(); + await expect(page.getByText("Model preview")).toBeVisible(); + + // Go back to connector step. + await page.getByRole("button", { name: "Back" }).click(); + + // Auth selection and values should persist. + await expect(page.getByText("Connector preview")).toBeVisible(); + await expect(page.getByRole("radio", { name: "HMAC keys" })).toBeChecked({ + timeout: 5000, + }); + await expect( + page.getByRole("textbox", { name: "Access Key ID" }), + ).toHaveValue(hmacKey!); + await expect( + page.getByRole("textbox", { name: "Secret Access Key" }), + ).toHaveValue(hmacSecret!); + }); }); From 5203e2b8cf16f503a2bc2f6b14ed6166eea3f221 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 22 Dec 2025 18:23:04 +0800 Subject: [PATCH 068/120] e2e for public auth option --- .../connectors/multi-step-connector.spec.ts | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/web-local/tests/connectors/multi-step-connector.spec.ts b/web-local/tests/connectors/multi-step-connector.spec.ts index 778f44fd9b3..39abf31954d 100644 --- a/web-local/tests/connectors/multi-step-connector.spec.ts +++ b/web-local/tests/connectors/multi-step-connector.spec.ts @@ -129,4 +129,59 @@ test.describe("Multi-step connector wrapper", () => { page.getByRole("textbox", { name: "Secret Access Key" }), ).toHaveValue(hmacSecret!); }); + + test("GCS connector - disables submit until auth requirements met", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + + // Default auth is credentials (file upload); switch to HMAC to check required fields. + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await expect(connectorCta).toBeDisabled(); + + // Fill key only -> still disabled. + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(connectorCta).toBeDisabled(); + + // Fill secret -> enabled. + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(connectorCta).toBeEnabled(); + }); + + test("GCS connector - public auth option keeps submit enabled and allows advancing", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + + await page.locator("#gcs").click(); + await page.waitForSelector('form[id*="gcs"]'); + + const connectorCta = page.getByRole("button", { + name: /Test and Connect|Continue/i, + }); + + // Switch to Public (no required fields) -> CTA should remain enabled and allow advancing. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(connectorCta).toBeEnabled(); + await connectorCta.click(); + + // Should land on source step without needing connector fields. + await expect(page.getByText("Model preview")).toBeVisible(); + await expect( + page.getByRole("button", { name: /Test and Add data|Add data/i }), + ).toBeVisible(); + }); }); From 1aa737b9ed6e00830c4c616aa90d513bb93138c9 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 10:42:40 +0800 Subject: [PATCH 069/120] import direction --- .../features/sources/modal/connector-schemas.ts | 14 -------------- web-common/src/features/sources/modal/utils.ts | 2 +- .../templates/JSONSchemaFormRenderer.svelte | 3 +-- web-common/src/features/templates/schema-utils.ts | 14 ++++++++++++++ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 0b94cffa19f..0b7cff5363b 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -17,17 +17,3 @@ export function getConnectorSchema( if (!schema?.properties) return null; return schema; } - -export function isStepMatch( - schema: MultiStepFormSchema | null, - key: string, - step?: "connector" | "source" | string, -): boolean { - if (!schema?.properties) return false; - const prop = schema.properties[key]; - if (!prop) return false; - if (!step) return true; - const propStep = prop["x-step"]; - if (!propStep) return true; - return propStep === step; -} diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 4ba3427df79..2934389ed8b 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -6,8 +6,8 @@ import { findRadioEnumKey, getRadioEnumOptions, getRequiredFieldsByEnumValue, + isStepMatch, } from "../../templates/schema-utils"; -import { isStepMatch } from "./connector-schemas"; /** * Returns true for undefined, null, empty string, or whitespace-only string. diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte index 525169d0430..d15e43b8800 100644 --- a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -5,8 +5,7 @@ JSONSchemaField, MultiStepFormSchema, } from "../sources/modal/types"; - import { isVisibleForValues } from "./schema-utils"; - import { isStepMatch } from "../sources/modal/connector-schemas"; + import { isStepMatch, isVisibleForValues } from "./schema-utils"; export let schema: MultiStepFormSchema | null = null; export let step: string | undefined = undefined; diff --git a/web-common/src/features/templates/schema-utils.ts b/web-common/src/features/templates/schema-utils.ts index 8cbe99d1df3..09ec2911432 100644 --- a/web-common/src/features/templates/schema-utils.ts +++ b/web-common/src/features/templates/schema-utils.ts @@ -10,6 +10,20 @@ export type RadioEnumOption = { hint?: string; }; +export function isStepMatch( + schema: MultiStepFormSchema | null, + key: string, + step?: "connector" | "source" | string, +): boolean { + if (!schema?.properties) return false; + const prop = schema.properties[key]; + if (!prop) return false; + if (!step) return true; + const propStep = prop["x-step"]; + if (!propStep) return true; + return propStep === step; +} + export function isVisibleForValues( schema: MultiStepFormSchema, key: string, From 3d87f285025818fdf7e56793a9e48513282bf020 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 11:08:11 +0800 Subject: [PATCH 070/120] fix find radio enum key fallback --- web-common/src/features/templates/schema-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/templates/schema-utils.ts b/web-common/src/features/templates/schema-utils.ts index 09ec2911432..a4544947297 100644 --- a/web-common/src/features/templates/schema-utils.ts +++ b/web-common/src/features/templates/schema-utils.ts @@ -57,7 +57,7 @@ export function findRadioEnumKey(schema: MultiStepFormSchema): string | null { return key; } } - return schema.properties.auth_method ? "auth_method" : null; + return null; } export function getRadioEnumOptions(schema: MultiStepFormSchema): { From 8222f10d13560baaceef05c52c0f49eab977c5d2 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 11:21:56 +0800 Subject: [PATCH 071/120] remove deadcode --- .../features/sources/modal/AddDataForm.svelte | 2 - .../modal/MultiStepConnectorFlow.svelte | 76 ++++--------------- 2 files changed, 14 insertions(+), 64 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 0bb6d33fb77..cf637fd22e1 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -401,8 +401,6 @@ | null = null; - let stepProperties: ConnectorDriverProperty[] | undefined = undefined; let selectedAuthMethod = ""; $: selectedAuthMethod = $selectedAuthMethodStore; - // Compute which properties to show for the current step. - $: stepProperties = - stepState.step === "source" - ? (connector.sourceProperties ?? []) - : properties; - // Initialize source step values from stored connector config. $: if (stepState.step === "source" && stepState.connectorConfig) { const sourceProperties = connector.sourceProperties ?? []; @@ -168,54 +157,17 @@ $: shouldShowSkipLink = stepState.step === "connector"; -{#if stepState.step === "connector"} - - {#if activeSchema} - - {:else} - - {/if} - -{:else} - - {#if activeSchema} - - {:else} - - {/if} - -{/if} + + + From 7e03a721aef0fc2a064d6134b8f6fb0dc69cab06 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 12:17:38 +0800 Subject: [PATCH 072/120] schemasafe init --- package-lock.json | 1 + web-common/package.json | 5 +- .../sources/modal/AddDataFormManager.ts | 65 +++++++++++------ .../features/sources/modal/FormValidation.ts | 70 ++----------------- .../src/features/sources/modal/yupSchemas.ts | 31 -------- web-common/vite.config.ts | 3 + 6 files changed, 55 insertions(+), 120 deletions(-) diff --git a/package-lock.json b/package-lock.json index a2476d53e9f..af85c8e0b54 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46859,6 +46859,7 @@ "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", "@dagrejs/dagre": "^1.1.3", + "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", diff --git a/web-common/package.json b/web-common/package.json index a34c03c267c..c1dfd35cc60 100644 --- a/web-common/package.json +++ b/web-common/package.json @@ -19,7 +19,6 @@ }, "devDependencies": { "@bufbuild/protobuf": "^1.0.0", - "@dagrejs/dagre": "^1.1.3", "@codemirror/autocomplete": "^6.18.1", "@codemirror/commands": "^6.7.0", "@codemirror/lang-json": "^6.0.1", @@ -34,6 +33,8 @@ "@codemirror/view": "^6.34.1", "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", + "@dagrejs/dagre": "^1.1.3", + "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", @@ -64,6 +65,7 @@ "@types/dompurify": "^3.0.5", "@types/luxon": "^3.4.2", "@types/memoizee": "^0.4.11", + "@xyflow/svelte": "^0.1.39", "autoprefixer": "^10.4.20", "bits-ui": "^0.22.0", "chroma-js": "^3.1.2", @@ -114,7 +116,6 @@ "vega-lite": "^5.23.0", "vega-typings": "^1.3.1", "vitest": "^3.1.1", - "@xyflow/svelte": "^0.1.39", "yaml": "^2.4.5", "yup": "^1.4.0" }, diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 0c8a9933fe4..f78a1f2727a 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -2,6 +2,7 @@ import { superForm, defaults } from "sveltekit-superforms"; import type { SuperValidated } from "sveltekit-superforms"; import { yup, + schemasafe, type Infer as YupInfer, type InferIn as YupInferIn, } from "sveltekit-superforms/adapters"; @@ -138,31 +139,51 @@ export class AddDataFormManager { ); // Superforms: params - const paramsSchemaDef = getValidationSchemaForConnector( - connector.name as string, - formType, - { - isMultiStepConnector: this.isMultiStepConnector, - authMethodGetter: this.getSelectedAuthMethod, - }, - ); - const paramsAdapter = yup(paramsSchemaDef); - type ParamsOut = YupInfer; - type ParamsIn = YupInferIn; const initialFormValues = getInitialFormValuesFromProperties( this.properties, ); - const paramsDefaults = defaults( - initialFormValues as Partial, - paramsAdapter, - ); - this.params = superForm(paramsDefaults, { - SPA: true, - validators: paramsAdapter, - onUpdate: onParamsUpdate, - resetForm: false, - validationMethod: "onsubmit", - }); + const multiStepSchema = this.isMultiStepConnector + ? getConnectorSchema(connector.name ?? "") + : null; + + if (multiStepSchema) { + const paramsAdapter = schemasafe(multiStepSchema); + type ParamsOut = Record; + type ParamsIn = Record; + const paramsDefaults = defaults( + initialFormValues as Partial, + paramsAdapter, + ); + this.params = superForm(paramsDefaults, { + SPA: true, + validators: paramsAdapter, + onUpdate: onParamsUpdate, + resetForm: false, + validationMethod: "onsubmit", + }); + } else { + const paramsSchemaDef = getValidationSchemaForConnector( + connector.name as string, + formType, + { + isMultiStepConnector: this.isMultiStepConnector, + }, + ); + const paramsAdapter = yup(paramsSchemaDef); + type ParamsOut = YupInfer; + type ParamsIn = YupInferIn; + const paramsDefaults = defaults( + initialFormValues as Partial, + paramsAdapter, + ); + this.params = superForm(paramsDefaults, { + SPA: true, + validators: paramsAdapter, + onUpdate: onParamsUpdate, + resetForm: false, + validationMethod: "onsubmit", + }); + } // Superforms: dsn const dsnAdapter = yup(dsnSchema); diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index b5853948fd2..5bc5f80e6d5 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,11 +1,6 @@ import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import { getConnectorSchema } from "./connector-schemas"; -import { - getFieldLabel, - getRequiredFieldsByEnumValue, -} from "../../templates/schema-utils"; -import type { AddDataFormType, MultiStepFormSchema } from "./types"; +import type { AddDataFormType } from "./types"; export { dsnSchema }; @@ -14,10 +9,9 @@ export function getValidationSchemaForConnector( formType: AddDataFormType, opts?: { isMultiStepConnector?: boolean; - authMethodGetter?: () => string | undefined; }, ) { - const { isMultiStepConnector, authMethodGetter } = opts || {}; + const { isMultiStepConnector } = opts || {}; // For multi-step source flows, prefer the connector-specific schema when present // so step 1 (connector) validation doesn't require source-only fields. @@ -28,61 +22,7 @@ export function getValidationSchemaForConnector( } } - // For multi-step connector step, prefer connector-specific schema when present. - if (isMultiStepConnector && formType === "connector") { - // Generic dynamic schema based on auth options, driven by config. - const dynamicSchema = makeAuthOptionValidationSchema( - name, - authMethodGetter, - ); - if (dynamicSchema) return dynamicSchema; - - const connectorKey = `${name}_connector`; - if (connectorKey in getYupSchema) { - return getYupSchema[connectorKey as keyof typeof getYupSchema]; - } - } - - return getYupSchema[name as keyof typeof getYupSchema]; -} - -/** - * Build a yup schema that enforces required fields for the selected auth option - * using the multi-step auth config. This keeps validation in sync with the UI - * definitions alongside the schema utilities. - */ -function makeAuthOptionValidationSchema( - connectorName: string, - getAuthMethod?: () => string | undefined, -) { - const schema = getConnectorSchema(connectorName); - if (!schema) return null; - - const fieldValidations: Record = {}; - const requiredByMethod = getRequiredFieldsByEnumValue(schema, { - step: "connector", - }); - - for (const [method, fields] of Object.entries(requiredByMethod || {})) { - for (const fieldId of fields) { - const label = getFieldLabel(schema as MultiStepFormSchema, fieldId); - fieldValidations[fieldId] = ( - fieldValidations[fieldId] || yup.string() - ).test( - `required-${fieldId}-${method}`, - `${label} is required`, - (value) => { - if (!getAuthMethod) return true; - const current = getAuthMethod(); - if (current !== method) return true; - return !!value; - }, - ); - } - } - - // If nothing to validate, skip dynamic schema. - if (!Object.keys(fieldValidations).length) return null; - - return yup.object().shape(fieldValidations); + return ( + getYupSchema[name as keyof typeof getYupSchema] || yup.object().shape({}) + ); } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 949cfecbbe6..e4b4e102d51 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,15 +5,6 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { - // Keep base auth fields optional; per-method required fields come from - // multi-step auth configs. This schema is a safe fallback. - s3_connector: yup.object().shape({ - aws_access_key_id: yup.string().optional(), - aws_secret_access_key: yup.string().optional(), - region: yup.string().optional(), - endpoint: yup.string().optional(), - }), - s3_source: yup.object().shape({ path: yup .string() @@ -25,18 +16,6 @@ export const getYupSchema = { .required(), }), - // Keep base auth fields optional; per-method required fields come from - // multi-step auth configs. This schema is a safe fallback. - gcs_connector: yup.object().shape({ - google_application_credentials: yup.string().optional(), - key_id: yup.string().optional(), - secret: yup.string().optional(), - path: yup - .string() - .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") - .optional(), - }), - gcs_source: yup.object().shape({ path: yup .string() @@ -86,16 +65,6 @@ export const getYupSchema = { .required("Google application credentials is required"), }), - // Keep these optional here; per-auth required fields are enforced dynamically - // via multi-step auth configs. This schema acts as a safe fallback (e.g. source - // step selection of `${name}_connector`). - azure_connector: yup.object().shape({ - azure_storage_account: yup.string().optional(), - azure_storage_key: yup.string().optional(), - azure_storage_sas_token: yup.string().optional(), - azure_storage_connection_string: yup.string().optional(), - }), - azure_source: yup.object().shape({ path: yup .string() diff --git a/web-common/vite.config.ts b/web-common/vite.config.ts index 4e9b1e458e3..a4416806f66 100644 --- a/web-common/vite.config.ts +++ b/web-common/vite.config.ts @@ -34,6 +34,9 @@ export default defineConfig(({ mode }) => { alias, }, plugins: [sveltekit()], + optimizeDeps: { + include: ["@exodus/schemasafe"], + }, test: { workspace: [ { From ea54ff981489f74eb81e3c8332e48266080d6309 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 12:26:17 +0800 Subject: [PATCH 073/120] remova source schema --- .../src/features/sources/modal/yupSchemas.ts | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index e4b4e102d51..18fe8befa14 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,28 +5,6 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { - s3_source: yup.object().shape({ - path: yup - .string() - .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") - .required(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required(), - }), - - gcs_source: yup.object().shape({ - path: yup - .string() - .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") - .optional(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required(), - }), - https: yup.object().shape({ path: yup .string() @@ -65,20 +43,6 @@ export const getYupSchema = { .required("Google application credentials is required"), }), - azure_source: yup.object().shape({ - path: yup - .string() - .matches( - /^azure:\/\//, - "Must be an Azure URI (e.g. azure://container/path)", - ) - .required("Path is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), - }), - postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), From 2042ef3ca7dce4d0a49aa545a12ef09f8a7f1bbc Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 21:28:07 +0800 Subject: [PATCH 074/120] Revert "remova source schema" This reverts commit ea54ff981489f74eb81e3c8332e48266080d6309. --- .../src/features/sources/modal/yupSchemas.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 18fe8befa14..e4b4e102d51 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,6 +5,28 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { + s3_source: yup.object().shape({ + path: yup + .string() + .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") + .required(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required(), + }), + + gcs_source: yup.object().shape({ + path: yup + .string() + .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") + .optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required(), + }), + https: yup.object().shape({ path: yup .string() @@ -43,6 +65,20 @@ export const getYupSchema = { .required("Google application credentials is required"), }), + azure_source: yup.object().shape({ + path: yup + .string() + .matches( + /^azure:\/\//, + "Must be an Azure URI (e.g. azure://container/path)", + ) + .required("Path is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Source name is required"), + }), + postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), From fb0a97002ee1dcd99a7578842d42b375f29e1a17 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 21:56:05 +0800 Subject: [PATCH 075/120] Revert "schemasafe init" This reverts commit 7e03a721aef0fc2a064d6134b8f6fb0dc69cab06. --- package-lock.json | 1 - web-common/package.json | 5 +- .../sources/modal/AddDataFormManager.ts | 65 ++++++----------- .../features/sources/modal/FormValidation.ts | 70 +++++++++++++++++-- .../src/features/sources/modal/yupSchemas.ts | 31 ++++++++ web-common/vite.config.ts | 3 - 6 files changed, 120 insertions(+), 55 deletions(-) diff --git a/package-lock.json b/package-lock.json index af85c8e0b54..a2476d53e9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46859,7 +46859,6 @@ "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", "@dagrejs/dagre": "^1.1.3", - "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", diff --git a/web-common/package.json b/web-common/package.json index c1dfd35cc60..a34c03c267c 100644 --- a/web-common/package.json +++ b/web-common/package.json @@ -19,6 +19,7 @@ }, "devDependencies": { "@bufbuild/protobuf": "^1.0.0", + "@dagrejs/dagre": "^1.1.3", "@codemirror/autocomplete": "^6.18.1", "@codemirror/commands": "^6.7.0", "@codemirror/lang-json": "^6.0.1", @@ -33,8 +34,6 @@ "@codemirror/view": "^6.34.1", "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-web": "^1.4.0", - "@dagrejs/dagre": "^1.1.3", - "@exodus/schemasafe": "^1.3.0", "@floating-ui/dom": "^1.7.4", "@replit/codemirror-indentation-markers": "^6.5.3", "@storybook/addon-actions": "^7.6.17", @@ -65,7 +64,6 @@ "@types/dompurify": "^3.0.5", "@types/luxon": "^3.4.2", "@types/memoizee": "^0.4.11", - "@xyflow/svelte": "^0.1.39", "autoprefixer": "^10.4.20", "bits-ui": "^0.22.0", "chroma-js": "^3.1.2", @@ -116,6 +114,7 @@ "vega-lite": "^5.23.0", "vega-typings": "^1.3.1", "vitest": "^3.1.1", + "@xyflow/svelte": "^0.1.39", "yaml": "^2.4.5", "yup": "^1.4.0" }, diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index f78a1f2727a..0c8a9933fe4 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -2,7 +2,6 @@ import { superForm, defaults } from "sveltekit-superforms"; import type { SuperValidated } from "sveltekit-superforms"; import { yup, - schemasafe, type Infer as YupInfer, type InferIn as YupInferIn, } from "sveltekit-superforms/adapters"; @@ -139,51 +138,31 @@ export class AddDataFormManager { ); // Superforms: params + const paramsSchemaDef = getValidationSchemaForConnector( + connector.name as string, + formType, + { + isMultiStepConnector: this.isMultiStepConnector, + authMethodGetter: this.getSelectedAuthMethod, + }, + ); + const paramsAdapter = yup(paramsSchemaDef); + type ParamsOut = YupInfer; + type ParamsIn = YupInferIn; const initialFormValues = getInitialFormValuesFromProperties( this.properties, ); - const multiStepSchema = this.isMultiStepConnector - ? getConnectorSchema(connector.name ?? "") - : null; - - if (multiStepSchema) { - const paramsAdapter = schemasafe(multiStepSchema); - type ParamsOut = Record; - type ParamsIn = Record; - const paramsDefaults = defaults( - initialFormValues as Partial, - paramsAdapter, - ); - this.params = superForm(paramsDefaults, { - SPA: true, - validators: paramsAdapter, - onUpdate: onParamsUpdate, - resetForm: false, - validationMethod: "onsubmit", - }); - } else { - const paramsSchemaDef = getValidationSchemaForConnector( - connector.name as string, - formType, - { - isMultiStepConnector: this.isMultiStepConnector, - }, - ); - const paramsAdapter = yup(paramsSchemaDef); - type ParamsOut = YupInfer; - type ParamsIn = YupInferIn; - const paramsDefaults = defaults( - initialFormValues as Partial, - paramsAdapter, - ); - this.params = superForm(paramsDefaults, { - SPA: true, - validators: paramsAdapter, - onUpdate: onParamsUpdate, - resetForm: false, - validationMethod: "onsubmit", - }); - } + const paramsDefaults = defaults( + initialFormValues as Partial, + paramsAdapter, + ); + this.params = superForm(paramsDefaults, { + SPA: true, + validators: paramsAdapter, + onUpdate: onParamsUpdate, + resetForm: false, + validationMethod: "onsubmit", + }); // Superforms: dsn const dsnAdapter = yup(dsnSchema); diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 5bc5f80e6d5..b5853948fd2 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,6 +1,11 @@ import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import type { AddDataFormType } from "./types"; +import { getConnectorSchema } from "./connector-schemas"; +import { + getFieldLabel, + getRequiredFieldsByEnumValue, +} from "../../templates/schema-utils"; +import type { AddDataFormType, MultiStepFormSchema } from "./types"; export { dsnSchema }; @@ -9,9 +14,10 @@ export function getValidationSchemaForConnector( formType: AddDataFormType, opts?: { isMultiStepConnector?: boolean; + authMethodGetter?: () => string | undefined; }, ) { - const { isMultiStepConnector } = opts || {}; + const { isMultiStepConnector, authMethodGetter } = opts || {}; // For multi-step source flows, prefer the connector-specific schema when present // so step 1 (connector) validation doesn't require source-only fields. @@ -22,7 +28,61 @@ export function getValidationSchemaForConnector( } } - return ( - getYupSchema[name as keyof typeof getYupSchema] || yup.object().shape({}) - ); + // For multi-step connector step, prefer connector-specific schema when present. + if (isMultiStepConnector && formType === "connector") { + // Generic dynamic schema based on auth options, driven by config. + const dynamicSchema = makeAuthOptionValidationSchema( + name, + authMethodGetter, + ); + if (dynamicSchema) return dynamicSchema; + + const connectorKey = `${name}_connector`; + if (connectorKey in getYupSchema) { + return getYupSchema[connectorKey as keyof typeof getYupSchema]; + } + } + + return getYupSchema[name as keyof typeof getYupSchema]; +} + +/** + * Build a yup schema that enforces required fields for the selected auth option + * using the multi-step auth config. This keeps validation in sync with the UI + * definitions alongside the schema utilities. + */ +function makeAuthOptionValidationSchema( + connectorName: string, + getAuthMethod?: () => string | undefined, +) { + const schema = getConnectorSchema(connectorName); + if (!schema) return null; + + const fieldValidations: Record = {}; + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { + step: "connector", + }); + + for (const [method, fields] of Object.entries(requiredByMethod || {})) { + for (const fieldId of fields) { + const label = getFieldLabel(schema as MultiStepFormSchema, fieldId); + fieldValidations[fieldId] = ( + fieldValidations[fieldId] || yup.string() + ).test( + `required-${fieldId}-${method}`, + `${label} is required`, + (value) => { + if (!getAuthMethod) return true; + const current = getAuthMethod(); + if (current !== method) return true; + return !!value; + }, + ); + } + } + + // If nothing to validate, skip dynamic schema. + if (!Object.keys(fieldValidations).length) return null; + + return yup.object().shape(fieldValidations); } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index e4b4e102d51..949cfecbbe6 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,6 +5,15 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + s3_connector: yup.object().shape({ + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + region: yup.string().optional(), + endpoint: yup.string().optional(), + }), + s3_source: yup.object().shape({ path: yup .string() @@ -16,6 +25,18 @@ export const getYupSchema = { .required(), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + gcs_connector: yup.object().shape({ + google_application_credentials: yup.string().optional(), + key_id: yup.string().optional(), + secret: yup.string().optional(), + path: yup + .string() + .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") + .optional(), + }), + gcs_source: yup.object().shape({ path: yup .string() @@ -65,6 +86,16 @@ export const getYupSchema = { .required("Google application credentials is required"), }), + // Keep these optional here; per-auth required fields are enforced dynamically + // via multi-step auth configs. This schema acts as a safe fallback (e.g. source + // step selection of `${name}_connector`). + azure_connector: yup.object().shape({ + azure_storage_account: yup.string().optional(), + azure_storage_key: yup.string().optional(), + azure_storage_sas_token: yup.string().optional(), + azure_storage_connection_string: yup.string().optional(), + }), + azure_source: yup.object().shape({ path: yup .string() diff --git a/web-common/vite.config.ts b/web-common/vite.config.ts index a4416806f66..4e9b1e458e3 100644 --- a/web-common/vite.config.ts +++ b/web-common/vite.config.ts @@ -34,9 +34,6 @@ export default defineConfig(({ mode }) => { alias, }, plugins: [sveltekit()], - optimizeDeps: { - include: ["@exodus/schemasafe"], - }, test: { workspace: [ { From 3e277752517408f41f1b06ef219d3d07a5572859 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 23 Dec 2025 21:59:44 +0800 Subject: [PATCH 076/120] rename schema field --- .../features/templates/JSONSchemaFormRenderer.svelte | 11 ++++------- .../SchemaField.svelte} | 6 ++++-- 2 files changed, 8 insertions(+), 9 deletions(-) rename web-common/src/features/{sources/modal/JSONSchemaFieldControl.svelte => templates/SchemaField.svelte} (92%) diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte index d15e43b8800..3d012838118 100644 --- a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -1,10 +1,7 @@ - -
-
- - {#if connectorType === "rill-managed"} -
- -
- {/if} -
- - {#if connectorType === "self-hosted" || connectorType === "clickhouse-cloud"} - - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {@const isPortField = propertyKey === "port"} - {@const isSSLField = propertyKey === "ssl"} - -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - options={connectorType === "clickhouse-cloud" && isPortField - ? [ - { value: "8443", label: "8443 (HTTPS)" }, - { value: "9440", label: "9440 (Native Secure)" }, - ] - : undefined} - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- - -
- {#each dsnProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- -
- {/each} -
- - - {:else} - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- {/if} -
From 8ef1e56e37b4697a48d23778d4be7dca2943f929 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 29 Dec 2025 15:04:05 -0800 Subject: [PATCH 095/120] move clickhouse logic to manager --- .../features/sources/modal/AddDataForm.svelte | 207 +++++------------- .../sources/modal/AddDataFormManager.ts | 192 +++++++++++++++- 2 files changed, 240 insertions(+), 159 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 74ddf19efd4..679c5834a0d 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -27,15 +27,15 @@ import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - - import { AddDataFormManager } from "./AddDataFormManager"; + import { + AddDataFormManager, + type ClickhouseUiState, + } from "./AddDataFormManager"; import AddDataFormSection from "./AddDataFormSection.svelte"; import { get, type Writable } from "svelte/store"; import { ConnectorDriverPropertyType, - type ConnectorDriverProperty, } from "@rilldata/web-common/runtime-client"; - import { getInitialFormValuesFromProperties } from "../sourceUtils"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -119,157 +119,49 @@ let clickhouseError: string | null = null; let clickhouseErrorDetails: string | undefined = undefined; - let clickhouseFormId: string = ""; - let clickhouseSubmitting: boolean; - let clickhouseIsSubmitDisabled: boolean; let clickhouseConnectorType: ClickHouseConnectorType = "self-hosted"; - let clickhouseParamsForm; - let clickhouseDsnForm; - let clickhouseProperties: ConnectorDriverProperty[] = []; - let clickhouseFilteredProperties: ConnectorDriverProperty[] = []; - let clickhouseDsnProperties: ConnectorDriverProperty[] = []; - let clickhouseShowSaveAnyway: boolean = false; - const clickhouseInitialValues = - connector.name === "clickhouse" - ? getInitialFormValuesFromProperties(connector.configProperties ?? []) - : {}; - let prevClickhouseConnectorType: ClickHouseConnectorType = - clickhouseConnectorType; + let clickhouseUiState: ClickhouseUiState | null = null; + let clickhouseSaving = false; + let effectiveClickhouseSubmitting = false; const paramsFormStore = paramsForm as unknown as Writable< Record >; const dsnFormStore = dsnForm as unknown as Writable>; - // Keep ClickHouse connector type in sync with form state and apply defaults + // ClickHouse-specific derived state handled by the manager $: if (connector.name === "clickhouse") { - // Always set connector_type on the params form - paramsForm.update( - ($form: any) => ({ - ...$form, - connector_type: clickhouseConnectorType, - }), - { taint: false } as any, - ); + clickhouseUiState = formManager.computeClickhouseState({ + connectorType: clickhouseConnectorType, + connectionTab, + paramsFormValues: $paramsForm, + dsnFormValues: $dsnForm, + paramsErrors: $paramsErrors, + dsnErrors: $dsnErrors, + paramsForm, + dsnForm, + paramsSubmitting: $paramsSubmitting, + dsnSubmitting: $dsnSubmitting, + }); if ( - clickhouseConnectorType === "rill-managed" && - Object.keys($paramsForm).length > 1 + clickhouseUiState?.enforcedConnectionTab && + clickhouseUiState.enforcedConnectionTab !== connectionTab ) { - paramsForm.update( - () => ({ managed: true, connector_type: "rill-managed" }), - { taint: false } as any, - ); + connectionTab = clickhouseUiState.enforcedConnectionTab; + } + + if (clickhouseUiState?.shouldClearErrors) { clickhouseError = null; clickhouseErrorDetails = undefined; - } else if ( - prevClickhouseConnectorType === "rill-managed" && - clickhouseConnectorType === "self-hosted" - ) { - paramsForm.update( - () => ({ ...clickhouseInitialValues, managed: false }), - { taint: false } as any, - ); - } else if ( - prevClickhouseConnectorType !== "clickhouse-cloud" && - clickhouseConnectorType === "clickhouse-cloud" - ) { - paramsForm.update( - () => ({ - ...clickhouseInitialValues, - managed: false, - port: "8443", - ssl: true, - }), - { taint: false } as any, - ); - } else if ( - prevClickhouseConnectorType === "clickhouse-cloud" && - clickhouseConnectorType === "self-hosted" - ) { - paramsForm.update( - () => ({ ...clickhouseInitialValues, managed: false }), - { taint: false } as any, - ); } - prevClickhouseConnectorType = clickhouseConnectorType; + } else { + clickhouseUiState = null; } - // Force parameters tab for Rill-managed - $: if ( - connector.name === "clickhouse" && - clickhouseConnectorType === "rill-managed" - ) { - connectionTab = "parameters"; - } - - // Use manager forms for ClickHouse - $: if (connector.name === "clickhouse") { - clickhouseParamsForm = paramsForm; - clickhouseDsnForm = dsnForm; - clickhouseFormId = connectionTab === "dsn" ? dsnFormId : paramsFormId; - clickhouseSubmitting = - connectionTab === "dsn" ? $dsnSubmitting : $paramsSubmitting; - } - - $: if (connector.name === "clickhouse") { - clickhouseShowSaveAnyway = showSaveAnyway; - } - - // ClickHouse-specific property filtering and disabled state - $: clickhouseProperties = + $: effectiveClickhouseSubmitting = connector.name === "clickhouse" - ? clickhouseConnectorType === "rill-managed" - ? (connector.sourceProperties ?? []) - : (connector.configProperties ?? []) - : []; - - $: clickhouseFilteredProperties = clickhouseProperties.filter( - (property) => - !property.noPrompt && - property.key !== "managed" && - (connectionTab !== "dsn" ? property.key !== "dsn" : true), - ); - - $: clickhouseDsnProperties = - connector.configProperties?.filter((property) => property.key === "dsn") ?? - []; - - $: clickhouseIsSubmitDisabled = (() => { - if (connector.name !== "clickhouse") return false; - if (clickhouseConnectorType === "rill-managed") { - for (const property of clickhouseFilteredProperties) { - if (property.required) { - const key = String(property.key); - const value = $paramsForm[key]; - if (isEmpty(value) || ($paramsErrors[key] as any)?.length) - return true; - } - } - return false; - } else if (connectionTab === "dsn") { - for (const property of clickhouseDsnProperties) { - if (property.required) { - const key = String(property.key); - const value = $dsnForm[key]; - if (isEmpty(value) || ($dsnErrors[key] as any)?.length) return true; - } - } - return false; - } else { - for (const property of clickhouseFilteredProperties) { - if (property.required && property.key !== "managed") { - const key = String(property.key); - const value = $paramsForm[key]; - if (isEmpty(value) || ($paramsErrors[key] as any)?.length) - return true; - } - } - if (clickhouseConnectorType === "clickhouse-cloud") { - if (!$paramsForm.ssl) return true; - } - return false; - } - })(); + ? clickhouseSaving || clickhouseUiState?.submitting || false + : submitting; // Hide Save Anyway once we advance to the model step in multi-step flows. $: if (isMultiStepConnector && stepState.step === "source") { @@ -331,7 +223,7 @@ step: stepState.step, submitting, clickhouseConnectorType, - clickhouseSubmitting, + clickhouseSubmitting: effectiveClickhouseSubmitting, selectedAuthMethod: activeAuthMethod ?? undefined, }); @@ -350,7 +242,8 @@ saveAnyway = false; } - $: isSubmitting = submitting; + $: isSubmitting = + connector.name === "clickhouse" ? effectiveClickhouseSubmitting : submitting; // Reset errors when form is modified $: (() => { @@ -385,13 +278,13 @@ const values = connector.name === "clickhouse" ? connectionTab === "dsn" - ? $clickhouseDsnForm - : $clickhouseParamsForm + ? $dsnForm + : $paramsForm : onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; if (connector.name === "clickhouse") { - clickhouseSubmitting = true; + clickhouseSaving = true; } const result = await formManager.saveConnectorAnyway({ queryClient, @@ -419,7 +312,7 @@ } saveAnyway = false; if (connector.name === "clickhouse") { - clickhouseSubmitting = false; + clickhouseSaving = false; } } @@ -434,14 +327,13 @@ paramsFormValues: $paramsForm, dsnFormValues: $dsnForm, clickhouseConnectorType, - clickhouseParamsValues: $clickhouseParamsForm, - clickhouseDsnValues: $clickhouseDsnForm, + clickhouseParamsValues: $paramsForm, + clickhouseDsnValues: $dsnForm, }); $: isClickhouse = connector.name === "clickhouse"; - $: shouldShowSaveAnywayButton = - isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway); + $: shouldShowSaveAnywayButton = isConnectorForm && showSaveAnyway; $: saveAnywayLoading = isClickhouse - ? clickhouseSubmitting && saveAnyway + ? effectiveClickhouseSubmitting && saveAnyway : submitting && saveAnyway; handleOnUpdate = formManager.makeOnUpdate({ @@ -506,7 +398,7 @@ enhance={paramsEnhance} onSubmit={paramsSubmit} > - {#each clickhouseFilteredProperties as property (property.key)} + {#each clickhouseUiState?.filteredProperties ?? [] as property (property.key)} {@const propertyKey = property.key ?? ""} {@const isPortField = propertyKey === "port"} {@const isSSLField = propertyKey === "ssl"} @@ -565,7 +457,7 @@ enhance={dsnEnhance} onSubmit={dsnSubmit} > - {#each clickhouseDsnProperties as property (property.key)} + {#each clickhouseUiState?.dsnProperties ?? [] as property (property.key)} {@const propertyKey = property.key ?? ""}
- {#each clickhouseFilteredProperties as property (property.key)} + {#each clickhouseUiState?.filteredProperties ?? [] as property (property.key)} {@const propertyKey = property.key ?? ""}
{#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} @@ -736,13 +628,16 @@