From c6c51e455a09ece5890df720ea2be016a31044ce Mon Sep 17 00:00:00 2001 From: hackolade-bot Date: Fri, 18 Apr 2025 16:56:59 +0300 Subject: [PATCH] HCK-10900: filter list of schemas by database from connection settings if provided, take all data if shcema is not provided --- reverse_engineering/api.js | 5 ++- .../connectionSettingsModalConfig.json | 6 +++ .../helpers/snowflakeHelper.js | 39 ++++++++++++------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index 9bb8e090..c65d4b9a 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -60,9 +60,10 @@ const getDbCollectionsNames = async (connectionInfo, logger, cb) => { try { logger.clear(); await snowflakeHelper.connect(logger, connectionInfo); - const schemasInfo = await snowflakeHelper.getSchemasInfo(); + const databaseName = connectionInfo.databaseName; + const schemasInfo = await snowflakeHelper.getSchemasInfo({ databaseName }); logger.log('info', { schemas: schemasInfo }, 'Found schemas'); - const namesBySchemas = await snowflakeHelper.getEntitiesNames({ logger }); + const namesBySchemas = await snowflakeHelper.getEntitiesNames({ databaseName, logger }); logger.log('info', { entities: namesBySchemas }, 'Found entities'); diff --git a/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json b/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json index aca905c3..1a87b710 100644 --- a/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json +++ b/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json @@ -58,6 +58,12 @@ "inputType": "text", "inputTooltip": "Optionally specify the active/current warehouse for the session", "defaultValue": "" + }, + { + "inputLabel": "Database name", + "inputKeyword": "databaseName", + "inputType": "text", + "inputTooltip": "Optionally specify the database name." } ] }, diff --git a/reverse_engineering/helpers/snowflakeHelper.js b/reverse_engineering/helpers/snowflakeHelper.js index 72280e9d..0f903bd6 100644 --- a/reverse_engineering/helpers/snowflakeHelper.js +++ b/reverse_engineering/helpers/snowflakeHelper.js @@ -40,8 +40,15 @@ const connect = async ( name, cloudPlatform, queryRequestTimeout, + databaseName, }, ) => { + if (connection) { + logger.log('info', 'connection already exists', 'Connection'); + + return connection; + } + const account = getAccount(host); const accessUrl = getAccessUrl(account); const timeout = _.toNumber(queryRequestTimeout) || 2 * 60 * 1000; @@ -54,7 +61,8 @@ const connect = async ( `Auth type: ${authType}\n` + `Username: ${username}\n` + `Warehouse: ${warehouse}\n` + - `Role: ${role}`, + `Role: ${role}\n` + + `Database name: ${databaseName}`, 'Connection', ); @@ -483,15 +491,20 @@ const showTablesByDatabases = async databases => databases.map(database => execute(`SHOW TABLES IN DATABASE "${removeQuotes(database.name)}";`)), ); +const showSchemasByDatabase = async databaseName => + databaseName ? showSchemasInDatabase(databaseName) : showSchemas(); + const showDatabases = () => execute('SHOW DATABASES;'); const showSchemas = () => execute('SHOW SCHEMAS;'); -const showExternalTables = () => execute('SHOW EXTERNAL TABLES;'); +const showSchemasInDatabase = databaseName => execute(`SHOW SCHEMAS IN DATABASE "${removeQuotes(databaseName)}";`); + +const showExternalTables = ({ options = '' } = {}) => execute(`SHOW EXTERNAL TABLES${options};`); -const showViews = () => execute('SHOW VIEWS;'); +const showViews = ({ options = '' } = {}) => execute(`SHOW VIEWS${options};`); -const showMaterializedViews = () => execute('SHOW MATERIALIZED VIEWS;'); +const showMaterializedViews = ({ options = '' } = {}) => execute(`SHOW MATERIALIZED VIEWS${options};`); const showIcebergTables = ({ options = '' } = {}) => execute(`SHOW ICEBERG TABLES${options};`); @@ -507,8 +520,8 @@ const splitEntityNames = names => { const isView = name => name.slice(-4) === ' (v)'; -const getSchemasInfo = async () => { - const schemas = await showSchemas().catch(err => [{ status: 'error', message: err.message }]); +const getSchemasInfo = async ({ databaseName }) => { + const schemas = await showSchemasByDatabase(databaseName).catch(err => [{ status: 'error', message: err.message }]); if (schemas[0]?.status === 'error') { return schemas; @@ -592,19 +605,19 @@ const logTablesMeta = async ({ logger, tables = [], icebergTables = [] }) => { logger.log('info', combinedMeta, 'Tables metadata'); }; -const getEntitiesNames = async ({ logger }) => { +const getEntitiesNames = async ({ databaseName, logger }) => { const logError = logErrorAndReturnEmptyArray({ logger, query: 'SHOW' }); - - const databases = await showDatabases().catch(logError); + const databaseQueryOptions = databaseName ? ` IN DATABASE "${removeQuotes(databaseName)}"` : ''; + const databases = databaseName ? [{ name: databaseName }] : await showDatabases().catch(logError); const tablesRows = await showTablesByDatabases(databases).catch(logError); const flatTableRows = tablesRows.flatMap(row => row.value).filter(Boolean); - const icebergTables = await showIcebergTables().catch(logError); + const icebergTables = await showIcebergTables({ options: databaseQueryOptions }).catch(logError); await logTablesMeta({ logger, tables: flatTableRows, icebergTables }); - const externalTableRows = await showExternalTables().catch(logError); - const viewsRows = await showViews().catch(logError); - const materializedViewsRows = await showMaterializedViews().catch(logError); + const externalTableRows = await showExternalTables({ options: databaseQueryOptions }).catch(logError); + const viewsRows = await showViews({ options: databaseQueryOptions }).catch(logError); + const materializedViewsRows = await showMaterializedViews({ options: databaseQueryOptions }).catch(logError); const entitiesRows = [ ...flatTableRows,