From bcb6ca3a39c6c6342ce8f0fc1fa0d3f40efbd42d Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 10 Apr 2025 16:21:38 +0300 Subject: [PATCH 1/2] HCK-10706: add config to get databases --- reverse_engineering/api.js | 22 +++++++++++++++---- reverse_engineering/config.json | 12 ++++++++++ .../connectionSettingsModalConfig.json | 6 +++++ .../helpers/snowflakeHelper.js | 10 +++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index 9bb8e090..d2e382e8 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -48,17 +48,31 @@ const getExternalBrowserUrl = async (connectionInfo, logger, cb) => { } }; -const getDatabases = (connectionInfo, logger, cb) => { - cb(); +const getDatabases = async (connectionInfo, logger, cb) => { + try { + logger.clear(); + + if (connectionInfo.databaseName) { + cb(null, [connectionInfo.databaseName]); + return; + } + + await snowflakeHelper.connect(logger, connectionInfo); + + const databaseNames = await snowflakeHelper.getDatabaseNames({ logger }); + + cb(null, databaseNames); + } catch (err) { + handleError(logger, err, cb); + } }; const getDocumentKinds = (connectionInfo, logger, cb) => { - cb(); + cb(null, []); }; const getDbCollectionsNames = async (connectionInfo, logger, cb) => { try { - logger.clear(); await snowflakeHelper.connect(logger, connectionInfo); const schemasInfo = await snowflakeHelper.getSchemasInfo(); logger.log('info', { schemas: schemasInfo }, 'Found schemas'); diff --git a/reverse_engineering/config.json b/reverse_engineering/config.json index a325b89d..6bd7e101 100644 --- a/reverse_engineering/config.json +++ b/reverse_engineering/config.json @@ -13,6 +13,18 @@ "value": "externalbrowser" }, "value": "connectByExternalBrowser" + }, + { + "dependency": { + "type": "not", + "values": [ + { + "key": "authType", + "value": "externalbrowser" + } + ] + }, + "value": "getDatabases" } ], "helpUrl": "https://hackolade.com/help/ConnecttoaSnowflakeinstance.html" diff --git a/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json b/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json index aca905c3..d5c8f73a 100644 --- a/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json +++ b/reverse_engineering/connection_settings_modal/connectionSettingsModalConfig.json @@ -58,6 +58,12 @@ "inputType": "text", "inputTooltip": "Optionally specify the active/current warehouse for the session", "defaultValue": "" + }, + { + "inputLabel": "Schema name", + "inputKeyword": "databaseName", + "inputType": "text", + "inputPlaceholder": "Optional" } ] }, diff --git a/reverse_engineering/helpers/snowflakeHelper.js b/reverse_engineering/helpers/snowflakeHelper.js index 72280e9d..2ef46c3f 100644 --- a/reverse_engineering/helpers/snowflakeHelper.js +++ b/reverse_engineering/helpers/snowflakeHelper.js @@ -636,6 +636,15 @@ const getEntitiesNames = async ({ logger }) => { }, []); }; +const getDatabaseNames = async ({ logger }) => { + const logError = logErrorAndReturnEmptyArray({ logger, query: 'SHOW' }); + const databases = await showDatabases().catch(logError); + + logger.log('info', { databases }, 'Found databases'); + + return databases.map(({ name }) => name); +}; + const getFullEntityName = (schemaName, tableName) => { return [...schemaName.split('.'), tableName].map(addQuotes).join('.'); }; @@ -1628,6 +1637,7 @@ module.exports = { disconnect, testConnection, getEntitiesNames, + getDatabaseNames, getDDL, getViewDDL, getSchemaDDL, From 1fe6b451fa89041b1f124f7f26a427f8c9fe28b1 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 10 Apr 2025 16:55:01 +0300 Subject: [PATCH 2/2] HCK-10706: RE entities by selected database --- reverse_engineering/api.js | 60 +++++++++------ .../helpers/snowflakeHelper.js | 75 +++++++++++-------- 2 files changed, 79 insertions(+), 56 deletions(-) diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index d2e382e8..e4a63d8a 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -74,9 +74,10 @@ const getDocumentKinds = (connectionInfo, logger, cb) => { const getDbCollectionsNames = async (connectionInfo, logger, cb) => { try { await snowflakeHelper.connect(logger, connectionInfo); - const schemasInfo = await snowflakeHelper.getSchemasInfo(); + const databaseName = connectionInfo.database || connectionInfo.databaseName; + const schemasInfo = await snowflakeHelper.getSchemasInfo(databaseName); logger.log('info', { schemas: schemasInfo }, 'Found schemas'); - const namesBySchemas = await snowflakeHelper.getEntitiesNames({ logger }); + const namesBySchemas = await snowflakeHelper.getEntitiesNames({ databaseName, logger }); logger.log('info', { entities: namesBySchemas }, 'Found entities'); @@ -90,20 +91,25 @@ const getDbCollectionsData = async (data, logger, cb) => { try { logger.log('info', data, 'Retrieving schema', data.hiddenKeys); const collections = data.collectionData.collections; - const dataBaseNames = data.collectionData.dataBaseNames; - const entitiesPromises = await dataBaseNames.reduce(async (packagesPromise, schema) => { + const schemaNames = data.collectionData.dataBaseNames; + const databaseName = data.database || data.databaseName; + const entitiesPromises = await schemaNames.reduce(async (packagesPromise, schemaName) => { const packages = await packagesPromise; - const entities = snowflakeHelper.splitEntityNames(collections[schema]); + const fullSchemaName = databaseName + '.' + schemaName; + const entities = snowflakeHelper.splitEntityNames(collections[schemaName]); - const containerData = await snowflakeHelper.getContainerData({ schema, logger }); - const [database, schemaName] = schema.split('.'); + const containerData = await snowflakeHelper.getContainerData({ databaseName, schemaName, logger }); const tablesPackages = entities.tables.map(async table => { - const fullTableName = snowflakeHelper.getFullEntityName(schema, table); - logger.progress({ message: `Start getting data from table`, containerName: schema, entityName: table }); + const fullTableName = snowflakeHelper.getFullEntityName(databaseName, schemaName, table); + logger.progress({ + message: `Start getting data from table`, + containerName: fullSchemaName, + entityName: table, + }); logger.log( 'info', - { message: `Start getting data from table`, containerName: schema, entityName: table }, + { message: `Start getting data from table`, containerName: fullSchemaName, entityName: table }, 'Getting schema', ); const ddl = await snowflakeHelper.getDDL(fullTableName, logger); @@ -111,12 +117,16 @@ const getDbCollectionsData = async (data, logger, cb) => { logger.progress({ message: `Fetching record for JSON schema inference`, - containerName: schema, + containerName: fullSchemaName, entityName: table, }); logger.log( 'info', - { message: `Fetching record for JSON schema inference`, containerName: schema, entityName: table }, + { + message: `Fetching record for JSON schema inference`, + containerName: fullSchemaName, + entityName: table, + }, 'Getting schema', ); @@ -128,19 +138,23 @@ const getDbCollectionsData = async (data, logger, cb) => { const entityData = await snowflakeHelper.getEntityData({ fullTableName, logger }); - logger.progress({ message: `Schema inference`, containerName: schema, entityName: table }); + logger.progress({ message: `Schema inference`, containerName: fullSchemaName, entityName: table }); logger.log( 'info', - { message: `Schema inference`, containerName: schema, entityName: table }, + { message: `Schema inference`, containerName: fullSchemaName, entityName: table }, 'Getting schema', ); const handledDocuments = snowflakeHelper.handleComplexTypesDocuments(jsonSchema, documents); - logger.progress({ message: `Data retrieved successfully`, containerName: schema, entityName: table }); + logger.progress({ + message: `Data retrieved successfully`, + containerName: fullSchemaName, + entityName: table, + }); logger.log( 'info', - { message: `Data retrieved successfully`, containerName: schema, entityName: table }, + { message: `Data retrieved successfully`, containerName: fullSchemaName, entityName: table }, 'Getting schema', ); @@ -161,7 +175,7 @@ const getDbCollectionsData = async (data, logger, cb) => { }, bucketInfo: { indexes: [], - database, + database: databaseName, ...containerData, }, }; @@ -169,15 +183,15 @@ const getDbCollectionsData = async (data, logger, cb) => { const views = await Promise.all( entities.views.map(async view => { - const fullViewName = snowflakeHelper.getFullEntityName(schema, view); + const fullViewName = snowflakeHelper.getFullEntityName(databaseName, schemaName, view); logger.progress({ message: `Start getting data from view`, - containerName: schema, + containerName: fullSchemaName, entityName: view, }); logger.log( 'info', - { message: `Start getting data from view`, containerName: schema, entityName: view }, + { message: `Start getting data from view`, containerName: fullSchemaName, entityName: view }, 'Getting schema', ); @@ -186,12 +200,12 @@ const getDbCollectionsData = async (data, logger, cb) => { logger.progress({ message: `Data retrieved successfully`, - containerName: schema, + containerName: fullSchemaName, entityName: view, }); logger.log( 'info', - { message: `Data retrieved successfully`, containerName: schema, entityName: view }, + { message: `Data retrieved successfully`, containerName: fullSchemaName, entityName: view }, 'Getting schema', ); @@ -218,7 +232,7 @@ const getDbCollectionsData = async (data, logger, cb) => { emptyBucket: false, bucketInfo: { indexes: [], - database, + database: databaseName, ...containerData, }, }); diff --git a/reverse_engineering/helpers/snowflakeHelper.js b/reverse_engineering/helpers/snowflakeHelper.js index 2ef46c3f..9723ae56 100644 --- a/reverse_engineering/helpers/snowflakeHelper.js +++ b/reverse_engineering/helpers/snowflakeHelper.js @@ -40,6 +40,7 @@ const connect = async ( name, cloudPlatform, queryRequestTimeout, + databaseName, }, ) => { const account = getAccount(host); @@ -54,7 +55,8 @@ const connect = async ( `Auth type: ${authType}\n` + `Username: ${username}\n` + `Warehouse: ${warehouse}\n` + - `Role: ${role}`, + `Role: ${role}\n`, + `Schema name: ${databaseName}`, 'Connection', ); @@ -483,15 +485,20 @@ const showTablesByDatabases = async databases => databases.map(database => execute(`SHOW TABLES IN DATABASE "${removeQuotes(database.name)}";`)), ); +const showSchemasByDatabase = async databaseName => + databaseName ? showSchemasInDatabase(databaseName) : showSchemas(); + const showDatabases = () => execute('SHOW DATABASES;'); const showSchemas = () => execute('SHOW SCHEMAS;'); -const showExternalTables = () => execute('SHOW EXTERNAL TABLES;'); +const showSchemasInDatabase = databaseName => execute(`SHOW SCHEMAS IN DATABASE "${removeQuotes(databaseName)}";`); + +const showExternalTables = ({ options = '' } = {}) => execute(`SHOW EXTERNAL TABLES${options};`); -const showViews = () => execute('SHOW VIEWS;'); +const showViews = ({ options = '' } = {}) => execute(`SHOW VIEWS${options};`); -const showMaterializedViews = () => execute('SHOW MATERIALIZED VIEWS;'); +const showMaterializedViews = ({ options = '' } = {}) => execute(`SHOW MATERIALIZED VIEWS${options};`); const showIcebergTables = ({ options = '' } = {}) => execute(`SHOW ICEBERG TABLES${options};`); @@ -507,8 +514,8 @@ const splitEntityNames = names => { const isView = name => name.slice(-4) === ' (v)'; -const getSchemasInfo = async () => { - const schemas = await showSchemas().catch(err => [{ status: 'error', message: err.message }]); +const getSchemasInfo = async databaseName => { + const schemas = await showSchemasByDatabase(databaseName).catch(err => [{ status: 'error', message: err.message }]); if (schemas[0]?.status === 'error') { return schemas; @@ -592,19 +599,19 @@ const logTablesMeta = async ({ logger, tables = [], icebergTables = [] }) => { logger.log('info', combinedMeta, 'Tables metadata'); }; -const getEntitiesNames = async ({ logger }) => { +const getEntitiesNames = async ({ databaseName, logger }) => { const logError = logErrorAndReturnEmptyArray({ logger, query: 'SHOW' }); - - const databases = await showDatabases().catch(logError); + const databaseQueryOptions = databaseName ? ` IN DATABASE "${removeQuotes(databaseName)}"` : ''; + const databases = databaseName ? [{ name: databaseName }] : await showDatabases().catch(logError); const tablesRows = await showTablesByDatabases(databases).catch(logError); const flatTableRows = tablesRows.flatMap(row => row.value).filter(Boolean); - const icebergTables = await showIcebergTables().catch(logError); + const icebergTables = await showIcebergTables({ options: databaseQueryOptions }).catch(logError); await logTablesMeta({ logger, tables: flatTableRows, icebergTables }); - const externalTableRows = await showExternalTables().catch(logError); - const viewsRows = await showViews().catch(logError); - const materializedViewsRows = await showMaterializedViews().catch(logError); + const externalTableRows = await showExternalTables({ options: databaseQueryOptions }).catch(logError); + const viewsRows = await showViews({ options: databaseQueryOptions }).catch(logError); + const materializedViewsRows = await showMaterializedViews({ options: databaseQueryOptions }).catch(logError); const entitiesRows = [ ...flatTableRows, @@ -626,7 +633,7 @@ const getEntitiesNames = async ({ logger }) => { return [ ...buckets, { - dbName: `${dbName}.${schema}`, + dbName: schema, dbCollections: entities, isEmpty: !entities.length, }, @@ -645,8 +652,8 @@ const getDatabaseNames = async ({ logger }) => { return databases.map(({ name }) => name); }; -const getFullEntityName = (schemaName, tableName) => { - return [...schemaName.split('.'), tableName].map(addQuotes).join('.'); +const getFullEntityName = (databaseName, schemaName, tableName) => { + return [databaseName, schemaName, tableName].map(addQuotes).join('.'); }; const addQuotes = string => { @@ -1511,12 +1518,12 @@ const getFileFormats = async (dbName, schemaName) => { ); }; -const getContainerData = async ({ schema, logger }) => { - if (containers[schema]) { - return containers[schema]; +const getContainerData = async ({ databaseName, schemaName, logger }) => { + if (containers[schemaName]) { + return containers[schemaName]; } - const [dbName, schemaName] = schema.split('.'); - const dbNameWithoutQuotes = removeQuotes(dbName); + + const dbNameWithoutQuotes = removeQuotes(databaseName); try { const dbRows = await execute( @@ -1528,13 +1535,13 @@ const getContainerData = async ({ schema, logger }) => { ); const isCaseSensitive = _.toUpper(schemaName) !== schemaName; const schemaData = _.first(schemaRows); - const functions = await getFunctions(dbName, schemaName); - const procedures = await getProcedures(dbName, schemaName); - const stages = await getStages(dbName, schemaName); - const sequences = await getSequences(dbName, schemaName); - const fileFormats = await getFileFormats(dbName, schemaName); - const tags = await getTags({ dbName, schemaName, logger }); - const schemaTags = await getSchemaTags({ dbName, schemaName, logger }); + const functions = await getFunctions(databaseName, schemaName); + const procedures = await getProcedures(databaseName, schemaName); + const stages = await getStages(databaseName, schemaName); + const sequences = await getSequences(databaseName, schemaName); + const fileFormats = await getFileFormats(databaseName, schemaName); + const tags = await getTags({ databaseName, schemaName, logger }); + const schemaTags = await getSchemaTags({ databaseName, schemaName, logger }); const data = { transient: Boolean(_.get(schemaData, 'IS_TRANSIENT', false) && _.get(schemaData, 'IS_TRANSIENT') !== 'NO'), @@ -1549,7 +1556,7 @@ const getContainerData = async ({ schema, logger }) => { tags, schemaTags, }; - containers[schema] = data; + containers[schemaName] = data; return data; } catch (error) { @@ -1573,9 +1580,11 @@ const getTagAllowedValues = ({ values, logger }) => { } }; -const getTags = async ({ dbName, schemaName, logger }) => { +const getTags = async ({ databaseName, schemaName, logger }) => { try { - const rows = await execute(`SHOW TAGS IN SCHEMA "${removeQuotes(dbName)}"."${removeQuotes(schemaName)}";`); + const rows = await execute( + `SHOW TAGS IN SCHEMA "${removeQuotes(databaseName)}"."${removeQuotes(schemaName)}";`, + ); return rows.map(row => ({ name: row.name, @@ -1589,10 +1598,10 @@ const getTags = async ({ dbName, schemaName, logger }) => { } }; -const getSchemaTags = async ({ dbName, schemaName, logger }) => { +const getSchemaTags = async ({ databaseName, schemaName, logger }) => { try { const rows = await execute( - `SELECT TAG_DATABASE, TAG_SCHEMA, TAG_NAME, TAG_VALUE FROM TABLE("${removeQuotes(dbName)}".information_schema.tag_references('"${removeQuotes(dbName)}"."${removeQuotes(schemaName)}"', 'SCHEMA'));`, + `SELECT TAG_DATABASE, TAG_SCHEMA, TAG_NAME, TAG_VALUE FROM TABLE("${removeQuotes(databaseName)}".information_schema.tag_references('"${removeQuotes(databaseName)}"."${removeQuotes(schemaName)}"', 'SCHEMA'));`, ); return rows.map(row => ({