diff --git a/.changeset/fair-cobras-flow.md b/.changeset/fair-cobras-flow.md new file mode 100644 index 000000000..23fd4f41e --- /dev/null +++ b/.changeset/fair-cobras-flow.md @@ -0,0 +1,8 @@ +--- +'@flatfile/plugin-automap': minor +--- + +Add onSuccess callback. +Add new options to only map if there is no unmatched columns (none, both, only-source, only-target) +Add options to change file name in success/failure case +Add options to overwrite file name patterns diff --git a/package-lock.json b/package-lock.json index 5d3712fdf..cdc1aea0e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17975,7 +17975,7 @@ }, "plugins/webhook-egress": { "name": "@flatfile/plugin-webhook-egress", - "version": "1.6.0", + "version": "1.6.1", "license": "ISC", "dependencies": { "@flatfile/plugin-job-handler": "^0.8.0", diff --git a/plugins/automap/src/automap.plugin.ts b/plugins/automap/src/automap.plugin.ts index 023c0c83a..f13e2359e 100644 --- a/plugins/automap/src/automap.plugin.ts +++ b/plugins/automap/src/automap.plugin.ts @@ -7,7 +7,8 @@ import { AutomapService } from './automap.service' * @param options - config options */ export function automap(options: AutomapOptions) { - const automapper = new AutomapService(options) + const optionsDefaulted = defaultOptions(options) + const automapper = new AutomapService(optionsDefaulted) return (listener: FlatfileListener): void => { automapper.assignListeners(listener) @@ -21,8 +22,17 @@ export function automap(options: AutomapOptions) { * @property {boolean} debug - show helpul messages useful for debugging (use intended for development). * @property {string} defaultTargetSheet - exact sheet name to import data to. * @property {RegExp} matchFilename - a regular expression to match specific files to perform automapping on. - * @property {Function} onFailure - callback to be executed when plugin bails. + * @property {string} allColumnsMustBeMapped - specify if all columns must be mapped. Values: 'none', 'both', 'only-source', 'only-target'. + * @property {Function} onSuccess - callback to be executed when plugin succeeds. + * @property {Function} onFailure - callback to be executed when plugin fails. * @property {string} targetWorkbook - specify destination Workbook id or name. + * @property {boolean} disableFileNameUpdate - disable filename update on automap. + * @property {boolean} disableFileNameUpdateOnSuccess - disable filename update on success. + * @property {boolean} disableFileNameUpdateOnFailure - disable filename update on failure. + * @property {string} filenameOnCheck - filename on check mapping. Can use {{fileName}}. + * @property {string} filenameOnStart - filename on start mapping. Can use {{fileName}}, {{destinationSheetName}}. + * @property {string} filenameOnSuccess - filename on success mapping. Can use {{fileName}}, {{destinationSheetName}}. + * @property {string} filenameOnFailure - filename on failure mapping. Can use {{fileName}}. */ export interface AutomapOptions { readonly accuracy: 'confident' | 'exact' @@ -31,7 +41,58 @@ export interface AutomapOptions { | string | ((fileName?: string, event?: FlatfileEvent) => string | Promise) readonly matchFilename?: RegExp + readonly allColumnsMustBeMapped?: + | 'none' + | 'both' + | 'only-source' + | 'only-target' + readonly onSuccess?: (event: FlatfileEvent) => void readonly onFailure?: (event: FlatfileEvent) => void readonly targetWorkbook?: string readonly disableFileNameUpdate?: boolean + readonly disableFileNameUpdateOnSuccess?: boolean + readonly disableFileNameUpdateOnFailure?: boolean + readonly filenameOnCheck?: string + readonly filenameOnStart?: string + readonly filenameOnSuccess?: string + readonly filenameOnFailure?: string +} + +export function defaultOptions(options: AutomapOptions): AutomapOptions { + const defaultedOptions = { + ...options, + accuracy: options.accuracy || 'confident', + allColumnsMustBeMapped: options.allColumnsMustBeMapped || 'none', + filenameOnCheck: options.filenameOnCheck || '⚡️ {{fileName}}', + filenameOnStart: + options.filenameOnStart || '⚡️ {{fileName}} 🔁 {{destinationSheetName}}', + filenameOnSuccess: + options.filenameOnSuccess || + '⚡️ {{fileName}} 🔁 {{destinationSheetName}}', + filenameOnFailure: + options.filenameOnFailure || + '⚡️ {{fileName}} 🔁 {{destinationSheetName}}', + } + + if (!defaultedOptions.filenameOnCheck.includes('{{fileName}}')) { + defaultedOptions.filenameOnCheck = + defaultedOptions.filenameOnCheck + ' {{fileName}}' + } + + if (!defaultedOptions.filenameOnStart.includes('{{fileName}}')) { + defaultedOptions.filenameOnStart = + defaultedOptions.filenameOnStart + ' {{fileName}}' + } + + if (!defaultedOptions.filenameOnSuccess.includes('{{fileName}}')) { + defaultedOptions.filenameOnSuccess = + defaultedOptions.filenameOnSuccess + ' {{fileName}}' + } + + if (!defaultedOptions.filenameOnFailure.includes('{{fileName}}')) { + defaultedOptions.filenameOnFailure = + defaultedOptions.filenameOnFailure + ' {{fileName}}' + } + + return defaultedOptions } diff --git a/plugins/automap/src/automap.service.ts b/plugins/automap/src/automap.service.ts index 9a54df704..3610fd0ab 100644 --- a/plugins/automap/src/automap.service.ts +++ b/plugins/automap/src/automap.service.ts @@ -43,8 +43,8 @@ export class AutomapService { if (!this.isFileNameMatch(file)) { return - } else if (!this.options.disableFileNameUpdate) { - await this.updateFileName(file.id, `⚡️ ${file.name}`) + } else { + await this.updateFileName('check', file.id) } if (this.isNil(file.workbookId)) { @@ -129,11 +129,8 @@ export class AutomapService { const actualJobs = jobs.filter((j) => !this.isNil(j)) - if (actualJobs.length > 0 && !this.options.disableFileNameUpdate) { - await this.updateFileName( - file.id, - `⚡️ ${file.name} 🔁 ${destinationSheet?.name}` - ) + if (actualJobs.length > 0) { + await this.updateFileName('start', file.id, destinationSheet?.name) } } catch (_mappingsError: unknown) { logError('@flatfile/plugin-automap', 'Unable to fetch mappings') @@ -196,6 +193,11 @@ export class AutomapService { const { jobId } = event.context const job = await api.jobs.get(jobId) + const workbook = await api.workbooks.get(job.data.destination) + const sheet = workbook.data.sheets.find( + (s) => s.slug === this.options.defaultTargetSheet + ) + const sheetName = sheet?.name if (!job.data.input?.isAutomap) { if (this.options.debug) { @@ -215,6 +217,11 @@ export class AutomapService { return } + //Get file + const sourceWorkbook = await api.workbooks.get(job.data.source) + const files = await api.files.list({ spaceId: sourceWorkbook.data.spaceId }) + const file = files.data.find((f) => f.workbookId === sourceWorkbook.data.id) + try { const { data: { plan }, @@ -244,39 +251,70 @@ export class AutomapService { } try { - switch (this.options.accuracy) { - case 'confident': - if (this.verifyConfidentMatchingStrategy(plan)) { - await api.jobs.execute(jobId) - } else { - if (this.options.debug) { - logWarn( - '@flatfile/plugin-automap', - 'Skipping automap due to lack of confidence' - ) + if (this.verifyMappedColumns(plan)) { + switch (this.options.accuracy) { + case 'confident': + if (this.verifyConfidentMatchingStrategy(plan)) { + await api.jobs.execute(jobId) + + await this.updateFileName('success', file.id, sheetName) + + if (!this.isNil(this.options.onSuccess)) { + this.options.onSuccess(event) + } + } else { + if (this.options.debug) { + logWarn( + '@flatfile/plugin-automap', + 'Skipping automap due to lack of confidence' + ) + } + + await this.updateFileName('failure', file.id, sheetName) + + if (!this.isNil(this.options.onFailure)) { + this.options.onFailure(event) + } } - - if (!this.isNil(this.options.onFailure)) { - this.options.onFailure(event) - } - } - break - case 'exact': - if (this.verifyAbsoluteMatchingStrategy(plan)) { - await api.jobs.execute(jobId) - } else { - if (this.options.debug) { - logWarn( - '@flatfile/plugin-automap', - 'Skipping automap due to lack of confidence' - ) + break + case 'exact': + if (this.verifyAbsoluteMatchingStrategy(plan)) { + await api.jobs.execute(jobId) + + await this.updateFileName('success', file.id, sheetName) + + if (!this.isNil(this.options.onSuccess)) { + this.options.onSuccess(event) + } + } else { + if (this.options.debug) { + logWarn( + '@flatfile/plugin-automap', + 'Skipping automap due to lack of confidence' + ) + } + + await this.updateFileName('failure', file.id, sheetName) + + if (!this.isNil(this.options.onFailure)) { + this.options.onFailure(event) + } } + break + } + } else { + if (this.options.debug) { + logWarn( + '@flatfile/plugin-automap', + 'Skipping automap due to lack of confidence' + ) + } - if (!this.isNil(this.options.onFailure)) { - this.options.onFailure(event) - } - } - break + await this.updateFileName('failure', file.id, sheetName) + + if (!this.isNil(this.options.onFailure)) { + this.options.onFailure(event) + } } } catch (_jobError: unknown) { logError( @@ -357,6 +395,35 @@ export class AutomapService { } } + private verifyMappedColumns(plan: Flatfile.JobExecutionPlan): boolean { + let mappedColumnsVerified = false + + if (this.options.allColumnsMustBeMapped === 'none') { + mappedColumnsVerified = true + } + if ( + this.options.allColumnsMustBeMapped === 'both' && + plan.unmappedDestinationFields?.length === 0 && + plan.unmappedSourceFields?.length === 0 + ) { + mappedColumnsVerified = true + } + if ( + this.options.allColumnsMustBeMapped === 'only-source' && + plan.unmappedSourceFields?.length === 0 + ) { + mappedColumnsVerified = true + } + if ( + this.options.allColumnsMustBeMapped === 'only-target' && + plan.unmappedDestinationFields?.length === 0 + ) { + mappedColumnsVerified = true + } + + return mappedColumnsVerified + } + private verifyAbsoluteMatchingStrategy( plan: Flatfile.JobExecutionPlan ): boolean { @@ -379,14 +446,136 @@ export class AutomapService { ) } - private updateFileName( + private async updateFileName( + stage: 'manual' | 'check' | 'start' | 'success' | 'failure', fileId: string, - fileName: string - ): Promise { - return api.files.update(fileId, { name: fileName }) + destinationSheetName?: string + ): Promise { + const file = await api.files.get(fileId) + const currentFileName = file.data.name + let newFileName = '' + if (this.options.disableFileNameUpdate) { + return + } + + if (stage === 'check') { + newFileName = this.resolveVariablesInFileName( + this.options.filenameOnCheck, + { fileName: currentFileName } + ) + return api.files.update(fileId, { name: newFileName }) + } + + if (stage === 'start') { + const oldFileName = await this.getFileNameFromOldFileName( + this.options.filenameOnCheck, + currentFileName + ) + newFileName = this.resolveVariablesInFileName( + this.options.filenameOnStart, + { fileName: oldFileName, destinationSheetName: destinationSheetName } + ) + return api.files.update(fileId, { name: newFileName }) + } + + if (stage === 'success' && !this.options.disableFileNameUpdateOnSuccess) { + const oldFileName = await this.getFileNameFromOldFileName( + this.options.filenameOnStart, + currentFileName + ) + newFileName = this.resolveVariablesInFileName( + this.options.filenameOnSuccess, + { fileName: oldFileName, destinationSheetName: destinationSheetName } + ) + return api.files.update(fileId, { name: newFileName }) + } + + if (stage === 'failure' && !this.options.disableFileNameUpdateOnFailure) { + const oldFileName = await this.getFileNameFromOldFileName( + this.options.filenameOnStart, + currentFileName + ) + newFileName = this.resolveVariablesInFileName( + this.options.filenameOnFailure, + { fileName: oldFileName, destinationSheetName: destinationSheetName } + ) + return api.files.update(fileId, { name: newFileName }) + } + + return } private isNil(value: any): value is null | undefined { return value === null || value === undefined } + + private resolveVariablesInFileName( + fileName: string, + variables: { destinationSheetName?: string; fileName?: string } + ): string { + if (variables.destinationSheetName) { + fileName = fileName.replace( + '{{destinationSheetName}}', + variables.destinationSheetName + ) + } + if (variables.fileName) { + fileName = fileName.replace('{{fileName}}', variables.fileName) + } + return fileName + } + + private async getFileNameFromOldFileName( + pattern: string, + filename: string + ): Promise { + try { + // Find the position of {{fileName}} in the pattern + const fileNameMatch = pattern.match(/{{fileName}}/) + if (!fileNameMatch) { + return filename + } + + // Get the text before and after {{fileName}} + const beforePattern = pattern.substring(0, fileNameMatch.index) + const afterPattern = pattern.substring( + fileNameMatch.index + '{{fileName}}'.length + ) + + // Escape special characters in the before/after patterns + const beforeRegex = this.escapeRegExp(beforePattern) + const afterRegex = this.escapeRegExp(afterPattern).replace( + this.escapeRegExp('{{destinationSheetName}}'), + '.*?' + ) + + // Create a safe regex pattern that matches the exact structure + const safeRegex = new RegExp(`^${beforeRegex}(.*?)${afterRegex}$`) + + // Add timeout protection + const MAX_EXECUTION_TIME = 100 + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Regex timeout')), MAX_EXECUTION_TIME) + }) + + const matchPromise = new Promise((resolve) => { + const match = filename.match(safeRegex) + if (match?.[1]) { + resolve(match[1].trim()) + } else { + resolve(filename) + } + }) + + return Promise.race([matchPromise, timeoutPromise]) + .then((result) => result) + .catch(() => filename) + } catch (error) { + return filename + } + } + + private escapeRegExp(string: string): string { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + } }