diff --git a/README.md b/README.md index 70c8731..0794767 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,14 @@ You can find all these credentials in a JSON file, which u can get from [here](h - To Deactivate this feature, simply set the value to `0`. 6. Target Department for Handover (optional) - Enter the department name where you want the visitor to be transferred upon handover. - 7. Handover Message (optional) + 7. Show only Text Messages + - If this setting is enabled, then the bot will always reply back with Text Message. However if this setting is disabled, then the bot will reply back with audio if the visitor has provided audio as input. + - Please note, currently the output audio will only work for Livechat widget. If you are using any other channels like WhatsApp or Messenger, then please keep this setting enabled. + 8. Handover Message (optional) - The Bot will send this message to Visitor upon handover - 8. Service Unavailable Message (optional) + 9. Service Unavailable Message (optional) - The Bot will send this message to Visitor if service is unavailable like suppose if no agents are online. - 9. Close Chat Message (optional) + 10. Close Chat Message (optional) - This message will be sent automatically when a chat is closed 10. Hide Quick Replies (required) - If enabled, then all quick-replies will hide when a visitor clicks on any one of them diff --git a/app.json b/app.json index 9203cd7..fb356db 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { "id": "21b7d3ba-031b-41d9-8ff2-fbbfa081ae90", - "version": "1.2.0", - "requiredApiVersion": "^1.17.0", + "version": "1.3.0", + "requiredApiVersion": "^1.18.0", "iconFile": "icon.png", "author": { "name": "Rocket.Chat", diff --git a/config/Settings.ts b/config/Settings.ts index f52cb84..0ae76fd 100644 --- a/config/Settings.ts +++ b/config/Settings.ts @@ -6,6 +6,7 @@ export enum AppSetting { DialogflowClientEmail = 'dialogflow_client_email', DialogFlowPrivateKey = 'dialogflow_private_key', DialogflowFallbackResponsesLimit = 'dialogflow_fallback_responses_limit', + DialogflowShowOnlyTextMessages = 'show_only_text_messages', FallbackTargetDepartment = 'fallback_target_department', DialogflowHandoverMessage = 'dialogflow_handover_message', DialogflowServiceUnavailableMessage = 'dialogflow_service_unavailable_message', @@ -17,6 +18,7 @@ export enum DefaultMessage { DEFAULT_DialogflowServiceUnavailableMessage = 'Sorry, I\'m having trouble answering your question.', DEFAULT_DialogflowHandoverMessage = 'Transferring to an online agent', DEFAULT_DialogflowCloseChatMessage = 'Closing the chat, Goodbye', + DEFAULT_UnsupportedAudioFormatMessage = 'Sorry! Only *.wav, *.opus and *.oga extension files are supported as audio input', } export const settings: Array = [ @@ -71,6 +73,16 @@ export const settings: Array = [ i18nDescription: 'target_department_for_handover_description', required: false, }, + { + id: AppSetting.DialogflowShowOnlyTextMessages, + public: true, + type: SettingType.BOOLEAN, + packageValue: true, + value: true, + i18nLabel: 'show_only_text_messages', + i18nDescription: 'show_only_text_messages_description', + required: true, + }, { id: AppSetting.DialogflowHandoverMessage, public: true, diff --git a/endpoints/IncomingEndpoint.ts b/endpoints/IncomingEndpoint.ts index 0f93180..b3cb5e5 100644 --- a/endpoints/IncomingEndpoint.ts +++ b/endpoints/IncomingEndpoint.ts @@ -1,7 +1,7 @@ import { HttpStatusCode, IHttp, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { ApiEndpoint, IApiEndpointInfo, IApiRequest, IApiResponse } from '@rocket.chat/apps-engine/definition/api'; import { ILivechatRoom } from '@rocket.chat/apps-engine/definition/livechat'; -import { IDialogflowMessage, DialogflowRequestType } from '../enum/Dialogflow'; +import { DialogflowRequestType, IDialogflowMessage } from '../enum/Dialogflow'; import { EndpointActionNames, IActionsEndpointContent } from '../enum/Endpoints'; import { Headers, Response } from '../enum/Http'; import { Logs } from '../enum/Logs'; diff --git a/enum/Dialogflow.ts b/enum/Dialogflow.ts index 0d7b550..ec43fa6 100644 --- a/enum/Dialogflow.ts +++ b/enum/Dialogflow.ts @@ -4,6 +4,7 @@ export interface IDialogflowMessage { messages?: Array; isFallback: boolean; sessionId?: string; + audio?: string; } export interface IDialogflowQuickReplies { @@ -47,13 +48,40 @@ export enum DialogflowJWT { export enum Base64 { BASE64_DICTIONARY = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', BASE64_PAD = '=', + BASE64 = 'base64', } export enum LanguageCode { EN = 'en', } +export enum AudioLanguageCode { + EN_US = 'en-US', +} + export enum DialogflowRequestType { MESSAGE = 'message', EVENT = 'event', + AUDIO = 'audio', + AUDIO_OGG = 'audio-ogg', +} + +export enum DialogflowOutputAudioEncoding { + LINEAR_16 = 'OUTPUT_AUDIO_ENCODING_LINEAR_16', +} + +export enum DialogflowInputAudioEncoding { + ENCODING_OGG = 'AUDIO_ENCODING_OGG_OPUS', +} + +export enum MIME_TYPE { + AUDIO_OGG = 'audio/ogg', + AUDIO_PREFIX = 'audio', +} + +// supported audio extensions +export enum AUDIO_EXTENSION { + OGA = 'oga', + WAV = 'wav', + OPUS = 'opus', } diff --git a/enum/Logs.ts b/enum/Logs.ts index 9a0212b..224f089 100644 --- a/enum/Logs.ts +++ b/enum/Logs.ts @@ -20,4 +20,6 @@ export enum Logs { HTTP_REQUEST_ERROR = 'Error occurred while sending a HTTP Request', CLOSE_CHAT_REQUEST_FAILED_ERROR = 'Error: Internal Server Error. Could not close the chat', HANDOVER_REQUEST_FAILED_ERROR = 'Error occurred while processing handover. Details', + ERROR_NULL_RESPONSE = 'Error! Null Response', + INVALID_AUDIO_FILE_NAME = 'Invalid audio file name', } diff --git a/handler/PostMessageSentHandler.ts b/handler/PostMessageSentHandler.ts index 3c3325e..a3aab62 100644 --- a/handler/PostMessageSentHandler.ts +++ b/handler/PostMessageSentHandler.ts @@ -3,9 +3,10 @@ import { IApp } from '@rocket.chat/apps-engine/definition/IApp'; import { ILivechatMessage, ILivechatRoom } from '@rocket.chat/apps-engine/definition/livechat'; import { RoomType } from '@rocket.chat/apps-engine/definition/rooms'; import { AppSetting, DefaultMessage } from '../config/Settings'; -import { DialogflowRequestType, IDialogflowMessage } from '../enum/Dialogflow'; +import { DialogflowRequestType, IDialogflowMessage, MIME_TYPE } from '../enum/Dialogflow'; import { Logs } from '../enum/Logs'; import { Dialogflow } from '../lib/Dialogflow'; +import { defineAudioFile } from '../lib/Helper'; import { createDialogflowMessage, createMessage } from '../lib/Message'; import { getAppSettingValue } from '../lib/Settings'; import { incFallbackIntent, resetFallbackIntent } from '../lib/SynchronousHandover'; @@ -19,7 +20,8 @@ export class PostMessageSentHandler { private readonly modify: IModify) {} public async run() { - const { text, editedAt, room, token, sender } = this.message; + const { text, editedAt, room, token, sender, file } = this.message; + const livechatRoom = room as ILivechatRoom; const { id: rid, type, servedBy, isOpen } = livechatRoom; @@ -30,7 +32,7 @@ export class PostMessageSentHandler { return; } - if (!isOpen || !token || editedAt || !text) { + if (!isOpen || !token || editedAt) { return; } @@ -42,33 +44,43 @@ export class PostMessageSentHandler { return; } - if (!text || (text && text.trim().length === 0)) { - return; - } - - let response: IDialogflowMessage; try { - response = (await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, text, DialogflowRequestType.MESSAGE)); + let response: IDialogflowMessage | undefined = undefined; + + if (text && text.trim().length !== 0) { + response = await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, text, DialogflowRequestType.MESSAGE); + } + + if (file && file.type.startsWith(MIME_TYPE.AUDIO_PREFIX)) { + const { content, contentType } = await defineAudioFile(this.read, this.modify, rid, file); + if (content && contentType) { + response = await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, content, contentType); + } + } + + if (!response) { + return; + } + + await createDialogflowMessage(rid, this.read, this.modify, response); + + // synchronous handover check + const { isFallback } = response; + if (isFallback) { + return incFallbackIntent(this.read, this.modify, rid); + } + return resetFallbackIntent(this.read, this.modify, rid); + } catch (error) { this.app.getLogger().error(`${Logs.DIALOGFLOW_REST_API_ERROR} ${error.message}`); const serviceUnavailable: string = await getAppSettingValue(this.read, AppSetting.DialogflowServiceUnavailableMessage); - await createMessage(rid, + return createMessage(rid, this.read, this.modify, { text: serviceUnavailable ? serviceUnavailable : DefaultMessage.DEFAULT_DialogflowServiceUnavailableMessage }); - return; - } - - await createDialogflowMessage(rid, this.read, this.modify, response); - - // synchronous handover check - const { isFallback } = response; - if (isFallback) { - return incFallbackIntent(this.read, this.modify, rid); } - return resetFallbackIntent(this.read, this.modify, rid); } } diff --git a/i18n/en.json b/i18n/en.json index fa36e66..fad1ee5 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -13,6 +13,8 @@ "dialogflow_service_unavailable_message_description": "The Bot will send this message to Visitor if service is unavailable", "dialogflow_close_chat_message": "Close Chat Message", "dialogflow_close_chat_message_description": "This message will be sent automatically when a chat is closed", + "show_only_text_messages": "Show only Text Messages", + "show_only_text_messages_description": "If this Setting is Enabled, then the bot will always reply back with a text message, even when a visitor sends a audio message.", "dialogflow_hide_quick_replies": "Hide Quick Replies", "dialogflow_hide_quick_replies_description": "If enabled, then all quick-replies will hide when a visitor clicks on any one of them" } diff --git a/lib/Dialogflow.ts b/lib/Dialogflow.ts index c26bf97..b6ff97c 100644 --- a/lib/Dialogflow.ts +++ b/lib/Dialogflow.ts @@ -1,8 +1,8 @@ -import { IHttp, IHttpRequest, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors'; +import { IHttp, IHttpRequest, IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { IRoom } from '@rocket.chat/apps-engine/definition/rooms'; import { createSign } from 'crypto'; import { AppSetting } from '../config/Settings'; -import { DialogflowJWT, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; +import { AudioLanguageCode, DialogflowInputAudioEncoding, DialogflowJWT, DialogflowOutputAudioEncoding, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; import { Headers } from '../enum/Http'; import { Logs } from '../enum/Logs'; import { base64urlEncode } from './Helper'; @@ -23,18 +23,30 @@ class DialogflowClass { const queryInput = { ...requestType === DialogflowRequestType.EVENT && { event: request }, ...requestType === DialogflowRequestType.MESSAGE && { text: { languageCode: LanguageCode.EN, text: request } }, + ...requestType === DialogflowRequestType.AUDIO && { audioConfig: { languageCode: AudioLanguageCode.EN_US } }, + ...requestType === DialogflowRequestType.AUDIO_OGG && + { audioConfig: { audioEncoding: DialogflowInputAudioEncoding.ENCODING_OGG, sampleRateHertz: 16000, languageCode: AudioLanguageCode.EN_US } }, }; + const { value: onlyTextMessage } = await read.getEnvironmentReader().getSettings().getById(AppSetting.DialogflowShowOnlyTextMessages); + const httpRequestContent: IHttpRequest = createHttpRequest( { 'Content-Type': Headers.CONTENT_TYPE_JSON, 'Accept': Headers.ACCEPT_JSON }, - { queryInput }, + { + queryInput, + ...(requestType === DialogflowRequestType.AUDIO || requestType === DialogflowRequestType.AUDIO_OGG) && + { + inputAudio: request, + ...!onlyTextMessage && { outputAudioConfig: { audioEncoding: DialogflowOutputAudioEncoding.LINEAR_16 } }, + }, + }, ); try { const response = await http.post(serverURL, httpRequestContent); return this.parseRequest(response.data); } catch (error) { - throw new Error(`${ Logs.HTTP_REQUEST_ERROR }`); + throw new Error(`${ Logs.HTTP_REQUEST_ERROR }. Details:- ${ error }`); } } @@ -81,7 +93,14 @@ class DialogflowClass { public parseRequest(response: any): IDialogflowMessage { if (!response) { throw new Error(Logs.INVALID_RESPONSE_FROM_DIALOGFLOW_CONTENT_UNDEFINED); } - const { session, queryResult } = response; + const { session, queryResult, outputAudio } = response; + if (outputAudio) { + const { intent: { isFallback } } = queryResult; + return { + audio: outputAudio, + isFallback: isFallback ? isFallback : false, + }; + } if (queryResult) { const { fulfillmentMessages, intent: { isFallback } } = queryResult; const parsedMessage: IDialogflowMessage = { diff --git a/lib/Helper.ts b/lib/Helper.ts index 1d19663..e9723de 100644 --- a/lib/Helper.ts +++ b/lib/Helper.ts @@ -1,4 +1,9 @@ -import { Base64 } from '../enum/Dialogflow'; +import { IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors'; +import { IMessageFile } from '@rocket.chat/apps-engine/definition/messages'; +import { DefaultMessage } from '../config/Settings'; +import { AUDIO_EXTENSION, Base64, DialogflowRequestType, MIME_TYPE } from '../enum/Dialogflow'; +import { Logs } from '../enum/Logs'; +import { createMessage } from './Message'; export const base64urlEncode = (str: any) => { const utf8str = unescape(encodeURIComponent(str)); @@ -35,10 +40,30 @@ export const base64EncodeData = (data: string, len: number, b64x: string, b64pad return dst; }; -export const uuid = (): string => { - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { - const r = Math.random() * 16 | 0; - const v = c === 'x' ? r : (r & 0x3 | 0x8); - return v.toString(16); - }); +export const defineAudioFile = async (read: IRead, modify: IModify, roomId: string, file: IMessageFile): Promise<{ content: string, contentType: DialogflowRequestType}> => { + const { name } = await read.getUploadReader().getById(file._id); + + if (!isSupportedAudioFormat(name)) { + await createMessage(roomId, read, modify, { text: DefaultMessage.DEFAULT_UnsupportedAudioFormatMessage }); + } + + const content = (await read.getUploadReader().getBufferById(file._id)).toString(Base64.BASE64); + const contentType = file && file.type === MIME_TYPE.AUDIO_OGG ? DialogflowRequestType.AUDIO_OGG : DialogflowRequestType.AUDIO; + + return { content, contentType }; +}; + +export const isSupportedAudioFormat = (fileName: string): boolean => { + const extension = fileName.split('.')[1]; + if (!extension) { + throw new Error(Logs.INVALID_AUDIO_FILE_NAME); + } + + if (extension === AUDIO_EXTENSION.OGA || + extension === AUDIO_EXTENSION.WAV || + extension === AUDIO_EXTENSION.OPUS) { + return true; + } + + return false; }; diff --git a/lib/Message.ts b/lib/Message.ts index b2f63aa..15a906f 100644 --- a/lib/Message.ts +++ b/lib/Message.ts @@ -1,15 +1,38 @@ import { IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { IVisitor } from '@rocket.chat/apps-engine/definition/livechat'; import { BlockElementType, BlockType, IActionsBlock, IButtonElement, TextObjectType } from '@rocket.chat/apps-engine/definition/uikit'; +import { IUploadDescriptor } from '@rocket.chat/apps-engine/definition/uploads/IUploadDescriptor'; import { IUser } from '@rocket.chat/apps-engine/definition/users'; +import { Buffer } from 'buffer'; import { AppSetting } from '../config/Settings'; -import { IDialogflowMessage, IDialogflowQuickReplies, IDialogflowQuickReplyOptions } from '../enum/Dialogflow'; +import { DialogflowJWT, IDialogflowMessage, IDialogflowQuickReplies, IDialogflowQuickReplyOptions } from '../enum/Dialogflow'; import { Logs } from '../enum/Logs'; -import { uuid } from './Helper'; import { getAppSettingValue } from './Settings'; export const createDialogflowMessage = async (rid: string, read: IRead, modify: IModify, dialogflowMessage: IDialogflowMessage): Promise => { - const { messages = [] } = dialogflowMessage; + const { messages = [], audio } = dialogflowMessage; + + const room = await read.getRoomReader().getById(rid); + if (!room) { + throw new Error(`${Logs.INVALID_ROOM_ID} ${rid}`); + } + const botUserName = await getAppSettingValue(read, AppSetting.DialogflowBotUsername); + if (!botUserName) { + throw new Error(Logs.EMPTY_BOT_USERNAME_SETTING); + } + const sender = await read.getUserReader().getByUsername(botUserName); + if (!sender) { + throw new Error(Logs.INVALID_BOT_USERNAME_SETTING); + } + if (audio) { + const buffer = Buffer.from(audio, DialogflowJWT.BASE_64); + const uploadDescriptor: IUploadDescriptor = { + filename: 'audio.wav', + room, + user: sender, + }; + await modify.getCreator().getUploadCreator().uploadBuffer(buffer, uploadDescriptor); + } for (const message of messages) { const { text, options } = message as IDialogflowQuickReplies; @@ -126,3 +149,13 @@ export const deleteAllActionBlocks = async (modify: IModify, appUser: IUser, msg msgBuilder.setEditor(appUser).setBlocks(modify.getCreator().getBlockBuilder().getBlocks()); return modify.getUpdater().finish(msgBuilder); }; + +export const uuid = (): string => { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { + // tslint:disable-next-line: no-bitwise + const r = Math.random() * 16 | 0; + // tslint:disable-next-line: no-bitwise + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); +}; diff --git a/package-lock.json b/package-lock.json index fd7d474..9e111c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,9 +29,9 @@ } }, "@rocket.chat/apps-engine": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/@rocket.chat/apps-engine/-/apps-engine-1.17.0.tgz", - "integrity": "sha512-4jsN9RUQR9zvZnOr4IZj9bnZR5/kAuSpKJ3JwqTv/DXBeHRSrCCCX0EDoZh5akmQu6x9tRpAXd1DRJWRfI5/Jw==", + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/@rocket.chat/apps-engine/-/apps-engine-1.18.0.tgz", + "integrity": "sha512-Y9XgRnG8v4HtujaMXuzTd9hjycQX9+wDOE30HhHL1NQl7de4+9MzPdWDetdy+84cBrDtXCymVYF/TzmLf+69Mw==", "dev": true, "requires": { "adm-zip": "^0.4.9", diff --git a/package.json b/package.json index 5ee6ffb..f040522 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "devDependencies": { - "@rocket.chat/apps-engine": "^1.17.0", + "@rocket.chat/apps-engine": "^1.18.0", "@types/node": "^10.17.24", "tslint": "^5.10.0", "typescript": "^2.9.1"