[NEW] Add support for audio file as input and output

README.md

@@ -55,11 +55,14 @@ You can find all these credentials in a JSON file, which u can get from [here](h
         - To Deactivate this feature, simply set the value to `0`.
     6. Target Department for Handover (optional)
         - Enter the department name where you want the visitor to be transferred upon handover.
-    7. Handover Message (optional)
+    7. Show only Text Messages
+        - If this setting is enabled, then the bot will always reply back with Text Message. However if this setting is disabled, then the bot will reply back with audio if the visitor has provided audio as input.
+        - Please note, currently the output audio will only work for Livechat widget. If you are using any other channels like WhatsApp or Messenger, then please keep this setting enabled. 
+    8. Handover Message (optional)
         - The Bot will send this message to Visitor upon handover
-    8. Service Unavailable Message (optional)
+    9. Service Unavailable Message (optional)
         - The Bot will send this message to Visitor if service is unavailable like suppose if no agents are online.
-    9. Close Chat Message (optional)
+    10. Close Chat Message (optional)
         - This message will be sent automatically when a chat is closed
     10. Hide Quick Replies (required)
         - If enabled, then all quick-replies will hide when a visitor clicks on any one of them

app.json

@@ -1,7 +1,7 @@
 {
     "id": "21b7d3ba-031b-41d9-8ff2-fbbfa081ae90",
-    "version": "1.2.0",
-    "requiredApiVersion": "^1.17.0",
+    "version": "1.3.0",
+    "requiredApiVersion": "^1.18.0",
     "iconFile": "icon.png",
     "author": {
         "name": "Rocket.Chat",

config/Settings.ts

@@ -6,6 +6,7 @@ export enum AppSetting {
     DialogflowClientEmail = 'dialogflow_client_email',
     DialogFlowPrivateKey = 'dialogflow_private_key',
     DialogflowFallbackResponsesLimit = 'dialogflow_fallback_responses_limit',
+    DialogflowShowOnlyTextMessages = 'show_only_text_messages',
     FallbackTargetDepartment = 'fallback_target_department',
     DialogflowHandoverMessage = 'dialogflow_handover_message',
     DialogflowServiceUnavailableMessage = 'dialogflow_service_unavailable_message',
@@ -17,6 +18,7 @@ export enum DefaultMessage {
     DEFAULT_DialogflowServiceUnavailableMessage = 'Sorry, I\'m having trouble answering your question.',
     DEFAULT_DialogflowHandoverMessage = 'Transferring to an online agent',
     DEFAULT_DialogflowCloseChatMessage = 'Closing the chat, Goodbye',
+    DEFAULT_UnsupportedAudioFormatMessage = 'Sorry! Only *.wav, *.opus and *.oga extension files are supported as audio input',
 }
 
 export const settings: Array<ISetting> = [
@@ -71,6 +73,16 @@ export const settings: Array<ISetting> = [
         i18nDescription: 'target_department_for_handover_description',
         required: false,
     },
+    {
+        id: AppSetting.DialogflowShowOnlyTextMessages,
+        public: true,
+        type: SettingType.BOOLEAN,
+        packageValue: true,
+        value: true,
+        i18nLabel: 'show_only_text_messages',
+        i18nDescription: 'show_only_text_messages_description',
+        required: true,
+    },
     {
         id: AppSetting.DialogflowHandoverMessage,
         public: true,

endpoints/IncomingEndpoint.ts

@@ -1,7 +1,7 @@
 import { HttpStatusCode, IHttp, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors';
 import { ApiEndpoint, IApiEndpointInfo, IApiRequest, IApiResponse } from '@rocket.chat/apps-engine/definition/api';
 import { ILivechatRoom } from '@rocket.chat/apps-engine/definition/livechat';
-import { IDialogflowMessage, DialogflowRequestType } from '../enum/Dialogflow';
+import { DialogflowRequestType, IDialogflowMessage } from '../enum/Dialogflow';
 import { EndpointActionNames, IActionsEndpointContent } from '../enum/Endpoints';
 import { Headers, Response } from '../enum/Http';
 import { Logs } from '../enum/Logs';

enum/Dialogflow.ts

@@ -4,6 +4,7 @@ export interface IDialogflowMessage {
     messages?: Array<string | IDialogflowQuickReplies>;
     isFallback: boolean;
     sessionId?: string;
+    audio?: string;
 }
 
 export interface IDialogflowQuickReplies {
@@ -47,13 +48,40 @@ export enum DialogflowJWT {
 export enum Base64 {
     BASE64_DICTIONARY = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_',
     BASE64_PAD = '=',
+    BASE64 = 'base64',
 }
 
 export enum LanguageCode {
     EN = 'en',
 }
 
+export enum AudioLanguageCode {
+    EN_US = 'en-US',
+}
+
 export enum DialogflowRequestType {
     MESSAGE = 'message',
     EVENT = 'event',
+    AUDIO = 'audio',
+    AUDIO_OGG = 'audio-ogg',
+}
+
+export enum DialogflowOutputAudioEncoding {
+    LINEAR_16 = 'OUTPUT_AUDIO_ENCODING_LINEAR_16',
+}
+
+export enum DialogflowInputAudioEncoding {
+    ENCODING_OGG = 'AUDIO_ENCODING_OGG_OPUS',
+}
+
+export enum MIME_TYPE {
+    AUDIO_OGG = 'audio/ogg',
+    AUDIO_PREFIX = 'audio',
+}
+
+// supported audio extensions
+export enum AUDIO_EXTENSION {
+    OGA = 'oga',
+    WAV = 'wav',
+    OPUS = 'opus',
 }

enum/Logs.ts

@@ -20,4 +20,6 @@ export enum Logs {
     HTTP_REQUEST_ERROR = 'Error occurred while sending a HTTP Request',
     CLOSE_CHAT_REQUEST_FAILED_ERROR = 'Error: Internal Server Error. Could not close the chat',
     HANDOVER_REQUEST_FAILED_ERROR = 'Error occurred while processing handover. Details',
+    ERROR_NULL_RESPONSE = 'Error! Null Response',
+    INVALID_AUDIO_FILE_NAME = 'Invalid audio file name',
 }

handler/PostMessageSentHandler.ts

@@ -3,9 +3,10 @@ import { IApp } from '@rocket.chat/apps-engine/definition/IApp';
 import { ILivechatMessage, ILivechatRoom } from '@rocket.chat/apps-engine/definition/livechat';
 import { RoomType } from '@rocket.chat/apps-engine/definition/rooms';
 import { AppSetting, DefaultMessage } from '../config/Settings';
-import { DialogflowRequestType, IDialogflowMessage } from '../enum/Dialogflow';
+import { DialogflowRequestType, IDialogflowMessage, MIME_TYPE } from '../enum/Dialogflow';
 import { Logs } from '../enum/Logs';
 import { Dialogflow } from '../lib/Dialogflow';
+import { defineAudioFile } from '../lib/Helper';
 import { createDialogflowMessage, createMessage } from '../lib/Message';
 import { getAppSettingValue } from '../lib/Settings';
 import { incFallbackIntent, resetFallbackIntent } from '../lib/SynchronousHandover';
@@ -19,7 +20,8 @@ export class PostMessageSentHandler {
                 private readonly modify: IModify) {}
 
     public async run() {
-        const { text, editedAt, room, token, sender } = this.message;
+        const { text, editedAt, room, token, sender, file } = this.message;
+
         const livechatRoom = room as ILivechatRoom;
 
         const { id: rid, type, servedBy, isOpen } = livechatRoom;
@@ -30,7 +32,7 @@ export class PostMessageSentHandler {
             return;
         }
 
-        if (!isOpen || !token || editedAt || !text) {
+        if (!isOpen || !token || editedAt) {
             return;
         }
 
@@ -42,33 +44,43 @@ export class PostMessageSentHandler {
             return;
         }
 
-        if (!text || (text && text.trim().length === 0)) {
-            return;
-        }
-
-        let response: IDialogflowMessage;
         try {
-            response = (await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, text, DialogflowRequestType.MESSAGE));
+            let response: IDialogflowMessage | undefined = undefined;
+
+            if (text && text.trim().length !== 0) {
+                response = await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, text, DialogflowRequestType.MESSAGE);
+            }
+
+            if (file && file.type.startsWith(MIME_TYPE.AUDIO_PREFIX)) {
+                const { content, contentType } = await defineAudioFile(this.read, this.modify, rid, file);
+                if (content && contentType) {
+                    response = await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, content, contentType);
+                }
+            }
+
+            if (!response) {
+                return;
+            }
+
+            await createDialogflowMessage(rid, this.read, this.modify, response);
+
+            // synchronous handover check
+            const { isFallback } = response;
+            if (isFallback) {
+                return incFallbackIntent(this.read, this.modify, rid);
+            }
+            return resetFallbackIntent(this.read, this.modify, rid);
+
         } catch (error) {
             this.app.getLogger().error(`${Logs.DIALOGFLOW_REST_API_ERROR} ${error.message}`);
 
             const serviceUnavailable: string = await getAppSettingValue(this.read, AppSetting.DialogflowServiceUnavailableMessage);
 
-            await createMessage(rid,
+            return createMessage(rid,
                                 this.read,
                                 this.modify,
                                 { text: serviceUnavailable ? serviceUnavailable : DefaultMessage.DEFAULT_DialogflowServiceUnavailableMessage });
 
-            return;
-        }
-
-        await createDialogflowMessage(rid, this.read, this.modify, response);
-
-        // synchronous handover check
-        const { isFallback } = response;
-        if (isFallback) {
-            return incFallbackIntent(this.read, this.modify, rid);
         }
-        return resetFallbackIntent(this.read, this.modify, rid);
     }
 }

i18n/en.json

@@ -13,6 +13,8 @@
     "dialogflow_service_unavailable_message_description": "The Bot will send this message to Visitor if service is unavailable",
     "dialogflow_close_chat_message": "Close Chat Message",
     "dialogflow_close_chat_message_description": "This message will be sent automatically when a chat is closed",
+    "show_only_text_messages": "Show only Text Messages",
+    "show_only_text_messages_description": "If this Setting is Enabled, then the bot will always reply back with a text message, even when a visitor sends a audio message.",
     "dialogflow_hide_quick_replies": "Hide Quick Replies",
     "dialogflow_hide_quick_replies_description": "If enabled, then all quick-replies will hide when a visitor clicks on any one of them"
 }

lib/Dialogflow.ts

@@ -1,8 +1,8 @@
-import { IHttp, IHttpRequest, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors';
+import { IHttp, IHttpRequest, IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors';
 import { IRoom } from '@rocket.chat/apps-engine/definition/rooms';
 import { createSign } from 'crypto';
 import { AppSetting } from '../config/Settings';
-import { DialogflowJWT, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow';
+import { AudioLanguageCode, DialogflowInputAudioEncoding, DialogflowJWT, DialogflowOutputAudioEncoding, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow';
 import { Headers } from '../enum/Http';
 import { Logs } from '../enum/Logs';
 import { base64urlEncode } from './Helper';
@@ -23,18 +23,30 @@ class DialogflowClass {
         const queryInput = {
             ...requestType === DialogflowRequestType.EVENT && { event: request },
             ...requestType === DialogflowRequestType.MESSAGE && { text: { languageCode: LanguageCode.EN, text: request } },
+            ...requestType === DialogflowRequestType.AUDIO && { audioConfig: { languageCode: AudioLanguageCode.EN_US } },
+            ...requestType === DialogflowRequestType.AUDIO_OGG &&
+                { audioConfig: { audioEncoding: DialogflowInputAudioEncoding.ENCODING_OGG, sampleRateHertz: 16000, languageCode: AudioLanguageCode.EN_US } },
         };
 
+        const { value: onlyTextMessage } = await read.getEnvironmentReader().getSettings().getById(AppSetting.DialogflowShowOnlyTextMessages);
+
         const httpRequestContent: IHttpRequest = createHttpRequest(
             { 'Content-Type': Headers.CONTENT_TYPE_JSON, 'Accept': Headers.ACCEPT_JSON },
-            { queryInput },
+            {
+                queryInput,
+                ...(requestType === DialogflowRequestType.AUDIO || requestType === DialogflowRequestType.AUDIO_OGG) &&
+                        {
+                            inputAudio: request,
+                            ...!onlyTextMessage && { outputAudioConfig: { audioEncoding: DialogflowOutputAudioEncoding.LINEAR_16 } },
+                        },
+            },
         );
 
         try {
             const response = await http.post(serverURL, httpRequestContent);
             return this.parseRequest(response.data);
         } catch (error) {
-            throw new Error(`${ Logs.HTTP_REQUEST_ERROR }`);
+            throw new Error(`${ Logs.HTTP_REQUEST_ERROR }. Details:- ${ error }`);
         }
     }
 
@@ -81,7 +93,14 @@ class DialogflowClass {
     public parseRequest(response: any): IDialogflowMessage {
         if (!response) { throw new Error(Logs.INVALID_RESPONSE_FROM_DIALOGFLOW_CONTENT_UNDEFINED); }
 
-        const { session, queryResult } = response;
+        const { session, queryResult, outputAudio } = response;
+        if (outputAudio) {
+            const { intent: { isFallback } } = queryResult;
+            return {
+                audio: outputAudio,
+                isFallback: isFallback ? isFallback : false,
+            };
+        }
         if (queryResult) {
             const { fulfillmentMessages, intent: { isFallback } } = queryResult;
             const parsedMessage: IDialogflowMessage = {

lib/Helper.ts

@@ -1,4 +1,9 @@
-import { Base64 } from '../enum/Dialogflow';
+import { IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors';
+import { IMessageFile } from '@rocket.chat/apps-engine/definition/messages';
+import { DefaultMessage } from '../config/Settings';
+import { AUDIO_EXTENSION, Base64, DialogflowRequestType, MIME_TYPE } from '../enum/Dialogflow';
+import { Logs } from '../enum/Logs';
+import { createMessage } from './Message';
 
 export const base64urlEncode = (str: any) => {
     const utf8str = unescape(encodeURIComponent(str));
@@ -35,10 +40,30 @@ export const base64EncodeData = (data: string, len: number, b64x: string, b64pad
     return dst;
 };
 
-export const uuid = (): string => {
-    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
-        const r = Math.random() * 16 | 0;
-        const v = c === 'x' ? r : (r & 0x3 | 0x8);
-        return v.toString(16);
-    });
+export const defineAudioFile = async (read: IRead, modify: IModify, roomId: string, file: IMessageFile): Promise<{ content: string, contentType: DialogflowRequestType}> => {
+    const { name } = await read.getUploadReader().getById(file._id);
+
+    if (!isSupportedAudioFormat(name)) {
+        await createMessage(roomId, read, modify, { text: DefaultMessage.DEFAULT_UnsupportedAudioFormatMessage });
+    }
+
+    const content = (await read.getUploadReader().getBufferById(file._id)).toString(Base64.BASE64);
+    const contentType = file && file.type === MIME_TYPE.AUDIO_OGG ? DialogflowRequestType.AUDIO_OGG : DialogflowRequestType.AUDIO;
+
+    return { content, contentType };
+};
+
+export const isSupportedAudioFormat = (fileName: string): boolean => {
+    const extension = fileName.split('.')[1];
+    if (!extension) {
+        throw new Error(Logs.INVALID_AUDIO_FILE_NAME);
+    }
+
+    if (extension === AUDIO_EXTENSION.OGA ||
+        extension === AUDIO_EXTENSION.WAV ||
+        extension === AUDIO_EXTENSION.OPUS) {
+        return true;
+    }
+
+    return false;
 };

lib/Message.ts

@@ -1,15 +1,38 @@
 import { IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors';
 import { IVisitor } from '@rocket.chat/apps-engine/definition/livechat';
 import { BlockElementType, BlockType, IActionsBlock, IButtonElement, TextObjectType } from '@rocket.chat/apps-engine/definition/uikit';
+import { IUploadDescriptor } from '@rocket.chat/apps-engine/definition/uploads/IUploadDescriptor';
 import { IUser } from '@rocket.chat/apps-engine/definition/users';
+import { Buffer } from 'buffer';
 import { AppSetting } from '../config/Settings';
-import { IDialogflowMessage, IDialogflowQuickReplies, IDialogflowQuickReplyOptions } from '../enum/Dialogflow';
+import { DialogflowJWT, IDialogflowMessage, IDialogflowQuickReplies, IDialogflowQuickReplyOptions } from '../enum/Dialogflow';
 import { Logs } from '../enum/Logs';
-import { uuid } from './Helper';
 import { getAppSettingValue } from './Settings';
 
 export const createDialogflowMessage = async (rid: string, read: IRead,  modify: IModify, dialogflowMessage: IDialogflowMessage): Promise<any> => {
-    const { messages = [] } = dialogflowMessage;
+    const { messages = [], audio } = dialogflowMessage;
+
+    const room = await read.getRoomReader().getById(rid);
+    if (!room) {
+        throw new Error(`${Logs.INVALID_ROOM_ID} ${rid}`);
+    }
+    const botUserName = await getAppSettingValue(read, AppSetting.DialogflowBotUsername);
+    if (!botUserName) {
+        throw new Error(Logs.EMPTY_BOT_USERNAME_SETTING);
+    }
+    const sender = await read.getUserReader().getByUsername(botUserName);
+    if (!sender) {
+        throw new Error(Logs.INVALID_BOT_USERNAME_SETTING);
+    }
+    if (audio) {
+        const buffer = Buffer.from(audio, DialogflowJWT.BASE_64);
+        const uploadDescriptor: IUploadDescriptor = {
+            filename: 'audio.wav',
+            room,
+            user: sender,
+        };
+        await modify.getCreator().getUploadCreator().uploadBuffer(buffer, uploadDescriptor);
+    }
 
     for (const message of messages) {
         const { text, options } = message as IDialogflowQuickReplies;
@@ -126,3 +149,13 @@ export const deleteAllActionBlocks = async (modify: IModify, appUser: IUser, msg
     msgBuilder.setEditor(appUser).setBlocks(modify.getCreator().getBlockBuilder().getBlocks());
     return modify.getUpdater().finish(msgBuilder);
 };
+
+export const uuid = (): string => {
+    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+        // tslint:disable-next-line: no-bitwise
+        const r = Math.random() * 16 | 0;
+        // tslint:disable-next-line: no-bitwise
+        const v = c === 'x' ? r : (r & 0x3 | 0x8);
+        return v.toString(16);
+    });
+};