From a6855fdb3c569f198c71df18c420302dde5c1dbb Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Sun, 2 Aug 2020 20:25:59 +0530 Subject: [PATCH 1/8] add support for audio input --- enum/Dialogflow.ts | 5 +++++ handler/PostMessageSentHandler.ts | 17 +++++++++++++---- lib/Dialogflow.ts | 8 ++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/enum/Dialogflow.ts b/enum/Dialogflow.ts index d5e6f3f..9f16f84 100644 --- a/enum/Dialogflow.ts +++ b/enum/Dialogflow.ts @@ -45,7 +45,12 @@ export enum LanguageCode { EN = 'en', } +export enum AudioLanguageCode { + EN_US = 'en-US', +} + export enum DialogflowRequestType { MESSAGE = 'message', EVENT = 'event', + AUDIO = 'audio', } diff --git a/handler/PostMessageSentHandler.ts b/handler/PostMessageSentHandler.ts index 3c3325e..ceba9f9 100644 --- a/handler/PostMessageSentHandler.ts +++ b/handler/PostMessageSentHandler.ts @@ -1,6 +1,7 @@ import { IHttp, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { IApp } from '@rocket.chat/apps-engine/definition/IApp'; import { ILivechatMessage, ILivechatRoom } from '@rocket.chat/apps-engine/definition/livechat'; +import { IMessageFile } from '@rocket.chat/apps-engine/definition/messages'; import { RoomType } from '@rocket.chat/apps-engine/definition/rooms'; import { AppSetting, DefaultMessage } from '../config/Settings'; import { DialogflowRequestType, IDialogflowMessage } from '../enum/Dialogflow'; @@ -19,7 +20,8 @@ export class PostMessageSentHandler { private readonly modify: IModify) {} public async run() { - const { text, editedAt, room, token, sender } = this.message; + const { text, editedAt, room, token, sender, file } = this.message; + const livechatRoom = room as ILivechatRoom; const { id: rid, type, servedBy, isOpen } = livechatRoom; @@ -30,7 +32,7 @@ export class PostMessageSentHandler { return; } - if (!isOpen || !token || editedAt || !text) { + if (!isOpen || !token || editedAt) { return; } @@ -42,13 +44,20 @@ export class PostMessageSentHandler { return; } - if (!text || (text && text.trim().length === 0)) { + if (!text && !file) { + return; + } + + if ((text && text.trim().length === 0) || (file && !file.type.startsWith('audio'))) { return; } let response: IDialogflowMessage; try { - response = (await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, text, DialogflowRequestType.MESSAGE)); + const content = text || (await this.read.getUploadReader().getBufferById((file as IMessageFile)._id)).toString('base64'); + const contentType = text ? DialogflowRequestType.MESSAGE : DialogflowRequestType.AUDIO; + + response = (await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, content, contentType)); } catch (error) { this.app.getLogger().error(`${Logs.DIALOGFLOW_REST_API_ERROR} ${error.message}`); diff --git a/lib/Dialogflow.ts b/lib/Dialogflow.ts index c26bf97..8324cc8 100644 --- a/lib/Dialogflow.ts +++ b/lib/Dialogflow.ts @@ -2,7 +2,7 @@ import { IHttp, IHttpRequest, IModify, IPersistence, IRead } from '@rocket.chat/ import { IRoom } from '@rocket.chat/apps-engine/definition/rooms'; import { createSign } from 'crypto'; import { AppSetting } from '../config/Settings'; -import { DialogflowJWT, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; +import { AudioLanguageCode, DialogflowJWT, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; import { Headers } from '../enum/Http'; import { Logs } from '../enum/Logs'; import { base64urlEncode } from './Helper'; @@ -23,11 +23,15 @@ class DialogflowClass { const queryInput = { ...requestType === DialogflowRequestType.EVENT && { event: request }, ...requestType === DialogflowRequestType.MESSAGE && { text: { languageCode: LanguageCode.EN, text: request } }, + ...requestType === DialogflowRequestType.AUDIO && { audioConfig: { languageCode: AudioLanguageCode.EN_US } }, }; const httpRequestContent: IHttpRequest = createHttpRequest( { 'Content-Type': Headers.CONTENT_TYPE_JSON, 'Accept': Headers.ACCEPT_JSON }, - { queryInput }, + { + queryInput, + ...requestType === DialogflowRequestType.AUDIO && { inputAudio: request }, + }, ); try { From 8d73f4e5872706e0f98f89b34f12a426d888f406 Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Tue, 4 Aug 2020 20:03:00 +0530 Subject: [PATCH 2/8] add support for output audio --- enum/Dialogflow.ts | 5 +++++ lib/Dialogflow.ts | 12 ++++++++++-- lib/Message.ts | 10 ++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/enum/Dialogflow.ts b/enum/Dialogflow.ts index 9f16f84..71c656c 100644 --- a/enum/Dialogflow.ts +++ b/enum/Dialogflow.ts @@ -2,6 +2,7 @@ export interface IDialogflowMessage { messages?: Array; isFallback: boolean; sessionId?: string; + audio?: string; } export interface IDialogflowQuickReplies { @@ -54,3 +55,7 @@ export enum DialogflowRequestType { EVENT = 'event', AUDIO = 'audio', } + +export enum DialogflowOutputAudioEncoding { + LINEAR_16 = 'OUTPUT_AUDIO_ENCODING_LINEAR_16', +} diff --git a/lib/Dialogflow.ts b/lib/Dialogflow.ts index 8324cc8..ff2d42f 100644 --- a/lib/Dialogflow.ts +++ b/lib/Dialogflow.ts @@ -2,7 +2,7 @@ import { IHttp, IHttpRequest, IModify, IPersistence, IRead } from '@rocket.chat/ import { IRoom } from '@rocket.chat/apps-engine/definition/rooms'; import { createSign } from 'crypto'; import { AppSetting } from '../config/Settings'; -import { AudioLanguageCode, DialogflowJWT, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; +import { AudioLanguageCode, DialogflowJWT, DialogflowOutputAudioEncoding, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; import { Headers } from '../enum/Http'; import { Logs } from '../enum/Logs'; import { base64urlEncode } from './Helper'; @@ -31,6 +31,7 @@ class DialogflowClass { { queryInput, ...requestType === DialogflowRequestType.AUDIO && { inputAudio: request }, + ...requestType === DialogflowRequestType.AUDIO && { outputAudioConfig: { audioEncoding: DialogflowOutputAudioEncoding.LINEAR_16 } }, }, ); @@ -85,7 +86,14 @@ class DialogflowClass { public parseRequest(response: any): IDialogflowMessage { if (!response) { throw new Error(Logs.INVALID_RESPONSE_FROM_DIALOGFLOW_CONTENT_UNDEFINED); } - const { session, queryResult } = response; + const { session, queryResult, outputAudio } = response; + if (outputAudio) { + const { intent: { isFallback } } = queryResult; + return { + audio: outputAudio, + isFallback: isFallback ? isFallback : false, + }; + } if (queryResult) { const { fulfillmentMessages, intent: { isFallback } } = queryResult; const parsedMessage: IDialogflowMessage = { diff --git a/lib/Message.ts b/lib/Message.ts index 72799c8..11f04d8 100644 --- a/lib/Message.ts +++ b/lib/Message.ts @@ -6,7 +6,13 @@ import { Logs } from '../enum/Logs'; import { getAppSettingValue } from './Settings'; export const createDialogflowMessage = async (rid: string, read: IRead, modify: IModify, dialogflowMessage: IDialogflowMessage): Promise => { - const { messages = [] } = dialogflowMessage; + const { messages = [], audio } = dialogflowMessage; + + if (audio) { + const uri = `data:audio/x-wav;base64,${ audio }`; + const attachment = { audioUrl: uri }; + await createMessage(rid, read, modify, { attachment }); + } for (const message of messages) { const { text, options } = message as IDialogflowQuickReplies; @@ -54,7 +60,7 @@ export const createMessage = async (rid: string, read: IRead, modify: IModify, return; } - const msg = modify.getCreator().startMessage().setRoom(room).setSender(sender); + const msg = modify.getCreator().startLivechatMessage().setRoom(room).setSender(sender); const { text, attachment } = message; if (text) { From b73c2cc94f00c0c61c10c07c630e09ed8bb43cd7 Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Fri, 7 Aug 2020 00:30:12 +0530 Subject: [PATCH 3/8] add additional support for whataspp audio file (oga) --- config/Settings.ts | 11 +++++++++++ enum/Dialogflow.ts | 5 +++++ handler/PostMessageSentHandler.ts | 4 +++- i18n/en.json | 4 +++- lib/Dialogflow.ts | 15 +++++++++++---- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/config/Settings.ts b/config/Settings.ts index 2f1fc63..216e02b 100644 --- a/config/Settings.ts +++ b/config/Settings.ts @@ -6,6 +6,7 @@ export enum AppSetting { DialogflowClientEmail = 'dialogflow_client_email', DialogFlowPrivateKey = 'dialogflow_private_key', DialogflowFallbackResponsesLimit = 'dialogflow_fallback_responses_limit', + DialogflowShowOnlyTextMessages = 'show_only_text_messages', FallbackTargetDepartment = 'fallback_target_department', DialogflowHandoverMessage = 'dialogflow_handover_message', DialogflowServiceUnavailableMessage = 'dialogflow_service_unavailable_message', @@ -70,6 +71,16 @@ export const settings: Array = [ i18nDescription: 'target_department_for_handover_description', required: false, }, + { + id: AppSetting.DialogflowShowOnlyTextMessages, + public: true, + type: SettingType.BOOLEAN, + packageValue: false, + value: false, + i18nLabel: 'show_only_text_messages', + i18nDescription: 'show_only_text_messages_description', + required: true, + }, { id: AppSetting.DialogflowHandoverMessage, public: true, diff --git a/enum/Dialogflow.ts b/enum/Dialogflow.ts index 71c656c..bc367c0 100644 --- a/enum/Dialogflow.ts +++ b/enum/Dialogflow.ts @@ -54,8 +54,13 @@ export enum DialogflowRequestType { MESSAGE = 'message', EVENT = 'event', AUDIO = 'audio', + AUDIO_OGG = 'audio-ogg', } export enum DialogflowOutputAudioEncoding { LINEAR_16 = 'OUTPUT_AUDIO_ENCODING_LINEAR_16', } + +export enum DialogflowInputAudioEncoding { + ENCODING_OGG = 'AUDIO_ENCODING_OGG_OPUS', +} diff --git a/handler/PostMessageSentHandler.ts b/handler/PostMessageSentHandler.ts index ceba9f9..f27c058 100644 --- a/handler/PostMessageSentHandler.ts +++ b/handler/PostMessageSentHandler.ts @@ -55,7 +55,9 @@ export class PostMessageSentHandler { let response: IDialogflowMessage; try { const content = text || (await this.read.getUploadReader().getBufferById((file as IMessageFile)._id)).toString('base64'); - const contentType = text ? DialogflowRequestType.MESSAGE : DialogflowRequestType.AUDIO; + const contentType = text ? + DialogflowRequestType.MESSAGE : + (file && file.type === 'audio/ogg') ? DialogflowRequestType.AUDIO_OGG : DialogflowRequestType.AUDIO; response = (await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, content, contentType)); } catch (error) { diff --git a/i18n/en.json b/i18n/en.json index 5d49d09..557cd7d 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -12,5 +12,7 @@ "dialogflow_service_unavailable_message": "Service Unavailable Message", "dialogflow_service_unavailable_message_description": "The Bot will send this message to Visitor if service is unavailable", "dialogflow_close_chat_message": "Close Chat Message", - "dialogflow_close_chat_message_description": "This message will be sent automatically when a chat is closed" + "dialogflow_close_chat_message_description": "This message will be sent automatically when a chat is closed", + "show_only_text_messages": "Show only Text Messages", + "show_only_text_messages_description": "If this Setting is Enabled, then the bot will always reply back with a text message, even when a visitor sends a audio message." } diff --git a/lib/Dialogflow.ts b/lib/Dialogflow.ts index ff2d42f..4078c4f 100644 --- a/lib/Dialogflow.ts +++ b/lib/Dialogflow.ts @@ -2,7 +2,7 @@ import { IHttp, IHttpRequest, IModify, IPersistence, IRead } from '@rocket.chat/ import { IRoom } from '@rocket.chat/apps-engine/definition/rooms'; import { createSign } from 'crypto'; import { AppSetting } from '../config/Settings'; -import { AudioLanguageCode, DialogflowJWT, DialogflowOutputAudioEncoding, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; +import { AudioLanguageCode, DialogflowInputAudioEncoding, DialogflowJWT, DialogflowOutputAudioEncoding, DialogflowRequestType, DialogflowUrl, IDialogflowAccessToken, IDialogflowEvent, IDialogflowMessage, IDialogflowQuickReplies, LanguageCode } from '../enum/Dialogflow'; import { Headers } from '../enum/Http'; import { Logs } from '../enum/Logs'; import { base64urlEncode } from './Helper'; @@ -24,14 +24,21 @@ class DialogflowClass { ...requestType === DialogflowRequestType.EVENT && { event: request }, ...requestType === DialogflowRequestType.MESSAGE && { text: { languageCode: LanguageCode.EN, text: request } }, ...requestType === DialogflowRequestType.AUDIO && { audioConfig: { languageCode: AudioLanguageCode.EN_US } }, + ...requestType === DialogflowRequestType.AUDIO_OGG && + { audioConfig: { audioEncoding: DialogflowInputAudioEncoding.ENCODING_OGG, sampleRateHertz: 16000, languageCode: AudioLanguageCode.EN_US } }, }; + const onlyTextMessage = await getAppSettingValue(read, AppSetting.DialogflowShowOnlyTextMessages); + const httpRequestContent: IHttpRequest = createHttpRequest( { 'Content-Type': Headers.CONTENT_TYPE_JSON, 'Accept': Headers.ACCEPT_JSON }, { queryInput, - ...requestType === DialogflowRequestType.AUDIO && { inputAudio: request }, - ...requestType === DialogflowRequestType.AUDIO && { outputAudioConfig: { audioEncoding: DialogflowOutputAudioEncoding.LINEAR_16 } }, + ...(requestType === DialogflowRequestType.AUDIO || requestType === DialogflowRequestType.AUDIO_OGG) && + { + inputAudio: request, + ...!onlyTextMessage && { outputAudioConfig: { audioEncoding: DialogflowOutputAudioEncoding.LINEAR_16 } }, + }, }, ); @@ -39,7 +46,7 @@ class DialogflowClass { const response = await http.post(serverURL, httpRequestContent); return this.parseRequest(response.data); } catch (error) { - throw new Error(`${ Logs.HTTP_REQUEST_ERROR }`); + throw new Error(`${ Logs.HTTP_REQUEST_ERROR }. Details:- ${ error }`); } } From b91b1c656b5a6d322bf0e9fde294796c5de20ad2 Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Fri, 7 Aug 2020 00:59:03 +0530 Subject: [PATCH 4/8] Update App to verion 1.2.0 --- app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.json b/app.json index 413ced8..2525c62 100644 --- a/app.json +++ b/app.json @@ -1,6 +1,6 @@ { "id": "21b7d3ba-031b-41d9-8ff2-fbbfa081ae90", - "version": "1.0.0", + "version": "1.2.0", "requiredApiVersion": "^1.15.0", "iconFile": "icon.png", "author": { From c8c4b0b89bd3aae4eecc90d380ae2bab1c30ced9 Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Mon, 17 Aug 2020 00:18:55 +0530 Subject: [PATCH 5/8] update readme n set default new setting value to true --- README.md | 9 ++++++--- config/Settings.ts | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8ec53e8..2d87a68 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,14 @@ You can find all these credentials in a JSON file, which u can get from [here](h - To Deactivate this feature, simply set the value to `0`. 6. Target Department for Handover (optional) - Enter the department name where you want the visitor to be transferred upon handover. - 7. Handover Message (optional) + 7. Show only Text Messages + - If this setting is enabled, then the bot will always reply back with Text Message. However if this setting is disabled, then the bot will reply back with audio if the visitor has provided audio as input. + - Please note, currently the output audio will only work for Livechat widget. If you are using any other channels like WhatsApp or Messenger, then please keep this setting enabled. + 8. Handover Message (optional) - The Bot will send this message to Visitor upon handover - 8. Service Unavailable Message (optional) + 9. Service Unavailable Message (optional) - The Bot will send this message to Visitor if service is unavailable like suppose if no agents are online. - 9. Close Chat Message (optional) + 10. Close Chat Message (optional) - This message will be sent automatically when a chat is closed 4. (Optional Step) Lastly you can test your Dialogflow Connection by viewing App Logs. To view the logs, goto App Page (`Setting > Apps > Dialogflow`). There click on menu item (3 vertical dots icon) and then select `View Logs`. There select the **most recent** `onSettingUpdated` title. If you see `------------------ Google Credentials validation Success ----------------` message, then it means your setup is fine. If you don't see this message, then recheck your Dialogflow credentials. diff --git a/config/Settings.ts b/config/Settings.ts index 216e02b..7660ed7 100644 --- a/config/Settings.ts +++ b/config/Settings.ts @@ -75,8 +75,8 @@ export const settings: Array = [ id: AppSetting.DialogflowShowOnlyTextMessages, public: true, type: SettingType.BOOLEAN, - packageValue: false, - value: false, + packageValue: true, + value: true, i18nLabel: 'show_only_text_messages', i18nDescription: 'show_only_text_messages_description', required: true, From e44d3f8d9949fd600d6aabbce3b5a2008270ea3d Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Sun, 4 Oct 2020 02:16:45 +0530 Subject: [PATCH 6/8] Using new Apps-engine interfaces to upload audio file --- app.json | 2 +- config/Settings.ts | 1 + enum/Dialogflow.ts | 13 ++++++++ enum/Logs.ts | 2 ++ handler/PostMessageSentHandler.ts | 53 ++++++++++++++++--------------- lib/Dialogflow.ts | 4 +-- lib/Helper.ts | 35 +++++++++++++++++++- lib/Message.ts | 26 ++++++++++++--- package-lock.json | 6 ++-- package.json | 2 +- 10 files changed, 106 insertions(+), 38 deletions(-) diff --git a/app.json b/app.json index 2525c62..09cf90d 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { "id": "21b7d3ba-031b-41d9-8ff2-fbbfa081ae90", "version": "1.2.0", - "requiredApiVersion": "^1.15.0", + "requiredApiVersion": "^1.18.0", "iconFile": "icon.png", "author": { "name": "Rocket.Chat", diff --git a/config/Settings.ts b/config/Settings.ts index 7660ed7..3b954fc 100644 --- a/config/Settings.ts +++ b/config/Settings.ts @@ -17,6 +17,7 @@ export enum DefaultMessage { DEFAULT_DialogflowServiceUnavailableMessage = 'Sorry, I\'m having trouble answering your question.', DEFAULT_DialogflowHandoverMessage = 'Transferring to an online agent', DEFAULT_DialogflowCloseChatMessage = 'Closing the chat, Goodbye', + DEFAULT_UnsupportedAudioFormatMessage = 'Sorry! Only *.wav, *.opus and *.oga extension files are supported as audio input', } export const settings: Array = [ diff --git a/enum/Dialogflow.ts b/enum/Dialogflow.ts index bc367c0..0c19623 100644 --- a/enum/Dialogflow.ts +++ b/enum/Dialogflow.ts @@ -40,6 +40,7 @@ export enum DialogflowJWT { export enum Base64 { BASE64_DICTIONARY = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', BASE64_PAD = '=', + BASE64 = 'base64', } export enum LanguageCode { @@ -64,3 +65,15 @@ export enum DialogflowOutputAudioEncoding { export enum DialogflowInputAudioEncoding { ENCODING_OGG = 'AUDIO_ENCODING_OGG_OPUS', } + +export enum MIME_TYPE { + AUDIO_OGG = 'audio/ogg', + AUDIO_PREFIX = 'audio', +} + +// supported audio extensions +export enum AUDIO_EXTENSION { + OGA = 'oga', + WAV = 'wav', + OPUS = 'opus', +} diff --git a/enum/Logs.ts b/enum/Logs.ts index 9a0212b..224f089 100644 --- a/enum/Logs.ts +++ b/enum/Logs.ts @@ -20,4 +20,6 @@ export enum Logs { HTTP_REQUEST_ERROR = 'Error occurred while sending a HTTP Request', CLOSE_CHAT_REQUEST_FAILED_ERROR = 'Error: Internal Server Error. Could not close the chat', HANDOVER_REQUEST_FAILED_ERROR = 'Error occurred while processing handover. Details', + ERROR_NULL_RESPONSE = 'Error! Null Response', + INVALID_AUDIO_FILE_NAME = 'Invalid audio file name', } diff --git a/handler/PostMessageSentHandler.ts b/handler/PostMessageSentHandler.ts index f27c058..a3aab62 100644 --- a/handler/PostMessageSentHandler.ts +++ b/handler/PostMessageSentHandler.ts @@ -1,12 +1,12 @@ import { IHttp, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { IApp } from '@rocket.chat/apps-engine/definition/IApp'; import { ILivechatMessage, ILivechatRoom } from '@rocket.chat/apps-engine/definition/livechat'; -import { IMessageFile } from '@rocket.chat/apps-engine/definition/messages'; import { RoomType } from '@rocket.chat/apps-engine/definition/rooms'; import { AppSetting, DefaultMessage } from '../config/Settings'; -import { DialogflowRequestType, IDialogflowMessage } from '../enum/Dialogflow'; +import { DialogflowRequestType, IDialogflowMessage, MIME_TYPE } from '../enum/Dialogflow'; import { Logs } from '../enum/Logs'; import { Dialogflow } from '../lib/Dialogflow'; +import { defineAudioFile } from '../lib/Helper'; import { createDialogflowMessage, createMessage } from '../lib/Message'; import { getAppSettingValue } from '../lib/Settings'; import { incFallbackIntent, resetFallbackIntent } from '../lib/SynchronousHandover'; @@ -44,42 +44,43 @@ export class PostMessageSentHandler { return; } - if (!text && !file) { - return; - } + try { + let response: IDialogflowMessage | undefined = undefined; - if ((text && text.trim().length === 0) || (file && !file.type.startsWith('audio'))) { - return; - } + if (text && text.trim().length !== 0) { + response = await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, text, DialogflowRequestType.MESSAGE); + } - let response: IDialogflowMessage; - try { - const content = text || (await this.read.getUploadReader().getBufferById((file as IMessageFile)._id)).toString('base64'); - const contentType = text ? - DialogflowRequestType.MESSAGE : - (file && file.type === 'audio/ogg') ? DialogflowRequestType.AUDIO_OGG : DialogflowRequestType.AUDIO; + if (file && file.type.startsWith(MIME_TYPE.AUDIO_PREFIX)) { + const { content, contentType } = await defineAudioFile(this.read, this.modify, rid, file); + if (content && contentType) { + response = await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, content, contentType); + } + } + + if (!response) { + return; + } + + await createDialogflowMessage(rid, this.read, this.modify, response); + + // synchronous handover check + const { isFallback } = response; + if (isFallback) { + return incFallbackIntent(this.read, this.modify, rid); + } + return resetFallbackIntent(this.read, this.modify, rid); - response = (await Dialogflow.sendRequest(this.http, this.read, this.modify, rid, content, contentType)); } catch (error) { this.app.getLogger().error(`${Logs.DIALOGFLOW_REST_API_ERROR} ${error.message}`); const serviceUnavailable: string = await getAppSettingValue(this.read, AppSetting.DialogflowServiceUnavailableMessage); - await createMessage(rid, + return createMessage(rid, this.read, this.modify, { text: serviceUnavailable ? serviceUnavailable : DefaultMessage.DEFAULT_DialogflowServiceUnavailableMessage }); - return; - } - - await createDialogflowMessage(rid, this.read, this.modify, response); - - // synchronous handover check - const { isFallback } = response; - if (isFallback) { - return incFallbackIntent(this.read, this.modify, rid); } - return resetFallbackIntent(this.read, this.modify, rid); } } diff --git a/lib/Dialogflow.ts b/lib/Dialogflow.ts index 4078c4f..b6ff97c 100644 --- a/lib/Dialogflow.ts +++ b/lib/Dialogflow.ts @@ -1,4 +1,4 @@ -import { IHttp, IHttpRequest, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors'; +import { IHttp, IHttpRequest, IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { IRoom } from '@rocket.chat/apps-engine/definition/rooms'; import { createSign } from 'crypto'; import { AppSetting } from '../config/Settings'; @@ -28,7 +28,7 @@ class DialogflowClass { { audioConfig: { audioEncoding: DialogflowInputAudioEncoding.ENCODING_OGG, sampleRateHertz: 16000, languageCode: AudioLanguageCode.EN_US } }, }; - const onlyTextMessage = await getAppSettingValue(read, AppSetting.DialogflowShowOnlyTextMessages); + const { value: onlyTextMessage } = await read.getEnvironmentReader().getSettings().getById(AppSetting.DialogflowShowOnlyTextMessages); const httpRequestContent: IHttpRequest = createHttpRequest( { 'Content-Type': Headers.CONTENT_TYPE_JSON, 'Accept': Headers.ACCEPT_JSON }, diff --git a/lib/Helper.ts b/lib/Helper.ts index 30a3cb5..e9723de 100644 --- a/lib/Helper.ts +++ b/lib/Helper.ts @@ -1,4 +1,9 @@ -import { Base64 } from '../enum/Dialogflow'; +import { IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors'; +import { IMessageFile } from '@rocket.chat/apps-engine/definition/messages'; +import { DefaultMessage } from '../config/Settings'; +import { AUDIO_EXTENSION, Base64, DialogflowRequestType, MIME_TYPE } from '../enum/Dialogflow'; +import { Logs } from '../enum/Logs'; +import { createMessage } from './Message'; export const base64urlEncode = (str: any) => { const utf8str = unescape(encodeURIComponent(str)); @@ -34,3 +39,31 @@ export const base64EncodeData = (data: string, len: number, b64x: string, b64pad return dst; }; + +export const defineAudioFile = async (read: IRead, modify: IModify, roomId: string, file: IMessageFile): Promise<{ content: string, contentType: DialogflowRequestType}> => { + const { name } = await read.getUploadReader().getById(file._id); + + if (!isSupportedAudioFormat(name)) { + await createMessage(roomId, read, modify, { text: DefaultMessage.DEFAULT_UnsupportedAudioFormatMessage }); + } + + const content = (await read.getUploadReader().getBufferById(file._id)).toString(Base64.BASE64); + const contentType = file && file.type === MIME_TYPE.AUDIO_OGG ? DialogflowRequestType.AUDIO_OGG : DialogflowRequestType.AUDIO; + + return { content, contentType }; +}; + +export const isSupportedAudioFormat = (fileName: string): boolean => { + const extension = fileName.split('.')[1]; + if (!extension) { + throw new Error(Logs.INVALID_AUDIO_FILE_NAME); + } + + if (extension === AUDIO_EXTENSION.OGA || + extension === AUDIO_EXTENSION.WAV || + extension === AUDIO_EXTENSION.OPUS) { + return true; + } + + return false; +}; diff --git a/lib/Message.ts b/lib/Message.ts index 11f04d8..5b45470 100644 --- a/lib/Message.ts +++ b/lib/Message.ts @@ -1,17 +1,35 @@ import { IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { IMessageAction, IMessageAttachment, MessageActionType, MessageProcessingType } from '@rocket.chat/apps-engine/definition/messages'; +import { IUploadDescriptor } from '@rocket.chat/apps-engine/definition/uploads/IUploadDescriptor'; +import { Buffer } from 'buffer'; import { AppSetting } from '../config/Settings'; -import { IDialogflowMessage, IDialogflowQuickReplies } from '../enum/Dialogflow'; +import { DialogflowJWT, IDialogflowMessage, IDialogflowQuickReplies } from '../enum/Dialogflow'; import { Logs } from '../enum/Logs'; import { getAppSettingValue } from './Settings'; export const createDialogflowMessage = async (rid: string, read: IRead, modify: IModify, dialogflowMessage: IDialogflowMessage): Promise => { const { messages = [], audio } = dialogflowMessage; + const room = await read.getRoomReader().getById(rid); + if (!room) { + throw new Error(`${Logs.INVALID_ROOM_ID} ${rid}`); + } + const botUserName = await getAppSettingValue(read, AppSetting.DialogflowBotUsername); + if (!botUserName) { + throw new Error(Logs.EMPTY_BOT_USERNAME_SETTING); + } + const sender = await read.getUserReader().getByUsername(botUserName); + if (!sender) { + throw new Error(Logs.INVALID_BOT_USERNAME_SETTING); + } if (audio) { - const uri = `data:audio/x-wav;base64,${ audio }`; - const attachment = { audioUrl: uri }; - await createMessage(rid, read, modify, { attachment }); + const buffer = Buffer.from(audio, DialogflowJWT.BASE_64); + const uploadDescriptor: IUploadDescriptor = { + filename: 'audio.wav', + room, + user: sender, + }; + await modify.getCreator().getUploadCreator().uploadBuffer(buffer, uploadDescriptor); } for (const message of messages) { diff --git a/package-lock.json b/package-lock.json index b540f6e..9e111c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,9 +29,9 @@ } }, "@rocket.chat/apps-engine": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@rocket.chat/apps-engine/-/apps-engine-1.15.0.tgz", - "integrity": "sha512-gwsHa/zTYMmoSG3PP3sZfmVRDRBmIDacOAdCv1FsgJog89ZBCICeoab3VyYAdOMliV5XoygygduYFtc6rinFHQ==", + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/@rocket.chat/apps-engine/-/apps-engine-1.18.0.tgz", + "integrity": "sha512-Y9XgRnG8v4HtujaMXuzTd9hjycQX9+wDOE30HhHL1NQl7de4+9MzPdWDetdy+84cBrDtXCymVYF/TzmLf+69Mw==", "dev": true, "requires": { "adm-zip": "^0.4.9", diff --git a/package.json b/package.json index 0c61971..f040522 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "devDependencies": { - "@rocket.chat/apps-engine": "^1.15.0", + "@rocket.chat/apps-engine": "^1.18.0", "@types/node": "^10.17.24", "tslint": "^5.10.0", "typescript": "^2.9.1" From dd1e3adc18e4089db19a77cd2669d603712c4083 Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Mon, 5 Oct 2020 09:57:55 +0530 Subject: [PATCH 7/8] Bump to version 1.3.0 --- app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.json b/app.json index 9203cd7..81a76de 100644 --- a/app.json +++ b/app.json @@ -1,6 +1,6 @@ { "id": "21b7d3ba-031b-41d9-8ff2-fbbfa081ae90", - "version": "1.2.0", + "version": "1.3.0", "requiredApiVersion": "^1.17.0", "iconFile": "icon.png", "author": { From 01f95b55c01659ada55d056ac9ff14e5ea148970 Mon Sep 17 00:00:00 2001 From: murtaza98 Date: Tue, 6 Oct 2020 22:50:51 +0530 Subject: [PATCH 8/8] Fix Typescript ---> Maximum call stack size exceeded <--- Error --- endpoints/IncomingEndpoint.ts | 2 +- lib/Helper.ts | 8 -------- lib/Message.ts | 14 +++++++++++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/endpoints/IncomingEndpoint.ts b/endpoints/IncomingEndpoint.ts index 0f93180..b3cb5e5 100644 --- a/endpoints/IncomingEndpoint.ts +++ b/endpoints/IncomingEndpoint.ts @@ -1,7 +1,7 @@ import { HttpStatusCode, IHttp, IModify, IPersistence, IRead } from '@rocket.chat/apps-engine/definition/accessors'; import { ApiEndpoint, IApiEndpointInfo, IApiRequest, IApiResponse } from '@rocket.chat/apps-engine/definition/api'; import { ILivechatRoom } from '@rocket.chat/apps-engine/definition/livechat'; -import { IDialogflowMessage, DialogflowRequestType } from '../enum/Dialogflow'; +import { DialogflowRequestType, IDialogflowMessage } from '../enum/Dialogflow'; import { EndpointActionNames, IActionsEndpointContent } from '../enum/Endpoints'; import { Headers, Response } from '../enum/Http'; import { Logs } from '../enum/Logs'; diff --git a/lib/Helper.ts b/lib/Helper.ts index e8086fc..e9723de 100644 --- a/lib/Helper.ts +++ b/lib/Helper.ts @@ -67,11 +67,3 @@ export const isSupportedAudioFormat = (fileName: string): boolean => { return false; }; - -export const uuid = (): string => { - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { - const r = Math.random() * 16 | 0; - const v = c === 'x' ? r : (r & 0x3 | 0x8); - return v.toString(16); - }); -}; diff --git a/lib/Message.ts b/lib/Message.ts index dfeb618..15a906f 100644 --- a/lib/Message.ts +++ b/lib/Message.ts @@ -1,14 +1,12 @@ import { IModify, IRead } from '@rocket.chat/apps-engine/definition/accessors'; -import { IMessageAction, IMessageAttachment, MessageActionType, MessageProcessingType } from '@rocket.chat/apps-engine/definition/messages'; -import { IUploadDescriptor } from '@rocket.chat/apps-engine/definition/uploads/IUploadDescriptor'; import { IVisitor } from '@rocket.chat/apps-engine/definition/livechat'; import { BlockElementType, BlockType, IActionsBlock, IButtonElement, TextObjectType } from '@rocket.chat/apps-engine/definition/uikit'; +import { IUploadDescriptor } from '@rocket.chat/apps-engine/definition/uploads/IUploadDescriptor'; import { IUser } from '@rocket.chat/apps-engine/definition/users'; import { Buffer } from 'buffer'; import { AppSetting } from '../config/Settings'; import { DialogflowJWT, IDialogflowMessage, IDialogflowQuickReplies, IDialogflowQuickReplyOptions } from '../enum/Dialogflow'; import { Logs } from '../enum/Logs'; -import { uuid } from './Helper'; import { getAppSettingValue } from './Settings'; export const createDialogflowMessage = async (rid: string, read: IRead, modify: IModify, dialogflowMessage: IDialogflowMessage): Promise => { @@ -151,3 +149,13 @@ export const deleteAllActionBlocks = async (modify: IModify, appUser: IUser, msg msgBuilder.setEditor(appUser).setBlocks(modify.getCreator().getBlockBuilder().getBlocks()); return modify.getUpdater().finish(msgBuilder); }; + +export const uuid = (): string => { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { + // tslint:disable-next-line: no-bitwise + const r = Math.random() * 16 | 0; + // tslint:disable-next-line: no-bitwise + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); +};