From 0cd55cad495e7ce520a078b12bf9850fb3120570 Mon Sep 17 00:00:00 2001 From: Dmitry Katson Date: Sun, 4 May 2025 15:22:03 +0600 Subject: [PATCH 1/2] Add new image processing capabilities to AOAI Chat Messages - Introduced new procedures in AOAIChatMessages.Codeunit.al to handle user messages with images from various sources (URL, InStream, MediaSet, Tenant Media, Temp Blob). - Added AOAIImageDetailLevel.Enum.al to define detail levels for image processing. - Implemented AOAIImagesImpl.Codeunit.al for image content preparation and encoding. - Updated app.json to include new modules: BLOB Storage and Base64 Convert. --- src/System Application/App/AI/app.json | 12 + .../AOAIChatMessages.Codeunit.al | 66 +++++ .../AOAIChatMessagesImpl.Codeunit.al | 106 +++++++- .../AOAIImageDetailLevel.Enum.al | 28 ++ .../Images/AOAIImagesImpl.Codeunit.al | 245 ++++++++++++++++++ 5 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIImageDetailLevel.Enum.al create mode 100644 src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Images/AOAIImagesImpl.Codeunit.al diff --git a/src/System Application/App/AI/app.json b/src/System Application/App/AI/app.json index 68aa8cb477..3699892cf1 100644 --- a/src/System Application/App/AI/app.json +++ b/src/System Application/App/AI/app.json @@ -100,6 +100,18 @@ "name": "Client Type Management", "publisher": "Microsoft", "version": "27.0.0.0" + }, + { + "id": "e31ad830-3d46-472e-afeb-1d3d35247943", + "name": "BLOB Storage", + "publisher": "Microsoft", + "version": "27.0.0.0" + }, + { + "id": "0846d207-5dec-4c1b-afd8-6a25e1e14b9d", + "name": "Base64 Convert", + "publisher": "Microsoft", + "version": "27.0.0.0" } ], "screenshots": [], diff --git a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessages.Codeunit.al b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessages.Codeunit.al index 18881a2518..ec5105b4ac 100644 --- a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessages.Codeunit.al +++ b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessages.Codeunit.al @@ -4,6 +4,8 @@ // ------------------------------------------------------------------------------------------------ namespace System.AI; +using System.Environment; +using System.Utilities; /// /// Helper functions for the AOAI Chat Message table. @@ -59,6 +61,70 @@ codeunit 7763 "AOAI Chat Messages" AOAIChatMessagesImpl.AddUserMessage(NewMessage, NewName); end; + /// + /// Adds a user message containing text and an image from a URL to the chat messages history. + /// + /// The text part of the message. + /// The URL of the image (HTTPS recommended). + /// The detail level for image processing. + [NonDebuggable] + procedure AddUserMessage(UserText: Text; ImageUrl: Text; DetailLevel: Enum "AOAI Image Detail Level") + begin + AOAIChatMessagesImpl.AddUserMessage(UserText, ImageUrl, DetailLevel); + end; + + /// + /// Adds a user message containing text and an image from a stream to the chat messages history. + /// The image stream will be Base64 encoded. + /// + /// The text part of the message. + /// The InStream containing the image data. + /// The file extension of the image (e.g., 'png', 'jpg') used to determine the MIME type. + /// The detail level for image processing. + [NonDebuggable] + procedure AddUserMessage(UserText: Text; var ImageStream: InStream; FileExtension: Text; DetailLevel: Enum "AOAI Image Detail Level") + begin + AOAIChatMessagesImpl.AddUserMessage(UserText, ImageStream, FileExtension, DetailLevel); + end; + + /// + /// Adds a user message containing text and images from a MediaSet to the chat messages history. + /// + /// The text part of the message. + /// The Guid of the MediaSet containing the images. + /// The detail level for image processing. + [NonDebuggable] + procedure AddUserMessage(UserText: Text; MediaSetId: Guid; DetailLevel: Enum "AOAI Image Detail Level") + begin + AOAIChatMessagesImpl.AddUserMessage(UserText, MediaSetId, DetailLevel); + end; + + + /// + /// Adds a user message containing text and an image from a Tenant Media record to the chat messages history. + /// + /// The text part of the message. + /// The Tenant Media record containing the image. + /// The detail level for image processing. + [NonDebuggable] + procedure AddUserMessage(UserText: Text; TenantMedia: Record "Tenant Media"; DetailLevel: Enum "AOAI Image Detail Level") + begin + AOAIChatMessagesImpl.AddUserMessage(UserText, TenantMedia, DetailLevel); + end; + + /// + /// Adds a user message containing text and an image from a Temp Blob to the chat messages history. + /// + /// The text part of the message. + /// The Temp Blob codeunit containing the image data. + /// The file extension of the image in the Temp Blob (e.g., 'png', 'jpg'). + /// The detail level for image processing. + [NonDebuggable] + procedure AddUserMessage(UserText: Text; var TempBlob: Codeunit "Temp Blob"; FileExtension: Text; DetailLevel: Enum "AOAI Image Detail Level") + begin + AOAIChatMessagesImpl.AddUserMessage(UserText, TempBlob, FileExtension, DetailLevel); + end; + /// /// Adds a assistant message to the chat messages history. /// diff --git a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessagesImpl.Codeunit.al b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessagesImpl.Codeunit.al index 332d4a97a7..2cf7386c58 100644 --- a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessagesImpl.Codeunit.al +++ b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIChatMessagesImpl.Codeunit.al @@ -7,6 +7,7 @@ namespace System.AI; using System.Azure.KeyVault; using System.Environment; +using System.Utilities; using System.Telemetry; codeunit 7764 "AOAI Chat Messages Impl" @@ -69,6 +70,76 @@ codeunit 7764 "AOAI Chat Messages Impl" AddMessage(NewMessage, NewName, Enum::"AOAI Chat Roles"::User); end; + [NonDebuggable] + procedure AddUserMessage(UserText: Text; ImageUrl: Text; DetailLevel: Enum "AOAI Image Detail Level") + var + AOAIImagesImpl: Codeunit "AOAI Images Impl"; + ContentAsText: Text; + begin + Initialize(); + ContentAsText := AOAIImagesImpl.PrepareUserMessageContentFromUrl(UserText, ImageUrl, DetailLevel); + if ContentAsText = '' then + exit; + + AddMessage(ContentAsText, '', Enum::"AOAI Chat Roles"::User); + end; + + [NonDebuggable] + procedure AddUserMessage(UserText: Text; var ImageStream: InStream; FileExtension: Text; DetailLevel: Enum "AOAI Image Detail Level") + var + AOAIImagesImpl: Codeunit "AOAI Images Impl"; + ContentAsText: Text; + begin + Initialize(); + ContentAsText := AOAIImagesImpl.PrepareUserMessageContentFromStream(UserText, ImageStream, FileExtension, DetailLevel); + if ContentAsText = '' then + exit; + + AddMessage(ContentAsText, '', Enum::"AOAI Chat Roles"::User); + end; + + [NonDebuggable] + procedure AddUserMessage(UserText: Text; MediaSetId: Guid; DetailLevel: Enum "AOAI Image Detail Level") + var + AOAIImagesImpl: Codeunit "AOAI Images Impl"; + ContentAsText: Text; + begin + Initialize(); + ContentAsText := AOAIImagesImpl.PrepareUserMessageContentFromMediaSet(UserText, MediaSetId, DetailLevel); + if ContentAsText = '' then + exit; + + AddMessage(ContentAsText, '', Enum::"AOAI Chat Roles"::User); + end; + + [NonDebuggable] + procedure AddUserMessage(UserText: Text; TenantMedia: Record "Tenant Media"; DetailLevel: Enum "AOAI Image Detail Level") + var + AOAIImagesImpl: Codeunit "AOAI Images Impl"; + ContentAsText: Text; + begin + Initialize(); + ContentAsText := AOAIImagesImpl.PrepareUserMessageContentFromMediaRecord(UserText, TenantMedia, DetailLevel); + if ContentAsText = '' then + exit; + + AddMessage(ContentAsText, '', Enum::"AOAI Chat Roles"::User); + end; + + [NonDebuggable] + procedure AddUserMessage(UserText: Text; var TempBlob: Codeunit "Temp Blob"; FileExtension: Text; DetailLevel: Enum "AOAI Image Detail Level") + var + AOAIImagesImpl: Codeunit "AOAI Images Impl"; + ContentAsText: Text; + begin + Initialize(); + ContentAsText := AOAIImagesImpl.PrepareUserMessageContentFromTempBlob(UserText, TempBlob, FileExtension, DetailLevel); + if ContentAsText = '' then + exit; + + AddMessage(ContentAsText, '', Enum::"AOAI Chat Roles"::User); + end; + [NonDebuggable] procedure AddAssistantMessage(NewMessage: Text) begin @@ -239,10 +310,11 @@ codeunit 7764 "AOAI Chat Messages Impl" Message := WrapUserMessages(AzureOpenAIImpl.RemoveProhibitedCharacters(Message)) else Message := AzureOpenAIImpl.RemoveProhibitedCharacters(Message); + if ToolCalls.Count() > 0 then MessageJsonObject.Add('tool_calls', ToolCalls) else - MessageJsonObject.Add('content', Message); + AddContentToMessage(Message, MessageJsonObject); if Name <> '' then MessageJsonObject.Add('name', Name); @@ -258,6 +330,38 @@ codeunit 7764 "AOAI Chat Messages Impl" MessagesTokenCount := AOAIToken.GetGPT4TokenCount(TotalMessages); end; + local procedure AddContentToMessage(Message: Text; var MessageJsonObject: JsonObject) + begin + AddTextContentToMessage(Message, MessageJsonObject); + AddImageContentToMessage(Message, MessageJsonObject); + end; + + local procedure AddTextContentToMessage(Message: Text; var MessageJsonObject: JsonObject) + begin + if CheckIfImageContent(Message) then + exit; + + MessageJsonObject.Add('content', Message); + end; + + local procedure AddImageContentToMessage(Message: Text; var MessageJsonObject: JsonObject) + var + AOAIImagesImpl: Codeunit "AOAI Images Impl"; + begin + if not CheckIfImageContent(Message) then + exit; + + MessageJsonObject.Add('content', AOAIImagesImpl.ReadImageContent(Message)); + end; + + local procedure CheckIfImageContent(Message: Text): Boolean + var + AOAIImagesImpl: Codeunit "AOAI Images Impl"; + begin + exit(AOAIImagesImpl.CheckIfImageContent(Message)); + end; + + local procedure Initialize() begin if Initialized then diff --git a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIImageDetailLevel.Enum.al b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIImageDetailLevel.Enum.al new file mode 100644 index 0000000000..d6a40f77a2 --- /dev/null +++ b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIImageDetailLevel.Enum.al @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.AI; + +/// +/// Specifies the detail level for image processing in vision-enabled models. +/// +enum 7770 "AOAI Image Detail Level" +{ + Extensible = true; + Access = Public; + + value(0; low) + { + Caption = 'Low'; + } + value(1; high) + { + Caption = 'High'; + } + value(2; auto) + { + Caption = 'Auto'; + } +} \ No newline at end of file diff --git a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Images/AOAIImagesImpl.Codeunit.al b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Images/AOAIImagesImpl.Codeunit.al new file mode 100644 index 0000000000..cca8259ec3 --- /dev/null +++ b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Images/AOAIImagesImpl.Codeunit.al @@ -0,0 +1,245 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.AI; + +using System.Utilities; +using System.Text; +using System.Environment; + +codeunit 7783 "AOAI Images Impl" +{ + Access = Internal; + InherentEntitlements = X; + InherentPermissions = X; + + var + FileExtensionToMimeTypeMappingInitialized: Boolean; + FileExtensionToMimeTypeMapping: Dictionary of [Text, Text]; + MimeTypeFromExtensionNotResolvedErr: Label 'Could not resolve Mime type for the provided file extension %1.', Comment = '%1 = file extension'; + DataUrlFormatLbl: Label 'data:%1;base64,%2', Comment = '%1 = mime type, %2 = base64 encoded data', Locked = true; + + internal procedure PrepareUserMessageContentFromTempBlob(UserText: Text; var TempBlob: Codeunit "Temp Blob"; FileExtension: Text; DetailLevel: Enum "AOAI Image Detail Level") ContentAsText: Text + var + ImageStream: InStream; + begin + if not TempBlob.HasValue() then + exit(''); + + if FileExtension = '' then + exit(''); + + TempBlob.CreateInStream(ImageStream); + exit(PrepareUserMessageContentFromStream(UserText, ImageStream, FileExtension, DetailLevel)); + end; + + internal procedure PrepareUserMessageContentFromStream(UserText: Text; ImageStream: InStream; FileExtension: Text; DetailLevel: Enum "AOAI Image Detail Level") ContentAsText: Text + var + MimeType: Text; + begin + if FileExtension = '' then + exit(''); + + MimeType := ConvertFileExtensionToMimeType(FileExtension); + if MimeType = '' then + Error(MimeTypeFromExtensionNotResolvedErr, FileExtension); + + exit(PrepareUserMessageContentFromStreamAndMimeType(UserText, ImageStream, MimeType, DetailLevel)); + end; + + internal procedure PrepareUserMessageContentFromMediaSet(UserText: Text; MediaSetId: Guid; DetailLevel: Enum "AOAI Image Detail Level") ContentAsText: Text + var + TenantMediaSet: Record "Tenant Media Set"; + TenantMedia: Record "Tenant Media"; + ImageStream: InStream; + ContentArray: JsonArray; + MaxImages: Integer; + ImageCount: Integer; + begin + if IsNullGuid(MediaSetId) then + exit(''); + + if not TenantMediaSet.Get(MediaSetId) then + exit(''); + + MaxImages := 10; + ImageCount := 0; + + AddTextPart(UserText, ContentArray); + + TenantMediaSet.SetRange(ID, MediaSetId); + if TenantMediaSet.IsEmpty() then + exit(''); + + if TenantMediaSet.FindSet() then + repeat + TenantMedia.Get(TenantMediaSet."Media ID".MediaId()); + TenantMedia.CalcFields(Content); + TenantMedia.Content.CreateInStream(ImageStream); + if ImageCount < MaxImages then begin + AddImagePart(GetBase64EncodedUrl(ImageStream, TenantMedia."Mime Type"), DetailLevel, ContentArray); + ImageCount := ImageCount + 1; + end; + until TenantMediaSet.Next() = 0; + + if ImageCount = 0 then + exit(''); + + ContentArray.WriteTo(ContentAsText); + exit(ContentAsText); + end; + + + internal procedure PrepareUserMessageContentFromMediaRecord(UserText: Text; TenantMedia: Record "Tenant Media"; DetailLevel: Enum "AOAI Image Detail Level") ContentAsText: Text + var + ImageStream: InStream; + begin + TenantMedia.CalcFields(Content); + if not TenantMedia.Content.HasValue then + exit(''); + + TenantMedia.Content.CreateInStream(ImageStream); + exit(PrepareUserMessageContentFromStreamAndMimeType(UserText, ImageStream, TenantMedia."Mime Type", DetailLevel)); + end; + + internal procedure PrepareUserMessageContentFromUrl(UserText: Text; ImageUrl: Text; DetailLevel: Enum "AOAI Image Detail Level") ContentAsText: Text + var + ContentArray: JsonArray; + begin + if ImageUrl = '' then + exit(''); + + AddTextPart(UserText, ContentArray); + AddImagePart(ImageUrl, DetailLevel, ContentArray); + + ContentArray.WriteTo(ContentAsText); + exit(ContentAsText); + end; + + local procedure PrepareUserMessageContentFromStreamAndMimeType(UserText: Text; ImageStream: InStream; MimeType: Text; DetailLevel: Enum "AOAI Image Detail Level") ContentAsText: Text + var + ContentArray: JsonArray; + DataUrl: Text; + begin + if MimeType = '' then + exit(''); + + DataUrl := GetBase64EncodedUrl(ImageStream, MimeType); + if DataUrl = '' then + exit(''); + + AddTextPart(UserText, ContentArray); + AddImagePart(DataUrl, DetailLevel, ContentArray); + + ContentArray.WriteTo(ContentAsText); + exit(ContentAsText); + end; + + local procedure AddTextPart(UserText: Text; var ContentArray: JsonArray) + var + TextObject: JsonObject; + begin + if UserText = '' then + exit; + + TextObject.Add('type', 'text'); + TextObject.Add('text', UserText); + ContentArray.Add(TextObject); + end; + + local procedure AddImagePart(ImageUrl: Text; DetailLevel: Enum "AOAI Image Detail Level"; var ContentArray: JsonArray) + var + ImageJsonObj: JsonObject; + UrlJsonObj: JsonObject; + begin + if ImageUrl = '' then + exit; + + UrlJsonObj.Add('url', ImageUrl); + if DetailLevel <> DetailLevel::auto then // Only add detail if not 'auto' + UrlJsonObj.Add('detail', Format(DetailLevel)); + + ImageJsonObj.Add('type', 'image_url'); + ImageJsonObj.Add('image_url', UrlJsonObj); + ContentArray.Add(ImageJsonObj); + end; + + local procedure GetBase64EncodedUrl(ImageStream: InStream; MimeType: Text): Text + var + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + Base64EncodedData: Text; + BlobOutStream: OutStream; + BlobInStream: InStream; + begin + if MimeType = '' then + exit(''); + + // Copy AL InStream to Temp Blob + TempBlob.CreateOutStream(BlobOutStream); + CopyStream(BlobOutStream, ImageStream); + + // Read from Temp Blob and Base64 Encode + TempBlob.CreateInStream(BlobInStream); + Base64EncodedData := Base64Convert.ToBase64(BlobInStream); + + // Construct the data URL + exit(StrSubstNo(DataUrlFormatLbl, MimeType, Base64EncodedData)); + end; + + local procedure ConvertFileExtensionToMimeType(FileExtension: Text) MimeType: Text + begin + InitializeFileExtensionToMimeTypeMapping(); + if not FileExtensionToMimeTypeMapping.Get(LowerCase(FileExtension), MimeType) then + exit(''); + + exit(MimeType); + end; + + local procedure InitializeFileExtensionToMimeTypeMapping() + begin + if FileExtensionToMimeTypeMappingInitialized then + exit; + + // Initialize default mappings + FileExtensionToMimeTypeMapping.Add('jpg', 'image/jpeg'); + FileExtensionToMimeTypeMapping.Add('jpeg', 'image/jpeg'); + FileExtensionToMimeTypeMapping.Add('png', 'image/png'); + FileExtensionToMimeTypeMapping.Add('gif', 'image/gif'); + FileExtensionToMimeTypeMapping.Add('bmp', 'image/bmp'); + FileExtensionToMimeTypeMapping.Add('webp', 'image/webp'); + + OnAfterInitializeFileExtensionToMimeTypeMapping(FileExtensionToMimeTypeMapping); + + FileExtensionToMimeTypeMappingInitialized := true; + end; + + internal procedure CheckIfImageContent(Message: Text): Boolean + var + ContentJArray: JsonArray; + begin + if not ContentJArray.ReadFrom(Message) then + exit(false); + + exit(ContentJArray.Count() > 0); + end; + + internal procedure ReadImageContent(Message: Text): JsonArray + var + ContentJArray: JsonArray; + DummyJsonArray: JsonArray; + begin + if not ContentJArray.ReadFrom(Message) then + exit(DummyJsonArray); + + exit(ContentJArray); + end; + + [IntegrationEvent(false, false)] + internal procedure OnAfterInitializeFileExtensionToMimeTypeMapping(var FileExtensionToMimeTypeMap: Dictionary of [Text, Text]) + begin + end; + +} \ No newline at end of file From 20ed36ca50387996143d33b514fa51992abc27f0 Mon Sep 17 00:00:00 2001 From: Dmitry Katson Date: Sun, 4 May 2025 18:16:46 +0600 Subject: [PATCH 2/2] Created AzureOpenAIVisionTest.Codeunit.al for comprehensive testing of image handling features. --- .../{ => Vision}/AOAIImageDetailLevel.Enum.al | 6 +- .../AOAIImagesImpl.Codeunit.al | 7 +- src/System Application/Test/AI/app.json | 18 + .../AI/src/AzureOpenAIVisionTest.Codeunit.al | 697 ++++++++++++++++++ 4 files changed, 720 insertions(+), 8 deletions(-) rename src/System Application/App/AI/src/Azure OpenAI/Chat Completion/{ => Vision}/AOAIImageDetailLevel.Enum.al (89%) rename src/System Application/App/AI/src/Azure OpenAI/Chat Completion/{Images => Vision}/AOAIImagesImpl.Codeunit.al (99%) create mode 100644 src/System Application/Test/AI/src/AzureOpenAIVisionTest.Codeunit.al diff --git a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIImageDetailLevel.Enum.al b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Vision/AOAIImageDetailLevel.Enum.al similarity index 89% rename from src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIImageDetailLevel.Enum.al rename to src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Vision/AOAIImageDetailLevel.Enum.al index d6a40f77a2..6177088eea 100644 --- a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/AOAIImageDetailLevel.Enum.al +++ b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Vision/AOAIImageDetailLevel.Enum.al @@ -15,14 +15,14 @@ enum 7770 "AOAI Image Detail Level" value(0; low) { - Caption = 'Low'; + Caption = 'low'; } value(1; high) { - Caption = 'High'; + Caption = 'high'; } value(2; auto) { - Caption = 'Auto'; + Caption = 'auto'; } } \ No newline at end of file diff --git a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Images/AOAIImagesImpl.Codeunit.al b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Vision/AOAIImagesImpl.Codeunit.al similarity index 99% rename from src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Images/AOAIImagesImpl.Codeunit.al rename to src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Vision/AOAIImagesImpl.Codeunit.al index cca8259ec3..f812f65acd 100644 --- a/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Images/AOAIImagesImpl.Codeunit.al +++ b/src/System Application/App/AI/src/Azure OpenAI/Chat Completion/Vision/AOAIImagesImpl.Codeunit.al @@ -61,7 +61,8 @@ codeunit 7783 "AOAI Images Impl" if IsNullGuid(MediaSetId) then exit(''); - if not TenantMediaSet.Get(MediaSetId) then + TenantMediaSet.SetRange(ID, MediaSetId); + if TenantMediaSet.IsEmpty() then exit(''); MaxImages := 10; @@ -69,10 +70,6 @@ codeunit 7783 "AOAI Images Impl" AddTextPart(UserText, ContentArray); - TenantMediaSet.SetRange(ID, MediaSetId); - if TenantMediaSet.IsEmpty() then - exit(''); - if TenantMediaSet.FindSet() then repeat TenantMedia.Get(TenantMediaSet."Media ID".MediaId()); diff --git a/src/System Application/Test/AI/app.json b/src/System Application/Test/AI/app.json index 590d5c7ea5..8a3004a944 100644 --- a/src/System Application/Test/AI/app.json +++ b/src/System Application/Test/AI/app.json @@ -46,6 +46,24 @@ "name": "Environment Information Test Library", "publisher": "Microsoft", "version": "27.0.0.0" + }, + { + "id": "e31ad830-3d46-472e-afeb-1d3d35247943", + "name": "BLOB Storage", + "publisher": "Microsoft", + "version": "27.0.0.0" + }, + { + "id": "0846d207-5dec-4c1b-afd8-6a25e1e14b9d", + "name": "Base64 Convert", + "publisher": "Microsoft", + "version": "27.0.0.0" + }, + { + "id": "6d4dab82-dc5c-4c77-8e1e-d9aaeb7d0420", + "name": "Data Administration Test", + "publisher": "Microsoft", + "version": "27.0.0.0" } ], "screenshots": [], diff --git a/src/System Application/Test/AI/src/AzureOpenAIVisionTest.Codeunit.al b/src/System Application/Test/AI/src/AzureOpenAIVisionTest.Codeunit.al new file mode 100644 index 0000000000..f4c7b222fd --- /dev/null +++ b/src/System Application/Test/AI/src/AzureOpenAIVisionTest.Codeunit.al @@ -0,0 +1,697 @@ +namespace System.Test.AI; + +using System.AI; +using System.Utilities; +using System.TestLibraries.AI; +using System.Test.DataAdministration; +using System.TestLibraries.Utilities; +using System.Environment; +using System.Text; + +codeunit 132691 "Azure OpenAI Vision Test" +{ + Subtype = Test; + + var + LibraryAssert: Codeunit "Library Assert"; + + [Test] + procedure TestAddUserMessageWithImageUrl() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + UserText: Text; + ImageUrl: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + ContentItem: JsonToken; + ContentItemType: JsonToken; + ImageUrlJsonTok: JsonToken; + ImageUrlObj: JsonToken; + DetailJsonTok: JsonToken; + begin + // [SCENARIO] AddUserMessage with text and image URL adds a message with proper JSON structure + + // [GIVEN] A user text and image URL + UserText := 'Describe this image'; + ImageUrl := 'https://example.com/image.jpg'; + + // [WHEN] Adding a user message with an image URL + AOAIChatMessages.AddUserMessage(UserText, ImageUrl, Enum::"AOAI Image Detail Level"::high); + + // [THEN] The history contains one message + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains both text and image parts in proper format + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + LibraryAssert.IsTrue(MessageJsonTok.AsObject().Get('content', ContentJsonTok), 'Content should exist in the message'); + + // Verify content is an array with 2 items (text and image) + LibraryAssert.IsTrue(ContentJsonTok.IsArray(), 'Content should be an array'); + LibraryAssert.AreEqual(2, ContentJsonTok.AsArray().Count(), 'Content should have 2 items (text and image)'); + + // Check first item is text + ContentJsonTok.AsArray().Get(0, ContentItem); + LibraryAssert.IsTrue(ContentItem.AsObject().Get('type', ContentItemType), 'Content item should have type'); + LibraryAssert.AreEqual('text', ContentItemType.AsValue().AsText(), 'First content item should be text'); + LibraryAssert.IsTrue(ContentItem.AsObject().Get('text', ContentItemType), 'Content item should have text'); + LibraryAssert.AreEqual(UserText, ContentItemType.AsValue().AsText(), 'Text content should match input'); + + // Check second item is image URL + ContentJsonTok.AsArray().Get(1, ContentItem); + LibraryAssert.IsTrue(ContentItem.AsObject().Get('type', ContentItemType), 'Content item should have type'); + LibraryAssert.AreEqual('image_url', ContentItemType.AsValue().AsText(), 'Second content item should be image_url'); + LibraryAssert.IsTrue(ContentItem.AsObject().Get('image_url', ImageUrlJsonTok), 'Content item should have image_url'); + LibraryAssert.IsTrue(ImageUrlJsonTok.AsObject().Get('url', ImageUrlObj), 'image_url should have url'); + LibraryAssert.AreEqual(ImageUrl, ImageUrlObj.AsValue().AsText(), 'URL should match input'); + + // Check detail level is properly set + LibraryAssert.IsTrue(ImageUrlJsonTok.AsObject().Get('detail', DetailJsonTok), 'image_url should have detail'); + LibraryAssert.AreEqual('high', DetailJsonTok.AsValue().AsText(), 'Detail should be set to high'); + end; + + [Test] + procedure TestAddUserMessageWithImageUrlAutoDetail() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + UserText: Text; + ImageUrl: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + ContentItem: JsonToken; + ImageUrlJsonTok: JsonToken; + DetailJsonTok: JsonToken; + begin + // [SCENARIO] AddUserMessage with auto detail level doesn't include detail in the JSON + + // [GIVEN] A user text and image URL + UserText := 'Describe this image'; + ImageUrl := 'https://example.com/image.jpg'; + + // [WHEN] Adding a user message with an image URL and auto detail level + AOAIChatMessages.AddUserMessage(UserText, ImageUrl, Enum::"AOAI Image Detail Level"::auto); + + // [THEN] The message JSON doesn't include a detail field + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + ContentJsonTok.AsArray().Get(1, ContentItem); + ContentItem.AsObject().Get('image_url', ImageUrlJsonTok); + LibraryAssert.IsFalse(ImageUrlJsonTok.AsObject().Get('detail', DetailJsonTok), 'Detail should not be present when set to auto'); + end; + + [Test] + procedure TestAddUserMessageWithImageStreamFromTempBlob() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + InStream: InStream; + UserText: Text; + FileExtension: Text; + ImageData: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + ContentItem: JsonToken; + ContentItemType: JsonToken; + ImageUrlJsonTok: JsonToken; + ImageUrlObj: JsonToken; + begin + // [SCENARIO] AddUserMessage with image from stream creates a data URL in the proper format + + // [GIVEN] A user text and image data in a TempBlob + UserText := 'Describe this image'; + FileExtension := 'png'; + + // Create a very small test image data + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + TempBlob.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlob.CreateInStream(InStream); + + // [WHEN] Adding a user message with an image from temp blob + AOAIChatMessages.AddUserMessage(UserText, TempBlob, FileExtension, Enum::"AOAI Image Detail Level"::low); + + // [THEN] The history contains one message with a proper data URL + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains both text and image parts in proper format + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + + // Verify content has 2 items (text and image) + LibraryAssert.AreEqual(2, ContentJsonTok.AsArray().Count(), 'Content should have 2 items (text and image)'); + + // Check second item is image data URL + ContentJsonTok.AsArray().Get(1, ContentItem); + ContentItem.AsObject().Get('type', ContentItemType); + LibraryAssert.AreEqual('image_url', ContentItemType.AsValue().AsText(), 'Second content item should be image_url'); + ContentItem.AsObject().Get('image_url', ImageUrlJsonTok); + ImageUrlJsonTok.AsObject().Get('url', ImageUrlObj); + + // Check URL starts with data:image/png;base64 + LibraryAssert.IsTrue(ImageUrlObj.AsValue().AsText().StartsWith('data:image/png;base64,'), 'URL should be a PNG data URL'); + end; + + [Test] + procedure TestAddUserMessageWithImageStream() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + TempBlobStream: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + InStream: InStream; + UserText: Text; + FileExtension: Text; + ImageData: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + ContentItem: JsonToken; + ImageUrlJsonTok: JsonToken; + ImageUrlObj: JsonToken; + begin + // [SCENARIO] AddUserMessage with an image stream creates a data URL in the proper format + + // [GIVEN] A user text and image data in a Stream + UserText := 'Describe this image'; + FileExtension := 'jpg'; + + // Create a very small test image data + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + TempBlobStream.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlobStream.CreateInStream(InStream); + + // [WHEN] Adding a user message with an image stream + AOAIChatMessages.AddUserMessage(UserText, InStream, FileExtension, Enum::"AOAI Image Detail Level"::high); + + // [THEN] The history contains one message with a proper data URL + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains both text and image parts in proper format + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + + // Verify content has 2 items (text and image) + LibraryAssert.AreEqual(2, ContentJsonTok.AsArray().Count(), 'Content should have 2 items (text and image)'); + + // Check second item is image data URL and starts with data:image/jpeg;base64 + ContentJsonTok.AsArray().Get(1, ContentItem); + ContentItem.AsObject().Get('image_url', ImageUrlJsonTok); + ImageUrlJsonTok.AsObject().Get('url', ImageUrlObj); + LibraryAssert.IsTrue(ImageUrlObj.AsValue().AsText().StartsWith('data:image/jpeg;base64,'), 'URL should be a JPEG data URL'); + end; + + [Test] + procedure TestAddUserMessageWithTenantMedia() + var + TenantMedia: Record "Tenant Media"; + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + BlobOutStream: OutStream; + InStream: InStream; + UserText: Text; + ImageData: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + begin + // [SCENARIO] AddUserMessage with TenantMedia adds a message with a proper data URL + + // [GIVEN] A user text and TenantMedia record with image content + UserText := 'Describe this image'; + + // Create test image data + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + TempBlob.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlob.CreateInStream(InStream); + + // Create a TenantMedia record with the image + TenantMedia.Init(); + TenantMedia."Mime Type" := 'image/png'; + TenantMedia.ID := CreateGuid(); + TenantMedia.Insert(); + + // Update the TenantMedia.Content with test image data + TenantMedia.Content.CreateOutStream(BlobOutStream); + CopyStream(BlobOutStream, InStream); + TenantMedia.Modify(); + + // [WHEN] Adding a user message with the TenantMedia + AOAIChatMessages.AddUserMessage(UserText, TenantMedia, Enum::"AOAI Image Detail Level"::auto); + + // [THEN] The history contains one message with a proper data URL + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains content in JSON format + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + LibraryAssert.IsTrue(MessageJsonTok.AsObject().Get('content', ContentJsonTok), 'Content should exist in the message'); + LibraryAssert.IsTrue(ContentJsonTok.IsArray(), 'Content should be an array'); + end; + + [Test] + procedure TestAddUserMessageWithMediaSet() + var + TestMediaCleanup: Record "Test Media Cleanup"; + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + InStream: InStream; + UserText: Text; + ImageData: Text; + MediaSetId: Guid; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + begin + // [SCENARIO] AddUserMessage with MediaSet adds a message with all images as data URLs + + // [GIVEN] A user text and a Test Media Cleanup record with MediaSet content + UserText := 'Describe these images'; + + // Create a Test Media Cleanup record + TestMediaCleanup.Init(); + TestMediaCleanup."Primary Key" := 1; + TestMediaCleanup.Insert(); + + // Create image data and upload to MediaSet field + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + TempBlob.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlob.CreateInStream(InStream); + + // Import stream into the Test Media Cleanup record's MediaSet field + TestMediaCleanup."Test Media Set".ImportStream(InStream, 'Test Image'); + TestMediaCleanup.Modify(); + + // Get the MediaSet ID + MediaSetId := TestMediaCleanup."Test Media Set".MediaId; + + // [WHEN] Adding a user message with the MediaSet ID + AOAIChatMessages.AddUserMessage(UserText, MediaSetId, Enum::"AOAI Image Detail Level"::low); + + // [THEN] The history contains one message with a proper data URL + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains content in JSON format + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + LibraryAssert.IsTrue(MessageJsonTok.AsObject().Get('content', ContentJsonTok), 'Content should exist in the message'); + LibraryAssert.IsTrue(ContentJsonTok.IsArray(), 'Content should be an array'); + end; + + [Test] + procedure TestAddUserMessageWithMultipleImagesMediaSet() + var + TestMediaCleanup: Record "Test Media Cleanup"; + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + UserText: Text; + MediaSetId: Guid; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + ImageCount: Integer; + begin + // [SCENARIO] AddUserMessage with MediaSet containing 5 images adds a message with all images as data URLs + + // [GIVEN] A user text and a Test Media Cleanup record with multiple images in MediaSet + UserText := 'Describe these 5 images'; + ImageCount := 5; + + // Create a Test Media Cleanup record + TestMediaCleanup.Init(); + TestMediaCleanup."Primary Key" := 2; + TestMediaCleanup.Insert(); + + // Add images to the MediaSet + AddImagesToMediaSet(TestMediaCleanup, ImageCount); + + // Get the MediaSet ID + MediaSetId := TestMediaCleanup."Test Media Set".MediaId; + + // [WHEN] Adding a user message with the MediaSet ID containing 5 images + AOAIChatMessages.AddUserMessage(UserText, MediaSetId, Enum::"AOAI Image Detail Level"::low); + + // [THEN] The history contains one message + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains content in JSON format with all 5 images + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + + // Content should be an array with 6 items (1 text + 5 images) + LibraryAssert.IsTrue(ContentJsonTok.IsArray(), 'Content should be an array'); + LibraryAssert.AreEqual(ImageCount + 1, ContentJsonTok.AsArray().Count(), 'Content should have 6 items (1 text + 5 images)'); + end; + + [Test] + procedure TestAddUserMessageWithMaxImagesMediaSet() + var + TestMediaCleanup: Record "Test Media Cleanup"; + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + UserText: Text; + MediaSetId: Guid; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + TotalImageCount: Integer; + MaxImagesAllowed: Integer; + begin + // [SCENARIO] AddUserMessage with MediaSet containing 11 images only includes the first 10 in the message + + // [GIVEN] A user text and a Test Media Cleanup record with 11 images in MediaSet + UserText := 'Describe these images'; + TotalImageCount := 11; + MaxImagesAllowed := 10; // Assuming a limit of 10 images per message + + // Create a Test Media Cleanup record + TestMediaCleanup.Init(); + TestMediaCleanup."Primary Key" := 3; + TestMediaCleanup.Insert(); + + // Add 11 images to the MediaSet + AddImagesToMediaSet(TestMediaCleanup, TotalImageCount); + + // Get the MediaSet ID + MediaSetId := TestMediaCleanup."Test Media Set".MediaId; + + // [WHEN] Adding a user message with the MediaSet ID containing 11 images + AOAIChatMessages.AddUserMessage(UserText, MediaSetId, Enum::"AOAI Image Detail Level"::low); + + // [THEN] The history contains one message + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains content in JSON format with max 10 images + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + + // Content should be an array with 11 items (1 text + 10 images max) + LibraryAssert.IsTrue(ContentJsonTok.IsArray(), 'Content should be an array'); + LibraryAssert.AreEqual(MaxImagesAllowed + 1, ContentJsonTok.AsArray().Count(), 'Content should have 11 items (1 text + 10 images)'); + end; + + [Test] + procedure TestAddUserMessageWithOnlyText() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + UserText: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + begin + // [SCENARIO] Regular AddUserMessage with just text doesn't create a JSON array structure + + // [GIVEN] A user text + UserText := 'This is a regular text message'; + + // [WHEN] Adding a regular user message + AOAIChatMessages.AddUserMessage(UserText); + + // [THEN] The history contains one message with just text content + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The content is a plain text string, not a JSON array + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + + LibraryAssert.IsTrue(ContentJsonTok.IsValue(), 'Content should be a value, not an array'); + LibraryAssert.AreEqual(UserText, ContentJsonTok.AsValue().AsText(), 'Content should match input text'); + end; + + [Test] + procedure TestAddUserMessageWithEmptyImage() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + UserText: Text; + EmptyUrl: Text; + begin + // [SCENARIO] AddUserMessage with empty image URL doesn't add a message + + // [GIVEN] A user text and empty image URL + UserText := 'Describe this image'; + EmptyUrl := ''; + + // [WHEN] Adding a user message with an empty image URL + AOAIChatMessages.AddUserMessage(UserText, EmptyUrl, Enum::"AOAI Image Detail Level"::high); + + // [THEN] The history should be empty + LibraryAssert.AreEqual(0, AOAIChatMessages.GetHistory().Count, 'The history should be empty when image URL is empty'); + end; + + [Test] + procedure TestAddUserMessageWithEmptyTextAndValidImage() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + EmptyText: Text; + ImageUrl: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + begin + // [SCENARIO] AddUserMessage with empty text but valid image URL adds a message with just the image + + // [GIVEN] An empty user text and valid image URL + EmptyText := ''; + ImageUrl := 'https://example.com/image.jpg'; + + // [WHEN] Adding a user message with empty text and valid image URL + AOAIChatMessages.AddUserMessage(EmptyText, ImageUrl, Enum::"AOAI Image Detail Level"::high); + + // [THEN] The history contains one message + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains only the image part (not text part) + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + + LibraryAssert.IsTrue(ContentJsonTok.IsArray(), 'Content should be an array'); + LibraryAssert.AreEqual(1, ContentJsonTok.AsArray().Count(), 'Content should have only 1 item (image)'); + end; + + [Test] + procedure TestMultipleImagesInChatSession() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + UserText1: Text; + UserText2: Text; + ImageUrl1: Text; + ImageUrl2: Text; + AssistantResponse: Text; + HistoryJsonArray: JsonArray; + begin + // [SCENARIO] Multiple images can be added in a chat session + + // [GIVEN] User texts and image URLs + UserText1 := 'What is in this image?'; + ImageUrl1 := 'https://example.com/image1.jpg'; + UserText2 := 'And what is in this one?'; + ImageUrl2 := 'https://example.com/image2.jpg'; + AssistantResponse := 'The first image shows a cat.'; + + // [WHEN] Adding multiple messages with images to the chat + AOAIChatMessages.AddUserMessage(UserText1, ImageUrl1, Enum::"AOAI Image Detail Level"::auto); + AOAIChatMessages.AddAssistantMessage(AssistantResponse); + AOAIChatMessages.AddUserMessage(UserText2, ImageUrl2, Enum::"AOAI Image Detail Level"::auto); + + // [THEN] The history contains three messages + LibraryAssert.AreEqual(3, AOAIChatMessages.GetHistory().Count, 'The history should contain three messages'); + + // [THEN] The JSON for the chat history is correct + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(3, AOAIChatMessages); + LibraryAssert.AreEqual(3, HistoryJsonArray.Count, 'The JSON history should contain three messages'); + end; + + [Test] + procedure TestUnsupportedFileExtension() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + InStream: InStream; + UserText: Text; + InvalidFileExtension: Text; + ImageData: Text; + begin + // [SCENARIO] Unsupported file extension should fail + + // [GIVEN] A user text and image data in a Stream with invalid extension + UserText := 'Describe this image'; + InvalidFileExtension := 'xyz'; // Unsupported extension + + // Create a test image data + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + TempBlob.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlob.CreateInStream(InStream); + + // [WHEN] Adding a user message with invalid file extension + asserterror AOAIChatMessages.AddUserMessage(UserText, InStream, InvalidFileExtension, Enum::"AOAI Image Detail Level"::high); + + // [THEN] An error should be raised + LibraryAssert.ExpectedError('Could not resolve Mime type for the provided file extension xyz.'); + end; + + [Test] + procedure TestSimulatedEndToEndVisionSession() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + InStream: InStream; + ImageStream: InStream; + UserText: Text; + FileExtension: Text; + ImageUrl: Text; + ImageData: Text; + HistoryJsonArray: JsonArray; + MockAIResponse: Text; + begin + // [SCENARIO] Simulating an end-to-end vision chat flow with both text and image inputs + + // [GIVEN] Test data for the vision session + UserText := 'What is in this image?'; + ImageUrl := 'https://example.com/image.jpg'; + FileExtension := 'png'; + + // Create a test image + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + TempBlob.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlob.CreateInStream(InStream); + ImageStream := InStream; + + // Mock AI setup (we won't actually call the API) + MockAIResponse := 'I can see a simple 1x1 pixel PNG image with a transparent background.'; + + // [WHEN] Setting up and using the vision capabilities + // Setup system message + AOAIChatMessages.AddSystemMessage('You are an AI image description assistant.'); + + // Add text and images in different ways + AOAIChatMessages.AddUserMessage(UserText); + AOAIChatMessages.AddUserMessage('', ImageStream, FileExtension, Enum::"AOAI Image Detail Level"::auto); + AOAIChatMessages.AddUserMessage('', ImageUrl, Enum::"AOAI Image Detail Level"::auto); + + // Simulate AI response (without actually calling Azure OpenAI) + AOAIChatMessages.AddAssistantMessage(MockAIResponse); + + // [THEN] Verify the chat history is constructed correctly + LibraryAssert.AreEqual(5, AOAIChatMessages.GetHistory().Count, 'The history should contain five messages'); + + // Check the content of the history + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(5, AOAIChatMessages); + LibraryAssert.AreEqual(5, HistoryJsonArray.Count(), 'The JSON history array should have 5 items'); + + // Check the last message (AI response) + LibraryAssert.AreEqual(MockAIResponse, AOAIChatMessages.GetLastMessage(), 'The last message should be the AI response'); + end; + + [Test] + procedure TestTextAndImageInOneMessage() + var + AOAIChatMessages: Codeunit "AOAI Chat Messages"; + AzureOpenAITestLibrary: Codeunit "Azure OpenAI Test Library"; + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + InStream: InStream; + UserText: Text; + FileExtension: Text; + ImageData: Text; + HistoryJsonArray: JsonArray; + MessageJsonTok: JsonToken; + ContentJsonTok: JsonToken; + ContentItem: JsonToken; + ContentItemType: JsonToken; + TextContent: JsonToken; + begin + // [SCENARIO] Adding text and image in one message works correctly + + // [GIVEN] A user text and image data + UserText := 'Please describe this image in detail:'; + FileExtension := 'png'; + + // Create a test image + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + TempBlob.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlob.CreateInStream(InStream); + + // [WHEN] Adding a user message with both text and image + AOAIChatMessages.AddUserMessage(UserText, InStream, FileExtension, Enum::"AOAI Image Detail Level"::high); + + // [THEN] The history contains one message + LibraryAssert.AreEqual(1, AOAIChatMessages.GetHistory().Count, 'The history should contain one message'); + + // [THEN] The message contains both text and image parts in proper format + HistoryJsonArray := AzureOpenAITestLibrary.GetAOAIHistory(1, AOAIChatMessages); + HistoryJsonArray.Get(0, MessageJsonTok); + MessageJsonTok.AsObject().Get('content', ContentJsonTok); + + // Verify content is an array with 2 items (text and image) + LibraryAssert.IsTrue(ContentJsonTok.IsArray(), 'Content should be an array'); + LibraryAssert.AreEqual(2, ContentJsonTok.AsArray().Count(), 'Content should have 2 items (text and image)'); + + // Check first item is text with the correct content + ContentJsonTok.AsArray().Get(0, ContentItem); + ContentItem.AsObject().Get('type', ContentItemType); + LibraryAssert.AreEqual('text', ContentItemType.AsValue().AsText(), 'First content item should be text'); + ContentItem.AsObject().Get('text', TextContent); + LibraryAssert.AreEqual(UserText, TextContent.AsValue().AsText(), 'Text content should match input'); + end; + + local procedure AddImagesToMediaSet(var TestMediaCleanup: Record "Test Media Cleanup"; ImageCount: Integer) + var + TempBlob: Codeunit "Temp Blob"; + Base64Convert: Codeunit "Base64 Convert"; + OutStream: OutStream; + InStream: InStream; + ImageData: Text; + i: Integer; + begin + // Sample 1x1 pixel PNG image data + ImageData := 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + + for i := 1 to ImageCount do begin + Clear(TempBlob); + TempBlob.CreateOutStream(OutStream); + Base64Convert.FromBase64(ImageData, OutStream); + TempBlob.CreateInStream(InStream); + + // Import stream into the MediaSet field + TestMediaCleanup."Test Media Set".ImportStream(InStream, 'Test Image ' + Format(i)); + end; + + TestMediaCleanup.Modify(); + end; +} \ No newline at end of file