diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 9801ccc105..b5fe15bb98 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -464,6 +464,8 @@ + + diff --git a/dotnet/tests/AgentConversation.IntegrationTests/.gitignore b/dotnet/tests/AgentConversation.IntegrationTests/.gitignore new file mode 100644 index 0000000000..e660fd93d3 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/dotnet/tests/AgentConversation.IntegrationTests/AgentConversation.IntegrationTests.csproj b/dotnet/tests/AgentConversation.IntegrationTests/AgentConversation.IntegrationTests.csproj new file mode 100644 index 0000000000..ac38a0629f --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/AgentConversation.IntegrationTests.csproj @@ -0,0 +1,14 @@ + + + + false + true + true + true + + + + + + + diff --git a/dotnet/tests/AgentConversation.IntegrationTests/ConversationAgentDefinition.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationAgentDefinition.cs new file mode 100644 index 0000000000..d485d94cd9 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationAgentDefinition.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace AgentConversation.IntegrationTests; + +/// +/// Defines an agent participating in a . +/// +public sealed class ConversationAgentDefinition +{ + /// + /// Gets or sets the unique name identifying this agent within the test case. + /// + public required string Name { get; init; } + + /// + /// Gets or sets the system instructions for the agent. + /// + public string Instructions { get; init; } = "You are a helpful assistant."; + + /// + /// Gets or sets the optional list of tools available to the agent. + /// + public IList? Tools { get; init; } +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/ConversationContextSerializer.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationContextSerializer.cs new file mode 100644 index 0000000000..a5d5aba7c4 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationContextSerializer.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentConversation.IntegrationTests; + +/// +/// Provides helpers for serializing and deserializing conversation contexts (lists of ) +/// to and from JSON, enabling the initial context of a test case to be captured once and reused across runs. +/// +public static class ConversationContextSerializer +{ + private static readonly JsonSerializerOptions s_serializerOptions = AgentAbstractionsJsonUtilities.DefaultOptions; + + /// + /// Serializes a list of instances to a JSON string. + /// + /// The messages to serialize. + /// A JSON string representation of the messages. + public static string Serialize(IList messages) => + JsonSerializer.Serialize(messages, s_serializerOptions); + + /// + /// Deserializes a JSON string into a list of instances. + /// + /// The JSON string to deserialize. + /// The deserialized list of messages. + /// + /// Thrown when the JSON cannot be deserialized into a list of instances. + /// + public static IList Deserialize(string json) + { + var messages = JsonSerializer.Deserialize>(json, s_serializerOptions); + return messages ?? throw new InvalidOperationException("Failed to deserialize chat messages from the provided JSON."); + } + + /// + /// Saves a list of instances to a JSON file. + /// + /// The path of the file to write. + /// The messages to save. + public static void SaveToFile(string filePath, IList messages) + { + var json = Serialize(messages); + File.WriteAllText(filePath, json); + } + + /// + /// Loads a list of instances from a JSON file. + /// + /// The path of the file to read. + /// The deserialized list of messages. + /// Thrown when does not exist. + public static IList LoadFromFile(string filePath) + { + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"Conversation context file not found: '{filePath}'. " + + "Run the context creation step first to generate this file.", filePath); + } + + var json = File.ReadAllText(filePath); + return Deserialize(json); + } +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarness.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarness.cs new file mode 100644 index 0000000000..16a49a9b83 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarness.cs @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentConversation.IntegrationTests; + +/// +/// Orchestrates the execution of a against a given +/// : restores the conversation context, optionally runs compaction, +/// executes each step, captures before/after metrics, and runs per-step validations. +/// +public sealed class ConversationHarness +{ + private readonly IConversationTestSystem _system; + + /// + /// Initializes a new instance of . + /// + /// The system under test that provides agent creation and compaction. + public ConversationHarness(IConversationTestSystem system) + { + if (system is null) + { + throw new ArgumentNullException(nameof(system)); + } + + this._system = system; + } + + /// + /// Runs the supplied from its serialized initial context, executing + /// every in order and returning the combined metrics report. + /// + /// The test case to execute. + /// A token to cancel the operation. + /// + /// A describing the before-and-after state of the + /// conversation context across all steps. + /// + /// is . + /// + /// Thrown when a step references an agent name that is not present in . + /// + public async Task RunAsync( + IConversationTestCase testCase, + CancellationToken cancellationToken = default) + { + if (testCase is null) + { + throw new ArgumentNullException(nameof(testCase)); + } + + // 1. Restore the initial context. + var initialMessages = testCase.GetInitialMessages(); + + // 2. Capture "before" metrics. + var beforeMetrics = MeasureMetrics(initialMessages); + + // 3. Create the agents defined for this test case. + var agents = new Dictionary(StringComparer.Ordinal); + foreach (var entry in testCase.AgentDefinitions) + { + agents[entry.Key] = await this._system.CreateAgentAsync(entry.Value, cancellationToken).ConfigureAwait(false); + } + + // 4. Create sessions and restore the initial messages for each agent. + var sessions = new Dictionary(StringComparer.Ordinal); + foreach (var entry in agents) + { + var session = await entry.Value.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + RestoreMessages(entry.Value, session, initialMessages); + sessions[entry.Key] = session; + } + + // 5. Optionally compact the messages. + var compacted = await this._system.CompactAsync(initialMessages, cancellationToken).ConfigureAwait(false); + if (compacted is not null) + { + // Apply the compacted history to all agent sessions. + foreach (var entry in agents) + { + RestoreMessages(entry.Value, sessions[entry.Key], compacted); + } + } + + // 6. Execute each step. + foreach (var step in testCase.Steps) + { + if (!agents.TryGetValue(step.AgentName, out var agent)) + { + throw new InvalidOperationException( + $"Step references agent '{step.AgentName}' which is not defined in the test case. " + + $"Defined agents: {string.Join(", ", agents.Keys)}"); + } + + var session = sessions[step.AgentName]; + AgentResponse response; + + if (step.Input is not null) + { + response = await agent.RunAsync(step.Input, session, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + response = await agent.RunAsync(session, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + // 7. Capture "after" metrics for this step and run the step's validation. + var currentMessages = GetCurrentMessages(agent, sessions[step.AgentName], initialMessages, compacted); + var afterMetrics = MeasureMetrics(currentMessages); + var metricsReport = new ConversationMetricsReport + { + Before = beforeMetrics, + After = afterMetrics, + }; + + step.Validate?.Invoke(response, metricsReport); + } + + // 8. Capture the final "after" metrics from the first agent's session. + var firstAgent = agents.Values.First(); + var firstSession = sessions[agents.Keys.First()]; + var finalMessages = GetCurrentMessages(firstAgent, firstSession, initialMessages, compacted); + var finalAfterMetrics = MeasureMetrics(finalMessages); + + return new ConversationMetricsReport + { + Before = beforeMetrics, + After = finalAfterMetrics, + }; + } + + /// + /// Drives a conversation with the agents defined in to produce the initial + /// context, then serializes that context to . + /// + /// + /// This method should be called once (outside of normal test execution) to generate the fixture + /// data that tests will subsequently restore via . + /// + /// The test case whose initial context should be created. + /// The file path to write the serialized context to. + /// A token to cancel the operation. + public async Task SerializeInitialContextAsync( + IConversationTestCase testCase, + string outputFilePath, + CancellationToken cancellationToken = default) + { + if (testCase is null) + { + throw new ArgumentNullException(nameof(testCase)); + } + + if (string.IsNullOrEmpty(outputFilePath)) + { + throw new ArgumentException("Output file path must not be null or empty.", nameof(outputFilePath)); + } + + // Create agents for context generation. + var agents = new Dictionary(StringComparer.Ordinal); + foreach (var entry in testCase.AgentDefinitions) + { + agents[entry.Key] = await this._system.CreateAgentAsync(entry.Value, cancellationToken).ConfigureAwait(false); + } + + var messages = await testCase.CreateInitialContextAsync(agents, cancellationToken).ConfigureAwait(false); + ConversationContextSerializer.SaveToFile(outputFilePath, messages); + } + + // ------------------------------------------------------------------------- + // Private helpers + // ------------------------------------------------------------------------- + + private static ConversationMetrics MeasureMetrics(IList messages) + { + var serialized = ConversationContextSerializer.Serialize(messages); + return new ConversationMetrics + { + MessageCount = messages.Count, + SerializedSizeBytes = System.Text.Encoding.UTF8.GetByteCount(serialized), + }; + } + + private static void RestoreMessages(AIAgent agent, AgentSession session, IList messages) + { + // InMemoryChatHistoryProvider is the standard history provider for ChatClientAgent. + // When found, load the messages directly into the provider's state for this session. + if (agent.GetService() is InMemoryChatHistoryProvider memProvider) + { + memProvider.SetMessages(session, messages.ToList()); + } + } + + private static IList GetCurrentMessages( + AIAgent agent, + AgentSession session, + IList fallbackInitial, + IList? compacted) + { + if (agent.GetService() is InMemoryChatHistoryProvider memProvider) + { + return memProvider.GetMessages(session); + } + + // Fall back to the compacted (or original) initial messages when the provider is unavailable. + return compacted ?? fallbackInitial; + } +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarnessTests.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarnessTests.cs new file mode 100644 index 0000000000..5b43122d34 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationHarnessTests.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using Xunit.Abstractions; + +namespace AgentConversation.IntegrationTests; + +/// +/// Abstract xunit base class for agent conversation integration tests. +/// +/// +/// +/// Subclasses must implement and to provide +/// the AI backend and the set of test cases to run. Each subclass will automatically inherit the +/// test method, which runs every case returned by +/// through the . +/// +/// +/// To generate (and serialize) the initial context for a test case, the same subclass inherits +/// , which should be run once outside of normal CI. +/// +/// +/// +/// The concrete implementation that provides agent creation +/// and compaction for the system under test. +/// +public abstract class ConversationHarnessTests + where TSystem : IConversationTestSystem +{ + private readonly ITestOutputHelper? _output; + + /// + /// Initializes a new instance of . + /// + protected ConversationHarnessTests() + { + } + + /// + /// Initializes a new instance of with xunit test output. + /// + /// The xunit test output helper used to log metrics and step results. + protected ConversationHarnessTests(ITestOutputHelper output) + { + this._output = output; + } + + /// + /// Creates the to use for agent creation and compaction. + /// + protected abstract TSystem CreateTestSystem(); + + /// + /// Returns the set of instances to exercise. + /// + protected abstract IEnumerable GetTestCases(); + + /// + /// Runs all test cases returned by and logs the metrics report for each. + /// + [Fact] + public virtual async Task RunAllTestCasesAsync() + { + var system = this.CreateTestSystem(); + var harness = new ConversationHarness(system); + + foreach (var testCase in this.GetTestCases()) + { + this.Log($"[{testCase.Name}] Running..."); + var stopwatch = Stopwatch.StartNew(); + + var report = await harness.RunAsync(testCase); + + stopwatch.Stop(); + this.Log($"[{testCase.Name}] Completed in {stopwatch.ElapsedMilliseconds}ms. Metrics: {report}"); + } + } + + /// + /// Generates and serializes the initial context for each test case returned by . + /// + /// + /// This test is skipped during normal test runs because generating contexts requires live AI calls and + /// can be expensive. Run it explicitly (e.g., with dotnet test --filter "FullyQualifiedName~Serialize") + /// to regenerate the fixture files. After running, commit the generated files alongside the test code. + /// + [Fact(Skip = "Run explicitly to regenerate initial context fixture files.")] + public virtual async Task SerializeAllInitialContextsAsync() + { + var system = this.CreateTestSystem(); + var harness = new ConversationHarness(system); + + foreach (var testCase in this.GetTestCases()) + { + var outputPath = GetDefaultContextFilePath(testCase); + this.Log($"[{testCase.Name}] Serializing initial context to '{outputPath}'..."); + + await harness.SerializeInitialContextAsync(testCase, outputPath); + + this.Log($"[{testCase.Name}] Context serialized successfully."); + } + } + + // ------------------------------------------------------------------------- + // Protected helpers for subclasses + // ------------------------------------------------------------------------- + + /// + /// Computes the default file path used by when + /// writing the initial context for . + /// + /// + /// Override this method to change the output location. The default path is + /// {TestCase.Name}.context.json relative to the current working directory. + /// + /// The test case whose default output path is required. + /// The absolute or relative file path to write the serialized context to. + protected virtual string GetDefaultContextFilePath(IConversationTestCase testCase) => + $"{testCase.Name}.context.json"; + + /// + /// Writes a message to the xunit test output, if available, otherwise to the console. + /// + protected void Log(string message) + { + if (this._output is not null) + { + this._output.WriteLine(message); + } + else + { + Console.WriteLine(message); + } + } +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetrics.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetrics.cs new file mode 100644 index 0000000000..d795055320 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetrics.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace AgentConversation.IntegrationTests; + +/// +/// Captures the size characteristics of a conversation context at a specific point in time. +/// +public sealed class ConversationMetrics +{ + /// + /// Gets the number of messages in the conversation context. + /// + public required int MessageCount { get; init; } + + /// + /// Gets the approximate serialized size of the conversation context in bytes. + /// This serves as a proxy for context window consumption. + /// + public required long SerializedSizeBytes { get; init; } + + /// + public override string ToString() => + $"Messages={MessageCount}, Size={SerializedSizeBytes}B"; +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetricsReport.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetricsReport.cs new file mode 100644 index 0000000000..7e32729032 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationMetricsReport.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace AgentConversation.IntegrationTests; + +/// +/// Captures the before-and-after for a single test case run, +/// enabling comparison and reporting of context size changes. +/// +public sealed class ConversationMetricsReport +{ + /// + /// Gets the metrics captured before the agent steps were executed. + /// + public required ConversationMetrics Before { get; init; } + + /// + /// Gets the metrics captured after the agent steps were executed. + /// + public required ConversationMetrics After { get; init; } + + /// + /// Gets the change in message count between and . + /// A positive value means messages were added; a negative value means compaction removed messages. + /// + public int MessageCountDelta => After.MessageCount - Before.MessageCount; + + /// + /// Gets the change in serialized size in bytes between and . + /// + public long SizeDeltaBytes => After.SerializedSizeBytes - Before.SerializedSizeBytes; + + /// + public override string ToString() => + $"Before=[{Before}] After=[{After}] Delta=[Messages={MessageCountDelta:+#;-#;0}, Size={SizeDeltaBytes:+#;-#;0}B]"; +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/ConversationStep.cs b/dotnet/tests/AgentConversation.IntegrationTests/ConversationStep.cs new file mode 100644 index 0000000000..58ff445952 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/ConversationStep.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentConversation.IntegrationTests; + +/// +/// Represents a single step within a , combining the agent to invoke, +/// an optional input message, and an optional validation delegate. +/// +public sealed class ConversationStep +{ + /// + /// Gets or sets the name of the agent to invoke for this step. + /// Must match a key in . + /// + public required string AgentName { get; init; } + + /// + /// Gets or sets the optional input message to send to the agent. + /// When , the agent is invoked with no new user input (useful for + /// eliciting a response from the existing conversation context). + /// + public ChatMessage? Input { get; init; } + + /// + /// Gets or sets an optional delegate that validates the agent response and metrics for this step. + /// When , no validation is performed. + /// + public Action? Validate { get; init; } +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestCase.cs b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestCase.cs new file mode 100644 index 0000000000..142f9c952f --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestCase.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentConversation.IntegrationTests; + +/// +/// Defines a single agent conversation test case. +/// +/// +/// Each test case describes the initial conversation context (as a list of instances), +/// the agents that participate in the conversation, the steps to execute, and the expected outcomes. +/// The initial context can either be loaded from a previously serialized file or generated on-demand +/// via . +/// +public interface IConversationTestCase +{ + /// + /// Gets the human-readable name that uniquely identifies this test case. + /// + string Name { get; } + + /// + /// Gets the agents involved in this test case, keyed by their names. + /// Each entry is a that describes how to create the agent. + /// + IReadOnlyDictionary AgentDefinitions { get; } + + /// + /// Returns the initial list of instances to restore into the conversation + /// context before any steps are executed. + /// + /// + /// The initial chat messages. These are typically loaded from a previously serialized JSON file + /// produced by . + /// + IList GetInitialMessages(); + + /// + /// Gets the ordered list of steps to execute against the restored conversation context. + /// + IReadOnlyList Steps { get; } + + /// + /// Creates the initial conversation context by actually driving a conversation with the provided agents, + /// then returns the resulting list of messages. + /// + /// + /// This method is intended to be called once (e.g., during a setup phase) to produce the serialized + /// context that subsequent test runs will deserialize. Implementations should build up a long or + /// complex conversation that is representative of the long-running operation scenario being validated. + /// + /// + /// The agents to use when building the initial context, keyed by their names as defined in + /// . + /// + /// A token to cancel the operation. + /// + /// The ordered list of instances that form the initial context. + /// + Task> CreateInitialContextAsync( + IReadOnlyDictionary agents, + CancellationToken cancellationToken = default); +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestSystem.cs b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestSystem.cs new file mode 100644 index 0000000000..08b0eaa466 --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/IConversationTestSystem.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentConversation.IntegrationTests; + +/// +/// Abstracts the system-specific concerns of an agent conversation test run: how agents are created +/// and how context compaction is performed. +/// +/// +/// Implement this interface to adapt the to a particular AI backend +/// (e.g., OpenAI Chat Completion, Azure AI, OpenAI Responses API). Each implementation controls: +/// +/// +/// How instances are turned into live objects. +/// +/// +/// How context compaction is applied to a list of messages. Compaction is optional; returning +/// means no compaction is performed. +/// +/// +/// +public interface IConversationTestSystem +{ + /// + /// Creates a live from the supplied . + /// + /// The definition describing the agent to create. + /// A token to cancel the operation. + /// A fully-initialised ready to participate in the conversation. + Task CreateAgentAsync( + ConversationAgentDefinition definition, + CancellationToken cancellationToken = default); + + /// + /// Optionally compacts (reduces) the supplied . + /// + /// The current list of messages to compact. + /// A token to cancel the operation. + /// + /// The compacted list of messages, or if no compaction was performed. + /// When is returned the original list is used unchanged. + /// + Task?> CompactAsync( + IList messages, + CancellationToken cancellationToken = default); +} diff --git a/dotnet/tests/AgentConversation.IntegrationTests/README.md b/dotnet/tests/AgentConversation.IntegrationTests/README.md new file mode 100644 index 0000000000..4ca230aa7c --- /dev/null +++ b/dotnet/tests/AgentConversation.IntegrationTests/README.md @@ -0,0 +1,163 @@ +# AgentConversation Integration Test Harness + +The `AgentConversation.IntegrationTests` project provides a **reusable test harness** for validating conversation dynamics in long-running, multi-agent scenarios that involve tool use. Instead of rebuilding a large conversation context in every test run, the harness allows you to: + +1. **Capture** a representative conversation context once (by driving a real conversation with your AI agents). +2. **Serialize** that context to a JSON file and commit it alongside your tests. +3. **Restore** the saved context in each test run. +4. **Solicit** one or more agent responses from the restored context. +5. **Validate** each response and compare before/after metrics. + +--- + +## Key Abstractions + +| Type | Role | +|------|------| +| `IConversationTestCase` | Defines a test case: agents, initial messages, ordered steps, and a method to automate context creation. | +| `IConversationTestSystem` | System-specific plugin: how to **create agents** and how to apply **context compaction** (both vary per AI backend). | +| `ConversationAgentDefinition` | Describes a participating agent — name, instructions, and optional tools. | +| `ConversationStep` | One step in the conversation: which agent to invoke, an optional input message, and an optional validation delegate. | +| `ConversationMetrics` | A snapshot of conversation context size — message count and serialized byte size. | +| `ConversationMetricsReport` | A before/after `ConversationMetrics` pair with delta helpers for reporting. | +| `ConversationContextSerializer` | Serializes and deserializes `IList` to/from JSON strings or files. | +| `ConversationHarness` | The core runner that ties everything together. | +| `ConversationHarnessTests` | Abstract xunit base class that subclasses inherit to get the `RunAllTestCasesAsync` test. | + +--- + +## How It Works + +### 1. Implement `IConversationTestSystem` + +Provide an implementation that knows how to create agents for your target AI backend and optionally compact messages: + +```csharp +public sealed class OpenAIConversationTestSystem : IConversationTestSystem +{ + public Task CreateAgentAsync(ConversationAgentDefinition definition, CancellationToken ct = default) + { + var chatClient = new OpenAIClient(apiKey) + .GetChatClient("gpt-4o") + .AsIChatClient(); + + AIAgent agent = new ChatClientAgent(chatClient, options: new() + { + Name = definition.Name, + ChatOptions = new() { Instructions = definition.Instructions, Tools = definition.Tools } + }); + + return Task.FromResult(agent); + } + + public Task?> CompactAsync(IList messages, CancellationToken ct = default) + { + // Return null for no compaction, or apply an IChatReducer here. + return Task.FromResult?>(null); + } +} +``` + +### 2. Implement `IConversationTestCase` + +Define the agents involved, the initial context to restore, and the steps to execute: + +```csharp +public sealed class MyConversationTestCase : IConversationTestCase +{ + public string Name => "MyConversation"; + + public IReadOnlyDictionary AgentDefinitions { get; } = + new Dictionary + { + ["Assistant"] = new() { Name = "Assistant", Instructions = "You are a helpful assistant." } + }; + + // Load the saved context from a JSON fixture file. + public IList GetInitialMessages() => + ConversationContextSerializer.LoadFromFile("fixtures/my-conversation.context.json"); + + public IReadOnlyList Steps { get; } = + [ + new ConversationStep + { + AgentName = "Assistant", + Input = new ChatMessage(ChatRole.User, "Summarize our conversation so far."), + Validate = (response, metrics) => + { + Assert.NotEmpty(response.Text); + Assert.True(metrics.After.MessageCount > metrics.Before.MessageCount); + } + } + ]; + + // Called once to generate the fixture file — not during normal CI runs. + public async Task> CreateInitialContextAsync( + IReadOnlyDictionary agents, CancellationToken ct = default) + { + var agent = agents["Assistant"]; + var session = await agent.CreateSessionAsync(ct); + + // Drive a rich, representative conversation to build up the context. + await agent.RunAsync(new ChatMessage(ChatRole.User, "Tell me about the weather."), session, cancellationToken: ct); + await agent.RunAsync(new ChatMessage(ChatRole.User, "What about tomorrow?"), session, cancellationToken: ct); + // ... more turns ... + + var provider = agent.GetService()!; + return provider.GetMessages(session); + } +} +``` + +### 3. Derive from `ConversationHarnessTests` + +Wire the system and test cases into a concrete test class: + +```csharp +public class MyConversationTests(ITestOutputHelper output) + : ConversationHarnessTests(output) +{ + protected override OpenAIConversationTestSystem CreateTestSystem() => new(); + + protected override IEnumerable GetTestCases() => + [ + new MyConversationTestCase(), + ]; +} +``` + +The inherited `RunAllTestCasesAsync` test method will automatically run all cases and log the before/after metrics to the xunit test output. + +--- + +## Generating Initial Context Fixtures + +The context fixture files need to be generated once and committed to the repository. Run the inherited `SerializeAllInitialContextsAsync` test to produce them: + +```bash +dotnet test --filter "FullyQualifiedName~SerializeAllInitialContexts" +``` + +This test is **skipped during normal CI runs** to avoid expensive AI calls. After generating the files, commit them alongside your test code so that all subsequent runs can restore the context without calling the AI service again. + +--- + +## Metrics Reporting + +After each test case runs, a `ConversationMetricsReport` is logged. It captures: + +- **`Before`** — message count and serialized byte size of the initial context. +- **`After`** — message count and byte size after all steps have executed. +- **`MessageCountDelta`** / **`SizeDeltaBytes`** — the change between before and after. + +Example output: + +``` +[MyConversation] Before=[Messages=12, Size=4096B] After=[Messages=14, Size=4712B] Delta=[Messages=+2, Size=+616B] +``` + +--- + +## Example + +See [`Microsoft.Agents.AI.Abstractions.IntegrationTests`](../Microsoft.Agents.AI.Abstractions.IntegrationTests) for a self-contained working example that uses an in-memory mock `IChatClient` so it runs without live AI credentials. diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/AgentConversationHarnessTests.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/AgentConversationHarnessTests.cs new file mode 100644 index 0000000000..c073a926b9 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/AgentConversationHarnessTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using AgentConversation.IntegrationTests; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// Example integration tests that exercise the using an +/// in-memory that does not require live AI credentials. +/// +/// +/// +/// This class derives from and provides the system +/// and test cases. The test +/// method is inherited automatically and will run all cases returned by . +/// +/// +/// To adapt these tests to a real AI backend, replace +/// with an implementation that constructs agents backed by your AI service. +/// +/// +public class AgentConversationHarnessTests(ITestOutputHelper output) + : ConversationHarnessTests(output) +{ + /// + protected override InMemoryConversationTestSystem CreateTestSystem() => + new InMemoryConversationTestSystem(); + + /// + protected override IEnumerable GetTestCases() => + [ + new MenuConversationTestCase(), + ]; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/InMemoryConversationTestSystem.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/InMemoryConversationTestSystem.cs new file mode 100644 index 0000000000..d09d2c25bd --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/InMemoryConversationTestSystem.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AgentConversation.IntegrationTests; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using Moq; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// An example that uses an in-memory mock +/// so the harness can be exercised without live AI service credentials. +/// +/// +/// In a real integration test against a live AI service (e.g., OpenAI Chat Completion), this class +/// would be replaced with an implementation that constructs instances +/// backed by the real . The compaction contract can similarly be wired up +/// to an of your choice. +/// +public sealed class InMemoryConversationTestSystem : IConversationTestSystem +{ + /// + /// A deterministic response suffix appended by the mock chat client to every assistant reply. + /// Test validations can assert on this value to confirm the mock was invoked. + /// + public const string MockResponseSuffix = "[mock-response]"; + + /// + public Task CreateAgentAsync( + ConversationAgentDefinition definition, + CancellationToken cancellationToken = default) + { + // Create a mock IChatClient that returns a deterministic response. + var mockClient = new Mock(); + mockClient + .Setup(c => c.GetResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(() => new ChatResponse( + new ChatMessage(ChatRole.Assistant, + $"Here are today's specials: Clam Chowder, Cobb Salad, Chai Tea. {MockResponseSuffix}"))); + + // GetService is called internally by the harness for metadata; return null for unknown types. + mockClient + .Setup(c => c.GetService(It.IsAny(), It.IsAny())) + .Returns((System.Type _, object? _) => null); + + AIAgent agent = new ChatClientAgent( + mockClient.Object, + options: new ChatClientAgentOptions + { + Name = definition.Name, + ChatOptions = new ChatOptions + { + Instructions = definition.Instructions, + Tools = definition.Tools is not null ? new System.Collections.Generic.List(definition.Tools) : null, + } + }); + + return Task.FromResult(agent); + } + + /// + public Task?> CompactAsync( + IList messages, + CancellationToken cancellationToken = default) + { + // No compaction in this example system. + return Task.FromResult?>(null); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuConversationTestCase.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuConversationTestCase.cs new file mode 100644 index 0000000000..f68c57d143 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuConversationTestCase.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AgentConversation.IntegrationTests; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// An example that validates the harness can restore a +/// pre-built conversation context and solicit a response from an agent. +/// +/// +/// This test case uses a fixed, in-memory conversation representing a menu-ordering interaction. +/// The messages are defined inline (no JSON fixture file is required), which makes this a +/// self-contained example that runs without live AI credentials. +/// +public sealed class MenuConversationTestCase : IConversationTestCase +{ + private const string AgentKey = "MenuAgent"; + + /// + public string Name => "MenuConversation"; + + /// + public IReadOnlyDictionary AgentDefinitions { get; } = + new Dictionary + { + [AgentKey] = new ConversationAgentDefinition + { + Name = AgentKey, + Instructions = "You are a helpful restaurant assistant. Answer questions about the menu.", + Tools = + [ + AIFunctionFactory.Create(MenuTools.GetSpecials), + AIFunctionFactory.Create(MenuTools.GetItemPrice), + ] + } + }; + + /// + public IReadOnlyList Steps { get; } = + [ + new ConversationStep + { + AgentName = AgentKey, + Input = new ChatMessage(ChatRole.User, "What are the specials today?"), + Validate = (response, metrics) => + { + Assert.NotNull(response); + Assert.NotEmpty(response.Text); + Assert.True(metrics.After.MessageCount > metrics.Before.MessageCount, + "Message count should grow after the step."); + } + } + ]; + + /// + public IList GetInitialMessages() => + // A short, representative conversation context that is already in memory. + [ + new ChatMessage(ChatRole.User, "Hello, I'd like to see the menu."), + new ChatMessage(ChatRole.Assistant, "Welcome! I'm happy to help you with our menu. Feel free to ask about today's specials or the price of any item."), + ]; + + /// + public async Task> CreateInitialContextAsync( + IReadOnlyDictionary agents, + CancellationToken cancellationToken = default) + { + // Build the initial context by running a short greeting exchange. + var agent = agents[AgentKey]; + var session = await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + + await agent.RunAsync( + new ChatMessage(ChatRole.User, "Hello, I'd like to see the menu."), + session, + cancellationToken: cancellationToken).ConfigureAwait(false); + + var historyProvider = agent.GetService() as InMemoryChatHistoryProvider; + if (historyProvider is not null) + { + return historyProvider.GetMessages(session); + } + + return GetInitialMessages(); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuTools.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuTools.cs new file mode 100644 index 0000000000..67b8195c87 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/MenuTools.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; + +namespace Microsoft.Agents.AI.Abstractions.IntegrationTests; + +/// +/// Example tools used by the to simulate a restaurant menu service. +/// +internal static class MenuTools +{ + [Description("Provides a list of today's specials from the restaurant menu.")] + public static string GetSpecials() => + """ + Special Soup: Clam Chowder + Special Salad: Cobb Salad + Special Drink: Chai Tea + """; + + [Description("Provides the price of the requested menu item.")] + public static string GetItemPrice( + [Description("The name of the menu item.")] string menuItem) => "$9.99"; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/Microsoft.Agents.AI.Abstractions.IntegrationTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/Microsoft.Agents.AI.Abstractions.IntegrationTests.csproj new file mode 100644 index 0000000000..faa9bda844 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.IntegrationTests/Microsoft.Agents.AI.Abstractions.IntegrationTests.csproj @@ -0,0 +1,16 @@ + + + + True + true + true + true + + + + + + + + +