test
Some checks failed
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
Python - Merge - Tests / paths-filter (push) Has been cancelled
Python - Merge - Tests / Python Tests - Core (integration, ubuntu-latest, 3.10) (push) Has been cancelled
Python - Merge - Tests / Python Tests - Azure AI (integration, ubuntu-latest, 3.10) (push) Has been cancelled
Python - Merge - Tests / python-integration-tests-check (push) Has been cancelled
Python - Lab Tests / paths-filter (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (ubuntu-latest, 3.10) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (ubuntu-latest, 3.11) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (ubuntu-latest, 3.12) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (ubuntu-latest, 3.13) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (ubuntu-latest, 3.14) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (windows-latest, 3.10) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (windows-latest, 3.11) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (windows-latest, 3.12) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (windows-latest, 3.13) (push) Has been cancelled
Python - Lab Tests / Python Lab Tests (windows-latest, 3.14) (push) Has been cancelled
Check .md links / markdown-link-check (push) Has been cancelled

This commit is contained in:
2026-01-24 03:05:12 +11:00
parent f78f2388b3
commit 539852f81c
2584 changed files with 287471 additions and 0 deletions

View File

@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsTestProject>false</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
</ItemGroup>
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net10.0'))">
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,25 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
namespace AgentConformance.IntegrationTests;
/// <summary>
/// Base class for all test classes used for testing agents.
/// </summary>
/// <typeparam name="TAgentFixture">The type of the agent fixture used in these tests.</typeparam>
/// <param name="createAgentFixture">Used to create a new fixture for this test suite.</param>
public abstract class AgentTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : IAsyncLifetime
where TAgentFixture : IAgentFixture
{
protected TAgentFixture Fixture { get; private set; } = default!;
public Task InitializeAsync()
{
this.Fixture = createAgentFixture();
return this.Fixture.InitializeAsync();
}
public Task DisposeAsync() => this.Fixture.DisposeAsync();
}

View File

@@ -0,0 +1,70 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Linq;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
namespace AgentConformance.IntegrationTests;
/// <summary>
/// Conformance tests that are specific to the <see cref="ChatClientAgent"/> in addition to those in <see cref="RunStreamingTests{TAgentFixture}"/>.
/// </summary>
/// <typeparam name="TAgentFixture">The type of test fixture used by the concrete test implementation.</typeparam>
/// <param name="createAgentFixture">Function to create the test fixture with.</param>
public abstract class ChatClientAgentRunStreamingTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : AgentTests<TAgentFixture>(createAgentFixture)
where TAgentFixture : IChatClientAgentFixture
{
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
{
// Arrange
var agent = await this.Fixture.CreateChatClientAgentAsync(instructions: "Always respond with 'Computer says no', even if there was no user input.");
var thread = await agent.GetNewThreadAsync();
await using var agentCleanup = new AgentCleanup(agent, this.Fixture);
await using var threadCleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var responseUpdates = await agent.RunStreamingAsync(thread).ToListAsync();
// Assert
var chatResponseText = string.Concat(responseUpdates.Select(x => x.Text));
Assert.Contains("Computer says no", chatResponseText, StringComparison.OrdinalIgnoreCase);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync()
{
// Arrange
var questionsAndAnswers = new[]
{
(Question: "Hello", ExpectedAnswer: string.Empty),
(Question: "What is the special soup?", ExpectedAnswer: "Clam Chowder"),
(Question: "What is the special drink?", ExpectedAnswer: "Chai Tea"),
(Question: "What is the special salad?", ExpectedAnswer: "Cobb Salad"),
(Question: "Thank you", ExpectedAnswer: string.Empty)
};
var agent = await this.Fixture.CreateChatClientAgentAsync(
aiTools:
[
AIFunctionFactory.Create(MenuPlugin.GetSpecials),
AIFunctionFactory.Create(MenuPlugin.GetItemPrice)
]);
var thread = await agent.GetNewThreadAsync();
foreach (var questionAndAnswer in questionsAndAnswers)
{
// Act
var responseUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, questionAndAnswer.Question),
thread).ToListAsync();
// Assert
var chatResponseText = string.Concat(responseUpdates.Select(x => x.Text));
Assert.Contains(questionAndAnswer.ExpectedAnswer, chatResponseText, StringComparison.OrdinalIgnoreCase);
}
}
}

View File

@@ -0,0 +1,70 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
namespace AgentConformance.IntegrationTests;
/// <summary>
/// Conformance tests that are specific to the <see cref="ChatClientAgent"/> in addition to those in <see cref="RunTests{TAgentFixture}"/>.
/// </summary>
/// <typeparam name="TAgentFixture">The type of test fixture used by the concrete test implementation.</typeparam>
/// <param name="createAgentFixture">Function to create the test fixture with.</param>
public abstract class ChatClientAgentRunTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : AgentTests<TAgentFixture>(createAgentFixture)
where TAgentFixture : IChatClientAgentFixture
{
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
{
// Arrange
var agent = await this.Fixture.CreateChatClientAgentAsync(instructions: "ALWAYS RESPOND WITH 'Computer says no', even if there was no user input.");
var thread = await agent.GetNewThreadAsync();
await using var agentCleanup = new AgentCleanup(agent, this.Fixture);
await using var threadCleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var response = await agent.RunAsync(thread);
// Assert
Assert.NotNull(response);
Assert.Single(response.Messages);
Assert.False(string.IsNullOrWhiteSpace(response.Text), "Agent should return non-empty response even without user input");
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync()
{
// Arrange
var questionsAndAnswers = new[]
{
(Question: "Hello", ExpectedAnswer: string.Empty),
(Question: "What is the special soup?", ExpectedAnswer: "Clam Chowder"),
(Question: "What is the special drink?", ExpectedAnswer: "Chai Tea"),
(Question: "What is the special salad?", ExpectedAnswer: "Cobb Salad"),
(Question: "Thank you", ExpectedAnswer: string.Empty)
};
var agent = await this.Fixture.CreateChatClientAgentAsync(
aiTools:
[
AIFunctionFactory.Create(MenuPlugin.GetSpecials),
AIFunctionFactory.Create(MenuPlugin.GetItemPrice)
]);
var thread = await agent.GetNewThreadAsync();
foreach (var questionAndAnswer in questionsAndAnswers)
{
// Act
var result = await agent.RunAsync(
new ChatMessage(ChatRole.User, questionAndAnswer.Question),
thread);
// Assert
Assert.NotNull(result);
Assert.Contains(questionAndAnswer.ExpectedAnswer, result.Text);
}
}
}

View File

@@ -0,0 +1,21 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
namespace AgentConformance.IntegrationTests;
/// <summary>
/// Interface for setting up and tearing down agents, to be used in tests.
/// Each agent type should have its own derived class.
/// </summary>
public interface IAgentFixture : IAsyncLifetime
{
AIAgent Agent { get; }
Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread);
Task DeleteThreadAsync(AgentThread thread);
}

View File

@@ -0,0 +1,24 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
namespace AgentConformance.IntegrationTests;
/// <summary>
/// Interface for setting up and tearing down <see cref="IChatClient"/> based agents, to be used in tests.
/// Each agent type should have its own derived class.
/// </summary>
public interface IChatClientAgentFixture : IAgentFixture
{
IChatClient ChatClient { get; }
Task<ChatClientAgent> CreateChatClientAgentAsync(
string name = "HelpfulAssistant",
string instructions = "You are a helpful assistant.",
IList<AITool>? aiTools = null);
Task DeleteAgentAsync(ChatClientAgent agent);
}

View File

@@ -0,0 +1,25 @@
// Copyright (c) Microsoft. All rights reserved.
using System.ComponentModel;
namespace AgentConformance.IntegrationTests;
#pragma warning disable CA1812 // Avoid uninstantiated internal classes
/// <summary>
/// A test plugin used to verify function invocation.
/// </summary>
internal static class MenuPlugin
{
[Description("Provides a list of specials from the menu.")]
public static string GetSpecials() => """
Special Soup: Clam Chowder
Special Salad: Cobb Salad
Special Drink: Chai Tea
""";
[Description("Provides the price of the requested menu item.")]
public static string GetItemPrice(
[Description("The name of the menu item.")]
string menuItem) => "$9.99";
}

View File

@@ -0,0 +1,118 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Linq;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
namespace AgentConformance.IntegrationTests;
/// <summary>
/// Conformance tests for run methods on agents.
/// </summary>
/// <typeparam name="TAgentFixture">The type of test fixture used by the concrete test implementation.</typeparam>
/// <param name="createAgentFixture">Function to create the test fixture with.</param>
public abstract class RunStreamingTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : AgentTests<TAgentFixture>(createAgentFixture)
where TAgentFixture : IAgentFixture
{
public virtual Func<Task<AgentRunOptions?>> AgentRunOptionsFactory { get; set; } = () => Task.FromResult(default(AgentRunOptions));
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithNoMessageDoesNotFailAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var chatResponses = await agent.RunStreamingAsync(thread, await this.AgentRunOptionsFactory.Invoke()).ToListAsync();
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithStringReturnsExpectedResultAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var responseUpdates = await agent.RunStreamingAsync("What is the capital of France.", thread, await this.AgentRunOptionsFactory.Invoke()).ToListAsync();
// Assert
var chatResponseText = string.Concat(responseUpdates.Select(x => x.Text));
Assert.Contains("Paris", chatResponseText);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithChatMessageReturnsExpectedResultAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var responseUpdates = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread, await this.AgentRunOptionsFactory.Invoke()).ToListAsync();
// Assert
var chatResponseText = string.Concat(responseUpdates.Select(x => x.Text));
Assert.Contains("Paris", chatResponseText);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithChatMessagesReturnsExpectedResultAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var responseUpdates = await agent.RunStreamingAsync(
[
new ChatMessage(ChatRole.User, "Hello."),
new ChatMessage(ChatRole.User, "What is the capital of France.")
],
thread,
await this.AgentRunOptionsFactory.Invoke()).ToListAsync();
// Assert
var chatResponseText = string.Concat(responseUpdates.Select(x => x.Text));
Assert.Contains("Paris", chatResponseText);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task ThreadMaintainsHistoryAsync()
{
// Arrange
const string Q1 = "What is the capital of France.";
const string Q2 = "And Austria?";
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var options = await this.AgentRunOptionsFactory.Invoke();
var responseUpdates1 = await agent.RunStreamingAsync(Q1, thread, options).ToListAsync();
var responseUpdates2 = await agent.RunStreamingAsync(Q2, thread, options).ToListAsync();
// Assert
var response1Text = string.Concat(responseUpdates1.Select(x => x.Text));
var response2Text = string.Concat(responseUpdates2.Select(x => x.Text));
Assert.Contains("Paris", response1Text);
Assert.Contains("Vienna", response2Text);
var chatHistory = await this.Fixture.GetChatHistoryAsync(thread);
Assert.Equal(4, chatHistory.Count);
Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.User));
Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.Assistant));
Assert.Equal(Q1, chatHistory[0].Text);
Assert.Equal(Q2, chatHistory[2].Text);
Assert.Contains("Paris", chatHistory[1].Text);
Assert.Contains("Vienna", chatHistory[3].Text);
}
}

View File

@@ -0,0 +1,123 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Linq;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
namespace AgentConformance.IntegrationTests;
/// <summary>
/// Conformance tests for run methods on agents.
/// </summary>
/// <typeparam name="TAgentFixture">The type of test fixture used by the concrete test implementation.</typeparam>
/// <param name="createAgentFixture">Function to create the test fixture with.</param>
public abstract class RunTests<TAgentFixture>(Func<TAgentFixture> createAgentFixture) : AgentTests<TAgentFixture>(createAgentFixture)
where TAgentFixture : IAgentFixture
{
public virtual Func<Task<AgentRunOptions?>> AgentRunOptionsFactory { get; set; } = () => Task.FromResult(default(AgentRunOptions));
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithNoMessageDoesNotFailAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var chatResponse = await agent.RunAsync(thread);
// Assert
Assert.NotNull(chatResponse);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithStringReturnsExpectedResultAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var response = await agent.RunAsync("What is the capital of France.", thread, await this.AgentRunOptionsFactory.Invoke());
// Assert
Assert.NotNull(response);
Assert.Single(response.Messages);
Assert.Contains("Paris", response.Text);
Assert.Equal(agent.Id, response.AgentId);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithChatMessageReturnsExpectedResultAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var response = await agent.RunAsync(new ChatMessage(ChatRole.User, "What is the capital of France."), thread, await this.AgentRunOptionsFactory.Invoke());
// Assert
Assert.NotNull(response);
Assert.Single(response.Messages);
Assert.Contains("Paris", response.Text);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task RunWithChatMessagesReturnsExpectedResultAsync()
{
// Arrange
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var response = await agent.RunAsync(
[
new ChatMessage(ChatRole.User, "Hello."),
new ChatMessage(ChatRole.User, "What is the capital of France.")
],
thread,
await this.AgentRunOptionsFactory.Invoke());
// Assert
Assert.NotNull(response);
Assert.Single(response.Messages);
Assert.Contains("Paris", response.Text);
}
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
public virtual async Task ThreadMaintainsHistoryAsync()
{
// Arrange
const string Q1 = "What is the capital of France.";
const string Q2 = "And Austria?";
var agent = this.Fixture.Agent;
var thread = await agent.GetNewThreadAsync();
await using var cleanup = new ThreadCleanup(thread, this.Fixture);
// Act
var options = await this.AgentRunOptionsFactory.Invoke();
var result1 = await agent.RunAsync(Q1, thread, options);
var result2 = await agent.RunAsync(Q2, thread, options);
// Assert
Assert.Contains("Paris", result1.Text);
Assert.Contains("Vienna", result2.Text);
var chatHistory = await this.Fixture.GetChatHistoryAsync(thread);
Assert.Equal(4, chatHistory.Count);
Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.User));
Assert.Equal(2, chatHistory.Count(x => x.Role == ChatRole.Assistant));
Assert.Equal(Q1, chatHistory[0].Text);
Assert.Contains("Paris", chatHistory[1].Text);
Assert.Equal(Q2, chatHistory[2].Text);
Assert.Contains("Vienna", chatHistory[3].Text);
}
}

View File

@@ -0,0 +1,18 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using Microsoft.Agents.AI;
namespace AgentConformance.IntegrationTests.Support;
/// <summary>
/// Helper class to delete agents after tests.
/// </summary>
/// <param name="agent">The agent to delete.</param>
/// <param name="fixture">The fixture that provides agent specific capabilities.</param>
internal sealed class AgentCleanup(ChatClientAgent agent, IChatClientAgentFixture fixture) : IAsyncDisposable
{
public async ValueTask DisposeAsync() =>
await fixture.DeleteAgentAsync(agent);
}

View File

@@ -0,0 +1,9 @@
// Copyright (c) Microsoft. All rights reserved.
namespace AgentConformance.IntegrationTests.Support;
internal static class Constants
{
public const int RetryCount = 3;
public const int RetryDelay = 5000;
}

View File

@@ -0,0 +1,40 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using Microsoft.Extensions.Configuration;
namespace AgentConformance.IntegrationTests.Support;
/// <summary>
/// Helper for loading test configuration settings.
/// </summary>
public sealed class TestConfiguration
{
private static readonly IConfiguration s_configuration = new ConfigurationBuilder()
.AddJsonFile(path: "testsettings.json", optional: true)
.AddJsonFile(path: "testsettings.development.json", optional: true)
.AddEnvironmentVariables()
.AddUserSecrets<TestConfiguration>()
.Build();
/// <summary>
/// Loads the type of configuration using a section name based on the type name.
/// </summary>
/// <typeparam name="T">The type of config to load.</typeparam>
/// <returns>The loaded configuration section of the specified type.</returns>
/// <exception cref="InvalidOperationException">Thrown if the configuration section cannot be loaded.</exception>
public static T LoadSection<T>()
{
var configType = typeof(T);
var configTypeName = configType.Name;
const string TrimText = "Configuration";
if (configTypeName.EndsWith(TrimText, StringComparison.OrdinalIgnoreCase))
{
configTypeName = configTypeName.Substring(0, configTypeName.Length - TrimText.Length);
}
return s_configuration.GetRequiredSection(configTypeName).Get<T>() ??
throw new InvalidOperationException($"Could not load config for {configTypeName}.");
}
}

View File

@@ -0,0 +1,18 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using Microsoft.Agents.AI;
namespace AgentConformance.IntegrationTests.Support;
/// <summary>
/// Helper class to delete threads after tests.
/// </summary>
/// <param name="thread">The thread to delete.</param>
/// <param name="fixture">The fixture that provides agent specific capabilities.</param>
internal sealed class ThreadCleanup(AgentThread thread, IAgentFixture fixture) : IAsyncDisposable
{
public async ValueTask DisposeAsync() =>
await fixture.DeleteThreadAsync(thread);
}