test

dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunStreamingTests.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests;
+using Microsoft.Agents.AI;
+
+namespace AzureAI.IntegrationTests;
+
+public class AIProjectClientAgentRunStreamingPreviousResponseTests() : RunStreamingTests<AIProjectClientFixture>(() => new())
+{
+    [Fact(Skip = "No messages is not supported")]
+    public override Task RunWithNoMessageDoesNotFailAsync()
+    {
+        return Task.CompletedTask;
+    }
+}
+
+public class AIProjectClientAgentRunStreamingConversationTests() : RunTests<AIProjectClientFixture>(() => new())
+{
+    public override Func<Task<AgentRunOptions?>> AgentRunOptionsFactory => async () =>
+    {
+        var conversationId = await this.Fixture.CreateConversationAsync();
+        return new ChatClientAgentRunOptions(new() { ConversationId = conversationId });
+    };
+
+    [Fact(Skip = "No messages is not supported")]
+    public override Task RunWithNoMessageDoesNotFailAsync()
+    {
+        return Task.CompletedTask;
+    }
+}

dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunTests.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests;
+using Microsoft.Agents.AI;
+
+namespace AzureAI.IntegrationTests;
+
+public class AIProjectClientAgentRunPreviousResponseTests() : RunTests<AIProjectClientFixture>(() => new())
+{
+    [Fact(Skip = "No messages is not supported")]
+    public override Task RunWithNoMessageDoesNotFailAsync()
+    {
+        return Task.CompletedTask;
+    }
+}
+
+public class AIProjectClientAgentRunConversationTests() : RunTests<AIProjectClientFixture>(() => new())
+{
+    public override Func<Task<AgentRunOptions?>> AgentRunOptionsFactory => async () =>
+    {
+        var conversationId = await this.Fixture.CreateConversationAsync();
+        return new ChatClientAgentRunOptions(new() { ConversationId = conversationId });
+    };
+
+    [Fact(Skip = "No messages is not supported")]
+    public override Task RunWithNoMessageDoesNotFailAsync()
+    {
+        return Task.CompletedTask;
+    }
+}

dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunStreamingTests.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests;
+
+namespace AzureAI.IntegrationTests;
+
+public class AIProjectClientChatClientAgentRunStreamingTests() : ChatClientAgentRunStreamingTests<AIProjectClientFixture>(() => new())
+{
+    [Fact(Skip = "No messages is not supported")]
+    public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
+    {
+        return Task.CompletedTask;
+    }
+}

dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunTests.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests;
+
+namespace AzureAI.IntegrationTests;
+
+public class AIProjectClientChatClientAgentRunTests() : ChatClientAgentRunTests<AIProjectClientFixture>(() => new())
+{
+    [Fact(Skip = "No messages is not supported")]
+    public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
+    {
+        return Task.CompletedTask;
+    }
+}

dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs

@@ -0,0 +1,232 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests.Support;
+using Azure.AI.Projects;
+using Azure.AI.Projects.OpenAI;
+using Azure.Identity;
+using Microsoft.Agents.AI;
+using Microsoft.Extensions.AI;
+using OpenAI.Files;
+using OpenAI.Responses;
+using Shared.IntegrationTests;
+
+namespace AzureAI.IntegrationTests;
+
+public class AIProjectClientCreateTests
+{
+    private static readonly AzureAIConfiguration s_config = TestConfiguration.LoadSection<AzureAIConfiguration>();
+    private readonly AIProjectClient _client = new(new Uri(s_config.Endpoint), new AzureCliCredential());
+
+    [Theory]
+    [InlineData("CreateWithChatClientAgentOptionsAsync")]
+    [InlineData("CreateWithFoundryOptionsAsync")]
+    public async Task CreateAgent_CreatesAgentWithCorrectMetadataAsync(string createMechanism)
+    {
+        // Arrange.
+        string AgentName = AIProjectClientFixture.GenerateUniqueAgentName("IntegrationTestAgent");
+        const string AgentDescription = "An agent created during integration tests";
+        const string AgentInstructions = "You are an integration test agent";
+
+        // Act.
+        var agent = createMechanism switch
+        {
+            "CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
+                model: s_config.DeploymentName,
+                options: new ChatClientAgentOptions()
+                {
+                    Name = AgentName,
+                    Description = AgentDescription,
+                    ChatOptions = new() { Instructions = AgentInstructions }
+                }),
+            "CreateWithFoundryOptionsAsync" => await this._client.CreateAIAgentAsync(
+                name: AgentName,
+                creationOptions: new AgentVersionCreationOptions(new PromptAgentDefinition(s_config.DeploymentName) { Instructions = AgentInstructions }) { Description = AgentDescription }),
+            _ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
+        };
+
+        try
+        {
+            // Assert.
+            Assert.NotNull(agent);
+            Assert.Equal(AgentName, agent.Name);
+            Assert.Equal(AgentDescription, agent.Description);
+            Assert.Equal(AgentInstructions, agent.Instructions);
+
+            var agentRecord = await this._client.Agents.GetAgentAsync(agent.Name);
+            Assert.NotNull(agentRecord);
+            Assert.Equal(AgentName, agentRecord.Value.Name);
+            var definition = Assert.IsType<PromptAgentDefinition>(agentRecord.Value.Versions.Latest.Definition);
+            Assert.Equal(AgentDescription, agentRecord.Value.Versions.Latest.Description);
+            Assert.Equal(AgentInstructions, definition.Instructions);
+        }
+        finally
+        {
+            // Cleanup.
+            await this._client.Agents.DeleteAgentAsync(agent.Name);
+        }
+    }
+
+    [Theory(Skip = "For manual testing only")]
+    [InlineData("CreateWithChatClientAgentOptionsAsync")]
+    [InlineData("CreateWithFoundryOptionsAsync")]
+    public async Task CreateAgent_CreatesAgentWithVectorStoresAsync(string createMechanism)
+    {
+        // Arrange.
+        string AgentName = AIProjectClientFixture.GenerateUniqueAgentName("VectorStoreAgent");
+        const string AgentInstructions = """
+            You are a helpful agent that can help fetch data from files you know about.
+            Use the File Search Tool to look up codes for words.
+            Do not answer a question unless you can find the answer using the File Search Tool.
+            """;
+
+        // Get the project OpenAI client.
+        var projectOpenAIClient = this._client.GetProjectOpenAIClient();
+
+        // Create a vector store.
+        var searchFilePath = Path.GetTempFileName() + "wordcodelookup.txt";
+        File.WriteAllText(
+            path: searchFilePath,
+            contents: "The word 'apple' uses the code 442345, while the word 'banana' uses the code 673457."
+        );
+        OpenAIFile uploadedAgentFile = projectOpenAIClient.GetProjectFilesClient().UploadFile(
+            filePath: searchFilePath,
+            purpose: FileUploadPurpose.Assistants
+        );
+        var vectorStoreMetadata = await projectOpenAIClient.GetProjectVectorStoresClient().CreateVectorStoreAsync(options: new() { FileIds = { uploadedAgentFile.Id }, Name = "WordCodeLookup_VectorStore" });
+
+        // Act.
+        var agent = createMechanism switch
+        {
+            "CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
+                model: s_config.DeploymentName,
+                name: AgentName,
+                instructions: AgentInstructions,
+                tools: [new HostedFileSearchTool() { Inputs = [new HostedVectorStoreContent(vectorStoreMetadata.Value.Id)] }]),
+            "CreateWithFoundryOptionsAsync" => await this._client.CreateAIAgentAsync(
+                model: s_config.DeploymentName,
+                name: AgentName,
+                instructions: AgentInstructions,
+                tools: [ResponseTool.CreateFileSearchTool(vectorStoreIds: [vectorStoreMetadata.Value.Id]).AsAITool()]),
+            _ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
+        };
+
+        try
+        {
+            // Assert.
+            // Verify that the agent can use the vector store to answer a question.
+            var result = await agent.RunAsync("Can you give me the documented code for 'banana'?");
+            Assert.Contains("673457", result.ToString());
+        }
+        finally
+        {
+            // Cleanup.
+            await this._client.Agents.DeleteAgentAsync(agent.Name);
+            await projectOpenAIClient.GetProjectVectorStoresClient().DeleteVectorStoreAsync(vectorStoreMetadata.Value.Id);
+            await projectOpenAIClient.GetProjectFilesClient().DeleteFileAsync(uploadedAgentFile.Id);
+            File.Delete(searchFilePath);
+        }
+    }
+
+    [Theory]
+    [InlineData("CreateWithChatClientAgentOptionsAsync")]
+    [InlineData("CreateWithFoundryOptionsAsync")]
+    public async Task CreateAgent_CreatesAgentWithCodeInterpreterAsync(string createMechanism)
+    {
+        // Arrange.
+        string AgentName = AIProjectClientFixture.GenerateUniqueAgentName("CodeInterpreterAgent");
+        const string AgentInstructions = """
+            You are a helpful coding agent. A Python file is provided. Use the Code Interpreter Tool to run the file
+            and report the SECRET_NUMBER value it prints. Respond only with the number.
+            """;
+
+        // Get the project OpenAI client.
+        var projectOpenAIClient = this._client.GetProjectOpenAIClient();
+
+        // Create a python file that prints a known value.
+        var codeFilePath = Path.GetTempFileName() + "secret_number.py";
+        File.WriteAllText(
+            path: codeFilePath,
+            contents: "print(\"SECRET_NUMBER=24601\")" // Deterministic output we will look for.
+        );
+        OpenAIFile uploadedCodeFile = projectOpenAIClient.GetProjectFilesClient().UploadFile(
+            filePath: codeFilePath,
+            purpose: FileUploadPurpose.Assistants
+        );
+
+        // Act.
+        var agent = createMechanism switch
+        {
+            // Hosted tool path (tools supplied via ChatClientAgentOptions)
+            "CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
+                model: s_config.DeploymentName,
+                name: AgentName,
+                instructions: AgentInstructions,
+                tools: [new HostedCodeInterpreterTool() { Inputs = [new HostedFileContent(uploadedCodeFile.Id)] }]),
+            // Foundry (definitions + resources provided directly)
+            "CreateWithFoundryOptionsAsync" => await this._client.CreateAIAgentAsync(
+                model: s_config.DeploymentName,
+                name: AgentName,
+                instructions: AgentInstructions,
+                tools: [ResponseTool.CreateCodeInterpreterTool(new CodeInterpreterToolContainer(CodeInterpreterToolContainerConfiguration.CreateAutomaticContainerConfiguration([uploadedCodeFile.Id]))).AsAITool()]),
+            _ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
+        };
+
+        try
+        {
+            // Assert.
+            var result = await agent.RunAsync("What is the SECRET_NUMBER?");
+            // We expect the model to run the code and surface the number.
+            Assert.Contains("24601", result.ToString());
+        }
+        finally
+        {
+            // Cleanup.
+            await this._client.Agents.DeleteAgentAsync(agent.Name);
+            await projectOpenAIClient.GetProjectFilesClient().DeleteFileAsync(uploadedCodeFile.Id);
+            File.Delete(codeFilePath);
+        }
+    }
+
+    [Theory]
+    [InlineData("CreateWithChatClientAgentOptionsAsync")]
+    public async Task CreateAgent_CreatesAgentWithAIFunctionToolsAsync(string createMechanism)
+    {
+        // Arrange.
+        string AgentName = AIProjectClientFixture.GenerateUniqueAgentName("WeatherAgent");
+        const string AgentInstructions = "You are a helpful weather assistant. Always call the GetWeather function to answer questions about weather.";
+
+        static string GetWeather(string location) => $"The weather in {location} is sunny with a high of 23C.";
+        var weatherFunction = AIFunctionFactory.Create(GetWeather);
+
+        ChatClientAgent agent = createMechanism switch
+        {
+            "CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync(
+                model: s_config.DeploymentName,
+                options: new ChatClientAgentOptions()
+                {
+                    Name = AgentName,
+                    ChatOptions = new() { Instructions = AgentInstructions, Tools = [weatherFunction] }
+                }),
+            _ => throw new InvalidOperationException($"Unknown create mechanism: {createMechanism}")
+        };
+
+        try
+        {
+            // Act.
+            var response = await agent.RunAsync("What is the weather like in Amsterdam?");
+
+            // Assert - ensure function was invoked and its output surfaced.
+            var text = response.Text;
+            Assert.Contains("Amsterdam", text, StringComparison.OrdinalIgnoreCase);
+            Assert.Contains("sunny", text, StringComparison.OrdinalIgnoreCase);
+            Assert.Contains("23", text, StringComparison.OrdinalIgnoreCase);
+        }
+        finally
+        {
+            await this._client.Agents.DeleteAgentAsync(agent.Name);
+        }
+    }
+}

dotnet/tests/AzureAI.IntegrationTests/AIProjectClientFixture.cs

@@ -0,0 +1,167 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using AgentConformance.IntegrationTests;
+using AgentConformance.IntegrationTests.Support;
+using Azure.AI.Projects;
+using Azure.AI.Projects.OpenAI;
+using Azure.Identity;
+using Microsoft.Agents.AI;
+using Microsoft.Extensions.AI;
+using OpenAI.Responses;
+using Shared.IntegrationTests;
+
+namespace AzureAI.IntegrationTests;
+
+public class AIProjectClientFixture : IChatClientAgentFixture
+{
+    private static readonly AzureAIConfiguration s_config = TestConfiguration.LoadSection<AzureAIConfiguration>();
+
+    private ChatClientAgent _agent = null!;
+    private AIProjectClient _client = null!;
+
+    public IChatClient ChatClient => this._agent.ChatClient;
+
+    public AIAgent Agent => this._agent;
+
+    public async Task<string> CreateConversationAsync()
+    {
+        var response = await this._client.GetProjectOpenAIClient().GetProjectConversationsClient().CreateProjectConversationAsync();
+        return response.Value.Id;
+    }
+
+    public async Task<List<ChatMessage>> GetChatHistoryAsync(AgentThread thread)
+    {
+        var chatClientThread = (ChatClientAgentThread)thread;
+
+        if (chatClientThread.ConversationId?.StartsWith("conv_", StringComparison.OrdinalIgnoreCase) == true)
+        {
+            // Conversation threads do not persist message history.
+            return await this.GetChatHistoryFromConversationAsync(chatClientThread.ConversationId);
+        }
+
+        if (chatClientThread.ConversationId?.StartsWith("resp_", StringComparison.OrdinalIgnoreCase) == true)
+        {
+            return await this.GetChatHistoryFromResponsesChainAsync(chatClientThread.ConversationId);
+        }
+
+        if (chatClientThread.MessageStore is null)
+        {
+            return [];
+        }
+
+        return (await chatClientThread.MessageStore.InvokingAsync(new([]))).ToList();
+    }
+
+    private async Task<List<ChatMessage>> GetChatHistoryFromResponsesChainAsync(string conversationId)
+    {
+        var openAIResponseClient = this._client.GetProjectOpenAIClient().GetProjectResponsesClient();
+        var inputItems = await openAIResponseClient.GetResponseInputItemsAsync(conversationId).ToListAsync();
+        var response = await openAIResponseClient.GetResponseAsync(conversationId);
+        var responseItem = response.Value.OutputItems.FirstOrDefault()!;
+
+        // Take the messages that were the chat history leading up to the current response
+        // remove the instruction messages, and reverse the order so that the most recent message is last.
+        var previousMessages = inputItems
+            .Select(ConvertToChatMessage)
+            .Where(x => x.Text != "You are a helpful assistant.")
+            .Reverse();
+
+        // Convert the response item to a chat message.
+        var responseMessage = ConvertToChatMessage(responseItem);
+
+        // Concatenate the previous messages with the response message to get a full chat history
+        // that includes the current response.
+        return [.. previousMessages, responseMessage];
+    }
+
+    private static ChatMessage ConvertToChatMessage(ResponseItem item)
+    {
+        if (item is MessageResponseItem messageResponseItem)
+        {
+            var role = messageResponseItem.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant;
+            return new ChatMessage(role, messageResponseItem.Content.FirstOrDefault()?.Text);
+        }
+
+        throw new NotSupportedException("This test currently only supports text messages");
+    }
+
+    private async Task<List<ChatMessage>> GetChatHistoryFromConversationAsync(string conversationId)
+    {
+        List<ChatMessage> messages = [];
+        await foreach (AgentResponseItem item in this._client.GetProjectOpenAIClient().GetProjectConversationsClient().GetProjectConversationItemsAsync(conversationId, order: "asc"))
+        {
+            var openAIItem = item.AsResponseResultItem();
+            if (openAIItem is MessageResponseItem messageItem)
+            {
+                messages.Add(new ChatMessage
+                {
+                    Role = new ChatRole(messageItem.Role.ToString()),
+                    Contents = messageItem.Content
+                        .Where(c => c.Kind is ResponseContentPartKind.OutputText or ResponseContentPartKind.InputText)
+                        .Select(c => new TextContent(c.Text))
+                        .ToList<AIContent>()
+                });
+            }
+        }
+
+        return messages;
+    }
+
+    public async Task<ChatClientAgent> CreateChatClientAgentAsync(
+        string name = "HelpfulAssistant",
+        string instructions = "You are a helpful assistant.",
+        IList<AITool>? aiTools = null)
+    {
+        return await this._client.CreateAIAgentAsync(GenerateUniqueAgentName(name), model: s_config.DeploymentName, instructions: instructions, tools: aiTools);
+    }
+
+    public static string GenerateUniqueAgentName(string baseName) =>
+        $"{baseName}-{Guid.NewGuid().ToString("N").Substring(0, 8)}";
+
+    public Task DeleteAgentAsync(ChatClientAgent agent) =>
+        this._client.Agents.DeleteAgentAsync(agent.Name);
+
+    public async Task DeleteThreadAsync(AgentThread thread)
+    {
+        var typedThread = (ChatClientAgentThread)thread;
+        if (typedThread.ConversationId?.StartsWith("conv_", StringComparison.OrdinalIgnoreCase) == true)
+        {
+            await this._client.GetProjectOpenAIClient().GetProjectConversationsClient().DeleteConversationAsync(typedThread.ConversationId);
+        }
+        else if (typedThread.ConversationId?.StartsWith("resp_", StringComparison.OrdinalIgnoreCase) == true)
+        {
+            await this.DeleteResponseChainAsync(typedThread.ConversationId!);
+        }
+    }
+
+    private async Task DeleteResponseChainAsync(string lastResponseId)
+    {
+        var response = await this._client.GetProjectOpenAIClient().GetProjectResponsesClient().GetResponseAsync(lastResponseId);
+        await this._client.GetProjectOpenAIClient().GetProjectResponsesClient().DeleteResponseAsync(lastResponseId);
+
+        if (response.Value.PreviousResponseId is not null)
+        {
+            await this.DeleteResponseChainAsync(response.Value.PreviousResponseId);
+        }
+    }
+
+    public Task DisposeAsync()
+    {
+        if (this._client is not null && this._agent is not null)
+        {
+            return this._client.Agents.DeleteAgentAsync(this._agent.Name);
+        }
+
+        return Task.CompletedTask;
+    }
+
+    public async Task InitializeAsync()
+    {
+        this._client = new(new Uri(s_config.Endpoint), new AzureCliCredential());
+        this._agent = await this.CreateChatClientAgentAsync();
+    }
+}

dotnet/tests/AzureAI.IntegrationTests/AzureAI.IntegrationTests.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <InjectSharedIntegrationTestCode>True</InjectSharedIntegrationTestCode>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
+    <ProjectReference Include="..\AgentConformance.IntegrationTests\AgentConformance.IntegrationTests.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Identity" />
+  </ItemGroup>
+
+</Project>