test

dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/ComputerUseUtil.cs

@@ -0,0 +1,98 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using OpenAI.Responses;
+
+namespace Demo.ComputerUse;
+
+/// <summary>
+/// Enum for tracking the state of the simulated web search flow.
+/// </summary>
+internal enum SearchState
+{
+    Initial,        // Browser search page
+    Typed,          // Text entered in search box
+    PressedEnter   // Enter key pressed, transitioning to results
+}
+
+internal static class ComputerUseUtil
+{
+    /// <summary>
+    /// Load and convert screenshot images to base64 data URLs.
+    /// </summary>
+    internal static Dictionary<string, byte[]> LoadScreenshotAssets()
+    {
+        string baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets");
+
+        ReadOnlySpan<(string key, string fileName)> screenshotFiles =
+            [
+                ("browser_search", "cua_browser_search.png"),
+                ("search_typed", "cua_search_typed.png"),
+                ("search_results", "cua_search_results.png")
+            ];
+
+        Dictionary<string, byte[]> screenshots = [];
+        foreach (var (key, fileName) in screenshotFiles)
+        {
+            string fullPath = Path.GetFullPath(Path.Combine(baseDir, fileName));
+            screenshots[key] = File.ReadAllBytes(fullPath);
+        }
+
+        return screenshots;
+    }
+
+    /// <summary>
+    /// Process a computer action and simulate its execution.
+    /// </summary>
+    internal static (SearchState CurrentState, byte[] ImageBytes) HandleComputerActionAndTakeScreenshot(
+        ComputerCallAction action,
+        SearchState currentState,
+        Dictionary<string, byte[]> screenshots)
+    {
+        Console.WriteLine($"Simulating the execution of computer action: {action.Kind}");
+
+        SearchState newState = DetermineNextState(action, currentState);
+        string imageKey = GetImageKey(newState);
+
+        return (newState, screenshots[imageKey]);
+    }
+
+    private static SearchState DetermineNextState(ComputerCallAction action, SearchState currentState)
+    {
+        string actionType = action.Kind.ToString();
+
+        if (actionType.Equals("type", StringComparison.OrdinalIgnoreCase) && action.TypeText is not null)
+        {
+            return SearchState.Typed;
+        }
+
+        if (IsEnterKeyAction(action, actionType))
+        {
+            Console.WriteLine("  -> Detected ENTER key press");
+            return SearchState.PressedEnter;
+        }
+
+        if (actionType.Equals("click", StringComparison.OrdinalIgnoreCase) && currentState == SearchState.Typed)
+        {
+            Console.WriteLine("  -> Detected click after typing");
+            return SearchState.PressedEnter;
+        }
+
+        return currentState;
+    }
+
+    private static bool IsEnterKeyAction(ComputerCallAction action, string actionType)
+    {
+        return (actionType.Equals("key", StringComparison.OrdinalIgnoreCase) ||
+                actionType.Equals("keypress", StringComparison.OrdinalIgnoreCase)) &&
+               action.KeyPressKeyCodes is not null &&
+               (action.KeyPressKeyCodes.Contains("Return", StringComparer.OrdinalIgnoreCase) ||
+                action.KeyPressKeyCodes.Contains("Enter", StringComparer.OrdinalIgnoreCase));
+    }
+
+    private static string GetImageKey(SearchState state) => state switch
+    {
+        SearchState.PressedEnter => "search_results",
+        SearchState.Typed => "search_typed",
+        _ => "browser_search"
+    };
+}

dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net10.0</TargetFrameworks>
+
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <NoWarn>$(NoWarn);OPENAICUA001</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="Assets\cua_browser_search.png">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\cua_search_results.png">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\cua_search_typed.png">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  
+</Project>

dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/Program.cs

@@ -0,0 +1,174 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+// This sample shows how to use Computer Use Tool with AI Agents.
+
+using Azure.AI.Projects;
+using Azure.AI.Projects.OpenAI;
+using Azure.Identity;
+using Microsoft.Agents.AI;
+using Microsoft.Extensions.AI;
+using OpenAI.Responses;
+
+namespace Demo.ComputerUse;
+
+internal sealed class Program
+{
+    private static async Task Main(string[] args)
+    {
+        string endpoint = Environment.GetEnvironmentVariable("AZURE_FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_FOUNDRY_PROJECT_ENDPOINT is not set.");
+        string deploymentName = Environment.GetEnvironmentVariable("AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME") ?? "computer-use-preview";
+
+        // Get a client to create/retrieve/delete server side agents with Azure Foundry Agents.
+        AIProjectClient aiProjectClient = new(new Uri(endpoint), new AzureCliCredential());
+        const string AgentInstructions = @"
+                    You are a computer automation assistant. 
+                    
+                    Be direct and efficient. When you reach the search results page, read and describe the actual search result titles and descriptions you can see.
+                ";
+
+        const string AgentNameMEAI = "ComputerAgent-MEAI";
+        const string AgentNameNative = "ComputerAgent-NATIVE";
+
+        // Option 1 - Using ComputerUseTool + AgentOptions (MEAI + AgentFramework)
+        // Create AIAgent directly
+        AIAgent agentOption1 = await aiProjectClient.CreateAIAgentAsync(
+            name: AgentNameMEAI,
+            model: deploymentName,
+            instructions: AgentInstructions,
+            description: "Computer automation agent with screen interaction capabilities.",
+            tools: [
+                    ResponseTool.CreateComputerTool(ComputerToolEnvironment.Browser, 1026, 769).AsAITool(),
+                ]);
+
+        // Option 2 - Using PromptAgentDefinition SDK native type
+        // Create the server side agent version
+        AIAgent agentOption2 = await aiProjectClient.CreateAIAgentAsync(
+            name: AgentNameNative,
+            creationOptions: new AgentVersionCreationOptions(
+                new PromptAgentDefinition(model: deploymentName)
+                {
+                    Instructions = AgentInstructions,
+                    Tools = { ResponseTool.CreateComputerTool(
+                environment: new ComputerToolEnvironment("windows"),
+                displayWidth: 1026,
+                displayHeight: 769) }
+                })
+        );
+
+        // Either invoke option1 or option2 agent, should have same result
+        // Option 1
+        await InvokeComputerUseAgentAsync(agentOption1);
+
+        // Option 2
+        //await InvokeComputerUseAgentAsync(agentOption2);
+
+        // Cleanup by agent name removes the agent version created.
+        await aiProjectClient.Agents.DeleteAgentAsync(agentOption1.Name);
+        await aiProjectClient.Agents.DeleteAgentAsync(agentOption2.Name);
+    }
+
+    private static async Task InvokeComputerUseAgentAsync(AIAgent agent)
+    {
+        // Load screenshot assets
+        Dictionary<string, byte[]> screenshots = ComputerUseUtil.LoadScreenshotAssets();
+
+        ChatOptions chatOptions = new();
+        CreateResponseOptions responseCreationOptions = new()
+        {
+            TruncationMode = ResponseTruncationMode.Auto
+        };
+        chatOptions.RawRepresentationFactory = (_) => responseCreationOptions;
+        ChatClientAgentRunOptions runOptions = new(chatOptions)
+        {
+            AllowBackgroundResponses = true,
+        };
+
+        AgentThread thread = await agent.GetNewThreadAsync();
+
+        ChatMessage message = new(ChatRole.User, [
+            new TextContent("I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete."),
+            new DataContent(new BinaryData(screenshots["browser_search"]), "image/png")
+        ]);
+
+        // Initial request with screenshot - start with Bing search page
+        Console.WriteLine("Starting computer automation session (initial screenshot: cua_browser_search.png)...");
+
+        AgentResponse response = await agent.RunAsync(message, thread: thread, options: runOptions);
+
+        // Main interaction loop
+        const int MaxIterations = 10;
+        int iteration = 0;
+        // Initialize state machine
+        SearchState currentState = SearchState.Initial;
+        string initialCallId = string.Empty;
+
+        while (true)
+        {
+            // Poll until the response is complete.
+            while (response.ContinuationToken is { } token)
+            {
+                // Wait before polling again.
+                await Task.Delay(TimeSpan.FromSeconds(2));
+
+                // Continue with the token.
+                runOptions.ContinuationToken = token;
+
+                response = await agent.RunAsync(thread, runOptions);
+            }
+
+            Console.WriteLine($"Agent response received (ID: {response.ResponseId})");
+
+            if (iteration >= MaxIterations)
+            {
+                Console.WriteLine($"\nReached maximum iterations ({MaxIterations}). Stopping.");
+                break;
+            }
+
+            iteration++;
+            Console.WriteLine($"\n--- Iteration {iteration} ---");
+
+            // Check for computer calls in the response
+            IEnumerable<ComputerCallResponseItem> computerCallResponseItems = response.Messages
+                .SelectMany(x => x.Contents)
+                .Where(c => c.RawRepresentation is ComputerCallResponseItem and not null)
+                .Select(c => (ComputerCallResponseItem)c.RawRepresentation!);
+
+            ComputerCallResponseItem? firstComputerCall = computerCallResponseItems.FirstOrDefault();
+            if (firstComputerCall is null)
+            {
+                Console.WriteLine("No computer call actions found. Ending interaction.");
+                Console.WriteLine($"Final Response: {response}");
+                break;
+            }
+
+            // Process the first computer call response
+            ComputerCallAction action = firstComputerCall.Action;
+            string currentCallId = firstComputerCall.CallId;
+
+            // Set the initial computer call ID for tracking and subsequent responses.
+            if (string.IsNullOrEmpty(initialCallId))
+            {
+                initialCallId = currentCallId;
+            }
+
+            Console.WriteLine($"Processing computer call (ID: {currentCallId})");
+
+            // Simulate executing the action and taking a screenshot
+            (SearchState CurrentState, byte[] ImageBytes) screenInfo = ComputerUseUtil.HandleComputerActionAndTakeScreenshot(action, currentState, screenshots);
+            currentState = screenInfo.CurrentState;
+
+            Console.WriteLine("Sending action result back to agent...");
+
+            AIContent content = new()
+            {
+                RawRepresentation = new ComputerCallOutputResponseItem(
+                    initialCallId,
+                    output: ComputerCallOutput.CreateScreenshotOutput(new BinaryData(screenInfo.ImageBytes), "image/png"))
+            };
+
+            // Follow-up message with action result and new screenshot
+            message = new(ChatRole.User, [content]);
+            response = await agent.RunAsync(message, thread: thread, options: runOptions);
+        }
+    }
+}

dotnet/samples/GettingStarted/FoundryAgents/FoundryAgents_Step15_ComputerUse/README.md

@@ -0,0 +1,55 @@
+# Using Computer Use Tool with AI Agents
+
+This sample demonstrates how to use the computer use tool with AI agents. The computer use tool allows agents to interact with a computer environment by viewing the screen, controlling the mouse and keyboard, and performing various actions to help complete tasks.
+
+## What this sample demonstrates
+
+- Creating agents with computer use capabilities
+- Using HostedComputerTool (MEAI abstraction)
+- Using native SDK computer use tools (ResponseTool.CreateComputerTool)
+- Extracting computer action information from agent responses
+- Handling computer tool results (text output and screenshots)
+- Managing agent lifecycle (creation and deletion)
+
+## Prerequisites
+
+Before you begin, ensure you have the following prerequisites:
+
+- .NET 10 SDK or later
+- Azure Foundry service endpoint and deployment configured
+- Azure CLI installed and authenticated (for Azure credential authentication)
+
+**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
+
+Set the following environment variables:
+
+```powershell
+$env:AZURE_FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint
+$env:AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME="computer-use-preview"  # Optional, defaults to computer-use-preview
+```
+
+## Run the sample
+
+Navigate to the FoundryAgents sample directory and run:
+
+```powershell
+cd dotnet/samples/GettingStarted/FoundryAgents
+dotnet run --project .\FoundryAgents_Step15_ComputerUse
+```
+
+## Expected behavior
+
+The sample will:
+
+1. Create two agents with computer use capabilities:
+   - Option 1: Using HostedComputerTool (MEAI abstraction)
+   - Option 2: Using native SDK computer use tools
+2. Run the agent with a task: "I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete."
+3. The agent will use the computer use tool to:
+   - Interpret the screenshots
+   - Issue action requests based on the task
+   - Analyze the search results for "OpenAI news" from the screenshots.
+4. Extract and display the computer actions performed
+5. Display the results from the computer tool execution
+6. Display the final response from the agent
+7. Clean up resources by deleting both agents