diff --git a/src/xAI.Tests/ChatClientTests.cs b/src/xAI.Tests/ChatClientTests.cs index 7323cd0..df19eb6 100644 --- a/src/xAI.Tests/ChatClientTests.cs +++ b/src/xAI.Tests/ChatClientTests.cs @@ -25,7 +25,7 @@ public async Task GrokInvokesTools() { "user", "What day is today?" }, }; - var chat = new GrokClient(Configuration["XAI_API_KEY"]!).AsIChatClient("grok-4") + var chat = new GrokClient(Configuration["XAI_API_KEY"]!).AsIChatClient("grok-4-1-fast") .AsBuilder() .UseLogging(output.AsLoggerFactory()) .Build(); @@ -60,7 +60,7 @@ public async Task GrokInvokesToolAndSearch() { "user", "What's Tesla stock worth today?" }, }; - var grok = new GrokClient(Configuration["XAI_API_KEY"]!).AsIChatClient("grok-4") + var grok = new GrokClient(Configuration["XAI_API_KEY"]!).AsIChatClient("grok-4-1-fast") .AsBuilder() .UseFunctionInvocation() .UseLogging(output.AsLoggerFactory()) diff --git a/src/xAI.Tests/SanityChecks.cs b/src/xAI.Tests/SanityChecks.cs index f5bda92..345807f 100644 --- a/src/xAI.Tests/SanityChecks.cs +++ b/src/xAI.Tests/SanityChecks.cs @@ -1,9 +1,13 @@ using System.Text.Json; using Devlooped.Extensions.AI; +using DotNetEnv; +using Grpc.Core; +using Grpc.Net.Client.Configuration; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; using xAI.Protocol; using Xunit.Abstractions; +using Xunit.Sdk; using ChatConversation = Devlooped.Extensions.AI.Chat; namespace xAI.Tests; @@ -151,65 +155,14 @@ public async Task ExecuteLocalFunctionWithWebSearch() Assert.NotEmpty(finalOutput.Message.Content); } - /// - /// Comprehensive integration test (non-streaming) that exercises all major features: - /// - Client-side tool invocation (AIFunctionFactory) - /// - Hosted web search tool - /// - Hosted code interpreter tool - /// - Hosted MCP server tool (GitHub) - /// - Citations and annotations - /// - [SecretsFact("CI_XAI_API_KEY", "GITHUB_TOKEN")] - public async Task IntegrationTest() - { - var (grok, options, getDateCalls) = SetupIntegrationTest(); - - var response = await grok.GetResponseAsync(CreateIntegrationChat(), options); - - AssertIntegrationTest(response, getDateCalls); - } - - [SecretsFact("CI_XAI_API_KEY", "GITHUB_TOKEN")] - public async Task IntegrationTestStreaming() - { - var (grok, options, getDateCalls) = SetupIntegrationTest(); - - var updates = await grok.GetStreamingResponseAsync(CreateIntegrationChat(), options).ToListAsync(); - var response = updates.ToChatResponse(); - - AssertIntegrationTest(response, getDateCalls); - } - - static ChatConversation CreateIntegrationChat() => new() - { - { "system", "You are a helpful assistant that uses all available tools to answer questions accurately." }, - { "user", - $$""" - Please answer the following questions using the appropriate tools: - 1. What is today's date? (use get_date tool) - 2. What is the current price of Tesla (TSLA) stock? (use Yahoo news web search, always include citations) - 3. What is the top news from Tesla on X? - 4. Calculate the earnings that would be produced by compound interest to $5k savings at 4% annually for 5 years (use code interpreter). - Return just the earnings, not the grand total of savings plus earnings). - 5. What is the latest release version of the {{ThisAssembly.Git.Url}} repository? (use GitHub MCP tool) - - Respond with a JSON object in this exact format: - { - "today": "[date from get_date in YYYY-MM-DD format]", - "tesla_price": [numeric price from web search], - "tesla_news": "[top news from X]", - "compound_interest": [numeric result from code interpreter], - "latest_release": "[version string from GitHub]" - } - """ - } - }; - - static (IChatClient grok, GrokChatOptions options, Func getDateCalls) SetupIntegrationTest() + [SecretsTheory("CI_XAI_API_KEY")] + [InlineData(false)] + [InlineData(true)] + public async Task ClientSideFunction(bool streaming) { var getDateCalls = 0; - var grok = new GrokClient(Environment.GetEnvironmentVariable("CI_XAI_API_KEY")!) - .AsIChatClient("grok-4-1-fast-reasoning") + var grok = new GrokClient(Env.GetString("CI_XAI_API_KEY")!) + .AsIChatClient("grok-4-1-fast") .AsBuilder() .UseFunctionInvocation() .Build(); @@ -217,14 +170,6 @@ 3. What is the top news from Tesla on X? var options = new GrokChatOptions { ResponseFormat = ChatResponseFormat.Json, - Include = - [ - IncludeOption.InlineCitations, - IncludeOption.WebSearchCallOutput, - IncludeOption.CodeExecutionCallOutput, - IncludeOption.McpCallOutput, - IncludeOption.XSearchCallOutput, - ], Tools = [ AIFunctionFactory.Create(() => @@ -232,33 +177,52 @@ 3. What is the top news from Tesla on X? getDateCalls++; return DateTime.Now.ToString("yyyy-MM-dd"); }, "get_date", "Gets the current date in YYYY-MM-DD format"), - new HostedWebSearchTool(), - new HostedCodeInterpreterTool(), - new HostedMcpServerTool("GitHub", "https://api.githubcopilot.com/mcp/") - { - AuthorizationToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN")!, - AllowedTools = ["list_releases", "get_release_by_tag"], - }, - new GrokXSearchTool - { - AllowedHandles = ["tesla"] - } ] }; - return (grok, options, () => getDateCalls); + var chat = new ChatConversation + { + { "system", "You are a helpful assistant." }, + { "user", """ + What is today's date? Use the get_date tool. + Respond with a JSON object: { "today": "[date in YYYY-MM-DD format]" } + """ } + }; + + var response = await GetResponseAsync(grok, chat, options, streaming); + + Assert.True(getDateCalls >= 1, "get_date function was not called"); + + var result = ParseJson(response, output); + Assert.Equal(DateTime.Today.ToString("yyyy-MM-dd"), result.Today); + output.WriteLine($"Today: {result.Today}"); } - void AssertIntegrationTest(ChatResponse response, Func getDateCalls) + [SecretsTheory("CI_XAI_API_KEY")] + [InlineData(false)] + [InlineData(true)] + public async Task AgenticWebSearch(bool streaming) { - // Verify response basics - Assert.NotNull(response); - Assert.NotNull(response.ModelId); - Assert.NotEmpty(response.Messages); - Assert.NotNull(response.Usage); + var grok = new GrokClient(Env.GetString("CI_XAI_API_KEY")!) + .AsIChatClient("grok-4-1-fast"); - // Verify client-side tool was invoked - Assert.True(getDateCalls() >= 1); + var options = new GrokChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + Include = [IncludeOption.WebSearchCallOutput], + Tools = [new HostedWebSearchTool()] + }; + + var chat = new ChatConversation + { + { "system", "You are a helpful assistant." }, + { "user", """ + What is the current price of Tesla (TSLA) stock? Use web search (Yahoo Finance or similar). + Respond with a JSON object: { "tesla_price": [numeric price] } + """ } + }; + + var response = await GetResponseAsync(grok, chat, options, streaming); // Verify web search tool was used var webSearchCalls = response.Messages @@ -267,91 +231,258 @@ void AssertIntegrationTest(ChatResponse response, Func getDateCalls) .ToList(); Assert.NotEmpty(webSearchCalls); - // Verify code interpreter tool was used - var codeInterpreterCalls = response.Messages + // Verify citations were produced + var citations = response.Messages .SelectMany(x => x.Contents) - .OfType() + .SelectMany(x => x.Annotations?.OfType() ?? []) + .Where(x => x.Url is not null) .ToList(); - Assert.NotEmpty(codeInterpreterCalls); + Assert.NotEmpty(citations); - // Verify code interpreter output was included - var codeInterpreterResults = response.Messages - .SelectMany(x => x.Contents) - .OfType() + var result = ParseJson(response, output); + Assert.True(result.TeslaPrice > 100, $"Tesla price {result.TeslaPrice} should be > 100"); + output.WriteLine($"Tesla price: {result.TeslaPrice}"); + } + + [SecretsTheory("CI_XAI_API_KEY")] + [InlineData(false)] + [InlineData(true)] + public async Task AgenticXSearch(bool streaming) + { + var grok = new GrokClient(Env.GetString("CI_XAI_API_KEY")!) + .AsIChatClient("grok-4-1-fast"); + + var options = new GrokChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + Include = [IncludeOption.XSearchCallOutput], + Tools = [new GrokXSearchTool { AllowedHandles = ["tesla"] }] + }; + + var chat = new ChatConversation + { + { "system", "You are a helpful assistant." }, + { "user", """ + What is the top news from Tesla on X? Use the X search tool. + Respond with a JSON object: { "tesla_news": "[top news headline or summary]" } + """ } + }; + + var response = await GetResponseAsync(grok, chat, options, streaming); + + // Verify X search tool was used + var xSearchCalls = response.Messages + .SelectMany(x => x.Contents.Select(c => c.RawRepresentation as xAI.Protocol.ToolCall)) + .Where(x => x?.Type == xAI.Protocol.ToolCallType.XSearchTool) .ToList(); - Assert.NotEmpty(codeInterpreterResults); + Assert.NotEmpty(xSearchCalls); + + var result = ParseJson(response, output); + Assert.NotNull(result.TeslaNews); + Assert.NotEmpty(result.TeslaNews); + output.WriteLine($"Tesla X news: {result.TeslaNews}"); + } + + [SecretsTheory("CI_XAI_API_KEY", "GITHUB_TOKEN")] + [InlineData(false)] + [InlineData(true)] + public async Task AgenticMcpServer(bool streaming) + { + var grok = new GrokClient(Env.GetString("CI_XAI_API_KEY")!) + .AsIChatClient("grok-4-1-fast"); + + var options = new GrokChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + Include = [IncludeOption.McpCallOutput], + Tools = + [ + new HostedMcpServerTool("GitHub", "https://api.githubcopilot.com/mcp/") + { + AuthorizationToken = Env.GetString("GITHUB_TOKEN")!, + AllowedTools = ["list_releases", "get_release_by_tag"], + } + ] + }; + + var chat = new ChatConversation + { + { "system", "You are a helpful assistant." }, + { "user", $$""" + What is the latest release version of the {{ThisAssembly.Git.Url}} repository? Use the GitHub MCP tool. + Respond with a JSON object: { "latest_release": "[version string]" } + """ } + }; + + var response = await GetResponseAsync(grok, chat, options, streaming); - // Verify MCP tool was used var mcpCalls = response.Messages .SelectMany(x => x.Contents) .OfType() .ToList(); Assert.NotEmpty(mcpCalls); - // Verify MCP output was included var mcpResults = response.Messages .SelectMany(x => x.Contents) .OfType() .ToList(); Assert.NotEmpty(mcpResults); - // Verify citations from web search - Assert.NotEmpty(response.Messages - .SelectMany(x => x.Contents) - .SelectMany(x => x.Annotations?.OfType() ?? []) - .Where(x => x.Url is not null) - .Select(x => x.Url!)); - - // Parse and validate the JSON response - var responseText = response.Messages.Last().Text; - Assert.NotNull(responseText); + var result = ParseJson(response, output); + Assert.NotNull(result.LatestRelease); + Assert.Contains(".", result.LatestRelease); + output.WriteLine($"Latest release: {result.LatestRelease}"); + output.WriteLine($"MCP calls: {mcpCalls.Count}"); + } - output.WriteLine("Response text:"); - output.WriteLine(responseText); + [SecretsTheory("CI_XAI_API_KEY")] + [InlineData(false)] + [InlineData(true)] + public async Task AgenticFileSearch(bool streaming) + { + var grok = new GrokClient(Env.GetString("CI_XAI_API_KEY")!) + .AsIChatClient("grok-4-1-fast"); - // Extract JSON from response (may be wrapped in markdown code blocks) - var jsonStart = responseText.IndexOf('{'); - var jsonEnd = responseText.LastIndexOf('}'); - if (jsonStart >= 0 && jsonEnd > jsonStart) + var options = new GrokChatOptions { - var json = responseText.Substring(jsonStart, jsonEnd - jsonStart + 1); - var result = JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web) - { - PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower - }); + ResponseFormat = ChatResponseFormat.Json, + Include = [IncludeOption.CollectionsSearchCallOutput], + ToolMode = ChatToolMode.RequireAny, + Tools = + [ + new HostedFileSearchTool + { + Inputs = [new HostedVectorStoreContent("collection_91559d9b-a55d-42fe-b2ad-ecf8904d9049")] + } + ] + }; - Assert.NotNull(result); + var chat = new ChatConversation + { + { "system", "You are a helpful assistant." }, + { "user", """ + What is the law number of Código Procesal Civil y Comercial de la Nación? + Use the collection search tool. + Respond with a JSON object: { "law_number": [numeric law number without group separator] } + """ } + }; - // Verify date is today - Assert.Equal(DateTime.Today.ToString("yyyy-MM-dd"), result.Today); + var response = await GetResponseAsync(grok, chat, options, streaming); - // Verify Tesla price is reasonable (greater than $100) - Assert.True(result.TeslaPrice > 100, $"Tesla price {result.TeslaPrice} should be > 100"); + Assert.Contains( + response.Messages.SelectMany(x => x.Contents).OfType() + .Select(x => x.RawRepresentation as xAI.Protocol.ToolCall), + x => x?.Type == xAI.Protocol.ToolCallType.CollectionsSearchTool); - // Verify compound interest calculation is approximately correct - // Formula: P(1 + r)^t - P = 5000 * (1.04)^5 - 5000 ≈ $1,083.26 - Assert.True(result.CompoundInterest > 1000 && result.CompoundInterest < 1200, - $"Compound interest {result.CompoundInterest} should be between 1000 and 1200"); + var files = response.Messages + .SelectMany(x => x.Contents).OfType() + .SelectMany(x => (x.Outputs ?? []).OfType()) + .ToList(); - // Verify latest release contains version pattern - Assert.NotNull(result.LatestRelease); - Assert.Contains(".", result.LatestRelease); + if (files.Count == 0) + { + var search = response.Messages + .SelectMany(x => x.Contents).OfType() + .First(); - output.WriteLine($"Parsed response: Today={result.Today}, TeslaPrice={result.TeslaPrice}, CompoundInterest={result.CompoundInterest}, LatestRelease={result.LatestRelease}"); + Assert.Fail("Expected at least one file in the collection search results: " + + new ChatMessage(ChatRole.Tool, search.Outputs).Text); } - else + + output.WriteLine(string.Join(", ", files.Select(x => x.Name))); + Assert.Contains(files, x => x.Name?.Contains("LNS0004592") == true); + + var result = ParseJson(response, output); + Assert.Equal(17454, result.LawNumber); + output.WriteLine($"Law number: {result.LawNumber}"); + } + + /// + /// Code execution is flaky and can produce: + /// Grpc.Core.RpcException : Status(StatusCode="Unavailable", Detail="Bad gRPC response. HTTP status code: 504") + /// + [SecretsTheory("CI_XAI_API_KEY")] + [InlineData(false)] + [InlineData(true)] + public async Task AgenticCodeInterpreter(bool streaming) + { + var client = new GrokClient(Env.GetString("CI_XAI_API_KEY")!); + + var grok = client.AsIChatClient("grok-4-1-fast"); + + var options = new GrokChatOptions { - Assert.Fail("Response did not contain expected JSON output"); - } + Include = [IncludeOption.CodeExecutionCallOutput], + Tools = [new HostedCodeInterpreterTool()] + }; + + var chat = new ChatConversation + { + { "system", "You are a helpful assistant." }, + { "user", """ + Calculate the earnings produced by compound interest on $5,000 at 4% annually for 5 years. + Use the code interpreter. Return just the earnings number (not the total principal + earnings), + no additional text or formatting, and no explanation. The output should be a single numeric value + parseable by a decimal parser. + """ } + }; + + var response = await GetResponseAsync(grok, chat, options, streaming); + output.WriteLine($"Compound interest: {response.Text}"); + + var codeInterpreterCalls = response.Messages + .SelectMany(x => x.Contents) + .OfType() + .ToList(); + Assert.NotEmpty(codeInterpreterCalls); + + var codeInterpreterResults = response.Messages + .SelectMany(x => x.Contents) + .OfType() + .ToList(); + Assert.NotEmpty(codeInterpreterResults); + + // Formula: P(1 + r)^t - P = 5000 * (1.04)^5 - 5000 ≈ $1,083.26 + Assert.NotEmpty(response.Text); + Assert.True(decimal.TryParse(response.Text, out var result), $"Could not parse response {response.Text}"); + Assert.True(result > 1000 && result < 1200, + $"Compound interest {result} should be between 1000 and 1200"); output.WriteLine($"Code interpreter calls: {codeInterpreterCalls.Count}"); - output.WriteLine($"MCP calls: {mcpCalls.Count}"); } - record IntegrationTestResponse( - string Today, - decimal TeslaPrice, - string TeslaNews, - decimal CompoundInterest, - string LatestRelease); + static async Task GetResponseAsync(IChatClient client, ChatConversation chat, GrokChatOptions options, bool streaming) + { + if (!streaming) + return await client.GetResponseAsync(chat, options); + + var updates = await client.GetStreamingResponseAsync(chat, options).ToListAsync(); + return updates.ToChatResponse(); + } + + static T ParseJson(ChatResponse response, ITestOutputHelper output) + { + var responseText = response.Messages.Last().Text; + Assert.NotNull(responseText); + output.WriteLine("Response text:"); + output.WriteLine(responseText); + + var jsonStart = responseText.IndexOf('{'); + var jsonEnd = responseText.LastIndexOf('}'); + Assert.True(jsonStart >= 0 && jsonEnd > jsonStart, "Response did not contain a JSON object"); + + var json = responseText[jsonStart..(jsonEnd + 1)]; + var result = JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower + }); + Assert.NotNull(result); + return result; + } + + record DateResult(string Today); + record TeslaPriceResult(decimal TeslaPrice); + record TeslaNewsResult(string TeslaNews); + record LatestReleaseResult(string LatestRelease); + record LawNumberResult(int LawNumber); } diff --git a/src/xAI.Tests/xAI.Tests.csproj b/src/xAI.Tests/xAI.Tests.csproj index d9ed57a..f27325f 100644 --- a/src/xAI.Tests/xAI.Tests.csproj +++ b/src/xAI.Tests/xAI.Tests.csproj @@ -29,9 +29,6 @@ - - - diff --git a/src/xAI/CollectionSearchToolCallContent.cs b/src/xAI/CollectionSearchToolCallContent.cs index f80f6ee..aaed26f 100644 --- a/src/xAI/CollectionSearchToolCallContent.cs +++ b/src/xAI/CollectionSearchToolCallContent.cs @@ -2,7 +2,7 @@ namespace xAI; -/// Represents a hosted tool agentic call. +/// Represents a hosted collection search call. public class CollectionSearchToolCallContent : AIContent { /// Gets or sets the tool call ID. diff --git a/src/xAI/CollectionSearchToolResultContent.cs b/src/xAI/CollectionSearchToolResultContent.cs index ac490b3..c728b6e 100644 --- a/src/xAI/CollectionSearchToolResultContent.cs +++ b/src/xAI/CollectionSearchToolResultContent.cs @@ -2,7 +2,7 @@ namespace xAI; -/// Represents a hosted tool agentic call. +/// Represents a hosted collection search call result. public class CollectionSearchToolResultContent : AIContent { /// Gets or sets the tool call ID. diff --git a/src/xAI/GrokChatClient.cs b/src/xAI/GrokChatClient.cs index 410a89c..cc20452 100644 --- a/src/xAI/GrokChatClient.cs +++ b/src/xAI/GrokChatClient.cs @@ -229,7 +229,6 @@ codeResult.RawRepresentation is ToolCall codeToolCall && request.Messages.Add(gmsg); } - IList includes = []; if (options is GrokChatOptions grokOptions) { request.Include.AddRange(grokOptions.Include); diff --git a/src/xAI/GrokProtocolExtensions.cs b/src/xAI/GrokProtocolExtensions.cs index 831ab01..df9e3d0 100644 --- a/src/xAI/GrokProtocolExtensions.cs +++ b/src/xAI/GrokProtocolExtensions.cs @@ -80,7 +80,7 @@ grokSearch.Region is not null || grokSearch.City is not null || grokSearch.Timezone is not null) { - websearch.UserLocation = new WebSearchUserLocation(); + websearch.UserLocation = new(); if (grokSearch.Country is not null) websearch.UserLocation.Country = grokSearch.Country; if (grokSearch.Region is not null) @@ -104,11 +104,11 @@ grokSearch.City is not null || } else { - return new Tool { WebSearch = new WebSearch() }; + return new Tool { WebSearch = new() }; } case HostedCodeInterpreterTool: - return new Tool { CodeExecution = new CodeExecution { } }; + return new Tool { CodeExecution = new() }; case HostedFileSearchTool fileSearch: var collectionTool = new CollectionsSearch(); @@ -194,6 +194,15 @@ static IEnumerable ToChatMessages(IEnumerable me RawRepresentation = completion }); } + //else if (completion.Role == MessageRole.RoleTool && message.Contents.Count == 0 && content?.Length > 0) + //{ + // // For tool messages with no content, we can still create a message to hold annotations and completion + // message.Contents.Add(new TextContent(content) + // { + // Annotations = annotations, + // RawRepresentation = completion + // }); + //} } if (message is not null) @@ -280,6 +289,18 @@ internal static IEnumerable AsContents(this IEnumerable too result.Outputs = outputs; yield return result; } + else + { + // If we can't parse the references, still return as single raw content + // for further inspection by consumers. + yield return new CollectionSearchToolResultContent + { + Annotations = annotations, + RawRepresentation = toolCall, + CallId = toolCall.Id, + Outputs = [new TextContent(content)] + }; + } } else {