From 917719d36e6ff9021d411774289365ac00fb8f57 Mon Sep 17 00:00:00 2001 From: Stevan Freeborn <65925598+StevanFreeborn@users.noreply.github.com> Date: Wed, 3 Jul 2024 00:11:48 -0500 Subject: [PATCH] tests: work on adding integration tests --- src/AnthropicClient/AnthropicApiClient.cs | 6 + src/AnthropicClient/Models/ChatResponse.cs | 4 +- .../AnthropicClient.Tests.csproj | 1 + .../Integration/AnthropicApiClientTests.cs | 227 +++++++++++ .../Integration/EventTestData.cs | 385 ++++++++++++++++++ .../Integration/IntegrationTest.cs | 4 +- tests/AnthropicClient.Tests/Usings.cs | 2 + 7 files changed, 625 insertions(+), 4 deletions(-) create mode 100644 tests/AnthropicClient.Tests/Integration/EventTestData.cs diff --git a/src/AnthropicClient/AnthropicApiClient.cs b/src/AnthropicClient/AnthropicApiClient.cs index 1d33ad7..2dd013e 100644 --- a/src/AnthropicClient/AnthropicApiClient.cs +++ b/src/AnthropicClient/AnthropicApiClient.cs @@ -213,6 +213,12 @@ public class AnthropicApiClient : IAnthropicApiClient if (line is null) { + if (string.IsNullOrWhiteSpace(currentEvent.Type) is false) + { + yield return currentEvent; + currentEvent = new AnthropicEvent(); + } + break; } diff --git a/src/AnthropicClient/Models/ChatResponse.cs b/src/AnthropicClient/Models/ChatResponse.cs index 087066c..6f61fad 100644 --- a/src/AnthropicClient/Models/ChatResponse.cs +++ b/src/AnthropicClient/Models/ChatResponse.cs @@ -26,13 +26,13 @@ public class ChatResponse /// Gets the stop reason of the chat response. /// [JsonPropertyName("stop_reason")] - public string StopReason { get; init; } = string.Empty; + public string? StopReason { get; init; } /// /// Gets the stop sequence of the chat response. /// [JsonPropertyName("stop_sequence")] - public string StopSequence { get; init; } = string.Empty; + public string? StopSequence { get; init; } /// /// Gets the type of the chat response. diff --git a/tests/AnthropicClient.Tests/AnthropicClient.Tests.csproj b/tests/AnthropicClient.Tests/AnthropicClient.Tests.csproj index f21321f..5dac8d0 100644 --- a/tests/AnthropicClient.Tests/AnthropicClient.Tests.csproj +++ b/tests/AnthropicClient.Tests/AnthropicClient.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/tests/AnthropicClient.Tests/Integration/AnthropicApiClientTests.cs b/tests/AnthropicClient.Tests/Integration/AnthropicApiClientTests.cs index d073564..f5babc0 100644 --- a/tests/AnthropicClient.Tests/Integration/AnthropicApiClientTests.cs +++ b/tests/AnthropicClient.Tests/Integration/AnthropicApiClientTests.cs @@ -31,4 +31,231 @@ public class AnthropicApiClientTests : IntegrationTest var actualErrorType = result.Error.Error!.GetType(); actualErrorType.Should().Be(errorType); } + + [Fact] + public async Task CreateChatMessageAsync_WhenCalledAndMessageCreatedWithTextContent_ItShouldReturnMessage() + { + _mockHttpMessageHandler + .WhenCreateMessageRequest() + .Respond( + HttpStatusCode.OK, + "application/json", + @"{ + ""content"": [ + { + ""text"": ""Hi! My name is Claude."", + ""type"": ""text"" + } + ], + ""id"": ""msg_013Zva2CMHLNnXjNJJKqJ2EF"", + ""model"": ""claude-3-5-sonnet-20240620"", + ""role"": ""assistant"", + ""stop_reason"": ""end_turn"", + ""stop_sequence"": null, + ""type"": ""message"", + ""usage"": { + ""input_tokens"": 10, + ""output_tokens"": 25 + } + }" + ); + + var request = new ChatMessageRequest( + model: AnthropicModels.Claude35Sonnet, + messages: [new(MessageRole.User, [new TextContent("Hello!")])] + ); + + var result = await Client.CreateChatMessageAsync(request); + + result.IsSuccess.Should().BeTrue(); + result.Value.Should().BeOfType(); + result.Error.Should().BeNull(); + + var message = result.Value; + message.Id.Should().Be("msg_013Zva2CMHLNnXjNJJKqJ2EF"); + message.Model.Should().Be("claude-3-5-sonnet-20240620"); + message.Role.Should().Be("assistant"); + message.StopReason.Should().Be("end_turn"); + message.StopSequence.Should().BeNull(); + message.Type.Should().Be("message"); + message.Usage.InputTokens.Should().Be(10); + message.Usage.OutputTokens.Should().Be(25); + message.Content.Should().HaveCount(1); + message.ToolCall.Should().BeNull(); + + var textContent = message.Content[0]; + textContent.Should().BeOfType(); + textContent.As().Text.Should().Be("Hi! My name is Claude."); + textContent.As().Type.Should().Be("text"); + } + + [Fact] + public async Task CreateChatMessageAsync_WhenCalledAndMessageCreatedWithToolUseContentAndToolIsProvided_ItShouldReturnMessageWithToolCall() + { + _mockHttpMessageHandler + .WhenCreateMessageRequest() + .Respond( + HttpStatusCode.OK, + "application/json", + @"{ + ""content"": [ + { + ""id"": ""toolu_01D7FLrfh4GYq7yT1ULFeyMV"", + ""name"": ""get_stock_price"", + ""input"": { ""ticker"": ""^GSPC"" }, + ""type"": ""tool_use"" + } + ], + ""id"": ""msg_01D7FLrfh4GYq7yT1ULFeyMV"", + ""model"": ""claude-3-5-sonnet-20240620"", + ""role"": ""assistant"", + ""stop_reason"": ""end_turn"", + ""stop_sequence"": null, + ""type"": ""message"", + ""usage"": { + ""input_tokens"": 10, + ""output_tokens"": 25 + } + }" + ); + + var func = (string ticker) => ticker; + + var request = new ChatMessageRequest( + model: AnthropicModels.Claude35Sonnet, + messages: [ + new(MessageRole.User, [new TextContent("Hello!")]), + ], + tools: [ + Tool.CreateFromFunction("get_stock_price", "Gets the stock price for a given ticker", func) + ] + ); + + var result = await Client.CreateChatMessageAsync(request); + + result.IsSuccess.Should().BeTrue(); + result.Value.Should().BeOfType(); + result.Error.Should().BeNull(); + + var message = result.Value; + message.Id.Should().Be("msg_01D7FLrfh4GYq7yT1ULFeyMV"); + message.Model.Should().Be("claude-3-5-sonnet-20240620"); + message.Role.Should().Be("assistant"); + message.StopReason.Should().Be("end_turn"); + message.StopSequence.Should().BeNull(); + message.Type.Should().Be("message"); + message.Usage.InputTokens.Should().Be(10); + message.Usage.OutputTokens.Should().Be(25); + message.Content.Should().HaveCount(1); + message.ToolCall.Should().NotBeNull(); + + var toolUseContent = message.Content[0]; + toolUseContent.Should().BeOfType(); + + toolUseContent.As().Id.Should().Be("toolu_01D7FLrfh4GYq7yT1ULFeyMV"); + toolUseContent.As().Name.Should().Be("get_stock_price"); + toolUseContent.As().Type.Should().Be("tool_use"); + + var input = toolUseContent.As().Input; + var ticker = input.GetValueOrDefault("ticker"); + ticker!.ToString().Should().Be("^GSPC"); + + var toolCallResult = await message.ToolCall!.InvokeAsync(); + toolCallResult.IsSuccess.Should().BeTrue(); + toolCallResult.Value!.ToString().Should().Be("^GSPC"); + toolCallResult.Error.Should().BeNull(); + } + + + [Fact] + public async Task CreateChatMessageAsync_WhenCalledAndMessageCreatedWithToolUseButNoToolProvided_ItShouldReturnMessageWithoutToolCall() + { + _mockHttpMessageHandler + .WhenCreateMessageRequest() + .Respond( + HttpStatusCode.OK, + "application/json", + @"{ + ""content"": [ + { + ""id"": ""toolu_01D7FLrfh4GYq7yT1ULFeyMV"", + ""name"": ""get_stock_price"", + ""input"": { ""ticker"": ""^GSPC"" }, + ""type"": ""tool_use"" + } + ], + ""id"": ""msg_01D7FLrfh4GYq7yT1ULFeyMV"", + ""model"": ""claude-3-5-sonnet-20240620"", + ""role"": ""assistant"", + ""stop_reason"": ""end_turn"", + ""stop_sequence"": null, + ""type"": ""message"", + ""usage"": { + ""input_tokens"": 10, + ""output_tokens"": 25 + } + }" + ); + + var request = new ChatMessageRequest( + model: AnthropicModels.Claude35Sonnet, + messages: [ + new(MessageRole.User, [new TextContent("Hello!")]), + ] + ); + + var result = await Client.CreateChatMessageAsync(request); + + result.IsSuccess.Should().BeTrue(); + result.Value.Should().BeOfType(); + result.Error.Should().BeNull(); + + var message = result.Value; + message.Id.Should().Be("msg_01D7FLrfh4GYq7yT1ULFeyMV"); + message.Model.Should().Be("claude-3-5-sonnet-20240620"); + message.Role.Should().Be("assistant"); + message.StopReason.Should().Be("end_turn"); + message.StopSequence.Should().BeNull(); + message.Type.Should().Be("message"); + message.Usage.InputTokens.Should().Be(10); + message.Usage.OutputTokens.Should().Be(25); + message.Content.Should().HaveCount(1); + message.ToolCall.Should().BeNull(); + + var toolUseContent = message.Content[0]; + toolUseContent.Should().BeOfType(); + + toolUseContent.As().Id.Should().Be("toolu_01D7FLrfh4GYq7yT1ULFeyMV"); + toolUseContent.As().Name.Should().Be("get_stock_price"); + toolUseContent.As().Type.Should().Be("tool_use"); + + var input = toolUseContent.As().Input; + var ticker = input.GetValueOrDefault("ticker"); + ticker!.ToString().Should().Be("^GSPC"); + } + + [Theory] + [ClassData(typeof(EventTestData))] + public async Task CreateChatMessageAsync_WhenCalledAndMessageIsStreamed_ItShouldReturnAllEvents(string eventString, AnthropicEvent anthropicEvent) + { + _mockHttpMessageHandler + .WhenCreateStreamMessageRequest() + .Respond( + HttpStatusCode.OK, + "text/event-stream", + new MemoryStream(Encoding.UTF8.GetBytes(eventString)) + ); + + var request = new StreamChatMessageRequest( + model: AnthropicModels.Claude35Sonnet, + messages: [ + new(MessageRole.User, [new TextContent("Hello!")]), + ] + ); + + var result = Client.CreateChatMessageAsync(request); + var e = await result.FirstOrDefaultAsync(); + + e.Should().BeEquivalentTo(anthropicEvent); + } } \ No newline at end of file diff --git a/tests/AnthropicClient.Tests/Integration/EventTestData.cs b/tests/AnthropicClient.Tests/Integration/EventTestData.cs new file mode 100644 index 0000000..5bdc571 --- /dev/null +++ b/tests/AnthropicClient.Tests/Integration/EventTestData.cs @@ -0,0 +1,385 @@ +namespace AnthropicClient.Tests.Integration; + +public class EventTestData : IEnumerable +{ + public IEnumerator GetEnumerator() + { + yield return new object[] + { + """ + event: message_start + data: {"type":"message_start","message":{"id":"msg_014p7gG3wDgGV9EUtLvnow3U","type":"message","role":"assistant","model":"claude-3-haiku-20240307","stop_sequence":null,"usage":{"input_tokens":472,"output_tokens":2},"content":[],"stop_reason":null}} + """, + new AnthropicEvent() + { + Type = EventType.MessageStart, + Data = new MessageStartEventData() + { + Message = new ChatResponse() + { + Id = "msg_014p7gG3wDgGV9EUtLvnow3U", + Type = "message", + Role = "assistant", + Model = "claude-3-haiku-20240307", + StopSequence = null, + Usage = new ChatUsage() + { + InputTokens = 472, + OutputTokens = 2, + }, + Content = [], + StopReason = null, + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockStart, + Data = new ContentStartEventData() + { + Index = 0, + ContentBlock = new TextContent() + { + Type = "text", + Text = "", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: ping + data: {"type": "ping"} + """, + new AnthropicEvent() + { + Type = EventType.Ping, + Data = new PingEventData(), + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Okay"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = "Okay", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = ",", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" let"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " let", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = "'s", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" check"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " check", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" the"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " the", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" weather"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " weather", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" for"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " for", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" San"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " San", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Francisco"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " Francisco", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " ,", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" CA"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " CA", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":":"}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockDelta, + Data = new ContentDeltaEventData() + { + Index = 0, + Delta = new TextDelta() + { + Type = "text_delta", + Text = " :", + }, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_stop + data: {"type":"content_block_stop","index":0} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockStop, + Data = new ContentStopEventData() + { + Index = 0, + }, + }, + }; + + yield return new object[] + { + """ + event: content_block_start + data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01T1x1fJ34qAmk2tNTrN7Up6","name":"get_weather","input":{}}} + """, + new AnthropicEvent() + { + Type = EventType.ContentBlockStart, + Data = new ContentStartEventData() + { + Index = 1, + ContentBlock = new ToolUseContent() + { + Type = "tool_use", + Id = "toolu_01T1x1fJ34qAmk2tNTrN7Up6", + Name = "get_weather", + Input = [], + }, + }, + }, + }; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} \ No newline at end of file diff --git a/tests/AnthropicClient.Tests/Integration/IntegrationTest.cs b/tests/AnthropicClient.Tests/Integration/IntegrationTest.cs index 857fb0f..29725ef 100644 --- a/tests/AnthropicClient.Tests/Integration/IntegrationTest.cs +++ b/tests/AnthropicClient.Tests/Integration/IntegrationTest.cs @@ -28,13 +28,13 @@ public static class MockHttpMessageHandlerExtensions { return mockHttpMessageHandler .SetupBaseRequest() - .WithJsonContent(r => r.Stream == false); + .WithJsonContent(r => r.Stream == false, JsonSerializationOptions.DefaultOptions); } public static MockedRequest WhenCreateStreamMessageRequest(this MockHttpMessageHandler mockHttpMessageHandler) { return mockHttpMessageHandler .SetupBaseRequest() - .WithJsonContent(r => r.Stream == true); + .WithJsonContent(r => r.Stream == true, JsonSerializationOptions.DefaultOptions); } } \ No newline at end of file diff --git a/tests/AnthropicClient.Tests/Usings.cs b/tests/AnthropicClient.Tests/Usings.cs index 27ef28a..69bde4f 100644 --- a/tests/AnthropicClient.Tests/Usings.cs +++ b/tests/AnthropicClient.Tests/Usings.cs @@ -1,8 +1,10 @@ +global using System.Text; global using System.Text.Json; global using System.Text.Json.JsonDiffPatch.Xunit; global using AnthropicClient.Models; global using AnthropicClient.Utils; +global using AnthropicClient.Json; global using FluentAssertions;