Further E2E tests for resume (#665)

Author: SteveSandersonMS

dotnet/test/SessionTests.cs

@@ -167,6 +167,11 @@ public async Task Should_Resume_A_Session_Using_The_Same_Client()
         var answer2 = await TestHelper.GetFinalAssistantMessageAsync(session2);
         Assert.NotNull(answer2);
         Assert.Contains("2", answer2!.Data.Content ?? string.Empty);
+
+        // Can continue the conversation statefully
+        var answer3 = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" });
+        Assert.NotNull(answer3);
+        Assert.Contains("4", answer3!.Data.Content ?? string.Empty);
     }
 
     [Fact]
@@ -187,6 +192,11 @@ public async Task Should_Resume_A_Session_Using_A_New_Client()
         var messages = await session2.GetMessagesAsync();
         Assert.Contains(messages, m => m is UserMessageEvent);
         Assert.Contains(messages, m => m is SessionResumeEvent);
+
+        // Can continue the conversation statefully
+        var answer2 = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" });
+        Assert.NotNull(answer2);
+        Assert.Contains("4", answer2!.Data.Content ?? string.Empty);
     }
 
     [Fact]
@@ -231,68 +241,6 @@ await session.SendAsync(new MessageOptions
         Assert.Contains("4", answer!.Data.Content ?? string.Empty);
     }
 
-    // TODO: This test requires the session-events.schema.json to include assistant.message_delta.
-    // The CLI v0.0.376 emits delta events at runtime, but the schema hasn't been updated yet.
-    // Once the schema is updated and types are regenerated, this test can be enabled.
-    [Fact(Skip = "Requires schema update for AssistantMessageDeltaEvent type")]
-    public async Task Should_Receive_Streaming_Delta_Events_When_Streaming_Is_Enabled()
-    {
-        var session = await CreateSessionAsync(new SessionConfig { Streaming = true });
-
-        var deltaContents = new List<string>();
-        var doneEvent = new TaskCompletionSource<bool>();
-
-        session.On(evt =>
-        {
-            switch (evt)
-            {
-                // TODO: Uncomment once AssistantMessageDeltaEvent is generated
-                // case AssistantMessageDeltaEvent delta:
-                //     if (!string.IsNullOrEmpty(delta.Data.DeltaContent))
-                //         deltaContents.Add(delta.Data.DeltaContent);
-                //     break;
-                case SessionIdleEvent:
-                    doneEvent.TrySetResult(true);
-                    break;
-            }
-        });
-
-        await session.SendAsync(new MessageOptions { Prompt = "What is 2+2?" });
-
-        // Wait for completion
-        var completed = await Task.WhenAny(doneEvent.Task, Task.Delay(TimeSpan.FromSeconds(60)));
-        Assert.Equal(doneEvent.Task, completed);
-
-        // Should have received delta events
-        Assert.NotEmpty(deltaContents);
-
-        // Get the final message to compare
-        var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session);
-        Assert.NotNull(assistantMessage);
-
-        // Accumulated deltas should equal the final message
-        var accumulated = string.Join("", deltaContents);
-        Assert.Equal(assistantMessage!.Data.Content, accumulated);
-
-        // Final message should contain the answer
-        Assert.Contains("4", assistantMessage.Data.Content ?? string.Empty);
-    }
-
-    [Fact]
-    public async Task Should_Pass_Streaming_Option_To_Session_Creation()
-    {
-        // Verify that the streaming option is accepted without errors
-        var session = await CreateSessionAsync(new SessionConfig { Streaming = true });
-
-        Assert.Matches(@"^[a-f0-9-]+$", session.SessionId);
-
-        // Session should still work normally
-        await session.SendAsync(new MessageOptions { Prompt = "What is 1+1?" });
-        var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session);
-        Assert.NotNull(assistantMessage);
-        Assert.Contains("2", assistantMessage!.Data.Content);
-    }
-
     [Fact]
     public async Task Should_Receive_Session_Events()
     {

dotnet/test/StreamingFidelityTests.cs

@@ -0,0 +1,99 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+
+using GitHub.Copilot.SDK.Test.Harness;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace GitHub.Copilot.SDK.Test;
+
+public class StreamingFidelityTests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "streaming_fidelity", output)
+{
+    [Fact]
+    public async Task Should_Produce_Delta_Events_When_Streaming_Is_Enabled()
+    {
+        var session = await CreateSessionAsync(new SessionConfig { Streaming = true });
+
+        var events = new List<SessionEvent>();
+        session.On(evt => events.Add(evt));
+
+        await session.SendAndWaitAsync(new MessageOptions { Prompt = "Count from 1 to 5, separated by commas." });
+
+        var types = events.Select(e => e.Type).ToList();
+
+        // Should have streaming deltas before the final message
+        var deltaEvents = events.OfType<AssistantMessageDeltaEvent>().ToList();
+        Assert.NotEmpty(deltaEvents);
+
+        // Deltas should have content
+        foreach (var delta in deltaEvents)
+        {
+            Assert.False(string.IsNullOrEmpty(delta.Data.DeltaContent));
+        }
+
+        // Should still have a final assistant.message
+        Assert.Contains("assistant.message", types);
+
+        // Deltas should come before the final message
+        var firstDeltaIdx = types.IndexOf("assistant.message_delta");
+        var lastAssistantIdx = types.LastIndexOf("assistant.message");
+        Assert.True(firstDeltaIdx < lastAssistantIdx);
+
+        await session.DisposeAsync();
+    }
+
+    [Fact]
+    public async Task Should_Not_Produce_Deltas_When_Streaming_Is_Disabled()
+    {
+        var session = await CreateSessionAsync(new SessionConfig { Streaming = false });
+
+        var events = new List<SessionEvent>();
+        session.On(evt => events.Add(evt));
+
+        await session.SendAndWaitAsync(new MessageOptions { Prompt = "Say 'hello world'." });
+
+        var deltaEvents = events.OfType<AssistantMessageDeltaEvent>().ToList();
+
+        // No deltas when streaming is off
+        Assert.Empty(deltaEvents);
+
+        // But should still have a final assistant.message
+        var assistantEvents = events.OfType<AssistantMessageEvent>().ToList();
+        Assert.NotEmpty(assistantEvents);
+
+        await session.DisposeAsync();
+    }
+
+    [Fact]
+    public async Task Should_Produce_Deltas_After_Session_Resume()
+    {
+        var session = await CreateSessionAsync(new SessionConfig { Streaming = false });
+        await session.SendAndWaitAsync(new MessageOptions { Prompt = "What is 3 + 6?" });
+        await session.DisposeAsync();
+
+        // Resume using a new client
+        using var newClient = Ctx.CreateClient();
+        var session2 = await newClient.ResumeSessionAsync(session.SessionId,
+            new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Streaming = true });
+
+        var events = new List<SessionEvent>();
+        session2.On(evt => events.Add(evt));
+
+        var answer = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" });
+        Assert.NotNull(answer);
+        Assert.Contains("18", answer!.Data.Content ?? string.Empty);
+
+        // Should have streaming deltas before the final message
+        var deltaEvents = events.OfType<AssistantMessageDeltaEvent>().ToList();
+        Assert.NotEmpty(deltaEvents);
+
+        // Deltas should have content
+        foreach (var delta in deltaEvents)
+        {
+            Assert.False(string.IsNullOrEmpty(delta.Data.DeltaContent));
+        }
+
+        await session2.DisposeAsync();
+    }
+}

go/internal/e2e/session_test.go

@@ -368,6 +368,15 @@ func TestSession(t *testing.T) {
 		if answer2.Data.Content == nil || !strings.Contains(*answer2.Data.Content, "2") {
 			t.Errorf("Expected resumed session answer to contain '2', got %v", answer2.Data.Content)
 		}
+
+		// Can continue the conversation statefully
+		answer3, err := session2.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Now if you double that, what do you get?"})
+		if err != nil {
+			t.Fatalf("Failed to send follow-up message: %v", err)
+		}
+		if answer3 == nil || answer3.Data.Content == nil || !strings.Contains(*answer3.Data.Content, "4") {
+			t.Errorf("Expected follow-up answer to contain '4', got %v", answer3)
+		}
 	})
 
 	t.Run("should resume a session using a new client", func(t *testing.T) {
@@ -432,6 +441,15 @@ func TestSession(t *testing.T) {
 		if !hasSessionResume {
 			t.Error("Expected messages to contain 'session.resume'")
 		}
+
+		// Can continue the conversation statefully
+		answer3, err := session2.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Now if you double that, what do you get?"})
+		if err != nil {
+			t.Fatalf("Failed to send follow-up message: %v", err)
+		}
+		if answer3 == nil || answer3.Data.Content == nil || !strings.Contains(*answer3.Data.Content, "4") {
+			t.Errorf("Expected follow-up answer to contain '4', got %v", answer3)
+		}
 	})
 
 	t.Run("should throw error when resuming non-existent session", func(t *testing.T) {
@@ -565,99 +583,6 @@ func TestSession(t *testing.T) {
 		}
 	})
 
-	t.Run("should receive streaming delta events when streaming is enabled", func(t *testing.T) {
-		ctx.ConfigureForTest(t)
-
-		session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
-			OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
-			Streaming:           true,
-		})
-		if err != nil {
-			t.Fatalf("Failed to create session with streaming: %v", err)
-		}
-
-		var deltaContents []string
-		done := make(chan bool)
-
-		session.On(func(event copilot.SessionEvent) {
-			switch event.Type {
-			case "assistant.message_delta":
-				if event.Data.DeltaContent != nil {
-					deltaContents = append(deltaContents, *event.Data.DeltaContent)
-				}
-			case "session.idle":
-				close(done)
-			}
-		})
-
-		_, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 2+2?"})
-		if err != nil {
-			t.Fatalf("Failed to send message: %v", err)
-		}
-
-		// Wait for completion
-		select {
-		case <-done:
-		case <-time.After(60 * time.Second):
-			t.Fatal("Timed out waiting for session.idle")
-		}
-
-		// Should have received delta events
-		if len(deltaContents) == 0 {
-			t.Error("Expected to receive delta events, got none")
-		}
-
-		// Get the final message to compare
-		assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session)
-		if err != nil {
-			t.Fatalf("Failed to get assistant message: %v", err)
-		}
-
-		// Accumulated deltas should equal the final message
-		accumulated := strings.Join(deltaContents, "")
-		if assistantMessage.Data.Content != nil && accumulated != *assistantMessage.Data.Content {
-			t.Errorf("Accumulated deltas don't match final message.\nAccumulated: %q\nFinal: %q", accumulated, *assistantMessage.Data.Content)
-		}
-
-		// Final message should contain the answer
-		if assistantMessage.Data.Content == nil || !strings.Contains(*assistantMessage.Data.Content, "4") {
-			t.Errorf("Expected assistant message to contain '4', got %v", assistantMessage.Data.Content)
-		}
-	})
-
-	t.Run("should pass streaming option to session creation", func(t *testing.T) {
-		ctx.ConfigureForTest(t)
-
-		// Verify that the streaming option is accepted without errors
-		session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
-			OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
-			Streaming:           true,
-		})
-		if err != nil {
-			t.Fatalf("Failed to create session with streaming: %v", err)
-		}
-
-		matched, _ := regexp.MatchString(`^[a-f0-9-]+$`, session.SessionID)
-		if !matched {
-			t.Errorf("Expected session ID to match UUID pattern, got %q", session.SessionID)
-		}
-
-		// Session should still work normally
-		_, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 1+1?"})
-		if err != nil {
-			t.Fatalf("Failed to send message: %v", err)
-		}
-
-		assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session)
-		if err != nil {
-			t.Fatalf("Failed to get assistant message: %v", err)
-		}
-
-		if assistantMessage.Data.Content == nil || !strings.Contains(*assistantMessage.Data.Content, "2") {
-			t.Errorf("Expected assistant message to contain '2', got %v", assistantMessage.Data.Content)
-		}
-	})
-
 	t.Run("should receive session events", func(t *testing.T) {
 		ctx.ConfigureForTest(t)