Skip to content

De-flake builtin_tools E2E tests with a longer send timeout #1538

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephentoub merged 1 commit into from
Jun 1, 2026
+224 −99
Merged

De-flake builtin_tools E2E tests with a longer send timeout #1538

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions dotnet/test/E2E/BuiltinToolsE2ETests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,20 @@ namespace GitHub.Copilot.Test.E2E;
public class BuiltinToolsE2ETests(E2ETestFixture fixture, ITestOutputHelper output)
: E2ETestBase(fixture, "builtin_tools", output)
{
// Built-in tool tests spawn a real CLI subprocess and execute actual shell /
// file tools. Under slow/concurrent CI (notably Windows) this agent loop can
// briefly exceed the 60s SendAndWaitAsync default, so give it extra headroom
// while still failing fast on a genuine hang.
private static readonly TimeSpan SendTimeout = TimeSpan.FromSeconds(120);

[Fact]
public async Task Should_Capture_Exit_Code_In_Output()
{
var session = await CreateSessionAsync();
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Run 'echo hello && echo world'. Tell me the exact output.",
});
}, SendTimeout);
var content = msg?.Data.Content ?? string.Empty;
Assert.Contains("hello", content);
Assert.Contains("world", content);
Expand All @@ -44,7 +50,7 @@ public async Task Should_Capture_Stderr_Output()
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Run 'echo error_msg >&2; echo ok' and tell me what stderr said. Reply with just the stderr content.",
});
}, SendTimeout);
Assert.Contains("error_msg", msg?.Data.Content ?? string.Empty);
}

Expand All @@ -56,7 +62,7 @@ public async Task Should_Read_File_With_Line_Range()
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Read lines 2 through 4 of the file 'lines.txt' in this directory. Tell me what those lines contain.",
});
}, SendTimeout);
var content = msg?.Data.Content ?? string.Empty;
Assert.Contains("line2", content);
Assert.Contains("line4", content);
Expand All @@ -69,7 +75,7 @@ public async Task Should_Handle_Nonexistent_File_Gracefully()
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Try to read the file 'does_not_exist.txt'. If it doesn't exist, say 'FILE_NOT_FOUND'.",
});
}, SendTimeout);
var content = (msg?.Data.Content ?? string.Empty).ToUpperInvariant();
// Match any of the common phrasings for a missing-file response.
Assert.True(
Expand All @@ -90,7 +96,7 @@ public async Task Should_Edit_A_File_Successfully()
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Edit the file 'edit_me.txt': replace 'Hello World' with 'Hi Universe'. Then read it back and tell me its contents.",
});
}, SendTimeout);
Assert.Contains("Hi Universe", msg?.Data.Content ?? string.Empty);
}

Expand All @@ -101,7 +107,7 @@ public async Task Should_Create_A_New_File()
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Create a file called 'new_file.txt' with the content 'Created by test'. Then read it back to confirm.",
});
}, SendTimeout);
Assert.Contains("Created by test", msg?.Data.Content ?? string.Empty);
}

Expand All @@ -113,7 +119,7 @@ public async Task Should_Search_For_Patterns_In_Files()
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Search for lines starting with 'ap' in the file 'data.txt'. Tell me which lines matched.",
});
}, SendTimeout);
var content = msg?.Data.Content ?? string.Empty;
Assert.Contains("apple", content);
Assert.Contains("apricot", content);
Expand All @@ -130,7 +136,7 @@ public async Task Should_Find_Files_By_Pattern()
var msg = await session.SendAndWaitAsync(new MessageOptions
{
Prompt = "Find all .ts files in this directory (recursively). List the filenames you found.",
});
}, SendTimeout);
Assert.Contains("index.ts", msg?.Data.Content ?? string.Empty);
}
}
40 changes: 32 additions & 8 deletions go/internal/e2e/builtin_tools_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,24 @@
package e2e

import (
"context"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"

copilot "github.com/github/copilot-sdk/go"
"github.com/github/copilot-sdk/go/internal/e2e/testharness"
)

// Built-in tool tests spawn a real CLI subprocess and execute actual shell /
// file tools. Under slow/concurrent CI (notably Windows) this agent loop can
// briefly exceed the 60s SendAndWait default, so give it extra headroom while
// still failing fast on a genuine hang.
const sendTimeout = 120 * time.Second

func TestBuiltinToolsE2E(t *testing.T) {
ctx := testharness.NewTestContext(t)
client := ctx.NewClient()
Expand All @@ -27,7 +35,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Run 'echo hello && echo world'. Tell me the exact output.",
})
if err != nil {
Expand Down Expand Up @@ -55,7 +65,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Run 'echo error_msg >&2; echo ok' and tell me what stderr said. Reply with just the stderr content.",
})
if err != nil {
Expand All @@ -82,7 +94,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Read lines 2 through 4 of the file 'lines.txt' in this directory. Tell me what those lines contain.",
})
if err != nil {
Expand All @@ -106,7 +120,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Try to read the file 'does_not_exist.txt'. If it doesn't exist, say 'FILE_NOT_FOUND'.",
})
if err != nil {
Expand Down Expand Up @@ -139,7 +155,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Edit the file 'edit_me.txt': replace 'Hello World' with 'Hi Universe'. Then read it back and tell me its contents.",
})
if err != nil {
Expand All @@ -162,7 +180,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Create a file called 'new_file.txt' with the content 'Created by test'. Then read it back to confirm.",
})
if err != nil {
Expand All @@ -189,7 +209,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Search for lines starting with 'ap' in the file 'data.txt'. Tell me which lines matched.",
})
if err != nil {
Expand Down Expand Up @@ -223,7 +245,9 @@ func TestBuiltinToolsE2E(t *testing.T) {
}
t.Cleanup(func() { _ = session.Disconnect() })

msg, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
sendCtx, cancel := context.WithTimeout(t.Context(), sendTimeout)
defer cancel()
msg, err := session.SendAndWait(sendCtx, copilot.MessageOptions{
Prompt: "Find all .ts files in this directory (recursively). List the filenames you found.",
})
if err != nil {
Expand Down
Loading
Loading