fix(ai): support Anthropic eager tool streaming compat

closes #3575
2026-04-23 23:12:45 +02:00
parent 6af10c9c7f
commit ffa0f31239
9 changed files with 412 additions and 16 deletions
--- a/packages/ai/scripts/generate-models.ts
+++ b/packages/ai/scripts/generate-models.ts
@@ -3,7 +3,13 @@
 import { writeFileSync } from "fs";
 import { join, dirname } from "path";
 import { fileURLToPath } from "url";
-import { Api, KnownProvider, Model, type OpenAICompletionsCompat } from "../src/types.js";
+import {
+	Api,
+	type AnthropicMessagesCompat,
+	KnownProvider,
+	Model,
+	type OpenAICompletionsCompat,
+} from "../src/types.js";

 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -60,6 +66,13 @@ const KIMI_STATIC_HEADERS = {
 const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1";
 const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh";
 const ZAI_TOOL_STREAM_UNSUPPORTED_MODELS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]);
+const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS = new Set(["github-copilot:claude-haiku-4.5"]);
+
+function getAnthropicMessagesCompat(provider: string, modelId: string): AnthropicMessagesCompat | undefined {
+	return EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS.has(`${provider}:${modelId}`)
+		? { supportsEagerToolInputStreaming: false }
+		: undefined;
+}

 function getBedrockBaseUrl(modelId: string): string {
 	return modelId.startsWith("eu.")
@@ -582,6 +595,9 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 						? "openai-responses"
 						: "openai-completions";

+				const anthropicCompat =
+					api === "anthropic-messages" ? getAnthropicMessagesCompat("github-copilot", modelId) : undefined;
+
 				const copilotModel: Model<any> = {
 					id: modelId,
 					name: m.name || modelId,
@@ -599,6 +615,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 128000,
 					maxTokens: m.limit?.output || 8192,
 					headers: { ...COPILOT_STATIC_HEADERS },
+					...(anthropicCompat ? { compat: anthropicCompat } : {}),
 					// compat only applies to openai-completions
 					...(api === "openai-completions" ? {
 						compat: {
--- a/packages/ai/src/models.generated.ts
+++ b/packages/ai/src/models.generated.ts
@@ -3014,6 +3014,7 @@ export const MODELS = {
 			provider: "github-copilot",
 			baseUrl: "https://api.individual.githubcopilot.com",
 			headers: {"User-Agent":"GitHubCopilotChat/0.35.0","Editor-Version":"vscode/1.107.0","Editor-Plugin-Version":"copilot-chat/0.35.0","Copilot-Integration-Id":"vscode-chat"},
+			compat: {"supportsEagerToolInputStreaming":false},
 			reasoning: true,
 			input: ["text", "image"],
 			cost: {
--- a/packages/ai/src/providers/anthropic.ts
+++ b/packages/ai/src/providers/anthropic.ts
@@ -9,6 +9,7 @@ import type {
 import { getEnvApiKey } from "../env-api-keys.js";
 import { calculateCost } from "../models.js";
 import type {
+	AnthropicMessagesCompat,
 	Api,
 	AssistantMessage,
 	CacheRetention,
@@ -159,6 +160,15 @@ export type AnthropicEffort = "low" | "medium" | "high" | "xhigh" | "max";

 export type AnthropicThinkingDisplay = "summarized" | "omitted";

+const FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14";
+const INTERLEAVED_THINKING_BETA = "interleaved-thinking-2025-05-14";
+
+function getAnthropicCompat(model: Model<"anthropic-messages">): Required<AnthropicMessagesCompat> {
+	return {
+		supportsEagerToolInputStreaming: model.compat?.supportsEagerToolInputStreaming ?? true,
+	};
+}
+
 export interface AnthropicOptions extends StreamOptions {
 	/**
 	 * Enable extended thinking.
@@ -433,6 +443,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
 					model,
 					apiKey,
 					options?.interleavedThinking ?? true,
+					shouldUseFineGrainedToolStreamingBeta(model, context),
 					options?.headers,
 					copilotDynamicHeaders,
 				);
@@ -728,6 +739,7 @@ function createClient(
 	model: Model<"anthropic-messages">,
 	apiKey: string,
 	interleavedThinking: boolean,
+	useFineGrainedToolStreamingBeta: boolean,
 	optionsHeaders?: Record<string, string>,
 	dynamicHeaders?: Record<string, string>,
 ): { client: Anthropic; isOAuthToken: boolean } {
@@ -735,11 +747,14 @@ function createClient(
 	// The beta header is deprecated on Opus 4.6 and redundant on Sonnet 4.6, so skip it.
 	const needsInterleavedBeta = interleavedThinking && !supportsAdaptiveThinking(model.id);

-	// Copilot: Bearer auth, selective betas (no fine-grained-tool-streaming)
+	// Copilot: Bearer auth, selective betas.
 	if (model.provider === "github-copilot") {
 		const betaFeatures: string[] = [];
+		if (useFineGrainedToolStreamingBeta) {
+			betaFeatures.push(FINE_GRAINED_TOOL_STREAMING_BETA);
+		}
 		if (needsInterleavedBeta) {
-			betaFeatures.push("interleaved-thinking-2025-05-14");
+			betaFeatures.push(INTERLEAVED_THINKING_BETA);
 		}

 		const client = new Anthropic({
@@ -763,8 +778,11 @@ function createClient(
 	}

 	const betaFeatures: string[] = [];
+	if (useFineGrainedToolStreamingBeta) {
+		betaFeatures.push(FINE_GRAINED_TOOL_STREAMING_BETA);
+	}
 	if (needsInterleavedBeta) {
-		betaFeatures.push("interleaved-thinking-2025-05-14");
+		betaFeatures.push(INTERLEAVED_THINKING_BETA);
 	}

 	// OAuth: Bearer auth, Claude Code identity headers
@@ -856,7 +874,12 @@ function buildParams(
 	}

 	if (context.tools) {
-		params.tools = convertTools(context.tools, isOAuthToken, cacheControl);
+		params.tools = convertTools(
+			context.tools,
+			isOAuthToken,
+			getAnthropicCompat(model).supportsEagerToolInputStreaming,
+			cacheControl,
+		);
 	}

 	// Configure thinking mode: adaptive (Opus 4.6+ and Sonnet 4.6),
@@ -1078,9 +1101,14 @@ function convertMessages(
 	return params;
 }

+function shouldUseFineGrainedToolStreamingBeta(model: Model<"anthropic-messages">, context: Context): boolean {
+	return !!context.tools?.length && !getAnthropicCompat(model).supportsEagerToolInputStreaming;
+}
+
 function convertTools(
 	tools: Tool[],
 	isOAuthToken: boolean,
+	supportsEagerToolInputStreaming: boolean,
 	cacheControl?: CacheControlEphemeral,
 ): Anthropic.Messages.Tool[] {
 	if (!tools) return [];
@@ -1091,7 +1119,7 @@ function convertTools(
 		return {
 			name: isOAuthToken ? toClaudeCodeName(tool.name) : tool.name,
 			description: tool.description,
-			eager_input_streaming: true,
+			...(supportsEagerToolInputStreaming ? { eager_input_streaming: true } : {}),
 			input_schema: {
 				type: "object",
 				properties: schema.properties ?? {},
--- a/packages/ai/src/types.ts
+++ b/packages/ai/src/types.ts
@@ -304,6 +304,18 @@ export interface OpenAIResponsesCompat {
 	sendSessionIdHeader?: boolean;
 }

+/** Compatibility settings for Anthropic Messages-compatible APIs. */
+export interface AnthropicMessagesCompat {
+	/**
+	 * Whether the provider accepts per-tool `eager_input_streaming`.
+	 * When false, the Anthropic provider omits `tools[].eager_input_streaming`
+	 * and sends the legacy `fine-grained-tool-streaming-2025-05-14` beta header
+	 * for tool-enabled requests.
+	 * Default: true.
+	 */
+	supportsEagerToolInputStreaming?: boolean;
+}
+
 /**
 * OpenRouter provider routing preferences.
 * Controls which upstream providers OpenRouter routes requests to.
@@ -414,5 +426,7 @@ export interface Model<TApi extends Api> {
 		? OpenAICompletionsCompat
 		: TApi extends "openai-responses"
 			? OpenAIResponsesCompat
-			: never;
+			: TApi extends "anthropic-messages"
+				? AnthropicMessagesCompat
+				: never;
 }
--- a/packages/ai/test/anthropic-eager-tool-input-compat.test.ts
+++ b/packages/ai/test/anthropic-eager-tool-input-compat.test.ts
@@ -0,0 +1,122 @@
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
+import type { AddressInfo } from "node:net";
+import { Type } from "typebox";
+import { describe, expect, it } from "vitest";
+import { streamAnthropic } from "../src/providers/anthropic.js";
+import type { Context, Model, Tool } from "../src/types.js";
+
+interface CapturedRequest {
+	headers: IncomingMessage["headers"];
+	body: Record<string, unknown>;
+}
+
+function createModel(baseUrl: string, compat?: Model<"anthropic-messages">["compat"]): Model<"anthropic-messages"> {
+	return {
+		id: "claude-opus-4-7",
+		name: "Claude Opus 4.7",
+		api: "anthropic-messages",
+		provider: "test-anthropic",
+		baseUrl,
+		reasoning: true,
+		input: ["text"],
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+		contextWindow: 200000,
+		maxTokens: 32000,
+		compat,
+	};
+}
+
+const tool: Tool = {
+	name: "lookup",
+	description: "Look up a value",
+	parameters: Type.Object({ value: Type.String() }),
+};
+
+function createContext(tools: Tool[] = [tool]): Context {
+	return {
+		messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }],
+		...(tools.length > 0 ? { tools } : {}),
+	};
+}
+
+async function readRequestBody(request: IncomingMessage): Promise<Record<string, unknown>> {
+	const chunks: Buffer[] = [];
+	for await (const chunk of request) {
+		chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+	}
+	return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;
+}
+
+function writeEmptySseResponse(response: ServerResponse): void {
+	response.writeHead(200, { "content-type": "text/event-stream" });
+	response.end();
+}
+
+async function captureAnthropicRequest(
+	compat: Model<"anthropic-messages">["compat"],
+	context: Context,
+): Promise<CapturedRequest> {
+	let capturedRequest: CapturedRequest | undefined;
+
+	const server = createServer(async (request, response) => {
+		capturedRequest = {
+			headers: request.headers,
+			body: await readRequestBody(request),
+		};
+		writeEmptySseResponse(response);
+	});
+
+	await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+	const address = server.address() as AddressInfo;
+
+	try {
+		const stream = streamAnthropic(createModel(`http://127.0.0.1:${address.port}`, compat), context, {
+			apiKey: "test-key",
+			cacheRetention: "none",
+		});
+
+		for await (const event of stream) {
+			if (event.type === "done" || event.type === "error") break;
+		}
+	} finally {
+		await new Promise<void>((resolve, reject) => {
+			server.close((error) => (error ? reject(error) : resolve()));
+		});
+	}
+
+	if (!capturedRequest) {
+		throw new Error("Anthropic request was not captured");
+	}
+	return capturedRequest;
+}
+
+function getFirstTool(body: Record<string, unknown>): Record<string, unknown> {
+	const tools = body.tools;
+	if (!Array.isArray(tools) || typeof tools[0] !== "object" || tools[0] === null) {
+		throw new Error("Expected first tool in request body");
+	}
+	return tools[0] as Record<string, unknown>;
+}
+
+describe("Anthropic eager tool input streaming compatibility", () => {
+	it("sends per-tool eager_input_streaming by default", async () => {
+		const request = await captureAnthropicRequest(undefined, createContext());
+
+		expect(getFirstTool(request.body).eager_input_streaming).toBe(true);
+		expect(request.headers["anthropic-beta"]).toBeUndefined();
+	});
+
+	it("uses the legacy fine-grained tool streaming beta when eager tool input streaming is disabled", async () => {
+		const request = await captureAnthropicRequest({ supportsEagerToolInputStreaming: false }, createContext());
+
+		expect(getFirstTool(request.body).eager_input_streaming).toBeUndefined();
+		expect(request.headers["anthropic-beta"]).toBe("fine-grained-tool-streaming-2025-05-14");
+	});
+
+	it("does not send the legacy fine-grained tool streaming beta when there are no tools", async () => {
+		const request = await captureAnthropicRequest({ supportsEagerToolInputStreaming: false }, createContext([]));
+
+		expect(request.body.tools).toBeUndefined();
+		expect(request.headers["anthropic-beta"]).toBeUndefined();
+	});
+});
--- a/packages/ai/test/anthropic-eager-tool-input-e2e.test.ts
+++ b/packages/ai/test/anthropic-eager-tool-input-e2e.test.ts
@@ -0,0 +1,143 @@
+import { Type } from "typebox";
+import { describe, expect, it } from "vitest";
+import { getEnvApiKey } from "../src/env-api-keys.js";
+import { getModels, getProviders } from "../src/models.js";
+import { complete } from "../src/stream.js";
+import type { Api, KnownProvider, Model, ProviderStreamOptions, Tool } from "../src/types.js";
+import { resolveApiKey } from "./oauth.js";
+
+const githubCopilotToken = await resolveApiKey("github-copilot");
+
+const echoToolSchema = Type.Object({
+	value: Type.String({ description: "The value to echo" }),
+});
+
+const echoTool: Tool<typeof echoToolSchema> = {
+	name: "echo_value",
+	description: "Echo a string value",
+	parameters: echoToolSchema,
+};
+
+interface AnthropicEagerE2ECase {
+	name: string;
+	provider: KnownProvider;
+	model: Model<"anthropic-messages">;
+	apiKey: string | undefined;
+}
+
+function getE2EApiKey(provider: KnownProvider): string | undefined {
+	if (provider === "github-copilot") {
+		return githubCopilotToken;
+	}
+	return getEnvApiKey(provider);
+}
+
+function getAnthropicMessagesModels(provider: KnownProvider): Model<"anthropic-messages">[] {
+	const models = getModels(provider) as Model<Api>[];
+	return models.filter((model) => model.api === "anthropic-messages") as Model<"anthropic-messages">[];
+}
+
+const anthropicMessagesCases: AnthropicEagerE2ECase[] = getProviders().flatMap((provider) =>
+	getAnthropicMessagesModels(provider).map((model) => ({
+		name: `${provider}/${model.id}`,
+		provider,
+		model,
+		apiKey: getE2EApiKey(provider),
+	})),
+);
+
+function getProbePriority(model: Model<"anthropic-messages">): number {
+	const modelId = model.id.toLowerCase();
+	const cost = model.cost.input + model.cost.output;
+	let priority = cost;
+
+	// Prefer current Claude 4 Haiku routes when present: they are cheap and avoid
+	// stale Claude 3.x aliases that can remain in catalogs after upstream removal.
+	if (modelId.includes("haiku") && (modelId.includes("4-5") || modelId.includes("4.5"))) {
+		priority -= 1000;
+	} else if (modelId.includes("claude") && (modelId.includes("4-") || modelId.includes("4."))) {
+		priority -= 500;
+	}
+
+	return priority;
+}
+
+function selectOneCasePerProvider(cases: AnthropicEagerE2ECase[]): AnthropicEagerE2ECase[] {
+	const byProvider = new Map<KnownProvider, AnthropicEagerE2ECase[]>();
+	for (const testCase of cases) {
+		const providerCases = byProvider.get(testCase.provider) ?? [];
+		providerCases.push(testCase);
+		byProvider.set(testCase.provider, providerCases);
+	}
+
+	return Array.from(byProvider.values()).map(
+		(providerCases) =>
+			providerCases.sort(
+				(a, b) => getProbePriority(a.model) - getProbePriority(b.model) || a.model.id.localeCompare(b.model.id),
+			)[0],
+	);
+}
+
+const probeCases = selectOneCasePerProvider(anthropicMessagesCases);
+
+function withEagerToolInputStreaming(model: Model<"anthropic-messages">): Model<"anthropic-messages"> {
+	return {
+		...model,
+		compat: {
+			...model.compat,
+			supportsEagerToolInputStreaming: true,
+		},
+	};
+}
+
+async function expectToolEnabledRequestAccepted(
+	model: Model<"anthropic-messages">,
+	apiKey: string | undefined,
+): Promise<void> {
+	const options: ProviderStreamOptions = {
+		apiKey,
+		maxTokens: 128,
+		thinkingEnabled: false,
+	};
+	const response = await complete(
+		model,
+		{
+			systemPrompt: "You are a concise assistant. Use tools when useful.",
+			messages: [
+				{
+					role: "user",
+					content: "Call echo_value with value set to eager-input-streaming-compat.",
+					timestamp: Date.now(),
+				},
+			],
+			tools: [echoTool],
+		},
+		options,
+	);
+
+	expect(response.errorMessage, response.errorMessage).toBeFalsy();
+	expect(response.stopReason, response.errorMessage).not.toBe("error");
+}
+
+describe("Anthropic Messages eager tool input streaming E2E", () => {
+	it("covers every generated anthropic-messages model", () => {
+		const expectedModels = getProviders().flatMap((provider) =>
+			getAnthropicMessagesModels(provider).map((model) => `${provider}/${model.id}`),
+		);
+		expect(anthropicMessagesCases.map((testCase) => testCase.name).sort()).toEqual(expectedModels.sort());
+	});
+
+	describe("forced eager_input_streaming probe", () => {
+		for (const testCase of probeCases) {
+			const model = withEagerToolInputStreaming(testCase.model);
+
+			it.skipIf(!testCase.apiKey)(
+				`${testCase.name} accepts forced eager_input_streaming`,
+				{ retry: 2 },
+				async () => {
+					await expectToolEnabledRequestAccepted(model, testCase.apiKey);
+				},
+			);
+		}
+	});
+});