Merge branch 'main' into feat/image-outputs
This commit is contained in:
@@ -10,10 +10,7 @@ import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Resolve OAuth tokens at module level (async, runs before tests)
|
||||
const [geminiCliToken, openaiCodexToken] = await Promise.all([
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [openaiCodexToken] = await Promise.all([resolveApiKey("openai-codex")]);
|
||||
|
||||
async function testAbortSignal<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
|
||||
const context: Context = {
|
||||
@@ -193,6 +190,54 @@ describe("AI Providers Abort Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider Abort", () => {
|
||||
const llm = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
|
||||
it("should abort mid-stream", { retry: 3 }, async () => {
|
||||
await testAbortSignal(llm);
|
||||
});
|
||||
|
||||
it("should handle immediate abort", { retry: 3 }, async () => {
|
||||
await testImmediateAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN) Provider Abort", () => {
|
||||
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
|
||||
it("should abort mid-stream", { retry: 3 }, async () => {
|
||||
await testAbortSignal(llm);
|
||||
});
|
||||
|
||||
it("should handle immediate abort", { retry: 3 }, async () => {
|
||||
await testImmediateAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS) Provider Abort", () => {
|
||||
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
|
||||
it("should abort mid-stream", { retry: 3 }, async () => {
|
||||
await testAbortSignal(llm);
|
||||
});
|
||||
|
||||
it("should handle immediate abort", { retry: 3 }, async () => {
|
||||
await testImmediateAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP) Provider Abort", () => {
|
||||
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
|
||||
it("should abort mid-stream", { retry: 3 }, async () => {
|
||||
await testAbortSignal(llm);
|
||||
});
|
||||
|
||||
it("should handle immediate abort", { retry: 3 }, async () => {
|
||||
await testImmediateAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Abort", () => {
|
||||
const llm = getModel("kimi-coding", "kimi-k2-thinking");
|
||||
|
||||
@@ -217,19 +262,6 @@ describe("AI Providers Abort Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// Google Gemini CLI / Antigravity share the same provider, so one test covers both
|
||||
describe("Google Gemini CLI Provider Abort", () => {
|
||||
it.skipIf(!geminiCliToken)("should abort mid-stream", { retry: 3 }, async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testAbortSignal(llm, { apiKey: geminiCliToken });
|
||||
});
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle immediate abort", { retry: 3 }, async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testImmediateAbort(llm, { apiKey: geminiCliToken });
|
||||
});
|
||||
});
|
||||
|
||||
describe("OpenAI Codex Provider Abort", () => {
|
||||
it.skipIf(!openaiCodexToken)("should abort mid-stream", { retry: 3 }, async () => {
|
||||
const llm = getModel("openai-codex", "gpt-5.2-codex");
|
||||
|
||||
9
packages/ai/test/cloudflare-utils.ts
Normal file
9
packages/ai/test/cloudflare-utils.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export function hasCloudflareWorkersAICredentials(): boolean {
|
||||
return !!process.env.CLOUDFLARE_API_KEY && !!process.env.CLOUDFLARE_ACCOUNT_ID;
|
||||
}
|
||||
|
||||
export function hasCloudflareAiGatewayCredentials(): boolean {
|
||||
return (
|
||||
!!process.env.CLOUDFLARE_API_KEY && !!process.env.CLOUDFLARE_ACCOUNT_ID && !!process.env.CLOUDFLARE_GATEWAY_ID
|
||||
);
|
||||
}
|
||||
291
packages/ai/test/codex-websocket-cached-probe.ts
Normal file
291
packages/ai/test/codex-websocket-cached-probe.ts
Normal file
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Live probe for OpenAI Codex Responses websocket-cached mode.
|
||||
*
|
||||
* Runs a simple tool loop directly against the pi-ai provider source so it does not
|
||||
* depend on built dist packages or coding-agent SDK wiring.
|
||||
*/
|
||||
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { Type } from "typebox";
|
||||
import { AuthStorage } from "../../coding-agent/src/core/auth-storage.js";
|
||||
import { getModel } from "../src/models.js";
|
||||
import {
|
||||
closeOpenAICodexWebSocketSessions,
|
||||
getOpenAICodexWebSocketDebugStats,
|
||||
resetOpenAICodexWebSocketDebugStats,
|
||||
streamOpenAICodexResponses,
|
||||
} from "../src/providers/openai-codex-responses.js";
|
||||
import type { AssistantMessage, Context, Message, Model, Tool, ToolResultMessage, Transport } from "../src/types.js";
|
||||
|
||||
type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh";
|
||||
|
||||
interface Args {
|
||||
turns: number;
|
||||
transport: Transport;
|
||||
maxTokens: number;
|
||||
reasoning: ThinkingLevel;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
const DEFAULT_TURNS = 20;
|
||||
const DEFAULT_MAX_TOKENS = 64;
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
let turns = DEFAULT_TURNS;
|
||||
let transport: Transport = "websocket-cached";
|
||||
let maxTokens = DEFAULT_MAX_TOKENS;
|
||||
let reasoning: ThinkingLevel = "low";
|
||||
let sessionId = `pi-ai-codex-ws-cached-probe-${Date.now()}`;
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
switch (arg) {
|
||||
case "--turns":
|
||||
turns = Number.parseInt(required(argv[++i], arg), 10);
|
||||
break;
|
||||
case "--transport": {
|
||||
const value = required(argv[++i], arg);
|
||||
if (value !== "sse" && value !== "websocket" && value !== "websocket-cached" && value !== "auto") {
|
||||
throw new Error(`Invalid --transport: ${value}`);
|
||||
}
|
||||
transport = value;
|
||||
break;
|
||||
}
|
||||
case "--max-tokens":
|
||||
maxTokens = Number.parseInt(required(argv[++i], arg), 10);
|
||||
break;
|
||||
case "--reasoning": {
|
||||
const value = required(argv[++i], arg);
|
||||
if (value !== "minimal" && value !== "low" && value !== "medium" && value !== "high" && value !== "xhigh") {
|
||||
throw new Error(`Invalid --reasoning: ${value}`);
|
||||
}
|
||||
reasoning = value;
|
||||
break;
|
||||
}
|
||||
case "--session-id":
|
||||
sessionId = required(argv[++i], arg);
|
||||
break;
|
||||
case "--help":
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { turns, transport, maxTokens, reasoning, sessionId };
|
||||
}
|
||||
|
||||
function required(value: string | undefined, flag: string): string {
|
||||
if (!value) throw new Error(`Missing value for ${flag}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`Usage: npx tsx test/codex-websocket-cached-probe.ts [options]
|
||||
|
||||
Options:
|
||||
--turns <n> Number of user turns. Default: ${DEFAULT_TURNS}
|
||||
--transport <mode> sse | websocket | websocket-cached | auto. Default: websocket-cached
|
||||
--reasoning <level> minimal | low | medium | high | xhigh. Default: low
|
||||
--max-tokens <n> Max output tokens per model request. Default: ${DEFAULT_MAX_TOKENS}
|
||||
--session-id <id> Session id for websocket/cache state
|
||||
`);
|
||||
}
|
||||
|
||||
function buildPrompt(turn: number): string {
|
||||
const marker = `TURN-${String(turn).padStart(2, "0")}-MARKER-${(turn * 17 + 13) % 97}`;
|
||||
const lines = [
|
||||
"This is an automated OpenAI Codex Responses websocket cache probe.",
|
||||
`Task for turn ${turn}: call deterministic_probe exactly once before your final answer.`,
|
||||
`Use tool arguments: turn=${turn}, marker=${marker}`,
|
||||
`After the tool result arrives, reply exactly: TURN ${turn} OK ${marker}`,
|
||||
"The following repeated block is intentional benchmark padding.",
|
||||
];
|
||||
for (let i = 1; i <= 180; i++) {
|
||||
lines.push(
|
||||
`Turn ${turn} synthetic record ${String(i).padStart(3, "0")}: alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega.`,
|
||||
);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function deterministicProbeTool(): Tool {
|
||||
return {
|
||||
name: "deterministic_probe",
|
||||
description: "Mandatory benchmark tool. Call exactly once with the turn and marker from the user prompt.",
|
||||
parameters: Type.Object({
|
||||
turn: Type.Number(),
|
||||
marker: Type.String(),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function executeTool(call: Extract<AssistantMessage["content"][number], { type: "toolCall" }>): ToolResultMessage {
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: call.id,
|
||||
toolName: call.name,
|
||||
content: [{ type: "text", text: `deterministic_probe_result ${JSON.stringify(call.arguments)} fixed=OK` }],
|
||||
details: { fixed: "OK" },
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function textOf(message: AssistantMessage): string {
|
||||
return message.content
|
||||
.filter((block): block is Extract<AssistantMessage["content"][number], { type: "text" }> => block.type === "text")
|
||||
.map((block) => block.text)
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function average(values: number[]): number {
|
||||
return values.reduce((sum, value) => sum + value, 0) / Math.max(1, values.length);
|
||||
}
|
||||
|
||||
function percentile(values: number[], p: number): number {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))];
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const model = getModel("openai-codex", "gpt-5.5") as Model<"openai-codex-responses"> | undefined;
|
||||
if (!model) throw new Error("Model openai-codex/gpt-5.5 not found");
|
||||
const modelWithMaxTokens = { ...model, maxTokens: args.maxTokens };
|
||||
const authStorage = AuthStorage.create();
|
||||
const apiKey = (await authStorage.getApiKey("openai-codex")) ?? (await authStorage.getApiKey("openai"));
|
||||
if (!apiKey) {
|
||||
throw new Error("No OpenAI Codex API key found in coding-agent auth storage.");
|
||||
}
|
||||
const context: Context = {
|
||||
systemPrompt:
|
||||
"You are participating in a benchmark. For each benchmark turn, call deterministic_probe exactly once before the final answer. Keep final answers minimal.",
|
||||
messages: [],
|
||||
tools: [deterministicProbeTool()],
|
||||
};
|
||||
const elapsed: number[] = [];
|
||||
resetOpenAICodexWebSocketDebugStats(args.sessionId);
|
||||
|
||||
console.log(`provider openai-codex, model gpt-5.5`);
|
||||
console.log(`sessionId ${args.sessionId}`);
|
||||
console.log(
|
||||
`turns ${args.turns}, transport ${args.transport}, reasoning ${args.reasoning}, maxTokens ${args.maxTokens}`,
|
||||
);
|
||||
console.log(`scratch ${resolve(join(tmpdir(), args.sessionId))}`);
|
||||
console.log("");
|
||||
|
||||
for (let turn = 1; turn <= args.turns; turn++) {
|
||||
context.messages.push({ role: "user", content: buildPrompt(turn), timestamp: Date.now() });
|
||||
const beforeStats = getOpenAICodexWebSocketDebugStats(args.sessionId);
|
||||
const started = Date.now();
|
||||
let requests = 0;
|
||||
let assistantCount = 0;
|
||||
let toolResults = 0;
|
||||
let finalText = "";
|
||||
let turnInput = 0;
|
||||
let turnOutput = 0;
|
||||
let turnCacheRead = 0;
|
||||
let turnCacheWrite = 0;
|
||||
|
||||
while (true) {
|
||||
requests++;
|
||||
const message = await streamOpenAICodexResponses(modelWithMaxTokens, context, {
|
||||
apiKey,
|
||||
sessionId: args.sessionId,
|
||||
transport: args.transport,
|
||||
reasoningEffort: args.reasoning,
|
||||
maxTokens: args.maxTokens,
|
||||
}).result();
|
||||
assistantCount++;
|
||||
context.messages.push(message);
|
||||
turnInput += message.usage.input;
|
||||
turnOutput += message.usage.output;
|
||||
turnCacheRead += message.usage.cacheRead;
|
||||
turnCacheWrite += message.usage.cacheWrite;
|
||||
const toolCalls = message.content.filter(
|
||||
(block): block is Extract<AssistantMessage["content"][number], { type: "toolCall" }> =>
|
||||
block.type === "toolCall",
|
||||
);
|
||||
console.log(
|
||||
[
|
||||
`turn ${String(turn).padStart(2, "0")}.${requests}`,
|
||||
`stop ${message.stopReason}`,
|
||||
`in ${message.usage.input}`,
|
||||
`out ${message.usage.output}`,
|
||||
`cache ${message.usage.cacheRead}/${message.usage.cacheWrite}`,
|
||||
`tools ${toolCalls.length}`,
|
||||
].join(" | "),
|
||||
);
|
||||
if (message.stopReason === "error" || message.stopReason === "aborted") {
|
||||
throw new Error(message.errorMessage ?? `request failed on turn ${turn}.${requests}`);
|
||||
}
|
||||
if (toolCalls.length === 0) {
|
||||
finalText = textOf(message);
|
||||
break;
|
||||
}
|
||||
for (const call of toolCalls) {
|
||||
context.messages.push(executeTool(call) as Message);
|
||||
toolResults++;
|
||||
}
|
||||
if (requests > 4) throw new Error(`Too many requests for turn ${turn}`);
|
||||
}
|
||||
|
||||
const elapsedMs = Date.now() - started;
|
||||
elapsed.push(elapsedMs);
|
||||
const afterStats = getOpenAICodexWebSocketDebugStats(args.sessionId);
|
||||
const statLine = afterStats
|
||||
? `ws requests ${afterStats.requests - (beforeStats?.requests ?? 0)} | new/reused ${afterStats.connectionsCreated - (beforeStats?.connectionsCreated ?? 0)}/${afterStats.connectionsReused - (beforeStats?.connectionsReused ?? 0)} | cached ${afterStats.cachedContextRequests - (beforeStats?.cachedContextRequests ?? 0)} | store ${afterStats.storeTrueRequests - (beforeStats?.storeTrueRequests ?? 0)} | full/delta ${afterStats.fullContextRequests - (beforeStats?.fullContextRequests ?? 0)}/${afterStats.deltaRequests - (beforeStats?.deltaRequests ?? 0)}`
|
||||
: "ws none";
|
||||
console.log(
|
||||
[
|
||||
`turn ${String(turn).padStart(2, "0")} agg`,
|
||||
`elapsed ${(elapsedMs / 1000).toFixed(1)}s`,
|
||||
`assistant ${assistantCount}`,
|
||||
`toolResults ${toolResults}`,
|
||||
`in ${turnInput}`,
|
||||
`out ${turnOutput}`,
|
||||
`cache ${turnCacheRead}/${turnCacheWrite}`,
|
||||
statLine,
|
||||
`final ${JSON.stringify(finalText).slice(0, 80)}`,
|
||||
].join(" | "),
|
||||
);
|
||||
}
|
||||
|
||||
const stats = getOpenAICodexWebSocketDebugStats(args.sessionId);
|
||||
console.log("");
|
||||
console.log(
|
||||
[
|
||||
"timing",
|
||||
`turns ${elapsed.length}`,
|
||||
`total ${(elapsed.reduce((sum, value) => sum + value, 0) / 1000).toFixed(1)}s`,
|
||||
`avg ${(average(elapsed) / 1000).toFixed(2)}s`,
|
||||
`p50 ${(percentile(elapsed, 50) / 1000).toFixed(2)}s`,
|
||||
`p95 ${(percentile(elapsed, 95) / 1000).toFixed(2)}s`,
|
||||
`max ${(Math.max(...elapsed) / 1000).toFixed(2)}s`,
|
||||
].join(" | "),
|
||||
);
|
||||
console.log(
|
||||
[
|
||||
"transport summary",
|
||||
`requested ${args.transport}`,
|
||||
`observed ${stats && stats.requests > 0 ? "websocket" : "sse/no-websocket"}`,
|
||||
`storeTrue ${stats ? `${stats.storeTrueRequests}/${stats.requests}` : "0/0"}`,
|
||||
`full/delta ${stats ? `${stats.fullContextRequests}/${stats.deltaRequests}` : "0/0"}`,
|
||||
`connections created/reused ${stats ? `${stats.connectionsCreated}/${stats.connectionsReused}` : "0/0"}`,
|
||||
`lastPreviousResponseId ${stats?.lastPreviousResponseId ?? "n/a"}`,
|
||||
].join(" | "),
|
||||
);
|
||||
closeOpenAICodexWebSocketSessions(args.sessionId);
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -23,13 +23,8 @@ import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Resolve OAuth tokens at module level (async, runs before tests)
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const oauthTokens = await Promise.all([resolveApiKey("github-copilot"), resolveApiKey("openai-codex")]);
|
||||
const [githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
// Lorem ipsum paragraph for realistic token estimation
|
||||
const LOREM_IPSUM = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. `;
|
||||
@@ -220,64 +215,12 @@ describe("Context overflow error handling", () => {
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Google Gemini CLI (OAuth)
|
||||
// Uses same API as Google, expects same error pattern
|
||||
// =============================================================================
|
||||
|
||||
describe("Google Gemini CLI (OAuth)", () => {
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should detect overflow via isContextOverflow",
|
||||
async () => {
|
||||
const model = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
const result = await testContextOverflow(model, geminiCliToken!);
|
||||
logResult(result);
|
||||
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toMatch(/input token count.*exceeds the maximum/i);
|
||||
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||
},
|
||||
120000,
|
||||
);
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Google Antigravity (OAuth)
|
||||
// Tests both Gemini and Anthropic models via Antigravity
|
||||
// =============================================================================
|
||||
|
||||
describe("Google Antigravity (OAuth)", () => {
|
||||
// Gemini model
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should detect overflow via isContextOverflow",
|
||||
async () => {
|
||||
const model = getModel("google-antigravity", "gemini-3-flash");
|
||||
const result = await testContextOverflow(model, antigravityToken!);
|
||||
logResult(result);
|
||||
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toMatch(/input token count.*exceeds the maximum/i);
|
||||
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||
},
|
||||
120000,
|
||||
);
|
||||
|
||||
// Anthropic model via Antigravity
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should detect overflow via isContextOverflow",
|
||||
async () => {
|
||||
const model = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
const result = await testContextOverflow(model, antigravityToken!);
|
||||
logResult(result);
|
||||
|
||||
expect(result.stopReason).toBe("error");
|
||||
// Anthropic models return "prompt is too long" pattern
|
||||
expect(result.errorMessage).toMatch(/prompt is too long/i);
|
||||
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||
},
|
||||
120000,
|
||||
);
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// OpenAI Codex (OAuth)
|
||||
// Uses ChatGPT Plus/Pro subscription via OAuth
|
||||
@@ -446,6 +389,61 @@ describe("Context overflow error handling", () => {
|
||||
}, 120000);
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Xiaomi MiMo
|
||||
// =============================================================================
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing)", () => {
|
||||
// Xiaomi silently truncates oversized input to fill the context window exactly,
|
||||
// then returns finish_reason "length" with output=0 (no room left to generate).
|
||||
// This is a detectable overflow signal but uses stopReason "length" rather than "error".
|
||||
it("mimo-v2.5-pro - should detect overflow via isContextOverflow", async () => {
|
||||
const model = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
const result = await testContextOverflow(model, process.env.XIAOMI_API_KEY!);
|
||||
logResult(result);
|
||||
|
||||
expect(result.stopReason).toBe("length");
|
||||
expect(result.usage.output).toBe(0);
|
||||
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||
}, 120000);
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN)", () => {
|
||||
it("mimo-v2.5-pro - should detect overflow via isContextOverflow", async () => {
|
||||
const model = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
const result = await testContextOverflow(model, process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY!);
|
||||
logResult(result);
|
||||
|
||||
expect(result.stopReason).toBe("length");
|
||||
expect(result.usage.output).toBe(0);
|
||||
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||
}, 120000);
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS)", () => {
|
||||
it("mimo-v2.5-pro - should detect overflow via isContextOverflow", async () => {
|
||||
const model = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
const result = await testContextOverflow(model, process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY!);
|
||||
logResult(result);
|
||||
|
||||
expect(result.stopReason).toBe("length");
|
||||
expect(result.usage.output).toBe(0);
|
||||
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||
}, 120000);
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP)", () => {
|
||||
it("mimo-v2.5-pro - should detect overflow via isContextOverflow", async () => {
|
||||
const model = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
const result = await testContextOverflow(model, process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY!);
|
||||
logResult(result);
|
||||
|
||||
expect(result.stopReason).toBe("length");
|
||||
expect(result.usage.output).toBe(0);
|
||||
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||
}, 120000);
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Kimi For Coding
|
||||
// =============================================================================
|
||||
|
||||
@@ -29,6 +29,7 @@ import { getModel } from "../src/models.js";
|
||||
import { completeSimple, getEnvApiKey } from "../src/stream.js";
|
||||
import type { Api, AssistantMessage, Message, Model, Tool, ToolResultMessage } from "../src/types.js";
|
||||
import { hasAzureOpenAICredentials } from "./azure-utils.js";
|
||||
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Simple tool for testing
|
||||
@@ -48,6 +49,7 @@ interface ProviderModelPair {
|
||||
model: string;
|
||||
label: string;
|
||||
apiOverride?: Api;
|
||||
upstreamApiKeyEnv?: string;
|
||||
}
|
||||
|
||||
const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [
|
||||
@@ -66,9 +68,6 @@ const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [
|
||||
{ provider: "azure-openai-responses", model: "gpt-4o-mini", label: "azure-openai-responses-gpt-4o-mini" },
|
||||
// OpenAI Codex
|
||||
{ provider: "openai-codex", model: "gpt-5.2-codex", label: "openai-codex-gpt-5.2-codex" },
|
||||
// Google Antigravity
|
||||
{ provider: "google-antigravity", model: "gemini-3-flash", label: "antigravity-gemini-3-flash" },
|
||||
{ provider: "google-antigravity", model: "claude-sonnet-4-5", label: "antigravity-claude-sonnet-4-5" },
|
||||
// GitHub Copilot
|
||||
{ provider: "github-copilot", model: "claude-sonnet-4.5", label: "copilot-claude-sonnet-4.5" },
|
||||
{ provider: "github-copilot", model: "gpt-5.1-codex", label: "copilot-gpt-5.1-codex" },
|
||||
@@ -86,6 +85,24 @@ const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [
|
||||
{ provider: "cerebras", model: "zai-glm-4.7", label: "cerebras-zai-glm-4.7" },
|
||||
// Cloudflare Workers AI
|
||||
{ provider: "cloudflare-workers-ai", model: "@cf/moonshotai/kimi-k2.6", label: "cloudflare-kimi-k2.6" },
|
||||
// Cloudflare AI Gateway
|
||||
{
|
||||
provider: "cloudflare-ai-gateway",
|
||||
model: "workers-ai/@cf/moonshotai/kimi-k2.6",
|
||||
label: "cloudflare-gateway-kimi-k2.6",
|
||||
},
|
||||
{
|
||||
provider: "cloudflare-ai-gateway",
|
||||
model: "claude-sonnet-4-5",
|
||||
label: "cloudflare-gateway-claude-sonnet-4-5",
|
||||
upstreamApiKeyEnv: "ANTHROPIC_API_KEY",
|
||||
},
|
||||
{
|
||||
provider: "cloudflare-ai-gateway",
|
||||
model: "gpt-5.1",
|
||||
label: "cloudflare-gateway-gpt-5.1",
|
||||
upstreamApiKeyEnv: "OPENAI_API_KEY",
|
||||
},
|
||||
// Groq
|
||||
{ provider: "groq", model: "openai/gpt-oss-120b", label: "groq-gpt-oss-120b" },
|
||||
// Hugging Face
|
||||
@@ -96,6 +113,7 @@ const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [
|
||||
{ provider: "mistral", model: "devstral-medium-latest", label: "mistral-devstral-medium" },
|
||||
// MiniMax
|
||||
{ provider: "minimax", model: "MiniMax-M2.7", label: "minimax-m2.7" },
|
||||
{ provider: "minimax-cn", model: "MiniMax-M2.7", label: "minimax-m2.7" },
|
||||
// OpenCode Zen
|
||||
{ provider: "opencode", model: "big-pickle", label: "zen-big-pickle" },
|
||||
{ provider: "opencode", model: "claude-sonnet-4-5", label: "zen-claude-sonnet-4-5" },
|
||||
@@ -106,6 +124,11 @@ const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [
|
||||
// OpenCode Go
|
||||
{ provider: "opencode-go", model: "kimi-k2.5", label: "go-kimi-k2.5" },
|
||||
{ provider: "opencode-go", model: "minimax-m2.5", label: "go-minimax-m2.5" },
|
||||
// Xiaomi MiMo
|
||||
{ provider: "xiaomi", model: "mimo-v2.5-pro", label: "xiaomi-mimo-v2.5-pro" },
|
||||
{ provider: "xiaomi-token-plan-cn", model: "mimo-v2.5-pro", label: "xiaomi-token-plan-cn-mimo-v2.5-pro" },
|
||||
{ provider: "xiaomi-token-plan-ams", model: "mimo-v2.5-pro", label: "xiaomi-token-plan-ams-mimo-v2.5-pro" },
|
||||
{ provider: "xiaomi-token-plan-sgp", model: "mimo-v2.5-pro", label: "xiaomi-token-plan-sgp-mimo-v2.5-pro" },
|
||||
];
|
||||
|
||||
// Cached context structure
|
||||
@@ -130,18 +153,31 @@ async function getApiKey(provider: string): Promise<string | undefined> {
|
||||
/**
|
||||
* Synchronous check for API key availability (env vars only, for skipIf)
|
||||
*/
|
||||
function hasApiKey(provider: string): boolean {
|
||||
if (provider === "azure-openai-responses") {
|
||||
function hasApiKey(pair: ProviderModelPair): boolean {
|
||||
if (pair.provider === "azure-openai-responses") {
|
||||
return hasAzureOpenAICredentials();
|
||||
}
|
||||
return !!getEnvApiKey(provider);
|
||||
if (pair.provider === "cloudflare-workers-ai") {
|
||||
return hasCloudflareWorkersAICredentials();
|
||||
}
|
||||
if (pair.provider === "cloudflare-ai-gateway") {
|
||||
if (!hasCloudflareAiGatewayCredentials()) return false;
|
||||
return pair.upstreamApiKeyEnv ? !!process.env[pair.upstreamApiKeyEnv] : true;
|
||||
}
|
||||
return !!getEnvApiKey(pair.provider);
|
||||
}
|
||||
|
||||
function getHeaders(pair: ProviderModelPair): Record<string, string> | undefined {
|
||||
if (!pair.upstreamApiKeyEnv) return undefined;
|
||||
const upstreamApiKey = process.env[pair.upstreamApiKeyEnv];
|
||||
return upstreamApiKey ? { Authorization: `Bearer ${upstreamApiKey}` } : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any provider has API keys available (for skipIf at describe level)
|
||||
*/
|
||||
function hasAnyApiKey(): boolean {
|
||||
return PROVIDER_MODEL_PAIRS.some((pair) => hasApiKey(pair.provider));
|
||||
return PROVIDER_MODEL_PAIRS.some((pair) => hasApiKey(pair));
|
||||
}
|
||||
|
||||
function dumpFailurePayload(params: { label: string; error: string; payload?: unknown; messages: Message[] }): void {
|
||||
@@ -179,6 +215,7 @@ async function generateContext(
|
||||
};
|
||||
|
||||
const supportsReasoning = model.reasoning === true;
|
||||
const headers = getHeaders(pair);
|
||||
let lastPayload: unknown;
|
||||
let assistantResponse: AssistantMessage;
|
||||
try {
|
||||
@@ -192,6 +229,7 @@ async function generateContext(
|
||||
{
|
||||
apiKey,
|
||||
reasoning: supportsReasoning ? "high" : undefined,
|
||||
headers,
|
||||
onPayload: (payload) => {
|
||||
lastPayload = payload;
|
||||
},
|
||||
@@ -253,6 +291,7 @@ async function generateContext(
|
||||
{
|
||||
apiKey,
|
||||
reasoning: supportsReasoning ? "high" : undefined,
|
||||
headers,
|
||||
onPayload: (payload) => {
|
||||
lastPayload = payload;
|
||||
},
|
||||
@@ -299,7 +338,7 @@ describe.skipIf(!hasAnyApiKey())("Cross-Provider Handoff", () => {
|
||||
|
||||
for (const pair of PROVIDER_MODEL_PAIRS) {
|
||||
const apiKey = await getApiKey(pair.provider);
|
||||
if (!apiKey) {
|
||||
if (!apiKey || !hasApiKey(pair)) {
|
||||
console.log(`[${pair.label}] Skipping - no auth for ${pair.provider}`);
|
||||
continue;
|
||||
}
|
||||
@@ -347,7 +386,7 @@ describe.skipIf(!hasAnyApiKey())("Cross-Provider Handoff", () => {
|
||||
|
||||
for (const targetPair of availablePairs) {
|
||||
const apiKey = await getApiKey(targetPair.provider);
|
||||
if (!apiKey) {
|
||||
if (!apiKey || !hasApiKey(targetPair)) {
|
||||
console.log(`[Target: ${targetPair.label}] Skipping - no auth`);
|
||||
continue;
|
||||
}
|
||||
@@ -387,6 +426,7 @@ describe.skipIf(!hasAnyApiKey())("Cross-Provider Handoff", () => {
|
||||
? { ...baseModel, api: targetPair.apiOverride }
|
||||
: baseModel;
|
||||
const supportsReasoning = model.reasoning === true;
|
||||
const headers = getHeaders(targetPair);
|
||||
|
||||
console.log(
|
||||
`[Target: ${targetPair.label}] Testing with ${otherMessages.length} messages from other providers...`,
|
||||
@@ -404,6 +444,7 @@ describe.skipIf(!hasAnyApiKey())("Cross-Provider Handoff", () => {
|
||||
{
|
||||
apiKey,
|
||||
reasoning: supportsReasoning ? "high" : undefined,
|
||||
headers,
|
||||
onPayload: (payload) => {
|
||||
lastPayload = payload;
|
||||
},
|
||||
|
||||
@@ -7,17 +7,16 @@ type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
|
||||
|
||||
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
|
||||
import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Resolve OAuth tokens at module level (async, runs before tests)
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("anthropic"),
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
async function testEmptyMessage<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
|
||||
// Test with completely empty content array
|
||||
@@ -308,28 +307,45 @@ describe("AI Providers Empty Message Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.CLOUDFLARE_API_KEY || !process.env.CLOUDFLARE_ACCOUNT_ID)(
|
||||
"Cloudflare Workers AI Provider Empty Messages",
|
||||
() => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI Provider Empty Messages", () => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider Empty Messages", () => {
|
||||
const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider Empty Messages", () => {
|
||||
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
|
||||
@@ -411,6 +427,95 @@ describe("AI Providers Empty Message Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider Empty Messages", () => {
|
||||
const llm = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (CN) Provider Empty Messages",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (AMS) Provider Empty Messages",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (SGP) Provider Empty Messages",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Empty Messages", () => {
|
||||
const llm = getModel("kimi-coding", "kimi-k2-thinking");
|
||||
|
||||
@@ -577,154 +682,6 @@ describe("AI Providers Empty Message Tests", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider Empty Messages", () => {
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle empty content array",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testEmptyMessage(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle empty string content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testEmptyStringMessage(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle whitespace-only content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testWhitespaceOnlyMessage(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle empty assistant message in conversation",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testEmptyAssistantMessage(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider Empty Messages", () => {
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle empty content array",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testEmptyMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle empty string content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testEmptyStringMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle whitespace-only content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testWhitespaceOnlyMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle empty assistant message in conversation",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testEmptyAssistantMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle empty content array",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testEmptyMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle empty string content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testEmptyStringMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle whitespace-only content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testWhitespaceOnlyMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle empty assistant message in conversation",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testEmptyAssistantMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should handle empty content array",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testEmptyMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should handle empty string content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testEmptyStringMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should handle whitespace-only content",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testWhitespaceOnlyMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should handle empty assistant message in conversation",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testEmptyAssistantMessage(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("OpenAI Codex Provider Empty Messages", () => {
|
||||
it.skipIf(!openaiCodexToken)(
|
||||
"gpt-5.2-codex - should handle empty content array",
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { streamGoogleGeminiCli } from "../src/providers/google-gemini-cli.js";
|
||||
import type { Context, Model } from "../src/types.js";
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
const apiKey = JSON.stringify({ token: "token", projectId: "project" });
|
||||
|
||||
const createSseResponse = () => {
|
||||
const sse = `${[
|
||||
`data: ${JSON.stringify({
|
||||
response: {
|
||||
candidates: [
|
||||
{
|
||||
content: { role: "model", parts: [{ text: "Hello" }] },
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
},
|
||||
})}`,
|
||||
].join("\n\n")}\n\n`;
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(sse));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("google-gemini-cli Claude thinking header", () => {
|
||||
const context: Context = {
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
it("adds anthropic-beta for Claude thinking models", async () => {
|
||||
const fetchMock = vi.fn(async (_input: string | URL, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers);
|
||||
expect(headers.get("anthropic-beta")).toBe("interleaved-thinking-2025-05-14");
|
||||
return createSseResponse();
|
||||
});
|
||||
|
||||
global.fetch = fetchMock as typeof fetch;
|
||||
|
||||
const model: Model<"google-gemini-cli"> = {
|
||||
id: "claude-opus-4-5-thinking",
|
||||
name: "Claude Opus 4.5 Thinking",
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-antigravity",
|
||||
baseUrl: "https://cloudcode-pa.googleapis.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
|
||||
const stream = streamGoogleGeminiCli(model, context, { apiKey });
|
||||
for await (const _event of stream) {
|
||||
// exhaust stream
|
||||
}
|
||||
await stream.result();
|
||||
});
|
||||
|
||||
it("does not add anthropic-beta for Gemini models", async () => {
|
||||
const fetchMock = vi.fn(async (_input: string | URL, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers);
|
||||
expect(headers.has("anthropic-beta")).toBe(false);
|
||||
return createSseResponse();
|
||||
});
|
||||
|
||||
global.fetch = fetchMock as typeof fetch;
|
||||
|
||||
const model: Model<"google-gemini-cli"> = {
|
||||
id: "gemini-2.5-flash",
|
||||
name: "Gemini 2.5 Flash",
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-gemini-cli",
|
||||
baseUrl: "https://cloudcode-pa.googleapis.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
|
||||
const stream = streamGoogleGeminiCli(model, context, { apiKey });
|
||||
for await (const _event of stream) {
|
||||
// exhaust stream
|
||||
}
|
||||
await stream.result();
|
||||
});
|
||||
});
|
||||
@@ -1,108 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { streamGoogleGeminiCli } from "../src/providers/google-gemini-cli.js";
|
||||
import type { Context, Model } from "../src/types.js";
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("google-gemini-cli empty stream retry", () => {
|
||||
it("retries empty SSE responses without duplicate start", async () => {
|
||||
const emptyStream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
const sse = `${[
|
||||
`data: ${JSON.stringify({
|
||||
response: {
|
||||
candidates: [
|
||||
{
|
||||
content: { role: "model", parts: [{ text: "Hello" }] },
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
usageMetadata: {
|
||||
promptTokenCount: 1,
|
||||
candidatesTokenCount: 1,
|
||||
totalTokenCount: 2,
|
||||
},
|
||||
},
|
||||
})}`,
|
||||
].join("\n\n")}\n\n`;
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const dataStream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(sse));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
let callCount = 0;
|
||||
const fetchMock = vi.fn(async () => {
|
||||
callCount += 1;
|
||||
if (callCount === 1) {
|
||||
return new Response(emptyStream, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
}
|
||||
return new Response(dataStream, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
});
|
||||
|
||||
global.fetch = fetchMock as typeof fetch;
|
||||
|
||||
const model: Model<"google-gemini-cli"> = {
|
||||
id: "gemini-2.5-flash",
|
||||
name: "Gemini 2.5 Flash",
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-gemini-cli",
|
||||
baseUrl: "https://cloudcode-pa.googleapis.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
|
||||
const context: Context = {
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
const stream = streamGoogleGeminiCli(model, context, {
|
||||
apiKey: JSON.stringify({ token: "token", projectId: "project" }),
|
||||
});
|
||||
|
||||
let startCount = 0;
|
||||
let doneCount = 0;
|
||||
let text = "";
|
||||
|
||||
for await (const event of stream) {
|
||||
if (event.type === "start") {
|
||||
startCount += 1;
|
||||
}
|
||||
if (event.type === "done") {
|
||||
doneCount += 1;
|
||||
}
|
||||
if (event.type === "text_delta") {
|
||||
text += event.delta;
|
||||
}
|
||||
}
|
||||
|
||||
const result = await stream.result();
|
||||
|
||||
expect(text).toBe("Hello");
|
||||
expect(result.stopReason).toBe("stop");
|
||||
expect(startCount).toBe(1);
|
||||
expect(doneCount).toBe(1);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -1,53 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { extractRetryDelay } from "../src/providers/google-gemini-cli.js";
|
||||
|
||||
describe("extractRetryDelay header parsing", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("prefers Retry-After seconds header", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:00Z"));
|
||||
|
||||
const response = new Response("", { headers: { "Retry-After": "5" } });
|
||||
const delay = extractRetryDelay("Please retry in 1s", response);
|
||||
|
||||
expect(delay).toBe(6000);
|
||||
});
|
||||
|
||||
it("parses Retry-After HTTP date header", () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2025-01-01T00:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
|
||||
const retryAt = new Date(now.getTime() + 12000).toUTCString();
|
||||
const response = new Response("", { headers: { "Retry-After": retryAt } });
|
||||
const delay = extractRetryDelay("", response);
|
||||
|
||||
expect(delay).toBe(13000);
|
||||
});
|
||||
|
||||
it("parses x-ratelimit-reset header", () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2025-01-01T00:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
|
||||
const resetAtMs = now.getTime() + 20000;
|
||||
const resetSeconds = Math.floor(resetAtMs / 1000).toString();
|
||||
const response = new Response("", { headers: { "x-ratelimit-reset": resetSeconds } });
|
||||
const delay = extractRetryDelay("", response);
|
||||
|
||||
expect(delay).toBe(21000);
|
||||
});
|
||||
|
||||
it("parses x-ratelimit-reset-after header", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:00Z"));
|
||||
|
||||
const response = new Response("", { headers: { "x-ratelimit-reset-after": "30" } });
|
||||
const delay = extractRetryDelay("", response);
|
||||
|
||||
expect(delay).toBe(31000);
|
||||
});
|
||||
});
|
||||
@@ -2,15 +2,17 @@ import { describe, expect, it } from "vitest";
|
||||
import { convertMessages } from "../src/providers/google-shared.js";
|
||||
import type { Context, Model } from "../src/types.js";
|
||||
|
||||
const SKIP_THOUGHT_SIGNATURE = "skip_thought_signature_validator";
|
||||
|
||||
function makeGemini3Model(id = "gemini-3-pro-preview"): Model<"google-generative-ai"> {
|
||||
function makeGemini3Model<TApi extends "google-generative-ai" | "google-vertex">(
|
||||
api: TApi,
|
||||
provider: Model<TApi>["provider"],
|
||||
id = "gemini-3-pro-preview",
|
||||
): Model<TApi> {
|
||||
return {
|
||||
id,
|
||||
name: "Gemini 3 Pro Preview",
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
api,
|
||||
provider,
|
||||
baseUrl: "https://example.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
@@ -19,154 +21,96 @@ function makeGemini3Model(id = "gemini-3-pro-preview"): Model<"google-generative
|
||||
};
|
||||
}
|
||||
|
||||
describe("google-shared convertMessages — Gemini 3 unsigned tool calls", () => {
|
||||
it("uses skip_thought_signature_validator for unsigned tool calls on Gemini 3", () => {
|
||||
const model = makeGemini3Model();
|
||||
const now = Date.now();
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{ role: "user", content: "Hi", timestamp: now },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "bash",
|
||||
arguments: { command: "ls -la" },
|
||||
// No thoughtSignature: simulates Claude via Antigravity.
|
||||
},
|
||||
],
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-antigravity",
|
||||
model: "claude-sonnet-4-6",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
function makeContext(model: { api: string; provider: string; id: string }, thoughtSignature?: string): Context {
|
||||
const now = Date.now();
|
||||
return {
|
||||
messages: [
|
||||
{ role: "user", content: "Hi", timestamp: now },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "bash",
|
||||
arguments: { command: "echo hi" },
|
||||
...(thoughtSignature && { thoughtSignature }),
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: now,
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_2",
|
||||
name: "bash",
|
||||
arguments: { command: "ls -la" },
|
||||
},
|
||||
],
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
],
|
||||
};
|
||||
stopReason: "toolUse",
|
||||
timestamp: now,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
describe("google-shared convertMessages — Gemini 3 unsigned tool calls", () => {
|
||||
it("does not add skip_thought_signature_validator for unsigned Google Gen AI tool calls", () => {
|
||||
const model = makeGemini3Model("google-generative-ai", "google");
|
||||
const contents = convertMessages(model, makeContext({ ...model, id: "other-model" }));
|
||||
|
||||
const modelTurn = contents.find((c) => c.role === "model");
|
||||
expect(modelTurn).toBeTruthy();
|
||||
|
||||
// Should be a structured functionCall, NOT text fallback
|
||||
const fcPart = modelTurn?.parts?.find((p) => p.functionCall !== undefined);
|
||||
expect(fcPart).toBeTruthy();
|
||||
expect(fcPart?.functionCall?.name).toBe("bash");
|
||||
expect(fcPart?.functionCall?.args).toEqual({ command: "ls -la" });
|
||||
expect(fcPart?.thoughtSignature).toBe(SKIP_THOUGHT_SIGNATURE);
|
||||
const functionCallParts = modelTurn?.parts?.filter((p) => p.functionCall !== undefined) ?? [];
|
||||
expect(functionCallParts).toHaveLength(2);
|
||||
expect(functionCallParts[0]?.thoughtSignature).toBeUndefined();
|
||||
expect(functionCallParts[1]?.thoughtSignature).toBeUndefined();
|
||||
expect(JSON.stringify(modelTurn)).not.toContain("skip_thought_signature_validator");
|
||||
|
||||
// No text fallback should exist
|
||||
const textParts = modelTurn?.parts?.filter((p) => p.text !== undefined) ?? [];
|
||||
const historicalText = textParts.filter((p) => p.text?.includes("Historical context"));
|
||||
expect(historicalText).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("preserves valid thoughtSignature when present (same provider/model)", () => {
|
||||
const model = makeGemini3Model();
|
||||
const now = Date.now();
|
||||
// Valid base64 signature (16 bytes = 24 chars base64)
|
||||
const validSig = "AAAAAAAAAAAAAAAAAAAAAA==";
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{ role: "user", content: "Hi", timestamp: now },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "bash",
|
||||
arguments: { command: "echo hi" },
|
||||
thoughtSignature: validSig,
|
||||
},
|
||||
],
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
model: "gemini-3-pro-preview",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: now,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
it("does not add skip_thought_signature_validator for unsigned Vertex tool calls", () => {
|
||||
const model = makeGemini3Model("google-vertex", "google-vertex");
|
||||
const contents = convertMessages(model, makeContext(model));
|
||||
const modelTurn = contents.find((c) => c.role === "model");
|
||||
const fcPart = modelTurn?.parts?.find((p) => p.functionCall !== undefined);
|
||||
const functionCallParts = modelTurn?.parts?.filter((p) => p.functionCall !== undefined) ?? [];
|
||||
|
||||
expect(fcPart).toBeTruthy();
|
||||
expect(fcPart?.thoughtSignature).toBe(validSig);
|
||||
expect(functionCallParts).toHaveLength(2);
|
||||
expect(functionCallParts[0]?.thoughtSignature).toBeUndefined();
|
||||
expect(functionCallParts[1]?.thoughtSignature).toBeUndefined();
|
||||
expect(JSON.stringify(modelTurn)).not.toContain("skip_thought_signature_validator");
|
||||
});
|
||||
|
||||
it("does not add sentinel for non-Gemini-3 models", () => {
|
||||
const model: Model<"google-generative-ai"> = {
|
||||
id: "gemini-2.5-flash",
|
||||
name: "Gemini 2.5 Flash",
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
const now = Date.now();
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{ role: "user", content: "Hi", timestamp: now },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "bash",
|
||||
arguments: { command: "ls" },
|
||||
// No thoughtSignature
|
||||
},
|
||||
],
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-antigravity",
|
||||
model: "claude-sonnet-4-6",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: now,
|
||||
},
|
||||
],
|
||||
};
|
||||
it("preserves valid thoughtSignature when present for the same provider and model", () => {
|
||||
const model = makeGemini3Model("google-generative-ai", "google");
|
||||
const validSig = "AAAAAAAAAAAAAAAAAAAAAA==";
|
||||
const contents = convertMessages(model, makeContext(model, validSig));
|
||||
const modelTurn = contents.find((c) => c.role === "model");
|
||||
const functionCallParts = modelTurn?.parts?.filter((p) => p.functionCall !== undefined) ?? [];
|
||||
|
||||
const contents = convertMessages(model, context);
|
||||
expect(functionCallParts).toHaveLength(2);
|
||||
expect(functionCallParts[0]?.thoughtSignature).toBe(validSig);
|
||||
expect(functionCallParts[1]?.thoughtSignature).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not add a thoughtSignature for non-Gemini-3 models", () => {
|
||||
const model = makeGemini3Model("google-generative-ai", "google", "gemini-2.5-flash");
|
||||
const contents = convertMessages(model, makeContext({ ...model, id: "other-model" }));
|
||||
const modelTurn = contents.find((c) => c.role === "model");
|
||||
const fcPart = modelTurn?.parts?.find((p) => p.functionCall !== undefined);
|
||||
|
||||
expect(fcPart).toBeTruthy();
|
||||
// No sentinel, no thoughtSignature at all
|
||||
expect(fcPart?.thoughtSignature).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { convertMessages } from "../src/providers/google-shared.js";
|
||||
import type { Context, Model } from "../src/types.js";
|
||||
|
||||
function makeModel<TApi extends "google-generative-ai" | "google-gemini-cli">(
|
||||
function makeModel<TApi extends "google-generative-ai">(
|
||||
api: TApi,
|
||||
provider: Model<TApi>["provider"],
|
||||
id: string,
|
||||
@@ -99,26 +99,4 @@ describe("google-shared image tool result routing", () => {
|
||||
expect(imageResponse?.parts).toHaveLength(1);
|
||||
expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy();
|
||||
});
|
||||
|
||||
it("nests image tool results for non-Gemini models on Antigravity / Cloud Code Assist", () => {
|
||||
const model = makeModel("google-gemini-cli", "google-antigravity", "claude-sonnet-4-6");
|
||||
const contents = convertMessages(model, makeContext(model));
|
||||
|
||||
expect(contents).toHaveLength(3);
|
||||
const toolResultTurn = contents[2];
|
||||
expect(toolResultTurn.parts).toHaveLength(3);
|
||||
const imageResponse = toolResultTurn.parts?.[1]?.functionResponse;
|
||||
expect(imageResponse).toBeTruthy();
|
||||
expect(imageResponse?.parts).toHaveLength(1);
|
||||
expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy();
|
||||
});
|
||||
|
||||
it("keeps separate synthetic image turn for Gemini 2.x Cloud Code Assist models", () => {
|
||||
const model = makeModel("google-gemini-cli", "google-gemini-cli", "gemini-2.5-flash");
|
||||
const contents = convertMessages(model, makeContext(model));
|
||||
|
||||
expect(contents).toHaveLength(5);
|
||||
expect(contents[3].parts?.[0]?.text).toBe("Tool result image:");
|
||||
expect(contents[3].parts?.[1]?.inlineData).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest";
|
||||
import { getModel } from "../src/models.js";
|
||||
import { streamSimple } from "../src/stream.js";
|
||||
import type { Api, Context, Model, SimpleStreamOptions } from "../src/types.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
type SimpleOptionsWithExtras = SimpleStreamOptions & Record<string, unknown>;
|
||||
|
||||
@@ -20,9 +19,6 @@ interface DisableExpectations {
|
||||
maxOutputTokens?: number;
|
||||
}
|
||||
|
||||
const oauthTokens = await Promise.all([resolveApiKey("google-gemini-cli"), resolveApiKey("google-antigravity")]);
|
||||
const [geminiCliToken, antigravityToken] = oauthTokens;
|
||||
|
||||
function makeContext(): Context {
|
||||
return {
|
||||
systemPrompt: "You are a precise assistant. Follow the requested output format exactly.",
|
||||
@@ -148,24 +144,6 @@ describe("Google Vertex thinking disable E2E", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI thinking disable E2E", () => {
|
||||
it.skipIf(!geminiCliToken)("disables thinking for Gemini 2.5", { retry: 2, timeout: 30000 }, async () => {
|
||||
await expectThinkingDisabledE2E(getModel("google-gemini-cli", "gemini-2.5-flash"), {
|
||||
requestOptions: { apiKey: geminiCliToken! },
|
||||
maxOutputTokens: 100,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Antigravity thinking disable E2E", () => {
|
||||
it.skipIf(!antigravityToken)("disables thinking for Gemini 3.x", { retry: 2, timeout: 30000 }, async () => {
|
||||
await expectThinkingDisabledE2E(getModel("google-antigravity", "gemini-3-flash"), {
|
||||
requestOptions: { apiKey: antigravityToken! },
|
||||
maxOutputTokens: 100,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI thinking disable E2E", () => {
|
||||
it("disables thinking for Responses reasoning models", { retry: 2, timeout: 30000 }, async () => {
|
||||
await expectThinkingDisabledE2E(getModel("openai", "gpt-5.4-mini"), {
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { streamGoogleGeminiCli } from "../src/providers/google-gemini-cli.js";
|
||||
import type { Context, Model, ToolCall } from "../src/types.js";
|
||||
|
||||
const emptySchema = Type.Object({});
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("google providers tool call missing args", () => {
|
||||
it("defaults arguments to empty object when provider omits args field", async () => {
|
||||
// Simulate a tool call response where args is missing (no-arg tool)
|
||||
const sse = `${[
|
||||
`data: ${JSON.stringify({
|
||||
response: {
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: "model",
|
||||
parts: [
|
||||
{
|
||||
functionCall: {
|
||||
name: "get_status",
|
||||
// args intentionally omitted
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
usageMetadata: {
|
||||
promptTokenCount: 10,
|
||||
candidatesTokenCount: 5,
|
||||
totalTokenCount: 15,
|
||||
},
|
||||
},
|
||||
})}`,
|
||||
].join("\n\n")}\n\n`;
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const dataStream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(sse));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
const fetchMock = vi.fn(async () => {
|
||||
return new Response(dataStream, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
});
|
||||
|
||||
global.fetch = fetchMock as typeof fetch;
|
||||
|
||||
const model: Model<"google-gemini-cli"> = {
|
||||
id: "gemini-2.5-flash",
|
||||
name: "Gemini 2.5 Flash",
|
||||
api: "google-gemini-cli",
|
||||
provider: "google-gemini-cli",
|
||||
baseUrl: "https://cloudcode-pa.googleapis.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
|
||||
const context: Context = {
|
||||
messages: [{ role: "user", content: "Check status", timestamp: Date.now() }],
|
||||
tools: [
|
||||
{
|
||||
name: "get_status",
|
||||
description: "Get current status",
|
||||
parameters: emptySchema,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const stream = streamGoogleGeminiCli(model, context, {
|
||||
apiKey: JSON.stringify({ token: "token", projectId: "project" }),
|
||||
});
|
||||
|
||||
for await (const _ of stream) {
|
||||
// consume stream
|
||||
}
|
||||
|
||||
const result = await stream.result();
|
||||
|
||||
expect(result.stopReason).toBe("toolUse");
|
||||
expect(result.content).toHaveLength(1);
|
||||
|
||||
const toolCall = result.content[0] as ToolCall;
|
||||
expect(toolCall.type).toBe("toolCall");
|
||||
expect(toolCall.name).toBe("get_status");
|
||||
expect(toolCall.arguments).toEqual({});
|
||||
});
|
||||
});
|
||||
@@ -16,11 +16,9 @@ import { resolveApiKey } from "./oauth.js";
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("anthropic"),
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
/**
|
||||
* Test that tool results containing only images work correctly across all providers.
|
||||
@@ -300,6 +298,76 @@ describe("Tool Results with Images", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider (mimo-v2.5-pro)", () => {
|
||||
const llm = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithImageResult(llm);
|
||||
});
|
||||
|
||||
// FIXME(xiaomi): when a tool_result contains both a descriptive text block
|
||||
// and an image block, MiMo locks onto the text and ignores the image (it
|
||||
// reports the text-derived diameter but never mentions the image's color).
|
||||
// The image-only case above proves the image reaches the model, and the
|
||||
// text-only path obviously works, so this is a multimodal-fusion quality
|
||||
// issue in the model, not a transport bug. Re-enable when upstream model
|
||||
// quality improves.
|
||||
it.skip("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithTextAndImageResult(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (CN) Provider (mimo-v2.5-pro)",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithImageResult(llm);
|
||||
});
|
||||
|
||||
// FIXME(xiaomi): see the API-billing block above — same multimodal-fusion
|
||||
// limitation applies to Token Plan endpoints (same model behind both).
|
||||
it.skip("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithTextAndImageResult(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (AMS) Provider (mimo-v2.5-pro)",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithImageResult(llm);
|
||||
});
|
||||
|
||||
// FIXME(xiaomi): see the API-billing block above — same multimodal-fusion
|
||||
// limitation applies to Token Plan endpoints (same model behind both).
|
||||
it.skip("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithTextAndImageResult(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (SGP) Provider (mimo-v2.5-pro)",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithImageResult(llm);
|
||||
});
|
||||
|
||||
// FIXME(xiaomi): see the API-billing block above — same multimodal-fusion
|
||||
// limitation applies to Token Plan endpoints (same model behind both).
|
||||
it.skip("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
|
||||
await handleToolWithTextAndImageResult(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider (kimi-for-coding)", () => {
|
||||
const llm = getModel("kimi-coding", "kimi-for-coding");
|
||||
|
||||
@@ -398,67 +466,6 @@ describe("Tool Results with Images", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider", () => {
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle tool result with only image",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await handleToolWithImageResult(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle tool result with text and image",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await handleToolWithTextAndImageResult(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider", () => {
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle tool result with only image",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await handleToolWithImageResult(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle tool result with text and image",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await handleToolWithTextAndImageResult(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
/** These two don't work, the model simply won't call the tool, works in pi
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle tool result with only image",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await handleToolWithImageResult(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle tool result with text and image",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await handleToolWithTextAndImageResult(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);**/
|
||||
|
||||
// Note: gpt-oss-120b-medium does not support images, so not tested here
|
||||
});
|
||||
|
||||
describe("OpenAI Codex Provider", () => {
|
||||
it.skipIf(!openaiCodexToken)(
|
||||
"gpt-5.2-codex - should handle tool result with only image",
|
||||
|
||||
@@ -63,4 +63,18 @@ describe("Mistral reasoning mode selection", () => {
|
||||
expect(payload.promptMode).toBe("reasoning");
|
||||
expect(payload.reasoningEffort).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses reasoning_effort for Mistral Medium 3.5", async () => {
|
||||
const payload = await capturePayload(getModel("mistral", "mistral-medium-3.5"), { reasoning: "medium" });
|
||||
|
||||
expect(payload.reasoningEffort).toBe("high");
|
||||
expect(payload.promptMode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("omits reasoning controls for Mistral Medium 3.5 when thinking is off", async () => {
|
||||
const payload = await capturePayload(getModel("mistral", "mistral-medium-3.5"));
|
||||
|
||||
expect(payload.reasoningEffort).toBeUndefined();
|
||||
expect(payload.promptMode).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,7 +53,6 @@ function saveAuthStorage(storage: AuthStorage): void {
|
||||
* For API key credentials, returns the key directly.
|
||||
* For OAuth credentials, returns the access token (refreshing if expired and saving back).
|
||||
*
|
||||
* For google-gemini-cli and google-antigravity, returns JSON-encoded { token, projectId }
|
||||
*/
|
||||
export async function resolveApiKey(provider: string): Promise<string | undefined> {
|
||||
const storage = loadAuthStorage();
|
||||
|
||||
@@ -3,21 +3,26 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
getOpenAICodexWebSocketDebugStats,
|
||||
resetOpenAICodexWebSocketDebugStats,
|
||||
streamOpenAICodexResponses,
|
||||
streamSimpleOpenAICodexResponses,
|
||||
} from "../src/providers/openai-codex-responses.js";
|
||||
import type { Context, Model } from "../src/types.js";
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
const originalWebSocket = globalThis.WebSocket;
|
||||
const originalAgentDir = process.env.PI_CODING_AGENT_DIR;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
globalThis.WebSocket = originalWebSocket;
|
||||
if (originalAgentDir === undefined) {
|
||||
delete process.env.PI_CODING_AGENT_DIR;
|
||||
} else {
|
||||
process.env.PI_CODING_AGENT_DIR = originalAgentDir;
|
||||
}
|
||||
resetOpenAICodexWebSocketDebugStats();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -237,7 +242,7 @@ describe("openai-codex streaming", () => {
|
||||
};
|
||||
|
||||
const result = await Promise.race([
|
||||
streamOpenAICodexResponses(model, context, { apiKey: token }).result(),
|
||||
streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }).result(),
|
||||
new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error("Timed out waiting for completed SSE stream")), 1000);
|
||||
}),
|
||||
@@ -296,7 +301,7 @@ describe("openai-codex streaming", () => {
|
||||
};
|
||||
|
||||
const result = await Promise.race([
|
||||
streamOpenAICodexResponses(model, context, { apiKey: token }).result(),
|
||||
streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }).result(),
|
||||
new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error("Timed out waiting for incomplete SSE stream")), 1000);
|
||||
}),
|
||||
@@ -446,6 +451,7 @@ describe("openai-codex streaming", () => {
|
||||
provider: "openai-codex",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { xhigh: "xhigh" },
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
@@ -746,4 +752,257 @@ describe("openai-codex streaming", () => {
|
||||
const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token });
|
||||
await streamResult.result();
|
||||
});
|
||||
it("forwards auto transport from streamSimple options and uses cached websocket context", async () => {
|
||||
const token = mockToken();
|
||||
const sentBodies: unknown[] = [];
|
||||
|
||||
global.fetch = vi.fn(async () => new Response("unexpected fetch", { status: 500 })) as typeof fetch;
|
||||
|
||||
class MockWebSocket {
|
||||
private listeners = new Map<string, Set<(event: unknown) => void>>();
|
||||
|
||||
constructor(_url: string, _protocols?: string | string[] | { headers?: Record<string, string> }) {
|
||||
queueMicrotask(() => this.dispatch("open", {}));
|
||||
}
|
||||
|
||||
addEventListener(type: string, listener: (event: unknown) => void): void {
|
||||
let listeners = this.listeners.get(type);
|
||||
if (!listeners) {
|
||||
listeners = new Set();
|
||||
this.listeners.set(type, listeners);
|
||||
}
|
||||
listeners.add(listener);
|
||||
}
|
||||
|
||||
removeEventListener(type: string, listener: (event: unknown) => void): void {
|
||||
this.listeners.get(type)?.delete(listener);
|
||||
}
|
||||
|
||||
send(data: string): void {
|
||||
sentBodies.push(JSON.parse(data));
|
||||
const events = [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] },
|
||||
},
|
||||
{ type: "response.content_part.added", part: { type: "output_text", text: "" } },
|
||||
{ type: "response.output_text.delta", delta: "Hello" },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "message",
|
||||
id: "msg_1",
|
||||
role: "assistant",
|
||||
status: "completed",
|
||||
content: [{ type: "output_text", text: "Hello" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
status: "completed",
|
||||
usage: {
|
||||
input_tokens: 5,
|
||||
output_tokens: 3,
|
||||
total_tokens: 8,
|
||||
input_tokens_details: { cached_tokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
queueMicrotask(() => {
|
||||
for (const event of events) {
|
||||
this.dispatch("message", { data: JSON.stringify(event) });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
close(): void {}
|
||||
|
||||
private dispatch(type: string, event: unknown): void {
|
||||
for (const listener of this.listeners.get(type) ?? []) {
|
||||
listener(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.WebSocket = MockWebSocket as unknown as typeof WebSocket;
|
||||
|
||||
const model: Model<"openai-codex-responses"> = {
|
||||
id: "gpt-5.1-codex",
|
||||
name: "GPT-5.1 Codex",
|
||||
api: "openai-codex-responses",
|
||||
provider: "openai-codex",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
};
|
||||
const context: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: 1 }],
|
||||
};
|
||||
|
||||
await streamSimpleOpenAICodexResponses(model, context, {
|
||||
apiKey: token,
|
||||
sessionId: "session-auto",
|
||||
transport: "auto",
|
||||
}).result();
|
||||
|
||||
expect(sentBodies).toHaveLength(1);
|
||||
expect(global.fetch).not.toHaveBeenCalled();
|
||||
expect(getOpenAICodexWebSocketDebugStats("session-auto")).toMatchObject({
|
||||
cachedContextRequests: 1,
|
||||
fullContextRequests: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("sends only response input deltas in websocket-cached mode", async () => {
|
||||
const token = mockToken();
|
||||
const sentBodies: unknown[] = [];
|
||||
const responses = [
|
||||
{ responseId: "resp_1", messageId: "msg_1", text: "Hello" },
|
||||
{ responseId: "resp_2", messageId: "msg_2", text: "Done" },
|
||||
];
|
||||
|
||||
class MockWebSocket {
|
||||
static OPEN = 1;
|
||||
readyState = MockWebSocket.OPEN;
|
||||
private listeners = new Map<string, Set<(event: unknown) => void>>();
|
||||
|
||||
constructor(_url: string, _protocols?: string | string[] | { headers?: Record<string, string> }) {
|
||||
queueMicrotask(() => this.dispatch("open", {}));
|
||||
}
|
||||
|
||||
addEventListener(type: string, listener: (event: unknown) => void): void {
|
||||
let listeners = this.listeners.get(type);
|
||||
if (!listeners) {
|
||||
listeners = new Set();
|
||||
this.listeners.set(type, listeners);
|
||||
}
|
||||
listeners.add(listener);
|
||||
}
|
||||
|
||||
removeEventListener(type: string, listener: (event: unknown) => void): void {
|
||||
this.listeners.get(type)?.delete(listener);
|
||||
}
|
||||
|
||||
send(data: string): void {
|
||||
sentBodies.push(JSON.parse(data));
|
||||
const response = responses.shift();
|
||||
if (!response) throw new Error("unexpected websocket request");
|
||||
const events = [
|
||||
{ type: "response.created", response: { id: response.responseId } },
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: {
|
||||
type: "message",
|
||||
id: response.messageId,
|
||||
role: "assistant",
|
||||
status: "in_progress",
|
||||
content: [],
|
||||
},
|
||||
},
|
||||
{ type: "response.content_part.added", part: { type: "output_text", text: "" } },
|
||||
{ type: "response.output_text.delta", delta: response.text },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "message",
|
||||
id: response.messageId,
|
||||
role: "assistant",
|
||||
status: "completed",
|
||||
content: [{ type: "output_text", text: response.text }],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: response.responseId,
|
||||
status: "completed",
|
||||
usage: {
|
||||
input_tokens: 5,
|
||||
output_tokens: 3,
|
||||
total_tokens: 8,
|
||||
input_tokens_details: { cached_tokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
queueMicrotask(() => {
|
||||
for (const event of events) {
|
||||
this.dispatch("message", { data: JSON.stringify(event) });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.readyState = 3;
|
||||
}
|
||||
|
||||
private dispatch(type: string, event: unknown): void {
|
||||
for (const listener of this.listeners.get(type) ?? []) {
|
||||
listener(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.WebSocket = MockWebSocket as unknown as typeof WebSocket;
|
||||
|
||||
const model: Model<"openai-codex-responses"> = {
|
||||
id: "gpt-5.1-codex",
|
||||
name: "GPT-5.1 Codex",
|
||||
api: "openai-codex-responses",
|
||||
provider: "openai-codex",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
};
|
||||
const firstContext: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [{ role: "user", content: "Say hello", timestamp: 1 }],
|
||||
};
|
||||
|
||||
const first = await streamOpenAICodexResponses(model, firstContext, {
|
||||
apiKey: token,
|
||||
sessionId: "session-1",
|
||||
transport: "websocket-cached",
|
||||
}).result();
|
||||
|
||||
const secondContext: Context = {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
messages: [...firstContext.messages, first, { role: "user", content: "Now finish", timestamp: 2 }],
|
||||
};
|
||||
await streamOpenAICodexResponses(model, secondContext, {
|
||||
apiKey: token,
|
||||
sessionId: "session-1",
|
||||
transport: "websocket-cached",
|
||||
}).result();
|
||||
|
||||
expect(sentBodies).toHaveLength(2);
|
||||
const firstBody = sentBodies[0] as { input: unknown[]; previous_response_id?: string; store?: boolean };
|
||||
const secondBody = sentBodies[1] as { input: unknown[]; previous_response_id?: string; store?: boolean };
|
||||
expect(firstBody.store).toBe(false);
|
||||
expect(firstBody.previous_response_id).toBeUndefined();
|
||||
expect(firstBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Say hello" }] }]);
|
||||
expect(secondBody.store).toBe(false);
|
||||
expect(secondBody.previous_response_id).toBe("resp_1");
|
||||
expect(secondBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Now finish" }] }]);
|
||||
expect(getOpenAICodexWebSocketDebugStats("session-1")).toMatchObject({
|
||||
requests: 2,
|
||||
connectionsCreated: 1,
|
||||
connectionsReused: 1,
|
||||
cachedContextRequests: 2,
|
||||
storeTrueRequests: 0,
|
||||
fullContextRequests: 1,
|
||||
deltaRequests: 1,
|
||||
lastDeltaInputItems: 1,
|
||||
lastPreviousResponseId: "resp_1",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,10 +9,15 @@ import { streamSimple } from "../src/stream.js";
|
||||
|
||||
const mockState = vi.hoisted(() => ({
|
||||
lastParams: undefined as unknown,
|
||||
lastClientOptions: undefined as unknown,
|
||||
}));
|
||||
|
||||
vi.mock("openai", () => {
|
||||
class FakeOpenAI {
|
||||
constructor(options: unknown) {
|
||||
mockState.lastClientOptions = options;
|
||||
}
|
||||
|
||||
chat = {
|
||||
completions: {
|
||||
create: (params: unknown) => {
|
||||
@@ -52,6 +57,7 @@ vi.mock("openai", () => {
|
||||
describe("openai-completions empty tools handling", () => {
|
||||
beforeEach(() => {
|
||||
mockState.lastParams = undefined;
|
||||
mockState.lastClientOptions = undefined;
|
||||
});
|
||||
|
||||
it("omits tools field when context.tools is an empty array", async () => {
|
||||
@@ -87,6 +93,79 @@ describe("openai-completions empty tools handling", () => {
|
||||
expect("tools" in (params as object)).toBe(false);
|
||||
});
|
||||
|
||||
it("uses conservative OpenAI-compatible fields for Cloudflare AI Gateway /compat models", async () => {
|
||||
process.env.CLOUDFLARE_ACCOUNT_ID = "account-id";
|
||||
process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id";
|
||||
const model = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6")!;
|
||||
|
||||
await streamSimple(
|
||||
model,
|
||||
{
|
||||
systemPrompt: "You are helpful.",
|
||||
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
|
||||
},
|
||||
{ apiKey: "test", reasoning: "high" },
|
||||
).result();
|
||||
|
||||
const params = mockState.lastParams as {
|
||||
messages: Array<{ role: string }>;
|
||||
max_tokens?: number;
|
||||
max_completion_tokens?: number;
|
||||
reasoning_effort?: string;
|
||||
store?: boolean;
|
||||
};
|
||||
expect(params.messages[0].role).toBe("system");
|
||||
expect(params.max_tokens).toBeDefined();
|
||||
expect(params.max_completion_tokens).toBeUndefined();
|
||||
expect(params.reasoning_effort).toBeUndefined();
|
||||
expect(params.store).toBeUndefined();
|
||||
|
||||
const clientOptions = mockState.lastClientOptions as {
|
||||
baseURL?: string;
|
||||
defaultHeaders?: Record<string, unknown>;
|
||||
};
|
||||
expect(clientOptions.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/compat");
|
||||
expect(clientOptions.defaultHeaders?.Authorization).toBeNull();
|
||||
expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer test");
|
||||
});
|
||||
|
||||
it("preserves inline upstream Authorization for Cloudflare AI Gateway BYOK requests", async () => {
|
||||
process.env.CLOUDFLARE_ACCOUNT_ID = "account-id";
|
||||
process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id";
|
||||
const model = getModel("cloudflare-ai-gateway", "gpt-5.1")!;
|
||||
|
||||
await streamSimple(
|
||||
model,
|
||||
{
|
||||
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
|
||||
},
|
||||
{ apiKey: "cf-token", headers: { Authorization: "Bearer upstream-token" } },
|
||||
).result();
|
||||
|
||||
const clientOptions = mockState.lastClientOptions as { defaultHeaders?: Record<string, unknown> };
|
||||
expect(clientOptions.defaultHeaders?.Authorization).toBe("Bearer upstream-token");
|
||||
expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer cf-token");
|
||||
});
|
||||
|
||||
it("sends session affinity headers for Workers AI through Cloudflare AI Gateway", async () => {
|
||||
process.env.CLOUDFLARE_ACCOUNT_ID = "account-id";
|
||||
process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id";
|
||||
const workersModel = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6")!;
|
||||
|
||||
await streamSimple(
|
||||
workersModel,
|
||||
{
|
||||
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
|
||||
},
|
||||
{ apiKey: "test", sessionId: "session-1" },
|
||||
).result();
|
||||
|
||||
const clientOptions = mockState.lastClientOptions as { defaultHeaders?: Record<string, string> };
|
||||
expect(clientOptions.defaultHeaders?.session_id).toBe("session-1");
|
||||
expect(clientOptions.defaultHeaders?.["x-client-request-id"]).toBe("session-1");
|
||||
expect(clientOptions.defaultHeaders?.["x-session-affinity"]).toBe("session-1");
|
||||
});
|
||||
|
||||
it("still emits tools: [] for Anthropic/LiteLLM proxy when conversation has tool history", async () => {
|
||||
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
|
||||
const model = { ...baseModel, api: "openai-completions" } as const;
|
||||
|
||||
140
packages/ai/test/openai-completions-response-model.test.ts
Normal file
140
packages/ai/test/openai-completions-response-model.test.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { complete } from "../src/stream.js";
|
||||
import type { Model } from "../src/types.js";
|
||||
|
||||
// Router/virtual ids (e.g. OpenRouter `auto`) keep `model` pinned to the
|
||||
// requested id and surface the routed concrete id on `responseModel`.
|
||||
|
||||
const mockState = vi.hoisted(() => ({
|
||||
chunks: [] as unknown[],
|
||||
}));
|
||||
|
||||
vi.mock("openai", () => {
|
||||
class FakeOpenAI {
|
||||
chat = {
|
||||
completions: {
|
||||
create: () => {
|
||||
const chunks = mockState.chunks;
|
||||
const stream = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of chunks) yield chunk;
|
||||
},
|
||||
};
|
||||
const promise = Promise.resolve(stream) as Promise<typeof stream> & {
|
||||
withResponse: () => Promise<{
|
||||
data: typeof stream;
|
||||
response: { status: number; headers: Headers };
|
||||
}>;
|
||||
};
|
||||
promise.withResponse = async () => ({
|
||||
data: stream,
|
||||
response: { status: 200, headers: new Headers() },
|
||||
});
|
||||
return promise;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return { default: FakeOpenAI };
|
||||
});
|
||||
|
||||
function openRouterAuto(): Model<"openai-completions"> {
|
||||
return {
|
||||
id: "openrouter/auto",
|
||||
name: "OpenRouter Auto",
|
||||
api: "openai-completions",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
}
|
||||
|
||||
describe("openai-completions responseModel", () => {
|
||||
beforeEach(() => {
|
||||
mockState.chunks = [];
|
||||
});
|
||||
|
||||
it("surfaces routed chunk.model on responseModel without changing model", async () => {
|
||||
mockState.chunks = [
|
||||
{ id: "chatcmpl-1", model: "anthropic/claude-opus-4.7", choices: [{ index: 0, delta: { content: "hi" } }] },
|
||||
{
|
||||
id: "chatcmpl-1",
|
||||
model: "anthropic/claude-opus-4.7",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
prompt_tokens_details: { cached_tokens: 0 },
|
||||
completion_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const message = await complete(
|
||||
openRouterAuto(),
|
||||
{ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] },
|
||||
{ apiKey: "test" },
|
||||
);
|
||||
|
||||
expect(message.model).toBe("openrouter/auto");
|
||||
expect(message.responseModel).toBe("anthropic/claude-opus-4.7");
|
||||
expect(message.provider).toBe("openrouter");
|
||||
expect(message.stopReason).toBe("stop");
|
||||
});
|
||||
|
||||
it("leaves responseModel undefined when chunks echo the requested id", async () => {
|
||||
mockState.chunks = [
|
||||
{ id: "chatcmpl-2", model: "openrouter/auto", choices: [{ index: 0, delta: { content: "hi" } }] },
|
||||
{
|
||||
id: "chatcmpl-2",
|
||||
model: "openrouter/auto",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: {
|
||||
prompt_tokens: 1,
|
||||
completion_tokens: 1,
|
||||
prompt_tokens_details: { cached_tokens: 0 },
|
||||
completion_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const message = await complete(
|
||||
openRouterAuto(),
|
||||
{ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] },
|
||||
{ apiKey: "test" },
|
||||
);
|
||||
|
||||
expect(message.model).toBe("openrouter/auto");
|
||||
expect(message.responseModel).toBeUndefined();
|
||||
});
|
||||
|
||||
it("ignores empty or missing chunk.model", async () => {
|
||||
mockState.chunks = [
|
||||
{ id: "chatcmpl-3", choices: [{ index: 0, delta: { content: "hi" } }] },
|
||||
{ id: "chatcmpl-3", model: "", choices: [{ index: 0, delta: { content: "!" } }] },
|
||||
{
|
||||
id: "chatcmpl-3",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: {
|
||||
prompt_tokens: 1,
|
||||
completion_tokens: 2,
|
||||
prompt_tokens_details: { cached_tokens: 0 },
|
||||
completion_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const message = await complete(
|
||||
openRouterAuto(),
|
||||
{ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] },
|
||||
{ apiKey: "test" },
|
||||
);
|
||||
|
||||
expect(message.model).toBe("openrouter/auto");
|
||||
expect(message.responseModel).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -25,7 +25,6 @@ const compat = {
|
||||
supportsStore: true,
|
||||
supportsDeveloperRole: true,
|
||||
supportsReasoningEffort: true,
|
||||
reasoningEffortMap: {},
|
||||
supportsUsageInStreaming: true,
|
||||
maxTokensField: "max_completion_tokens",
|
||||
requiresToolResultName: false,
|
||||
|
||||
@@ -23,7 +23,6 @@ const compat: Required<OpenAICompletionsCompat> = {
|
||||
supportsStore: true,
|
||||
supportsDeveloperRole: true,
|
||||
supportsReasoningEffort: true,
|
||||
reasoningEffortMap: {},
|
||||
supportsUsageInStreaming: true,
|
||||
maxTokensField: "max_completion_tokens",
|
||||
requiresToolResultName: false,
|
||||
|
||||
@@ -61,4 +61,39 @@ describe("isContextOverflow", () => {
|
||||
const message = createErrorMessage("Too many requests. Please slow down.");
|
||||
expect(isContextOverflow(message, 200000)).toBe(false);
|
||||
});
|
||||
|
||||
function createLengthStopMessage(input: number, cacheRead: number, output: number): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "openai-completions",
|
||||
provider: "xiaomi",
|
||||
model: "mimo-v2.5-pro",
|
||||
usage: {
|
||||
input,
|
||||
output,
|
||||
cacheRead,
|
||||
cacheWrite: 0,
|
||||
totalTokens: input + cacheRead + output,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "length",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
it("detects Xiaomi-style overflow (length stop with zero output and filled context)", () => {
|
||||
const message = createLengthStopMessage(58, 1048512, 0);
|
||||
expect(isContextOverflow(message, 1048576)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat normal length stops with output as overflow", () => {
|
||||
const message = createLengthStopMessage(1000, 0, 4096);
|
||||
expect(isContextOverflow(message, 200000)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not treat length stops far below context as overflow", () => {
|
||||
const message = createLengthStopMessage(100, 0, 0);
|
||||
expect(isContextOverflow(message, 200000)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,13 +7,8 @@ import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
|
||||
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const oauthTokens = await Promise.all([resolveApiKey("github-copilot"), resolveApiKey("openai-codex")]);
|
||||
const [githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
async function expectResponseId<TApi extends Api>(model: Model<TApi>, options: StreamOptionsWithExtras = {}) {
|
||||
const context: Context = {
|
||||
@@ -117,25 +112,6 @@ describe("responseId E2E Tests", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider", () => {
|
||||
it.skipIf(!geminiCliToken)("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await expectResponseId(llm, { apiKey: geminiCliToken });
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider", () => {
|
||||
it.skipIf(!antigravityToken)("Gemini path should expose responseId", { retry: 3, timeout: 30000 }, async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3.1-pro-high");
|
||||
await expectResponseId(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("Claude path should expose responseId", { retry: 3, timeout: 30000 }, async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-6");
|
||||
await expectResponseId(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
});
|
||||
|
||||
describe("OpenAI Codex Provider", () => {
|
||||
it.skipIf(!openaiCodexToken)("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
|
||||
const llm = getModel("openai-codex", "gpt-5.2-codex");
|
||||
|
||||
@@ -13,6 +13,7 @@ type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
|
||||
import { StringEnum } from "../src/utils/typebox-helpers.js";
|
||||
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
|
||||
import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
@@ -22,11 +23,9 @@ const __dirname = dirname(__filename);
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("anthropic"),
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
// Calculator tool definition (same as examples)
|
||||
// Note: Using StringEnum helper because Google's API doesn't support anyOf/const patterns
|
||||
@@ -616,7 +615,7 @@ describe("Generate E2E Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.CLOUDFLARE_API_KEY || !process.env.CLOUDFLARE_ACCOUNT_ID)(
|
||||
describe.skipIf(!hasCloudflareWorkersAICredentials())(
|
||||
"Cloudflare Workers AI Provider (Kimi K2.6 via OpenAI Completions)",
|
||||
() => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
@@ -643,6 +642,99 @@ describe("Generate E2E Tests", () => {
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials())(
|
||||
"Cloudflare AI Gateway → Workers AI (Kimi K2.6 via /compat)",
|
||||
() => {
|
||||
const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm);
|
||||
});
|
||||
|
||||
it("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm);
|
||||
});
|
||||
|
||||
it("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm);
|
||||
});
|
||||
|
||||
it("should handle thinking mode", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, { reasoningEffort: "medium" });
|
||||
});
|
||||
|
||||
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, { reasoningEffort: "medium" });
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials() || !process.env.OPENAI_API_KEY)(
|
||||
"Cloudflare AI Gateway → OpenAI BYOK (gpt-5.1 via /openai responses)",
|
||||
() => {
|
||||
const llm = getModel("cloudflare-ai-gateway", "gpt-5.1");
|
||||
const options = { headers: { Authorization: `Bearer ${process.env.OPENAI_API_KEY}` } };
|
||||
const thinkingOptions = {
|
||||
...options,
|
||||
thinkingEnabled: true,
|
||||
reasoningEffort: "medium",
|
||||
} satisfies StreamOptionsWithExtras;
|
||||
|
||||
it("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm, options);
|
||||
});
|
||||
|
||||
it("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm, options);
|
||||
});
|
||||
|
||||
it("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm, options);
|
||||
});
|
||||
|
||||
it("should handle thinking mode", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, thinkingOptions);
|
||||
});
|
||||
|
||||
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, thinkingOptions);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials() || !process.env.ANTHROPIC_API_KEY)(
|
||||
"Cloudflare AI Gateway → Anthropic BYOK (claude-sonnet-4-5 via /anthropic messages)",
|
||||
() => {
|
||||
const llm = getModel("cloudflare-ai-gateway", "claude-sonnet-4-5");
|
||||
const options = { headers: { Authorization: `Bearer ${process.env.ANTHROPIC_API_KEY}` } };
|
||||
const thinkingOptions = {
|
||||
...options,
|
||||
thinkingEnabled: true,
|
||||
reasoningEffort: "high",
|
||||
} satisfies StreamOptionsWithExtras;
|
||||
|
||||
it("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm, options);
|
||||
});
|
||||
|
||||
it("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm, options);
|
||||
});
|
||||
|
||||
it("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm, options);
|
||||
});
|
||||
|
||||
it("should handle thinking mode", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, thinkingOptions);
|
||||
});
|
||||
|
||||
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, thinkingOptions);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider (Kimi-K2.5 via OpenAI Completions)", () => {
|
||||
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
|
||||
|
||||
@@ -901,6 +993,130 @@ describe("Generate E2E Tests", () => {
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)(
|
||||
"Xiaomi MiMo (API billing) Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages)",
|
||||
() => {
|
||||
const llm = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
const thinkingOptions = {
|
||||
thinkingEnabled: true,
|
||||
reasoningEffort: "high",
|
||||
} satisfies StreamOptionsWithExtras;
|
||||
|
||||
it("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm);
|
||||
});
|
||||
|
||||
it("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm);
|
||||
});
|
||||
|
||||
it("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm);
|
||||
});
|
||||
|
||||
it("should handle thinking mode", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, thinkingOptions);
|
||||
});
|
||||
|
||||
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, thinkingOptions);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages, CN region)",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
const thinkingOptions = {
|
||||
thinkingEnabled: true,
|
||||
reasoningEffort: "high",
|
||||
} satisfies StreamOptionsWithExtras;
|
||||
|
||||
it("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm);
|
||||
});
|
||||
|
||||
it("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm);
|
||||
});
|
||||
|
||||
it("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm);
|
||||
});
|
||||
|
||||
it("should handle thinking mode", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, thinkingOptions);
|
||||
});
|
||||
|
||||
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, thinkingOptions);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages, AMS region)",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
const thinkingOptions = {
|
||||
thinkingEnabled: true,
|
||||
reasoningEffort: "high",
|
||||
} satisfies StreamOptionsWithExtras;
|
||||
|
||||
it("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm);
|
||||
});
|
||||
|
||||
it("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm);
|
||||
});
|
||||
|
||||
it("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm);
|
||||
});
|
||||
|
||||
it("should handle thinking mode", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, thinkingOptions);
|
||||
});
|
||||
|
||||
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, thinkingOptions);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages, SGP region)",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
const thinkingOptions = {
|
||||
thinkingEnabled: true,
|
||||
reasoningEffort: "high",
|
||||
} satisfies StreamOptionsWithExtras;
|
||||
|
||||
it("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm);
|
||||
});
|
||||
|
||||
it("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm);
|
||||
});
|
||||
|
||||
it("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm);
|
||||
});
|
||||
|
||||
it("should handle thinking mode", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, thinkingOptions);
|
||||
});
|
||||
|
||||
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, thinkingOptions);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
// =========================================================================
|
||||
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
|
||||
// Tokens are resolved at module level (see oauthTokens above)
|
||||
@@ -1028,124 +1244,6 @@ describe("Generate E2E Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider (gemini-2.5-flash)", () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
|
||||
it.skipIf(!geminiCliToken)("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm, { apiKey: geminiCliToken });
|
||||
});
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm, { apiKey: geminiCliToken });
|
||||
});
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm, { apiKey: geminiCliToken });
|
||||
});
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle thinking", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, { apiKey: geminiCliToken, thinking: { enabled: true, budgetTokens: 1024 } });
|
||||
});
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, { apiKey: geminiCliToken, thinking: { enabled: true, budgetTokens: 2048 } });
|
||||
});
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle image input", { retry: 3 }, async () => {
|
||||
await handleImage(llm, { apiKey: geminiCliToken });
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider (gemini-3-flash-preview with thinkingLevel)", () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-3-flash-preview");
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle thinking with thinkingLevel", { retry: 3 }, async () => {
|
||||
await handleThinking(llm, { apiKey: geminiCliToken, thinking: { enabled: true, level: "LOW" } });
|
||||
});
|
||||
|
||||
it.skipIf(!geminiCliToken)("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, { apiKey: geminiCliToken, thinking: { enabled: true, level: "MEDIUM" } });
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider (gemini-3.1-pro-high)", () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3.1-pro-high");
|
||||
|
||||
it.skipIf(!antigravityToken)("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle thinking with thinkingLevel", { retry: 3 }, async () => {
|
||||
// gemini-3-pro only supports LOW/HIGH
|
||||
await handleThinking(llm, {
|
||||
apiKey: antigravityToken,
|
||||
thinking: { enabled: true, level: "LOW" },
|
||||
});
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
await multiTurn(llm, { apiKey: antigravityToken, thinking: { enabled: true, level: "HIGH" } });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle image input", { retry: 3 }, async () => {
|
||||
await handleImage(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider (gemini-3.1-pro-high with thinkingLevel)", () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3.1-pro-high");
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle thinking with thinkingLevel HIGH", { retry: 3 }, async () => {
|
||||
// gemini-3-pro only supports LOW/HIGH
|
||||
await handleThinking(llm, {
|
||||
apiKey: antigravityToken,
|
||||
thinking: { enabled: true, level: "HIGH" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider (claude-sonnet-4-5)", () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
|
||||
it.skipIf(!antigravityToken)("should complete basic text generation", { retry: 3 }, async () => {
|
||||
await basicTextGeneration(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle tool calling", { retry: 3 }, async () => {
|
||||
await handleToolCall(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle streaming", { retry: 3 }, async () => {
|
||||
await handleStreaming(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle thinking", { retry: 3 }, async () => {
|
||||
// claude-sonnet-4-5 has reasoning: false, use claude-sonnet-4-5-thinking
|
||||
const thinkingModel = getModel("google-antigravity", "claude-sonnet-4-5-thinking");
|
||||
await handleThinking(thinkingModel, {
|
||||
apiKey: antigravityToken,
|
||||
thinking: { enabled: true, budgetTokens: 4096 },
|
||||
});
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
|
||||
const thinkingModel = getModel("google-antigravity", "claude-sonnet-4-5-thinking");
|
||||
await multiTurn(thinkingModel, { apiKey: antigravityToken, thinking: { enabled: true, budgetTokens: 4096 } });
|
||||
});
|
||||
|
||||
it.skipIf(!antigravityToken)("should handle image input", { retry: 3 }, async () => {
|
||||
await handleImage(llm, { apiKey: antigravityToken });
|
||||
});
|
||||
});
|
||||
|
||||
describe("OpenAI Codex Provider (gpt-5.4)", () => {
|
||||
const llm = getModel("openai-codex", "gpt-5.4");
|
||||
|
||||
|
||||
@@ -1,34 +1,52 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getModel, supportsXhigh } from "../src/models.js";
|
||||
import { getModel, getSupportedThinkingLevels } from "../src/models.js";
|
||||
|
||||
describe("supportsXhigh", () => {
|
||||
it("returns true for Anthropic Opus 4.6 on anthropic-messages API", () => {
|
||||
describe("getSupportedThinkingLevels", () => {
|
||||
it("includes xhigh for Anthropic Opus 4.6 on anthropic-messages API", () => {
|
||||
const model = getModel("anthropic", "claude-opus-4-6");
|
||||
expect(model).toBeDefined();
|
||||
expect(supportsXhigh(model!)).toBe(true);
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("xhigh");
|
||||
});
|
||||
|
||||
it("returns true for Anthropic Opus 4.7 on anthropic-messages API", () => {
|
||||
it("includes xhigh for Anthropic Opus 4.7 on anthropic-messages API", () => {
|
||||
const model = getModel("anthropic", "claude-opus-4-7");
|
||||
expect(model).toBeDefined();
|
||||
expect(supportsXhigh(model!)).toBe(true);
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("xhigh");
|
||||
});
|
||||
|
||||
it("returns false for non-Opus Anthropic models", () => {
|
||||
it("does not include xhigh for non-Opus Anthropic models", () => {
|
||||
const model = getModel("anthropic", "claude-sonnet-4-5");
|
||||
expect(model).toBeDefined();
|
||||
expect(supportsXhigh(model!)).toBe(false);
|
||||
expect(getSupportedThinkingLevels(model!)).not.toContain("xhigh");
|
||||
});
|
||||
|
||||
it.each(["gpt-5.4", "gpt-5.5"] as const)("returns true for %s models", (modelId) => {
|
||||
it.each(["gpt-5.4", "gpt-5.5"] as const)("includes xhigh for %s models", (modelId) => {
|
||||
const model = getModel("openai-codex", modelId);
|
||||
expect(model).toBeDefined();
|
||||
expect(supportsXhigh(model!)).toBe(true);
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("xhigh");
|
||||
});
|
||||
|
||||
it("returns true for OpenRouter Opus 4.6 (openai-completions API)", () => {
|
||||
it("includes only high/xhigh plus off for DeepSeek V4 Flash on the DeepSeek provider", () => {
|
||||
const model = getModel("deepseek", "deepseek-v4-flash");
|
||||
expect(model).toBeDefined();
|
||||
expect(getSupportedThinkingLevels(model!)).toEqual(["off", "high", "xhigh"]);
|
||||
});
|
||||
|
||||
it("includes only high/xhigh plus off for DeepSeek V4 Flash on opencode-go", () => {
|
||||
const model = getModel("opencode-go", "deepseek-v4-flash");
|
||||
expect(model).toBeDefined();
|
||||
expect(getSupportedThinkingLevels(model!)).toEqual(["off", "high", "xhigh"]);
|
||||
});
|
||||
|
||||
it("includes only high/xhigh plus off for DeepSeek V4 Flash on OpenRouter", () => {
|
||||
const model = getModel("openrouter", "deepseek/deepseek-v4-flash");
|
||||
expect(model).toBeDefined();
|
||||
expect(getSupportedThinkingLevels(model!)).toEqual(["off", "high", "xhigh"]);
|
||||
});
|
||||
|
||||
it("includes xhigh for OpenRouter Opus 4.6 (openai-completions API)", () => {
|
||||
const model = getModel("openrouter", "anthropic/claude-opus-4.6");
|
||||
expect(model).toBeDefined();
|
||||
expect(supportsXhigh(model!)).toBe(true);
|
||||
expect(getSupportedThinkingLevels(model!)).toContain("xhigh");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,17 +7,16 @@ type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
|
||||
|
||||
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
|
||||
import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Resolve OAuth tokens at module level (async, runs before tests)
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("anthropic"),
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
async function testTokensOnAbort<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
|
||||
const context: Context = {
|
||||
@@ -50,7 +49,7 @@ async function testTokensOnAbort<TApi extends Api>(llm: Model<TApi>, options: St
|
||||
|
||||
expect(msg.stopReason).toBe("aborted");
|
||||
|
||||
// OpenAI providers, OpenAI Codex, Gemini CLI, zai, Amazon Bedrock, and the GPT-OSS model on Antigravity only send usage in the final chunk,
|
||||
// OpenAI providers, OpenAI Codex, zai, and Amazon Bedrock only send usage in the final chunk,
|
||||
// so when aborted they have no token stats. Anthropic and Google send usage information early in the stream.
|
||||
// MiniMax and Kimi report input tokens but not output tokens differently on aborted requests.
|
||||
if (
|
||||
@@ -59,11 +58,9 @@ async function testTokensOnAbort<TApi extends Api>(llm: Model<TApi>, options: St
|
||||
llm.api === "openai-responses" ||
|
||||
llm.api === "azure-openai-responses" ||
|
||||
llm.api === "openai-codex-responses" ||
|
||||
llm.provider === "google-gemini-cli" ||
|
||||
llm.provider === "zai" ||
|
||||
llm.provider === "amazon-bedrock" ||
|
||||
llm.provider === "vercel-ai-gateway" ||
|
||||
(llm.provider === "google-antigravity" && llm.id.includes("gpt-oss"))
|
||||
llm.provider === "vercel-ai-gateway"
|
||||
) {
|
||||
expect(msg.usage.input).toBe(0);
|
||||
expect(msg.usage.output).toBe(0);
|
||||
@@ -79,7 +76,7 @@ async function testTokensOnAbort<TApi extends Api>(llm: Model<TApi>, options: St
|
||||
expect(msg.usage.input).toBeGreaterThan(0);
|
||||
expect(msg.usage.output).toBeGreaterThan(0);
|
||||
|
||||
// Some providers (Antigravity, Copilot) have zero cost rates
|
||||
// Some providers (Copilot) have zero cost rates
|
||||
if (llm.cost.input > 0) {
|
||||
expect(msg.usage.cost.input).toBeGreaterThan(0);
|
||||
expect(msg.usage.cost.total).toBeGreaterThan(0);
|
||||
@@ -159,16 +156,21 @@ describe("Token Statistics on Abort", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.CLOUDFLARE_API_KEY || !process.env.CLOUDFLARE_ACCOUNT_ID)(
|
||||
"Cloudflare Workers AI Provider",
|
||||
() => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI Provider", () => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testTokensOnAbort(llm);
|
||||
});
|
||||
},
|
||||
);
|
||||
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testTokensOnAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider", () => {
|
||||
const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testTokensOnAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider", () => {
|
||||
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
|
||||
@@ -218,6 +220,49 @@ describe("Token Statistics on Abort", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider", () => {
|
||||
const llm = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
|
||||
// FIXME(xiaomi): Xiaomi's Anthropic-compatible stream does not populate
|
||||
// usage in the message_start event the way Anthropic does — usage only
|
||||
// arrives at message_stop. Aborting mid-stream therefore loses input/output
|
||||
// token counts. Non-streaming usage works (see total-tokens.test.ts).
|
||||
// Re-enable once upstream sends usage in message_start.
|
||||
it.skip("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testTokensOnAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN) Provider", () => {
|
||||
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
|
||||
// FIXME(xiaomi): see the API-billing block above — same upstream streaming
|
||||
// usage limitation applies to Token Plan endpoints.
|
||||
it.skip("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testTokensOnAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS) Provider", () => {
|
||||
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
|
||||
// FIXME(xiaomi): see the API-billing block above — same upstream streaming
|
||||
// usage limitation applies to Token Plan endpoints.
|
||||
it.skip("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testTokensOnAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP) Provider", () => {
|
||||
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
|
||||
// FIXME(xiaomi): see the API-billing block above — same upstream streaming
|
||||
// usage limitation applies to Token Plan endpoints.
|
||||
it.skip("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testTokensOnAbort(llm);
|
||||
});
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
|
||||
// =========================================================================
|
||||
@@ -254,46 +299,6 @@ describe("Token Statistics on Abort", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider", () => {
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should include token stats when aborted mid-stream",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testTokensOnAbort(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider", () => {
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should include token stats when aborted mid-stream",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testTokensOnAbort(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-6 - should include token stats when aborted mid-stream",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-6");
|
||||
await testTokensOnAbort(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should include token stats when aborted mid-stream",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testTokensOnAbort(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("OpenAI Codex Provider", () => {
|
||||
it.skipIf(!openaiCodexToken)(
|
||||
"gpt-5.2-codex - should include token stats when aborted mid-stream",
|
||||
|
||||
@@ -8,17 +8,16 @@ type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
|
||||
|
||||
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
|
||||
import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Resolve OAuth tokens at module level (async, runs before tests)
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("anthropic"),
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
// Simple calculate tool
|
||||
const calculateSchema = Type.Object({
|
||||
@@ -168,20 +167,21 @@ describe("Tool Call Without Result Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.CLOUDFLARE_API_KEY || !process.env.CLOUDFLARE_ACCOUNT_ID)(
|
||||
"Cloudflare Workers AI Provider",
|
||||
() => {
|
||||
const model = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI Provider", () => {
|
||||
const model = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it(
|
||||
"should filter out tool calls without corresponding tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
await testToolCallWithoutResult(model);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testToolCallWithoutResult(model);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider", () => {
|
||||
const model = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testToolCallWithoutResult(model);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider", () => {
|
||||
const model = getModel("huggingface", "moonshotai/Kimi-K2.5");
|
||||
@@ -215,6 +215,38 @@ describe("Tool Call Without Result Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider", () => {
|
||||
const model = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
|
||||
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testToolCallWithoutResult(model);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN) Provider", () => {
|
||||
const model = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
|
||||
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testToolCallWithoutResult(model);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS) Provider", () => {
|
||||
const model = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
|
||||
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testToolCallWithoutResult(model);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP) Provider", () => {
|
||||
const model = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
|
||||
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testToolCallWithoutResult(model);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider", () => {
|
||||
const model = getModel("kimi-coding", "kimi-k2-thinking");
|
||||
|
||||
@@ -275,46 +307,6 @@ describe("Tool Call Without Result Tests", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider", () => {
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should filter out tool calls without corresponding tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const model = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testToolCallWithoutResult(model, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider", () => {
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should filter out tool calls without corresponding tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const model = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testToolCallWithoutResult(model, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should filter out tool calls without corresponding tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const model = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testToolCallWithoutResult(model, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should filter out tool calls without corresponding tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const model = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testToolCallWithoutResult(model, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("OpenAI Codex Provider", () => {
|
||||
it.skipIf(!openaiCodexToken)(
|
||||
"gpt-5.2-codex - should filter out tool calls without corresponding tool results",
|
||||
|
||||
@@ -21,17 +21,16 @@ type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
|
||||
|
||||
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
|
||||
import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Resolve OAuth tokens at module level (async, runs before tests)
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("anthropic"),
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
// Generate a long system prompt to trigger caching (>2k bytes for most providers)
|
||||
const LONG_SYSTEM_PROMPT = `You are a helpful assistant. Be concise in your responses.
|
||||
@@ -310,29 +309,51 @@ describe("totalTokens field", () => {
|
||||
// Cloudflare Workers AI
|
||||
// =========================================================================
|
||||
|
||||
describe.skipIf(!process.env.CLOUDFLARE_API_KEY || !process.env.CLOUDFLARE_ACCOUNT_ID)(
|
||||
"Cloudflare Workers AI",
|
||||
() => {
|
||||
it(
|
||||
"@cf/moonshotai/kimi-k2.6 - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI", () => {
|
||||
it(
|
||||
"@cf/moonshotai/kimi-k2.6 - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
console.log(`\nCloudflare Workers AI / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, {
|
||||
apiKey: process.env.CLOUDFLARE_API_KEY,
|
||||
});
|
||||
console.log(`\nCloudflare Workers AI / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, {
|
||||
apiKey: process.env.CLOUDFLARE_API_KEY,
|
||||
});
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Cloudflare AI Gateway
|
||||
// =========================================================================
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway", () => {
|
||||
it(
|
||||
"workers-ai/@cf/moonshotai/kimi-k2.6 - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
console.log(`\nCloudflare AI Gateway / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, {
|
||||
apiKey: process.env.CLOUDFLARE_API_KEY,
|
||||
});
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Hugging Face
|
||||
@@ -422,6 +443,104 @@ describe("totalTokens field", () => {
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Xiaomi MiMo
|
||||
// =========================================================================
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing)", () => {
|
||||
it(
|
||||
"mimo-v2.5-pro - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
|
||||
console.log(`\nXiaomi MiMo / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.XIAOMI_API_KEY });
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Xiaomi MiMo Token Plan CN
|
||||
// =========================================================================
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN)", () => {
|
||||
it(
|
||||
"mimo-v2.5-pro - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
|
||||
console.log(`\nXiaomi MiMo Token Plan CN / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, {
|
||||
apiKey: process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY,
|
||||
});
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Xiaomi MiMo Token Plan AMS
|
||||
// =========================================================================
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS)", () => {
|
||||
it(
|
||||
"mimo-v2.5-pro - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
|
||||
console.log(`\nXiaomi MiMo Token Plan AMS / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, {
|
||||
apiKey: process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY,
|
||||
});
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Xiaomi MiMo Token Plan SGP
|
||||
// =========================================================================
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP)", () => {
|
||||
it(
|
||||
"mimo-v2.5-pro - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
|
||||
console.log(`\nXiaomi MiMo Token Plan SGP / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, {
|
||||
apiKey: process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY,
|
||||
});
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Kimi For Coding
|
||||
// =========================================================================
|
||||
@@ -600,85 +719,11 @@ describe("totalTokens field", () => {
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Google Gemini CLI (OAuth)
|
||||
// =========================================================================
|
||||
|
||||
describe("Google Gemini CLI (OAuth)", () => {
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
|
||||
console.log(`\nGoogle Gemini CLI / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: geminiCliToken });
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Google Antigravity (OAuth)
|
||||
// =========================================================================
|
||||
|
||||
describe("Google Antigravity (OAuth)", () => {
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
|
||||
console.log(`\nGoogle Antigravity / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: antigravityToken });
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
|
||||
console.log(`\nGoogle Antigravity / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: antigravityToken });
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should return totalTokens equal to sum of components",
|
||||
{ retry: 3, timeout: 60000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
|
||||
console.log(`\nGoogle Antigravity / ${llm.id}:`);
|
||||
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: antigravityToken });
|
||||
|
||||
logUsage("First request", first);
|
||||
logUsage("Second request", second);
|
||||
|
||||
assertTotalTokensEqualsComponents(first);
|
||||
assertTotalTokensEqualsComponents(second);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock", () => {
|
||||
it(
|
||||
"claude-sonnet-4-5 - should return totalTokens equal to sum of components",
|
||||
|
||||
@@ -8,6 +8,7 @@ type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
|
||||
|
||||
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
|
||||
import { hasBedrockCredentials } from "./bedrock-utils.js";
|
||||
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
|
||||
import { resolveApiKey } from "./oauth.js";
|
||||
|
||||
// Empty schema for test tools - must be proper OBJECT type for Cloud Code Assist
|
||||
@@ -17,11 +18,9 @@ const emptySchema = Type.Object({});
|
||||
const oauthTokens = await Promise.all([
|
||||
resolveApiKey("anthropic"),
|
||||
resolveApiKey("github-copilot"),
|
||||
resolveApiKey("google-gemini-cli"),
|
||||
resolveApiKey("google-antigravity"),
|
||||
resolveApiKey("openai-codex"),
|
||||
]);
|
||||
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
|
||||
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
|
||||
|
||||
/**
|
||||
* Test for Unicode surrogate pair handling in tool results.
|
||||
@@ -451,118 +450,6 @@ describe("AI Providers Unicode Surrogate Pair Tests", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Gemini CLI Provider Unicode Handling", () => {
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle emoji in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testEmojiInToolResults(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle real-world LinkedIn comment data with emoji",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testRealWorldLinkedInData(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!geminiCliToken)(
|
||||
"gemini-2.5-flash - should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
|
||||
await testUnpairedHighSurrogate(llm, { apiKey: geminiCliToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("Google Antigravity Provider Unicode Handling", () => {
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle emoji in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testEmojiInToolResults(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle real-world LinkedIn comment data with emoji",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testRealWorldLinkedInData(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gemini-3-flash - should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gemini-3-flash");
|
||||
await testUnpairedHighSurrogate(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle emoji in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testEmojiInToolResults(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle real-world LinkedIn comment data with emoji",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testRealWorldLinkedInData(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"claude-sonnet-4-5 - should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
|
||||
await testUnpairedHighSurrogate(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should handle emoji in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testEmojiInToolResults(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should handle real-world LinkedIn comment data with emoji",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testRealWorldLinkedInData(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(!antigravityToken)(
|
||||
"gpt-oss-120b-medium - should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
|
||||
await testUnpairedHighSurrogate(llm, { apiKey: antigravityToken });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider Unicode Handling", () => {
|
||||
const llm = getModel("xai", "grok-3");
|
||||
|
||||
@@ -611,28 +498,37 @@ describe("AI Providers Unicode Surrogate Pair Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.CLOUDFLARE_API_KEY || !process.env.CLOUDFLARE_ACCOUNT_ID)(
|
||||
"Cloudflare Workers AI Provider Unicode Handling",
|
||||
() => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI Provider Unicode Handling", () => {
|
||||
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmojiInToolResults(llm);
|
||||
});
|
||||
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmojiInToolResults(llm);
|
||||
});
|
||||
|
||||
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testRealWorldLinkedInData(llm);
|
||||
});
|
||||
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testRealWorldLinkedInData(llm);
|
||||
});
|
||||
|
||||
it(
|
||||
"should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
await testUnpairedHighSurrogate(llm);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testUnpairedHighSurrogate(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider Unicode Handling", () => {
|
||||
const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6");
|
||||
|
||||
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmojiInToolResults(llm);
|
||||
});
|
||||
|
||||
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testRealWorldLinkedInData(llm);
|
||||
});
|
||||
|
||||
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testUnpairedHighSurrogate(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider Unicode Handling", () => {
|
||||
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
|
||||
@@ -698,6 +594,91 @@ describe("AI Providers Unicode Surrogate Pair Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider Unicode Handling", () => {
|
||||
const llm = getModel("xiaomi", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmojiInToolResults(llm);
|
||||
});
|
||||
|
||||
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testRealWorldLinkedInData(llm);
|
||||
});
|
||||
|
||||
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testUnpairedHighSurrogate(llm);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (CN) Provider Unicode Handling",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmojiInToolResults(llm);
|
||||
});
|
||||
|
||||
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testRealWorldLinkedInData(llm);
|
||||
});
|
||||
|
||||
it(
|
||||
"should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
await testUnpairedHighSurrogate(llm);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (AMS) Provider Unicode Handling",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmojiInToolResults(llm);
|
||||
});
|
||||
|
||||
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testRealWorldLinkedInData(llm);
|
||||
});
|
||||
|
||||
it(
|
||||
"should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
await testUnpairedHighSurrogate(llm);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)(
|
||||
"Xiaomi MiMo Token Plan (SGP) Provider Unicode Handling",
|
||||
() => {
|
||||
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
|
||||
|
||||
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testEmojiInToolResults(llm);
|
||||
});
|
||||
|
||||
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||
await testRealWorldLinkedInData(llm);
|
||||
});
|
||||
|
||||
it(
|
||||
"should handle unpaired high surrogate (0xD83D) in tool results",
|
||||
{ retry: 3, timeout: 30000 },
|
||||
async () => {
|
||||
await testUnpairedHighSurrogate(llm);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Unicode Handling", () => {
|
||||
const llm = getModel("kimi-coding", "kimi-k2-thinking");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user