feat(ai): allowed custom idle timeout for openai responses streams
- Pass `streamIdleTimeoutMs` from model compatibility settings to the streaming logic. - Update catalog definitions for Sakana models to include a 300,000ms idle timeout. - Add a test case to verify that the streaming client honors the model-defined idle timeout.
This commit is contained in:
@@ -436,7 +436,8 @@ const streamOpenAIResponsesOnce = (
|
||||
? buildOpenAIResponsesChainedParams(params, trailingScaffoldingItems, chainState)
|
||||
: { params };
|
||||
sentPreviousResponseId = chained.previousResponseId;
|
||||
const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs();
|
||||
const idleTimeoutMs =
|
||||
options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs(model.compat.streamIdleTimeoutMs);
|
||||
const firstEventTimeoutMs =
|
||||
options?.streamFirstEventTimeoutMs ?? getOpenAIStreamFirstEventTimeoutMs(idleTimeoutMs);
|
||||
const requestTimeoutMs =
|
||||
|
||||
@@ -790,4 +790,29 @@ describe("OpenAI-family first-event timeouts", () => {
|
||||
{ type: "text", text: "Truncated output", textSignature: '{"v":1,"id":"msg_length_limited"}' },
|
||||
]);
|
||||
});
|
||||
|
||||
it("honors streamIdleTimeoutMs from model.compat for OpenAI responses streams", async () => {
|
||||
const customResponsesModel: Model<"openai-responses"> = buildModel({
|
||||
id: "fugu-test",
|
||||
name: "Fugu Test",
|
||||
api: "openai-responses",
|
||||
provider: "sakana",
|
||||
baseUrl: "https://api.sakana.ai/v1",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 16384,
|
||||
compat: { streamIdleTimeoutMs: 20 },
|
||||
});
|
||||
|
||||
const fetchMock = () => Promise.resolve(createNoProgressOpenAIResponsesStream(undefined));
|
||||
const result = await streamOpenAIResponses(customResponsesModel, baseContext(), {
|
||||
apiKey: "test-key",
|
||||
fetch: fetchMock as any,
|
||||
}).result();
|
||||
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toBe("OpenAI responses stream stalled while waiting for the next event");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import type { FetchImpl } from "@oh-my-pi/pi-ai/types";
|
||||
import { getProxyForProvider, isLocalOrMetadataHost, shouldBypassProxy, wrapFetchForProxy } from "@oh-my-pi/pi-ai/utils/proxy";
|
||||
import {
|
||||
getProxyForProvider,
|
||||
isLocalOrMetadataHost,
|
||||
shouldBypassProxy,
|
||||
wrapFetchForProxy,
|
||||
} from "@oh-my-pi/pi-ai/utils/proxy";
|
||||
|
||||
const PROXY = "http://127.0.0.1:24560";
|
||||
|
||||
|
||||
@@ -535,6 +535,7 @@ export function buildOpenAIResponsesCompat(spec: OpenAIResponsesSpecLike): Resol
|
||||
emptyLengthFinishIsContextError: spec.provider === "ollama",
|
||||
usesOpenAIToolCallIdLimit: spec.provider === "openai",
|
||||
promptCacheSessionHeader: spec.provider === "xai-oauth" ? "x-grok-conv-id" : undefined,
|
||||
streamIdleTimeoutMs: spec.compat?.streamIdleTimeoutMs,
|
||||
};
|
||||
applyCompatOverrides(compat, spec.compat);
|
||||
if (spec.compat?.reasoningDisableMode === undefined) {
|
||||
|
||||
@@ -69272,7 +69272,8 @@
|
||||
}
|
||||
},
|
||||
"compat": {
|
||||
"includeEncryptedReasoning": false
|
||||
"includeEncryptedReasoning": false,
|
||||
"streamIdleTimeoutMs": 300000
|
||||
}
|
||||
},
|
||||
"fugu-ultra": {
|
||||
@@ -69304,7 +69305,8 @@
|
||||
}
|
||||
},
|
||||
"compat": {
|
||||
"includeEncryptedReasoning": false
|
||||
"includeEncryptedReasoning": false,
|
||||
"streamIdleTimeoutMs": 300000
|
||||
}
|
||||
},
|
||||
"fugu-ultra-20260615": {
|
||||
@@ -69336,7 +69338,8 @@
|
||||
}
|
||||
},
|
||||
"compat": {
|
||||
"includeEncryptedReasoning": false
|
||||
"includeEncryptedReasoning": false,
|
||||
"streamIdleTimeoutMs": 300000
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -2534,6 +2534,7 @@ const SAKANA_FUGU_THINKING: ThinkingConfig = {
|
||||
};
|
||||
const SAKANA_RESPONSES_COMPAT: ModelSpec<"openai-responses">["compat"] = {
|
||||
includeEncryptedReasoning: false,
|
||||
streamIdleTimeoutMs: 300_000,
|
||||
};
|
||||
|
||||
function normalizeSakanaBaseUrl(baseUrl: string | undefined): string {
|
||||
|
||||
@@ -534,6 +534,7 @@ export interface ResolvedOpenAIResponsesCompat extends ResolvedOpenAISharedCompa
|
||||
supportsImageDetailOriginal: boolean;
|
||||
requiresJuiceZeroHack: boolean;
|
||||
supportsObfuscationOptOut: boolean;
|
||||
streamIdleTimeoutMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user