smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(ai): migrated Unicode handling to native toWellFormed() API

Browse Source 
- Replaced custom `sanitizeSurrogates()` utility with native `String.prototype.toWellFormed()` across all provider implementations.
- Removed `sanitize-unicode.ts` utility module and all imports of `sanitizeSurrogates` function from 10 provider files.
- Updated Unicode surrogate handling in message conversion, response processing, and system prompt handling to use built-in JavaScript API.
- Documented removal of `sanitizeSurrogates()` utility and migration to native `toWellFormed()` in CHANGELOG.
This commit is contained in:
can1357
2026-02-28 21:10:36 +01:00
parent 7d0807ec36
commit 924a70d9b4
13 changed files with 65 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/CHANGELOG.md
+6 -2
View File
@@ -1,7 +1,6 @@
# Changelog
## [Unreleased]
### Added
- `hasUnrepresentableStrictObjectMap()` pre-flight check in `tryEnforceStrictSchema`: schemas with `patternProperties` or schema-valued `additionalProperties` now degrade gracefully to non-strict mode instead of throwing during enforcement
@@ -26,6 +25,7 @@
### Changed
- Replaced `sanitizeSurrogates()` utility with native `String.prototype.toWellFormed()` for handling unpaired Unicode surrogates across all providers
- Extended `ANTHROPIC_OAUTH_BETA` constant in the OpenAI-compat Anthropic route with `interleaved-thinking-2025-05-14`, `context-management-2025-06-27`, and `prompt-caching-scope-2026-01-05` beta flags
- `claudeCodeVersion` bumped to `2.1.63`; `claudeCodeSystemInstruction` updated to identify as Claude Agent SDK
- `claudeCodeHeaders`: removed `X-Stainless-Helper-Method`, updated package version to `0.74.0`, runtime version to `v24.3.0`
@@ -50,6 +50,10 @@
- `maxOutputTokens` removed from Antigravity generation config for non-Claude models
- System instruction injection for Antigravity scoped to Claude and `gemini-3-pro-high` models only
### Removed
- Removed `sanitizeSurrogates()` utility function; use native `String.prototype.toWellFormed()` instead
## [13.3.14] - 2026-02-28
### Added
@@ -1471,4 +1475,4 @@ _Dedicated to Peter's shoulder ([@steipete](https://twitter.com/steipete))_
## [0.9.4] - 2025-11-26
Initial release with multi-provider LLM support.
Initial release with multi-provider LLM support.
packages/ai/src/providers/amazon-bedrock.ts
+8 -9
View File
@@ -44,7 +44,6 @@ import { normalizeToolCallId, resolveCacheRetention } from "../utils";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { appendRawHttpRequestDumpFor400, type RawHttpRequestDump, withHttpStatus } from "../utils/http-inspector";
import { parseStreamingJson } from "../utils/json-parse";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import { transformMessages } from "./transform-messages";
export interface BedrockOptions extends StreamOptions {
@@ -363,7 +362,7 @@ function buildSystemPrompt(
): SystemContentBlock[] | undefined {
if (!systemPrompt) return undefined;
const blocks: SystemContentBlock[] = [{ text: sanitizeSurrogates(systemPrompt) }];
const blocks: SystemContentBlock[] = [{ text: systemPrompt.toWellFormed() }];
// Add cache point for supported Claude models
if (cacheRetention !== "none" && supportsPromptCaching(model)) {
@@ -394,14 +393,14 @@ function convertMessages(
if (!m.content || m.content.trim() === "") continue;
result.push({
role: ConversationRole.USER,
content: [{ text: sanitizeSurrogates(m.content) }],
content: [{ text: m.content.toWellFormed() }],
});
} else {
const contentBlocks = m.content
.map(c => {
switch (c.type) {
case "text":
return { text: sanitizeSurrogates(c.text) };
return { text: c.text.toWellFormed() };
case "image":
return { image: createImageBlock(c.mimeType, c.data) };
default:
@@ -435,7 +434,7 @@ function convertMessages(
case "text":
// Skip empty text blocks
if (c.text.trim().length === 0) continue;
contentBlocks.push({ text: sanitizeSurrogates(c.text) });
contentBlocks.push({ text: c.text.toWellFormed() });
break;
case "toolCall":
contentBlocks.push({
@@ -455,13 +454,13 @@ function convertMessages(
if (supportsThinkingSignature(model)) {
contentBlocks.push({
reasoningContent: {
reasoningText: { text: sanitizeSurrogates(c.thinking), signature: c.thinkingSignature },
reasoningText: { text: c.thinking.toWellFormed(), signature: c.thinkingSignature },
},
});
} else {
contentBlocks.push({
reasoningContent: {
reasoningText: { text: sanitizeSurrogates(c.thinking) },
reasoningText: { text: c.thinking.toWellFormed() },
},
});
}
@@ -492,7 +491,7 @@ function convertMessages(
content: m.content.map(c =>
c.type === "image"
? { image: createImageBlock(c.mimeType, c.data) }
: { text: sanitizeSurrogates(c.text) },
: { text: c.text.toWellFormed() },
),
status: m.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS,
},
@@ -508,7 +507,7 @@ function convertMessages(
content: nextMsg.content.map(c =>
c.type === "image"
? { image: createImageBlock(c.mimeType, c.data) }
: { text: sanitizeSurrogates(c.text) },
: { text: c.text.toWellFormed() },
),
status: nextMsg.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS,
},
packages/ai/src/providers/anthropic.ts
+12 -10
View File
@@ -31,7 +31,6 @@ import { isAnthropicOAuthToken, normalizeToolCallId, resolveCacheRetention } fro
import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import { parseStreamingJson } from "../utils/json-parse";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import {
buildCopilotDynamicHeaders,
getCopilotInitiatorOverride,
@@ -286,7 +285,10 @@ function convertContentBlocks(content: (TextContent | ImageContent)[]):
// If only text blocks, return as concatenated string for simplicity
const hasImages = content.some(c => c.type === "image");
if (!hasImages) {
return sanitizeSurrogates(content.map(c => (c as TextContent).text).join("\n"));
return content
.map(c => (c as TextContent).text)
.join("\n")
.toWellFormed();
}
// If we have images, convert to content block array
@@ -294,7 +296,7 @@ function convertContentBlocks(content: (TextContent | ImageContent)[]):
if (block.type === "text") {
return {
type: "text" as const,
text: sanitizeSurrogates(block.text),
text: block.text.toWellFormed(),
};
}
return {
@@ -752,7 +754,7 @@ export function buildAnthropicSystemBlocks(
): AnthropicSystemBlock[] | undefined {
const { includeClaudeCodeInstruction = false, extraInstructions = [], billingPayload } = options;
const blocks: AnthropicSystemBlock[] = [];
const sanitizedPrompt = systemPrompt ? sanitizeSurrogates(systemPrompt) : "";
const sanitizedPrompt = systemPrompt ? systemPrompt.toWellFormed() : "";
const trimmedInstructions = extraInstructions.map(instruction => instruction.trim()).filter(Boolean);
const hasBillingHeader = sanitizedPrompt.includes(CLAUDE_BILLING_HEADER_PREFIX);
const claudeCodeSystemCacheControl: AnthropicCacheControl = { type: "ephemeral", ttl: "1h" };
@@ -1086,7 +1088,7 @@ function buildParams(
const billingPayload = shouldInjectClaudeCodeInstruction
? {
...params,
...(context.systemPrompt ? { system: sanitizeSurrogates(context.systemPrompt) } : {}),
...(context.systemPrompt ? { system: context.systemPrompt.toWellFormed() } : {}),
}
: undefined;
const systemBlocks = buildAnthropicSystemBlocks(context.systemPrompt, {
@@ -1122,7 +1124,7 @@ export function convertAnthropicMessages(
if (msg.content.trim().length > 0) {
params.push({
role: "user",
content: sanitizeSurrogates(msg.content),
content: msg.content.toWellFormed(),
});
}
} else {
@@ -1130,7 +1132,7 @@ export function convertAnthropicMessages(
if (item.type === "text") {
return {
type: "text",
text: sanitizeSurrogates(item.text),
text: item.text.toWellFormed(),
};
}
return {
@@ -1163,19 +1165,19 @@ export function convertAnthropicMessages(
if (block.text.trim().length === 0) continue;
blocks.push({
type: "text",
text: sanitizeSurrogates(block.text),
text: block.text.toWellFormed(),
});
} else if (block.type === "thinking") {
if (block.thinking.trim().length === 0) continue;
if (!block.thinkingSignature || block.thinkingSignature.trim().length === 0) {
blocks.push({
type: "text",
text: sanitizeSurrogates(block.thinking),
text: block.thinking.toWellFormed(),
});
} else {
blocks.push({
type: "thinking",
thinking: sanitizeSurrogates(block.thinking),
thinking: block.thinking.toWellFormed(),
signature: block.thinkingSignature,
});
}
packages/ai/src/providers/azure-openai-responses.ts
+7 -8
View File
@@ -33,7 +33,6 @@ import { normalizeResponsesToolCallId } from "../utils";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import { parseStreamingJson } from "../utils/json-parse";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice";
import { transformMessages } from "./transform-messages";
@@ -554,7 +553,7 @@ function convertMessages(
const role = model.reasoning ? "developer" : "system";
messages.push({
role,
content: sanitizeSurrogates(context.systemPrompt),
content: context.systemPrompt.toWellFormed(),
});
}
@@ -566,14 +565,14 @@ function convertMessages(
if (!msg.content || msg.content.trim() === "") continue;
messages.push({
role: "user",
content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }],
content: [{ type: "input_text", text: msg.content.toWellFormed() }],
});
} else {
const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => {
if (item.type === "text") {
return {
type: "input_text",
text: sanitizeSurrogates(item.text),
text: item.text.toWellFormed(),
} satisfies ResponseInputText;
}
return {
@@ -604,14 +603,14 @@ function convertMessages(
if (!msg.content || msg.content.trim() === "") continue;
messages.push({
role: devRole,
content: sanitizeSurrogates(msg.content),
content: msg.content.toWellFormed(),
});
} else {
const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => {
if (item.type === "text") {
return {
type: "input_text",
text: sanitizeSurrogates(item.text),
text: item.text.toWellFormed(),
} satisfies ResponseInputText;
}
return {
@@ -666,7 +665,7 @@ function convertMessages(
output.push({
type: "message",
role: "assistant",
content: [{ type: "output_text", text: sanitizeSurrogates(textBlock.text), annotations: [] }],
content: [{ type: "output_text", text: textBlock.text.toWellFormed(), annotations: [] }],
status: "completed",
id: msgId,
} satisfies ResponseOutputMessage);
@@ -711,7 +710,7 @@ function convertMessages(
messages.push({
type: "function_call_output",
call_id: normalized.callId,
output: sanitizeSurrogates(hasText ? textResult : "(see attached image)"),
output: (hasText ? textResult : "(see attached image)").toWellFormed(),
});
// If there are images and model supports them, send a follow-up user message with images
packages/ai/src/providers/google-gemini-cli.ts
+1 -2
View File
@@ -22,7 +22,6 @@ import { AssistantMessageEventStream } from "../utils/event-stream";
import { appendRawHttpRequestDumpFor400, type RawHttpRequestDump, withHttpStatus } from "../utils/http-inspector";
import { refreshAntigravityToken } from "../utils/oauth/google-antigravity";
import { refreshGoogleCloudToken } from "../utils/oauth/google-gemini-cli";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import {
convertMessages,
convertTools,
@@ -1036,7 +1035,7 @@ export function buildRequest(
// System instruction must be object with parts, not plain string
if (context.systemPrompt) {
request.systemInstruction = {
parts: [{ text: sanitizeSurrogates(context.systemPrompt) }],
parts: [{ text: context.systemPrompt.toWellFormed() }],
};
}
packages/ai/src/providers/google-shared.ts
+8 -11
View File
@@ -3,7 +3,6 @@
*/
import { type Content, FinishReason, FunctionCallingConfigMode, type Part } from "@google/genai";
import type { Context, ImageContent, Model, StopReason, TextContent, Tool } from "../types";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import { prepareSchemaForCCA, sanitizeSchemaForGoogle } from "../utils/schema";
import { transformMessages } from "./transform-messages";
@@ -89,12 +88,12 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
if (!msg.content || msg.content.trim() === "") continue;
contents.push({
role: "user",
parts: [{ text: sanitizeSurrogates(msg.content) }],
parts: [{ text: msg.content.toWellFormed() }],
});
} else {
const parts: Part[] = msg.content.map(item => {
if (item.type === "text") {
return { text: sanitizeSurrogates(item.text) };
return { text: item.text.toWellFormed() };
} else {
return {
inlineData: {
@@ -129,7 +128,7 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
if (!block.text || block.text.trim() === "") continue;
const thoughtSignature = resolveThoughtSignature(isSameProviderAndModel, block.textSignature);
parts.push({
text: sanitizeSurrogates(block.text),
text: block.text.toWellFormed(),
...(thoughtSignature && { thoughtSignature }),
});
} else if (block.type === "thinking") {
@@ -141,12 +140,12 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
const thoughtSignature = resolveThoughtSignature(isSameProviderAndModel, block.thinkingSignature);
parts.push({
thought: true,
text: sanitizeSurrogates(block.thinking),
text: block.thinking.toWellFormed(),
...(thoughtSignature && { thoughtSignature }),
});
} else {
parts.push({
text: sanitizeSurrogates(block.thinking),
text: block.thinking.toWellFormed(),
});
}
} else if (block.type === "toolCall") {
@@ -160,12 +159,10 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
.join("\n");
parts.push({
text: sanitizeSurrogates(
`<call_record tool="${block.name}">
text: `<call_record tool="${block.name}">
<critical>Historical context only. You cannot invoke tools this way—use proper function calling.</critical>
${params}
</call_record>`,
),
</call_record>`.toWellFormed(),
});
continue;
}
@@ -209,7 +206,7 @@ ${params}
const supportsMultimodalFunctionResponse = model.id.includes("gemini-3");
// Use "output" key for success, "error" key for errors as per SDK documentation
const responseValue = hasText ? sanitizeSurrogates(textResult) : hasImages ? "(see attached image)" : "";
const responseValue = hasText ? textResult.toWellFormed() : hasImages ? "(see attached image)" : "";
const imageParts: Part[] = imageContent.map(imageBlock => ({
inlineData: {
packages/ai/src/providers/google-vertex.ts
+1 -2
View File
@@ -20,7 +20,6 @@ import type {
} from "../types";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import type { GoogleThinkingLevel } from "./google-gemini-cli";
import {
convertMessages,
@@ -371,7 +370,7 @@ function buildParams(
const config: GenerateContentConfig = {
...(Object.keys(generationConfig).length > 0 && generationConfig),
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
...(context.systemPrompt && { systemInstruction: context.systemPrompt.toWellFormed() }),
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools, model) }),
};
packages/ai/src/providers/google.ts
+1 -2
View File
@@ -19,7 +19,6 @@ import type {
} from "../types";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import type { GoogleThinkingLevel } from "./google-gemini-cli";
import {
convertMessages,
@@ -334,7 +333,7 @@ function buildParams(
const config: GenerateContentConfig = {
...(Object.keys(generationConfig).length > 0 && generationConfig),
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
...(context.systemPrompt && { systemInstruction: context.systemPrompt.toWellFormed() }),
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools, model) }),
};
packages/ai/src/providers/openai-codex-responses.ts
+6 -7
View File
@@ -31,7 +31,6 @@ import { normalizeResponsesToolCallId } from "../utils";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import { parseStreamingJson } from "../utils/json-parse";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import { adaptSchemaForStrict, NO_STRICT } from "../utils/schema";
import {
CODEX_BASE_URL,
@@ -1551,14 +1550,14 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex
if (!msg.content || msg.content.trim() === "") continue;
messages.push({
role: "user",
content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }],
content: [{ type: "input_text", text: msg.content.toWellFormed() }],
});
} else {
const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => {
if (item.type === "text") {
return {
type: "input_text",
text: sanitizeSurrogates(item.text),
text: item.text.toWellFormed(),
} satisfies ResponseInputText;
}
return {
@@ -1588,14 +1587,14 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex
if (!msg.content || msg.content.trim() === "") continue;
messages.push({
role: "developer",
content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }],
content: [{ type: "input_text", text: msg.content.toWellFormed() }],
});
} else {
const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => {
if (item.type === "text") {
return {
type: "input_text",
text: sanitizeSurrogates(item.text),
text: item.text.toWellFormed(),
} satisfies ResponseInputText;
}
return {
@@ -1639,7 +1638,7 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex
output.push({
type: "message",
role: "assistant",
content: [{ type: "output_text", text: sanitizeSurrogates(textBlock.text), annotations: [] }],
content: [{ type: "output_text", text: textBlock.text.toWellFormed(), annotations: [] }],
status: "completed",
id: msgId,
} satisfies ResponseOutputMessage);
@@ -1669,7 +1668,7 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex
messages.push({
type: "function_call_output",
call_id: normalized.callId,
output: sanitizeSurrogates(hasText ? textResult : "(see attached image)"),
output: (hasText ? textResult : "(see attached image)").toWellFormed(),
});
if (hasImages && model.input.includes("image")) {
packages/ai/src/providers/openai-completions.ts
+6 -7
View File
@@ -31,7 +31,6 @@ import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import { parseStreamingJson } from "../utils/json-parse";
import { getKimiCommonHeaders } from "../utils/oauth/kimi";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import { adaptSchemaForStrict, NO_STRICT } from "../utils/schema";
import { mapToOpenAICompletionsToolChoice } from "../utils/tool-choice";
import {
@@ -719,7 +718,7 @@ export function convertMessages(
if (context.systemPrompt) {
const useDeveloperRole = model.reasoning && compat.supportsDeveloperRole;
const role = useDeveloperRole ? "developer" : "system";
params.push({ role: role, content: sanitizeSurrogates(context.systemPrompt) });
params.push({ role: role, content: context.systemPrompt.toWellFormed() });
}
let lastRole: string | null = null;
@@ -743,7 +742,7 @@ export function convertMessages(
if (msg.role === "user" || msg.role === "developer") {
const role = !devAsUser && msg.role === "developer" ? "developer" : "user";
if (typeof msg.content === "string") {
const text = sanitizeSurrogates(msg.content);
const text = msg.content.toWellFormed();
if (text.trim().length === 0) continue;
params.push({
role: role,
@@ -753,7 +752,7 @@ export function convertMessages(
const content: ChatCompletionContentPart[] = [];
for (const item of msg.content) {
if (item.type === "text") {
const text = sanitizeSurrogates(item.text);
const text = item.text.toWellFormed();
if (text.trim().length === 0) continue;
content.push({
type: "text",
@@ -791,10 +790,10 @@ export function convertMessages(
// GitHub Copilot requires assistant content as a string, not an array.
// Sending as array causes Claude models to re-answer all previous prompts.
if (model.provider === "github-copilot") {
assistantMsg.content = nonEmptyTextBlocks.map(b => sanitizeSurrogates(b.text)).join("");
assistantMsg.content = nonEmptyTextBlocks.map(b => b.text.toWellFormed()).join("");
} else {
assistantMsg.content = nonEmptyTextBlocks.map(b => {
return { type: "text", text: sanitizeSurrogates(b.text) };
return { type: "text", text: b.text.toWellFormed() };
});
}
}
@@ -918,7 +917,7 @@ export function convertMessages(
remappedToolCallId ?? ensureToolCallId(toolMsg.toolCallId, `${j}:${toolMsg.toolName ?? "tool"}`);
const toolResultMsg: ChatCompletionToolMessageParam = {
role: "tool",
content: sanitizeSurrogates(hasText ? textResult : "(see attached image)"),
content: (hasText ? textResult : "(see attached image)").toWellFormed(),
tool_call_id: normalizeMistralToolId(resolvedToolCallId, compat.requiresMistralToolIds),
};
if (compat.requiresToolResultName && toolMsg.toolName) {
packages/ai/src/providers/openai-responses.ts
+5 -6
View File
@@ -32,7 +32,6 @@ import { normalizeResponsesToolCallId, resolveCacheRetention } from "../utils";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import { parseStreamingJson } from "../utils/json-parse";
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
import { adaptSchemaForStrict, NO_STRICT } from "../utils/schema";
import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice";
import {
@@ -539,7 +538,7 @@ function convertMessages(
const role = model.reasoning ? "developer" : "system";
messages.push({
role,
content: sanitizeSurrogates(context.systemPrompt),
content: context.systemPrompt.toWellFormed(),
});
}
@@ -551,14 +550,14 @@ function convertMessages(
if (!msg.content || msg.content.trim() === "") continue;
messages.push({
role: "user",
content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }],
content: [{ type: "input_text", text: msg.content.toWellFormed() }],
});
} else {
const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => {
if (item.type === "text") {
return {
type: "input_text",
text: sanitizeSurrogates(item.text),
text: item.text.toWellFormed(),
} satisfies ResponseInputText;
} else {
return {
@@ -615,7 +614,7 @@ function convertMessages(
output.push({
type: "message",
role: "assistant",
content: [{ type: "output_text", text: sanitizeSurrogates(textBlock.text), annotations: [] }],
content: [{ type: "output_text", text: textBlock.text.toWellFormed(), annotations: [] }],
status: "completed",
id: msgId,
} satisfies ResponseOutputMessage);
@@ -660,7 +659,7 @@ function convertMessages(
messages.push({
type: "function_call_output",
call_id: normalized.callId,
output: sanitizeSurrogates(hasText ? textResult : "(see attached image)"),
output: (hasText ? textResult : "(see attached image)").toWellFormed(),
});
// If there are images and model supports them, send a follow-up user message with images
packages/ai/src/utils/sanitize-unicode.ts
-25
View File
@@ -1,25 +0,0 @@
/**
* Removes unpaired Unicode surrogate characters from a string.
*
* Unpaired surrogates (high surrogates 0xD800-0xDBFF without matching low surrogates 0xDC00-0xDFFF,
* or vice versa) cause JSON serialization errors in many API providers.
*
* Valid emoji and other characters outside the Basic Multilingual Plane use properly paired
* surrogates and will NOT be affected by this function.
*
* @param text - The text to sanitize
* @returns The sanitized text with unpaired surrogates removed
*
* @example
* // Valid emoji (properly paired surrogates) are preserved
* sanitizeSurrogates("Hello 🙈 World") // => "Hello 🙈 World"
*
* // Unpaired high surrogate is removed
* const unpaired = String.fromCharCode(0xD83D); // high surrogate without low
* sanitizeSurrogates(`Text ${unpaired} here`) // => "Text here"
*/
export function sanitizeSurrogates(text: string): string {
// Replace unpaired high surrogates (0xD800-0xDBFF not followed by low surrogate)
// Replace unpaired low surrogates (0xDC00-0xDFFF not preceded by high surrogate)
return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "");
}
packages/ai/test/claude-usage-headers.test.ts
+4 -1
View File
@@ -14,7 +14,10 @@ function createMemoryCache(): UsageCache {
};
}
function getHeaderCaseInsensitive(headers: HeadersInit | undefined, name: string): string | undefined {
function getHeaderCaseInsensitive(
headers: Headers | Record<string, string | ReadonlyArray<string>> | string[][] | undefined,
name: string,
): string | undefined {
if (!headers) return undefined;
const target = name.toLowerCase();