From 924a70d9b4f45975b6685b96d205fffaf27e69c7 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 28 Feb 2026 21:10:36 +0100 Subject: [PATCH] refactor(ai): migrated Unicode handling to native toWellFormed() API - Replaced custom `sanitizeSurrogates()` utility with native `String.prototype.toWellFormed()` across all provider implementations. - Removed `sanitize-unicode.ts` utility module and all imports of `sanitizeSurrogates` function from 10 provider files. - Updated Unicode surrogate handling in message conversion, response processing, and system prompt handling to use built-in JavaScript API. - Documented removal of `sanitizeSurrogates()` utility and migration to native `toWellFormed()` in CHANGELOG. --- packages/ai/CHANGELOG.md | 8 ++++-- packages/ai/src/providers/amazon-bedrock.ts | 17 ++++++------- packages/ai/src/providers/anthropic.ts | 22 ++++++++-------- .../src/providers/azure-openai-responses.ts | 15 ++++++----- .../ai/src/providers/google-gemini-cli.ts | 3 +-- packages/ai/src/providers/google-shared.ts | 19 ++++++-------- packages/ai/src/providers/google-vertex.ts | 3 +-- packages/ai/src/providers/google.ts | 3 +-- .../src/providers/openai-codex-responses.ts | 13 +++++----- .../ai/src/providers/openai-completions.ts | 13 +++++----- packages/ai/src/providers/openai-responses.ts | 11 ++++---- packages/ai/src/utils/sanitize-unicode.ts | 25 ------------------- packages/ai/test/claude-usage-headers.test.ts | 5 +++- 13 files changed, 65 insertions(+), 92 deletions(-) delete mode 100644 packages/ai/src/utils/sanitize-unicode.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 03674ddf4..9c365e074 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,7 +1,6 @@ # Changelog ## [Unreleased] - ### Added - `hasUnrepresentableStrictObjectMap()` pre-flight check in `tryEnforceStrictSchema`: schemas with `patternProperties` or schema-valued `additionalProperties` now degrade gracefully to non-strict mode instead of throwing during enforcement @@ -26,6 +25,7 @@ ### Changed +- Replaced `sanitizeSurrogates()` utility with native `String.prototype.toWellFormed()` for handling unpaired Unicode surrogates across all providers - Extended `ANTHROPIC_OAUTH_BETA` constant in the OpenAI-compat Anthropic route with `interleaved-thinking-2025-05-14`, `context-management-2025-06-27`, and `prompt-caching-scope-2026-01-05` beta flags - `claudeCodeVersion` bumped to `2.1.63`; `claudeCodeSystemInstruction` updated to identify as Claude Agent SDK - `claudeCodeHeaders`: removed `X-Stainless-Helper-Method`, updated package version to `0.74.0`, runtime version to `v24.3.0` @@ -50,6 +50,10 @@ - `maxOutputTokens` removed from Antigravity generation config for non-Claude models - System instruction injection for Antigravity scoped to Claude and `gemini-3-pro-high` models only +### Removed + +- Removed `sanitizeSurrogates()` utility function; use native `String.prototype.toWellFormed()` instead + ## [13.3.14] - 2026-02-28 ### Added @@ -1471,4 +1475,4 @@ _Dedicated to Peter's shoulder ([@steipete](https://twitter.com/steipete))_ ## [0.9.4] - 2025-11-26 -Initial release with multi-provider LLM support. +Initial release with multi-provider LLM support. \ No newline at end of file diff --git a/packages/ai/src/providers/amazon-bedrock.ts b/packages/ai/src/providers/amazon-bedrock.ts index 4c3a20c51..77e1a735f 100644 --- a/packages/ai/src/providers/amazon-bedrock.ts +++ b/packages/ai/src/providers/amazon-bedrock.ts @@ -44,7 +44,6 @@ import { normalizeToolCallId, resolveCacheRetention } from "../utils"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { appendRawHttpRequestDumpFor400, type RawHttpRequestDump, withHttpStatus } from "../utils/http-inspector"; import { parseStreamingJson } from "../utils/json-parse"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { transformMessages } from "./transform-messages"; export interface BedrockOptions extends StreamOptions { @@ -363,7 +362,7 @@ function buildSystemPrompt( ): SystemContentBlock[] | undefined { if (!systemPrompt) return undefined; - const blocks: SystemContentBlock[] = [{ text: sanitizeSurrogates(systemPrompt) }]; + const blocks: SystemContentBlock[] = [{ text: systemPrompt.toWellFormed() }]; // Add cache point for supported Claude models if (cacheRetention !== "none" && supportsPromptCaching(model)) { @@ -394,14 +393,14 @@ function convertMessages( if (!m.content || m.content.trim() === "") continue; result.push({ role: ConversationRole.USER, - content: [{ text: sanitizeSurrogates(m.content) }], + content: [{ text: m.content.toWellFormed() }], }); } else { const contentBlocks = m.content .map(c => { switch (c.type) { case "text": - return { text: sanitizeSurrogates(c.text) }; + return { text: c.text.toWellFormed() }; case "image": return { image: createImageBlock(c.mimeType, c.data) }; default: @@ -435,7 +434,7 @@ function convertMessages( case "text": // Skip empty text blocks if (c.text.trim().length === 0) continue; - contentBlocks.push({ text: sanitizeSurrogates(c.text) }); + contentBlocks.push({ text: c.text.toWellFormed() }); break; case "toolCall": contentBlocks.push({ @@ -455,13 +454,13 @@ function convertMessages( if (supportsThinkingSignature(model)) { contentBlocks.push({ reasoningContent: { - reasoningText: { text: sanitizeSurrogates(c.thinking), signature: c.thinkingSignature }, + reasoningText: { text: c.thinking.toWellFormed(), signature: c.thinkingSignature }, }, }); } else { contentBlocks.push({ reasoningContent: { - reasoningText: { text: sanitizeSurrogates(c.thinking) }, + reasoningText: { text: c.thinking.toWellFormed() }, }, }); } @@ -492,7 +491,7 @@ function convertMessages( content: m.content.map(c => c.type === "image" ? { image: createImageBlock(c.mimeType, c.data) } - : { text: sanitizeSurrogates(c.text) }, + : { text: c.text.toWellFormed() }, ), status: m.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS, }, @@ -508,7 +507,7 @@ function convertMessages( content: nextMsg.content.map(c => c.type === "image" ? { image: createImageBlock(c.mimeType, c.data) } - : { text: sanitizeSurrogates(c.text) }, + : { text: c.text.toWellFormed() }, ), status: nextMsg.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS, }, diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index cc889363c..9a02c35ca 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -31,7 +31,6 @@ import { isAnthropicOAuthToken, normalizeToolCallId, resolveCacheRetention } fro import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; import { parseStreamingJson } from "../utils/json-parse"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { buildCopilotDynamicHeaders, getCopilotInitiatorOverride, @@ -286,7 +285,10 @@ function convertContentBlocks(content: (TextContent | ImageContent)[]): // If only text blocks, return as concatenated string for simplicity const hasImages = content.some(c => c.type === "image"); if (!hasImages) { - return sanitizeSurrogates(content.map(c => (c as TextContent).text).join("\n")); + return content + .map(c => (c as TextContent).text) + .join("\n") + .toWellFormed(); } // If we have images, convert to content block array @@ -294,7 +296,7 @@ function convertContentBlocks(content: (TextContent | ImageContent)[]): if (block.type === "text") { return { type: "text" as const, - text: sanitizeSurrogates(block.text), + text: block.text.toWellFormed(), }; } return { @@ -752,7 +754,7 @@ export function buildAnthropicSystemBlocks( ): AnthropicSystemBlock[] | undefined { const { includeClaudeCodeInstruction = false, extraInstructions = [], billingPayload } = options; const blocks: AnthropicSystemBlock[] = []; - const sanitizedPrompt = systemPrompt ? sanitizeSurrogates(systemPrompt) : ""; + const sanitizedPrompt = systemPrompt ? systemPrompt.toWellFormed() : ""; const trimmedInstructions = extraInstructions.map(instruction => instruction.trim()).filter(Boolean); const hasBillingHeader = sanitizedPrompt.includes(CLAUDE_BILLING_HEADER_PREFIX); const claudeCodeSystemCacheControl: AnthropicCacheControl = { type: "ephemeral", ttl: "1h" }; @@ -1086,7 +1088,7 @@ function buildParams( const billingPayload = shouldInjectClaudeCodeInstruction ? { ...params, - ...(context.systemPrompt ? { system: sanitizeSurrogates(context.systemPrompt) } : {}), + ...(context.systemPrompt ? { system: context.systemPrompt.toWellFormed() } : {}), } : undefined; const systemBlocks = buildAnthropicSystemBlocks(context.systemPrompt, { @@ -1122,7 +1124,7 @@ export function convertAnthropicMessages( if (msg.content.trim().length > 0) { params.push({ role: "user", - content: sanitizeSurrogates(msg.content), + content: msg.content.toWellFormed(), }); } } else { @@ -1130,7 +1132,7 @@ export function convertAnthropicMessages( if (item.type === "text") { return { type: "text", - text: sanitizeSurrogates(item.text), + text: item.text.toWellFormed(), }; } return { @@ -1163,19 +1165,19 @@ export function convertAnthropicMessages( if (block.text.trim().length === 0) continue; blocks.push({ type: "text", - text: sanitizeSurrogates(block.text), + text: block.text.toWellFormed(), }); } else if (block.type === "thinking") { if (block.thinking.trim().length === 0) continue; if (!block.thinkingSignature || block.thinkingSignature.trim().length === 0) { blocks.push({ type: "text", - text: sanitizeSurrogates(block.thinking), + text: block.thinking.toWellFormed(), }); } else { blocks.push({ type: "thinking", - thinking: sanitizeSurrogates(block.thinking), + thinking: block.thinking.toWellFormed(), signature: block.thinkingSignature, }); } diff --git a/packages/ai/src/providers/azure-openai-responses.ts b/packages/ai/src/providers/azure-openai-responses.ts index 9ebce90a4..1820b0d2f 100644 --- a/packages/ai/src/providers/azure-openai-responses.ts +++ b/packages/ai/src/providers/azure-openai-responses.ts @@ -33,7 +33,6 @@ import { normalizeResponsesToolCallId } from "../utils"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; import { parseStreamingJson } from "../utils/json-parse"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice"; import { transformMessages } from "./transform-messages"; @@ -554,7 +553,7 @@ function convertMessages( const role = model.reasoning ? "developer" : "system"; messages.push({ role, - content: sanitizeSurrogates(context.systemPrompt), + content: context.systemPrompt.toWellFormed(), }); } @@ -566,14 +565,14 @@ function convertMessages( if (!msg.content || msg.content.trim() === "") continue; messages.push({ role: "user", - content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }], + content: [{ type: "input_text", text: msg.content.toWellFormed() }], }); } else { const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => { if (item.type === "text") { return { type: "input_text", - text: sanitizeSurrogates(item.text), + text: item.text.toWellFormed(), } satisfies ResponseInputText; } return { @@ -604,14 +603,14 @@ function convertMessages( if (!msg.content || msg.content.trim() === "") continue; messages.push({ role: devRole, - content: sanitizeSurrogates(msg.content), + content: msg.content.toWellFormed(), }); } else { const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => { if (item.type === "text") { return { type: "input_text", - text: sanitizeSurrogates(item.text), + text: item.text.toWellFormed(), } satisfies ResponseInputText; } return { @@ -666,7 +665,7 @@ function convertMessages( output.push({ type: "message", role: "assistant", - content: [{ type: "output_text", text: sanitizeSurrogates(textBlock.text), annotations: [] }], + content: [{ type: "output_text", text: textBlock.text.toWellFormed(), annotations: [] }], status: "completed", id: msgId, } satisfies ResponseOutputMessage); @@ -711,7 +710,7 @@ function convertMessages( messages.push({ type: "function_call_output", call_id: normalized.callId, - output: sanitizeSurrogates(hasText ? textResult : "(see attached image)"), + output: (hasText ? textResult : "(see attached image)").toWellFormed(), }); // If there are images and model supports them, send a follow-up user message with images diff --git a/packages/ai/src/providers/google-gemini-cli.ts b/packages/ai/src/providers/google-gemini-cli.ts index 9d98125a9..f09b01644 100644 --- a/packages/ai/src/providers/google-gemini-cli.ts +++ b/packages/ai/src/providers/google-gemini-cli.ts @@ -22,7 +22,6 @@ import { AssistantMessageEventStream } from "../utils/event-stream"; import { appendRawHttpRequestDumpFor400, type RawHttpRequestDump, withHttpStatus } from "../utils/http-inspector"; import { refreshAntigravityToken } from "../utils/oauth/google-antigravity"; import { refreshGoogleCloudToken } from "../utils/oauth/google-gemini-cli"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { convertMessages, convertTools, @@ -1036,7 +1035,7 @@ export function buildRequest( // System instruction must be object with parts, not plain string if (context.systemPrompt) { request.systemInstruction = { - parts: [{ text: sanitizeSurrogates(context.systemPrompt) }], + parts: [{ text: context.systemPrompt.toWellFormed() }], }; } diff --git a/packages/ai/src/providers/google-shared.ts b/packages/ai/src/providers/google-shared.ts index 850cba7fe..fe3c88261 100644 --- a/packages/ai/src/providers/google-shared.ts +++ b/packages/ai/src/providers/google-shared.ts @@ -3,7 +3,6 @@ */ import { type Content, FinishReason, FunctionCallingConfigMode, type Part } from "@google/genai"; import type { Context, ImageContent, Model, StopReason, TextContent, Tool } from "../types"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { prepareSchemaForCCA, sanitizeSchemaForGoogle } from "../utils/schema"; import { transformMessages } from "./transform-messages"; @@ -89,12 +88,12 @@ export function convertMessages(model: Model, contex if (!msg.content || msg.content.trim() === "") continue; contents.push({ role: "user", - parts: [{ text: sanitizeSurrogates(msg.content) }], + parts: [{ text: msg.content.toWellFormed() }], }); } else { const parts: Part[] = msg.content.map(item => { if (item.type === "text") { - return { text: sanitizeSurrogates(item.text) }; + return { text: item.text.toWellFormed() }; } else { return { inlineData: { @@ -129,7 +128,7 @@ export function convertMessages(model: Model, contex if (!block.text || block.text.trim() === "") continue; const thoughtSignature = resolveThoughtSignature(isSameProviderAndModel, block.textSignature); parts.push({ - text: sanitizeSurrogates(block.text), + text: block.text.toWellFormed(), ...(thoughtSignature && { thoughtSignature }), }); } else if (block.type === "thinking") { @@ -141,12 +140,12 @@ export function convertMessages(model: Model, contex const thoughtSignature = resolveThoughtSignature(isSameProviderAndModel, block.thinkingSignature); parts.push({ thought: true, - text: sanitizeSurrogates(block.thinking), + text: block.thinking.toWellFormed(), ...(thoughtSignature && { thoughtSignature }), }); } else { parts.push({ - text: sanitizeSurrogates(block.thinking), + text: block.thinking.toWellFormed(), }); } } else if (block.type === "toolCall") { @@ -160,12 +159,10 @@ export function convertMessages(model: Model, contex .join("\n"); parts.push({ - text: sanitizeSurrogates( - ` + text: ` Historical context only. You cannot invoke tools this way—use proper function calling. ${params} -`, - ), +`.toWellFormed(), }); continue; } @@ -209,7 +206,7 @@ ${params} const supportsMultimodalFunctionResponse = model.id.includes("gemini-3"); // Use "output" key for success, "error" key for errors as per SDK documentation - const responseValue = hasText ? sanitizeSurrogates(textResult) : hasImages ? "(see attached image)" : ""; + const responseValue = hasText ? textResult.toWellFormed() : hasImages ? "(see attached image)" : ""; const imageParts: Part[] = imageContent.map(imageBlock => ({ inlineData: { diff --git a/packages/ai/src/providers/google-vertex.ts b/packages/ai/src/providers/google-vertex.ts index 0add6f813..1ee23235a 100644 --- a/packages/ai/src/providers/google-vertex.ts +++ b/packages/ai/src/providers/google-vertex.ts @@ -20,7 +20,6 @@ import type { } from "../types"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import type { GoogleThinkingLevel } from "./google-gemini-cli"; import { convertMessages, @@ -371,7 +370,7 @@ function buildParams( const config: GenerateContentConfig = { ...(Object.keys(generationConfig).length > 0 && generationConfig), - ...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }), + ...(context.systemPrompt && { systemInstruction: context.systemPrompt.toWellFormed() }), ...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools, model) }), }; diff --git a/packages/ai/src/providers/google.ts b/packages/ai/src/providers/google.ts index 6f6be73fe..c49c02f8d 100644 --- a/packages/ai/src/providers/google.ts +++ b/packages/ai/src/providers/google.ts @@ -19,7 +19,6 @@ import type { } from "../types"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import type { GoogleThinkingLevel } from "./google-gemini-cli"; import { convertMessages, @@ -334,7 +333,7 @@ function buildParams( const config: GenerateContentConfig = { ...(Object.keys(generationConfig).length > 0 && generationConfig), - ...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }), + ...(context.systemPrompt && { systemInstruction: context.systemPrompt.toWellFormed() }), ...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools, model) }), }; diff --git a/packages/ai/src/providers/openai-codex-responses.ts b/packages/ai/src/providers/openai-codex-responses.ts index 72188a8f0..e7fa690ce 100644 --- a/packages/ai/src/providers/openai-codex-responses.ts +++ b/packages/ai/src/providers/openai-codex-responses.ts @@ -31,7 +31,6 @@ import { normalizeResponsesToolCallId } from "../utils"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; import { parseStreamingJson } from "../utils/json-parse"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { adaptSchemaForStrict, NO_STRICT } from "../utils/schema"; import { CODEX_BASE_URL, @@ -1551,14 +1550,14 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex if (!msg.content || msg.content.trim() === "") continue; messages.push({ role: "user", - content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }], + content: [{ type: "input_text", text: msg.content.toWellFormed() }], }); } else { const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => { if (item.type === "text") { return { type: "input_text", - text: sanitizeSurrogates(item.text), + text: item.text.toWellFormed(), } satisfies ResponseInputText; } return { @@ -1588,14 +1587,14 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex if (!msg.content || msg.content.trim() === "") continue; messages.push({ role: "developer", - content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }], + content: [{ type: "input_text", text: msg.content.toWellFormed() }], }); } else { const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => { if (item.type === "text") { return { type: "input_text", - text: sanitizeSurrogates(item.text), + text: item.text.toWellFormed(), } satisfies ResponseInputText; } return { @@ -1639,7 +1638,7 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex output.push({ type: "message", role: "assistant", - content: [{ type: "output_text", text: sanitizeSurrogates(textBlock.text), annotations: [] }], + content: [{ type: "output_text", text: textBlock.text.toWellFormed(), annotations: [] }], status: "completed", id: msgId, } satisfies ResponseOutputMessage); @@ -1669,7 +1668,7 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex messages.push({ type: "function_call_output", call_id: normalized.callId, - output: sanitizeSurrogates(hasText ? textResult : "(see attached image)"), + output: (hasText ? textResult : "(see attached image)").toWellFormed(), }); if (hasImages && model.input.includes("image")) { diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 0ec5c9440..bfec2f9ce 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -31,7 +31,6 @@ import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; import { parseStreamingJson } from "../utils/json-parse"; import { getKimiCommonHeaders } from "../utils/oauth/kimi"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { adaptSchemaForStrict, NO_STRICT } from "../utils/schema"; import { mapToOpenAICompletionsToolChoice } from "../utils/tool-choice"; import { @@ -719,7 +718,7 @@ export function convertMessages( if (context.systemPrompt) { const useDeveloperRole = model.reasoning && compat.supportsDeveloperRole; const role = useDeveloperRole ? "developer" : "system"; - params.push({ role: role, content: sanitizeSurrogates(context.systemPrompt) }); + params.push({ role: role, content: context.systemPrompt.toWellFormed() }); } let lastRole: string | null = null; @@ -743,7 +742,7 @@ export function convertMessages( if (msg.role === "user" || msg.role === "developer") { const role = !devAsUser && msg.role === "developer" ? "developer" : "user"; if (typeof msg.content === "string") { - const text = sanitizeSurrogates(msg.content); + const text = msg.content.toWellFormed(); if (text.trim().length === 0) continue; params.push({ role: role, @@ -753,7 +752,7 @@ export function convertMessages( const content: ChatCompletionContentPart[] = []; for (const item of msg.content) { if (item.type === "text") { - const text = sanitizeSurrogates(item.text); + const text = item.text.toWellFormed(); if (text.trim().length === 0) continue; content.push({ type: "text", @@ -791,10 +790,10 @@ export function convertMessages( // GitHub Copilot requires assistant content as a string, not an array. // Sending as array causes Claude models to re-answer all previous prompts. if (model.provider === "github-copilot") { - assistantMsg.content = nonEmptyTextBlocks.map(b => sanitizeSurrogates(b.text)).join(""); + assistantMsg.content = nonEmptyTextBlocks.map(b => b.text.toWellFormed()).join(""); } else { assistantMsg.content = nonEmptyTextBlocks.map(b => { - return { type: "text", text: sanitizeSurrogates(b.text) }; + return { type: "text", text: b.text.toWellFormed() }; }); } } @@ -918,7 +917,7 @@ export function convertMessages( remappedToolCallId ?? ensureToolCallId(toolMsg.toolCallId, `${j}:${toolMsg.toolName ?? "tool"}`); const toolResultMsg: ChatCompletionToolMessageParam = { role: "tool", - content: sanitizeSurrogates(hasText ? textResult : "(see attached image)"), + content: (hasText ? textResult : "(see attached image)").toWellFormed(), tool_call_id: normalizeMistralToolId(resolvedToolCallId, compat.requiresMistralToolIds), }; if (compat.requiresToolResultName && toolMsg.toolName) { diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index 5719de3f0..60af103c2 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -32,7 +32,6 @@ import { normalizeResponsesToolCallId, resolveCacheRetention } from "../utils"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; import { parseStreamingJson } from "../utils/json-parse"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { adaptSchemaForStrict, NO_STRICT } from "../utils/schema"; import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice"; import { @@ -539,7 +538,7 @@ function convertMessages( const role = model.reasoning ? "developer" : "system"; messages.push({ role, - content: sanitizeSurrogates(context.systemPrompt), + content: context.systemPrompt.toWellFormed(), }); } @@ -551,14 +550,14 @@ function convertMessages( if (!msg.content || msg.content.trim() === "") continue; messages.push({ role: "user", - content: [{ type: "input_text", text: sanitizeSurrogates(msg.content) }], + content: [{ type: "input_text", text: msg.content.toWellFormed() }], }); } else { const content: ResponseInputContent[] = msg.content.map((item): ResponseInputContent => { if (item.type === "text") { return { type: "input_text", - text: sanitizeSurrogates(item.text), + text: item.text.toWellFormed(), } satisfies ResponseInputText; } else { return { @@ -615,7 +614,7 @@ function convertMessages( output.push({ type: "message", role: "assistant", - content: [{ type: "output_text", text: sanitizeSurrogates(textBlock.text), annotations: [] }], + content: [{ type: "output_text", text: textBlock.text.toWellFormed(), annotations: [] }], status: "completed", id: msgId, } satisfies ResponseOutputMessage); @@ -660,7 +659,7 @@ function convertMessages( messages.push({ type: "function_call_output", call_id: normalized.callId, - output: sanitizeSurrogates(hasText ? textResult : "(see attached image)"), + output: (hasText ? textResult : "(see attached image)").toWellFormed(), }); // If there are images and model supports them, send a follow-up user message with images diff --git a/packages/ai/src/utils/sanitize-unicode.ts b/packages/ai/src/utils/sanitize-unicode.ts deleted file mode 100644 index d869ee9dc..000000000 --- a/packages/ai/src/utils/sanitize-unicode.ts +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Removes unpaired Unicode surrogate characters from a string. - * - * Unpaired surrogates (high surrogates 0xD800-0xDBFF without matching low surrogates 0xDC00-0xDFFF, - * or vice versa) cause JSON serialization errors in many API providers. - * - * Valid emoji and other characters outside the Basic Multilingual Plane use properly paired - * surrogates and will NOT be affected by this function. - * - * @param text - The text to sanitize - * @returns The sanitized text with unpaired surrogates removed - * - * @example - * // Valid emoji (properly paired surrogates) are preserved - * sanitizeSurrogates("Hello 🙈 World") // => "Hello 🙈 World" - * - * // Unpaired high surrogate is removed - * const unpaired = String.fromCharCode(0xD83D); // high surrogate without low - * sanitizeSurrogates(`Text ${unpaired} here`) // => "Text here" - */ -export function sanitizeSurrogates(text: string): string { - // Replace unpaired high surrogates (0xD800-0xDBFF not followed by low surrogate) - // Replace unpaired low surrogates (0xDC00-0xDFFF not preceded by high surrogate) - return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?> | string[][] | undefined, + name: string, +): string | undefined { if (!headers) return undefined; const target = name.toLowerCase();