fix(ai): drop image content for dashscope compatible-mode text-only qwen
DashScope's consumer compatible-mode endpoint (dashscope.aliyuncs.com/compatible-mode/v1) rejects multipart message content with '400 InternalError.Algo.InvalidParameter: The provided messages input is invalid. The error info is [Unexpected item type in content.]' whenever a non-vision Qwen id (qwen-max, qwen3.7-max, qwen3-coder-*, ...) is asked to consume an image_url part. The agent hit this whenever a screenshot or other image-bearing tool result was batched into the synthetic user turn convertMessages emits between successive turns. Add isDashscopeCompatibleModeTextOnlyQwen in vision-guard.ts and gate both supportsImages checks in convertMessages on it, so a misconfigured custom provider claiming input: ["text", "image"] for a text-only Qwen on that endpoint falls back to the standard [image omitted: model does not support vision] placeholder. The coding-intl.dashscope.aliyuncs.com coding-plan endpoint and explicit qwen-vl-* / qwen-omni-* / qwen-audio-* ids are unaffected. Fixes #1859
This commit is contained in:
roboomp
@@ -13,6 +13,7 @@
|
||||
- Fixed Xiaomi Token Plan MiMo OpenAI-compatible tool-call continuations omitting required `reasoning_content` replay. ([#1846](https://github.com/can1357/oh-my-pi/issues/1846))
|
||||
- Fixed Anthropic prompt caching for OpenAI-compatible Claude proxies by honoring `compat.cacheControlFormat: "anthropic"` outside OpenRouter. ([#1845](https://github.com/can1357/oh-my-pi/issues/1845))
|
||||
- Fixed Moonshot Kimi K2.6 silently pausing for many seconds between tool calls because the server discarded the `reasoning_content` that omp was already sending with every assistant tool-call replay. The K2.6 `thinking` parameter takes an extra `keep` field whose default (`null`) ignores historical reasoning, so K2.6 had to re-derive its full chain-of-thought from the user prompt on every iteration of the agent loop. The Moonshot direct (`api.moonshot.ai`) and Kimi Code (`api.kimi.com`) wire bodies now send `thinking: { type: "enabled", keep: "all" }` for `kimi-k2.6` requests with reasoning enabled, matching Moonshot's documented best practice for multi-step tool-calling agents. The flag is gated on the K2.6 id and the two native hosts because earlier Moonshot models (K2.5 and below) 400 on the unknown field and every Kimi gateway (OpenRouter, OpenCode, Kilo, Fireworks, …) speaks its own thinking shape. ([#1838](https://github.com/can1357/oh-my-pi/issues/1838))
|
||||
- Fixed Alibaba DashScope (Bailian) compatible-mode endpoint `400 InternalError.Algo.InvalidParameter: The provided messages input is invalid. The error info is [Unexpected item type in content.]` when a screenshot or other image-producing tool result was folded into a text-only Qwen turn (e.g. `qwen3.7-max`, `qwen-max`, `qwen3-coder-*`) hosted at `dashscope.aliyuncs.com/compatible-mode/v1`. `convertMessages` in `openai-completions` no longer forwards `image_url` content parts for non-`vl` / non-`omni` / non-`audio` Qwen ids on that endpoint even when a misconfigured custom provider claims `input: ["text", "image"]`; the tool-result branch and the user-content branch both fall back to the standard `[image omitted: model does not support vision]` placeholder so the model still sees the attachment intent. The `coding-intl.dashscope.aliyuncs.com` coding-plan endpoint (whose `qwen3.5-plus`/`qwen3.6-plus` SKUs are genuinely multimodal) is unaffected. ([#1859](https://github.com/can1357/oh-my-pi/issues/1859))
|
||||
|
||||
## [15.9.0] - 2026-06-04
|
||||
|
||||
|
||||
@@ -72,7 +72,11 @@ import {
|
||||
import { detectOpenAICompat, type ResolvedOpenAICompat, resolveOpenAICompat } from "./openai-completions-compat";
|
||||
import { createInitialResponsesAssistantMessage } from "./openai-responses-shared";
|
||||
import { transformMessages } from "./transform-messages";
|
||||
import { joinTextWithImagePlaceholder, NON_VISION_IMAGE_PLACEHOLDER } from "./vision-guard";
|
||||
import {
|
||||
isDashscopeCompatibleModeTextOnlyQwen,
|
||||
joinTextWithImagePlaceholder,
|
||||
NON_VISION_IMAGE_PLACEHOLDER,
|
||||
} from "./vision-guard";
|
||||
|
||||
/**
|
||||
* Normalize tool call ID for Mistral.
|
||||
@@ -1587,7 +1591,7 @@ export function convertMessages(
|
||||
content: text,
|
||||
});
|
||||
} else {
|
||||
const supportsImages = model.input.includes("image");
|
||||
const supportsImages = model.input.includes("image") && !isDashscopeCompatibleModeTextOnlyQwen(model);
|
||||
const content: ChatCompletionContentPart[] = [];
|
||||
let omittedImages = false;
|
||||
for (const item of msg.content) {
|
||||
@@ -1830,7 +1834,7 @@ export function convertMessages(
|
||||
.filter(c => c.type === "text")
|
||||
.map(c => (c as TextContent).text)
|
||||
.join("\n");
|
||||
const supportsImages = model.input.includes("image");
|
||||
const supportsImages = model.input.includes("image") && !isDashscopeCompatibleModeTextOnlyQwen(model);
|
||||
const hasImages = toolMsg.content.some(c => c.type === "image");
|
||||
const omittedImages = hasImages && !supportsImages;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { ImageContent, TextContent } from "../types";
|
||||
import type { ImageContent, Model, TextContent } from "../types";
|
||||
|
||||
export const NON_VISION_IMAGE_PLACEHOLDER = "[image omitted: model does not support vision]";
|
||||
|
||||
@@ -29,3 +29,28 @@ export function joinTextWithImagePlaceholder(text: string, omittedImages: boolea
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect Qwen models served via Alibaba DashScope's consumer
|
||||
* `compatible-mode` endpoint that the upstream chat-completions API rejects
|
||||
* multimodal content arrays for. Vision support on that endpoint is gated by
|
||||
* id — only `-vl-` / `-omni-` / `-audio-` variants accept `image_url` parts;
|
||||
* everything else (text-only `qwen-max`, `qwen3.7-max`, `qwen3-coder-*`, …)
|
||||
* 400s with `Unexpected item type in content.` when sent an image.
|
||||
*
|
||||
* The check intentionally ignores the `coding-intl.dashscope.aliyuncs.com`
|
||||
* coding-plan endpoint — its SKUs (e.g. `qwen3.5-plus`, `qwen3.6-plus`) are
|
||||
* genuinely multimodal and rely on the catalog `input` field.
|
||||
*
|
||||
* Used as a defensive override in `convertMessages` so a misconfigured custom
|
||||
* provider (issue #1859) can't drive the request into an unrecoverable 400.
|
||||
*/
|
||||
export function isDashscopeCompatibleModeTextOnlyQwen(model: Model<"openai-completions">): boolean {
|
||||
const baseUrl = model.baseUrl.toLowerCase();
|
||||
if (!baseUrl.includes("dashscope") || !baseUrl.includes("aliyuncs.com") || !baseUrl.includes("/compatible-mode")) {
|
||||
return false;
|
||||
}
|
||||
const id = model.id.toLowerCase();
|
||||
if (!id.includes("qwen")) return false;
|
||||
return !/-(?:vl|omni|audio)\b/.test(id);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from "bun:test";
|
||||
import { getBundledModel } from "@oh-my-pi/pi-ai/models";
|
||||
import { convertMessages } from "@oh-my-pi/pi-ai/providers/openai-completions";
|
||||
import type { ResolvedOpenAICompat } from "@oh-my-pi/pi-ai/providers/openai-completions-compat";
|
||||
import { NON_VISION_IMAGE_PLACEHOLDER } from "@oh-my-pi/pi-ai/providers/vision-guard";
|
||||
import type { AssistantMessage, Context, Model, ToolResultMessage, Usage } from "@oh-my-pi/pi-ai/types";
|
||||
|
||||
const emptyUsage: Usage = {
|
||||
@@ -229,4 +230,128 @@ describe("openai-completions convertMessages", () => {
|
||||
const serializedArgs = assistantParam!.tool_calls![0].function.arguments;
|
||||
expect(JSON.parse(serializedArgs)).toEqual({ path: "README.md" });
|
||||
});
|
||||
it("strips image_url content for DashScope compatible-mode text-only Qwen models (issue #1859)", () => {
|
||||
// Reproduces the bailian/qwen3.7-max + dashscope.aliyuncs.com/compatible-mode
|
||||
// 400 from issue #1859: even when a misconfigured `model.input` claims
|
||||
// image support, sending an `image_url` part to a non-VL/Omni Qwen on
|
||||
// the consumer DashScope endpoint server-errors with
|
||||
// "Unexpected item type in content.". Verify both the synthesized user
|
||||
// turn (text-only message) and the tool-result image batching path drop
|
||||
// images for that combination, substituting the standard placeholder.
|
||||
const baseModel = getBundledModel("openai", "gpt-4o-mini") as Model<"openai-completions">;
|
||||
const model: Model<"openai-completions"> = {
|
||||
...baseModel,
|
||||
id: "qwen3.7-max",
|
||||
provider: "bailian" as Model<"openai-completions">["provider"],
|
||||
baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
api: "openai-completions",
|
||||
// Wrong-but-realistic user config: claims vision capability even
|
||||
// though qwen3.7-max on this endpoint is text-only upstream.
|
||||
input: ["text", "image"],
|
||||
};
|
||||
|
||||
const now = Date.now();
|
||||
const assistantMessage: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "toolCall", id: "tool-1", name: "browser", arguments: { action: "screenshot" } }],
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: emptyUsage,
|
||||
stopReason: "toolUse",
|
||||
timestamp: now,
|
||||
};
|
||||
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Take a screenshot" },
|
||||
{ type: "image", data: "ZmFrZQ==", mimeType: "image/webp" },
|
||||
],
|
||||
timestamp: now - 2,
|
||||
},
|
||||
assistantMessage,
|
||||
buildToolResult("tool-1", now + 1),
|
||||
],
|
||||
};
|
||||
|
||||
const messages = convertMessages(model, context, compat);
|
||||
|
||||
// No converted turn carries an image_url part — the dashscope guard
|
||||
// must veto both the user-content and tool-result-batching emit paths.
|
||||
for (const m of messages) {
|
||||
if (Array.isArray(m.content)) {
|
||||
for (const part of m.content as Array<{ type?: string }>) {
|
||||
expect(part?.type).not.toBe("image_url");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// User content with the dropped image keeps the text and appends the
|
||||
// standard "image omitted" placeholder so the model still sees that an
|
||||
// attachment existed.
|
||||
const userMessage = messages.find(m => m.role === "user");
|
||||
expect(userMessage).toBeDefined();
|
||||
const userContent = userMessage?.content;
|
||||
const userText =
|
||||
typeof userContent === "string"
|
||||
? userContent
|
||||
: (userContent as Array<{ type?: string; text?: string }>)
|
||||
.filter(p => p?.type === "text")
|
||||
.map(p => p?.text ?? "")
|
||||
.join("\n");
|
||||
expect(userText).toContain("Take a screenshot");
|
||||
expect(userText).toContain(NON_VISION_IMAGE_PLACEHOLDER);
|
||||
|
||||
// Tool-result image is folded into the tool text content with the same
|
||||
// placeholder rather than emitted as a follow-up multimodal user turn.
|
||||
const toolMessage = messages.find(m => m.role === "tool");
|
||||
expect(toolMessage).toBeDefined();
|
||||
expect(typeof toolMessage?.content).toBe("string");
|
||||
expect(toolMessage?.content as string).toContain(NON_VISION_IMAGE_PLACEHOLDER);
|
||||
});
|
||||
|
||||
it("preserves image_url for DashScope compatible-mode VL/Omni Qwen models", () => {
|
||||
// Counter-case for the issue #1859 guard: qwen-vl-max on the same
|
||||
// endpoint IS genuinely vision-capable, so the dashscope-text-only
|
||||
// override must NOT kick in there.
|
||||
const baseModel = getBundledModel("openai", "gpt-4o-mini") as Model<"openai-completions">;
|
||||
const model: Model<"openai-completions"> = {
|
||||
...baseModel,
|
||||
id: "qwen-vl-max",
|
||||
provider: "bailian" as Model<"openai-completions">["provider"],
|
||||
baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
api: "openai-completions",
|
||||
input: ["text", "image"],
|
||||
};
|
||||
|
||||
const now = Date.now();
|
||||
const assistantMessage: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "toolCall", id: "tool-1", name: "read", arguments: { path: "img.png" } }],
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: emptyUsage,
|
||||
stopReason: "toolUse",
|
||||
timestamp: now,
|
||||
};
|
||||
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{ role: "user", content: "Read the image", timestamp: now - 2 },
|
||||
assistantMessage,
|
||||
buildToolResult("tool-1", now + 1),
|
||||
],
|
||||
};
|
||||
|
||||
const messages = convertMessages(model, context, compat);
|
||||
const trailingUser = messages[messages.length - 1];
|
||||
expect(trailingUser.role).toBe("user");
|
||||
expect(Array.isArray(trailingUser.content)).toBe(true);
|
||||
const imageParts = (trailingUser.content as Array<{ type?: string }>).filter(p => p?.type === "image_url");
|
||||
expect(imageParts.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user