fix(coding-agent): split mixed file mentions so text stays on developer
Reviewer caught that demoting the whole mixed payload to `user` (`@notes.md @screenshot.png`) regressed the developer-priority treatment text-only mentions still get for image-free turns. `generateFileMentionMessages` packs every `@…` into one `fileMention`, so the previous `hasImage` toggle collapsed the source-file context into the user slot whenever an image was attached. `convertToLlm` now returns up to two messages per `fileMention` via `flatMap`: text-only files keep their existing `developer` envelope, and image-bearing files emit a separate `user` envelope that carries their `<file>` wrappers plus the `input_image` block. Pure-text and pure-image turns still collapse to a single message. Tests cover the mixed case (split into developer + user), the image-only case (single user message), and the existing text-only case (single developer message). Fixes #3443
This commit is contained in:
roboomp
@@ -5,7 +5,7 @@
|
||||
### Fixed
|
||||
|
||||
- Fixed `skill://` tool resolution losing loaded session skills when a tool runs outside the session-initialization module state. Internal URL resolution now prefers the caller's `session.skills` snapshot before falling back to the process-global skill list, so `read skill://<name>` works across tool execution boundaries. ([#3436](https://github.com/can1357/oh-my-pi/issues/3436))
|
||||
- Fixed `@image` mentions on OpenAI Codex Responses (chatgpt.com `gpt-5.5` and siblings) failing with `Codex error event: [OneOfParam] [input[N].content[…]] [invalid_enum_value] Invalid value: 'input_image'. Supported values are: 'input_text'.`. `convertToLlm` for `fileMention` always emitted a `developer`-role message, so the auto-attached image landed in a Responses content array that the Codex backend (and OpenAI Responses generally) only allows to carry `input_text`. #3421's previous fix only stopped the Codex Responses Lite header from going out on image-bearing turns; the full transport kept rejecting the same payload. Image-bearing file mentions now ride as `user` (text-only mentions keep `developer` to preserve the existing instruction-priority intent). ([#3443](https://github.com/can1357/oh-my-pi/issues/3443))
|
||||
- Fixed `@image` mentions on OpenAI Codex Responses (chatgpt.com `gpt-5.5` and siblings) failing with `Codex error event: [OneOfParam] [input[N].content[…]] [invalid_enum_value] Invalid value: 'input_image'. Supported values are: 'input_text'.`. `convertToLlm` for `fileMention` always emitted a `developer`-role message, so the auto-attached image landed in a Responses content array that the Codex backend (and OpenAI Responses generally) only allows to carry `input_text`. #3421's previous fix only stopped the Codex Responses Lite header from going out on image-bearing turns; the full transport kept rejecting the same payload. `convertToLlm` now splits a mixed-content `fileMention` into two messages — text-only files stay on `developer` (so the auto-read context keeps instruction priority), while image-bearing files ride on `user` (the only Responses content slot that accepts `input_image`). ([#3443](https://github.com/can1357/oh-my-pi/issues/3443))
|
||||
|
||||
## [16.1.18] - 2026-06-25
|
||||
|
||||
|
||||
@@ -482,72 +482,88 @@ export function sanitizeRehydratedOpenAIResponsesAssistantMessage(message: Assis
|
||||
* - Custom extensions and tools
|
||||
*/
|
||||
export function convertToLlm(messages: AgentMessage[]): Message[] {
|
||||
return messages
|
||||
.map((m): Message | undefined => {
|
||||
switch (m.role) {
|
||||
case "bashExecution":
|
||||
if (m.excludeFromContext) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
return messages.flatMap((m): Message[] => {
|
||||
switch (m.role) {
|
||||
case "bashExecution":
|
||||
if (m.excludeFromContext) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: bashExecutionToText(m) }],
|
||||
attribution: "user",
|
||||
timestamp: m.timestamp,
|
||||
};
|
||||
case "pythonExecution":
|
||||
if (m.excludeFromContext) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
},
|
||||
];
|
||||
case "pythonExecution":
|
||||
if (m.excludeFromContext) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: pythonExecutionToText(m) }],
|
||||
attribution: "user",
|
||||
timestamp: m.timestamp,
|
||||
};
|
||||
case "fileMention": {
|
||||
const fileContents = m.files
|
||||
.map(file => {
|
||||
const inner = file.content ? `\n${file.content}\n` : "\n";
|
||||
return `<file path="${file.path}">${inner}</file>`;
|
||||
})
|
||||
.join("\n");
|
||||
const content: (TextContent | ImageContent)[] = [{ type: "text" as const, text: fileContents }];
|
||||
let hasImage = false;
|
||||
for (const file of m.files) {
|
||||
if (file.image) {
|
||||
content.push(file.image);
|
||||
hasImage = true;
|
||||
}
|
||||
},
|
||||
];
|
||||
case "fileMention": {
|
||||
// One `fileMention` can mix `@notes.md` (text) and `@screenshot.png` (image)
|
||||
// in the same turn (`generateFileMentionMessages` packs every `@…` into a
|
||||
// single message). Splitting by image presence keeps text-only mentions on
|
||||
// the higher-priority `developer` slot while routing image attachments
|
||||
// through `user`, the only Responses content slot that legitimately accepts
|
||||
// `input_image` (Codex chatgpt.com /codex/responses rejects everything else
|
||||
// with `Invalid value: 'input_image'`, #3443).
|
||||
const wrap = (file: FileMentionMessage["files"][number]): string => {
|
||||
const inner = file.content ? `\n${file.content}\n` : "\n";
|
||||
return `<file path="${file.path}">${inner}</file>`;
|
||||
};
|
||||
const textFiles = m.files.filter(file => !file.image);
|
||||
const imageFiles = m.files.filter(file => file.image);
|
||||
const out: Message[] = [];
|
||||
if (textFiles.length > 0) {
|
||||
out.push({
|
||||
role: "developer",
|
||||
content: [{ type: "text" as const, text: textFiles.map(wrap).join("\n") }],
|
||||
attribution: "user",
|
||||
timestamp: m.timestamp,
|
||||
});
|
||||
}
|
||||
if (imageFiles.length > 0) {
|
||||
const content: (TextContent | ImageContent)[] = [
|
||||
{ type: "text" as const, text: imageFiles.map(wrap).join("\n") },
|
||||
];
|
||||
for (const file of imageFiles) {
|
||||
if (file.image) content.push(file.image);
|
||||
}
|
||||
// Image-bearing mentions must ride as `user`: developer/system
|
||||
// messages only accept text on OpenAI Responses + Codex; sending
|
||||
// an `input_image` in a developer-role content array gets
|
||||
// rejected with `Invalid value: 'input_image'. Supported values
|
||||
// are: 'input_text'.` (Codex chatgpt.com backend, #3443).
|
||||
return {
|
||||
role: hasImage ? "user" : "developer",
|
||||
out.push({
|
||||
role: "user",
|
||||
content,
|
||||
attribution: "user",
|
||||
timestamp: m.timestamp,
|
||||
};
|
||||
});
|
||||
}
|
||||
case "custom":
|
||||
case "hookMessage":
|
||||
case "branchSummary":
|
||||
case "compactionSummary":
|
||||
case "user":
|
||||
case "developer":
|
||||
case "assistant":
|
||||
case "toolResult":
|
||||
// Core roles share one transformer with agent-core —
|
||||
// duplicating them here is how snapcompact frames once
|
||||
// silently fell off the provider request.
|
||||
return convertMessageToLlm(m);
|
||||
default:
|
||||
m satisfies never;
|
||||
return undefined;
|
||||
return out;
|
||||
}
|
||||
})
|
||||
.filter(m => m !== undefined);
|
||||
case "custom":
|
||||
case "hookMessage":
|
||||
case "branchSummary":
|
||||
case "compactionSummary":
|
||||
case "user":
|
||||
case "developer":
|
||||
case "assistant":
|
||||
case "toolResult": {
|
||||
// Core roles share one transformer with agent-core —
|
||||
// duplicating them here is how snapcompact frames once
|
||||
// silently fell off the provider request.
|
||||
const converted = convertMessageToLlm(m);
|
||||
return converted ? [converted] : [];
|
||||
}
|
||||
default:
|
||||
m satisfies never;
|
||||
return [];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -136,12 +136,13 @@ describe("convertToLlm custom message mapping", () => {
|
||||
expect(text).toContain("export const config = {};");
|
||||
});
|
||||
|
||||
it("routes image-bearing file mentions through user role to satisfy Codex schema (#3443)", () => {
|
||||
// `developer` (and `system`) Responses messages reject `input_image`
|
||||
// with `Invalid value: 'input_image'. Supported values are: 'input_text'.`
|
||||
// Issue #3421's earlier fix only disabled Codex Responses Lite for image
|
||||
// turns; the role-promotion here closes the bug at the source so every
|
||||
// Responses-family provider accepts attached images.
|
||||
it("splits mixed text + image file mentions into developer + user messages (#3443)", () => {
|
||||
// `developer` (and `system`) Responses messages reject `input_image` with
|
||||
// `Invalid value: 'input_image'. Supported values are: 'input_text'.`
|
||||
// `generateFileMentionMessages` packs every `@…` into one `fileMention`,
|
||||
// so a `@notes.md @diagram.png` turn would have demoted the text payload
|
||||
// to `user` (losing the instruction-priority intent) before #3443; now the
|
||||
// text-only file stays on `developer` and only the image file rides as `user`.
|
||||
const image: ImageContent = { type: "image", data: "aGVsbG8=", mimeType: "image/png" };
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
@@ -156,17 +157,50 @@ describe("convertToLlm custom message mapping", () => {
|
||||
|
||||
const converted = convertToLlm(messages);
|
||||
|
||||
expect(converted).toHaveLength(2);
|
||||
|
||||
const dev = converted[0];
|
||||
expect(dev?.role).toBe("developer");
|
||||
expectAttribution(dev, "user");
|
||||
if (dev?.role !== "developer" || !Array.isArray(dev.content)) {
|
||||
throw new Error("Expected developer array content for text mention");
|
||||
}
|
||||
const devText = dev.content.find(content => content.type === "text")?.text ?? "";
|
||||
expect(devText).toContain('<file path="notes/log.txt">');
|
||||
expect(devText).toContain("alpha");
|
||||
expect(devText).not.toContain('<file path="diagram.png">');
|
||||
expect(dev.content.some(content => content.type === "image")).toBe(false);
|
||||
|
||||
const user = converted[1];
|
||||
expect(user?.role).toBe("user");
|
||||
expectAttribution(user, "user");
|
||||
if (user?.role !== "user" || !Array.isArray(user.content)) {
|
||||
throw new Error("Expected user array content for image mention");
|
||||
}
|
||||
const userText = user.content.find(content => content.type === "text")?.text ?? "";
|
||||
expect(userText).toContain('<file path="diagram.png">');
|
||||
expect(userText).not.toContain('<file path="notes/log.txt">');
|
||||
expect(user.content.filter(content => content.type === "image")).toEqual([image]);
|
||||
});
|
||||
|
||||
it("emits a user-only message when every mention is an image (#3443)", () => {
|
||||
const image: ImageContent = { type: "image", data: "aGVsbG8=", mimeType: "image/png" };
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "fileMention",
|
||||
files: [{ path: "screenshot.png", content: "", image }],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
];
|
||||
|
||||
const converted = convertToLlm(messages);
|
||||
|
||||
expect(converted).toHaveLength(1);
|
||||
expect(converted[0]?.role).toBe("user");
|
||||
expectAttribution(converted[0], "user");
|
||||
if (converted[0]?.role !== "user" || !Array.isArray(converted[0].content)) {
|
||||
throw new Error("Expected user array content");
|
||||
}
|
||||
const text = converted[0].content.find(content => content.type === "text")?.text ?? "";
|
||||
expect(text).toContain('<file path="notes/log.txt">');
|
||||
expect(text).toContain('<file path="diagram.png">');
|
||||
const images = converted[0].content.filter(content => content.type === "image") as ImageContent[];
|
||||
expect(images).toEqual([image]);
|
||||
expect(converted[0].content.filter(content => content.type === "image")).toEqual([image]);
|
||||
});
|
||||
|
||||
it("allows custom messages to opt into user attribution", () => {
|
||||
|
||||
Reference in New Issue
Block a user