feat(coding-agent): added matplotlib rendering and image persistence across session reload

- Added Matplotlib figure PNG rendering and display tracking in Python runner to emit PNG output immediately when figures are displayed via display(fig). - Extended session persistence to externalize oversized image payloads in both content and details.images, enabling tool result images to survive session reload. - Enhanced session loader to resolve image data payloads and blob references across content and details.images during session reconstruction. - Added image cache invalidation in TUI image component when image protocol, cell dimensions, or Kitty Unicode placeholder mode changes. - Added comprehensive test coverage for Matplotlib display, image persistence across reload, and TUI image rendering with protocol and dimension changes.
2026-06-17 13:19:28 +02:00
parent 6472566818
commit e4a87fa3ce
12 changed files with 723 additions and 59 deletions
@@ -6,11 +6,37 @@
 */
 import { afterEach, describe, expect, it } from "bun:test";
 import * as path from "node:path";
+import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
 import { disposeAllKernelSessions, executePythonWithKernel } from "@oh-my-pi/pi-coding-agent/eval/py/executor";
 import { PythonKernel } from "@oh-my-pi/pi-coding-agent/eval/py/kernel";
+import { filterEnv, resolvePythonRuntime } from "@oh-my-pi/pi-coding-agent/eval/py/runtime";
 import { TempDir } from "@oh-my-pi/pi-utils";

 const SHOULD_RUN = Bun.env.PI_PYTHON_INTEGRATION === "1";
+const MATPLOTLIB_TEST_CWD = process.cwd();
+
+async function hasMatplotlib(cwd: string): Promise<boolean> {
+	if (!SHOULD_RUN) return false;
+	try {
+		const { env } = (await Settings.init()).getShellConfig();
+		const runtime = resolvePythonRuntime(cwd, filterEnv(env));
+		const spawnEnv: Record<string, string> = {};
+		for (const [key, value] of Object.entries(runtime.env)) {
+			if (typeof value === "string") spawnEnv[key] = value;
+		}
+		const result = Bun.spawnSync([runtime.pythonPath, "-c", "import matplotlib"], {
+			cwd,
+			env: spawnEnv,
+			stdout: "ignore",
+			stderr: "ignore",
+		});
+		return result.exitCode === 0;
+	} catch {
+		return false;
+	}
+}
+
+const HAS_MATPLOTLIB = await hasMatplotlib(MATPLOTLIB_TEST_CWD);

 describe.skipIf(!SHOULD_RUN)("python runner subprocess", () => {
 	afterEach(async () => {
@@ -108,6 +134,51 @@ describe.skipIf(!SHOULD_RUN)("python runner subprocess", () => {
 		}
 	});

+	it.skipIf(!HAS_MATPLOTLIB)("captures display(fig) as a PNG before the figure is closed", async () => {
+		const kernel = await PythonKernel.start({ cwd: MATPLOTLIB_TEST_CWD });
+		try {
+			const result = await executePythonWithKernel(
+				kernel,
+				[
+					"import matplotlib.pyplot as plt",
+					"fig, ax = plt.subplots()",
+					"ax.plot([0, 1], [0, 1])",
+					"display(fig)",
+					"plt.close(fig)",
+				].join("\n"),
+			);
+
+			expect(result.exitCode).toBe(0);
+			const images = result.displayOutputs.filter(output => output.type === "image");
+			expect(images).toHaveLength(1);
+			expect(images[0]).toMatchObject({ mimeType: "image/png" });
+			expect(images[0]?.data).not.toContain("blob:");
+			expect(result.output).toContain("<Figure");
+		} finally {
+			await kernel.shutdown();
+		}
+	});
+
+	it.skipIf(!HAS_MATPLOTLIB)("does not flush a second PNG for a displayed open figure", async () => {
+		const kernel = await PythonKernel.start({ cwd: MATPLOTLIB_TEST_CWD });
+		try {
+			const result = await executePythonWithKernel(
+				kernel,
+				[
+					"import matplotlib.pyplot as plt",
+					"fig, ax = plt.subplots()",
+					"ax.plot([0, 1], [1, 0])",
+					"display(fig)",
+				].join("\n"),
+			);
+
+			expect(result.exitCode).toBe(0);
+			expect(result.displayOutputs.filter(output => output.type === "image")).toHaveLength(1);
+		} finally {
+			await kernel.shutdown();
+		}
+	});
+
 	it("translates %pwd magic to the user namespace", async () => {
 		using tempDir = TempDir.createSync("@python-runner-magic-");
 		const kernel = await PythonKernel.start({ cwd: tempDir.path() });