From d7c64728245cd0a0ae3cae6900ee187be33f7cb7 Mon Sep 17 00:00:00 2001 From: roboomp Date: Tue, 30 Jun 2026 05:48:37 +0000 Subject: [PATCH 1/5] fix(edit): preserved hashline utf-8 bom Added binary BOM detection to the hashline patcher and filesystem adapters so edits restore BOM bytes when text decoding hides U+FEFF. Fixes #3867 --- packages/coding-agent/CHANGELOG.md | 4 +++ .../src/edit/hashline/filesystem.ts | 10 ++++++++ .../coding-agent/test/core/hashline.test.ts | 17 +++++++++++++ packages/hashline/CHANGELOG.md | 4 +++ packages/hashline/src/fs.ts | 12 +++++++++ packages/hashline/src/patcher.ts | 13 +++++++++- packages/hashline/test/patcher.test.ts | 25 +++++++++++++++++++ 7 files changed, 84 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index c44e8b2da..e4922c2a0 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed hashline edit mode preserving UTF-8 BOM bytes on edited files. ([#3867](https://github.com/can1357/oh-my-pi/issues/3867)) + ## [16.2.7] - 2026-06-30 ### Breaking Changes diff --git a/packages/coding-agent/src/edit/hashline/filesystem.ts b/packages/coding-agent/src/edit/hashline/filesystem.ts index 63b9ffd17..765854275 100644 --- a/packages/coding-agent/src/edit/hashline/filesystem.ts +++ b/packages/coding-agent/src/edit/hashline/filesystem.ts @@ -123,6 +123,16 @@ export class HashlineFilesystem extends Filesystem { return content; } + async readBinary(relativePath: string): Promise { + const absolutePath = this.resolveAbsolute(relativePath); + try { + return await fs.readFile(absolutePath); + } catch (error) { + if (isEnoent(error)) throw new NotFoundError(relativePath, error); + throw error; + } + } + async preflightWrite(relativePath: string, options?: PreflightWriteOptions): Promise { const fileOp = options?.fileOp; if (fileOp?.kind === "rem") { diff --git a/packages/coding-agent/test/core/hashline.test.ts b/packages/coding-agent/test/core/hashline.test.ts index fad0130ae..1a09e9a50 100644 --- a/packages/coding-agent/test/core/hashline.test.ts +++ b/packages/coding-agent/test/core/hashline.test.ts @@ -111,6 +111,23 @@ describe("hashline executor", () => { }); }); + it("preserves UTF-8 BOM bytes when hashline edits decoded text", async () => { + await withTempDir(async tempDir => { + const filePath = path.join(tempDir, "Program.cs"); + const source = "using A;\n"; + await Bun.write(filePath, new Uint8Array([0xef, 0xbb, 0xbf, ...new TextEncoder().encode(source)])); + const session = makeHashlineSession(tempDir); + const sourceTag = recordFullSnapshot(getFileReadCache(session), filePath, source); + const input = `${header("Program.cs", sourceTag)}\n${sameLineRange(tag(1, source))}\n${repl("using B;")}\n`; + + await executeHashlineSingle(hashlineExecuteOptions(tempDir, input, undefined, session)); + + const bytes = await fs.readFile(filePath); + expect(Array.from(bytes.subarray(0, 3))).toEqual([0xef, 0xbb, 0xbf]); + expect(new TextDecoder().decode(bytes.subarray(3))).toBe("using B;\n"); + }); + }); + it("emits an actionable no-op diagnostic when the payload matches the file byte-for-byte", async () => { await withTempDir(async tempDir => { const filePath = path.join(tempDir, "a.ts"); diff --git a/packages/hashline/CHANGELOG.md b/packages/hashline/CHANGELOG.md index bc5589f3e..f2d477c34 100644 --- a/packages/hashline/CHANGELOG.md +++ b/packages/hashline/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed hashline writes preserving UTF-8 BOM bytes when the host text decoder hides the leading `U+FEFF`. ([#3867](https://github.com/can1357/oh-my-pi/issues/3867)) + ## [16.2.6] - 2026-06-29 ### Fixed diff --git a/packages/hashline/src/fs.ts b/packages/hashline/src/fs.ts index c1ec70ad0..425e78576 100644 --- a/packages/hashline/src/fs.ts +++ b/packages/hashline/src/fs.ts @@ -65,6 +65,9 @@ export abstract class Filesystem { /** Read the file's full text content. Throw on missing file. */ abstract readText(path: string): Promise; + /** Read the file's raw bytes when text decoding may hide leading bytes such as a UTF-8 BOM. */ + readBinary?(path: string): Promise; + /** Validate that `path` is writable before a prepared batch starts committing. */ async preflightWrite(_path: string, _options?: PreflightWriteOptions): Promise {} @@ -196,6 +199,15 @@ export class NodeFilesystem extends Filesystem { return file.text(); } + async readBinary(path: string): Promise { + try { + return await fs.readFile(path); + } catch (error) { + if (isNotFound(error)) throw new NotFoundError(path, error); + throw error; + } + } + async writeText(path: string, content: string): Promise { await Bun.write(path, content); return { text: content }; diff --git a/packages/hashline/src/patcher.ts b/packages/hashline/src/patcher.ts index ba5cc9a75..d87585756 100644 --- a/packages/hashline/src/patcher.ts +++ b/packages/hashline/src/patcher.ts @@ -148,6 +148,10 @@ function mergeWarnings(...sources: ReadonlyArray) return out; } +function hasUtf8Bom(bytes: Uint8Array | undefined): boolean { + return bytes !== undefined && bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf; +} + function assertUniqueCanonicalPaths(prepared: readonly PreparedSection[]): void { const seen = new Map(); for (const entry of prepared) { @@ -295,7 +299,8 @@ export class Patcher { throw new Error(`MV destination is the same as ${target.path}.`); } - const { bom, text } = stripBom(read.rawContent); + const { bom: bomFromText, text } = stripBom(read.rawContent); + const bom = bomFromText || (await this.#readBinaryBom(target.path)); const lineEnding = detectLineEnding(text); const normalized = normalizeToLF(text); @@ -453,6 +458,12 @@ export class Patcher { }; } + async #readBinaryBom(path: string): Promise { + if (!this.fs.readBinary) return ""; + const bytes = await this.fs.readBinary(path); + return hasUtf8Bom(bytes) ? "\uFEFF" : ""; + } + async #tryRead(path: string): Promise<{ exists: boolean; rawContent: string }> { try { const content = await this.fs.readText(path); diff --git a/packages/hashline/test/patcher.test.ts b/packages/hashline/test/patcher.test.ts index e4758025e..a9ccf75ed 100644 --- a/packages/hashline/test/patcher.test.ts +++ b/packages/hashline/test/patcher.test.ts @@ -1,10 +1,15 @@ import { describe, expect, it } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; import { computeFileHash, + formatHashlineHeader, HEADTAIL_DRIFT_WARNING, InMemoryFilesystem, InMemorySnapshotStore, MismatchError, + NodeFilesystem, Patch, Patcher, } from "@oh-my-pi/hashline"; @@ -33,6 +38,26 @@ describe("Patcher snapshot tag integrity", () => { expect(fs.get(PATH)).toBe("after\n"); }); + it("restores a UTF-8 BOM hidden by Bun text decoding", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "hashline-bom-")); + try { + const filePath = path.join(tempDir, "Program.cs"); + const source = "using A;\n"; + await Bun.write(filePath, new Uint8Array([0xef, 0xbb, 0xbf, ...new TextEncoder().encode(source)])); + const snapshots = new InMemorySnapshotStore(); + const tag = snapshots.record(filePath, source); + const patch = Patch.parse([formatHashlineHeader(filePath, tag), "SWAP 1.=1:", "+using B;"].join("\n")); + + await new Patcher({ fs: new NodeFilesystem(), snapshots }).apply(patch); + + const bytes = await fs.readFile(filePath); + expect(Array.from(bytes.subarray(0, 3))).toEqual([0xef, 0xbb, 0xbf]); + expect(new TextDecoder().decode(bytes.subarray(3))).toBe("using B;\n"); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + it("validates any anchor purely from the content hash, even with no recorded snapshot", async () => { // The core fix: the tag fingerprints the WHOLE file. An edit anchored at // a line the model never saw recorded applies whenever the live file From 45d2bf86417707f532f8e136b712887ee9e3d4ac Mon Sep 17 00:00:00 2001 From: roboomp Date: Tue, 30 Jun 2026 05:49:41 +0000 Subject: [PATCH 2/5] style: bun run fix --- packages/coding-agent/test/tools/web-search-duckduckgo.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts index 4da54bddd..8c3ea24be 100644 --- a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts +++ b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts @@ -73,7 +73,7 @@ describe("DuckDuckGo web search provider", () => { const headers = capturedInit?.headers as Record; expect(headers["Content-Type"]).toBe("application/x-www-form-urlencoded"); expect(headers["User-Agent"]).toContain("Mozilla/5.0"); - expect(headers["Referer"]).toBe("https://html.duckduckgo.com/"); + expect(headers.Referer).toBe("https://html.duckduckgo.com/"); expect(headers["Accept-Language"]).toContain("en"); expect(headers["Sec-Fetch-Mode"]).toBe("navigate"); expect(headers["Sec-Ch-Ua"]).toContain("Chromium"); From d1e412eeff41f334630da6e2eca00015b6d93ff6 Mon Sep 17 00:00:00 2001 From: roboomp Date: Tue, 30 Jun 2026 05:51:53 +0000 Subject: [PATCH 3/5] Revert "style: bun run fix" This reverts commit 45d2bf86417707f532f8e136b712887ee9e3d4ac. --- packages/coding-agent/test/tools/web-search-duckduckgo.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts index 8c3ea24be..4da54bddd 100644 --- a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts +++ b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts @@ -73,7 +73,7 @@ describe("DuckDuckGo web search provider", () => { const headers = capturedInit?.headers as Record; expect(headers["Content-Type"]).toBe("application/x-www-form-urlencoded"); expect(headers["User-Agent"]).toContain("Mozilla/5.0"); - expect(headers.Referer).toBe("https://html.duckduckgo.com/"); + expect(headers["Referer"]).toBe("https://html.duckduckgo.com/"); expect(headers["Accept-Language"]).toContain("en"); expect(headers["Sec-Fetch-Mode"]).toBe("navigate"); expect(headers["Sec-Ch-Ua"]).toContain("Chromium"); From dd4eb68b378658a3eea2fca08b2485093c4464ed Mon Sep 17 00:00:00 2001 From: roboomp Date: Tue, 30 Jun 2026 05:52:02 +0000 Subject: [PATCH 4/5] style: bun run fix --- packages/coding-agent/test/tools/web-search-duckduckgo.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts index 4da54bddd..8c3ea24be 100644 --- a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts +++ b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts @@ -73,7 +73,7 @@ describe("DuckDuckGo web search provider", () => { const headers = capturedInit?.headers as Record; expect(headers["Content-Type"]).toBe("application/x-www-form-urlencoded"); expect(headers["User-Agent"]).toContain("Mozilla/5.0"); - expect(headers["Referer"]).toBe("https://html.duckduckgo.com/"); + expect(headers.Referer).toBe("https://html.duckduckgo.com/"); expect(headers["Accept-Language"]).toContain("en"); expect(headers["Sec-Fetch-Mode"]).toBe("navigate"); expect(headers["Sec-Ch-Ua"]).toContain("Chromium"); From 7ec00a50076a39e84174969847cea813d3968875 Mon Sep 17 00:00:00 2001 From: roboomp Date: Tue, 30 Jun 2026 05:58:01 +0000 Subject: [PATCH 5/5] fix(edit): skipped notebook bom sniffing Skipped binary BOM detection for notebook-backed hashline reads so virtual cell text is serialized without a leading U+FEFF marker. --- .../src/edit/hashline/filesystem.ts | 4 ++- .../coding-agent/test/core/hashline.test.ts | 33 +++++++++++++++++++ packages/hashline/src/fs.ts | 4 +-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/packages/coding-agent/src/edit/hashline/filesystem.ts b/packages/coding-agent/src/edit/hashline/filesystem.ts index 765854275..41f15842b 100644 --- a/packages/coding-agent/src/edit/hashline/filesystem.ts +++ b/packages/coding-agent/src/edit/hashline/filesystem.ts @@ -28,6 +28,7 @@ import { invalidateFsScanAfterWrite } from "../../tools/fs-cache-invalidation"; import { isInternalUrlPath } from "../../tools/path-utils"; import { enforcePlanModeWrite, resolvePlanPath, targetsLocalSandbox } from "../../tools/plan-mode-guard"; import { canonicalSnapshotKey } from "../file-snapshot-store"; +import { isNotebookPath } from "../notebook"; import { readEditFileText, serializeEditFileText } from "../read-file"; import type { LspBatchRequest } from "../renderer"; @@ -123,8 +124,9 @@ export class HashlineFilesystem extends Filesystem { return content; } - async readBinary(relativePath: string): Promise { + async readBinary(relativePath: string): Promise { const absolutePath = this.resolveAbsolute(relativePath); + if (isNotebookPath(absolutePath)) return undefined; try { return await fs.readFile(absolutePath); } catch (error) { diff --git a/packages/coding-agent/test/core/hashline.test.ts b/packages/coding-agent/test/core/hashline.test.ts index 1a09e9a50..6627351ff 100644 --- a/packages/coding-agent/test/core/hashline.test.ts +++ b/packages/coding-agent/test/core/hashline.test.ts @@ -128,6 +128,39 @@ describe("hashline executor", () => { }); }); + it("edits BOM-prefixed notebooks through the virtual cell text", async () => { + await withTempDir(async tempDir => { + const filePath = path.join(tempDir, "notebook.ipynb"); + const notebook = { + cells: [ + { + cell_type: "markdown", + metadata: { keep: true }, + source: ["# Title\n"], + }, + ], + metadata: {}, + nbformat: 4, + nbformat_minor: 5, + }; + await Bun.write( + filePath, + new Uint8Array([0xef, 0xbb, 0xbf, ...new TextEncoder().encode(JSON.stringify(notebook))]), + ); + const session = makeHashlineSession(tempDir); + const editableText = "# %% [markdown] cell:0\n# Title\n"; + const sourceTag = recordFullSnapshot(getFileReadCache(session), filePath, editableText); + const input = `${header("notebook.ipynb", sourceTag)}\n${sameLineRange(tag(2, "# Title"))}\n${repl("# Updated")}\n`; + + await executeHashlineSingle(hashlineExecuteOptions(tempDir, input, undefined, session)); + + const updated = await Bun.file(filePath).json(); + expect(updated.cells).toHaveLength(1); + expect(updated.cells[0].source).toEqual(["# Updated\n"]); + expect(updated.cells[0].metadata).toEqual({ keep: true }); + }); + }); + it("emits an actionable no-op diagnostic when the payload matches the file byte-for-byte", async () => { await withTempDir(async tempDir => { const filePath = path.join(tempDir, "a.ts"); diff --git a/packages/hashline/src/fs.ts b/packages/hashline/src/fs.ts index 425e78576..bca93496a 100644 --- a/packages/hashline/src/fs.ts +++ b/packages/hashline/src/fs.ts @@ -65,8 +65,8 @@ export abstract class Filesystem { /** Read the file's full text content. Throw on missing file. */ abstract readText(path: string): Promise; - /** Read the file's raw bytes when text decoding may hide leading bytes such as a UTF-8 BOM. */ - readBinary?(path: string): Promise; + /** Read raw bytes for backends whose text is a direct decode of persisted bytes. */ + readBinary?(path: string): Promise; /** Validate that `path` is writable before a prepared batch starts committing. */ async preflightWrite(_path: string, _options?: PreflightWriteOptions): Promise {}