can1357
9324fc4432
- Fixed Vim path normalization to allow colon-prefixed targets without raising ToolError. - Documented the new Vim path behavior in the package changelog. - Updated chunk-edit benchmark prompts to read `test.rs` for edit and retry operations. - Updated vim benchmark flow to use `vim`+`read`, import expected content, and display final expected output. - Expanded benchmark fixtures to generate Rust `test.rs`, compute unified diffs, and add richer pool logic.
37 lines
1.0 KiB
Python
Executable File
37 lines
1.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Chunk edit benchmark: tests chunk-mode edit tool usage across models with a simple edit task.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from edit_benchmark_common import BenchmarkSpec, EDIT_DIFF, EXPECTED_CONTENT, run_benchmark_main
|
|
|
|
EDIT_PROMPT = f"""\
|
|
Use the `read` tool to inspect `test.rs`, then use the `edit` tool in chunk mode to make `test.rs` exactly match the requested change.
|
|
|
|
Apply this diff:
|
|
```diff
|
|
{EDIT_DIFF}```
|
|
|
|
Final expected file content:
|
|
```python
|
|
{EXPECTED_CONTENT}```
|
|
"""
|
|
|
|
CHUNK_BENCHMARK = BenchmarkSpec(
|
|
description="Benchmark chunk-mode edit tool across models with simple edit tasks.",
|
|
workspace_prefix="chunk-benchmark",
|
|
tools=("edit", "read"),
|
|
env={"PI_EDIT_VARIANT": "chunk", "PI_STRICT_EDIT_MODE": "1"},
|
|
initial_prompt=EDIT_PROMPT,
|
|
retry_instruction='Use `read(path="test.rs")` to refresh chunk selectors if needed, then try again using the edit tool.',
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
return run_benchmark_main(CHUNK_BENCHMARK)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|