can1357
ed2fd6e3f3
- Removed the standalone vim tool and normalized built-in/requested tooling to edit. - Updated session and SDK tool activation to dedupe lowercase names and track edit state via the edit key. - Added vim-mode argument detection and delegated edit rendering/execution into Vim handlers under edit. - Updated Vim step handling to auto-reorder numeric-positioned commands, including cc/C/S/s/i/I/A cases. - Renamed prompt/changelog text and test expectations to reflect edit-only tool naming and usage.
38 lines
1004 B
Python
Executable File
38 lines
1004 B
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Vim edit-mode benchmark: tests the edit tool in vim mode across models with a simple edit task.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from edit_benchmark_common import BenchmarkSpec, EDIT_DIFF, EXPECTED_CONTENT, run_benchmark_main
|
|
|
|
|
|
EDIT_PROMPT = f"""\
|
|
Use the `read` tool to inspect `test.rs`, then use the `edit` tool in vim mode to make `test.rs` exactly match the requested change.
|
|
|
|
Apply this diff:
|
|
```diff
|
|
{EDIT_DIFF}```
|
|
|
|
Final expected file content:
|
|
```rust
|
|
{EXPECTED_CONTENT}```
|
|
"""
|
|
|
|
VIM_BENCHMARK = BenchmarkSpec(
|
|
description="Benchmark edit tool in vim mode across models with simple edit tasks.",
|
|
workspace_prefix="vim-benchmark",
|
|
tools=("edit", "read"),
|
|
env={"PI_EDIT_VARIANT": "vim", "PI_STRICT_EDIT_MODE": "1"},
|
|
initial_prompt=EDIT_PROMPT,
|
|
retry_instruction="Please try again using the edit tool in vim mode.",
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
return run_benchmark_main(VIM_BENCHMARK)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|