smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(robomp): run sandbox setup/teardown off the event loop safely

Browse Source 
Workspace setup/teardown (git clone/fetch, worktree add/remove, chown)
ran synchronously on the asyncio dispatcher loop, so one stalled
subprocess froze the entire process.

- Offload every ensure_workspace/remove_workspace call to a worker thread
  via a new _run_workspace_op helper that drains the thread to completion
  on cancellation, so a cancelled event cannot reap/release a slot the
  setup thread still owns.
- Serialize same-repo setup with a per-repo threading.RLock while letting
  distinct repos run concurrently.
- Bound the direct git/chown subprocesses with a 120s timeout
  (returncode 124); treat a timed-out branch probe as an error rather
  than "branch absent" to avoid silently rebasing a follow-up onto the
  default branch and losing the PR's commits.
- When a timed-out worktree remove leaves the checkout behind, rmtree it
  and run `git worktree prune` so the pool's dangling registration cannot
  trip a later worktree add for the same path.

Adds regression tests for event-loop liveness, cancellation-safe offload,
per-repo lock serialization, subprocess timeout mapping, the branch-probe
timeout guard, and worktree-prune after a failed remove.

Op: correct
Restores: spec:dispatcher-event-loop-never-blocks-on-workspace-io
This commit is contained in:
metaphorics
2026-07-02 07:46:36 +09:00
parent 0823892295
commit 6d16c19a04
5 changed files with 655 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/robomp/src/natives_cache.py
+1
View File
@@ -152,6 +152,7 @@ def compute_key(repo_dir: Path, *, target: str | None = None) -> str:
capture_output=True,
check=True,
env=_git_safe_directory_env(repo_dir),
timeout=120.0,
)
lines = proc.stdout.splitlines()
if len(lines) != len(CACHE_KEY_PATHS):
python/robomp/src/sandbox.py
+194 -135
View File
@@ -47,6 +47,7 @@ import shutil
import signal
import stat
import subprocess
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Protocol
@@ -298,16 +299,31 @@ class LocalGitTransport:
# ---------- low-level helpers retained for callers expecting old shape ----------
def _safe_run(cmd: list[str], *, cwd: Path | None = None, **kwargs: Any) -> subprocess.CompletedProcess[str]:
_DEFAULT_SANDBOX_SUBPROCESS_TIMEOUT = 120.0
def _safe_run(
cmd: list[str],
*,
cwd: Path | None = None,
timeout: float | None = _DEFAULT_SANDBOX_SUBPROCESS_TIMEOUT,
**kwargs: Any,
) -> subprocess.CompletedProcess[str]:
"""Run without raising; caller decides on returncode. Credentials are redacted from any captured output."""
proc = subprocess.run(
cmd,
cwd=str(cwd) if cwd else None,
check=False,
capture_output=True,
text=True,
**kwargs,
)
try:
proc = subprocess.run(
cmd,
cwd=str(cwd) if cwd else None,
check=False,
capture_output=True,
text=True,
timeout=timeout,
**kwargs,
)
except subprocess.TimeoutExpired as exc:
out = redact_credentials(exc.stdout or "") if isinstance(exc.stdout, str) else ""
err = redact_credentials(exc.stderr or "") if isinstance(exc.stderr, str) else ""
return subprocess.CompletedProcess(cmd, 124, out, f"{err}\ntimed out after {timeout:.0f}s")
if proc.stdout:
proc.stdout = redact_credentials(proc.stdout)
if proc.stderr:
@@ -315,15 +331,24 @@ def _safe_run(cmd: list[str], *, cwd: Path | None = None, **kwargs: Any) -> subp
return proc
def _run(cmd: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
def _run(
cmd: list[str],
*,
cwd: Path | None = None,
timeout: float | None = _DEFAULT_SANDBOX_SUBPROCESS_TIMEOUT,
) -> subprocess.CompletedProcess[str]:
"""Legacy raising helper (still used by a sandbox test). Forwards to subprocess.run."""
proc = subprocess.run(
cmd,
cwd=str(cwd) if cwd else None,
check=False,
capture_output=True,
text=True,
)
try:
proc = subprocess.run(
cmd,
cwd=str(cwd) if cwd else None,
check=False,
capture_output=True,
text=True,
timeout=timeout,
)
except subprocess.TimeoutExpired as exc:
raise GitCommandError(cmd, 124, "", f"git timed out after {timeout:.0f}s") from exc
if proc.returncode != 0:
raise GitCommandError(cmd, proc.returncode, proc.stdout, proc.stderr)
return proc
@@ -607,8 +632,16 @@ def _chown_workspace(ws_root: Path, slot_uid: int | None) -> None:
return
uid = slot_uid if slot_uid is not None else os.geteuid()
gid = slot_uid if slot_uid is not None else os.getegid()
subprocess.run(["chown", "-R", f"{uid}:{gid}", str(ws_root)], check=True)
subprocess.run(["chmod", "-R", "u=rwX,g=rwX,o=", str(ws_root)], check=True)
subprocess.run(
["chown", "-R", f"{uid}:{gid}", str(ws_root)],
check=True,
timeout=_DEFAULT_SANDBOX_SUBPROCESS_TIMEOUT,
)
subprocess.run(
["chmod", "-R", "u=rwX,g=rwX,o=", str(ws_root)],
check=True,
timeout=_DEFAULT_SANDBOX_SUBPROCESS_TIMEOUT,
)
# ---------- SandboxManager ----------
@@ -634,6 +667,16 @@ class SandboxManager:
self.natives_cache = natives_cache
root.mkdir(parents=True, exist_ok=True)
self.pool.mkdir(parents=True, exist_ok=True)
self._repo_locks: dict[str, threading.RLock] = {}
self._repo_locks_guard = threading.Lock()
def _repo_lock(self, repo: str) -> threading.RLock:
with self._repo_locks_guard:
lock = self._repo_locks.get(repo)
if lock is None:
lock = threading.RLock()
self._repo_locks[repo] = lock
return lock
# ---- pool ----
def pool_path(self, repo: str) -> Path:
@@ -696,121 +739,130 @@ class SandboxManager:
slot_uid: int | None = None,
) -> Workspace:
"""Create or resume a per-issue worktree."""
if pr_head is not None and existing_branch is not None:
raise ValueError("ensure_workspace accepts either pr_head or existing_branch, not both")
pool = self.ensure_clone(repo=repo, clone_url=clone_url, default_branch=default_branch)
ws_root = self.workspace_root(repo, number)
repo_dir = ws_root / "repo"
session_dir = ws_root / ".omp-session"
context_dir = ws_root / "context"
artifacts_dir = ws_root / "artifacts"
for path in (ws_root, session_dir, context_dir, context_dir / "repro", artifacts_dir):
path.mkdir(parents=True, exist_ok=True)
with self._repo_lock(repo):
if pr_head is not None and existing_branch is not None:
raise ValueError("ensure_workspace accepts either pr_head or existing_branch, not both")
pool = self.ensure_clone(repo=repo, clone_url=clone_url, default_branch=default_branch)
ws_root = self.workspace_root(repo, number)
repo_dir = ws_root / "repo"
session_dir = ws_root / ".omp-session"
context_dir = ws_root / "context"
artifacts_dir = ws_root / "artifacts"
for path in (ws_root, session_dir, context_dir, context_dir / "repro", artifacts_dir):
path.mkdir(parents=True, exist_ok=True)
branch = (
f"review/pr-{pr_head}"
if pr_head is not None
else existing_branch
or make_branch(
issue_number=number,
title=title,
seed=f"{repo}#{number}",
branch = (
f"review/pr-{pr_head}"
if pr_head is not None
else existing_branch
or make_branch(
issue_number=number,
title=title,
seed=f"{repo}#{number}",
)
)
)
repo_exists = (repo_dir / ".git").exists()
workspace_prepared = False
slot_git_kwargs = _slot_subprocess_kwargs(slot_uid)
slot_git_env: dict[str, str] | None = None
if repo_exists:
# Existing workspaces are already slot-owned from the previous run.
# Refresh pool-side group bits, then hand the tree to the current
# slot before running any git command inside the worktree; root's
# uid-0 bypass does not bypass git's safe.directory ownership check.
_share_git_metadata_with_slots(repo_dir, slot_uid)
_provision_runtime_dirs(ws_root)
_chown_workspace(ws_root, slot_uid)
workspace_prepared = True
if not repo_exists:
if pr_head is not None:
self.transport.fetch_pr_head(repo=repo, pool_dir=pool, pr_number=pr_head)
_run(["git", "worktree", "add", "--detach", str(repo_dir), "FETCH_HEAD"], cwd=pool)
else:
# Make sure the requested start point exists locally (best-effort).
# For follow-ups on an existing PR, `existing_branch` is the remote
# head branch we need to amend; starting from default would silently
# lose the PR's current commits if the local pool branch is absent.
self.transport.fetch_base_ref(repo=repo, pool_dir=pool, ref=existing_branch or default_branch)
check = _safe_run(["git", "rev-parse", "--verify", f"refs/heads/{branch}"], cwd=pool)
if check.returncode == 0:
_run(["git", "worktree", "add", str(repo_dir), branch], cwd=pool)
repo_exists = (repo_dir / ".git").exists()
workspace_prepared = False
slot_git_kwargs = _slot_subprocess_kwargs(slot_uid)
slot_git_env: dict[str, str] | None = None
if repo_exists:
# Existing workspaces are already slot-owned from the previous run.
# Refresh pool-side group bits, then hand the tree to the current
# slot before running any git command inside the worktree; root's
# uid-0 bypass does not bypass git's safe.directory ownership check.
_share_git_metadata_with_slots(repo_dir, slot_uid)
_provision_runtime_dirs(ws_root)
_chown_workspace(ws_root, slot_uid)
workspace_prepared = True
if not repo_exists:
if pr_head is not None:
self.transport.fetch_pr_head(repo=repo, pool_dir=pool, pr_number=pr_head)
_run(["git", "worktree", "add", "--detach", str(repo_dir), "FETCH_HEAD"], cwd=pool)
else:
start_point = f"origin/{default_branch}"
if existing_branch:
remote = _safe_run(
["git", "rev-parse", "--verify", f"refs/remotes/origin/{existing_branch}"],
# Make sure the requested start point exists locally (best-effort).
# For follow-ups on an existing PR, `existing_branch` is the remote
# head branch we need to amend; starting from default would silently
# lose the PR's current commits if the local pool branch is absent.
self.transport.fetch_base_ref(repo=repo, pool_dir=pool, ref=existing_branch or default_branch)
probe = ["git", "rev-parse", "--verify", f"refs/heads/{branch}"]
check = _safe_run(probe, cwd=pool)
if check.returncode == 124:
# A timed-out probe is indeterminate, not "branch absent".
# Falling through would create the worktree from the wrong
# start point; fail instead so the event retries.
raise GitCommandError(probe, check.returncode, check.stdout, check.stderr)
if check.returncode == 0:
_run(["git", "worktree", "add", str(repo_dir), branch], cwd=pool)
else:
start_point = f"origin/{default_branch}"
if existing_branch:
remote_probe = ["git", "rev-parse", "--verify", f"refs/remotes/origin/{existing_branch}"]
remote = _safe_run(remote_probe, cwd=pool)
if remote.returncode == 124:
# Same: an indeterminate probe must not silently fall
# back to origin/default and lose the PR's commits.
raise GitCommandError(remote_probe, remote.returncode, remote.stdout, remote.stderr)
if remote.returncode == 0:
start_point = f"origin/{existing_branch}"
_run(
[
"git",
"worktree",
"add",
"-b",
branch,
str(repo_dir),
start_point,
],
cwd=pool,
)
if remote.returncode == 0:
start_point = f"origin/{existing_branch}"
_run(
[
"git",
"worktree",
"add",
"-b",
else:
slot_git_env = _git_env_for_repo(repo_dir)
current = _safe_run(
["git", "symbolic-ref", "--quiet", "--short", "HEAD"],
cwd=repo_dir,
env=slot_git_env,
**slot_git_kwargs,
)
if current.returncode == 0 and current.stdout.strip():
branch = current.stdout.strip()
if existing_branch is not None and existing_branch != branch:
log.warning(
"workspace branch mapping %r differs from checked-out branch %r; using checkout",
existing_branch,
branch,
str(repo_dir),
start_point,
],
cwd=pool,
)
else:
slot_git_env = _git_env_for_repo(repo_dir)
current = _safe_run(
["git", "symbolic-ref", "--quiet", "--short", "HEAD"],
cwd=repo_dir,
env=slot_git_env,
**slot_git_kwargs,
)
if current.returncode == 0 and current.stdout.strip():
branch = current.stdout.strip()
if existing_branch is not None and existing_branch != branch:
log.warning(
"workspace branch mapping %r differs from checked-out branch %r; using checkout",
existing_branch,
branch,
)
if not workspace_prepared:
)
if not workspace_prepared:
_share_git_metadata_with_slots(repo_dir, slot_uid)
_provision_runtime_dirs(ws_root)
_chown_workspace(ws_root, slot_uid)
if slot_git_env is None:
slot_git_env = _git_env_for_repo(repo_dir)
# Identity is set on the worktree's shared config; idempotent. Run as
# the slot after the chown so git never trips over safe.directory.
for command in (["git", "config", "user.email", author_email], ["git", "config", "user.name", author_name]):
proc = _safe_run(command, cwd=repo_dir, env=slot_git_env, **slot_git_kwargs)
if proc.returncode != 0:
raise GitCommandError(command, proc.returncode, proc.stdout, proc.stderr)
_share_git_metadata_with_slots(repo_dir, slot_uid)
_provision_runtime_dirs(ws_root)
_chown_workspace(ws_root, slot_uid)
if slot_git_env is None:
slot_git_env = _git_env_for_repo(repo_dir)
# Identity is set on the worktree's shared config; idempotent. Run as
# the slot after the chown so git never trips over safe.directory.
for command in (["git", "config", "user.email", author_email], ["git", "config", "user.name", author_name]):
proc = _safe_run(command, cwd=repo_dir, env=slot_git_env, **slot_git_kwargs)
if proc.returncode != 0:
raise GitCommandError(command, proc.returncode, proc.stdout, proc.stderr)
_share_git_metadata_with_slots(repo_dir, slot_uid)
workspace = Workspace(
root=ws_root,
repo_dir=repo_dir,
session_dir=session_dir,
context_dir=context_dir,
artifacts_dir=artifacts_dir,
branch=branch,
repo_full_name=repo,
issue_number=number,
)
# Best-effort: hardlink pre-built natives in if we've cached this
# source state before. Runs AFTER the slot chown so the cache inode
# keeps its `root:omp` ownership (the slot reads through group `omp`);
# write-temp + rename in the napi build replaces with a new inode if
# the agent rebuilds, so the cached file is never mutated.
self._populate_natives_cache(workspace, slot_uid=slot_uid)
return workspace
workspace = Workspace(
root=ws_root,
repo_dir=repo_dir,
session_dir=session_dir,
context_dir=context_dir,
artifacts_dir=artifacts_dir,
branch=branch,
repo_full_name=repo,
issue_number=number,
)
# Best-effort: hardlink pre-built natives in if we've cached this
# source state before. Runs AFTER the slot chown so the cache inode
# keeps its `root:omp` ownership (the slot reads through group `omp`);
# write-temp + rename in the napi build replaces with a new inode if
# the agent rebuilds, so the cached file is never mutated.
self._populate_natives_cache(workspace, slot_uid=slot_uid)
return workspace
def _populate_natives_cache(self, workspace: Workspace, *, slot_uid: int | None = None) -> None:
"""Try to hardlink cached pi-natives artifacts into the worktree.
@@ -838,7 +890,7 @@ class SandboxManager:
# produces natives, so creating the dir is correct.
try:
key = natives_compute_key(workspace.repo_dir)
except (subprocess.CalledProcessError, RuntimeError, OSError) as exc:
except (subprocess.SubprocessError, RuntimeError, OSError) as exc:
log.debug(
"natives_cache key compute failed",
extra={"workspace": workspace.workspace_key, "err": redact_credentials(str(exc))},
@@ -895,15 +947,22 @@ class SandboxManager:
)
def remove_workspace(self, *, repo: str, number: int) -> None:
ws_root = self.workspace_root(repo, number)
repo_dir = ws_root / "repo"
if repo_dir.exists():
pool = self.pool_path(repo)
_safe_run(["git", "worktree", "remove", "--force", str(repo_dir)], cwd=pool)
with self._repo_lock(repo):
ws_root = self.workspace_root(repo, number)
repo_dir = ws_root / "repo"
if repo_dir.exists():
shutil.rmtree(repo_dir, ignore_errors=True)
if ws_root.exists():
shutil.rmtree(ws_root, ignore_errors=True)
pool = self.pool_path(repo)
_safe_run(["git", "worktree", "remove", "--force", str(repo_dir)], cwd=pool)
if repo_dir.exists():
# `git worktree remove` did not clean up (nonzero exit, incl. a
# 124 timeout). Delete the checkout ourselves, then prune the
# pool's now-dangling worktree registration so a later
# `git worktree add` for the same path does not trip on stale
# metadata.
shutil.rmtree(repo_dir, ignore_errors=True)
_safe_run(["git", "worktree", "prune"], cwd=pool)
if ws_root.exists():
shutil.rmtree(ws_root, ignore_errors=True)
__all__ = [
python/robomp/src/tasks.py
+52 -12
View File
@@ -2,9 +2,10 @@
from __future__ import annotations
import asyncio
import logging
from collections.abc import Mapping
from typing import Any
from collections.abc import Callable, Mapping
from typing import Any, TypeVar
from robomp import persona
from robomp.config import Settings
@@ -23,6 +24,38 @@ from robomp.worker import DirectiveInfo, TaskInputs, ThreadMessage, run_task
log = logging.getLogger(__name__)
_T = TypeVar("_T")
async def _run_workspace_op(func: Callable[..., _T], /, **kwargs: object) -> _T:
"""Offload a blocking sandbox workspace op to a worker thread, uncancellably.
Workspace setup/teardown (git clone/fetch, worktree add/remove, chown) is
blocking, so it runs off the event loop via a thread. Unlike a bare
``await asyncio.to_thread(...)``, cancelling the awaiting coroutine here does
NOT detach the still-running thread: the thread holds the per-repo lock and
owns the slot mid-setup, and ``_run_event``'s ``finally`` reaps/releases that
slot on cancellation. If the await detached, the reaped slot could be reused
while the thread is still touching it. So on cancellation we drain the thread
to completion before propagating, gating the caller's ``finally`` behind the
thread. The inner subprocesses are timeout-bounded, so completion is assured.
"""
inner = asyncio.ensure_future(asyncio.to_thread(func, **kwargs))
try:
return await asyncio.shield(inner)
except asyncio.CancelledError:
# A repeated cancel can interrupt even a shielded await, so loop until
# the thread is actually done; swallow the inner's own outcome and
# re-raise the cancellation the caller expects.
while not inner.done():
try:
await asyncio.shield(inner)
except asyncio.CancelledError:
continue
except BaseException:
break
raise
def _comment_from_payload(payload: Mapping[str, Any]) -> CommentInfo:
c = payload.get("comment") or {}
@@ -267,7 +300,8 @@ async def triage_issue(
return
db.upsert_issue(key=key, repo=repo.full_name, number=issue.number, state="reproducing")
clone_url = repo.clone_url
workspace = sandbox.ensure_workspace(
workspace = await _run_workspace_op(
sandbox.ensure_workspace,
repo=repo.full_name,
number=issue.number,
title=issue.title,
@@ -341,7 +375,8 @@ async def review_pr(
)
db.upsert_issue(key=key, repo=repo.full_name, number=pr_number, state="reviewing", pr_number=pr_number)
workspace = sandbox.ensure_workspace(
workspace = await _run_workspace_op(
sandbox.ensure_workspace,
repo=repo.full_name,
number=pr_number,
title=issue.title,
@@ -406,7 +441,8 @@ async def handle_comment(
# first and executes the directive in the same RPC turn.
log.info("directive bootstrap", extra={"key": key, "author": directive.author})
db.upsert_issue(key=key, repo=repo.full_name, number=issue.number, state="reproducing")
workspace = sandbox.ensure_workspace(
workspace = await _run_workspace_op(
sandbox.ensure_workspace,
repo=repo.full_name,
number=issue.number,
title=issue.title,
@@ -456,9 +492,10 @@ async def handle_comment(
# Maintainer reopen: tear down stale workspace, reset state, branch
# afresh from default. The old branch may have been merged/deleted.
log.info("directive reopen", extra={"key": key, "from_state": existing.state, "author": directive.author})
sandbox.remove_workspace(repo=repo.full_name, number=issue.number)
await _run_workspace_op(sandbox.remove_workspace, repo=repo.full_name, number=issue.number)
db.upsert_issue(key=key, repo=repo.full_name, number=issue.number, state="reproducing")
workspace = sandbox.ensure_workspace(
workspace = await _run_workspace_op(
sandbox.ensure_workspace,
repo=repo.full_name,
number=issue.number,
title=issue.title,
@@ -493,7 +530,8 @@ async def handle_comment(
await run_task(task_kind="handle_comment", inputs=inputs, comment=comment, directive=directive)
return
workspace = sandbox.ensure_workspace(
workspace = await _run_workspace_op(
sandbox.ensure_workspace,
repo=repo.full_name,
number=issue.number,
title=issue.title,
@@ -567,7 +605,8 @@ async def handle_review(
log.warning("review fetch failed", extra={"err": str(exc)})
return
clone_url = repo.clone_url
workspace = sandbox.ensure_workspace(
workspace = await _run_workspace_op(
sandbox.ensure_workspace,
repo=repo.full_name,
number=issue.number,
title=issue.title,
@@ -677,7 +716,7 @@ async def handle_pr_conversation(
"directive reopen (pr)",
extra={"key": issue_row.key, "from_state": issue_row.state, "author": directive.author},
)
sandbox.remove_workspace(repo=issue_row.repo, number=issue_row.number)
await _run_workspace_op(sandbox.remove_workspace, repo=issue_row.repo, number=issue_row.number)
db.upsert_issue(key=issue_row.key, repo=issue_row.repo, number=issue_row.number, state="reproducing")
issue_row = db.get_issue(issue_row.key) or issue_row
# Bare @mention with no request body — the route stashes an empty
@@ -713,7 +752,8 @@ async def handle_pr_conversation(
if existing_branch is None and not (directive and issue_row.state == "reproducing"):
log.info("skip: pr-conversation PR missing branch mapping", extra={"repo": repo_full, "pr": pr_number})
return
workspace = sandbox.ensure_workspace(
workspace = await _run_workspace_op(
sandbox.ensure_workspace,
repo=repo.full_name,
number=issue.number,
title=issue.title,
@@ -797,7 +837,7 @@ async def cleanup_workspace(
issue_row = db.get_issue(issue_key(repo_full, number))
if issue_row is None:
return
sandbox.remove_workspace(repo=issue_row.repo, number=issue_row.number)
await _run_workspace_op(sandbox.remove_workspace, repo=issue_row.repo, number=issue_row.number)
db.set_issue_state(issue_row.key, target_state)
log.info("cleanup", extra={"key": issue_row.key, "state": target_state})