feat(python/robomp): added automatic event retry scheduling with backoff delays

- Added event retry settings with parsed delay schedules and jittered delay computation.
- Extended event persistence to persist an `available_at` timestamp, honor it during dequeuing, and clear it when re-queuing.
- Updated worker failure handling to queue bounded retries with backoff and transition to failed only when the retry budget is exhausted.

python/robomp/src/config.py

@@ -67,6 +67,17 @@ class Settings(BaseSettings):
     task_timeout_seconds: float = Field(2400.0, alias="ROBOMP_TASK_TIMEOUT_SECONDS")
     task_timeout_hard_grace_seconds: float = Field(60.0, alias="ROBOMP_TASK_TIMEOUT_HARD_GRACE_SECONDS")
     request_timeout_seconds: float = Field(120.0, alias="ROBOMP_REQUEST_TIMEOUT_SECONDS")
+
+    # Automatic retry of transiently-failed events. When an event handler
+    # raises (and it isn't an operator cancel or a shutdown interrupt), the
+    # dispatcher re-queues the delivery with escalating backoff instead of
+    # giving up, so ephemeral failures (git fetch timeouts, upstream 5xx/429,
+    # flaky RPC startup) self-heal. After `event_max_retries` retries the row
+    # stays `failed`. `event_retry_delays_seconds` is a comma-separated backoff
+    # schedule: the Nth retry waits the Nth value (last value repeats), jittered.
+    # Set `event_max_retries=0` to restore fail-fast behavior.
+    event_max_retries: int = Field(3, alias="ROBOMP_EVENT_MAX_RETRIES")
+    event_retry_delays_raw: str = Field("30,120,600", alias="ROBOMP_EVENT_RETRY_DELAYS_SECONDS")
     # Premature-end reminder. When a `triage_issue` turn ends without the
     # agent having reached a terminal tool (`gh_open_pr`,
     # `mark_unable_to_reproduce`, `abort_task`) for a `bug`/`documentation`
@@ -297,6 +308,41 @@ class Settings(BaseSettings):
         """Random selection from the pool (uniform). One-element pools return that one."""
         return random.choice(self.model_pool)
 
+    @field_validator("event_retry_delays_raw", mode="before")
+    @classmethod
+    def _coerce_retry_delays(cls, v: object) -> str:
+        if v is None:
+            return ""
+        if isinstance(v, (list, tuple)):
+            return ",".join(str(item) for item in v)
+        return str(v)
+
+    @property
+    def event_retry_delays(self) -> tuple[float, ...]:
+        """Parsed backoff schedule in seconds; always non-empty."""
+        vals: list[float] = []
+        for piece in self.event_retry_delays_raw.split(","):
+            piece = piece.strip()
+            if not piece:
+                continue
+            try:
+                seconds = float(piece)
+            except ValueError:
+                continue
+            if seconds >= 0:
+                vals.append(seconds)
+        return tuple(vals) or (30.0,)
+
+    def retry_delay_seconds(self, retry_index: int) -> float:
+        """Backoff before the `retry_index`-th retry (1-based), with jitter.
+
+        Clamps to the last configured delay; applies ±20% jitter so a
+        fleet-wide outage doesn't replay every event in lockstep.
+        """
+        delays = self.event_retry_delays
+        idx = min(max(retry_index, 1), len(delays)) - 1
+        return delays[idx] * (0.8 + random.random() * 0.4)
+
     @property
     def resolved_author_name(self) -> str:
         """Falls back to bot_login if ROBOMP_GIT_AUTHOR_NAME isn't set."""

python/robomp/src/db.py

@@ -119,6 +119,11 @@ def _utcnow() -> str:
     return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
 
+def _utc_after(seconds: float) -> str:
+    """UTC timestamp `seconds` in the future, same sortable format as `_utcnow`."""
+    return (datetime.now(UTC) + timedelta(seconds=max(seconds, 0.0))).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
 def iso_seconds_ago(seconds: float) -> str:
     """ISO-UTC timestamp for `seconds` ago, matching the format `_utcnow` writes."""
     return (datetime.now(UTC) - timedelta(seconds=seconds)).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
@@ -241,6 +246,8 @@ class Database:
         event_cols = {row[1] for row in self._conn.execute("PRAGMA table_info(events)").fetchall()}
         if "model" not in event_cols:
             self._conn.execute("ALTER TABLE events ADD COLUMN model TEXT")
+        if "available_at" not in event_cols:
+            self._conn.execute("ALTER TABLE events ADD COLUMN available_at TEXT")
 
     def close(self) -> None:
         with self._lock:
@@ -298,6 +305,7 @@ class Database:
     def claim_next_event(self) -> EventRow | None:
         """Atomically dequeue one unblocked queued event into running state."""
         with self._txn() as conn:
+            now = _utcnow()
             row = conn.execute(
                 """
                 SELECT queued.delivery_id, queued.event_type, queued.repo, queued.issue_key,
@@ -305,6 +313,7 @@ class Database:
                        queued.last_error
                 FROM events AS queued
                 WHERE queued.state = 'queued'
+                  AND (queued.available_at IS NULL OR queued.available_at <= ?)
                   AND (
                     queued.issue_key IS NULL
                     OR NOT EXISTS (
@@ -316,11 +325,11 @@ class Database:
                   )
                 ORDER BY queued.received_at
                 LIMIT 1
-                """
+                """,
+                (now,),
             ).fetchone()
             if row is None:
                 return None
-            now = _utcnow()
             conn.execute(
                 "UPDATE events SET state='running', attempts=attempts+1, started_at=? WHERE delivery_id=?",
                 (now, row["delivery_id"]),
@@ -360,7 +369,7 @@ class Database:
         """Recover events that were running at shutdown."""
         with self._lock:
             cur = self._conn.execute(
-                "UPDATE events SET state='queued' WHERE state='running'",
+                "UPDATE events SET state='queued', available_at=NULL WHERE state='running'",
             )
             return cur.rowcount
 
@@ -613,7 +622,7 @@ class Database:
         with self._lock:
             if from_states is None:
                 cur = self._conn.execute(
-                    "UPDATE events SET state='queued' WHERE delivery_id=?",
+                    "UPDATE events SET state='queued', available_at=NULL WHERE delivery_id=?",
                     (delivery_id,),
                 )
             elif not from_states:
@@ -621,11 +630,29 @@ class Database:
             else:
                 placeholders = ",".join("?" for _ in from_states)
                 cur = self._conn.execute(
-                    f"UPDATE events SET state='queued' WHERE delivery_id=? AND state IN ({placeholders})",
+                    f"UPDATE events SET state='queued', available_at=NULL WHERE delivery_id=? AND state IN ({placeholders})",
                     (delivery_id, *from_states),
                 )
             return cur.rowcount > 0
 
+    def schedule_retry(self, delivery_id: str, *, delay_seconds: float, error: str | None = None) -> bool:
+        """Re-queue a delivery for a future retry with backoff.
+
+        Flips state back to 'queued' but stamps `available_at` so
+        `claim_next_event` skips the row until the backoff elapses. `attempts`
+        is left untouched (it was already incremented at claim) so the retry
+        budget keeps counting down; `last_error` retains the failure reason for
+        the dashboard. Only transitions a 'running'/'failed' row; returns
+        whether a row changed.
+        """
+        with self._lock:
+            cur = self._conn.execute(
+                "UPDATE events SET state='queued', last_error=?, available_at=?, finished_at=NULL "
+                "WHERE delivery_id=? AND state IN ('running','failed')",
+                (error, _utc_after(delay_seconds), delivery_id),
+            )
+            return cur.rowcount > 0
+
     # ---- issues ----
     def upsert_issue(
         self,

python/robomp/src/queue.py

@@ -301,8 +301,25 @@ class WorkerPool:
                 self.db.mark_event(row.delivery_id, "failed", error="cancelled by operator")
             else:
                 tb = traceback.format_exc(limit=20)
-                log.exception("event handler failed", extra={"delivery": row.delivery_id})
-                self.db.mark_event(row.delivery_id, "failed", error=f"{exc}\n{tb}")
+                err = f"{exc}\n{tb}"
+                max_retries = self.settings.event_max_retries
+                delay = self.settings.retry_delay_seconds(row.attempts)
+                if 0 < row.attempts <= max_retries and self.db.schedule_retry(
+                    row.delivery_id, delay_seconds=delay, error=err
+                ):
+                    log.warning(
+                        "event retry scheduled",
+                        extra={
+                            "delivery": row.delivery_id,
+                            "key": row.issue_key,
+                            "attempt": row.attempts,
+                            "max_retries": max_retries,
+                            "retry_in_seconds": round(delay, 1),
+                        },
+                    )
+                else:
+                    log.exception("event handler failed", extra={"delivery": row.delivery_id})
+                    self.db.mark_event(row.delivery_id, "failed", error=err)
         finally:
             self._cancelled.discard(row.delivery_id)
             self._shutdown_cancelled.discard(row.delivery_id)