fix: let dead, quarantined failed executions stop blocking unrelated work

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ming01.wang 2026-10-03 21:17:40 +08:00
parent 10663a800f
commit 2eb29caa83
2 changed files with 65 additions and 1 deletions

View File

@ -168,6 +168,16 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict:
if (rd / "execution.json").exists():
owner = execution.Execution(rd)
record = owner.snapshot()
# Failed history stays failed and isolated; do not require its unknown
# stage receipt to become trustworthy before scheduling unrelated work.
if (record['execution_state'] == 'TERMINAL' and record['cleanup_state'] == 'QUARANTINED'
and row['state'] in {'INFRA_ERROR', 'CANCELLED'}
and record.get('execution_outcome') in {'AGENT_ERROR', 'INFRA_ERROR', 'CANCELLED'}
and execution.process_status(record.get('owner', {})) == 'DEAD'
and execution.owned_processes(record) == []):
reports.append({'job_id': row['job_id'], 'kk': row['kk'], 'execution': record,
'ready': True, 'reason': 'FAILED_EXECUTION_ISOLATED'})
continue
if ((execution.process_status(record.get('owner', {})) == 'DEAD'
or record['execution_state'] == 'TERMINAL')
and execution.owned_processes(record) == []):
@ -216,7 +226,8 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict:
if (not ready and record["execution_state"] == "TERMINAL"
and ((row["state"] == "INFRA_ERROR"
and execution.process_status(record.get("owner", {})) == "DEAD")
or (settled_state == record.get("execution_outcome")
or ((settled_state == record.get("execution_outcome")
or settled_state == 'INFRA_ERROR' and record.get('execution_outcome') == 'AGENT_ERROR')
and settled_state in {"INFRA_ERROR", "CANCELLED", "LOST", "AGENT_ERROR",
"DEADLINE_EXCEEDED", "BUDGET_EXHAUSTED"}
and scheduler._local_cleanup_verified(record)))

View File

@ -185,6 +185,59 @@ class Recovery(unittest.TestCase):
self.assertEqual(report["executions"], [{"job_id": "j", "kk": 0, "ready": False,
"reason": "LEGACY_EXECUTION_OWNERSHIP_UNKNOWN"}])
def test_shared_owner_agent_error_uses_recorded_infra_state(self):
from harness.stageworker import Worker
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state="INFRA_ERROR")
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
"execution_state": "TERMINAL", "execution_outcome": "AGENT_ERROR",
"cleanup_state": "QUARANTINED"}
p = self.rd / "execution.json"
p.write_text(json.dumps(record))
before = p.read_bytes()
for verified in (True, False):
with self.subTest(local_cleanup_verified=verified), \
patch.object(execution.Execution, "reconcile", return_value=record), \
patch.object(execution, "process_status", return_value="ALIVE"), \
patch.object(execution, "owned_processes", return_value=[]), \
patch.object(scheduler, "_local_cleanup_verified", return_value=verified), \
patch.object(scheduler, "reconcile_leases", return_value=[]), \
patch.object(Worker, "reconcile_completed", return_value=True):
self.assertEqual(recovery.reconcile_run("r")["ready"], verified)
self.assertEqual(p.read_bytes(), before)
def test_dead_failed_quarantine_does_not_replay_unverifiable_stage(self):
from harness.stageworker import Worker
cases = [("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [], True),
("CANCELLED", "CANCELLED", "TERMINAL", "DEAD", [], True),
("RUNNING", "AGENT_ERROR", "TERMINAL", "DEAD", [], False),
("PASS", "COMPLETED", "TERMINAL", "DEAD", [], False),
("INFRA_ERROR", "AGENT_ERROR", "PAUSED_INFRA", "DEAD", [], False),
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "UNKNOWN", [], False),
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [{"pid": 1}], False)]
for db_state, outcome, status, owner, members, allowed in cases:
with self.subTest(db_state=db_state, status=status, owner=owner, members=members):
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state=db_state)
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
"execution_state": status, "execution_outcome": outcome,
"cleanup_state": "QUARANTINED"}
p = self.rd / "execution.json"
p.write_text(json.dumps(record))
before, rows = p.read_bytes(), self.rows()
with patch.object(execution.Execution, "reconcile", return_value=record), \
patch.object(execution.Execution, "quarantine", return_value=record), \
patch.object(execution, "process_status", return_value=owner), \
patch.object(execution, "owned_processes", return_value=members), \
patch.object(scheduler, "_local_cleanup_verified", return_value=False), \
patch.object(scheduler, "reconcile_leases", return_value=[]), \
patch.object(Worker, "reconcile_completed", side_effect=ValueError("scope cleanup unconfirmed")) as replay:
self.assertEqual(recovery.reconcile_run("r")["ready"], allowed)
if allowed:
replay.assert_not_called()
self.assertEqual(p.read_bytes(), before)
self.assertEqual(self.rows(), rows)
if __name__ == "__main__":
import unittest