From ea0335bc240d26f6c213232ecc2f8bc88969fb20 Mon Sep 17 00:00:00 2001 From: "ming01.wang" Date: Wed, 7 Oct 2026 23:47:04 +0800 Subject: [PATCH] fix: release an unknown-outcome pause once the group has nothing open The probe after an unknown request was only allowed at the moment the unknown attempt was resolved, and only if no other attempt was open. On 2026-10-07 08:26 the Qwen group's unknown attempt was auto-abandoned while an innocent request was still streaming (ended 08:27), probe_allowed was false, and nothing re-checked: the group stayed PAUSED for 15 hours. abandon_unknown, confirm_end and reconcile() now share _probe_if_settled(): an unknown-outcome pause turns into one COOLING probe as soon as no attempt in the group is open, whichever of them sees that first. Co-Authored-By: Claude Opus 5.5 --- harness/quota.py | 30 +++++++++++++++++++----------- scripts/test_quota_auto_abandon.py | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/harness/quota.py b/harness/quota.py index 2c54fc4..4450095 100644 --- a/harness/quota.py +++ b/harness/quota.py @@ -579,12 +579,27 @@ class Broker: con.execute("UPDATE attempts SET state='UNKNOWN',reason=COALESCE(reason,'REQUEST_OUTCOME_UNKNOWN') WHERE attempt_id=?", (attempt['attempt_id'],)) con.execute("UPDATE generations SET state='PAUSED',reason=COALESCE(reason,'REQUEST_OUTCOME_UNKNOWN'),updated_at=? WHERE generation_id=?", (self._now(), attempt['generation_id'])) stale += [row[0] for row in con.execute("SELECT attempt_id FROM attempts WHERE group_id=? AND state='UNKNOWN' AND ended_at IS NULL AND deadline<=?", (group, self._now() - policy.window_s))] + if self._probe_if_settled(con, group): + # An innocent request was still in flight when the unknown one was resolved. + self._event(con, group, 'UNKNOWN_SETTLED_PROBE_ALLOWED') for attempt_id in stale: # outside the group flock: abandon_unknown takes it itself try: self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed') except QuotaError: pass # another broker settled it first + def _probe_if_settled(self, con, group): + """End an unknown-outcome pause with one probe once nothing in the group is open. + Every resolver and reconcile() call it, so a request that finishes after the + unknown one was resolved cannot leave the group paused forever.""" + remaining = con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'", (group,)).fetchone()[0] + state = con.execute('SELECT * FROM quota_groups WHERE group_id=?', (group,)).fetchone() + if remaining or state['reason'] != 'REQUEST_OUTCOME_UNKNOWN' or state['state'] not in ('RECONCILING', 'PAUSED'): + return False + now = self._now() + con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group)) + return True + def abandon_unknown(self, attempt_id: str, *, operator_reason: str): """Explicit operator release of local capacity, never provider-end proof. @@ -612,11 +627,7 @@ class Broker: con.execute("UPDATE attempts SET state='ABANDONED_UNKNOWN' WHERE attempt_id=?", (attempt_id,)) con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation)) con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,)) - remaining = con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'", (group,)).fetchone()[0] - state = con.execute('SELECT * FROM quota_groups WHERE group_id=?', (group,)).fetchone() - probe_allowed = not remaining and state['reason'] == 'REQUEST_OUTCOME_UNKNOWN' and state['state'] in ('RECONCILING', 'PAUSED') - if probe_allowed: - con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group)) + probe_allowed = self._probe_if_settled(con, group) self._event(con, group, 'OPERATOR_ABANDONED_UNKNOWN', generation, attempt_id=attempt_id, operator_reason=operator_reason, released_at=now, provider_end_confirmed=False, @@ -643,12 +654,9 @@ class Broker: if changed!=1: raise QuotaError('NOT_RECONCILABLE',409) self._event(con, row['group_id'], 'END_CONFIRMED', row['generation_id'], attempt_id=attempt_id, receipt_id=receipt_id) - remaining=con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND state <> 'ABANDONED_UNKNOWN' AND ended_at IS NULL",(row['group_id'],)).fetchone()[0] - group=con.execute('SELECT * FROM quota_groups WHERE group_id=?',(row['group_id'],)).fetchone() - if not remaining and group['reason']=='REQUEST_OUTCOME_UNKNOWN' and group['state'] in ('RECONCILING','PAUSED'): - # The trusted end receipt authorizes one future probe. The old - # generation remains paused and can never replay automatically. - con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?",(self._now(),self._now(),row['group_id'])) + # The trusted end receipt authorizes one future probe. The old + # generation remains paused and can never replay automatically. + if self._probe_if_settled(con, row['group_id']): self._event(con,row['group_id'],'RECONCILED_PROBE_ALLOWED',receipt_id=receipt_id) def _resume_readiness(self, con, execution_id, now): diff --git a/scripts/test_quota_auto_abandon.py b/scripts/test_quota_auto_abandon.py index 86fdced..fbef106 100644 --- a/scripts/test_quota_auto_abandon.py +++ b/scripts/test_quota_auto_abandon.py @@ -42,6 +42,26 @@ class AutoAbandon(unittest.TestCase): self.assertEqual(len([e for e in broker.snapshot()['quota_events'] if e['event'] == 'OPERATOR_ABANDONED_UNKNOWN']), 1) + def test_request_finishing_after_the_abandon_still_releases_the_group(self): + # 2026-10-07: the unknown attempt was abandoned while an innocent one was + # in flight, so no probe was allowed and the group stayed PAUSED for 15h. + _, broker, _, attempt = self.unknown() + self.assertEqual(self.request(broker.issue_scope('victim', 'teacher', 'glm-test'))[0], 503) + with broker._db() as con: + row = dict(con.execute('SELECT * FROM attempts WHERE attempt_id=?', (attempt['attempt_id'],)).fetchone()) + row.update(attempt_id='inflight', state='SENT', deadline=attempt['deadline'] + 3600, ended_at=None) + con.execute(f"INSERT INTO attempts ({','.join(row)}) VALUES ({','.join('?' * len(row))})", tuple(row.values())) + broker._now = lambda: attempt['deadline'] + broker.groups['shared'].window_s + .01 + broker.reconcile() + state = lambda: broker.snapshot()['groups'][0]['state'] + self.assertEqual(state(), 'PAUSED') # something is still open: no probe yet + with broker._db() as con: + con.execute("UPDATE attempts SET state='FINISHED', ended_at=? WHERE attempt_id='inflight'", (broker._now(),)) + broker.reconcile() + self.assertEqual(state(), 'COOLING') + self.assertTrue(broker.resume_readiness('victim')['eligible']) + self.assertFalse(broker.resume_readiness('old')['eligible']) + if __name__ == '__main__': unittest.main()