fix: release an unknown-outcome pause once the group has nothing open

The probe after an unknown request was only allowed at the moment the
unknown attempt was resolved, and only if no other attempt was open. On
2026-10-07 08:26 the Qwen group's unknown attempt was auto-abandoned while
an innocent request was still streaming (ended 08:27), probe_allowed was
false, and nothing re-checked: the group stayed PAUSED for 15 hours.

abandon_unknown, confirm_end and reconcile() now share _probe_if_settled():
an unknown-outcome pause turns into one COOLING probe as soon as no attempt
in the group is open, whichever of them sees that first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ming01.wang 2026-10-07 23:47:04 +08:00
parent f9b3fa6871
commit ea0335bc24
2 changed files with 39 additions and 11 deletions

View File

@ -579,12 +579,27 @@ class Broker:
con.execute("UPDATE attempts SET state='UNKNOWN',reason=COALESCE(reason,'REQUEST_OUTCOME_UNKNOWN') WHERE attempt_id=?", (attempt['attempt_id'],))
con.execute("UPDATE generations SET state='PAUSED',reason=COALESCE(reason,'REQUEST_OUTCOME_UNKNOWN'),updated_at=? WHERE generation_id=?", (self._now(), attempt['generation_id']))
stale += [row[0] for row in con.execute("SELECT attempt_id FROM attempts WHERE group_id=? AND state='UNKNOWN' AND ended_at IS NULL AND deadline<=?", (group, self._now() - policy.window_s))]
if self._probe_if_settled(con, group):
# An innocent request was still in flight when the unknown one was resolved.
self._event(con, group, 'UNKNOWN_SETTLED_PROBE_ALLOWED')
for attempt_id in stale: # outside the group flock: abandon_unknown takes it itself
try:
self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed')
except QuotaError:
pass # another broker settled it first
def _probe_if_settled(self, con, group):
"""End an unknown-outcome pause with one probe once nothing in the group is open.
Every resolver and reconcile() call it, so a request that finishes after the
unknown one was resolved cannot leave the group paused forever."""
remaining = con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'", (group,)).fetchone()[0]
state = con.execute('SELECT * FROM quota_groups WHERE group_id=?', (group,)).fetchone()
if remaining or state['reason'] != 'REQUEST_OUTCOME_UNKNOWN' or state['state'] not in ('RECONCILING', 'PAUSED'):
return False
now = self._now()
con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group))
return True
def abandon_unknown(self, attempt_id: str, *, operator_reason: str):
"""Explicit operator release of local capacity, never provider-end proof.
@ -612,11 +627,7 @@ class Broker:
con.execute("UPDATE attempts SET state='ABANDONED_UNKNOWN' WHERE attempt_id=?", (attempt_id,))
con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation))
con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,))
remaining = con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'", (group,)).fetchone()[0]
state = con.execute('SELECT * FROM quota_groups WHERE group_id=?', (group,)).fetchone()
probe_allowed = not remaining and state['reason'] == 'REQUEST_OUTCOME_UNKNOWN' and state['state'] in ('RECONCILING', 'PAUSED')
if probe_allowed:
con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group))
probe_allowed = self._probe_if_settled(con, group)
self._event(con, group, 'OPERATOR_ABANDONED_UNKNOWN', generation,
attempt_id=attempt_id, operator_reason=operator_reason,
released_at=now, provider_end_confirmed=False,
@ -643,12 +654,9 @@ class Broker:
if changed!=1:
raise QuotaError('NOT_RECONCILABLE',409)
self._event(con, row['group_id'], 'END_CONFIRMED', row['generation_id'], attempt_id=attempt_id, receipt_id=receipt_id)
remaining=con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND state <> 'ABANDONED_UNKNOWN' AND ended_at IS NULL",(row['group_id'],)).fetchone()[0]
group=con.execute('SELECT * FROM quota_groups WHERE group_id=?',(row['group_id'],)).fetchone()
if not remaining and group['reason']=='REQUEST_OUTCOME_UNKNOWN' and group['state'] in ('RECONCILING','PAUSED'):
# The trusted end receipt authorizes one future probe. The old
# generation remains paused and can never replay automatically.
con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?",(self._now(),self._now(),row['group_id']))
# The trusted end receipt authorizes one future probe. The old
# generation remains paused and can never replay automatically.
if self._probe_if_settled(con, row['group_id']):
self._event(con,row['group_id'],'RECONCILED_PROBE_ALLOWED',receipt_id=receipt_id)
def _resume_readiness(self, con, execution_id, now):

View File

@ -42,6 +42,26 @@ class AutoAbandon(unittest.TestCase):
self.assertEqual(len([e for e in broker.snapshot()['quota_events']
if e['event'] == 'OPERATOR_ABANDONED_UNKNOWN']), 1)
def test_request_finishing_after_the_abandon_still_releases_the_group(self):
# 2026-10-07: the unknown attempt was abandoned while an innocent one was
# in flight, so no probe was allowed and the group stayed PAUSED for 15h.
_, broker, _, attempt = self.unknown()
self.assertEqual(self.request(broker.issue_scope('victim', 'teacher', 'glm-test'))[0], 503)
with broker._db() as con:
row = dict(con.execute('SELECT * FROM attempts WHERE attempt_id=?', (attempt['attempt_id'],)).fetchone())
row.update(attempt_id='inflight', state='SENT', deadline=attempt['deadline'] + 3600, ended_at=None)
con.execute(f"INSERT INTO attempts ({','.join(row)}) VALUES ({','.join('?' * len(row))})", tuple(row.values()))
broker._now = lambda: attempt['deadline'] + broker.groups['shared'].window_s + .01
broker.reconcile()
state = lambda: broker.snapshot()['groups'][0]['state']
self.assertEqual(state(), 'PAUSED') # something is still open: no probe yet
with broker._db() as con:
con.execute("UPDATE attempts SET state='FINISHED', ended_at=? WHERE attempt_id='inflight'", (broker._now(),))
broker.reconcile()
self.assertEqual(state(), 'COOLING')
self.assertTrue(broker.resume_readiness('victim')['eligible'])
self.assertFalse(broker.resume_readiness('old')['eligible'])
if __name__ == '__main__':
unittest.main()