121 lines
6.0 KiB
Python
121 lines
6.0 KiB
Python
"""Explicit local abandonment preserves unknown evidence and fences old work."""
|
|
import json
|
|
import time
|
|
import unittest
|
|
|
|
from harness.quota import QuotaError, quota_snapshot
|
|
from scripts import test_quota as fixtures
|
|
|
|
|
|
class Abandon(unittest.TestCase):
|
|
setUp = fixtures.Quota.setUp
|
|
provider = fixtures.Quota.provider
|
|
policy = fixtures.Quota.policy
|
|
broker = fixtures.Quota.broker
|
|
request = fixtures.Quota.request
|
|
|
|
def unknown(self):
|
|
def respond(n, body):
|
|
if n == 1:
|
|
time.sleep(.1)
|
|
return fixtures.ok(n, body)
|
|
provider = self.provider(respond)
|
|
broker = self.broker(provider, self.policy(first_byte_timeout_s=.02))
|
|
scope = broker.issue_scope('old', 'teacher', 'glm-test')
|
|
self.assertEqual(self.request(scope, 'old-g')[0], 503)
|
|
attempt = broker.snapshot()['attempts'][0]
|
|
self.assertEqual(attempt['state'], 'UNKNOWN')
|
|
return provider, broker, scope, attempt
|
|
|
|
def expire(self, broker, attempt):
|
|
now = attempt['deadline'] + .01
|
|
broker._now = lambda: now
|
|
|
|
def test_release_keeps_unknown_evidence_and_allows_same_key_probe(self):
|
|
provider, broker, old, attempt = self.unknown()
|
|
self.expire(broker, attempt)
|
|
self.assertTrue(broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit operator discard'))
|
|
snapshot = quota_snapshot(self.ledger)
|
|
self.assertEqual(snapshot['groups'][0]['inflight'], 0)
|
|
self.assertEqual(snapshot['groups'][0]['tpm_reserved'], 0)
|
|
self.assertEqual(snapshot['groups'][0]['state'], 'COOLING')
|
|
after = broker.snapshot()['attempts'][0]
|
|
self.assertEqual(after, attempt | {'state': 'ABANDONED_UNKNOWN'})
|
|
self.assertIsNone(after['ended_at'])
|
|
self.assertEqual(self.request(broker.issue_scope('new', 'teacher', 'glm-test'))[0], 200)
|
|
self.assertEqual(len(provider.requests), 2)
|
|
self.assertEqual(provider.requests[0]['authorization'], provider.requests[1]['authorization'])
|
|
self.assertEqual(broker.snapshot()['attempts'][-1]['state_at_send'], 'HALF_OPEN')
|
|
self.assertEqual(self.request(old, 'old-g')[0], 409)
|
|
self.assertEqual(self.request(broker.issue_scope('old', 'teacher', 'glm-test'), 'replacement')[0], 409)
|
|
self.assertEqual(len(provider.requests), 2)
|
|
self.assertFalse(broker.resume_readiness('old')['eligible'])
|
|
|
|
def test_late_headers_and_completion_cannot_resurrect(self):
|
|
_, broker, _, attempt = self.unknown()
|
|
self.expire(broker, attempt)
|
|
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
|
|
before = broker.snapshot()
|
|
broker._response_headers('shared', 'old-g', attempt['attempt_id'], broker.groups['shared'], 429, {'Retry-After':'900'})
|
|
broker._finish('shared', 'old-g', attempt['attempt_id'], broker.groups['shared'], 'COMPLETE', known_end=True, status=200, usage=3)
|
|
self.assertEqual(before, broker.snapshot())
|
|
self.assertEqual(broker.execution_status('old')['generations'][0]['state'], 'ABANDONED_UNKNOWN')
|
|
|
|
def test_idempotence_and_restart_do_not_restore_reservation(self):
|
|
provider, broker, _, attempt = self.unknown()
|
|
self.expire(broker, attempt)
|
|
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
|
|
before = broker.snapshot()
|
|
self.assertFalse(broker.abandon_unknown(attempt['attempt_id'], operator_reason='again'))
|
|
self.assertEqual(before, broker.snapshot())
|
|
broker.close()
|
|
reopened = self.broker(provider, self.policy(first_byte_timeout_s=.02))
|
|
reopened.reconcile()
|
|
self.assertEqual(reopened.snapshot()['attempts'][0]['state'], 'ABANDONED_UNKNOWN')
|
|
self.assertEqual(quota_snapshot(self.ledger)['groups'][0]['inflight'], 0)
|
|
self.assertEqual(self.request(reopened.issue_scope('old', 'teacher', 'glm-test'))[0], 409)
|
|
self.assertEqual(len(provider.requests), 1)
|
|
|
|
def test_requires_explicit_reason_expiry_and_unknown_state(self):
|
|
_, broker, _, attempt = self.unknown()
|
|
before = broker.snapshot()
|
|
with self.assertRaises(ValueError):
|
|
broker.abandon_unknown(attempt['attempt_id'], operator_reason=' ')
|
|
with self.assertRaises(QuotaError):
|
|
broker.abandon_unknown(attempt['attempt_id'], operator_reason='too soon')
|
|
with self.assertRaises(QuotaError):
|
|
broker.abandon_unknown('absent', operator_reason='explicit')
|
|
self.assertEqual(before, broker.snapshot())
|
|
self.expire(broker, attempt)
|
|
with broker._db() as con:
|
|
con.execute("UPDATE attempts SET state='FINISHED', ended_at=1")
|
|
before = broker.snapshot()
|
|
with self.assertRaises(QuotaError):
|
|
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
|
|
self.assertEqual(before, broker.snapshot())
|
|
|
|
def test_unrelated_auth_pause_is_not_cleared(self):
|
|
_, broker, _, attempt = self.unknown()
|
|
self.expire(broker, attempt)
|
|
with broker._db() as con:
|
|
con.execute("UPDATE quota_groups SET state='PAUSED', reason='UPSTREAM_AUTH_OR_HARD_QUOTA'")
|
|
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
|
|
group = broker.snapshot()['groups'][0]
|
|
self.assertEqual((group['state'], group['reason']), ('PAUSED', 'UPSTREAM_AUTH_OR_HARD_QUOTA'))
|
|
|
|
def test_other_unknown_still_holds_group(self):
|
|
_, broker, _, attempt = self.unknown()
|
|
self.expire(broker, attempt)
|
|
with broker._db() as con:
|
|
columns = list(attempt)
|
|
other = attempt | {'attempt_id':'another-unknown'}
|
|
con.execute('INSERT INTO attempts (' + ','.join(columns) + ') VALUES (' + ','.join('?' for _ in columns) + ')', [other[k] for k in columns])
|
|
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
|
|
group = quota_snapshot(self.ledger)['groups'][0]
|
|
self.assertEqual(group['inflight'], 1)
|
|
self.assertEqual(group['reason'], 'REQUEST_OUTCOME_UNKNOWN')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|