droboticflow/tests/release/test_release.py
Codex 0040988fe9
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
🧭 Enforce dev mainline and release identity
2026-11-10 01:35:25 -08:00

501 lines
24 KiB
Python

"""No cluster or customer credentials: exercise release transitions with fake Kubernetes."""
import argparse
import copy
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import shutil
import sys
import tempfile
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location('release', ROOT / 'scripts/release.py')
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
IMAGE = 'registry.example/api@sha256:' + 'a' * 64
def fixture(name, container):
config = {'data': {'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true',
'ANNOTATION_PROCESSOR_MODE': 'external', 'ANNOTATION_WORKSPACE_BUCKET': 'drobodata-fanyu',
'ANNOTATION_WORKSPACE_CONNECTION_ID': 'storage-example', 'WORKSPACE_MOUNT_PATH': '/workspace'},
'metadata': {'uid': 'config-uid'}}
env = [{'name': 'MONGO_URI', 'valueFrom': {'secretKeyRef': {'name': 'droboticflow-mongo-secret', 'key': 'MONGO_URI'}}}]
if container == 'processor':
env += [{'name': k, 'value': v} for k, v in {
'ANNOTATION_PROCESSOR_MODE': 'processor', 'ANNOTATION_PROCESSOR_CONCURRENCY': '1',
'ANNOTATION_INDEX_BACKFILL': '0'}.items()]
d = {'metadata': {'name': name, 'uid': name + '-uid', 'resourceVersion': '1', 'generation': 1},
'spec': {'replicas': 1, 'strategy': {'type': 'Recreate'}, 'template': {'spec': {
'containers': [{'name': container, 'image': 'registry.example/api@sha256:' + 'b' * 64,
'envFrom': [{'configMapRef': {'name': 'config'}}], 'env': env,
'resources': {'limits': {'memory': '8Gi'}},
'volumeMounts': [{'name': 'workspace', 'mountPath': '/workspace'}]}],
'volumes': [{'name': 'workspace', 'persistentVolumeClaim': {'claimName': 'droboticflow-workspace'}}]}}},
'status': {'observedGeneration': 1, 'replicas': 1, 'updatedReplicas': 1, 'readyReplicas': 1}}
return d, config
class ReleaseTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.records = Path(self.tmp.name) / 'records'
self.p = m.policy()
self.p['localChecks'] = [['fake-check']]
self.start_patch('RECORDS', self.records)
self.start_patch('policy', lambda: copy.deepcopy(self.p))
self.start_patch('source', lambda: {'commit': 'c' * 40, 'dirty': False})
self.start_patch('require_mainline', lambda *args, **kwargs: None)
self.start_patch('run', lambda *args, **kwargs: 'feature/release')
self.start_patch('manifest_platforms', lambda image: ['linux/amd64'])
self.ds = {}
for n, c in self.p['deployments'].items():
self.ds[n], config = fixture(n, c)
self.configs = {'config': config}
self.writes = []
self.rollout_waits = []
self.fail_rollout = False
self.fail_patch = None
self.before_snapshot = None
self.start_patch('snapshot', self.snapshot)
self.start_patch('kubectl', self.kubectl)
m.prepare(argparse.Namespace(id='test', image=IMAGE, context='fanyu', server='https://cluster.example',
namespace_uid='namespace-uid', kubeconfig='/example/config', purpose='test', rollback='review old image'))
self.r = self.load()
def start_patch(self, name, value):
p = patch.object(m, name, value)
p.start()
self.addCleanup(p.stop)
def load(self):
return json.loads(m.record_path('test').read_text())
def snapshot(self, r):
if self.before_snapshot:
self.before_snapshot()
return {'at': m.now(), 'deployments': {
n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p) for n, d in self.ds.items()},
'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}},
'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}}
def kubectl(self, r, args, *, stdin=None, timeout=60):
if args[:2] == ['rollout', 'status']:
self.rollout_waits.append((args[2], list(self.writes)))
if self.fail_rollout:
raise m.Blocked('simulated rollout timeout')
return 'successfully rolled out'
self.assertEqual(args[:2], ['patch', 'deployment'])
name = args[2]
self.writes.append(name)
if self.fail_patch == len(self.writes):
raise m.Blocked('simulated timeout')
ops = json.loads(stdin)
self.assertEqual(ops[0]['path'], '/metadata/resourceVersion')
d = self.ds[name]
self.assertEqual(ops[0]['value'], d['metadata']['resourceVersion'])
c = d['spec']['template']['spec']['containers'][0]
c['image'] = ops[3]['value']
if ops[4]['op'] == 'add':
c['env'].append(ops[4]['value'])
else:
c['env'][int(ops[4]['path'].split('/')[-1])] = ops[4]['value']
d['metadata']['resourceVersion'] = str(int(d['metadata']['resourceVersion']) + 1)
return '{}'
def plan(self):
m.check(self.r, argparse.Namespace(local=True, cluster=True))
m.make_plan(self.r)
def evidence(self, key, result='passed'):
f = Path(self.tmp.name) / (key + '.txt')
f.write_text('synthetic test evidence for current release')
m.record_evidence(self.r, argparse.Namespace(check=key, file=str(f), result=result, note='test evidence only'))
return f
def approve(self):
self.evidence('image-provenance')
self.evidence('change-safety')
return argparse.Namespace(confirm_plan=self.r['planHash'], approval_ref='user-message:test')
def use_current_source_profile(self):
profile = json.loads((ROOT / 'docs/capacity/source-index-100.json').read_text())
for name, d in self.ds.items():
desired = profile['deployments'][name]
d['spec']['replicas'] = desired['replicas']
c = d['spec']['template']['spec']['containers'][0]
c['resources'] = {k: copy.deepcopy(desired[k]) for k in ('requests', 'limits')}
overrides = dict(desired['env'])
for e in c['env']:
if e['name'] in overrides:
e['value'] = overrides.pop(e['name'])
c['env'] += [{'name': k, 'value': v} for k, v in overrides.items()]
d['status'].update({k: desired['replicas'] for k in ('replicas', 'updatedReplicas', 'readyReplicas')})
def test_mainline_mismatch_blocks_before_cluster_access(self):
self.plan()
approval = self.approve()
with patch.object(m, 'require_mainline', side_effect=m.Blocked('dev changed')) as guard:
with patch.object(m, 'snapshot') as read_cluster:
with self.assertRaises(m.Blocked):
m.deploy(self.r, approval)
guard.assert_called_once_with(self.r['commit'], refresh=True)
read_cluster.assert_not_called()
self.assertEqual(self.writes, [])
def test_current_source_profile_keeps_resources_and_only_updates_image(self):
self.use_current_source_profile()
before = {name: copy.deepcopy(d['spec']) for name, d in self.ds.items()}
self.plan()
m.deploy(self.r, self.approve())
for name, d in self.ds.items():
old = m.inspect_deployment({'metadata': d['metadata'], 'spec': before[name]}, self.p['deployments'][name], self.configs, self.p)
current = m.inspect_deployment(d, self.p['deployments'][name], self.configs, self.p)
self.assertEqual(old['nonReleaseHash'], current['nonReleaseHash'])
self.assertEqual(self.writes, ['droboticflow-annotation-processor', 'droboticflow-api'])
def test_source_profile_rejects_resource_or_mode_drift(self):
self.use_current_source_profile()
name = 'droboticflow-annotation-processor'
good = copy.deepcopy(self.ds[name])
mutations = [
('ANNOTATION_PROCESSOR_CONCURRENCY', '16'),
('ANNOTATION_SOURCE_INDEX', 'false'),
('ANNOTATION_INDEX_CONCURRENCY', '8'),
('ANNOTATION_INDEX_BACKFILL', '1'),
('DROBOTICFLOW_DATABASE_BOOTSTRAP', 'migrate'),
('DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE', 'legacy'),
('ANNOTATION_PROCESSOR_ROLE', 'unknown'),
]
for key, value in mutations:
d = copy.deepcopy(good)
c = d['spec']['template']['spec']['containers'][0]
next(e for e in c['env'] if e['name'] == key)['value'] = value
with self.subTest(key=key), self.assertRaises(m.Blocked):
m.inspect_deployment(d, 'processor', self.configs, self.p)
for key in ('replicas', 'resources', 'strategy'):
d = copy.deepcopy(good)
if key == 'replicas':
d['spec']['replicas'] = 3
elif key == 'resources':
d['spec']['template']['spec']['containers'][0]['resources']['limits']['memory'] = '2Gi'
else:
d['spec']['strategy']['type'] = 'RollingUpdate'
with self.subTest(key=key), self.assertRaises(m.Blocked):
m.inspect_deployment(d, 'processor', self.configs, self.p)
def test_historical_status_is_readonly_without_fabricated_evidence(self):
import contextlib, io
for record in [
{'id': 'legacy', 'state': 'accepted_with_evidence', 'commit': 'old',
'evidence': {'review-acceptance': {'result': 'passed'}}},
{'id': 'operator', 'state': 'deployed_business_verified_pilot_recovery_queued',
'commit': 'old', 'plan': {'custom': True}},
]:
original = copy.deepcopy(record)
output = io.StringIO()
with contextlib.redirect_stdout(output):
m.status(record)
result = json.loads(output.getvalue())
self.assertEqual(result['recordFormat'], 'historical-readonly')
self.assertFalse(result['executableFormat'])
self.assertIn('review-acceptance', result['missingEvidence'])
self.assertEqual(record, original)
with self.assertRaises(m.Blocked):
m.verify(record)
self.assertFalse(self.writes)
def test_superseded_plan_keeps_backup_and_cannot_be_executed(self):
self.plan()
original = m.record_path('test').read_bytes()
replacement = copy.deepcopy(self.r)
replacement['id'] = 'replacement'
m.save(replacement)
m.supersede(self.r, argparse.Namespace(by='replacement', reason='already replaced'))
saved = self.load()
self.assertEqual(saved['state'], 'awaiting_approval')
backup = m.record_path('test').parent / saved['supersession']['backup']
self.assertEqual(backup.read_bytes(), original)
self.assertEqual(saved['planHash'], replacement['planHash'])
for fn in [lambda: m.check(saved, argparse.Namespace(local=True, cluster=False)),
lambda: m.make_plan(saved), lambda: m.deploy(saved, None), lambda: m.verify(saved)]:
with self.assertRaises(m.Blocked):
fn()
self.assertFalse(self.writes)
def test_cannot_supersede_executed_or_different_environment(self):
replacement = copy.deepcopy(self.r)
replacement['id'] = 'replacement'
replacement['target']['namespaceUID'] = 'other'
m.save(replacement)
with self.assertRaises(m.Blocked):
m.supersede(self.r, argparse.Namespace(by='replacement', reason='wrong target'))
self.r['operations'] = [{'result': 'unknown'}]
with self.assertRaises(m.Blocked):
m.supersede(self.r, argparse.Namespace(by='replacement', reason='unknown result'))
def test_prepare_is_offline_and_private(self):
self.assertEqual(self.r['state'], 'prepared')
self.assertEqual(m.record_path('test').stat().st_mode & 0o777, 0o600)
self.assertFalse(self.writes)
def test_cannot_plan_without_both_checks(self):
with self.assertRaises(m.Blocked):
m.make_plan(self.r)
m.check(self.r, argparse.Namespace(local=True, cluster=False))
with self.assertRaises(m.Blocked):
m.make_plan(self.r)
def test_local_failure_cannot_pass(self):
with patch.object(m, 'run', side_effect=m.Blocked('failure')):
with self.assertRaises(m.Blocked):
m.check(self.r, argparse.Namespace(local=True, cluster=False))
self.assertEqual(self.load()['state'], 'blocked')
def test_arm64_only_image_is_blocked_at_local_check(self):
# 2026-10-14: an arm64-only manifest passed prepare/deploy and the
# rollout wedged on ImagePullBackOff; the guard must fire on check --local.
self.start_patch('manifest_platforms', lambda image: ['linux/arm64'])
with self.assertRaises(m.Blocked):
m.check(self.r, argparse.Namespace(local=True, cluster=False))
self.assertEqual(self.load()['state'], 'blocked')
def test_recheck_invalidates_previous_plan_and_failed_snapshot(self):
self.plan()
with patch.object(m, 'snapshot', side_effect=m.Blocked('offline')):
with self.assertRaises(m.Blocked):
m.check(self.r, argparse.Namespace(local=False, cluster=True))
saved = self.load()
self.assertIsNone(saved['baseline'])
self.assertIsNone(saved['planHash'])
def test_source_and_policy_change_block_deployment(self):
self.plan()
a = self.approve()
for s in [{'commit': 'd' * 40, 'dirty': False}, {'commit': 'c' * 40, 'dirty': True}]:
with patch.object(m, 'source', return_value=s), self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.p['database'] = 'another'
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertFalse(self.writes)
def test_exact_plan_and_explicit_reference_required(self):
self.plan()
a = self.approve()
a.confirm_plan = 'wrong'
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
a.confirm_plan = self.r['planHash']
a.approval_ref = ''
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertFalse(self.writes)
def test_changed_evidence_rejected(self):
self.plan()
a = self.approve()
Path(self.r['evidence']['image-provenance']['path']).write_text('changed')
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertFalse(self.writes)
def test_colleague_live_change_blocks_before_patch(self):
self.plan()
a = self.approve()
self.ds['droboticflow-api']['spec']['replicas'] = 3
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertFalse(self.writes)
def test_config_change_between_patches_stops_second(self):
self.plan()
a = self.approve()
def change():
if len(self.writes) == 1:
self.configs['config']['data']['unrelated'] = 'changed'
self.before_snapshot = change
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertEqual(len(self.writes), 1)
self.assertEqual(len(self.load()['operations']), 1)
def test_timeout_preserves_unknown_intent_and_cannot_redeploy(self):
self.plan()
a = self.approve()
self.fail_patch = 2
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
saved = self.load()
self.assertEqual(saved['state'], 'outcome_unknown')
self.assertEqual([o['result'] for o in saved['operations']], ['acknowledged', 'unknown'])
with self.assertRaises(m.Blocked):
m.deploy(saved, a)
with self.assertRaises(m.Blocked):
m.verify(saved)
self.assertNotEqual(self.load()['state'], 'accepted_with_evidence')
def test_deploy_and_ready_are_not_business_acceptance(self):
self.plan()
m.deploy(self.r, self.approve())
self.assertEqual(self.r['state'], 'deployed_pending_verification')
m.verify(self.r)
self.assertEqual(self.r['state'], 'deployed_pending_acceptance')
for key in self.p['requiredEvidence']:
if key not in self.r['evidence']:
self.evidence(key)
m.verify(self.r)
self.assertEqual(self.r['state'], 'accepted_with_evidence')
Path(self.r['evidence']['video-playback']['path']).unlink()
m.verify(self.r)
self.assertEqual(self.r['state'], 'deployed_pending_acceptance')
def test_processor_rollout_precedes_api_patch(self):
self.plan()
m.deploy(self.r, self.approve())
self.assertEqual(self.writes, ['droboticflow-annotation-processor', 'droboticflow-api'])
self.assertEqual(self.rollout_waits, [
('deployment/droboticflow-annotation-processor', ['droboticflow-annotation-processor'])])
def test_processor_rollout_timeout_leaves_api_untouched(self):
self.plan()
a = self.approve()
self.fail_rollout = True
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
saved = self.load()
self.assertEqual(saved['state'], 'deployed_pending_verification')
self.assertEqual([o['result'] for o in saved['operations']], ['acknowledged'])
with self.assertRaises(m.Blocked):
m.deploy(saved, a)
def test_processor_not_converged_leaves_api_untouched(self):
self.plan()
a = self.approve()
def not_ready():
if self.writes:
self.ds['droboticflow-annotation-processor']['status']['readyReplicas'] = 0
self.before_snapshot = not_ready
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
self.assertEqual(self.load()['state'], 'deployed_pending_verification')
def test_no_predeployment_business_evidence(self):
self.plan()
with self.assertRaises(m.Blocked):
self.evidence('video-playback')
def test_verify_rejects_unplanned_settings(self):
self.plan()
m.deploy(self.r, self.approve())
self.ds['droboticflow-api']['spec']['template']['spec']['containers'][0]['resources'] = {'limits': {'memory': '1Gi'}}
with self.assertRaises(m.Blocked):
m.verify(self.r)
self.assertEqual(self.load()['state'], 'deployed_pending_verification')
def test_configuration_regressions_block(self):
for key, value in [('MONGO_DB', 'droboticflow'), ('DROBOTICFLOW_REQUIRE_SESSION_AUTH', 'false'),
('ANNOTATION_PROCESSOR_MODE', 'inprocess'), ('ANNOTATION_WORKSPACE_BUCKET', ''),
('WORKSPACE_MOUNT_PATH', '/wrong')]:
configs = copy.deepcopy(self.configs)
configs['config']['data'][key] = value
with self.subTest(key=key), self.assertRaises(m.Blocked):
m.inspect_deployment(self.ds['droboticflow-api'], 'api', configs, self.p)
def test_unknown_explicit_env_does_not_fall_back_to_envfrom(self):
d = copy.deepcopy(self.ds['droboticflow-api'])
d['spec']['template']['spec']['containers'][0]['env'].append({
'name': 'MONGO_DB', 'valueFrom': {'secretKeyRef': {'name': 'other', 'key': 'db'}}})
with self.assertRaises(m.Blocked):
m.inspect_deployment(d, 'api', self.configs, self.p)
def test_snapshot_identity_and_no_secret_reads(self):
r = self.r
replies = {'namespace': {'metadata': {'uid': 'namespace-uid'}},
'pvc': {'metadata': {'uid': 'pvc'}, 'spec': {}, 'status': {'phase': 'Bound'}},
'service': {'metadata': {'uid': 'svc'}, 'spec': {}}}
reads = []
def fake_get(_, kind, name):
reads.append(kind)
if kind == 'deployment': return self.ds[name]
if kind == 'configmap': return self.configs[name]
return replies[kind]
# setUp replaced snapshot; recover implementation through a second module import.
real = importlib.util.module_from_spec(spec)
spec.loader.exec_module(real)
with patch.object(real, 'policy', return_value=self.p), patch.object(real, 'get', side_effect=fake_get), patch.object(real, 'kubectl', return_value=json.dumps({'clusters': [{'cluster': {'server': r['target']['server']}}]})):
real.snapshot(r)
self.assertNotIn('secret', reads)
replies['namespace']['metadata']['uid'] = 'wrong'
with self.assertRaises(real.Blocked): real.snapshot(r)
def test_stale_cluster_check_blocks_deploy(self):
self.plan()
self.r['baseline']['at'] = '2000-01-01T00:00:00+00:00'
m.make_plan(self.r)
with self.assertRaises(m.Blocked): m.deploy(self.r, self.approve())
self.assertFalse(self.writes)
def test_record_path_traversal_rejected(self):
with self.assertRaises(m.Blocked): m.record_path('../other')
def test_record_lock_prevents_two_writers(self):
with m.locked('test'):
with self.assertRaises(m.Blocked):
with m.locked('test'): pass
class EntrypointTests(unittest.TestCase):
def test_cli_persists_and_resumes_offline(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / 'scripts').mkdir()
(root / 'docs/release').mkdir(parents=True)
shutil.copy(ROOT / 'scripts/release.py', root / 'scripts/release.py')
shutil.copy(ROOT / 'docs/release/fanyu-policy.json', root / 'docs/release/fanyu-policy.json')
(root / '.gitignore').write_text('local-data/\n')
for args in [ ['git', 'init', '-q'], ['git', 'add', '.'],
['git', '-c', 'user.name=ReleaseTest', '-c', 'user.email=test@example.invalid',
'commit', '-qm', 'fixture'] ]:
subprocess.run(args, cwd=root, check=True, capture_output=True)
commit = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip()
for ref in ('refs/heads/dev', 'refs/remotes/origin/dev'):
subprocess.run(['git', 'update-ref', ref, commit], cwd=root, check=True, capture_output=True)
def cli(*args):
return subprocess.run([sys.executable, 'scripts/release.py', *args], cwd=root,
text=True, capture_output=True)
r = cli('prepare', '--id', 'offline-demo', '--image', IMAGE, '--context', 'example',
'--kubeconfig', '/does-not-exist', '--server', 'https://cluster.example',
'--namespace-uid', 'example-uid', '--purpose', 'offline smoke', '--rollback', 'not deployed')
self.assertEqual(r.returncode, 0, r.stderr)
resumed = cli('status', 'offline-demo')
self.assertEqual(resumed.returncode, 0, resumed.stderr)
self.assertEqual(json.loads(resumed.stdout)['state'], 'prepared')
blocked = cli('deploy', 'offline-demo', '--confirm-plan', 'unprepared', '--approval-ref', 'none')
self.assertEqual(blocked.returncode, 2)
self.assertIn('No fresh deployable plan', blocked.stderr)
def test_legacy_fanyu_deploy_stops_before_kubectl(self):
with tempfile.TemporaryDirectory() as tmp:
fake = Path(tmp) / 'kubectl'
called = Path(tmp) / 'called'
fake.write_text('#!/bin/sh\ntouch "' + str(called) + '"\nexit 99\n')
fake.chmod(0o755)
env = dict(os.environ, PATH=tmp + ':' + os.environ['PATH'], DEPLOY_ENV='fanyu',
DROBOTICFLOW_DEPLOY_ENV_LOADED='test', K8S_NAMESPACE='droboticflow-fanyu')
p = subprocess.run(['bash', 'scripts/deploy-k8s.sh'], cwd=ROOT, env=env, text=True, capture_output=True)
self.assertEqual(p.returncode, 2)
self.assertIn('scripts/release.py', p.stderr)
self.assertFalse(called.exists())
if __name__ == '__main__':
unittest.main()