forked from longtao.wu/droboticflow
634 lines
33 KiB
Python
634 lines
33 KiB
Python
"""No cluster or customer credentials: exercise release transitions with fake Kubernetes."""
|
|
import argparse
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
spec = importlib.util.spec_from_file_location('release', ROOT / 'scripts/release.py')
|
|
m = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(m)
|
|
IMAGE = 'registry.example/api@sha256:' + 'a' * 64
|
|
|
|
|
|
def fixture(name, container):
|
|
config = {'data': {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true',
|
|
'ANNOTATION_PROCESSOR_MODE': 'external', 'ANNOTATION_WORKSPACE_BUCKET': 'drobodata-fanyu',
|
|
'ANNOTATION_WORKSPACE_CONNECTION_ID': 'storage-example', 'WORKSPACE_MOUNT_PATH': '/workspace'},
|
|
'metadata': {'uid': 'config-uid'}}
|
|
env = [{'name': 'MONGO_URI', 'valueFrom': {'secretKeyRef': {'name': 'droboticflow-mongo-secret', 'key': 'MONGO_URI'}}}]
|
|
if container == 'processor':
|
|
env += [{'name': k, 'value': v} for k, v in {
|
|
'ANNOTATION_PROCESSOR_MODE': 'processor', 'ANNOTATION_PROCESSOR_CONCURRENCY': '1',
|
|
'ANNOTATION_INDEX_BACKFILL': '0'}.items()]
|
|
d = {'metadata': {'name': name, 'uid': name + '-uid', 'resourceVersion': '1', 'generation': 1},
|
|
'spec': {'replicas': 1, 'strategy': {'type': 'Recreate'}, 'template': {'spec': {
|
|
'containers': [{'name': container, 'image': 'registry.example/api@sha256:' + 'b' * 64,
|
|
'envFrom': [{'configMapRef': {'name': 'config'}}], 'env': env,
|
|
'resources': {'limits': {'memory': '8Gi'}},
|
|
'volumeMounts': [{'name': 'workspace', 'mountPath': '/workspace'}]}],
|
|
'volumes': [{'name': 'workspace', 'persistentVolumeClaim': {'claimName': 'droboticflow-workspace'}}]}}},
|
|
'status': {'observedGeneration': 1, 'replicas': 1, 'updatedReplicas': 1, 'readyReplicas': 1}}
|
|
return d, config
|
|
|
|
|
|
class ReleaseTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.records = Path(self.tmp.name) / 'records'
|
|
self.p = m.policy()
|
|
self.p['localChecks'] = [['fake-check']]
|
|
self.start_patch('RECORDS', self.records)
|
|
self.start_patch('policy', lambda: copy.deepcopy(self.p))
|
|
self.start_patch('source', lambda: {'commit': 'c' * 40, 'dirty': False})
|
|
self.start_patch('require_mainline', lambda *args, **kwargs: None)
|
|
self.start_patch('run', lambda *args, **kwargs: 'feature/release')
|
|
self.start_patch('manifest_platforms', lambda image: ['linux/amd64'])
|
|
self.start_patch('check_identity', lambda r: {'testEnvironment': 'stable', 'complete': True})
|
|
self.ds = {}
|
|
for n, c in self.p['deployments'].items():
|
|
self.ds[n], config = fixture(n, c)
|
|
self.configs = {'config': config}
|
|
self.writes = []
|
|
self.rollout_waits = []
|
|
self.fail_rollout = False
|
|
self.fail_patch = None
|
|
self.before_snapshot = None
|
|
self.start_patch('snapshot', self.snapshot)
|
|
self.start_patch('kubectl', self.kubectl)
|
|
m.prepare(argparse.Namespace(id='test', image=IMAGE, context='fanyu', server='https://cluster.example',
|
|
namespace_uid='namespace-uid', kubeconfig='/example/config', purpose='test', rollback='review old image'))
|
|
self.r = self.load()
|
|
|
|
def start_patch(self, name, value):
|
|
p = patch.object(m, name, value)
|
|
p.start()
|
|
self.addCleanup(p.stop)
|
|
|
|
def load(self):
|
|
return json.loads(m.record_path('test').read_text())
|
|
|
|
def snapshot(self, r, **kwargs):
|
|
if self.before_snapshot:
|
|
self.before_snapshot()
|
|
return {'at': m.now(), 'deployments': {
|
|
n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in self.ds.items()},
|
|
'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}},
|
|
'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}}
|
|
|
|
def kubectl(self, r, args, *, stdin=None, timeout=60):
|
|
if args[:2] == ['rollout', 'status']:
|
|
self.rollout_waits.append((args[2], list(self.writes)))
|
|
if self.fail_rollout:
|
|
raise m.Blocked('simulated rollout timeout')
|
|
return 'successfully rolled out'
|
|
self.assertEqual(args[:2], ['patch', 'deployment'])
|
|
name = args[2]
|
|
self.writes.append(name)
|
|
if self.fail_patch == len(self.writes):
|
|
raise m.Blocked('simulated timeout')
|
|
ops = json.loads(stdin)
|
|
self.assertEqual(ops[0]['path'], '/metadata/resourceVersion')
|
|
d = self.ds[name]
|
|
self.assertEqual(ops[0]['value'], d['metadata']['resourceVersion'])
|
|
c = d['spec']['template']['spec']['containers'][0]
|
|
c['image'] = ops[3]['value']
|
|
if ops[4]['op'] == 'add':
|
|
c['env'].append(ops[4]['value'])
|
|
else:
|
|
c['env'][int(ops[4]['path'].split('/')[-1])] = ops[4]['value']
|
|
for extra in ops[5:]:
|
|
if extra['op']=='add': c['env'].append(extra['value'])
|
|
else: c['env'][int(extra['path'].split('/')[-1])] = extra['value']
|
|
d['metadata']['resourceVersion'] = str(int(d['metadata']['resourceVersion']) + 1)
|
|
return '{}'
|
|
|
|
def plan(self):
|
|
m.check(self.r, argparse.Namespace(local=True, cluster=True))
|
|
m.make_plan(self.r)
|
|
|
|
def evidence(self, key, result='passed'):
|
|
f = Path(self.tmp.name) / (key + '.txt')
|
|
f.write_text('synthetic test evidence for current release')
|
|
m.record_evidence(self.r, argparse.Namespace(check=key, file=str(f), result=result, note='test evidence only'))
|
|
return f
|
|
|
|
def approve(self):
|
|
self.evidence('image-provenance')
|
|
self.evidence('change-safety')
|
|
return argparse.Namespace(confirm_plan=self.r['planHash'], approval_ref='user-message:test')
|
|
|
|
def use_current_source_profile(self):
|
|
profile = json.loads((ROOT / 'docs/capacity/source-index-100.json').read_text())
|
|
for name, d in self.ds.items():
|
|
desired = profile['deployments'][name]
|
|
d['spec']['replicas'] = desired['replicas']
|
|
c = d['spec']['template']['spec']['containers'][0]
|
|
c['resources'] = {k: copy.deepcopy(desired[k]) for k in ('requests', 'limits')}
|
|
overrides = dict(desired['env'])
|
|
for e in c['env']:
|
|
if e['name'] in overrides:
|
|
e['value'] = overrides.pop(e['name'])
|
|
c['env'] += [{'name': k, 'value': v} for k, v in overrides.items()]
|
|
d['status'].update({k: desired['replicas'] for k in ('replicas', 'updatedReplicas', 'readyReplicas')})
|
|
|
|
def test_mainline_mismatch_blocks_before_cluster_access(self):
|
|
self.plan()
|
|
approval = self.approve()
|
|
with patch.object(m, 'require_mainline', side_effect=m.Blocked('dev changed')) as guard:
|
|
with patch.object(m, 'snapshot') as read_cluster:
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, approval)
|
|
guard.assert_called_once_with(self.r['commit'], refresh=True)
|
|
read_cluster.assert_not_called()
|
|
self.assertEqual(self.writes, [])
|
|
|
|
def test_current_source_profile_keeps_resources_and_only_updates_image(self):
|
|
self.use_current_source_profile()
|
|
before = {name: copy.deepcopy(d['spec']) for name, d in self.ds.items()}
|
|
self.plan()
|
|
m.deploy(self.r, self.approve())
|
|
for name, d in self.ds.items():
|
|
old = m.inspect_deployment({'metadata': d['metadata'], 'spec': before[name]}, self.p['deployments'][name], self.configs, self.p)
|
|
current = m.inspect_deployment(d, self.p['deployments'][name], self.configs, self.p)
|
|
self.assertEqual(old['nonReleaseHash'], current['nonReleaseHash'])
|
|
self.assertEqual(self.writes, ['droboticflow-annotation-processor', 'droboticflow-api'])
|
|
|
|
def test_source_profile_rejects_resource_or_mode_drift(self):
|
|
self.use_current_source_profile()
|
|
name = 'droboticflow-annotation-processor'
|
|
good = copy.deepcopy(self.ds[name])
|
|
mutations = [
|
|
('ANNOTATION_PROCESSOR_CONCURRENCY', '16'),
|
|
('ANNOTATION_SOURCE_INDEX', 'false'),
|
|
('ANNOTATION_INDEX_CONCURRENCY', '8'),
|
|
('ANNOTATION_INDEX_BACKFILL', '1'),
|
|
('DROBOTICFLOW_DATABASE_BOOTSTRAP', 'migrate'),
|
|
('DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE', 'legacy'),
|
|
('ANNOTATION_PROCESSOR_ROLE', 'unknown'),
|
|
]
|
|
for key, value in mutations:
|
|
d = copy.deepcopy(good)
|
|
c = d['spec']['template']['spec']['containers'][0]
|
|
next(e for e in c['env'] if e['name'] == key)['value'] = value
|
|
with self.subTest(key=key), self.assertRaises(m.Blocked):
|
|
m.inspect_deployment(d, 'processor', self.configs, self.p)
|
|
for key in ('replicas', 'resources', 'strategy'):
|
|
d = copy.deepcopy(good)
|
|
if key == 'replicas':
|
|
d['spec']['replicas'] = 3
|
|
elif key == 'resources':
|
|
d['spec']['template']['spec']['containers'][0]['resources']['limits']['memory'] = '2Gi'
|
|
else:
|
|
d['spec']['strategy']['type'] = 'RollingUpdate'
|
|
with self.subTest(key=key), self.assertRaises(m.Blocked):
|
|
m.inspect_deployment(d, 'processor', self.configs, self.p)
|
|
|
|
def test_historical_status_is_readonly_without_fabricated_evidence(self):
|
|
import contextlib, io
|
|
for record in [
|
|
{'id': 'legacy', 'state': 'accepted_with_evidence', 'commit': 'old',
|
|
'evidence': {'review-acceptance': {'result': 'passed'}}},
|
|
{'id': 'operator', 'state': 'deployed_business_verified_pilot_recovery_queued',
|
|
'commit': 'old', 'plan': {'custom': True}},
|
|
]:
|
|
original = copy.deepcopy(record)
|
|
output = io.StringIO()
|
|
with contextlib.redirect_stdout(output):
|
|
m.status(record)
|
|
result = json.loads(output.getvalue())
|
|
self.assertEqual(result['recordFormat'], 'historical-readonly')
|
|
self.assertFalse(result['executableFormat'])
|
|
self.assertIn('review-acceptance', result['missingEvidence'])
|
|
self.assertEqual(record, original)
|
|
with self.assertRaises(m.Blocked):
|
|
m.verify(record)
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_superseded_plan_keeps_backup_and_cannot_be_executed(self):
|
|
self.plan()
|
|
original = m.record_path('test').read_bytes()
|
|
replacement = copy.deepcopy(self.r)
|
|
replacement['id'] = 'replacement'
|
|
m.save(replacement)
|
|
m.supersede(self.r, argparse.Namespace(by='replacement', reason='already replaced'))
|
|
saved = self.load()
|
|
self.assertEqual(saved['state'], 'awaiting_approval')
|
|
backup = m.record_path('test').parent / saved['supersession']['backup']
|
|
self.assertEqual(backup.read_bytes(), original)
|
|
self.assertEqual(saved['planHash'], replacement['planHash'])
|
|
for fn in [lambda: m.check(saved, argparse.Namespace(local=True, cluster=False)),
|
|
lambda: m.make_plan(saved), lambda: m.deploy(saved, None), lambda: m.verify(saved)]:
|
|
with self.assertRaises(m.Blocked):
|
|
fn()
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_cannot_supersede_executed_or_different_environment(self):
|
|
replacement = copy.deepcopy(self.r)
|
|
replacement['id'] = 'replacement'
|
|
replacement['target']['namespaceUID'] = 'other'
|
|
m.save(replacement)
|
|
with self.assertRaises(m.Blocked):
|
|
m.supersede(self.r, argparse.Namespace(by='replacement', reason='wrong target'))
|
|
self.r['operations'] = [{'result': 'unknown'}]
|
|
with self.assertRaises(m.Blocked):
|
|
m.supersede(self.r, argparse.Namespace(by='replacement', reason='unknown result'))
|
|
|
|
def test_prepare_is_offline_and_private(self):
|
|
self.assertEqual(self.r['state'], 'prepared')
|
|
self.assertEqual(m.record_path('test').stat().st_mode & 0o777, 0o600)
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_cannot_plan_without_both_checks(self):
|
|
with self.assertRaises(m.Blocked):
|
|
m.make_plan(self.r)
|
|
m.check(self.r, argparse.Namespace(local=True, cluster=False))
|
|
with self.assertRaises(m.Blocked):
|
|
m.make_plan(self.r)
|
|
|
|
def test_local_failure_cannot_pass(self):
|
|
with patch.object(m, 'run', side_effect=m.Blocked('failure')):
|
|
with self.assertRaises(m.Blocked):
|
|
m.check(self.r, argparse.Namespace(local=True, cluster=False))
|
|
self.assertEqual(self.load()['state'], 'blocked')
|
|
|
|
def test_arm64_only_image_is_blocked_at_local_check(self):
|
|
# 2026-10-14: an arm64-only manifest passed prepare/deploy and the
|
|
# rollout wedged on ImagePullBackOff; the guard must fire on check --local.
|
|
self.start_patch('manifest_platforms', lambda image: ['linux/arm64'])
|
|
with self.assertRaises(m.Blocked):
|
|
m.check(self.r, argparse.Namespace(local=True, cluster=False))
|
|
self.assertEqual(self.load()['state'], 'blocked')
|
|
|
|
def test_recheck_invalidates_previous_plan_and_failed_snapshot(self):
|
|
self.plan()
|
|
with patch.object(m, 'snapshot', side_effect=m.Blocked('offline')):
|
|
with self.assertRaises(m.Blocked):
|
|
m.check(self.r, argparse.Namespace(local=False, cluster=True))
|
|
saved = self.load()
|
|
self.assertIsNone(saved['baseline'])
|
|
self.assertIsNone(saved['planHash'])
|
|
|
|
def test_source_and_policy_change_block_deployment(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
for s in [{'commit': 'd' * 40, 'dirty': False}, {'commit': 'c' * 40, 'dirty': True}]:
|
|
with patch.object(m, 'source', return_value=s), self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.p['database'] = 'another'
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_exact_plan_and_explicit_reference_required(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
a.confirm_plan = 'wrong'
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
a.confirm_plan = self.r['planHash']
|
|
a.approval_ref = ''
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_changed_evidence_rejected(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
Path(self.r['evidence']['image-provenance']['path']).write_text('changed')
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_colleague_live_change_blocks_before_patch(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
self.ds['droboticflow-api']['spec']['replicas'] = 3
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_config_change_between_patches_stops_second(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
def change():
|
|
if len(self.writes) == 1:
|
|
self.configs['config']['data']['unrelated'] = 'changed'
|
|
self.before_snapshot = change
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.assertEqual(len(self.writes), 1)
|
|
self.assertEqual(len(self.load()['operations']), 1)
|
|
|
|
def test_timeout_preserves_unknown_intent_and_cannot_redeploy(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
self.fail_patch = 2
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
saved = self.load()
|
|
self.assertEqual(saved['state'], 'outcome_unknown')
|
|
self.assertEqual([o['result'] for o in saved['operations']], ['acknowledged', 'unknown'])
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(saved, a)
|
|
with self.assertRaises(m.Blocked):
|
|
m.verify(saved)
|
|
self.assertNotEqual(self.load()['state'], 'accepted_with_evidence')
|
|
|
|
def test_deploy_and_ready_are_not_business_acceptance(self):
|
|
self.plan()
|
|
m.deploy(self.r, self.approve())
|
|
self.assertEqual(self.r['state'], 'deployed_pending_verification')
|
|
m.verify(self.r)
|
|
self.assertEqual(self.r['state'], 'deployed_pending_acceptance')
|
|
for key in self.p['requiredEvidence']:
|
|
if key not in self.r['evidence']:
|
|
self.evidence(key)
|
|
m.verify(self.r)
|
|
self.assertEqual(self.r['state'], 'accepted_with_evidence')
|
|
Path(self.r['evidence']['video-playback']['path']).unlink()
|
|
m.verify(self.r)
|
|
self.assertEqual(self.r['state'], 'deployed_pending_acceptance')
|
|
|
|
def test_processor_rollout_precedes_api_patch(self):
|
|
self.plan()
|
|
m.deploy(self.r, self.approve())
|
|
self.assertEqual(self.writes, ['droboticflow-annotation-processor', 'droboticflow-api'])
|
|
self.assertEqual(self.rollout_waits, [
|
|
('deployment/droboticflow-annotation-processor', ['droboticflow-annotation-processor'])])
|
|
|
|
def test_processor_rollout_timeout_leaves_api_untouched(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
self.fail_rollout = True
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
|
|
saved = self.load()
|
|
self.assertEqual(saved['state'], 'deployed_pending_verification')
|
|
self.assertEqual([o['result'] for o in saved['operations']], ['acknowledged'])
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(saved, a)
|
|
|
|
def test_processor_not_converged_leaves_api_untouched(self):
|
|
self.plan()
|
|
a = self.approve()
|
|
def not_ready():
|
|
if self.writes:
|
|
self.ds['droboticflow-annotation-processor']['status']['readyReplicas'] = 0
|
|
self.before_snapshot = not_ready
|
|
with self.assertRaises(m.Blocked):
|
|
m.deploy(self.r, a)
|
|
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
|
|
self.assertEqual(self.load()['state'], 'deployed_pending_verification')
|
|
|
|
def test_no_predeployment_business_evidence(self):
|
|
self.plan()
|
|
with self.assertRaises(m.Blocked):
|
|
self.evidence('video-playback')
|
|
|
|
def test_verify_rejects_unplanned_settings(self):
|
|
self.plan()
|
|
m.deploy(self.r, self.approve())
|
|
self.ds['droboticflow-api']['spec']['template']['spec']['containers'][0]['resources'] = {'limits': {'memory': '1Gi'}}
|
|
with self.assertRaises(m.Blocked):
|
|
m.verify(self.r)
|
|
self.assertEqual(self.load()['state'], 'deployed_pending_verification')
|
|
|
|
def test_configuration_regressions_block(self):
|
|
for key, value in [('MONGO_DB', 'droboticflow'), ('DROBOTICFLOW_REQUIRE_SESSION_AUTH', 'false'),
|
|
('ANNOTATION_PROCESSOR_MODE', 'inprocess'), ('ANNOTATION_WORKSPACE_BUCKET', ''),
|
|
('WORKSPACE_MOUNT_PATH', '/wrong')]:
|
|
configs = copy.deepcopy(self.configs)
|
|
configs['config']['data'][key] = value
|
|
with self.subTest(key=key), self.assertRaises(m.Blocked):
|
|
m.inspect_deployment(self.ds['droboticflow-api'], 'api', configs, self.p)
|
|
|
|
def test_unknown_explicit_env_does_not_fall_back_to_envfrom(self):
|
|
d = copy.deepcopy(self.ds['droboticflow-api'])
|
|
d['spec']['template']['spec']['containers'][0]['env'].append({
|
|
'name': 'MONGO_DB', 'valueFrom': {'secretKeyRef': {'name': 'other', 'key': 'db'}}})
|
|
with self.assertRaises(m.Blocked):
|
|
m.inspect_deployment(d, 'api', self.configs, self.p)
|
|
|
|
def test_snapshot_identity_and_no_secret_reads(self):
|
|
r = self.r
|
|
replies = {'namespace': {'metadata': {'uid': 'namespace-uid'}},
|
|
'pvc': {'metadata': {'uid': 'pvc'}, 'spec': {}, 'status': {'phase': 'Bound'}},
|
|
'service': {'metadata': {'uid': 'svc'}, 'spec': {}}}
|
|
reads = []
|
|
def fake_get(_, kind, name):
|
|
reads.append(kind)
|
|
if kind == 'deployment': return self.ds[name]
|
|
if kind == 'configmap': return self.configs[name]
|
|
return replies[kind]
|
|
# setUp replaced snapshot; recover implementation through a second module import.
|
|
real = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(real)
|
|
with patch.object(real, 'policy', return_value=self.p), patch.object(real, 'get', side_effect=fake_get), patch.object(real, 'kubectl', return_value=json.dumps({'clusters': [{'cluster': {'server': r['target']['server']}}]})):
|
|
real.snapshot(r)
|
|
self.assertNotIn('secret', reads)
|
|
r['baseline'] = self.snapshot(r)
|
|
r['sourceMode'] = 'strict'
|
|
env = self.ds['droboticflow-annotation-processor']['spec']['template']['spec']['containers'][0]['env']
|
|
env.append({'name':'ANNOTATION_SOURCE_VALIDATION_MODE','value':'strict'})
|
|
with self.assertRaises(real.Blocked): real.snapshot(r)
|
|
real.snapshot(r, allow_planned_transition=True)
|
|
env[-1]['value'] = 'disabled'
|
|
with self.assertRaises(real.Blocked): real.snapshot(r, allow_planned_transition=True)
|
|
env.pop()
|
|
replies['namespace']['metadata']['uid'] = 'wrong'
|
|
with self.assertRaises(real.Blocked): real.snapshot(r)
|
|
|
|
def test_stale_cluster_check_blocks_deploy(self):
|
|
self.plan()
|
|
self.r['baseline']['at'] = '2000-01-01T00:00:00+00:00'
|
|
m.make_plan(self.r)
|
|
with self.assertRaises(m.Blocked): m.deploy(self.r, self.approve())
|
|
self.assertFalse(self.writes)
|
|
|
|
def test_record_path_traversal_rejected(self):
|
|
with self.assertRaises(m.Blocked): m.record_path('../other')
|
|
|
|
def test_record_lock_prevents_two_writers(self):
|
|
with m.locked('test'):
|
|
with self.assertRaises(m.Blocked):
|
|
with m.locked('test'): pass
|
|
|
|
def test_combined_plan_changes_only_reviewed_mode_in_processor_api_order(self):
|
|
self.configs['config']['data']['ANNOTATION_SOURCE_VALIDATION_MODE']='strict'
|
|
self.r['sourceMode']='advisory'
|
|
self.plan()
|
|
approval=self.approve()
|
|
m.deploy(self.r,approval)
|
|
self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api'])
|
|
self.assertEqual(self.rollout_waits[0][1],['droboticflow-annotation-processor'])
|
|
for d in self.ds.values():
|
|
env={e['name']:e.get('value') for e in d['spec']['template']['spec']['containers'][0]['env']}
|
|
self.assertEqual(env['ANNOTATION_SOURCE_VALIDATION_MODE'],'advisory')
|
|
m.verify(self.r)
|
|
self.assertEqual(self.r['state'],'deployed_pending_acceptance')
|
|
|
|
def test_legacy_unset_mode_can_be_repaired_by_explicit_combined_plan(self):
|
|
self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE')
|
|
self.r['sourceMode']='advisory'
|
|
self.plan();m.deploy(self.r,self.approve())
|
|
self.assertEqual(len(self.writes),2)
|
|
|
|
def test_new_release_cannot_plan_an_implicit_strict_fallback(self):
|
|
self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE')
|
|
m.check(self.r,argparse.Namespace(local=True,cluster=True))
|
|
with self.assertRaises(m.Blocked):m.make_plan(self.r)
|
|
self.assertEqual(self.writes,[])
|
|
|
|
def test_mode_change_after_approval_invalidates_plan_without_writes(self):
|
|
self.r['sourceMode']='advisory';self.plan();a=self.approve()
|
|
self.r['sourceMode']='strict'
|
|
with self.assertRaises(m.Blocked):m.deploy(self.r,a)
|
|
self.assertEqual(self.writes,[])
|
|
|
|
def test_strict_activation_without_inventory_never_writes(self):
|
|
self.r['sourceMode']='strict'
|
|
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=True))
|
|
self.assertEqual(self.writes,[])
|
|
|
|
def test_combined_mode_rollout_failure_leaves_api_unmodified(self):
|
|
self.r['sourceMode']='advisory';self.plan();self.fail_rollout=True
|
|
with self.assertRaises(m.Blocked):m.deploy(self.r,self.approve())
|
|
self.assertEqual(self.writes,['droboticflow-annotation-processor'])
|
|
self.assertEqual(self.r['state'],'deployed_pending_verification')
|
|
|
|
def test_failed_check_log_is_private_and_does_not_discard_failure(self):
|
|
failure=m.Blocked('failed');failure.check_output='password: canary-secret\nTraceback: canary-failure'
|
|
with patch.object(m,'run',side_effect=failure):
|
|
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False))
|
|
e=self.load()['localChecks'][-1];text=Path(e['log']).read_text()
|
|
self.assertNotIn('canary-secret',text);self.assertIn('canary-failure',text)
|
|
self.assertEqual(Path(e['log']).stat().st_mode&0o777,0o600)
|
|
|
|
def test_explicit_reuse_keeps_only_verified_successful_pure_checks(self):
|
|
self.p['localChecks']=[['npm','run','lint:web']]
|
|
self.r['policyHash']=m.digest(self.p)
|
|
m.check(self.r,argparse.Namespace(local=True,cluster=False))
|
|
with patch.object(m,'run',side_effect=m.Blocked('must not rerun pure check')):
|
|
m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True))
|
|
self.assertIn('reusedAt',self.r['localChecks'][0])
|
|
Path(self.r['localChecks'][0]['log']).write_text('tampered')
|
|
with patch.object(m,'run',side_effect=m.Blocked('must rerun tampered evidence')):
|
|
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True))
|
|
|
|
def test_current_complete_deployment_report_binds_strict_mode_and_detects_tampering(self):
|
|
for d in self.ds.values():
|
|
d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']})
|
|
self.r['sourceMode']='strict'
|
|
report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'],
|
|
'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True,
|
|
'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(),
|
|
'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]}
|
|
path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report))
|
|
m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path)))
|
|
m.make_plan(self.r);a=self.approve()
|
|
report['rows'][0]['allowed']=0;report['rows'][0]['failed']=1;path.write_text(json.dumps(report))
|
|
with self.assertRaises(m.Blocked):m.deploy(self.r,a)
|
|
self.assertEqual(self.writes,[])
|
|
|
|
def test_complete_report_allows_reviewed_strict_activation(self):
|
|
for d in self.ds.values():
|
|
d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']})
|
|
self.r['sourceMode']='strict'
|
|
report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'],
|
|
'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True,
|
|
'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(),
|
|
'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]}
|
|
path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report))
|
|
m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path)))
|
|
m.make_plan(self.r);m.deploy(self.r,self.approve());m.verify(self.r)
|
|
self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api'])
|
|
self.assertEqual(self.r['state'],'deployed_pending_acceptance')
|
|
|
|
def test_changed_strict_rule_needs_advisory_preparation_not_old_report(self):
|
|
self.r['sourceMode']='strict';self.r['baseline']=self.snapshot(self.r)
|
|
self.r['baseline']['deployments']['droboticflow-api']['buildCommit']='b'*40
|
|
self.r['sourceReadiness']={'path':'not-read-yet'}
|
|
with self.assertRaisesRegex(m.Blocked,'deploy advisory first'):m.validate_source_transition(self.r)
|
|
|
|
|
|
|
|
class EntrypointTests(unittest.TestCase):
|
|
def test_cli_persists_and_resumes_offline(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / 'scripts').mkdir()
|
|
(root / 'docs/release').mkdir(parents=True)
|
|
shutil.copy(ROOT / 'scripts/release.py', root / 'scripts/release.py')
|
|
shutil.copytree(ROOT / 'scripts/lib', root / 'scripts/lib', ignore=shutil.ignore_patterns('__pycache__','*.pyc'))
|
|
shutil.copy(ROOT / 'docs/release/fanyu-policy.json', root / 'docs/release/fanyu-policy.json')
|
|
(root / '.gitignore').write_text('local-data/\n__pycache__/\n')
|
|
for args in [ ['git', 'init', '-q'], ['git', 'add', '.'],
|
|
['git', '-c', 'user.name=ReleaseTest', '-c', 'user.email=test@example.invalid',
|
|
'commit', '-qm', 'fixture'] ]:
|
|
subprocess.run(args, cwd=root, check=True, capture_output=True)
|
|
commit = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip()
|
|
for ref in ('refs/heads/dev', 'refs/remotes/origin/dev'):
|
|
subprocess.run(['git', 'update-ref', ref, commit], cwd=root, check=True, capture_output=True)
|
|
def cli(*args):
|
|
return subprocess.run([sys.executable, 'scripts/release.py', *args], cwd=root,
|
|
text=True, capture_output=True)
|
|
r = cli('prepare', '--id', 'offline-demo', '--image', IMAGE, '--context', 'example',
|
|
'--kubeconfig', '/does-not-exist', '--server', 'https://cluster.example',
|
|
'--namespace-uid', 'example-uid', '--purpose', 'offline smoke', '--rollback', 'not deployed')
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
|
resumed = cli('status', 'offline-demo')
|
|
self.assertEqual(resumed.returncode, 0, resumed.stderr)
|
|
self.assertEqual(json.loads(resumed.stdout)['state'], 'prepared')
|
|
blocked = cli('deploy', 'offline-demo', '--confirm-plan', 'unprepared', '--approval-ref', 'none')
|
|
self.assertEqual(blocked.returncode, 2)
|
|
self.assertIn('No fresh deployable plan', blocked.stderr)
|
|
|
|
def test_legacy_fanyu_deploy_stops_before_kubectl(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fake = Path(tmp) / 'kubectl'
|
|
called = Path(tmp) / 'called'
|
|
fake.write_text('#!/bin/sh\ntouch "' + str(called) + '"\nexit 99\n')
|
|
fake.chmod(0o755)
|
|
env = dict(os.environ, PATH=tmp + ':' + os.environ['PATH'], DEPLOY_ENV='fanyu',
|
|
DROBOTICFLOW_DEPLOY_ENV_LOADED='test', K8S_NAMESPACE='droboticflow-fanyu')
|
|
p = subprocess.run(['bash', 'scripts/deploy-k8s.sh'], cwd=ROOT, env=env, text=True, capture_output=True)
|
|
self.assertEqual(p.returncode, 2)
|
|
self.assertIn('scripts/release.py', p.stderr)
|
|
self.assertFalse(called.exists())
|
|
|
|
|
|
class LegacySourceCutoffTests(unittest.TestCase):
|
|
def test_advisory_mode_requires_matching_components(self):
|
|
m.validate_legacy_source_cutoffs([{'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory'}] * 2)
|
|
for modes in [('advisory', None), ('advisory', 'strict'), ('disabled', 'disabled')]:
|
|
with self.subTest(modes=modes), self.assertRaises(m.Blocked):
|
|
m.validate_legacy_source_cutoffs([{'ANNOTATION_SOURCE_VALIDATION_MODE': v} for v in modes])
|
|
|
|
def test_strict_default_and_approved_equal_cutoff(self):
|
|
m.validate_legacy_source_cutoffs([{}, {}])
|
|
v = {'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-11-12T00:00:00Z'}
|
|
m.validate_legacy_source_cutoffs([v, v])
|
|
|
|
def test_mismatched_and_invalid_cutoffs_block(self):
|
|
for values in [({}, {'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-11-12T00:00:00Z'}),
|
|
({'ANNOTATION_LEGACY_SOURCE_CUTOFF': 'bad'},)*2,
|
|
({'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-02-30T00:00:00Z'},)*2,
|
|
({'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-11-12T00:00:00.1Z'},)*2]:
|
|
with self.subTest(values=values), self.assertRaises(m.Blocked):
|
|
m.validate_legacy_source_cutoffs(values)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|