706 lines
39 KiB
Python

#!/usr/bin/env python3
"""Fanyu image/build-stamp release journal. Secret values are never persisted."""
import argparse
import contextlib
import copy
import datetime as dt
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import uuid
import time
sys.path.insert(0, str(Path(__file__).resolve().parent))
from lib.release_checks import check_environment, identity, reusable, write_log
from lib.source_readiness import validate_report
ROOT = Path(__file__).resolve().parents[1]
RECORDS = ROOT / 'local-data/releases'
POLICY = ROOT / 'docs/release/fanyu-policy.json'
SAFE_ENV = ('MONGO_DB', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH',
'ANNOTATION_PROCESSOR_MODE', 'ANNOTATION_PROCESSOR_CONCURRENCY',
'ANNOTATION_PROCESSOR_ROLE', 'ANNOTATION_SOURCE_INDEX', 'ANNOTATION_INDEX_CONCURRENCY',
'DROBOTICFLOW_DATABASE_BOOTSTRAP', 'DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE',
'ANNOTATION_INDEX_BACKFILL', 'ANNOTATION_LEGACY_SOURCE_CUTOFF', 'ANNOTATION_SOURCE_VALIDATION_MODE', 'ANNOTATION_WORKSPACE_CONNECTION_ID',
'ANNOTATION_WORKSPACE_BUCKET', 'WORKSPACE_MOUNT_PATH')
class Blocked(Exception):
pass
def require(ok, message):
if not ok:
raise Blocked(message)
def now():
return dt.datetime.now(dt.timezone.utc).isoformat()
def digest(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
def run(argv, *, stdin=None, timeout=60, env=None):
try:
p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout, env=env)
except (OSError, subprocess.TimeoutExpired) as e:
error = Blocked('Command unavailable or timed out; outcome must be checked before retry.')
error.check_output = str(getattr(e, 'stdout', '') or '') + str(getattr(e, 'stderr', '') or '')
raise error from None
# Only policy-local checks may persist the private redacted output.
if p.returncode != 0:
error = Blocked('Command failed: ' + argv[0] + ' (no success recorded).')
error.check_output = p.stdout + '\n' + p.stderr
raise error
return p.stdout
def source():
return {'commit': run(['git', 'rev-parse', 'HEAD']).strip(),
'dirty': bool(run(['git', 'status', '--porcelain', '--untracked-files=normal']).strip())}
def require_mainline(commit, *, refresh=False):
if refresh:
run(['git', 'fetch', '--quiet', 'origin', 'dev'], timeout=30)
for ref in ('refs/heads/dev', 'refs/remotes/origin/dev'):
tip = run(['git', 'rev-parse', '--verify', ref + '^{commit}']).strip()
require(tip == commit, 'Release commit must equal local dev and origin/dev; merge and synchronize dev first.')
def policy():
return json.loads(POLICY.read_text())
def record_path(release_id):
require(bool(re.fullmatch(r'[a-zA-Z0-9][a-zA-Z0-9_-]{0,79}', release_id)), 'Invalid release id.')
return RECORDS / release_id / 'release.json'
def save(r):
path = record_path(r['id'])
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
temp = path.with_suffix('.tmp')
with os.fdopen(os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
json.dump(r, f, indent=2, ensure_ascii=False)
f.write('\n')
os.replace(temp, path)
def event(r, name):
r['events'].append({'at': now(), 'event': name})
save(r)
@contextlib.contextmanager
def locked(release_id):
path = record_path(release_id)
require(path.exists(), 'Release record not found; use prepare first.')
with path.with_suffix('.lock').open('a') as f:
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise Blocked('Another command is updating this release.') from None
yield json.loads(path.read_text())
def record_format(r):
required = ('id', 'commit', 'image', 'policyHash', 'target', 'localChecks',
'baseline', 'planHash', 'operations', 'evidence', 'events', 'state')
if r.get('schemaVersion') != 1 or any(k not in r for k in required):
return 'historical-readonly' if r.get('schemaVersion') is None else 'incomplete-readonly'
if not isinstance(r['evidence'], dict) or not isinstance(r['events'], list):
return 'incomplete-readonly'
return 'v1'
def actionable(r):
require(record_format(r) == 'v1', 'Historical/incomplete record is read-only; prepare a new release, never synthesize approval or evidence.')
require(not r.get('supersession'), 'This plan has been superseded; use status to inspect its replacement.')
def supersede(r, a):
actionable(r)
require(not r['operations'], 'Only unexecuted plans can be superseded; deployed records retain their outcomes.')
require(a.by != r['id'] and bool(a.reason.strip()), 'Provide a different replacement and a reason.')
path = record_path(a.by)
require(path.is_file(), 'Replacement record does not exist.')
replacement = json.loads(path.read_text())
require(record_format(replacement) == 'v1' and not replacement.get('supersession'), 'Replacement must be a current v1 record.')
require(all(r['target'].get(k) == replacement['target'].get(k) for k in ('server', 'context', 'namespace', 'namespaceUID')),
'Replacement must target the same verified environment.')
original = record_path(r['id']).read_bytes()
backup = record_path(r['id']).with_name('release-before-supersession-' + hashlib.sha256(original).hexdigest()[:16] + '.json')
with backup.open('xb') as f:
os.chmod(backup, 0o600)
f.write(original)
r['supersession'] = {'by': a.by, 'reason': a.reason.strip(), 'at': now(),
'backup': backup.name, 'backupSHA256': hashlib.sha256(original).hexdigest()}
event(r, 'unexecuted plan superseded by ' + a.by + '; original state/evidence retained')
def fresh(r):
actionable(r)
s = source()
require(not s['dirty'], 'Checkout is dirty; commit task changes in a clean checkout first.')
require(s['commit'] == r['commit'], 'Checkout commit changed; prepare a new release.')
require(digest(policy()) == r['policyHash'], 'Release policy changed; prepare a new release.')
def kubectl(r, args, *, stdin=None, timeout=60):
t = r['target']
return run(['kubectl', '--kubeconfig', t['kubeconfig'], '--context', t['context'],
'--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=timeout)
def get(r, kind, name):
return json.loads(kubectl(r, ['get', kind, name, '-o', 'json']))
def env_values(container, configs):
values = {}
for entry in container.get('envFrom', []):
ref = entry.get('configMapRef')
require(bool(ref), 'Secret/unknown envFrom is unsupported by this preflight.')
for key, value in configs[ref['name']].get('data', {}).items():
values[entry.get('prefix', '') + key] = value
for entry in container.get('env', []):
key = entry['name']
if 'value' in entry:
values[key] = entry['value']
elif 'configMapKeyRef' in entry.get('valueFrom', {}):
ref = entry['valueFrom']['configMapKeyRef']
require(ref['key'] in configs[ref['name']].get('data', {}), 'Missing ConfigMap key.')
values[key] = configs[ref['name']]['data'][ref['key']]
else:
# Explicit unknowns must override envFrom rather than silently use its value.
values[key] = None
return values
def inspect_deployment(deployment, container_name, configs, p, *, ignore_source_mode=False):
spec = deployment['spec']
pod = spec['template']['spec']
containers = pod['containers']
matches = [(i, c) for i, c in enumerate(containers) if c['name'] == container_name]
require(len(matches) == 1, 'Expected deployment container missing.')
index, c = matches[0]
e = env_values(c, configs)
require(e.get('MONGO_DB') == p['database'], 'Database name mismatch.')
require(e.get('ANNOTATION_WORKSPACE_BUCKET') == p['bucket'], 'Workspace bucket mismatch.')
require(bool(e.get('ANNOTATION_WORKSPACE_CONNECTION_ID')), 'Missing workspace storage connection.')
require(e.get('WORKSPACE_MOUNT_PATH') == p['workspaceMount'], 'Workspace mount mismatch.')
mongo = next((x for x in c.get('env', []) if x['name'] == 'MONGO_URI'), {})
require(mongo.get('valueFrom', {}).get('secretKeyRef') == {'name': p['mongoSecret'], 'key': 'MONGO_URI'},
'Mongo secret binding mismatch; preflight does not read secret values.')
mounts = {v['name']: v['mountPath'] for v in c.get('volumeMounts', [])}
claims = [v for v in pod.get('volumes', []) if mounts.get(v['name']) == p['workspaceMount']]
require(len(claims) == 1 and claims[0].get('persistentVolumeClaim', {}).get('claimName') == p['workspacePVC'],
'Workspace PVC binding mismatch.')
require(spec.get('replicas', 1) > 0, 'Deployment has no replicas.')
if container_name == 'api':
require(e.get('DROBOTICFLOW_REQUIRE_SESSION_AUTH') in ('true', '1'), 'Session authentication disabled.')
require(e.get('ANNOTATION_PROCESSOR_MODE') == 'external', 'API must use external processor.')
else:
require(e.get('ANNOTATION_PROCESSOR_MODE') == 'processor', 'Processor mode mismatch.')
require(e.get('ANNOTATION_INDEX_BACKFILL') == '0', 'Historical backfill must remain disabled.')
if e.get('ANNOTATION_PROCESSOR_ROLE') == 'revision':
# Recognize the previously approved deployed profile, never resize
# it. nonReleaseHash and conditional patches still prohibit drift.
profile = json.loads((ROOT / p['processorCapacityProfile']).read_text())
require(profile['schemaVersion'] == 1 and profile['profile'] == 'fanyu-source-index-100-v1'
and profile['namespace'] == p['namespace'], 'Unsupported capacity profile.')
desired = profile['deployments']['droboticflow-annotation-processor']
require(all(e.get(k) == v for k, v in desired['env'].items()), 'Source-index processor settings differ from approved capacity profile.')
require(c.get('resources') == {k: desired[k] for k in ('requests', 'limits')},
'Source-index processor resources differ from approved capacity profile.')
require(spec.get('replicas') == desired['replicas'] and spec.get('strategy', {}).get('type') == 'Recreate',
'Source-index processor replicas/strategy differ from approved deployed profile.')
else:
require(not e.get('ANNOTATION_PROCESSOR_ROLE'), 'Unsupported processor role.')
require(e.get('ANNOTATION_PROCESSOR_CONCURRENCY') == '1', 'First release profile requires concurrency 1.')
require(c.get('resources', {}).get('limits', {}).get('memory') == '8Gi', 'First release profile requires processor memory limit 8Gi.')
require(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate',
'Processor must have one replica and Recreate strategy.')
status = deployment.get('status', {})
ready = (status.get('observedGeneration', 0) >= deployment['metadata'].get('generation', 1)
and status.get('updatedReplicas', 0) == spec.get('replicas', 1)
and status.get('readyReplicas', 0) == spec.get('replicas', 1)
and status.get('replicas', 0) == spec.get('replicas', 1))
m = deployment['metadata']
neutral = copy.deepcopy(spec)
nc = neutral['template']['spec']['containers'][index]
nc['image'] = '<release-image>'
nc['env'] = [x for x in nc.get('env', []) if x['name'] not in (('DROBOTICFLOW_BUILD_COMMIT', 'ANNOTATION_SOURCE_VALIDATION_MODE') if ignore_source_mode else ('DROBOTICFLOW_BUILD_COMMIT',))]
stamp = next((i for i, x in enumerate(c.get('env', [])) if x['name'] == 'DROBOTICFLOW_BUILD_COMMIT'), None)
return {'uid': m['uid'], 'specHash': digest(spec), 'nonReleaseHash': digest(neutral),
'sourceModeIndex': next((i for i,x in enumerate(c.get('env',[])) if x['name']=='ANNOTATION_SOURCE_VALIDATION_MODE'),None), 'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'],
'resourceVersion': m['resourceVersion'], 'containerIndex': index,
'ready': ready, 'env': {k: e.get(k) for k in SAFE_ENV}}
def validate_legacy_source_cutoffs(envs):
modes = {e.get('ANNOTATION_SOURCE_VALIDATION_MODE') for e in envs}
require(len(modes) == 1 and next(iter(modes)) in (None, '', 'strict', 'advisory'),
'API/processor source validation modes differ or are invalid.')
cutoffs = {e.get('ANNOTATION_LEGACY_SOURCE_CUTOFF') for e in envs}
require(len(cutoffs) == 1, 'API/processor legacy source admission cutoffs differ.')
cutoff = next(iter(cutoffs))
if cutoff:
require(bool(re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', cutoff)), 'Invalid legacy source admission cutoff.')
try:
dt.datetime.strptime(cutoff, '%Y-%m-%dT%H:%M:%SZ')
except ValueError:
raise Blocked('Invalid legacy source admission cutoff.')
def snapshot(r, *, allow_planned_transition=False):
t, p = r['target'], policy()
view = json.loads(kubectl(r, ['config', 'view', '--minify', '-o', 'json']))
require(view['clusters'][0]['cluster']['server'] == t['server'], 'Kube API server mismatch.')
ns = get(r, 'namespace', t['namespace'])
require(ns['metadata']['uid'] == t['namespaceUID'], 'Namespace UID mismatch.')
deployments = {n: get(r, 'deployment', n) for n in p['deployments']}
config_names = set()
for d in deployments.values():
for c in d['spec']['template']['spec']['containers']:
config_names.update(x['configMapRef']['name'] for x in c.get('envFrom', []) if 'configMapRef' in x)
config_names.update(x['valueFrom']['configMapKeyRef']['name'] for x in c.get('env', [])
if 'configMapKeyRef' in x.get('valueFrom', {}))
configs = {n: get(r, 'configmap', n) for n in sorted(config_names)}
deps = {n: inspect_deployment(d, p['deployments'][n], configs, p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in deployments.items()}
envs = [d['env'] for d in deps.values()]
require(len({e['ANNOTATION_WORKSPACE_CONNECTION_ID'] for e in envs}) == 1, 'API/processor storage connections differ.')
if allow_planned_transition and r.get("sourceMode"):
for e in envs: require(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in (r["sourceMode"], *(d["env"].get("ANNOTATION_SOURCE_VALIDATION_MODE") for d in r["baseline"]["deployments"].values())), "Unexpected source mode during rollout.")
normalized = [dict(e, ANNOTATION_SOURCE_VALIDATION_MODE=r["sourceMode"]) for e in envs]
validate_legacy_source_cutoffs(normalized)
else:
validate_legacy_source_cutoffs(envs)
if r.get("sourceModeExplicit") and not r.get("sourceMode"):
require(all(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in ("advisory", "strict") for e in envs), "New releases require an explicit source admission mode on both components.")
pvc = get(r, 'pvc', p['workspacePVC'])
require(pvc.get('status', {}).get('phase') == 'Bound', 'Workspace PVC is not Bound.')
svc = get(r, 'service', 'droboticflow-api-public-eip')
ann = svc['metadata'].get('annotations', {})
return {'at': now(), 'deployments': deps,
'configs': {n: {'uid': x['metadata']['uid'], 'hash': digest(x.get('data', {}))} for n, x in configs.items()},
'pvc': {'uid': pvc['metadata']['uid'], 'specHash': digest(pvc['spec'])},
'publicService': {'uid': svc['metadata']['uid'], 'specHash': digest(svc['spec']),
'annotationHash': digest(ann),
'bandwidthMbps': ann.get('service.beta.kubernetes.io/cce-elastic-ip-bandwidth-in-mbps', 'unknown')}}
def stable(s):
return {**{k: s[k] for k in ('configs', 'pvc', 'publicService')},
'deployments': {n: {k: d[k] for k in ('uid', 'specHash', 'image')} for n, d in s['deployments'].items()}}
def plan_data(r):
data = {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')}
data.update({k:r[k] for k in ('sourceModeExplicit','sourceMode','sourceReadiness') if k in r})
return data
def manifest_platforms(image_ref):
"""Platforms carried by the pushed image manifest (registry read, no cluster access).
2026-10-14: an arm64-only manifest built on Apple Silicon passed every gate
and wedged the rollout on ImagePullBackOff ("no match for platform") —
cluster nodes are linux/amd64. check --local refuses images without an
amd64 entry so the class of failure dies before deploy.
"""
out = run(['docker', 'buildx', 'imagetools', 'inspect', image_ref], timeout=180)
return sorted(set(re.findall(r'Platform:\s*(\S+/\S+)', out)))
def prepare(a):
p, s = policy(), source()
branch = run(['git', 'branch', '--show-current']).strip()
require(not re.search(r'(^|/)poc($|[-/])', branch), 'POC branches cannot prepare Fanyu releases.')
require(not s['dirty'], 'Prepare requires a clean checkout (use an isolated checkout for unrelated edits).')
require_mainline(s['commit'])
require(bool(re.fullmatch(r'[^\s@]+@sha256:[0-9a-f]{64}', a.image)), 'Use an immutable repository@sha256 image.')
require(a.server.startswith('https://') and '@' not in a.server and '?' not in a.server, 'Use a credential-free HTTPS Kubernetes server URL.')
release_id = a.id or dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ-') + uuid.uuid4().hex[:8]
require(not record_path(release_id).exists(), 'Release id already exists.')
r = {'schemaVersion': 1, 'id': release_id, 'createdAt': now(), 'commit': s['commit'],
'image': a.image, 'policyHash': digest(p), 'purpose': a.purpose, 'rollback': a.rollback,
'target': {'kubeconfig': str(Path(a.kubeconfig).resolve()), 'context': a.context,
'server': a.server.rstrip('/'), 'namespace': p['namespace'], 'namespaceUID': a.namespace_uid},
'state': 'prepared', 'localChecks': [], 'baseline': None, 'planHash': None,
'operations': [], 'evidence': {}, 'events': [], 'sourceModeExplicit': True,
'sourceMode': getattr(a, 'source_mode', None), 'sourceReadiness': None}
event(r, 'prepared; no network access')
print(json.dumps({'id': r['id'], 'state': r['state']}, ensure_ascii=False))
def check_identity(r):
return identity(ROOT, r['commit'], r['policyHash'])
def run_local_check(r, argv, expected):
entry = {'argv': argv, 'startedAt': now(), 'identity': expected}
try:
output = run(argv, timeout=1800, env=check_environment())
except Blocked as e:
entry.update(result='failed', finishedUnix=time.time())
entry.update(write_log(record_path(r['id']).parent/'logs', argv, getattr(e, 'check_output', ''), 'failed'))
r['localChecks'].append(entry)
r['state'] = 'blocked'
event(r, 'local check failed; private redacted log: ' + entry['log'])
raise Blocked(str(e) + ' Log: ' + entry['log']) from None
entry.update(result='passed', outputHash=digest(output), finishedUnix=time.time())
entry.update(write_log(record_path(r['id']).parent/'logs', argv, output, 'passed'))
return entry
def check(r, a):
fresh(r)
require(not r['operations'], 'A deployment was attempted; check outcome with verify, never rewrite its plan.')
r['planHash'] = None
r['evidence'] = {}
r['state'] = 'checking'
event(r, 'check started; previous plan invalidated')
if a.local:
inspect_argv = ['docker', 'buildx', 'imagetools', 'inspect', r['image']]
started = now()
try:
platforms = manifest_platforms(r['image'])
require('linux/amd64' in platforms,
'Image manifest lacks linux/amd64 (has: %s); rebuild with --platform linux/amd64 before releasing.'
% (', '.join(platforms) or 'no platforms'))
except Blocked:
r['localChecks'] = [{'argv': inspect_argv, 'startedAt': started, 'result': 'failed'}]
r['state'] = 'blocked'
event(r, 'local check failed')
raise
previous = {tuple(x['argv']): x for x in r['localChecks']}
expected = check_identity(r)
r['localChecks'] = []
for argv in policy()['localChecks']:
old = previous.get(tuple(argv), {})
if getattr(a, 'reuse_successful', False) and reusable(old, expected, argv):
entry = {**old, 'reusedAt': now()}
else:
entry = run_local_check(r, argv, expected)
r['localChecks'].append(entry)
save(r)
require(check_identity(r) == expected, 'Check environment/dependencies changed; evidence is not reusable.')
fresh(r)
if a.cluster:
# Never preserve an earlier successful snapshot when a new read fails.
r['baseline'] = None
save(r)
try:
s = snapshot(r)
require(all(d['ready'] for d in s['deployments'].values()), 'Current deployments are not fully ready.')
r['baseline'] = s
if strict_transition(r):
path = getattr(a, 'source_readiness_file', None)
require(bool(path), 'Strict activation needs a fresh deployment-wide --source-readiness-file.')
api = s['deployments']['droboticflow-api']
try:
r['sourceReadiness'] = validate_report(path, namespace=r['target']['namespace'], database=policy()['database'],
cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit'])
except (ValueError,KeyError,TypeError,AttributeError,OSError):
raise Blocked('Strict readiness incomplete, blocked, expired or wrong scope; keep advisory.') from None
except Blocked:
r['state'] = 'blocked'
event(r, 'cluster check failed')
raise
r['state'] = 'checked_partial'
event(r, 'checks recorded; plan must still be generated')
def target_source_mode(r, name):
return r.get('sourceMode') or r['baseline']['deployments'][name]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE')
def validate_planned_source_mode(r):
if r.get("sourceModeExplicit"):
require(all(target_source_mode(r,n) in ("advisory","strict") for n in r["baseline"]["deployments"]), "New releases require an explicit planned source mode.")
def strict_transition(r):
if not any(target_source_mode(r,n)=='strict' for n in r['baseline']['deployments']): return False
if any(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE')!='strict' for d in r['baseline']['deployments'].values()): return True
return strict_rules_changed(r)
def strict_rules_changed(r):
live=r['baseline']['deployments']['droboticflow-api']['buildCommit']
if live==r['commit']: return False
require(isinstance(live,str) and bool(re.fullmatch(r'[0-9a-f]{40}',live)), 'Unknown current source; strict activation requires advisory preparation first.')
paths=['internal/store/annotation_source_admission.go','internal/store/annotation_availability.go',
'internal/api/annotation_source_admission.go','internal/api/annotation_source_preparation.go',
'internal/api/annotation_source_certificate.go','internal/api/annotation_source_availability.go']
return bool(run(['git','diff','--name-only',live,r['commit'],'--',*paths]).strip())
def validate_source_transition(r):
if not strict_transition(r): return
proof = r.get('sourceReadiness')
require(bool(proof), 'Strict activation requires a complete deployment-wide source readiness report.')
require(not strict_rules_changed(r), 'Strict admission rules changed; deploy advisory first, then inventory with the candidate rules.')
api = r['baseline']['deployments']['droboticflow-api']
try:
current = validate_report(proof['path'], namespace=r['target']['namespace'], database=policy()['database'],
cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit'])
require(current['sha256'] == proof['sha256'], 'Source readiness report changed; recheck and approve a new plan.')
except (ValueError, KeyError, TypeError, AttributeError, OSError) as e:
raise Blocked('Source readiness report invalid: ' + type(e).__name__) from None
def make_plan(r):
fresh(r)
require(not r['operations'], 'Deployment already attempted.')
require([x['argv'] for x in r['localChecks']] == policy()['localChecks']
and all(x['result'] == 'passed' for x in r['localChecks']), 'Required local checks incomplete.')
require(r['baseline'] is not None, 'Read-only cluster check required.')
validate_planned_source_mode(r)
validate_source_transition(r)
r['planHash'] = digest(plan_data(r))
r['state'] = 'awaiting_approval'
event(r, 'plan prepared; no authorization inferred')
print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit'], 'sourceMode': {'from': r['baseline']['deployments'][n]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE'), 'to': target_source_mode(r,n)}} for n in sorted(r['baseline']['deployments'])],
'sequence': 'Processor first; verify its complete rollout before patching API.',
'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'],
'warnings': ['Only image/build stamp and the explicitly planned source mode may change; startup migrations require change-safety evidence.',
'Image-to-source provenance requires independent evidence.',
'Bandwidth adequacy and business acceptance remain manual evidence.'],
'rollback': r['rollback']}, ensure_ascii=False, indent=2))
def deploy(r, a):
fresh(r)
require(r['state'] == 'awaiting_approval' and not r['operations'], 'No fresh deployable plan; inspect status.')
require(r['planHash'] == a.confirm_plan == digest(plan_data(r)), 'Plan hash does not match.')
require(bool(a.approval_ref.strip()), 'Record the explicit user approval reference for this exact plan.')
require_mainline(r['commit'], refresh=True)
validate_planned_source_mode(r)
validate_source_transition(r)
age = (dt.datetime.now(dt.timezone.utc) - dt.datetime.fromisoformat(r['baseline']['at'])).total_seconds()
require(0 <= age <= 3600, 'Cluster check is older than one hour; recheck and obtain a new plan approval.')
for key in ('image-provenance', 'change-safety'):
require(key in r['evidence'] and evidence_valid(r['evidence'][key], r), key + ' evidence required before deployment.')
current = snapshot(r)
require(stable(current) == stable(r['baseline']), 'Live resources changed since check; prepare/check a new plan.')
require(all(d['ready'] for d in current['deployments'].values()), 'Live deployment not ready.')
r['approval'] = {'reference': a.approval_ref, 'at': now(), 'planHash': r['planHash']}
r['state'] = 'deploying'
event(r, 'explicit approval reference recorded; deployment starting')
# The processor must understand new durable payloads before API replicas can
# enqueue them. The fixed Fanyu names sort processor before API.
for name in sorted(current['deployments']):
latest = snapshot(r, allow_planned_transition=True)
require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.')
for n, live in latest['deployments'].items():
old = current['deployments'][n]
done = any(o['deployment'] == n and o['result'] == 'acknowledged' for o in r['operations'])
require(live['uid'] == old['uid'] and live['nonReleaseHash'] == old['nonReleaseHash'], 'Deployment changed during release.')
require((live['image'] == r['image'] and live['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or live['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n))) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.')
dep = latest['deployments'][name]
patch = [{'op': 'test', 'path': '/metadata/resourceVersion', 'value': dep['resourceVersion']},
{'op': 'test', 'path': '/metadata/uid', 'value': dep['uid']},
{'op': 'test', 'path': f"/spec/template/spec/containers/{dep['containerIndex']}/image", 'value': dep['image']},
{'op': 'replace', 'path': f"/spec/template/spec/containers/{dep['containerIndex']}/image", 'value': r['image']}]
stamp = {'name': 'DROBOTICFLOW_BUILD_COMMIT', 'value': r['commit']}
env_path = f"/spec/template/spec/containers/{dep['containerIndex']}/env/"
patch.append({'op': 'add' if dep['buildCommitIndex'] is None else 'replace',
'path': env_path + ('-' if dep['buildCommitIndex'] is None else str(dep['buildCommitIndex'])), 'value': stamp})
if r.get('sourceMode'):
mode_index = dep['sourceModeIndex']
# If the stamp is appended first, a pre-existing mode keeps its index.
patch.append({'op': 'add' if mode_index is None else 'replace',
'path': env_path + ('-' if mode_index is None else str(mode_index)),
'value': {'name': 'ANNOTATION_SOURCE_VALIDATION_MODE', 'value': r['sourceMode']}})
op = {'deployment': name, 'before': dep['image'], 'desired': r['image'], 'at': now(), 'result': 'unknown'}
r['operations'].append(op)
save(r) # Persist intent before sending any mutation.
try:
kubectl(r, ['patch', 'deployment', name, '--type=json', '--patch-file=/dev/stdin'], stdin=json.dumps(patch))
except Blocked:
r['state'] = 'outcome_unknown'
event(r, 'patch outcome unknown; verify before any further operation')
raise
op['result'] = 'acknowledged'
save(r)
if policy()['deployments'][name] == 'processor':
event(r, 'processor patch acknowledged; waiting for rollout before API')
try:
kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330)
ready = snapshot(r, allow_planned_transition=True)['deployments'][name]
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or ready['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,name)),
'Processor rollout has not converged to the candidate.')
except Blocked:
r['state'] = 'deployed_pending_verification'
event(r, 'processor rollout not verified; API was not patched; inspect partial outcome')
raise
r['state'] = 'deployed_pending_verification'
event(r, 'image patches acknowledged; rollout and acceptance not yet verified')
def record_evidence(r, a):
actionable(r)
require(r['planHash'] is not None, 'Generate a plan before attaching evidence.')
require(a.check in policy()['requiredEvidence'], 'Unknown evidence category.')
require(a.check in ('image-provenance', 'change-safety') or r['state'] in ('deployed_pending_acceptance', 'accepted_with_evidence'),
'Business evidence can be attached only after infrastructure verification.')
path = Path(a.file).resolve()
require(path.is_file() and path.stat().st_size > 0, 'Evidence must be a nonempty local file.')
r['evidence'][a.check] = {'result': a.result, 'path': str(path),
'sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'at': now(), 'planHash': r['planHash'], 'note': a.note,
'kind': 'operator-attested; content not automatically evaluated'}
if a.result != 'passed' and r['state'] == 'accepted_with_evidence':
r['state'] = 'deployed_pending_acceptance'
event(r, 'evidence recorded: ' + a.check)
def evidence_valid(e, r):
# Historical attestations may not contain file hashes or plan bindings.
# Display them without manufacturing a current acceptance pass.
try:
path = Path(e['path'])
return (e['result'] == 'passed' and e['planHash'] == r['planHash'] and path.is_file()
and hashlib.sha256(path.read_bytes()).hexdigest() == e['sha256'])
except (KeyError, TypeError, ValueError, OSError):
return False
def verify(r):
actionable(r)
require(bool(r['operations']), 'No deployment attempt in this record.')
# Failure cannot retain a stale accepted state.
r['state'] = 'deployed_pending_verification'
save(r)
require(digest(policy()) == r['policyHash'], 'Policy changed; verify with the original policy revision.')
s = snapshot(r)
r['lastObservation'] = s
expected = r['baseline']
require(all(s[k] == expected[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration drift detected.')
for n, d in s['deployments'].items():
require(d['uid'] == expected['deployments'][n]['uid'], 'Deployment was replaced.')
require(d['nonReleaseHash'] == expected['deployments'][n]['nonReleaseHash'], 'Deployment settings drifted beyond the approved image/build stamp.')
if r.get('sourceModeExplicit'): require(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n), 'Source mode differs from approved plan.')
require(d['image'] == r['image'] and d['buildCommit'] == r['commit'] and d['ready'], 'Target image rollout incomplete; outcome remains pending.')
r['state'] = 'deployed_pending_acceptance'
missing = [k for k in policy()['requiredEvidence'] if k not in r['evidence'] or not evidence_valid(r['evidence'][k], r)]
if not missing:
r['state'] = 'accepted_with_evidence'
event(r, 'infrastructure verified; remaining evidence: ' + ', '.join(missing))
def status(r):
p = policy()
fmt = record_format(r)
evidence = r.get('evidence') if isinstance(r.get('evidence'), dict) else {}
missing = [k for k in p['requiredEvidence'] if k not in evidence or not evidence_valid(evidence[k], r)]
state = r.get('state', 'unknown')
if fmt != 'v1':
state = fmt
if fmt == 'v1' and state == 'accepted_with_evidence' and missing:
state = 'acceptance_evidence_invalid'
if r.get('supersession'):
state = 'superseded'
events = r.get('events') if isinstance(r.get('events'), list) else []
print(json.dumps({'id': r.get('id'), 'state': state, 'recordedState': r.get('state'),
'recordFormat': fmt, 'executableFormat': fmt == 'v1' and not bool(r.get('supersession')),
'commit': r.get('commit'), 'image': r.get('image'),
'policyMatches': digest(p) == r.get('policyHash'), 'checkout': source(),
'planHash': r.get('planHash'), 'missingEvidence': missing,
'operations': r.get('operations', []), 'lastEvent': events[-1] if events else None,
'supersession': r.get('supersession'),
'next': ('Historical/incomplete record: read-only display; original claims are not revalidated. Prepare a new record for release work.'
if fmt != 'v1' else 'Use replacement record; do not resume this superseded plan.' if r.get('supersession')
else 'Resume from recorded state; fresh approval is required for any new deployment or rollback.')},
indent=2, ensure_ascii=False))
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest='command', required=True)
prep = sub.add_parser('prepare', help='Offline: create a release record from a clean committed checkout.')
prep.add_argument('--id')
for name in ('image', 'kubeconfig', 'context', 'server', 'namespace-uid', 'purpose', 'rollback'):
prep.add_argument('--' + name, required=True)
for cmd in ('check', 'plan', 'deploy', 'verify', 'status', 'evidence', 'supersede'):
c = sub.add_parser(cmd)
if cmd == 'status':
c.add_argument('id', nargs='?')
else:
c.add_argument('id')
if cmd == 'supersede':
c.add_argument('--by', required=True)
c.add_argument('--reason', required=True)
elif cmd == 'check':
c.add_argument('--local', action='store_true')
c.add_argument('--cluster', action='store_true')
c.add_argument('--source-readiness-file')
c.add_argument('--reuse-successful', action='store_true', help='Reuse only unchanged successful pure local checks; database/browser checks always rerun.')
elif cmd == 'deploy':
c.add_argument('--confirm-plan', required=True)
c.add_argument('--approval-ref', required=True)
elif cmd == 'evidence':
c.add_argument('--check', required=True)
c.add_argument('--result', choices=['passed', 'failed', 'unverified'], required=True)
c.add_argument('--file', required=True)
c.add_argument('--note', required=True)
prep.add_argument('--source-mode', choices=['strict','advisory'], help='Optional reviewed source-mode change in the same image plan.')
a = parser.parse_args()
try:
if a.command == 'prepare':
prepare(a)
return
if a.command == 'status' and not a.id:
summaries = []
for path in sorted(RECORDS.glob('*/release.json')):
item = json.loads(path.read_text())
summary = {k: item.get(k) for k in ('id', 'state', 'commit', 'createdAt')}
summary['recordFormat'] = record_format(item)
if summary['recordFormat'] != 'v1':
summary.update(state=summary['recordFormat'], recordedState=item.get('state'))
if item.get('supersession'):
summary.update(state='superseded', recordedState=item.get('state'), supersededBy=item['supersession']['by'])
summaries.append(summary)
print(json.dumps(summaries, indent=2, ensure_ascii=False))
return
with locked(a.id) as r:
if a.command == 'supersede':
supersede(r, a)
elif a.command == 'check':
require(a.local or a.cluster, 'Choose --local and/or --cluster.')
check(r, a)
elif a.command == 'plan':
make_plan(r)
elif a.command == 'deploy':
try:
deploy(r, a)
except Blocked:
if r['state'] == 'deploying':
r['state'] = 'outcome_unknown'
event(r, 'interrupted; inspect partial outcome before further changes')
raise
elif a.command == 'verify':
verify(r)
elif a.command == 'evidence':
record_evidence(r, a)
status(r)
except (Blocked, KeyError, ValueError, OSError) as e:
# Validation errors are authored here; generic errors omit possibly sensitive values.
print('BLOCKED: ' + (str(e) if isinstance(e, Blocked) else type(e).__name__), file=sys.stderr)
sys.exit(2)
if __name__ == '__main__':
main()