578 lines
31 KiB
Python
578 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
"""Fanyu image/build-stamp release journal. Secret values are never persisted."""
|
|
import argparse
|
|
import contextlib
|
|
import copy
|
|
import datetime as dt
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
RECORDS = ROOT / 'local-data/releases'
|
|
POLICY = ROOT / 'docs/release/fanyu-policy.json'
|
|
SAFE_ENV = ('MONGO_DB', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH',
|
|
'ANNOTATION_PROCESSOR_MODE', 'ANNOTATION_PROCESSOR_CONCURRENCY',
|
|
'ANNOTATION_PROCESSOR_ROLE', 'ANNOTATION_SOURCE_INDEX', 'ANNOTATION_INDEX_CONCURRENCY',
|
|
'DROBOTICFLOW_DATABASE_BOOTSTRAP', 'DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE',
|
|
'ANNOTATION_INDEX_BACKFILL', 'ANNOTATION_WORKSPACE_CONNECTION_ID',
|
|
'ANNOTATION_WORKSPACE_BUCKET', 'WORKSPACE_MOUNT_PATH')
|
|
|
|
|
|
class Blocked(Exception):
|
|
pass
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise Blocked(message)
|
|
|
|
|
|
def now():
|
|
return dt.datetime.now(dt.timezone.utc).isoformat()
|
|
|
|
|
|
def digest(value):
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
|
|
|
|
|
def run(argv, *, stdin=None, timeout=60):
|
|
try:
|
|
p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
raise Blocked('Command unavailable or timed out; outcome must be checked before retry.') from None
|
|
# Do not echo kubectl output, URLs or credential-bearing errors.
|
|
require(p.returncode == 0, 'Command failed: ' + argv[0] + ' (output withheld; no success recorded).')
|
|
return p.stdout
|
|
|
|
|
|
def source():
|
|
return {'commit': run(['git', 'rev-parse', 'HEAD']).strip(),
|
|
'dirty': bool(run(['git', 'status', '--porcelain', '--untracked-files=normal']).strip())}
|
|
|
|
|
|
def policy():
|
|
return json.loads(POLICY.read_text())
|
|
|
|
|
|
def record_path(release_id):
|
|
require(bool(re.fullmatch(r'[a-zA-Z0-9][a-zA-Z0-9_-]{0,79}', release_id)), 'Invalid release id.')
|
|
return RECORDS / release_id / 'release.json'
|
|
|
|
|
|
def save(r):
|
|
path = record_path(r['id'])
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
temp = path.with_suffix('.tmp')
|
|
with os.fdopen(os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
|
|
json.dump(r, f, indent=2, ensure_ascii=False)
|
|
f.write('\n')
|
|
os.replace(temp, path)
|
|
|
|
|
|
def event(r, name):
|
|
r['events'].append({'at': now(), 'event': name})
|
|
save(r)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def locked(release_id):
|
|
path = record_path(release_id)
|
|
require(path.exists(), 'Release record not found; use prepare first.')
|
|
with path.with_suffix('.lock').open('a') as f:
|
|
try:
|
|
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise Blocked('Another command is updating this release.') from None
|
|
yield json.loads(path.read_text())
|
|
|
|
|
|
def record_format(r):
|
|
required = ('id', 'commit', 'image', 'policyHash', 'target', 'localChecks',
|
|
'baseline', 'planHash', 'operations', 'evidence', 'events', 'state')
|
|
if r.get('schemaVersion') != 1 or any(k not in r for k in required):
|
|
return 'historical-readonly' if r.get('schemaVersion') is None else 'incomplete-readonly'
|
|
if not isinstance(r['evidence'], dict) or not isinstance(r['events'], list):
|
|
return 'incomplete-readonly'
|
|
return 'v1'
|
|
|
|
|
|
def actionable(r):
|
|
require(record_format(r) == 'v1', 'Historical/incomplete record is read-only; prepare a new release, never synthesize approval or evidence.')
|
|
require(not r.get('supersession'), 'This plan has been superseded; use status to inspect its replacement.')
|
|
|
|
|
|
def supersede(r, a):
|
|
actionable(r)
|
|
require(not r['operations'], 'Only unexecuted plans can be superseded; deployed records retain their outcomes.')
|
|
require(a.by != r['id'] and bool(a.reason.strip()), 'Provide a different replacement and a reason.')
|
|
path = record_path(a.by)
|
|
require(path.is_file(), 'Replacement record does not exist.')
|
|
replacement = json.loads(path.read_text())
|
|
require(record_format(replacement) == 'v1' and not replacement.get('supersession'), 'Replacement must be a current v1 record.')
|
|
require(all(r['target'].get(k) == replacement['target'].get(k) for k in ('server', 'context', 'namespace', 'namespaceUID')),
|
|
'Replacement must target the same verified environment.')
|
|
original = record_path(r['id']).read_bytes()
|
|
backup = record_path(r['id']).with_name('release-before-supersession-' + hashlib.sha256(original).hexdigest()[:16] + '.json')
|
|
with backup.open('xb') as f:
|
|
os.chmod(backup, 0o600)
|
|
f.write(original)
|
|
r['supersession'] = {'by': a.by, 'reason': a.reason.strip(), 'at': now(),
|
|
'backup': backup.name, 'backupSHA256': hashlib.sha256(original).hexdigest()}
|
|
event(r, 'unexecuted plan superseded by ' + a.by + '; original state/evidence retained')
|
|
|
|
|
|
def fresh(r):
|
|
actionable(r)
|
|
s = source()
|
|
require(not s['dirty'], 'Checkout is dirty; commit task changes in a clean checkout first.')
|
|
require(s['commit'] == r['commit'], 'Checkout commit changed; prepare a new release.')
|
|
require(digest(policy()) == r['policyHash'], 'Release policy changed; prepare a new release.')
|
|
|
|
|
|
def kubectl(r, args, *, stdin=None, timeout=60):
|
|
t = r['target']
|
|
return run(['kubectl', '--kubeconfig', t['kubeconfig'], '--context', t['context'],
|
|
'--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=timeout)
|
|
|
|
|
|
def get(r, kind, name):
|
|
return json.loads(kubectl(r, ['get', kind, name, '-o', 'json']))
|
|
|
|
|
|
def env_values(container, configs):
|
|
values = {}
|
|
for entry in container.get('envFrom', []):
|
|
ref = entry.get('configMapRef')
|
|
require(bool(ref), 'Secret/unknown envFrom is unsupported by this preflight.')
|
|
for key, value in configs[ref['name']].get('data', {}).items():
|
|
values[entry.get('prefix', '') + key] = value
|
|
for entry in container.get('env', []):
|
|
key = entry['name']
|
|
if 'value' in entry:
|
|
values[key] = entry['value']
|
|
elif 'configMapKeyRef' in entry.get('valueFrom', {}):
|
|
ref = entry['valueFrom']['configMapKeyRef']
|
|
require(ref['key'] in configs[ref['name']].get('data', {}), 'Missing ConfigMap key.')
|
|
values[key] = configs[ref['name']]['data'][ref['key']]
|
|
else:
|
|
# Explicit unknowns must override envFrom rather than silently use its value.
|
|
values[key] = None
|
|
return values
|
|
|
|
|
|
def inspect_deployment(deployment, container_name, configs, p):
|
|
spec = deployment['spec']
|
|
pod = spec['template']['spec']
|
|
containers = pod['containers']
|
|
matches = [(i, c) for i, c in enumerate(containers) if c['name'] == container_name]
|
|
require(len(matches) == 1, 'Expected deployment container missing.')
|
|
index, c = matches[0]
|
|
e = env_values(c, configs)
|
|
require(e.get('MONGO_DB') == p['database'], 'Database name mismatch.')
|
|
require(e.get('ANNOTATION_WORKSPACE_BUCKET') == p['bucket'], 'Workspace bucket mismatch.')
|
|
require(bool(e.get('ANNOTATION_WORKSPACE_CONNECTION_ID')), 'Missing workspace storage connection.')
|
|
require(e.get('WORKSPACE_MOUNT_PATH') == p['workspaceMount'], 'Workspace mount mismatch.')
|
|
mongo = next((x for x in c.get('env', []) if x['name'] == 'MONGO_URI'), {})
|
|
require(mongo.get('valueFrom', {}).get('secretKeyRef') == {'name': p['mongoSecret'], 'key': 'MONGO_URI'},
|
|
'Mongo secret binding mismatch; preflight does not read secret values.')
|
|
mounts = {v['name']: v['mountPath'] for v in c.get('volumeMounts', [])}
|
|
claims = [v for v in pod.get('volumes', []) if mounts.get(v['name']) == p['workspaceMount']]
|
|
require(len(claims) == 1 and claims[0].get('persistentVolumeClaim', {}).get('claimName') == p['workspacePVC'],
|
|
'Workspace PVC binding mismatch.')
|
|
require(spec.get('replicas', 1) > 0, 'Deployment has no replicas.')
|
|
if container_name == 'api':
|
|
require(e.get('DROBOTICFLOW_REQUIRE_SESSION_AUTH') in ('true', '1'), 'Session authentication disabled.')
|
|
require(e.get('ANNOTATION_PROCESSOR_MODE') == 'external', 'API must use external processor.')
|
|
else:
|
|
require(e.get('ANNOTATION_PROCESSOR_MODE') == 'processor', 'Processor mode mismatch.')
|
|
require(e.get('ANNOTATION_INDEX_BACKFILL') == '0', 'Historical backfill must remain disabled.')
|
|
if e.get('ANNOTATION_PROCESSOR_ROLE') == 'revision':
|
|
# Recognize the previously approved deployed profile, never resize
|
|
# it. nonReleaseHash and conditional patches still prohibit drift.
|
|
profile = json.loads((ROOT / p['processorCapacityProfile']).read_text())
|
|
require(profile['schemaVersion'] == 1 and profile['profile'] == 'fanyu-source-index-100-v1'
|
|
and profile['namespace'] == p['namespace'], 'Unsupported capacity profile.')
|
|
desired = profile['deployments']['droboticflow-annotation-processor']
|
|
require(all(e.get(k) == v for k, v in desired['env'].items()), 'Source-index processor settings differ from approved capacity profile.')
|
|
require(c.get('resources') == {k: desired[k] for k in ('requests', 'limits')},
|
|
'Source-index processor resources differ from approved capacity profile.')
|
|
require(spec.get('replicas') == desired['replicas'] and spec.get('strategy', {}).get('type') == 'Recreate',
|
|
'Source-index processor replicas/strategy differ from approved deployed profile.')
|
|
else:
|
|
require(not e.get('ANNOTATION_PROCESSOR_ROLE'), 'Unsupported processor role.')
|
|
require(e.get('ANNOTATION_PROCESSOR_CONCURRENCY') == '1', 'First release profile requires concurrency 1.')
|
|
require(c.get('resources', {}).get('limits', {}).get('memory') == '8Gi', 'First release profile requires processor memory limit 8Gi.')
|
|
require(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate',
|
|
'Processor must have one replica and Recreate strategy.')
|
|
status = deployment.get('status', {})
|
|
ready = (status.get('observedGeneration', 0) >= deployment['metadata'].get('generation', 1)
|
|
and status.get('updatedReplicas', 0) == spec.get('replicas', 1)
|
|
and status.get('readyReplicas', 0) == spec.get('replicas', 1)
|
|
and status.get('replicas', 0) == spec.get('replicas', 1))
|
|
m = deployment['metadata']
|
|
neutral = copy.deepcopy(spec)
|
|
nc = neutral['template']['spec']['containers'][index]
|
|
nc['image'] = '<release-image>'
|
|
nc['env'] = [x for x in nc.get('env', []) if x['name'] != 'DROBOTICFLOW_BUILD_COMMIT']
|
|
stamp = next((i for i, x in enumerate(c.get('env', [])) if x['name'] == 'DROBOTICFLOW_BUILD_COMMIT'), None)
|
|
return {'uid': m['uid'], 'specHash': digest(spec), 'nonReleaseHash': digest(neutral),
|
|
'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'],
|
|
'resourceVersion': m['resourceVersion'], 'containerIndex': index,
|
|
'ready': ready, 'env': {k: e.get(k) for k in SAFE_ENV}}
|
|
|
|
|
|
def snapshot(r):
|
|
t, p = r['target'], policy()
|
|
view = json.loads(kubectl(r, ['config', 'view', '--minify', '-o', 'json']))
|
|
require(view['clusters'][0]['cluster']['server'] == t['server'], 'Kube API server mismatch.')
|
|
ns = get(r, 'namespace', t['namespace'])
|
|
require(ns['metadata']['uid'] == t['namespaceUID'], 'Namespace UID mismatch.')
|
|
deployments = {n: get(r, 'deployment', n) for n in p['deployments']}
|
|
config_names = set()
|
|
for d in deployments.values():
|
|
for c in d['spec']['template']['spec']['containers']:
|
|
config_names.update(x['configMapRef']['name'] for x in c.get('envFrom', []) if 'configMapRef' in x)
|
|
config_names.update(x['valueFrom']['configMapKeyRef']['name'] for x in c.get('env', [])
|
|
if 'configMapKeyRef' in x.get('valueFrom', {}))
|
|
configs = {n: get(r, 'configmap', n) for n in sorted(config_names)}
|
|
deps = {n: inspect_deployment(d, p['deployments'][n], configs, p) for n, d in deployments.items()}
|
|
envs = [d['env'] for d in deps.values()]
|
|
require(len({e['ANNOTATION_WORKSPACE_CONNECTION_ID'] for e in envs}) == 1, 'API/processor storage connections differ.')
|
|
pvc = get(r, 'pvc', p['workspacePVC'])
|
|
require(pvc.get('status', {}).get('phase') == 'Bound', 'Workspace PVC is not Bound.')
|
|
svc = get(r, 'service', 'droboticflow-api-public-eip')
|
|
ann = svc['metadata'].get('annotations', {})
|
|
return {'at': now(), 'deployments': deps,
|
|
'configs': {n: {'uid': x['metadata']['uid'], 'hash': digest(x.get('data', {}))} for n, x in configs.items()},
|
|
'pvc': {'uid': pvc['metadata']['uid'], 'specHash': digest(pvc['spec'])},
|
|
'publicService': {'uid': svc['metadata']['uid'], 'specHash': digest(svc['spec']),
|
|
'annotationHash': digest(ann),
|
|
'bandwidthMbps': ann.get('service.beta.kubernetes.io/cce-elastic-ip-bandwidth-in-mbps', 'unknown')}}
|
|
|
|
|
|
def stable(s):
|
|
return {**{k: s[k] for k in ('configs', 'pvc', 'publicService')},
|
|
'deployments': {n: {k: d[k] for k in ('uid', 'specHash', 'image')} for n, d in s['deployments'].items()}}
|
|
|
|
|
|
def plan_data(r):
|
|
return {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')}
|
|
|
|
|
|
def manifest_platforms(image_ref):
|
|
"""Platforms carried by the pushed image manifest (registry read, no cluster access).
|
|
|
|
2026-10-14: an arm64-only manifest built on Apple Silicon passed every gate
|
|
and wedged the rollout on ImagePullBackOff ("no match for platform") —
|
|
cluster nodes are linux/amd64. check --local refuses images without an
|
|
amd64 entry so the class of failure dies before deploy.
|
|
"""
|
|
out = run(['docker', 'buildx', 'imagetools', 'inspect', image_ref], timeout=180)
|
|
return sorted(set(re.findall(r'Platform:\s*(\S+/\S+)', out)))
|
|
|
|
|
|
def prepare(a):
|
|
p, s = policy(), source()
|
|
branch = run(['git', 'branch', '--show-current']).strip()
|
|
require(not re.search(r'(^|/)poc($|[-/])', branch), 'POC branches cannot prepare Fanyu releases.')
|
|
require(not s['dirty'], 'Prepare requires a clean checkout (use an isolated checkout for unrelated edits).')
|
|
require(bool(re.fullmatch(r'[^\s@]+@sha256:[0-9a-f]{64}', a.image)), 'Use an immutable repository@sha256 image.')
|
|
require(a.server.startswith('https://') and '@' not in a.server and '?' not in a.server, 'Use a credential-free HTTPS Kubernetes server URL.')
|
|
release_id = a.id or dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ-') + uuid.uuid4().hex[:8]
|
|
require(not record_path(release_id).exists(), 'Release id already exists.')
|
|
r = {'schemaVersion': 1, 'id': release_id, 'createdAt': now(), 'commit': s['commit'],
|
|
'image': a.image, 'policyHash': digest(p), 'purpose': a.purpose, 'rollback': a.rollback,
|
|
'target': {'kubeconfig': str(Path(a.kubeconfig).resolve()), 'context': a.context,
|
|
'server': a.server.rstrip('/'), 'namespace': p['namespace'], 'namespaceUID': a.namespace_uid},
|
|
'state': 'prepared', 'localChecks': [], 'baseline': None, 'planHash': None,
|
|
'operations': [], 'evidence': {}, 'events': []}
|
|
event(r, 'prepared; no network access')
|
|
print(json.dumps({'id': r['id'], 'state': r['state']}, ensure_ascii=False))
|
|
|
|
|
|
def check(r, a):
|
|
fresh(r)
|
|
require(not r['operations'], 'A deployment was attempted; check outcome with verify, never rewrite its plan.')
|
|
r['planHash'] = None
|
|
r['evidence'] = {}
|
|
r['state'] = 'checking'
|
|
event(r, 'check started; previous plan invalidated')
|
|
if a.local:
|
|
inspect_argv = ['docker', 'buildx', 'imagetools', 'inspect', r['image']]
|
|
started = now()
|
|
try:
|
|
platforms = manifest_platforms(r['image'])
|
|
require('linux/amd64' in platforms,
|
|
'Image manifest lacks linux/amd64 (has: %s); rebuild with --platform linux/amd64 before releasing.'
|
|
% (', '.join(platforms) or 'no platforms'))
|
|
except Blocked:
|
|
r['localChecks'] = [{'argv': inspect_argv, 'startedAt': started, 'result': 'failed'}]
|
|
r['state'] = 'blocked'
|
|
event(r, 'local check failed')
|
|
raise
|
|
r['localChecks'] = []
|
|
for argv in policy()['localChecks']:
|
|
started = now()
|
|
try:
|
|
output = run(argv, timeout=1800)
|
|
except Blocked:
|
|
r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'failed'})
|
|
r['state'] = 'blocked'
|
|
event(r, 'local check failed')
|
|
raise
|
|
r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'passed', 'outputHash': digest(output)})
|
|
save(r)
|
|
fresh(r)
|
|
if a.cluster:
|
|
# Never preserve an earlier successful snapshot when a new read fails.
|
|
r['baseline'] = None
|
|
save(r)
|
|
try:
|
|
s = snapshot(r)
|
|
require(all(d['ready'] for d in s['deployments'].values()), 'Current deployments are not fully ready.')
|
|
r['baseline'] = s
|
|
except Blocked:
|
|
r['state'] = 'blocked'
|
|
event(r, 'cluster check failed')
|
|
raise
|
|
r['state'] = 'checked_partial'
|
|
event(r, 'checks recorded; plan must still be generated')
|
|
|
|
|
|
def make_plan(r):
|
|
fresh(r)
|
|
require(not r['operations'], 'Deployment already attempted.')
|
|
require([x['argv'] for x in r['localChecks']] == policy()['localChecks']
|
|
and all(x['result'] == 'passed' for x in r['localChecks']), 'Required local checks incomplete.')
|
|
require(r['baseline'] is not None, 'Read-only cluster check required.')
|
|
r['planHash'] = digest(plan_data(r))
|
|
r['state'] = 'awaiting_approval'
|
|
event(r, 'plan prepared; no authorization inferred')
|
|
print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [
|
|
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit']} for n in sorted(r['baseline']['deployments'])],
|
|
'sequence': 'Processor first; verify its complete rollout before patching API.',
|
|
'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'],
|
|
'warnings': ['Only image/build stamp changes; startup migrations require change-safety evidence.',
|
|
'Image-to-source provenance requires independent evidence.',
|
|
'Bandwidth adequacy and business acceptance remain manual evidence.'],
|
|
'rollback': r['rollback']}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
def deploy(r, a):
|
|
fresh(r)
|
|
require(r['state'] == 'awaiting_approval' and not r['operations'], 'No fresh deployable plan; inspect status.')
|
|
require(r['planHash'] == a.confirm_plan == digest(plan_data(r)), 'Plan hash does not match.')
|
|
require(bool(a.approval_ref.strip()), 'Record the explicit user approval reference for this exact plan.')
|
|
age = (dt.datetime.now(dt.timezone.utc) - dt.datetime.fromisoformat(r['baseline']['at'])).total_seconds()
|
|
require(0 <= age <= 3600, 'Cluster check is older than one hour; recheck and obtain a new plan approval.')
|
|
for key in ('image-provenance', 'change-safety'):
|
|
require(key in r['evidence'] and evidence_valid(r['evidence'][key], r), key + ' evidence required before deployment.')
|
|
current = snapshot(r)
|
|
require(stable(current) == stable(r['baseline']), 'Live resources changed since check; prepare/check a new plan.')
|
|
require(all(d['ready'] for d in current['deployments'].values()), 'Live deployment not ready.')
|
|
r['approval'] = {'reference': a.approval_ref, 'at': now(), 'planHash': r['planHash']}
|
|
r['state'] = 'deploying'
|
|
event(r, 'explicit approval reference recorded; deployment starting')
|
|
# The processor must understand new durable payloads before API replicas can
|
|
# enqueue them. The fixed Fanyu names sort processor before API.
|
|
for name in sorted(current['deployments']):
|
|
latest = snapshot(r)
|
|
require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.')
|
|
for n, live in latest['deployments'].items():
|
|
old = current['deployments'][n]
|
|
done = any(o['deployment'] == n and o['result'] == 'acknowledged' for o in r['operations'])
|
|
require(live['uid'] == old['uid'] and live['nonReleaseHash'] == old['nonReleaseHash'], 'Deployment changed during release.')
|
|
require((live['image'] == r['image'] and live['buildCommit'] == r['commit']) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.')
|
|
dep = latest['deployments'][name]
|
|
patch = [{'op': 'test', 'path': '/metadata/resourceVersion', 'value': dep['resourceVersion']},
|
|
{'op': 'test', 'path': '/metadata/uid', 'value': dep['uid']},
|
|
{'op': 'test', 'path': f"/spec/template/spec/containers/{dep['containerIndex']}/image", 'value': dep['image']},
|
|
{'op': 'replace', 'path': f"/spec/template/spec/containers/{dep['containerIndex']}/image", 'value': r['image']}]
|
|
stamp = {'name': 'DROBOTICFLOW_BUILD_COMMIT', 'value': r['commit']}
|
|
env_path = f"/spec/template/spec/containers/{dep['containerIndex']}/env/"
|
|
patch.append({'op': 'add' if dep['buildCommitIndex'] is None else 'replace',
|
|
'path': env_path + ('-' if dep['buildCommitIndex'] is None else str(dep['buildCommitIndex'])), 'value': stamp})
|
|
op = {'deployment': name, 'before': dep['image'], 'desired': r['image'], 'at': now(), 'result': 'unknown'}
|
|
r['operations'].append(op)
|
|
save(r) # Persist intent before sending any mutation.
|
|
try:
|
|
kubectl(r, ['patch', 'deployment', name, '--type=json', '--patch-file=/dev/stdin'], stdin=json.dumps(patch))
|
|
except Blocked:
|
|
r['state'] = 'outcome_unknown'
|
|
event(r, 'patch outcome unknown; verify before any further operation')
|
|
raise
|
|
op['result'] = 'acknowledged'
|
|
save(r)
|
|
if policy()['deployments'][name] == 'processor':
|
|
event(r, 'processor patch acknowledged; waiting for rollout before API')
|
|
try:
|
|
kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330)
|
|
ready = snapshot(r)['deployments'][name]
|
|
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'],
|
|
'Processor rollout has not converged to the candidate.')
|
|
except Blocked:
|
|
r['state'] = 'deployed_pending_verification'
|
|
event(r, 'processor rollout not verified; API was not patched; inspect partial outcome')
|
|
raise
|
|
r['state'] = 'deployed_pending_verification'
|
|
event(r, 'image patches acknowledged; rollout and acceptance not yet verified')
|
|
|
|
|
|
def record_evidence(r, a):
|
|
actionable(r)
|
|
require(r['planHash'] is not None, 'Generate a plan before attaching evidence.')
|
|
require(a.check in policy()['requiredEvidence'], 'Unknown evidence category.')
|
|
require(a.check in ('image-provenance', 'change-safety') or r['state'] in ('deployed_pending_acceptance', 'accepted_with_evidence'),
|
|
'Business evidence can be attached only after infrastructure verification.')
|
|
path = Path(a.file).resolve()
|
|
require(path.is_file() and path.stat().st_size > 0, 'Evidence must be a nonempty local file.')
|
|
r['evidence'][a.check] = {'result': a.result, 'path': str(path),
|
|
'sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
'at': now(), 'planHash': r['planHash'], 'note': a.note,
|
|
'kind': 'operator-attested; content not automatically evaluated'}
|
|
if a.result != 'passed' and r['state'] == 'accepted_with_evidence':
|
|
r['state'] = 'deployed_pending_acceptance'
|
|
event(r, 'evidence recorded: ' + a.check)
|
|
|
|
|
|
def evidence_valid(e, r):
|
|
# Historical attestations may not contain file hashes or plan bindings.
|
|
# Display them without manufacturing a current acceptance pass.
|
|
try:
|
|
path = Path(e['path'])
|
|
return (e['result'] == 'passed' and e['planHash'] == r['planHash'] and path.is_file()
|
|
and hashlib.sha256(path.read_bytes()).hexdigest() == e['sha256'])
|
|
except (KeyError, TypeError, ValueError, OSError):
|
|
return False
|
|
|
|
|
|
def verify(r):
|
|
actionable(r)
|
|
require(bool(r['operations']), 'No deployment attempt in this record.')
|
|
# Failure cannot retain a stale accepted state.
|
|
r['state'] = 'deployed_pending_verification'
|
|
save(r)
|
|
require(digest(policy()) == r['policyHash'], 'Policy changed; verify with the original policy revision.')
|
|
s = snapshot(r)
|
|
r['lastObservation'] = s
|
|
expected = r['baseline']
|
|
require(all(s[k] == expected[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration drift detected.')
|
|
for n, d in s['deployments'].items():
|
|
require(d['uid'] == expected['deployments'][n]['uid'], 'Deployment was replaced.')
|
|
require(d['nonReleaseHash'] == expected['deployments'][n]['nonReleaseHash'], 'Deployment settings drifted beyond the approved image/build stamp.')
|
|
require(d['image'] == r['image'] and d['buildCommit'] == r['commit'] and d['ready'], 'Target image rollout incomplete; outcome remains pending.')
|
|
r['state'] = 'deployed_pending_acceptance'
|
|
missing = [k for k in policy()['requiredEvidence'] if k not in r['evidence'] or not evidence_valid(r['evidence'][k], r)]
|
|
if not missing:
|
|
r['state'] = 'accepted_with_evidence'
|
|
event(r, 'infrastructure verified; remaining evidence: ' + ', '.join(missing))
|
|
|
|
|
|
def status(r):
|
|
p = policy()
|
|
fmt = record_format(r)
|
|
evidence = r.get('evidence') if isinstance(r.get('evidence'), dict) else {}
|
|
missing = [k for k in p['requiredEvidence'] if k not in evidence or not evidence_valid(evidence[k], r)]
|
|
state = r.get('state', 'unknown')
|
|
if fmt != 'v1':
|
|
state = fmt
|
|
if fmt == 'v1' and state == 'accepted_with_evidence' and missing:
|
|
state = 'acceptance_evidence_invalid'
|
|
if r.get('supersession'):
|
|
state = 'superseded'
|
|
events = r.get('events') if isinstance(r.get('events'), list) else []
|
|
print(json.dumps({'id': r.get('id'), 'state': state, 'recordedState': r.get('state'),
|
|
'recordFormat': fmt, 'executableFormat': fmt == 'v1' and not bool(r.get('supersession')),
|
|
'commit': r.get('commit'), 'image': r.get('image'),
|
|
'policyMatches': digest(p) == r.get('policyHash'), 'checkout': source(),
|
|
'planHash': r.get('planHash'), 'missingEvidence': missing,
|
|
'operations': r.get('operations', []), 'lastEvent': events[-1] if events else None,
|
|
'supersession': r.get('supersession'),
|
|
'next': ('Historical/incomplete record: read-only display; original claims are not revalidated. Prepare a new record for release work.'
|
|
if fmt != 'v1' else 'Use replacement record; do not resume this superseded plan.' if r.get('supersession')
|
|
else 'Resume from recorded state; fresh approval is required for any new deployment or rollback.')},
|
|
indent=2, ensure_ascii=False))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
sub = parser.add_subparsers(dest='command', required=True)
|
|
prep = sub.add_parser('prepare', help='Offline: create a release record from a clean committed checkout.')
|
|
prep.add_argument('--id')
|
|
for name in ('image', 'kubeconfig', 'context', 'server', 'namespace-uid', 'purpose', 'rollback'):
|
|
prep.add_argument('--' + name, required=True)
|
|
for cmd in ('check', 'plan', 'deploy', 'verify', 'status', 'evidence', 'supersede'):
|
|
c = sub.add_parser(cmd)
|
|
if cmd == 'status':
|
|
c.add_argument('id', nargs='?')
|
|
else:
|
|
c.add_argument('id')
|
|
if cmd == 'supersede':
|
|
c.add_argument('--by', required=True)
|
|
c.add_argument('--reason', required=True)
|
|
elif cmd == 'check':
|
|
c.add_argument('--local', action='store_true')
|
|
c.add_argument('--cluster', action='store_true')
|
|
elif cmd == 'deploy':
|
|
c.add_argument('--confirm-plan', required=True)
|
|
c.add_argument('--approval-ref', required=True)
|
|
elif cmd == 'evidence':
|
|
c.add_argument('--check', required=True)
|
|
c.add_argument('--result', choices=['passed', 'failed', 'unverified'], required=True)
|
|
c.add_argument('--file', required=True)
|
|
c.add_argument('--note', required=True)
|
|
a = parser.parse_args()
|
|
try:
|
|
if a.command == 'prepare':
|
|
prepare(a)
|
|
return
|
|
if a.command == 'status' and not a.id:
|
|
summaries = []
|
|
for path in sorted(RECORDS.glob('*/release.json')):
|
|
item = json.loads(path.read_text())
|
|
summary = {k: item.get(k) for k in ('id', 'state', 'commit', 'createdAt')}
|
|
summary['recordFormat'] = record_format(item)
|
|
if summary['recordFormat'] != 'v1':
|
|
summary.update(state=summary['recordFormat'], recordedState=item.get('state'))
|
|
if item.get('supersession'):
|
|
summary.update(state='superseded', recordedState=item.get('state'), supersededBy=item['supersession']['by'])
|
|
summaries.append(summary)
|
|
print(json.dumps(summaries, indent=2, ensure_ascii=False))
|
|
return
|
|
with locked(a.id) as r:
|
|
if a.command == 'supersede':
|
|
supersede(r, a)
|
|
elif a.command == 'check':
|
|
require(a.local or a.cluster, 'Choose --local and/or --cluster.')
|
|
check(r, a)
|
|
elif a.command == 'plan':
|
|
make_plan(r)
|
|
elif a.command == 'deploy':
|
|
try:
|
|
deploy(r, a)
|
|
except Blocked:
|
|
if r['state'] == 'deploying':
|
|
r['state'] = 'outcome_unknown'
|
|
event(r, 'interrupted; inspect partial outcome before further changes')
|
|
raise
|
|
elif a.command == 'verify':
|
|
verify(r)
|
|
elif a.command == 'evidence':
|
|
record_evidence(r, a)
|
|
status(r)
|
|
except (Blocked, KeyError, ValueError, OSError) as e:
|
|
# Validation errors are authored here; generic errors omit possibly sensitive values.
|
|
print('BLOCKED: ' + (str(e) if isinstance(e, Blocked) else type(e).__name__), file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|