data-gen-bpu/scripts/package_release.py

74 lines
3.2 KiB
Python

#!/usr/bin/env python3
"""Create a deterministic source-only release; never follow symlinks or copy state."""
from __future__ import annotations
import argparse
import gzip
import hashlib
import io
import importlib.util
import json
import re
from pathlib import Path
import tarfile
try:
from scripts.release_files import BLOCKED, source_files
except ModuleNotFoundError:
# Also support release verification loading this file by absolute path.
spec = importlib.util.spec_from_file_location('bench_release_files', Path(__file__).with_name('release_files.py'))
rules = importlib.util.module_from_spec(spec)
spec.loader.exec_module(rules)
BLOCKED, source_files = rules.BLOCKED, rules.source_files
def add_bytes(archive, name, data, mode=0o644):
info = tarfile.TarInfo(name)
info.size, info.mode, info.mtime = len(data), mode, 0
info.uid = info.gid = 0
archive.addfile(info, io.BytesIO(data))
def package(root: Path, output: Path):
if output.exists():
raise ValueError(f'refusing to overwrite existing release: {output}')
if root == output or root in output.parents:
raise ValueError('release output must be outside the source tree')
entries = []
# Read source once; hashes describe exactly the bytes stored despite concurrent edits.
payloads = []
for p in sorted(source_files(root)):
raw = p.read_bytes()
if re.search(rb'-----BEGIN [A-Z ]*PRIVATE KEY-----', raw):
raise ValueError(f'private key marker in source: {p.relative_to(root)}')
mode = 0o755 if p.stat().st_mode & 0o111 else 0o644
entry = {'path': p.relative_to(root).as_posix(), 'sha256': hashlib.sha256(raw).hexdigest(),
'size': len(raw), 'mode': oct(mode)}
entries.append(entry)
payloads.append((entry, raw, mode))
manifest = {'schema': 'bench-source-release.v1', 'files': entries,
'excludes': sorted(BLOCKED), 'symlinks': 'excluded',
'scope': 'control-plane source only; external runtimes and data are not included'}
manifest_bytes = (json.dumps(manifest, indent=2, ensure_ascii=False) + '\n').encode()
output.parent.mkdir(parents=True, exist_ok=True)
with output.open('xb') as dest:
with gzip.GzipFile(filename='', mode='wb', fileobj=dest, mtime=0) as gz:
with tarfile.open(fileobj=gz, mode='w') as archive:
for entry, raw, mode in payloads:
add_bytes(archive, 'bench/' + entry['path'], raw, mode)
add_bytes(archive, 'bench/RELEASE_MANIFEST.json', manifest_bytes)
digest = hashlib.sha256(output.read_bytes()).hexdigest()
return {'archive': str(output), 'sha256': digest, 'files': len(entries),
'manifest_sha256': hashlib.sha256(manifest_bytes).hexdigest()}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[1])
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
try:
result = package(args.root.resolve(), args.output.resolve())
except (ValueError, OSError) as exc:
parser.exit(2, f'{exc}\n')
print(json.dumps(result, indent=2))
if __name__ == '__main__':
main()