droboticflow-new/scripts/build_fanyu_v5_s100_bundle.py

332 lines
15 KiB
Python

#!/usr/bin/env python3
"""Build a deterministic, secret-free Fanyu v5 S100 edge deployment bundle."""
from __future__ import annotations
import argparse
import gzip
import hashlib
import io
import json
import os
import re
import shutil
import struct
import subprocess
import sys
import tarfile
from pathlib import Path
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from verify_qwen3vl_model_manifest import REVISION, verify as verify_qwen_manifest
from operator_release_manifest import source_tree_sha256
REQUIRED_IMAGES = {
"FANYU_IMAGE_MERGE",
"FANYU_IMAGE_PACKAGE",
"FANYU_IMAGE_VIDEO_FACE_BLUR",
"FANYU_IMAGE_QWEN_TEMPORAL",
"FANYU_IMAGE_QWEN_VALID_EVENT",
"FANYU_IMAGE_MS_SWIFT_EXPORT",
"FANYU_IMAGE_QWEN_TRAIN",
}
IMAGE_TO_OPERATOR = {
"FANYU_IMAGE_MERGE": "merge",
"FANYU_IMAGE_PACKAGE": "package",
"FANYU_IMAGE_VIDEO_FACE_BLUR": "video_face_blur",
"FANYU_IMAGE_QWEN_TEMPORAL": "qwen_temporal_localization",
"FANYU_IMAGE_QWEN_VALID_EVENT": "wilor_valid_event_detection",
"FANYU_IMAGE_MS_SWIFT_EXPORT": "ms_swift_dataset_export",
"FANYU_IMAGE_QWEN_TRAIN": "qwen_label_crop_classification",
}
REQUIRED_VALUES = {
"FANYU_S100_MODE",
"FANYU_REMOTE_API_BASE_URL",
"FANYU_REMOTE_BOS_ENDPOINT",
"FANYU_HEADPOSE_API_BASE_URL",
"FANYU_HEADPOSE_SERVICE_IMAGE_ID",
"FANYU_HEADPOSE_MODEL_SHA256",
"FANYU_HEADPOSE_LICENSE_ID",
"FANYU_KUBECONFIG_CONTEXT",
"FANYU_MONGO_URI_REF",
"FANYU_BOS_CONNECTION_REF",
"FANYU_WORKSPACE_PVC",
"FANYU_QWEN_MODEL_PVC",
"FANYU_QWEN_MODEL_ROOT",
"FANYU_QWEN_MODEL_MANIFEST_SHA256",
"FANYU_NODE_ARCH",
"FANYU_V5_IMAGE_ARCH",
"FANYU_DROSYNC_PLATFORM",
"FANYU_DROSYNC_DEB_SHA256",
"FANYU_DROSYNC_PORTABLE_SHA256",
*REQUIRED_IMAGES,
}
_COMMIT = re.compile(r"^[a-f0-9]{40}$")
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
def parse_values(path: Path) -> dict[str, str]:
values: dict[str, str] = {}
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
key, separator, value = line.partition("=")
if not separator or not key.strip():
raise RuntimeError(f"invalid values line: {raw}")
values[key.strip()] = value.strip()
return values
def _ar_entries(path: Path) -> dict[str, bytes]:
payload = path.read_bytes()
if not payload.startswith(b"!<arch>\n"):
raise RuntimeError("drosync DEB is not an ar archive")
entries: dict[str, bytes] = {}
offset = 8
while offset < len(payload):
header = payload[offset : offset + 60]
if len(header) != 60 or header[58:60] != b"`\n":
raise RuntimeError("drosync DEB has an invalid ar member header")
name = header[:16].decode("ascii", errors="strict").strip().rstrip("/")
try:
size = int(header[48:58].decode("ascii").strip())
except ValueError as exc:
raise RuntimeError("drosync DEB has an invalid ar member size") from exc
start = offset + 60
end = start + size
if end > len(payload):
raise RuntimeError("drosync DEB has a truncated ar member")
entries[name] = payload[start:end]
offset = end + (size % 2)
return entries
def _tar_member_bytes(archive: tarfile.TarFile, suffix: str) -> bytes:
matches = [member for member in archive.getmembers() if member.isfile() and member.name.endswith(suffix)]
if len(matches) != 1:
raise RuntimeError(f"edge artifact must contain exactly one {suffix}")
handle = archive.extractfile(matches[0])
if handle is None:
raise RuntimeError(f"cannot read {suffix} from edge artifact")
return handle.read()
def _require_aarch64_elf(payload: bytes, label: str) -> None:
if len(payload) < 20 or payload[:4] != b"\x7fELF" or payload[4] != 2 or payload[5] != 1:
raise RuntimeError(f"{label} must be a little-endian ELF64 binary")
if struct.unpack("<H", payload[18:20])[0] != 183:
raise RuntimeError(f"{label} must target AArch64")
def verify_drosync_artifacts(deb: Path, portable: Path, values: dict[str, str]) -> None:
if values["FANYU_DROSYNC_PLATFORM"] != "linux-arm64":
raise RuntimeError("FANYU_DROSYNC_PLATFORM must be linux-arm64")
for path, key in ((deb, "FANYU_DROSYNC_DEB_SHA256"), (portable, "FANYU_DROSYNC_PORTABLE_SHA256")):
if not path.is_file() or path.stat().st_size <= 0:
raise RuntimeError(f"missing drosync edge artifact: {path}")
expected = values[key].lower()
if not _SHA256.fullmatch(expected) or sha256(path) != expected:
raise RuntimeError(f"{key} does not verify {path.name}")
entries = _ar_entries(deb)
control_name = next((name for name in entries if name.startswith("control.tar")), "")
data_name = next((name for name in entries if name.startswith("data.tar")), "")
if not control_name or not data_name:
raise RuntimeError("drosync DEB must contain control and data archives")
with tarfile.open(fileobj=io.BytesIO(entries[control_name]), mode="r:*") as control_archive:
control = _tar_member_bytes(control_archive, "/control").decode("utf-8")
fields = dict(line.split(":", 1) for line in control.splitlines() if ":" in line)
if fields.get("Architecture", "").strip() != "arm64":
raise RuntimeError("drosync DEB Architecture must be arm64")
with tarfile.open(fileobj=io.BytesIO(entries[data_name]), mode="r:*") as data_archive:
names = [member.name for member in data_archive.getmembers()]
if any("drosync-gui" in name for name in names):
raise RuntimeError("headless S100 DEB must not contain drosync-gui")
_require_aarch64_elf(_tar_member_bytes(data_archive, "/drosync"), "DEB drosync")
_require_aarch64_elf(_tar_member_bytes(data_archive, "/drosync-daemon"), "DEB drosync-daemon")
with tarfile.open(portable, mode="r:gz") as portable_archive:
names = [member.name for member in portable_archive.getmembers()]
required_suffixes = ("/drosync", "/drosync-daemon", "/drosync-portable.service", "/install-systemd-portable.sh")
if any(not any(name.endswith(suffix) for name in names) for suffix in required_suffixes):
raise RuntimeError("portable drosync bundle is missing binaries or systemd installer")
if any("drosync-gui" in name for name in names):
raise RuntimeError("headless S100 portable bundle must not contain drosync-gui")
_require_aarch64_elf(_tar_member_bytes(portable_archive, "/drosync"), "portable drosync")
_require_aarch64_elf(_tar_member_bytes(portable_archive, "/drosync-daemon"), "portable drosync-daemon")
def current_source_commit(explicit: str) -> str:
value = explicit.strip()
if not value:
value = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True
).stdout.strip()
if not _COMMIT.fullmatch(value):
raise RuntimeError("--source-commit must be a full 40-character Git commit")
return value
def validate(
values: dict[str, str],
*,
verify_model: bool,
verified_model_manifest_sha256: str,
deb: Path,
portable: Path,
) -> str:
missing = sorted(key for key in REQUIRED_VALUES if not values.get(key, "").strip())
if missing:
raise RuntimeError("missing required S100 values: " + ", ".join(missing))
if values["FANYU_S100_MODE"] != "edge_remote":
raise RuntimeError("only FANYU_S100_MODE=edge_remote is deployable on ARM64 S100; full_local is fail-closed")
if values["FANYU_NODE_ARCH"].lower() not in {"aarch64", "arm64"} or values["FANYU_V5_IMAGE_ARCH"].lower() not in {"amd64", "x86_64"}:
raise RuntimeError("S100 must be ARM64 edge while the external control-plane images remain x86_64/amd64")
verify_drosync_artifacts(deb, portable, values)
for key in ("FANYU_REMOTE_API_BASE_URL", "FANYU_REMOTE_BOS_ENDPOINT", "FANYU_HEADPOSE_API_BASE_URL"):
parsed = urlparse(values[key])
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise RuntimeError(f"{key} must be an absolute HTTP(S) URL")
for key in REQUIRED_IMAGES | {"FANYU_HEADPOSE_SERVICE_IMAGE_ID"}:
if "@sha256:" not in values[key] or len(values[key].rsplit("@sha256:", 1)[1]) != 64:
raise RuntimeError(f"{key} must be digest-pinned")
if values.get("FANYU_QWEN_MODEL_REVISION", REVISION) != REVISION:
raise RuntimeError("FANYU_QWEN_MODEL_REVISION must match the pinned Qwen3-VL revision")
expected_model_sha = values["FANYU_QWEN_MODEL_MANIFEST_SHA256"].lower()
if not _SHA256.fullmatch(expected_model_sha):
raise RuntimeError("FANYU_QWEN_MODEL_MANIFEST_SHA256 must be a SHA-256 digest")
if verify_model:
supplied = verified_model_manifest_sha256.strip().lower()
if supplied:
if not _SHA256.fullmatch(supplied) or supplied != expected_model_sha:
raise RuntimeError("verified model manifest SHA-256 does not match values.env")
return supplied
verified = verify_qwen_manifest(Path(values["FANYU_QWEN_MODEL_ROOT"]))
if verified != expected_model_sha:
raise RuntimeError("local Qwen manifest SHA-256 does not match values.env")
return verified
return "model-verification-skipped-by-explicit-test-flag"
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def copy(source: Path, destination: Path) -> None:
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, destination)
def build(output: Path, values: dict[str, str], model_manifest_sha: str, deb: Path, portable: Path, source_commit: str) -> Path:
bundle = output / "fanyu-v5-s100-edge-bundle"
if bundle.exists():
shutil.rmtree(bundle)
bundle.mkdir(parents=True)
files = [
"deploy/fanyu-v5-s100/README.md",
"deploy/fanyu-v5-s100/ARCHITECTURE.md",
"docs/fanyu-v5-workflow.md",
"docs/drosync/installation.md",
"scripts/s100_edge_preflight.sh",
"scripts/verify_qwen3vl_model_manifest.py",
"tools/drosync/packaging/linux/PORTABLE-SYSTEMD-README.md",
"internal/api/builtin_operators_catalog.json",
]
for relative in files:
copy(ROOT / relative, bundle / relative)
release_dir = bundle / "operator-releases"
release_dir.mkdir()
for value_key, slug in sorted(IMAGE_TO_OPERATOR.items()):
image_id = values[value_key]
image = image_id.split("@sha256:", 1)[0]
digest = "sha256:" + image_id.rsplit("@sha256:", 1)[1]
release = {
"schemaVersion": "droboticflow.operator.release/v1",
"operatorId": "operator-builtin-" + slug.replace("_", "-"),
"slug": slug,
"sourceCommit": source_commit,
"sourceTreeSha256": source_tree_sha256(slug),
"image": image,
"imageId": image_id,
"digest": digest,
"environmentReleases": {"s100-edge": {"image": image, "imageId": image_id, "digest": digest}},
"acceptance": {"projectId": "", "workflowId": "", "runId": "", "artifactId": "", "verifiedAt": ""},
}
target = release_dir / slug / "release.json"
target.parent.mkdir(parents=True)
target.write_text(json.dumps(release, indent=2, sort_keys=True) + "\n", encoding="utf-8")
overrides = {
"schemaVersion": "droboticflow.fanyu-v5.s100-edge/v1",
"mode": "edge_remote",
"nodeArchitecture": values["FANYU_NODE_ARCH"],
"controlPlaneImageArchitecture": values["FANYU_V5_IMAGE_ARCH"],
"qwenModelRevision": values.get("FANYU_QWEN_MODEL_REVISION", REVISION),
"qwenModelManifestSha256": model_manifest_sha,
"sourceCommit": source_commit,
"drosync": {
"platform": values["FANYU_DROSYNC_PLATFORM"],
"deb": {"file": deb.name, "sha256": values["FANYU_DROSYNC_DEB_SHA256"]},
"portable": {"file": portable.name, "sha256": values["FANYU_DROSYNC_PORTABLE_SHA256"]},
},
"imageReleases": {key: values[key] for key in sorted(REQUIRED_IMAGES | {"FANYU_HEADPOSE_SERVICE_IMAGE_ID"})},
"externalReferences": {key: values[key] for key in sorted(REQUIRED_VALUES - REQUIRED_IMAGES - {"FANYU_HEADPOSE_SERVICE_IMAGE_ID"}) if key not in {"FANYU_QWEN_MODEL_ROOT"}},
}
(bundle / "operator-release-overrides.json").write_text(json.dumps(overrides, indent=2, sort_keys=True) + "\n", encoding="utf-8")
copy(deb, bundle / "edge-artifacts" / deb.name)
copy(portable, bundle / "edge-artifacts" / portable.name)
checksums = []
for path in sorted(candidate for candidate in bundle.rglob("*") if candidate.is_file()):
checksums.append(f"{sha256(path)} {path.relative_to(bundle).as_posix()}")
(bundle / "SHA256SUMS").write_text("\n".join(checksums) + "\n", encoding="utf-8")
archive = output / "fanyu-v5-s100-edge-bundle.tar.gz"
if archive.exists():
archive.unlink()
with archive.open("wb") as raw_archive:
with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as compressed:
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.GNU_FORMAT) as handle:
for path in sorted(candidate for candidate in bundle.rglob("*") if candidate.is_file()):
info = handle.gettarinfo(str(path), arcname=(Path(bundle.name) / path.relative_to(bundle)).as_posix())
info.uid = 0
info.gid = 0
info.uname = ""
info.gname = ""
info.mtime = 0
with path.open("rb") as source:
handle.addfile(info, source)
(archive.with_suffix(archive.suffix + ".sha256")).write_text(f"{sha256(archive)} {archive.name}\n", encoding="utf-8")
return bundle
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--values", required=True, type=Path)
parser.add_argument("--output-dir", required=True, type=Path)
parser.add_argument("--drosync-deb", required=True, type=Path)
parser.add_argument("--drosync-portable", required=True, type=Path)
parser.add_argument("--source-commit", default="")
parser.add_argument("--verified-model-manifest-sha256", default="", help="digest from an independently verified remote preload Job")
parser.add_argument("--skip-model-verification", action="store_true", help="test-only; production bundle must verify the local model manifest")
args = parser.parse_args()
values = parse_values(args.values)
source_commit = current_source_commit(args.source_commit)
manifest_sha = validate(
values,
verify_model=not args.skip_model_verification,
verified_model_manifest_sha256=args.verified_model_manifest_sha256,
deb=args.drosync_deb,
portable=args.drosync_portable,
)
bundle = build(args.output_dir, values, manifest_sha, args.drosync_deb, args.drosync_portable, source_commit)
print(bundle)
return 0
if __name__ == "__main__":
raise SystemExit(main())