forked from longtao.wu/droboticflow
332 lines
15 KiB
Python
332 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Build a deterministic, secret-free Fanyu v5 S100 edge deployment bundle."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from verify_qwen3vl_model_manifest import REVISION, verify as verify_qwen_manifest
|
|
from operator_release_manifest import source_tree_sha256
|
|
|
|
REQUIRED_IMAGES = {
|
|
"FANYU_IMAGE_MERGE",
|
|
"FANYU_IMAGE_PACKAGE",
|
|
"FANYU_IMAGE_VIDEO_FACE_BLUR",
|
|
"FANYU_IMAGE_QWEN_TEMPORAL",
|
|
"FANYU_IMAGE_QWEN_VALID_EVENT",
|
|
"FANYU_IMAGE_MS_SWIFT_EXPORT",
|
|
"FANYU_IMAGE_QWEN_TRAIN",
|
|
}
|
|
IMAGE_TO_OPERATOR = {
|
|
"FANYU_IMAGE_MERGE": "merge",
|
|
"FANYU_IMAGE_PACKAGE": "package",
|
|
"FANYU_IMAGE_VIDEO_FACE_BLUR": "video_face_blur",
|
|
"FANYU_IMAGE_QWEN_TEMPORAL": "qwen_temporal_localization",
|
|
"FANYU_IMAGE_QWEN_VALID_EVENT": "wilor_valid_event_detection",
|
|
"FANYU_IMAGE_MS_SWIFT_EXPORT": "ms_swift_dataset_export",
|
|
"FANYU_IMAGE_QWEN_TRAIN": "qwen_label_crop_classification",
|
|
}
|
|
REQUIRED_VALUES = {
|
|
"FANYU_S100_MODE",
|
|
"FANYU_REMOTE_API_BASE_URL",
|
|
"FANYU_REMOTE_BOS_ENDPOINT",
|
|
"FANYU_HEADPOSE_API_BASE_URL",
|
|
"FANYU_HEADPOSE_SERVICE_IMAGE_ID",
|
|
"FANYU_HEADPOSE_MODEL_SHA256",
|
|
"FANYU_HEADPOSE_LICENSE_ID",
|
|
"FANYU_KUBECONFIG_CONTEXT",
|
|
"FANYU_MONGO_URI_REF",
|
|
"FANYU_BOS_CONNECTION_REF",
|
|
"FANYU_WORKSPACE_PVC",
|
|
"FANYU_QWEN_MODEL_PVC",
|
|
"FANYU_QWEN_MODEL_ROOT",
|
|
"FANYU_QWEN_MODEL_MANIFEST_SHA256",
|
|
"FANYU_NODE_ARCH",
|
|
"FANYU_V5_IMAGE_ARCH",
|
|
"FANYU_DROSYNC_PLATFORM",
|
|
"FANYU_DROSYNC_DEB_SHA256",
|
|
"FANYU_DROSYNC_PORTABLE_SHA256",
|
|
*REQUIRED_IMAGES,
|
|
}
|
|
|
|
_COMMIT = re.compile(r"^[a-f0-9]{40}$")
|
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
|
|
|
|
|
def parse_values(path: Path) -> dict[str, str]:
|
|
values: dict[str, str] = {}
|
|
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
line = raw.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
key, separator, value = line.partition("=")
|
|
if not separator or not key.strip():
|
|
raise RuntimeError(f"invalid values line: {raw}")
|
|
values[key.strip()] = value.strip()
|
|
return values
|
|
|
|
|
|
def _ar_entries(path: Path) -> dict[str, bytes]:
|
|
payload = path.read_bytes()
|
|
if not payload.startswith(b"!<arch>\n"):
|
|
raise RuntimeError("drosync DEB is not an ar archive")
|
|
entries: dict[str, bytes] = {}
|
|
offset = 8
|
|
while offset < len(payload):
|
|
header = payload[offset : offset + 60]
|
|
if len(header) != 60 or header[58:60] != b"`\n":
|
|
raise RuntimeError("drosync DEB has an invalid ar member header")
|
|
name = header[:16].decode("ascii", errors="strict").strip().rstrip("/")
|
|
try:
|
|
size = int(header[48:58].decode("ascii").strip())
|
|
except ValueError as exc:
|
|
raise RuntimeError("drosync DEB has an invalid ar member size") from exc
|
|
start = offset + 60
|
|
end = start + size
|
|
if end > len(payload):
|
|
raise RuntimeError("drosync DEB has a truncated ar member")
|
|
entries[name] = payload[start:end]
|
|
offset = end + (size % 2)
|
|
return entries
|
|
|
|
|
|
def _tar_member_bytes(archive: tarfile.TarFile, suffix: str) -> bytes:
|
|
matches = [member for member in archive.getmembers() if member.isfile() and member.name.endswith(suffix)]
|
|
if len(matches) != 1:
|
|
raise RuntimeError(f"edge artifact must contain exactly one {suffix}")
|
|
handle = archive.extractfile(matches[0])
|
|
if handle is None:
|
|
raise RuntimeError(f"cannot read {suffix} from edge artifact")
|
|
return handle.read()
|
|
|
|
|
|
def _require_aarch64_elf(payload: bytes, label: str) -> None:
|
|
if len(payload) < 20 or payload[:4] != b"\x7fELF" or payload[4] != 2 or payload[5] != 1:
|
|
raise RuntimeError(f"{label} must be a little-endian ELF64 binary")
|
|
if struct.unpack("<H", payload[18:20])[0] != 183:
|
|
raise RuntimeError(f"{label} must target AArch64")
|
|
|
|
|
|
def verify_drosync_artifacts(deb: Path, portable: Path, values: dict[str, str]) -> None:
|
|
if values["FANYU_DROSYNC_PLATFORM"] != "linux-arm64":
|
|
raise RuntimeError("FANYU_DROSYNC_PLATFORM must be linux-arm64")
|
|
for path, key in ((deb, "FANYU_DROSYNC_DEB_SHA256"), (portable, "FANYU_DROSYNC_PORTABLE_SHA256")):
|
|
if not path.is_file() or path.stat().st_size <= 0:
|
|
raise RuntimeError(f"missing drosync edge artifact: {path}")
|
|
expected = values[key].lower()
|
|
if not _SHA256.fullmatch(expected) or sha256(path) != expected:
|
|
raise RuntimeError(f"{key} does not verify {path.name}")
|
|
entries = _ar_entries(deb)
|
|
control_name = next((name for name in entries if name.startswith("control.tar")), "")
|
|
data_name = next((name for name in entries if name.startswith("data.tar")), "")
|
|
if not control_name or not data_name:
|
|
raise RuntimeError("drosync DEB must contain control and data archives")
|
|
with tarfile.open(fileobj=io.BytesIO(entries[control_name]), mode="r:*") as control_archive:
|
|
control = _tar_member_bytes(control_archive, "/control").decode("utf-8")
|
|
fields = dict(line.split(":", 1) for line in control.splitlines() if ":" in line)
|
|
if fields.get("Architecture", "").strip() != "arm64":
|
|
raise RuntimeError("drosync DEB Architecture must be arm64")
|
|
with tarfile.open(fileobj=io.BytesIO(entries[data_name]), mode="r:*") as data_archive:
|
|
names = [member.name for member in data_archive.getmembers()]
|
|
if any("drosync-gui" in name for name in names):
|
|
raise RuntimeError("headless S100 DEB must not contain drosync-gui")
|
|
_require_aarch64_elf(_tar_member_bytes(data_archive, "/drosync"), "DEB drosync")
|
|
_require_aarch64_elf(_tar_member_bytes(data_archive, "/drosync-daemon"), "DEB drosync-daemon")
|
|
with tarfile.open(portable, mode="r:gz") as portable_archive:
|
|
names = [member.name for member in portable_archive.getmembers()]
|
|
required_suffixes = ("/drosync", "/drosync-daemon", "/drosync-portable.service", "/install-systemd-portable.sh")
|
|
if any(not any(name.endswith(suffix) for name in names) for suffix in required_suffixes):
|
|
raise RuntimeError("portable drosync bundle is missing binaries or systemd installer")
|
|
if any("drosync-gui" in name for name in names):
|
|
raise RuntimeError("headless S100 portable bundle must not contain drosync-gui")
|
|
_require_aarch64_elf(_tar_member_bytes(portable_archive, "/drosync"), "portable drosync")
|
|
_require_aarch64_elf(_tar_member_bytes(portable_archive, "/drosync-daemon"), "portable drosync-daemon")
|
|
|
|
|
|
def current_source_commit(explicit: str) -> str:
|
|
value = explicit.strip()
|
|
if not value:
|
|
value = subprocess.run(
|
|
["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True
|
|
).stdout.strip()
|
|
if not _COMMIT.fullmatch(value):
|
|
raise RuntimeError("--source-commit must be a full 40-character Git commit")
|
|
return value
|
|
|
|
|
|
def validate(
|
|
values: dict[str, str],
|
|
*,
|
|
verify_model: bool,
|
|
verified_model_manifest_sha256: str,
|
|
deb: Path,
|
|
portable: Path,
|
|
) -> str:
|
|
missing = sorted(key for key in REQUIRED_VALUES if not values.get(key, "").strip())
|
|
if missing:
|
|
raise RuntimeError("missing required S100 values: " + ", ".join(missing))
|
|
if values["FANYU_S100_MODE"] != "edge_remote":
|
|
raise RuntimeError("only FANYU_S100_MODE=edge_remote is deployable on ARM64 S100; full_local is fail-closed")
|
|
if values["FANYU_NODE_ARCH"].lower() not in {"aarch64", "arm64"} or values["FANYU_V5_IMAGE_ARCH"].lower() not in {"amd64", "x86_64"}:
|
|
raise RuntimeError("S100 must be ARM64 edge while the external control-plane images remain x86_64/amd64")
|
|
verify_drosync_artifacts(deb, portable, values)
|
|
for key in ("FANYU_REMOTE_API_BASE_URL", "FANYU_REMOTE_BOS_ENDPOINT", "FANYU_HEADPOSE_API_BASE_URL"):
|
|
parsed = urlparse(values[key])
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
raise RuntimeError(f"{key} must be an absolute HTTP(S) URL")
|
|
for key in REQUIRED_IMAGES | {"FANYU_HEADPOSE_SERVICE_IMAGE_ID"}:
|
|
if "@sha256:" not in values[key] or len(values[key].rsplit("@sha256:", 1)[1]) != 64:
|
|
raise RuntimeError(f"{key} must be digest-pinned")
|
|
if values.get("FANYU_QWEN_MODEL_REVISION", REVISION) != REVISION:
|
|
raise RuntimeError("FANYU_QWEN_MODEL_REVISION must match the pinned Qwen3-VL revision")
|
|
expected_model_sha = values["FANYU_QWEN_MODEL_MANIFEST_SHA256"].lower()
|
|
if not _SHA256.fullmatch(expected_model_sha):
|
|
raise RuntimeError("FANYU_QWEN_MODEL_MANIFEST_SHA256 must be a SHA-256 digest")
|
|
if verify_model:
|
|
supplied = verified_model_manifest_sha256.strip().lower()
|
|
if supplied:
|
|
if not _SHA256.fullmatch(supplied) or supplied != expected_model_sha:
|
|
raise RuntimeError("verified model manifest SHA-256 does not match values.env")
|
|
return supplied
|
|
verified = verify_qwen_manifest(Path(values["FANYU_QWEN_MODEL_ROOT"]))
|
|
if verified != expected_model_sha:
|
|
raise RuntimeError("local Qwen manifest SHA-256 does not match values.env")
|
|
return verified
|
|
return "model-verification-skipped-by-explicit-test-flag"
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def copy(source: Path, destination: Path) -> None:
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(source, destination)
|
|
|
|
|
|
def build(output: Path, values: dict[str, str], model_manifest_sha: str, deb: Path, portable: Path, source_commit: str) -> Path:
|
|
bundle = output / "fanyu-v5-s100-edge-bundle"
|
|
if bundle.exists():
|
|
shutil.rmtree(bundle)
|
|
bundle.mkdir(parents=True)
|
|
files = [
|
|
"deploy/fanyu-v5-s100/README.md",
|
|
"deploy/fanyu-v5-s100/ARCHITECTURE.md",
|
|
"docs/fanyu-v5-workflow.md",
|
|
"docs/drosync/installation.md",
|
|
"scripts/s100_edge_preflight.sh",
|
|
"scripts/verify_qwen3vl_model_manifest.py",
|
|
"tools/drosync/packaging/linux/PORTABLE-SYSTEMD-README.md",
|
|
"internal/api/builtin_operators_catalog.json",
|
|
]
|
|
for relative in files:
|
|
copy(ROOT / relative, bundle / relative)
|
|
release_dir = bundle / "operator-releases"
|
|
release_dir.mkdir()
|
|
for value_key, slug in sorted(IMAGE_TO_OPERATOR.items()):
|
|
image_id = values[value_key]
|
|
image = image_id.split("@sha256:", 1)[0]
|
|
digest = "sha256:" + image_id.rsplit("@sha256:", 1)[1]
|
|
release = {
|
|
"schemaVersion": "droboticflow.operator.release/v1",
|
|
"operatorId": "operator-builtin-" + slug.replace("_", "-"),
|
|
"slug": slug,
|
|
"sourceCommit": source_commit,
|
|
"sourceTreeSha256": source_tree_sha256(slug),
|
|
"image": image,
|
|
"imageId": image_id,
|
|
"digest": digest,
|
|
"environmentReleases": {"s100-edge": {"image": image, "imageId": image_id, "digest": digest}},
|
|
"acceptance": {"projectId": "", "workflowId": "", "runId": "", "artifactId": "", "verifiedAt": ""},
|
|
}
|
|
target = release_dir / slug / "release.json"
|
|
target.parent.mkdir(parents=True)
|
|
target.write_text(json.dumps(release, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
overrides = {
|
|
"schemaVersion": "droboticflow.fanyu-v5.s100-edge/v1",
|
|
"mode": "edge_remote",
|
|
"nodeArchitecture": values["FANYU_NODE_ARCH"],
|
|
"controlPlaneImageArchitecture": values["FANYU_V5_IMAGE_ARCH"],
|
|
"qwenModelRevision": values.get("FANYU_QWEN_MODEL_REVISION", REVISION),
|
|
"qwenModelManifestSha256": model_manifest_sha,
|
|
"sourceCommit": source_commit,
|
|
"drosync": {
|
|
"platform": values["FANYU_DROSYNC_PLATFORM"],
|
|
"deb": {"file": deb.name, "sha256": values["FANYU_DROSYNC_DEB_SHA256"]},
|
|
"portable": {"file": portable.name, "sha256": values["FANYU_DROSYNC_PORTABLE_SHA256"]},
|
|
},
|
|
"imageReleases": {key: values[key] for key in sorted(REQUIRED_IMAGES | {"FANYU_HEADPOSE_SERVICE_IMAGE_ID"})},
|
|
"externalReferences": {key: values[key] for key in sorted(REQUIRED_VALUES - REQUIRED_IMAGES - {"FANYU_HEADPOSE_SERVICE_IMAGE_ID"}) if key not in {"FANYU_QWEN_MODEL_ROOT"}},
|
|
}
|
|
(bundle / "operator-release-overrides.json").write_text(json.dumps(overrides, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
copy(deb, bundle / "edge-artifacts" / deb.name)
|
|
copy(portable, bundle / "edge-artifacts" / portable.name)
|
|
checksums = []
|
|
for path in sorted(candidate for candidate in bundle.rglob("*") if candidate.is_file()):
|
|
checksums.append(f"{sha256(path)} {path.relative_to(bundle).as_posix()}")
|
|
(bundle / "SHA256SUMS").write_text("\n".join(checksums) + "\n", encoding="utf-8")
|
|
archive = output / "fanyu-v5-s100-edge-bundle.tar.gz"
|
|
if archive.exists():
|
|
archive.unlink()
|
|
with archive.open("wb") as raw_archive:
|
|
with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as compressed:
|
|
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.GNU_FORMAT) as handle:
|
|
for path in sorted(candidate for candidate in bundle.rglob("*") if candidate.is_file()):
|
|
info = handle.gettarinfo(str(path), arcname=(Path(bundle.name) / path.relative_to(bundle)).as_posix())
|
|
info.uid = 0
|
|
info.gid = 0
|
|
info.uname = ""
|
|
info.gname = ""
|
|
info.mtime = 0
|
|
with path.open("rb") as source:
|
|
handle.addfile(info, source)
|
|
(archive.with_suffix(archive.suffix + ".sha256")).write_text(f"{sha256(archive)} {archive.name}\n", encoding="utf-8")
|
|
return bundle
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--values", required=True, type=Path)
|
|
parser.add_argument("--output-dir", required=True, type=Path)
|
|
parser.add_argument("--drosync-deb", required=True, type=Path)
|
|
parser.add_argument("--drosync-portable", required=True, type=Path)
|
|
parser.add_argument("--source-commit", default="")
|
|
parser.add_argument("--verified-model-manifest-sha256", default="", help="digest from an independently verified remote preload Job")
|
|
parser.add_argument("--skip-model-verification", action="store_true", help="test-only; production bundle must verify the local model manifest")
|
|
args = parser.parse_args()
|
|
values = parse_values(args.values)
|
|
source_commit = current_source_commit(args.source_commit)
|
|
manifest_sha = validate(
|
|
values,
|
|
verify_model=not args.skip_model_verification,
|
|
verified_model_manifest_sha256=args.verified_model_manifest_sha256,
|
|
deb=args.drosync_deb,
|
|
portable=args.drosync_portable,
|
|
)
|
|
bundle = build(args.output_dir, values, manifest_sha, args.drosync_deb, args.drosync_portable, source_commit)
|
|
print(bundle)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|