droboticflow/scripts/verify_qwen3vl_model_manifest.py

277 lines
13 KiB
Python

#!/usr/bin/env python3
"""Create or verify the cryptographic local Qwen3-VL preload manifest.
The v5 operators never trust a caller-supplied model hash. They require the
two files generated/validated here beside the local checkpoint:
``MODEL_MANIFEST.json`` and ``MODEL_MANIFEST.sha256``.
The manifest deliberately records the exact production preload evidence:
Qwen3-VL config type, processor class, safetensors index size, four shard
names and every shard hash. A cache directory is not model evidence and is
excluded from the file inventory.
"""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
MODEL_ID = "Qwen/Qwen3-VL-8B-Instruct"
REVISION = "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b"
MANIFEST_NAME = "MODEL_MANIFEST.json"
SIDECAR_NAME = "MODEL_MANIFEST.sha256"
MODEL_TYPE = "qwen3_vl"
PROCESSOR_CLASS = "Qwen3VLProcessor"
WEIGHT_INDEX_PATH = "model.safetensors.index.json"
WEIGHT_SHARD_COUNT = 4
_EXCLUDED_DIRECTORY_NAMES = {".cache"}
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _json(path: Path, label: str) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception as exc:
raise RuntimeError(f"{label} is not valid JSON: {path}") from exc
if not isinstance(payload, dict):
raise RuntimeError(f"{label} must be a JSON object: {path}")
return payload
def _relative_path(value: Any, *, label: str) -> str:
if not isinstance(value, str) or not value.strip():
raise RuntimeError(f"{label} must be a non-empty relative path")
path = Path(value)
if path.is_absolute() or ".." in path.parts or "\\" in value:
raise RuntimeError(f"{label} must stay under the model root")
return path.as_posix()
def _included_file(root: Path, path: Path) -> bool:
relative = path.relative_to(root)
return (
path.is_file()
and path.name not in {MANIFEST_NAME, SIDECAR_NAME}
and not any(part in _EXCLUDED_DIRECTORY_NAMES for part in relative.parts)
)
def _snapshot_files(root: Path) -> list[dict[str, Any]]:
return [
{"path": path.relative_to(root).as_posix(), "size_bytes": path.stat().st_size, "sha256": sha256(path)}
for path in sorted(candidate for candidate in root.rglob("*") if _included_file(root, candidate))
]
def _snapshot_layout(root: Path, files: list[dict[str, Any]]) -> dict[str, Any]:
config_path = root / "config.json"
config = _json(config_path, "Qwen3-VL config") if config_path.is_file() else None
if config is None or config.get("model_type") != MODEL_TYPE:
raise RuntimeError(f"Qwen3-VL config.json must declare model_type={MODEL_TYPE}")
processor_path = root / "preprocessor_config.json"
processor = _json(processor_path, "Qwen3-VL processor config") if processor_path.is_file() else None
if processor is None or processor.get("processor_class") != PROCESSOR_CLASS:
raise RuntimeError(f"Qwen3-VL preprocessor_config.json must declare processor_class={PROCESSOR_CLASS}")
index_path = root / WEIGHT_INDEX_PATH
index = _json(index_path, "Qwen3-VL safetensors index") if index_path.is_file() else None
weight_map = index.get("weight_map") if isinstance(index, dict) else None
metadata = index.get("metadata") if isinstance(index, dict) else None
total_size = metadata.get("total_size") if isinstance(metadata, dict) else None
if not isinstance(weight_map, dict) or not weight_map or not isinstance(total_size, int) or total_size <= 0:
raise RuntimeError("Qwen3-VL safetensors index must contain weight_map and positive metadata.total_size")
shards = sorted({str(value) for value in weight_map.values() if isinstance(value, str) and value.strip()})
if len(shards) != WEIGHT_SHARD_COUNT:
raise RuntimeError(f"Qwen3-VL safetensors index must reference exactly {WEIGHT_SHARD_COUNT} distinct shards")
file_by_path = {item["path"]: item for item in files}
for shard in shards:
relative = _relative_path(shard, label="Qwen3-VL safetensors shard")
if not relative.endswith(".safetensors") or relative not in file_by_path or file_by_path[relative]["size_bytes"] <= 0:
raise RuntimeError(f"Qwen3-VL safetensors shard is missing or invalid: {relative}")
return {
"config_model_type": MODEL_TYPE,
"processor_class": PROCESSOR_CLASS,
"safetensors_index_total_size": total_size,
"safetensors_index_shards": shards,
"shards": [file_by_path[shard] for shard in shards],
}
def _validate_source(source: Any) -> dict[str, Any]:
if not isinstance(source, dict):
raise RuntimeError("manifest source must be an object")
# Validate the pinned subset, not an exact object: the canonical preloader
# records source_url alongside repository and revision.
if source.get("repository") != MODEL_ID or source.get("revision") != REVISION:
raise RuntimeError("manifest repository/revision do not match the pinned Qwen3-VL release")
source_url = source.get("source_url")
if not isinstance(source_url, str) or not source_url.strip():
raise RuntimeError("manifest source.source_url is required")
parsed = urlparse(source_url)
if parsed.scheme != "https" or not parsed.netloc:
raise RuntimeError("manifest source.source_url must be an absolute HTTPS URL")
return source
def _canonical_source(existing: dict[str, Any] | None) -> dict[str, Any]:
source = dict(existing.get("source", {})) if isinstance(existing, dict) and isinstance(existing.get("source"), dict) else {}
if source and (source.get("repository") not in {None, MODEL_ID} or source.get("revision") not in {None, REVISION}):
raise RuntimeError("existing manifest source does not match the pinned Qwen3-VL release")
source["repository"] = MODEL_ID
source["revision"] = REVISION
source.setdefault("source_url", f"https://huggingface.co/{MODEL_ID}/tree/{REVISION}")
_validate_source(source)
return source
def _validate_shard_entries(value: Any, expected: list[dict[str, Any]]) -> None:
if not isinstance(value, list):
raise RuntimeError("manifest shards must be an array")
actual: dict[str, tuple[int, str]] = {}
for item in value:
if not isinstance(item, dict):
raise RuntimeError("manifest shard entry must be an object")
path = _relative_path(item.get("path"), label="manifest shard")
size = item.get("size_bytes")
digest = item.get("sha256")
if path in actual or not isinstance(size, int) or size <= 0 or not isinstance(digest, str) or len(digest) != 64 or digest.lower() != digest:
raise RuntimeError("manifest shard entry is invalid")
actual[path] = (size, digest)
required = {item["path"]: (item["size_bytes"], item["sha256"]) for item in expected}
if actual != required:
raise RuntimeError("manifest shards do not exactly match the Qwen3-VL safetensors index")
def _validate_validation_evidence(value: Any, expected: dict[str, Any]) -> None:
if not isinstance(value, dict):
raise RuntimeError("manifest validation evidence is required")
if value.get("config_model_type") != expected["config_model_type"]:
raise RuntimeError("manifest validation config_model_type does not match Qwen3-VL")
if value.get("processor_class") != expected["processor_class"]:
raise RuntimeError("manifest validation processor_class does not match Qwen3-VL")
if value.get("safetensors_index_total_size") != expected["safetensors_index_total_size"]:
raise RuntimeError("manifest validation safetensors_index_total_size does not match the local index")
shards = value.get("safetensors_index_shards")
if not isinstance(shards, list) or shards != expected["safetensors_index_shards"]:
raise RuntimeError("manifest validation safetensors_index_shards does not match the four local shards")
if value.get("all_expected_shard_sha256_verified") is not True:
raise RuntimeError("manifest validation must confirm all expected shard SHA-256 values")
def _validate_file_entries(root: Path, value: Any, expected: list[dict[str, Any]]) -> None:
if not isinstance(value, list) or not value:
raise RuntimeError("manifest has no file list")
actual: dict[str, tuple[int, str]] = {}
for item in value:
if not isinstance(item, dict):
raise RuntimeError("manifest file entry must be an object")
relative = _relative_path(item.get("path"), label="manifest file")
if any(part in _EXCLUDED_DIRECTORY_NAMES for part in Path(relative).parts):
raise RuntimeError(f"manifest must not include cache file: {relative}")
size = item.get("size_bytes")
digest = item.get("sha256")
if relative in actual or not isinstance(size, int) or size <= 0 or not isinstance(digest, str) or len(digest) != 64 or digest.lower() != digest:
raise RuntimeError(f"manifest file entry is invalid: {relative}")
path = (root / relative).resolve()
if root.resolve() not in path.parents or not path.is_file():
raise RuntimeError(f"manifest file is missing: {relative}")
actual[relative] = (size, digest)
required = {item["path"]: (item["size_bytes"], item["sha256"]) for item in expected}
if actual != required:
raise RuntimeError("manifest file list does not exactly match the non-cache Qwen3-VL snapshot")
def _existing_manifest(root: Path) -> dict[str, Any] | None:
path = root / MANIFEST_NAME
if not path.exists():
return None
if not path.is_file():
raise RuntimeError(f"{MANIFEST_NAME} must be a regular file")
return _json(path, MANIFEST_NAME)
def build(root: Path) -> dict[str, Any]:
existing = _existing_manifest(root)
files = _snapshot_files(root)
if not files:
raise RuntimeError("no Qwen3-VL files exist under model root")
layout = _snapshot_layout(root, files)
if existing is not None and "validation" in existing:
_validate_source(existing.get("source"))
_validate_validation_evidence(existing.get("validation"), layout)
_validate_shard_entries(existing.get("shards"), layout["shards"])
validation = existing["validation"]
shards = existing["shards"]
else:
validation = {
"config_model_type": layout["config_model_type"],
"processor_class": layout["processor_class"],
"safetensors_index_total_size": layout["safetensors_index_total_size"],
"safetensors_index_shards": layout["safetensors_index_shards"],
"all_expected_shard_sha256_verified": True,
}
shards = layout["shards"]
return {
"schema_version": 1,
"source": _canonical_source(existing),
"shards": shards,
"validation": validation,
"files": files,
}
def verify(root: Path) -> str:
manifest_path = root / MANIFEST_NAME
sidecar_path = root / SIDECAR_NAME
manifest = _json(manifest_path, MANIFEST_NAME)
if manifest.get("schema_version") != 1:
raise RuntimeError("manifest schema_version must be 1")
_validate_source(manifest.get("source"))
files = _snapshot_files(root)
layout = _snapshot_layout(root, files)
_validate_validation_evidence(manifest.get("validation"), layout)
_validate_shard_entries(manifest.get("shards"), layout["shards"])
_validate_file_entries(root, manifest.get("files"), files)
if not sidecar_path.is_file() or sidecar_path.stat().st_size == 0:
raise RuntimeError("MODEL_MANIFEST.sha256 is required")
tokens = sidecar_path.read_text(encoding="utf-8").strip().split()
if not tokens:
raise RuntimeError("MODEL_MANIFEST.sha256 is empty")
expected_manifest = tokens[0].lower()
actual_manifest = sha256(manifest_path)
if expected_manifest != actual_manifest:
raise RuntimeError("MODEL_MANIFEST.sha256 does not verify MODEL_MANIFEST.json")
return actual_manifest
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model-root", default="/workspace/models/Qwen3-VL-8B-Instruct")
parser.add_argument("--write", action="store_true", help="write a canonical manifest after validating the local model files")
args = parser.parse_args()
root = Path(args.model_root).resolve()
if not root.is_dir():
raise SystemExit(f"model root does not exist: {root}")
if args.write:
manifest_path = root / MANIFEST_NAME
manifest_path.write_text(json.dumps(build(root), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
digest = sha256(manifest_path)
(root / SIDECAR_NAME).write_text(f"{digest} {MANIFEST_NAME}\n", encoding="utf-8")
print(verify(root))
return 0
if __name__ == "__main__":
raise SystemExit(main())