277 lines
13 KiB
Python
277 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Create or verify the cryptographic local Qwen3-VL preload manifest.
|
|
|
|
The v5 operators never trust a caller-supplied model hash. They require the
|
|
two files generated/validated here beside the local checkpoint:
|
|
``MODEL_MANIFEST.json`` and ``MODEL_MANIFEST.sha256``.
|
|
|
|
The manifest deliberately records the exact production preload evidence:
|
|
Qwen3-VL config type, processor class, safetensors index size, four shard
|
|
names and every shard hash. A cache directory is not model evidence and is
|
|
excluded from the file inventory.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
MODEL_ID = "Qwen/Qwen3-VL-8B-Instruct"
|
|
REVISION = "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b"
|
|
MANIFEST_NAME = "MODEL_MANIFEST.json"
|
|
SIDECAR_NAME = "MODEL_MANIFEST.sha256"
|
|
MODEL_TYPE = "qwen3_vl"
|
|
PROCESSOR_CLASS = "Qwen3VLProcessor"
|
|
WEIGHT_INDEX_PATH = "model.safetensors.index.json"
|
|
WEIGHT_SHARD_COUNT = 4
|
|
_EXCLUDED_DIRECTORY_NAMES = {".cache"}
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _json(path: Path, label: str) -> dict[str, Any]:
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception as exc:
|
|
raise RuntimeError(f"{label} is not valid JSON: {path}") from exc
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"{label} must be a JSON object: {path}")
|
|
return payload
|
|
|
|
|
|
def _relative_path(value: Any, *, label: str) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
raise RuntimeError(f"{label} must be a non-empty relative path")
|
|
path = Path(value)
|
|
if path.is_absolute() or ".." in path.parts or "\\" in value:
|
|
raise RuntimeError(f"{label} must stay under the model root")
|
|
return path.as_posix()
|
|
|
|
|
|
def _included_file(root: Path, path: Path) -> bool:
|
|
relative = path.relative_to(root)
|
|
return (
|
|
path.is_file()
|
|
and path.name not in {MANIFEST_NAME, SIDECAR_NAME}
|
|
and not any(part in _EXCLUDED_DIRECTORY_NAMES for part in relative.parts)
|
|
)
|
|
|
|
|
|
def _snapshot_files(root: Path) -> list[dict[str, Any]]:
|
|
return [
|
|
{"path": path.relative_to(root).as_posix(), "size_bytes": path.stat().st_size, "sha256": sha256(path)}
|
|
for path in sorted(candidate for candidate in root.rglob("*") if _included_file(root, candidate))
|
|
]
|
|
|
|
|
|
def _snapshot_layout(root: Path, files: list[dict[str, Any]]) -> dict[str, Any]:
|
|
config_path = root / "config.json"
|
|
config = _json(config_path, "Qwen3-VL config") if config_path.is_file() else None
|
|
if config is None or config.get("model_type") != MODEL_TYPE:
|
|
raise RuntimeError(f"Qwen3-VL config.json must declare model_type={MODEL_TYPE}")
|
|
|
|
processor_path = root / "preprocessor_config.json"
|
|
processor = _json(processor_path, "Qwen3-VL processor config") if processor_path.is_file() else None
|
|
if processor is None or processor.get("processor_class") != PROCESSOR_CLASS:
|
|
raise RuntimeError(f"Qwen3-VL preprocessor_config.json must declare processor_class={PROCESSOR_CLASS}")
|
|
|
|
index_path = root / WEIGHT_INDEX_PATH
|
|
index = _json(index_path, "Qwen3-VL safetensors index") if index_path.is_file() else None
|
|
weight_map = index.get("weight_map") if isinstance(index, dict) else None
|
|
metadata = index.get("metadata") if isinstance(index, dict) else None
|
|
total_size = metadata.get("total_size") if isinstance(metadata, dict) else None
|
|
if not isinstance(weight_map, dict) or not weight_map or not isinstance(total_size, int) or total_size <= 0:
|
|
raise RuntimeError("Qwen3-VL safetensors index must contain weight_map and positive metadata.total_size")
|
|
shards = sorted({str(value) for value in weight_map.values() if isinstance(value, str) and value.strip()})
|
|
if len(shards) != WEIGHT_SHARD_COUNT:
|
|
raise RuntimeError(f"Qwen3-VL safetensors index must reference exactly {WEIGHT_SHARD_COUNT} distinct shards")
|
|
file_by_path = {item["path"]: item for item in files}
|
|
for shard in shards:
|
|
relative = _relative_path(shard, label="Qwen3-VL safetensors shard")
|
|
if not relative.endswith(".safetensors") or relative not in file_by_path or file_by_path[relative]["size_bytes"] <= 0:
|
|
raise RuntimeError(f"Qwen3-VL safetensors shard is missing or invalid: {relative}")
|
|
return {
|
|
"config_model_type": MODEL_TYPE,
|
|
"processor_class": PROCESSOR_CLASS,
|
|
"safetensors_index_total_size": total_size,
|
|
"safetensors_index_shards": shards,
|
|
"shards": [file_by_path[shard] for shard in shards],
|
|
}
|
|
|
|
|
|
def _validate_source(source: Any) -> dict[str, Any]:
|
|
if not isinstance(source, dict):
|
|
raise RuntimeError("manifest source must be an object")
|
|
# Validate the pinned subset, not an exact object: the canonical preloader
|
|
# records source_url alongside repository and revision.
|
|
if source.get("repository") != MODEL_ID or source.get("revision") != REVISION:
|
|
raise RuntimeError("manifest repository/revision do not match the pinned Qwen3-VL release")
|
|
source_url = source.get("source_url")
|
|
if not isinstance(source_url, str) or not source_url.strip():
|
|
raise RuntimeError("manifest source.source_url is required")
|
|
parsed = urlparse(source_url)
|
|
if parsed.scheme != "https" or not parsed.netloc:
|
|
raise RuntimeError("manifest source.source_url must be an absolute HTTPS URL")
|
|
return source
|
|
|
|
|
|
def _canonical_source(existing: dict[str, Any] | None) -> dict[str, Any]:
|
|
source = dict(existing.get("source", {})) if isinstance(existing, dict) and isinstance(existing.get("source"), dict) else {}
|
|
if source and (source.get("repository") not in {None, MODEL_ID} or source.get("revision") not in {None, REVISION}):
|
|
raise RuntimeError("existing manifest source does not match the pinned Qwen3-VL release")
|
|
source["repository"] = MODEL_ID
|
|
source["revision"] = REVISION
|
|
source.setdefault("source_url", f"https://huggingface.co/{MODEL_ID}/tree/{REVISION}")
|
|
_validate_source(source)
|
|
return source
|
|
|
|
|
|
def _validate_shard_entries(value: Any, expected: list[dict[str, Any]]) -> None:
|
|
if not isinstance(value, list):
|
|
raise RuntimeError("manifest shards must be an array")
|
|
actual: dict[str, tuple[int, str]] = {}
|
|
for item in value:
|
|
if not isinstance(item, dict):
|
|
raise RuntimeError("manifest shard entry must be an object")
|
|
path = _relative_path(item.get("path"), label="manifest shard")
|
|
size = item.get("size_bytes")
|
|
digest = item.get("sha256")
|
|
if path in actual or not isinstance(size, int) or size <= 0 or not isinstance(digest, str) or len(digest) != 64 or digest.lower() != digest:
|
|
raise RuntimeError("manifest shard entry is invalid")
|
|
actual[path] = (size, digest)
|
|
required = {item["path"]: (item["size_bytes"], item["sha256"]) for item in expected}
|
|
if actual != required:
|
|
raise RuntimeError("manifest shards do not exactly match the Qwen3-VL safetensors index")
|
|
|
|
|
|
def _validate_validation_evidence(value: Any, expected: dict[str, Any]) -> None:
|
|
if not isinstance(value, dict):
|
|
raise RuntimeError("manifest validation evidence is required")
|
|
if value.get("config_model_type") != expected["config_model_type"]:
|
|
raise RuntimeError("manifest validation config_model_type does not match Qwen3-VL")
|
|
if value.get("processor_class") != expected["processor_class"]:
|
|
raise RuntimeError("manifest validation processor_class does not match Qwen3-VL")
|
|
if value.get("safetensors_index_total_size") != expected["safetensors_index_total_size"]:
|
|
raise RuntimeError("manifest validation safetensors_index_total_size does not match the local index")
|
|
shards = value.get("safetensors_index_shards")
|
|
if not isinstance(shards, list) or shards != expected["safetensors_index_shards"]:
|
|
raise RuntimeError("manifest validation safetensors_index_shards does not match the four local shards")
|
|
if value.get("all_expected_shard_sha256_verified") is not True:
|
|
raise RuntimeError("manifest validation must confirm all expected shard SHA-256 values")
|
|
|
|
|
|
def _validate_file_entries(root: Path, value: Any, expected: list[dict[str, Any]]) -> None:
|
|
if not isinstance(value, list) or not value:
|
|
raise RuntimeError("manifest has no file list")
|
|
actual: dict[str, tuple[int, str]] = {}
|
|
for item in value:
|
|
if not isinstance(item, dict):
|
|
raise RuntimeError("manifest file entry must be an object")
|
|
relative = _relative_path(item.get("path"), label="manifest file")
|
|
if any(part in _EXCLUDED_DIRECTORY_NAMES for part in Path(relative).parts):
|
|
raise RuntimeError(f"manifest must not include cache file: {relative}")
|
|
size = item.get("size_bytes")
|
|
digest = item.get("sha256")
|
|
if relative in actual or not isinstance(size, int) or size <= 0 or not isinstance(digest, str) or len(digest) != 64 or digest.lower() != digest:
|
|
raise RuntimeError(f"manifest file entry is invalid: {relative}")
|
|
path = (root / relative).resolve()
|
|
if root.resolve() not in path.parents or not path.is_file():
|
|
raise RuntimeError(f"manifest file is missing: {relative}")
|
|
actual[relative] = (size, digest)
|
|
required = {item["path"]: (item["size_bytes"], item["sha256"]) for item in expected}
|
|
if actual != required:
|
|
raise RuntimeError("manifest file list does not exactly match the non-cache Qwen3-VL snapshot")
|
|
|
|
|
|
def _existing_manifest(root: Path) -> dict[str, Any] | None:
|
|
path = root / MANIFEST_NAME
|
|
if not path.exists():
|
|
return None
|
|
if not path.is_file():
|
|
raise RuntimeError(f"{MANIFEST_NAME} must be a regular file")
|
|
return _json(path, MANIFEST_NAME)
|
|
|
|
|
|
def build(root: Path) -> dict[str, Any]:
|
|
existing = _existing_manifest(root)
|
|
files = _snapshot_files(root)
|
|
if not files:
|
|
raise RuntimeError("no Qwen3-VL files exist under model root")
|
|
layout = _snapshot_layout(root, files)
|
|
if existing is not None and "validation" in existing:
|
|
_validate_source(existing.get("source"))
|
|
_validate_validation_evidence(existing.get("validation"), layout)
|
|
_validate_shard_entries(existing.get("shards"), layout["shards"])
|
|
validation = existing["validation"]
|
|
shards = existing["shards"]
|
|
else:
|
|
validation = {
|
|
"config_model_type": layout["config_model_type"],
|
|
"processor_class": layout["processor_class"],
|
|
"safetensors_index_total_size": layout["safetensors_index_total_size"],
|
|
"safetensors_index_shards": layout["safetensors_index_shards"],
|
|
"all_expected_shard_sha256_verified": True,
|
|
}
|
|
shards = layout["shards"]
|
|
return {
|
|
"schema_version": 1,
|
|
"source": _canonical_source(existing),
|
|
"shards": shards,
|
|
"validation": validation,
|
|
"files": files,
|
|
}
|
|
|
|
|
|
def verify(root: Path) -> str:
|
|
manifest_path = root / MANIFEST_NAME
|
|
sidecar_path = root / SIDECAR_NAME
|
|
manifest = _json(manifest_path, MANIFEST_NAME)
|
|
if manifest.get("schema_version") != 1:
|
|
raise RuntimeError("manifest schema_version must be 1")
|
|
_validate_source(manifest.get("source"))
|
|
files = _snapshot_files(root)
|
|
layout = _snapshot_layout(root, files)
|
|
_validate_validation_evidence(manifest.get("validation"), layout)
|
|
_validate_shard_entries(manifest.get("shards"), layout["shards"])
|
|
_validate_file_entries(root, manifest.get("files"), files)
|
|
if not sidecar_path.is_file() or sidecar_path.stat().st_size == 0:
|
|
raise RuntimeError("MODEL_MANIFEST.sha256 is required")
|
|
tokens = sidecar_path.read_text(encoding="utf-8").strip().split()
|
|
if not tokens:
|
|
raise RuntimeError("MODEL_MANIFEST.sha256 is empty")
|
|
expected_manifest = tokens[0].lower()
|
|
actual_manifest = sha256(manifest_path)
|
|
if expected_manifest != actual_manifest:
|
|
raise RuntimeError("MODEL_MANIFEST.sha256 does not verify MODEL_MANIFEST.json")
|
|
return actual_manifest
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model-root", default="/workspace/models/Qwen3-VL-8B-Instruct")
|
|
parser.add_argument("--write", action="store_true", help="write a canonical manifest after validating the local model files")
|
|
args = parser.parse_args()
|
|
root = Path(args.model_root).resolve()
|
|
if not root.is_dir():
|
|
raise SystemExit(f"model root does not exist: {root}")
|
|
if args.write:
|
|
manifest_path = root / MANIFEST_NAME
|
|
manifest_path.write_text(json.dumps(build(root), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
digest = sha256(manifest_path)
|
|
(root / SIDECAR_NAME).write_text(f"{digest} {MANIFEST_NAME}\n", encoding="utf-8")
|
|
print(verify(root))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|