155 lines
6.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""根目录、config.yaml、.env.local。所有路径解析只在这里做。"""
from __future__ import annotations
import os
import hashlib
import json
from functools import lru_cache
from pathlib import Path
import yaml
def resolve_installation(root=None, config_path=None, environ=None) -> tuple[Path, Path, Path]:
"""Pure path resolution shared by runtime and offline inventory.
Explicit root/config wins over environment. Relative override paths follow
the caller's working directory, as the existing runtime config path does.
No config/secret contents, database or runtime directories are touched.
"""
env = os.environ if environ is None else environ
source = Path(root or env.get("BENCH_ROOT") or Path(__file__).resolve().parent.parent).expanduser().resolve()
config_file = Path(config_path or env.get("BENCH_CONFIG") or source / "config.yaml").expanduser().resolve()
state = Path(env.get("BENCH_STATE_DIR") or source / "state").expanduser().resolve()
return source, config_file, state
BENCH_ROOT, CONFIG_PATH, STATE = resolve_installation()
PROJECT_ROOT = BENCH_ROOT.parent
REGISTRY = BENCH_ROOT / "registry"
DATA = BENCH_ROOT / "data"
CACHE = BENCH_ROOT / "cache"
RUNS = BENCH_ROOT / "runs"
REPORTS = BENCH_ROOT / "reports"
DATASETS = BENCH_ROOT / "datasets"
AGENT = BENCH_ROOT / "agent"
DB_PATH = STATE / "jobs.sqlite"
STAGE_NAMES = [
"01-fetch", "02-baseline", "03-export", # model-level → cache/
"04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval", # target-level → job dir
]
MODEL_LEVEL = STAGE_NAMES[:3]
TARGET_LEVEL = STAGE_NAMES[3:]
TASKS = ["cls", "det", "seg", "iseg", "pose", "mde", "stereo", "flow", "feat", "occ"]
MARCHES = ["nash-e", "nash-m", "nash-p"]
def expand(p: str | os.PathLike | None, base: Path = BENCH_ROOT) -> Path | None:
if p is None:
return None
p = os.path.expandvars(os.path.expanduser(str(p)))
q = Path(p)
return q if q.is_absolute() else (base / q).resolve()
@lru_cache(maxsize=1)
def config() -> dict:
with open(CONFIG_PATH, encoding="utf-8") as f:
cfg = yaml.safe_load(f) or {}
cfg.setdefault("paths", {})
if not cfg["paths"].get("project_root"):
cfg["paths"]["project_root"] = str(PROJECT_ROOT)
return cfg
def venv_python(backend: str, job: dict | None = None) -> Path:
"""按 backend 选解释器。这是 harness 的职责,agent 无权改。"""
if job and job.get("toolchain") and backend == job.get("backend"):
from .toolchains import validate
return Path(validate(job)["venv"]) / "bin" / "python"
v = config()["paths"]["venv"].get(backend)
if not v:
raise KeyError(f"config.yaml paths.venv 没有 backend={backend}")
return expand(v) / "bin" / "python"
def venv_bin(backend: str, tool: str, job: dict | None = None) -> Path:
return venv_python(backend, job).parent / tool
def model_runtime_path(value: str | os.PathLike) -> Path:
"""Keep the venv launch path: different venvs often symlink the same Python binary."""
path = Path(os.path.expandvars(os.path.expanduser(str(value))))
return Path(os.path.abspath(path if path.is_absolute() else BENCH_ROOT / path))
def model_python(job: dict) -> Path:
"""New model recipes may require an isolated upstream-compatible runtime."""
runtime = job.get("model_spec", {}).get("runtime_python")
frozen = job.get("model_runtime")
if frozen:
current = (config().get("model_runtimes") or {}).get(job.get("model"), frozen["configured"])
matches = str(model_runtime_path(current)) == frozen["python"]
if not matches and "cache_contract" not in job:
matches = str(expand(current)) == frozen["python"] # Historical snapshots may have followed symlinks.
if not matches:
raise ValueError("frozen model runtime changed; create a new run")
return Path(frozen["python"])
if runtime:
return model_runtime_path(runtime) if "cache_contract" in job else expand(runtime)
return venv_python("bench")
def freeze_cache_contract(job: dict) -> dict:
"""New jobs explicitly opt into versioned caches, including the default Python."""
from .tools.export_onnx import exporter_version
runtime = job.get("model_spec", {}).get("runtime_python")
python = model_runtime_path(runtime) if runtime and not job.get("model_runtime") else model_python(job)
return {"version": 1, "model_python": str(model_runtime_path(python)),
"exporter_version": "task-recipe-v1" if job.get("model_recipe") else exporter_version(job["model"])}
def cache_contract(job: dict, *, validate: bool = False) -> dict | None:
"""Missing means legacy, never an invitation to relabel an existing cache."""
if "cache_contract" not in job:
if validate and job.get("model_runtime"):
model_python(job)
return None
contract = job["cache_contract"]
if (not isinstance(contract, dict) or type(contract.get("version")) is not int or contract["version"] != 1
or not isinstance(contract.get("model_python"), str) or not Path(contract["model_python"]).is_absolute()
or not isinstance(contract.get("exporter_version"), str) or not contract["exporter_version"]):
raise ValueError("invalid or unsupported cache contract")
if validate and str(model_runtime_path(model_python(job))) != contract["model_python"]:
raise ValueError("frozen cache runtime changed; create a new run")
return contract
def cache_key(identity: dict) -> str:
return hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
@lru_cache(maxsize=1)
def load_env_local() -> dict:
"""读取 bench/.env.local。只给 harness 用;rollout 的 agent 子进程环境由 stagerun/rollout 单独构造,永远不含这些。"""
env = {}
p = BENCH_ROOT / ".env.local"
if p.exists():
for line in p.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
env[k.strip()] = v.strip().strip('"').strip("'")
return env
def secret(name: str) -> str | None:
return os.environ.get(name) or load_env_local().get(name)
def ensure_dirs() -> None:
for d in (STATE, RUNS, REPORTS, DATASETS, DATA / "pool", DATA / "t1", DATA / "calib",
CACHE / "weights", CACHE / "onnx", CACHE / "baseline"):
d.mkdir(parents=True, exist_ok=True)