data-gen-bpu/harness/coverage.py

460 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""bench coverage — 338 个模型里,哪些做了、哪些没做、哪些在跑。
bench coverage 总览 + 按任务分组
bench coverage --missing 只列"还没做"的
bench coverage --task cls --detail 逐模型一行
bench coverage --csv cov.csv 导出给别人看
三个层次,别混:
registry models.csv 里 338 行,status=ready 的才会被 matrix 展开(needs_adapter/needs_eval/skip 是"还没接进来")
oracle 每个 (model × march × backend × qcfg) 一个 job:PENDING 没开始 / ORACLE_RUNNING 在跑 / PASS·FAIL·UNVERIFIED 已结束
rollout 每个 job 计划 k 条轨迹:QUEUED 没采 / RUNNING 在采 / PASS·FAIL·BLOCKED·TIMEOUT 已采
"是不是还在跑"看顶部那行 LIVE:有没有调度进程、池占了几个槽、最近一次阶段心跳多久以前。
"""
from __future__ import annotations
import csv as csvmod
import json
import time
from collections import Counter, defaultdict
from pathlib import Path
from . import paths, db
from .registry import models, expand_matrix
from .scheduler import pool_usage, pool_size
DONE = ("PASS", "FAIL", "UNVERIFIED")
ROLLOUT_DONE = ("PASS", "FAIL", "BLOCKED", "TIMEOUT", "UNVERIFIED")
def _fmt_age(s: float | None) -> str:
if s is None:
return "从未"
s = int(s)
if s < 90:
return f"{s}s 前"
if s < 5400:
return f"{s // 60}m 前"
return f"{s // 3600}h{(s % 3600) // 60:02d}m 前"
def live_line() -> list[str]:
"""是不是还在跑:调度进程 + 池占用 + 最近心跳。"""
from .pipeline import scheduler_alive
out = []
alive = scheduler_alive()
pools = pool_usage()
busy = sum(pools.values())
with db.connect() as con:
row = con.execute("SELECT MAX(COALESCE(finished_at, started_at)) FROM stages").fetchone()
last = row[0] if row and row[0] else None
n_run = con.execute("SELECT COUNT(*) FROM stages WHERE state='RUNNING'").fetchone()[0]
age = (time.time() - last) if last else None
if alive or busy or n_run:
state = "🟢 在跑"
elif age is not None and age < 300:
state = "🟡 刚停(进程没了但 5 分钟内还有活动)"
else:
state = "🔴 没在跑"
out.append(f"LIVE {state} 调度进程={'有' if alive else '无'} "
+ " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pools.items())
+ f" RUNNING 阶段={n_run} 最近心跳={_fmt_age(age)}")
if not alive and not busy:
out.append(" 没有调度进程:要继续跑就 scripts/start_run.sh --resume(先 reset-stale 再接着跑 PENDING)")
return out
def exported_keys() -> set[tuple]:
"""已经进过 datasets/ 的 (run_id, job_id, kk)。"""
import json
keys = set()
files = sorted(paths.DATASETS.glob("*/teacher.jsonl")) + sorted(paths.DATASETS.glob("*/student.jsonl"))
for f in files:
if (f.parent / "VOID").exists():
continue
try:
with f.open(errors="replace") as stream:
for line in stream:
r = json.loads(line)
if (r.get("source") in (None, "agent_events") and isinstance(r.get("messages"), list)
and isinstance(r.get("rollout"), int)):
keys.add((r.get("run_id"), r.get("job_id"), r["rollout"]))
except (OSError, json.JSONDecodeError):
continue
return keys
def collect(run_id: str | None) -> dict:
"""汇总指定 run 的实际计划;run_id=None 时只展示 registry 的预期。"""
rows = models()
by_model = {r["model_id"]: r for r in rows}
jobs, ro = [], []
if run_id is not None:
with db.connect() as con:
jobs = db.list_jobs(con, run_id, mode="oracle")
ro = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=?", (run_id,))]
planned_rollouts = Counter(r["job_id"] for r in ro)
exported = exported_keys() if run_id is not None else set()
per_model: dict[str, dict] = {}
for m, r in by_model.items():
per_model[m] = {"model": m, "task": r["task"], "priority": r["priority"], "status": r.get("status") or "ready",
"jobs": [], "oracle": Counter(), "rollout": Counter(),
"planned_rollouts": 0, "exported": 0}
if run_id is None:
for spec in expand_matrix():
if spec.mode != "oracle" and spec.model in per_model:
per_model[spec.model]["planned_rollouts"] += spec.k
for j in jobs:
d = per_model.get(j["model"])
if d is None:
continue
d["jobs"].append(j)
d["oracle"][j["state"]] += 1
d["planned_rollouts"] += planned_rollouts.get(j["job_id"], 0)
for r in ro:
jrow = next((x for x in jobs if x["job_id"] == r["job_id"]), None)
if jrow:
model = jrow["model"]
else:
jp = paths.RUNS / run_id / r["job_id"] / "job.json"
model = json.loads(jp.read_text()).get("model") if jp.exists() else None
d = per_model.get(model)
if d is None:
continue
if not jrow:
d["planned_rollouts"] += 1
d["rollout"][r["state"]] += 1
if (run_id, r["job_id"], r["kk"]) in exported:
d["exported"] += 1
return per_model
def render(run_id: str, task: str | None = None, missing: bool = False, detail: bool = False) -> str:
per_model = collect(run_id)
lines = [f"bench coverage · run {run_id} · {time.strftime('%Y-%m-%d %H:%M:%S')}"]
lines += live_line()
lines.append("")
# ---- registry 层
st = Counter(d["status"] for d in per_model.values())
ready = [d for d in per_model.values() if d["status"] == "ready"]
lines.append(f"registry: 共 {len(per_model)} 个模型 · " + " · ".join(f"{k}={v}" for k, v in st.most_common())
+ " (只有 ready 会被展开成 job;其余是「还没接进来」,不是「失败」)")
# ---- oracle 层
oc = Counter()
for d in ready:
oc.update(d["oracle"])
total_jobs = sum(oc.values())
done_jobs = sum(oc[k] for k in DONE)
lines.append(f"oracle: {total_jobs} 个 job · 已结束 {done_jobs}"
+ (f"(PASS {oc['PASS']} / FAIL {oc['FAIL']} / UNVERIFIED {oc['UNVERIFIED']})" if done_jobs else "")
+ f" · 在跑 {oc['ORACLE_RUNNING']} · 没开始 {oc['PENDING']}")
# ---- rollout 层
rc = Counter()
planned = exported = 0
for d in ready:
rc.update(d["rollout"]); planned += d["planned_rollouts"]; exported += d["exported"]
got = sum(rc[k] for k in ROLLOUT_DONE)
lines.append(f"rollout: 计划 {planned} 条 · 已采 {got}"
+ (f"(PASS {rc['PASS']} / FAIL {rc['FAIL']} / BLOCKED {rc['BLOCKED']} / TIMEOUT {rc['TIMEOUT']} / UNVERIFIED {rc['UNVERIFIED']})" if got else "")
+ f" · 在采 {rc['RUNNING'] + rc['VERIFYING']}(RUNNING {rc['RUNNING']} / VERIFYING {rc['VERIFYING']})"
+ f" · 没采 {rc['QUEUED']} · 已导出 {exported}")
lines.append("")
# ---- 按任务分组
groups: dict[str, list[dict]] = defaultdict(list)
for d in per_model.values():
groups[d["task"]].append(d)
lines.append(f" {'task':<7} {'模型':>5} {'ready':>6} | {'job':>5} {'已结束':>6} {'PASS':>5} {'FAIL':>5} {'在跑':>4} {'没开始':>6} | "
f"{'轨迹计划':>8} {'已采':>5} {'PASS':>5} {'没采':>5}")
for t in sorted(groups):
g = groups[t]
gr = [d for d in g if d["status"] == "ready"]
o, r = Counter(), Counter()
pl = 0
for d in gr:
o.update(d["oracle"]); r.update(d["rollout"]); pl += d["planned_rollouts"]
lines.append(f" {t:<7} {len(g):>5} {len(gr):>6} | {sum(o.values()):>5} {sum(o[k] for k in DONE):>6} "
f"{o['PASS']:>5} {o['FAIL']:>5} {o['ORACLE_RUNNING']:>4} {o['PENDING']:>6} | "
f"{pl:>8} {sum(r[k] for k in ROLLOUT_DONE):>5} {r['PASS']:>5} {r['QUEUED']:>5}")
# ---- 逐模型
sel = [d for d in per_model.values() if (not task or d["task"] == task)]
if missing:
sel = [d for d in sel if d["status"] != "ready"
or d["oracle"].get("PENDING") or not d["jobs"]
or d["rollout"].get("QUEUED")]
lines.append("")
lines.append(f" 还没做的({len(sel)}):status≠ready,或 oracle 有 PENDING,或轨迹有 QUEUED")
if missing or detail:
lines.append("")
lines.append(f" {'model':<26} {'task':<6} {'pri':<4} {'status':<14} {'oracle':<34} {'rollout':<26}")
for d in sorted(sel, key=lambda x: (x["task"], x["priority"], x["model"])):
o = " ".join(f"{k}={v}" for k, v in sorted(d["oracle"].items())) or ("—(未展开)" if d["status"] != "ready" else "—")
r = " ".join(f"{k}={v}" for k, v in sorted(d["rollout"].items())) or "—"
if d["exported"]:
r += f" 导出={d['exported']}"
lines.append(f" {d['model']:<26} {d['task']:<6} {d['priority']:<4} {d['status']:<14} {o:<34} {r:<26}")
else:
lines.append("")
lines.append(" 逐模型明细:--detail(可加 --task cls);只看没做的:--missing;导出表格:--csv cov.csv")
return "\n".join(lines)
def to_csv(run_id: str, out: str) -> str:
per_model = collect(run_id)
fields = ["model", "task", "priority", "status", "jobs", "oracle_done", "oracle_pass", "oracle_fail",
"oracle_unverified", "oracle_running", "oracle_pending", "rollout_planned", "rollout_done",
"rollout_pass", "rollout_queued", "exported"]
with open(out, "w", newline="", encoding="utf-8") as f:
w = csvmod.DictWriter(f, fieldnames=fields)
w.writeheader()
for d in sorted(per_model.values(), key=lambda x: (x["task"], x["priority"], x["model"])):
o, r = d["oracle"], d["rollout"]
w.writerow({"model": d["model"], "task": d["task"], "priority": d["priority"], "status": d["status"],
"jobs": len(d["jobs"]), "oracle_done": sum(o[k] for k in DONE), "oracle_pass": o["PASS"],
"oracle_fail": o["FAIL"], "oracle_unverified": o["UNVERIFIED"],
"oracle_running": o["ORACLE_RUNNING"], "oracle_pending": o["PENDING"],
"rollout_planned": d["planned_rollouts"], "rollout_done": sum(r[k] for k in ROLLOUT_DONE),
"rollout_pass": r["PASS"], "rollout_queued": r["QUEUED"], "exported": d["exported"]})
return out
def model_blockers(row: dict, cache: dict | None = None, evaluation: dict | None = None) -> list[str]:
"""Check concrete local prerequisites, independently of CSV status labels.
This is admission checking, not a claim of successful conversion or accuracy.
The latter can only come from stages and verdicts.
"""
from .eval import TASK_MODULES
from .registry import datasets
from .adapters import model_spec, calibration_dir
row = model_spec(row)
cache = {} if cache is None else cache
def once(key, read):
if key not in cache:
cache[key] = read()
return cache[key]
def exists(p):
return once(("file", str(p)), p.is_file)
def read_json(p):
return once(("json", str(p)), lambda: json.loads(p.read_text()))
task, mid, source = row["task"], row["model_id"], row.get("source")
if source == "opencv":
return ["non_neural_algorithm: no neural weights/PTQ target; retain in inventory"]
reasons = []
uri = row.get("weights", "")
if uri.startswith(("https://github.com/", "https://docs.", "https://huggingface.co/")) and not any(
part in uri for part in ("/releases/download/", "/resolve/")):
reasons.append("weights_is_catalog_page: concrete checkpoint/revision missing")
elif not uri.startswith(("torchvision://", "ultralytics://", "hf://", "file://", "https://", "http://")):
reasons.append("weights_uri_unsupported_or_missing")
adapter = paths.BENCH_ROOT / "harness" / "adapters" / (mid.replace(".", "_").replace("-", "_") + ".py")
adapters = once("adapter_files", lambda: {p.name for p in adapter.parent.glob("*.py")})
if source not in ("torchvision", "ultralytics", "recipe") and adapter.name not in adapters:
reasons.append(f"model_builder_export_adapter_missing: {adapter.relative_to(paths.BENCH_ROOT)}")
if mid.startswith("fastsam."):
reasons.append("metric_semantics: FastSAM class-agnostic masks cannot use 80-class COCO mask AP")
if task not in TASK_MODULES:
reasons.append(f"task_evaluator_missing: {task}")
if source == "ultralytics" and task in ("iseg", "pose") and not mid.startswith("fastsam."):
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
proof = read_json(probe) if exists(probe) else {}
if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid:
reasons.append(f"real_checkpoint_export_probe_missing_or_failed: {probe}")
if task == "pose":
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
proof = read_json(probe) if exists(probe) else {}
if (proof.get("pose_export_shift", {}).get("status") != "measured"
or proof["pose_export_shift"]["keypoint_mean_shift_px"] > 2):
reasons.append("auxiliary_gate_probe_missing: actual FP32/ONNX keypoint correspondence and <=2px shift required")
evidence = proof.get("pose_export_evidence") or {}
if not evidence.get("sha256") or not exists(Path(evidence.get("path", "__missing__"))):
reasons.append("auxiliary_gate_raw_evidence_missing: replayable matched keypoint coordinates required")
if source not in ("torchvision", "ultralytics", "opencv", "recipe") and adapter.name in adapters:
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
proof = read_json(probe) if exists(probe) else {}
if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid:
reasons.append("custom_adapter_unvalidated: builder present; environment/preprocessing/real forward/export proof still required")
if row.get("preprocess") and json.dumps(proof.get("preprocess"), sort_keys=True) != json.dumps(row["preprocess"], sort_keys=True):
reasons.append("preprocessing_recipe_not_validated: probe must use the current model recipe")
if task in ("stereo", "flow", "feat", "occ"):
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
proof = read_json(probe) if exists(probe) else {}
contract = proof.get("input_contract") or {}
if (proof.get("status") != "validated_fp32_export_only" or contract.get("status") != "validated"
or contract.get("input_spec") != row.get("input_spec")):
reasons.append("input_contract_unverified: real pair/multi-input/camera metadata and export proof required")
ds = row.get("dataset", "")
ds_spec = datasets().get("datasets", {}).get(ds, {})
if task not in ds_spec.get("tasks", []):
reasons.append(f"dataset_task_mismatch: {ds} is not registered for {task}")
if evaluation is None:
probe_path = paths.CACHE / "adapter-probes" / mid / "probe.json"
proof = read_json(probe_path) if exists(probe_path) else {}
if proof.get("status") == "validated_fp32_export_only":
evaluation = proof.get("eval_set")
manifest = (Path(evaluation["path"]).parent / "manifest.json" if evaluation else
paths.DATA / "t1" / ds / "manifest.json")
if not exists(manifest):
reasons.append(f"t1_missing: {manifest}")
else:
m = read_json(manifest)
if not m.get("sha256") or not exists(manifest.parent / m.get("tar", "__missing__")):
reasons.append("t1_package_or_identity_missing")
elif evaluation and (m.get("dataset") != ds or m.get("n") != evaluation.get("n")
or m.get("sha256") != evaluation.get("manifest_sha")
or m.get("content_sha256") != evaluation.get("content_sha")):
reasons.append("t1_explicit_identity_mismatch")
elif task in ("iseg", "pose") and ds == "coco2017":
import tarfile
def annotations():
with tarfile.open(manifest.parent / m["tar"]) as archive:
return json.load(archive.extractfile("annotations.json"))
ann = once(("annotations", ds), annotations)
if task == "pose" and not ann.get("keypoints", {}).get("annotations"):
reasons.append("person_keypoint_annotations_missing")
if task == "iseg" and not any(a.get("segmentation") for a in ann.get("annotations", [])):
reasons.append("instance_mask_annotations_missing")
calib = calibration_dir(row)
if not exists(calib / "manifest.json"):
reasons.append(f"calibration_missing: {calib}")
else:
cm = read_json(calib / "manifest.json")
count = once(("calibration_count", str(calib)), lambda: len(list((calib / cm.get("npy_dir", "npy_f32")).glob("*.npy"))))
if not cm.get("sha256") or count < cm.get("count", 128) or not count:
reasons.append("calibration_samples_or_identity_missing")
return reasons
def master_inventory() -> dict:
"""All CSV models, all runs, with evidence joins. Never changes task state."""
from .verify import completion_summary
with db.readonly() as con:
jobs = [dict(r) for r in con.execute("SELECT * FROM jobs WHERE mode='oracle'")]
rollouts = [dict(r) for r in con.execute("SELECT * FROM rollouts")]
stages = [dict(r) for r in con.execute("SELECT * FROM stages")]
by_job = {(j["run_id"], j["job_id"]): j for j in jobs}
exports = {}
for f in sorted(paths.DATASETS.glob("*/teacher.jsonl")):
if (f.parent / "VOID").exists():
continue
for line in f.open():
record = json.loads(line)
if record.get("source") == "agent_events" and record.get("messages"):
key = (record["run_id"], record["job_id"], record["rollout"])
exports[key] = {"path": str(f), "id": record["id"], "model": record.get("model"),
"sft_eligible": record.get("sft_eligible", False)}
per_model, prerequisite_cache = {}, {}
for row in models():
from .adapters import model_spec
row = model_spec(row)
mid = row["model_id"]
per_model[mid] = {"model": mid, "name": row.get("name"), "task": row["task"],
"priority": row["priority"], "registry_status": row.get("status"),
"source": row.get("source"), "weights": row.get("weights"),
"blockers": model_blockers(row, prerequisite_cache), "jobs": [], "rounds": []}
probe = paths.CACHE / "adapter-probes" / mid / "probe.json"
per_model[mid]["adapter_probe"] = {"path": str(probe), **json.loads(probe.read_text())} if probe.is_file() else None
for j in jobs:
d = per_model.get(j["model"])
if d is None:
continue
result = json.loads(j["result"]) if j.get("result") else {}
directory = paths.RUNS / j["run_id"] / j["job_id"]
d["jobs"].append({"run_id": j["run_id"], "job_id": j["job_id"], "state": j["state"],
"error_class": j["error_class"], "stage_failed": j["stage_failed"],
"path": str(directory / "oracle"),
"completion": result.get("completion") or completion_summary(result.get("gates", {}), result.get("stages", {})),
"gates": result.get("gates", {}), "eval_set": result.get("eval_set"),
"stages": [s for s in stages if s["run_id"] == j["run_id"] and s["job_id"] == j["job_id"] and s["rollout"] == -1]})
for r in rollouts:
job = by_job.get((r["run_id"], r["job_id"]))
mid = job["model"] if job else r["job_id"].split("--")[1]
if mid not in per_model:
continue
rd = paths.RUNS / r["run_id"] / r["job_id"] / "rollouts" / f"{r['kk']:02d}"
# Hundreds of QUEUED rows have no directory yet. Avoid FUSE lookups for
# each absent file; pending plans are not execution evidence.
started = r["state"] != "QUEUED" or r.get("started_at") is not None
meta = json.loads((rd / "rollout.json").read_text()) if started and (rd / "rollout.json").is_file() else {}
verdict = json.loads(r["verdict"]) if r.get("verdict") else {}
raw = {str(p): p.stat().st_size for p in (rd / "agent" / "events.jsonl", rd / "agent" / "session.jsonl") if started and p.is_file()}
observed_models = set()
events = rd / "agent" / "events.jsonl"
if str(events) in raw:
with events.open() as stream:
for line in stream:
try:
event = json.loads(line)
except json.JSONDecodeError: # an active writer's last partial line
continue
message = event.get("message") or {}
if (event.get("type") == "message_end" and isinstance(message, dict)
and message.get("role") == "assistant" and message.get("model") and message.get("content")):
observed_models.add(message["model"])
break # first real response identity; raw full stream stays linked
exported = exports.get((r["run_id"], r["job_id"], r["kk"]))
per_model[mid]["rounds"].append({"run_id": r["run_id"], "job_id": r["job_id"], "kk": r["kk"],
"state": r["state"], "mode": r["mode"], "policy": r["policy"], "model": meta.get("model"),
"raw_traces": raw, "observed_models": sorted(observed_models), "path": str(rd), "export": exported,
"completion": verdict.get("completion") or completion_summary(verdict.get("gates", {}), verdict.get("stages", {})),
"gates": verdict.get("gates", {}),
"eval_set": verdict.get("eval_set"), "error_class": verdict.get("error_class")})
return {"schema": "bench.coverage/2", "captured_at": time.time(), "models": list(per_model.values())}
def write_master(out: str) -> str:
"""CSV summary + adjacent JSON containing every run/job/round/evidence link."""
import os
import tempfile
inventory = master_inventory()
destination = Path(out)
destination.parent.mkdir(parents=True, exist_ok=True)
fields = ["model", "name", "task", "priority", "registry_status", "source", "weights", "admission",
"blockers", "adapter_validation", "adapter_evidence", "job_ids", "latest_run", "latest_oracle_state", "latest_stage", "latest_error",
"oracle_states", "calibration", "quantization", "compile", "hbm_generated", "board", "accuracy", "accuracy_passed",
"first_glm53_trace", "glm53_exported", "sft_eligible", "round_states", "runtime_errors", "evidence_json"]
fd, tmp = tempfile.mkstemp(dir=destination.parent, prefix=destination.name + ".")
try:
with os.fdopen(fd, "w") as stream:
writer = csvmod.DictWriter(stream, fieldnames=fields)
writer.writeheader()
for d in inventory["models"]:
all_results = d["jobs"] + d["rounds"]
def done(key):
return sum(bool(x.get("completion", {}).get(key)) for x in all_results)
glm = [r for r in d["rounds"] if r["model"] == "glm-5.3" and "glm-5.3" in r["observed_models"]]
exported = [r for r in glm if r["export"] and r["export"]["model"] == "glm-5.3"]
latest = max(d["jobs"], key=lambda j: j["run_id"], default={})
latest_stage = max(latest.get("stages", []), key=lambda s: s.get("started_at") or 0, default={})
writer.writerow({**{k: d.get(k) for k in fields if k in d and k != "blockers"},
"admission": "blocked" if d["blockers"] else "prerequisites_present",
"blockers": " | ".join(d["blockers"]),
"adapter_validation": (d.get("adapter_probe") or {}).get("status", ""),
"adapter_evidence": (d.get("adapter_probe") or {}).get("path", ""),
"job_ids": " | ".join(j["run_id"] + "/" + j["job_id"] for j in d["jobs"]),
"latest_run": latest.get("run_id", ""), "latest_oracle_state": latest.get("state", "not_queued"),
"latest_stage": latest_stage.get("stage", ""), "latest_error": latest.get("error_class", ""),
"oracle_states": json.dumps(dict(Counter(j["state"] for j in d["jobs"]))),
"calibration": sum(any(s["stage"] == "04-calibrate" and s["state"] in ("OK", "SKIPPED") for s in j["stages"]) for j in d["jobs"]),
"quantization": done("compiled"), "compile": done("compiled"), "hbm_generated": done("hbm_generated"),
"board": done("board_completed"), "accuracy": done("accuracy_evaluated"), "accuracy_passed": done("accuracy_passed"),
"first_glm53_trace": glm[0]["path"] if glm else "", "glm53_exported": len(exported),
"sft_eligible": sum(r["export"]["sft_eligible"] for r in exported),
"round_states": json.dumps(dict(Counter(r["state"] for r in d["rounds"]))),
"runtime_errors": " | ".join(sorted({str(x.get("stage_failed") or "verdict") + ":" + x["error_class"] for x in all_results if x.get("error_class")})),
"evidence_json": str(destination.with_suffix(".json"))})
os.replace(tmp, destination)
fdj, tmpj = tempfile.mkstemp(dir=destination.parent, prefix=destination.stem + ".json.")
with os.fdopen(fdj, "w") as stream:
json.dump(inventory, stream, ensure_ascii=False, indent=2)
os.replace(tmpj, destination.with_suffix(".json"))
finally:
if os.path.exists(tmp):
os.unlink(tmp)
return str(destination)