343 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""state/jobs.sqlite — 状态、error_class、耗时历史的唯一真相。文件系统只存内容。"""
from __future__ import annotations
import json
import os
import sqlite3
import socket
import time
from contextlib import contextmanager
from pathlib import Path
from .paths import DB_PATH, STATE
SCHEMA = """
CREATE TABLE IF NOT EXISTS runs (
run_id TEXT PRIMARY KEY,
created_at REAL NOT NULL,
manifest TEXT NOT NULL, -- json:venv pip-freeze sha、板固件、registry 指纹、数据 manifest sha
note TEXT
);
CREATE TABLE IF NOT EXISTS jobs (
run_id TEXT NOT NULL,
job_id TEXT NOT NULL,
task TEXT, model TEXT, march TEXT, backend TEXT, qcfg TEXT, priority TEXT,
mode TEXT NOT NULL, -- oracle | e2e | repair
policy TEXT, -- teacher | student | NULL
k INTEGER DEFAULT 1,
state TEXT NOT NULL, -- PENDING ORACLE_RUNNING PREPARED PASS FAIL UNVERIFIED
error_class TEXT,
stage_failed TEXT,
estimate_s REAL,
started_at REAL, finished_at REAL,
result TEXT, -- oracle/result.json 副本
note TEXT,
PRIMARY KEY (run_id, job_id, mode, policy)
);
CREATE TABLE IF NOT EXISTS stages (
run_id TEXT NOT NULL, job_id TEXT NOT NULL,
rollout INTEGER NOT NULL DEFAULT -1, -- -1 = oracle
stage TEXT NOT NULL,
attempt INTEGER NOT NULL DEFAULT 0,
lane TEXT DEFAULT 'normal', -- normal | long
state TEXT NOT NULL, -- RUNNING OK FAIL STALLED SKIPPED
exit_code INTEGER,
error_class TEXT,
started_at REAL, finished_at REAL, duration_s REAL,
PRIMARY KEY (run_id, job_id, rollout, stage, attempt)
);
CREATE TABLE IF NOT EXISTS rollouts (
run_id TEXT NOT NULL, job_id TEXT NOT NULL, kk INTEGER NOT NULL,
mode TEXT, policy TEXT,
state TEXT NOT NULL, -- QUEUED RUNNING VERIFYING PASS FAIL BLOCKED TIMEOUT
claim_terminal TEXT,
verdict_terminal TEXT,
claim_consistent INTEGER,
review INTEGER DEFAULT 0, -- oracle FAIL / rollout PASS → 1,进人工抽查
turns INTEGER, tokens INTEGER, wall_s REAL,
verdict TEXT,
started_at REAL, finished_at REAL,
PRIMARY KEY (run_id, job_id, kk)
);
CREATE TABLE IF NOT EXISTS rollout_context (
run_id TEXT NOT NULL, job_id TEXT NOT NULL, kk INTEGER NOT NULL,
runner TEXT NOT NULL DEFAULT 'pi',
owner_host TEXT, owner_pid INTEGER,
PRIMARY KEY (run_id, job_id, kk)
);
CREATE TABLE IF NOT EXISTS annotations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id TEXT, job_id TEXT, ts REAL, note TEXT
);
CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(run_id, state);
CREATE INDEX IF NOT EXISTS idx_stages_hist ON stages(job_id, stage, state);
"""
def _open(path: Path, *, timeout_s: float = 60) -> sqlite3.Connection:
con = sqlite3.connect(path, timeout=timeout_s)
con.row_factory = sqlite3.Row
return con
def is_busy(exc: sqlite3.OperationalError) -> bool:
# Python 3.10 does not expose sqlite_errorcode or SQLITE_BUSY constants.
code = getattr(exc, 'sqlite_errorcode', None)
if code is not None:
return (code & 255) in (5, 6)
return str(exc).lower().split(':', 1)[0] in (
'database is locked', 'database table is locked', 'database schema is locked', 'database is busy')
@contextmanager
def connect(path: Path = DB_PATH, *, timeout_s: float = 60):
Path(path).parent.mkdir(parents=True, exist_ok=True)
con = _open(path, timeout_s=timeout_s)
# 默认 TRUNCATE:WAL 在 FUSE(JuiceFS 等)上会 disk I/O error(-shm 需要 mmap)。
# state/ 放本地盘(BENCH_STATE_DIR=/home/<user>/bench-state)时可设 BENCH_SQLITE_JOURNAL=WAL。
try:
jm = os.environ.get("BENCH_SQLITE_JOURNAL", "TRUNCATE")
con.execute(f"PRAGMA journal_mode={jm}")
con.executescript(SCHEMA)
yield con
con.commit()
finally:
con.close()
@contextmanager
def readonly(path: Path = DB_PATH):
con = sqlite3.connect(f'file:{path}?mode=ro', uri=True, timeout=60)
con.row_factory = sqlite3.Row
try:
yield con
finally:
con.close()
# ---------------------------------------------------------------- runs
def create_run(con, run_id: str, manifest: dict, note: str = "") -> None:
if con.execute("SELECT 1 FROM runs WHERE run_id=?", (run_id,)).fetchone():
raise RuntimeError(f"run_id 已存在(append-only,不允许覆盖): {run_id}")
con.execute("INSERT INTO runs VALUES (?,?,?,?)",
(run_id, time.time(), json.dumps(manifest, ensure_ascii=False), note))
def get_run(con, run_id: str) -> dict | None:
r = con.execute("SELECT * FROM runs WHERE run_id=?", (run_id,)).fetchone()
return dict(r) if r else None
def assert_run_open(con, run_id: str) -> None:
run = get_run(con, run_id)
target = json.loads(run["manifest"]).get("continued_to") if run else None
if target:
raise ValueError(f"run {run_id} handed off its remaining work to {target}; execution is closed")
def latest_run_id(con) -> str | None:
r = con.execute("SELECT run_id FROM runs ORDER BY created_at DESC LIMIT 1").fetchone()
return r[0] if r else None
def next_run_id(con, date_str: str) -> str:
rows = con.execute("SELECT run_id FROM runs WHERE run_id LIKE ?", (f"r{date_str}-%",)).fetchall()
seq = 1 + max([int(r[0].rsplit("-", 1)[1]) for r in rows if r[0].rsplit("-", 1)[1].isdigit()] or [0])
return f"r{date_str}-{seq:02d}"
# ---------------------------------------------------------------- jobs
def upsert_job(con, run_id: str, spec_row: dict, estimate_s: float | None) -> bool:
"""展开幂等:已存在的 (run_id, job_id, mode, policy) 不动。返回是否新插入。"""
key = (run_id, spec_row["job_id"], spec_row["mode"], spec_row.get("policy"))
if con.execute("SELECT 1 FROM jobs WHERE run_id=? AND job_id=? AND mode=? AND policy IS ?", key).fetchone():
return False
con.execute(
"INSERT INTO jobs (run_id, job_id, task, model, march, backend, qcfg, priority, mode, policy, k, state, estimate_s)"
" VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
(run_id, spec_row["job_id"], spec_row["task"], spec_row["model"], spec_row["march"],
spec_row["backend"], spec_row["qcfg"], spec_row["priority"], spec_row["mode"],
spec_row.get("policy"), spec_row.get("k", 1), "PENDING", estimate_s))
return True
def claim_job(con, run_id: str, job_id: str, mode: str = "oracle", force: bool = False,
prepare_only: bool = False) -> bool:
"""原子认领:准备只取 PENDING;完整 Oracle 也可取 PREPARED,同一 job 只有一个进程赢。
force=True 只允许完整 Oracle 重跑已结束的 job(显式 --oracle --job,原地覆盖 result.json)。"""
con.execute("BEGIN IMMEDIATE")
assert_run_open(con, run_id)
allowed = "('PENDING','PREPARED')" if not force else "('PENDING','PREPARED','PASS','FAIL','UNVERIFIED')"
if prepare_only:
allowed = "('PENDING')" # Preparation never overwrites a finished job, even with --job.
cur = con.execute(f"UPDATE jobs SET state='ORACLE_RUNNING', started_at=? WHERE run_id=? AND job_id=? AND mode=? AND state IN {allowed}",
(time.time(), run_id, job_id, mode))
con.commit()
return cur.rowcount == 1
def set_job_state(con, run_id: str, job_id: str, state: str, *, mode: str = "oracle",
error_class: str | None = None, stage_failed: str | None = None,
result: dict | None = None) -> None:
now = time.time()
if state == "ORACLE_RUNNING":
con.execute("UPDATE jobs SET state=?, started_at=? WHERE run_id=? AND job_id=? AND mode=?",
(state, now, run_id, job_id, mode))
else:
con.execute("UPDATE jobs SET state=?, finished_at=?, error_class=?, stage_failed=?, result=?"
" WHERE run_id=? AND job_id=? AND mode=?",
(state, now, error_class, stage_failed,
json.dumps(result, ensure_ascii=False) if result else None, run_id, job_id, mode))
def get_job(con, run_id: str, job_id: str, mode: str = "oracle") -> dict | None:
r = con.execute("SELECT * FROM jobs WHERE run_id=? AND job_id=? AND mode=?", (run_id, job_id, mode)).fetchone()
return dict(r) if r else None
def list_jobs(con, run_id: str, mode: str | None = None, state: str | None = None) -> list[dict]:
q, args = "SELECT * FROM jobs WHERE run_id=?", [run_id]
if mode:
q += " AND mode=?"; args.append(mode)
if state:
q += " AND state=?"; args.append(state)
q += " ORDER BY CASE priority WHEN 'A' THEN 0 WHEN 'B' THEN 1 ELSE 2 END, estimate_s, job_id"
return [dict(r) for r in con.execute(q, args)]
# ---------------------------------------------------------------- stages
def stage_start(con, run_id, job_id, stage, attempt=0, rollout=-1, lane="normal") -> None:
con.execute("INSERT OR REPLACE INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, started_at)"
" VALUES (?,?,?,?,?,?,'RUNNING',?)", (run_id, job_id, rollout, stage, attempt, lane, time.time()))
def stage_finish(con, run_id, job_id, stage, attempt, state, exit_code=None, error_class=None, rollout=-1) -> None:
now = time.time()
r = con.execute("SELECT started_at FROM stages WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?",
(run_id, job_id, rollout, stage, attempt)).fetchone()
started = r[0] if r and r[0] else now
con.execute("UPDATE stages SET state=?, exit_code=?, error_class=?, finished_at=?, duration_s=?"
" WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?",
(state, exit_code, error_class, now, now - started, run_id, job_id, rollout, stage, attempt))
def stage_history(con, job_id: str, stage: str) -> list[float]:
"""同 job_id(跨 run)同阶段的历史成功耗时,用于估算。"""
return [r[0] for r in con.execute(
"SELECT duration_s FROM stages WHERE job_id=? AND stage=? AND state='OK' AND rollout=-1 AND duration_s IS NOT NULL",
(job_id, stage))]
def stage_history_by_model(con, model: str, stage: str) -> list[float]:
return [r[0] for r in con.execute(
"SELECT s.duration_s FROM stages s JOIN jobs j ON s.run_id=j.run_id AND s.job_id=j.job_id"
" WHERE j.model=? AND s.stage=? AND s.state='OK' AND s.rollout=-1 AND s.duration_s IS NOT NULL",
(model, stage))]
# ---------------------------------------------------------------- rollouts
def rollout_runner(con, run_id, job_id, kk) -> str:
# A read-only old database may not yet contain the additive metadata table.
if not con.execute("SELECT 1 FROM sqlite_master WHERE name='rollout_context'").fetchone():
return "pi"
row = con.execute("SELECT runner FROM rollout_context WHERE run_id=? AND job_id=? AND kk=?",
(run_id, job_id, kk)).fetchone()
return row[0] if row else "pi"
def rollout_upsert(con, run_id, job_id, kk, mode, policy, state="QUEUED", *, runner="pi") -> None:
existing = con.execute("SELECT mode,policy FROM rollouts WHERE run_id=? AND job_id=? AND kk=?",
(run_id, job_id, kk)).fetchone()
if existing and (existing[0], existing[1], rollout_runner(con, run_id, job_id, kk)) != (mode, policy, runner):
raise ValueError(f"rollout identity conflict: {run_id}/{job_id}/{kk}")
inserted = con.execute("INSERT OR IGNORE INTO rollouts (run_id, job_id, kk, mode, policy, state) VALUES (?,?,?,?,?,?)",
(run_id, job_id, kk, mode, policy, state)).rowcount
if not inserted:
actual = con.execute("SELECT mode,policy FROM rollouts WHERE run_id=? AND job_id=? AND kk=?",
(run_id, job_id, kk)).fetchone()
if (actual[0], actual[1], rollout_runner(con, run_id, job_id, kk)) != (mode, policy, runner):
raise ValueError(f"rollout identity conflict: {run_id}/{job_id}/{kk}")
con.execute("INSERT OR IGNORE INTO rollout_context(run_id,job_id,kk,runner) VALUES (?,?,?,?)",
(run_id, job_id, kk, runner))
def next_rollout_kk(con, run_id, job_id) -> int:
return con.execute("SELECT COALESCE(MAX(kk),-1)+1 FROM rollouts WHERE run_id=? AND job_id=?",
(run_id, job_id)).fetchone()[0]
def ensure_rollout_plan(con, run_id, job_id, mode, policy, k, *, runner="pi") -> int:
"""Caller holds BEGIN IMMEDIATE; all modes/policies/runners share kk space."""
if not con.in_transaction:
raise RuntimeError("rollout allocation requires a transaction")
if not isinstance(k, int) or k < 0:
raise ValueError("rollout k must be a nonnegative integer")
rows = list_rollouts(con, run_id, job_id)
count = sum((r["mode"], r["policy"], r["runner"]) == (mode, policy, runner) for r in rows)
added = max(0, k - count)
kk = next_rollout_kk(con, run_id, job_id)
for i in range(added):
rollout_upsert(con, run_id, job_id, kk + i, mode, policy, runner=runner)
return added
def claim_rollout(con, run_id: str, job_id: str, kk: int, max_active: int | None = None) -> bool:
"""原子认领一条 QUEUED;同一 job 只允许一条运行,跨 collector 也不重复。"""
con.execute("BEGIN IMMEDIATE")
assert_run_open(con, run_id)
if max_active is not None:
active = con.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0]
if active >= max_active:
con.commit()
return False
cur = con.execute(
"UPDATE rollouts SET state='RUNNING', started_at=?"
" WHERE run_id=? AND job_id=? AND kk=? AND state='QUEUED'"
" AND NOT EXISTS (SELECT 1 FROM rollouts active WHERE active.run_id=? AND active.job_id=?"
" AND active.state IN ('RUNNING','VERIFYING'))",
(time.time(), run_id, job_id, kk, run_id, job_id))
if cur.rowcount:
con.execute("INSERT OR IGNORE INTO rollout_context(run_id,job_id,kk,runner) VALUES (?,?,?,'pi')",
(run_id, job_id, kk))
con.execute("UPDATE rollout_context SET owner_host=?,owner_pid=? WHERE run_id=? AND job_id=? AND kk=?",
(socket.gethostname(), os.getpid(), run_id, job_id, kk))
con.commit()
return cur.rowcount == 1
def rollout_set(con, run_id, job_id, kk, **fields) -> None:
if not fields:
return
cols = ", ".join(f"{k}=?" for k in fields)
con.execute(f"UPDATE rollouts SET {cols} WHERE run_id=? AND job_id=? AND kk=?",
(*fields.values(), run_id, job_id, kk))
def list_rollouts(con, run_id, job_id) -> list[dict]:
rows = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=? AND job_id=? ORDER BY kk", (run_id, job_id))]
for row in rows:
row["runner"] = rollout_runner(con, run_id, job_id, row["kk"])
return rows
# ---------------------------------------------------------------- annotations / status
def annotate(con, run_id, job_id, note) -> None:
con.execute("INSERT INTO annotations (run_id, job_id, ts, note) VALUES (?,?,?,?)", (run_id, job_id, time.time(), note))
def status_counts(con, run_id: str, mode: str = "oracle") -> list[dict]:
return [dict(r) for r in con.execute(
"SELECT state, task, march, COUNT(*) n FROM jobs WHERE run_id=? AND mode=? GROUP BY state, task, march ORDER BY state, task, march",
(run_id, mode))]
def error_class_counts(con, run_id: str) -> list[dict]:
return [dict(r) for r in con.execute(
"SELECT COALESCE(error_class,'(none)') error_class, stage_failed, COUNT(*) n FROM jobs WHERE run_id=? AND state='FAIL'"
" GROUP BY error_class, stage_failed ORDER BY n DESC", (run_id,))]