343 lines
16 KiB
Python
343 lines
16 KiB
Python
"""state/jobs.sqlite — 状态、error_class、耗时历史的唯一真相。文件系统只存内容。"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import socket
|
||
import time
|
||
from contextlib import contextmanager
|
||
from pathlib import Path
|
||
|
||
from .paths import DB_PATH, STATE
|
||
|
||
SCHEMA = """
|
||
CREATE TABLE IF NOT EXISTS runs (
|
||
run_id TEXT PRIMARY KEY,
|
||
created_at REAL NOT NULL,
|
||
manifest TEXT NOT NULL, -- json:venv pip-freeze sha、板固件、registry 指纹、数据 manifest sha
|
||
note TEXT
|
||
);
|
||
CREATE TABLE IF NOT EXISTS jobs (
|
||
run_id TEXT NOT NULL,
|
||
job_id TEXT NOT NULL,
|
||
task TEXT, model TEXT, march TEXT, backend TEXT, qcfg TEXT, priority TEXT,
|
||
mode TEXT NOT NULL, -- oracle | e2e | repair
|
||
policy TEXT, -- teacher | student | NULL
|
||
k INTEGER DEFAULT 1,
|
||
state TEXT NOT NULL, -- PENDING ORACLE_RUNNING PREPARED PASS FAIL UNVERIFIED
|
||
error_class TEXT,
|
||
stage_failed TEXT,
|
||
estimate_s REAL,
|
||
started_at REAL, finished_at REAL,
|
||
result TEXT, -- oracle/result.json 副本
|
||
note TEXT,
|
||
PRIMARY KEY (run_id, job_id, mode, policy)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS stages (
|
||
run_id TEXT NOT NULL, job_id TEXT NOT NULL,
|
||
rollout INTEGER NOT NULL DEFAULT -1, -- -1 = oracle
|
||
stage TEXT NOT NULL,
|
||
attempt INTEGER NOT NULL DEFAULT 0,
|
||
lane TEXT DEFAULT 'normal', -- normal | long
|
||
state TEXT NOT NULL, -- RUNNING OK FAIL STALLED SKIPPED
|
||
exit_code INTEGER,
|
||
error_class TEXT,
|
||
started_at REAL, finished_at REAL, duration_s REAL,
|
||
PRIMARY KEY (run_id, job_id, rollout, stage, attempt)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS rollouts (
|
||
run_id TEXT NOT NULL, job_id TEXT NOT NULL, kk INTEGER NOT NULL,
|
||
mode TEXT, policy TEXT,
|
||
state TEXT NOT NULL, -- QUEUED RUNNING VERIFYING PASS FAIL BLOCKED TIMEOUT
|
||
claim_terminal TEXT,
|
||
verdict_terminal TEXT,
|
||
claim_consistent INTEGER,
|
||
review INTEGER DEFAULT 0, -- oracle FAIL / rollout PASS → 1,进人工抽查
|
||
turns INTEGER, tokens INTEGER, wall_s REAL,
|
||
verdict TEXT,
|
||
started_at REAL, finished_at REAL,
|
||
PRIMARY KEY (run_id, job_id, kk)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS rollout_context (
|
||
run_id TEXT NOT NULL, job_id TEXT NOT NULL, kk INTEGER NOT NULL,
|
||
runner TEXT NOT NULL DEFAULT 'pi',
|
||
owner_host TEXT, owner_pid INTEGER,
|
||
PRIMARY KEY (run_id, job_id, kk)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS annotations (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
run_id TEXT, job_id TEXT, ts REAL, note TEXT
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(run_id, state);
|
||
CREATE INDEX IF NOT EXISTS idx_stages_hist ON stages(job_id, stage, state);
|
||
"""
|
||
|
||
|
||
def _open(path: Path, *, timeout_s: float = 60) -> sqlite3.Connection:
|
||
con = sqlite3.connect(path, timeout=timeout_s)
|
||
con.row_factory = sqlite3.Row
|
||
return con
|
||
|
||
|
||
def is_busy(exc: sqlite3.OperationalError) -> bool:
|
||
# Python 3.10 does not expose sqlite_errorcode or SQLITE_BUSY constants.
|
||
code = getattr(exc, 'sqlite_errorcode', None)
|
||
if code is not None:
|
||
return (code & 255) in (5, 6)
|
||
return str(exc).lower().split(':', 1)[0] in (
|
||
'database is locked', 'database table is locked', 'database schema is locked', 'database is busy')
|
||
|
||
|
||
@contextmanager
|
||
def connect(path: Path = DB_PATH, *, timeout_s: float = 60):
|
||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||
con = _open(path, timeout_s=timeout_s)
|
||
# 默认 TRUNCATE:WAL 在 FUSE(JuiceFS 等)上会 disk I/O error(-shm 需要 mmap)。
|
||
# state/ 放本地盘(BENCH_STATE_DIR=/home/<user>/bench-state)时可设 BENCH_SQLITE_JOURNAL=WAL。
|
||
try:
|
||
jm = os.environ.get("BENCH_SQLITE_JOURNAL", "TRUNCATE")
|
||
con.execute(f"PRAGMA journal_mode={jm}")
|
||
con.executescript(SCHEMA)
|
||
yield con
|
||
con.commit()
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
@contextmanager
|
||
def readonly(path: Path = DB_PATH):
|
||
con = sqlite3.connect(f'file:{path}?mode=ro', uri=True, timeout=60)
|
||
con.row_factory = sqlite3.Row
|
||
try:
|
||
yield con
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
# ---------------------------------------------------------------- runs
|
||
|
||
def create_run(con, run_id: str, manifest: dict, note: str = "") -> None:
|
||
if con.execute("SELECT 1 FROM runs WHERE run_id=?", (run_id,)).fetchone():
|
||
raise RuntimeError(f"run_id 已存在(append-only,不允许覆盖): {run_id}")
|
||
con.execute("INSERT INTO runs VALUES (?,?,?,?)",
|
||
(run_id, time.time(), json.dumps(manifest, ensure_ascii=False), note))
|
||
|
||
|
||
def get_run(con, run_id: str) -> dict | None:
|
||
r = con.execute("SELECT * FROM runs WHERE run_id=?", (run_id,)).fetchone()
|
||
return dict(r) if r else None
|
||
|
||
|
||
def assert_run_open(con, run_id: str) -> None:
|
||
run = get_run(con, run_id)
|
||
target = json.loads(run["manifest"]).get("continued_to") if run else None
|
||
if target:
|
||
raise ValueError(f"run {run_id} handed off its remaining work to {target}; execution is closed")
|
||
|
||
|
||
def latest_run_id(con) -> str | None:
|
||
r = con.execute("SELECT run_id FROM runs ORDER BY created_at DESC LIMIT 1").fetchone()
|
||
return r[0] if r else None
|
||
|
||
|
||
def next_run_id(con, date_str: str) -> str:
|
||
rows = con.execute("SELECT run_id FROM runs WHERE run_id LIKE ?", (f"r{date_str}-%",)).fetchall()
|
||
seq = 1 + max([int(r[0].rsplit("-", 1)[1]) for r in rows if r[0].rsplit("-", 1)[1].isdigit()] or [0])
|
||
return f"r{date_str}-{seq:02d}"
|
||
|
||
|
||
# ---------------------------------------------------------------- jobs
|
||
|
||
def upsert_job(con, run_id: str, spec_row: dict, estimate_s: float | None) -> bool:
|
||
"""展开幂等:已存在的 (run_id, job_id, mode, policy) 不动。返回是否新插入。"""
|
||
key = (run_id, spec_row["job_id"], spec_row["mode"], spec_row.get("policy"))
|
||
if con.execute("SELECT 1 FROM jobs WHERE run_id=? AND job_id=? AND mode=? AND policy IS ?", key).fetchone():
|
||
return False
|
||
con.execute(
|
||
"INSERT INTO jobs (run_id, job_id, task, model, march, backend, qcfg, priority, mode, policy, k, state, estimate_s)"
|
||
" VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||
(run_id, spec_row["job_id"], spec_row["task"], spec_row["model"], spec_row["march"],
|
||
spec_row["backend"], spec_row["qcfg"], spec_row["priority"], spec_row["mode"],
|
||
spec_row.get("policy"), spec_row.get("k", 1), "PENDING", estimate_s))
|
||
return True
|
||
|
||
|
||
def claim_job(con, run_id: str, job_id: str, mode: str = "oracle", force: bool = False,
|
||
prepare_only: bool = False) -> bool:
|
||
"""原子认领:准备只取 PENDING;完整 Oracle 也可取 PREPARED,同一 job 只有一个进程赢。
|
||
force=True 只允许完整 Oracle 重跑已结束的 job(显式 --oracle --job,原地覆盖 result.json)。"""
|
||
con.execute("BEGIN IMMEDIATE")
|
||
assert_run_open(con, run_id)
|
||
allowed = "('PENDING','PREPARED')" if not force else "('PENDING','PREPARED','PASS','FAIL','UNVERIFIED')"
|
||
if prepare_only:
|
||
allowed = "('PENDING')" # Preparation never overwrites a finished job, even with --job.
|
||
cur = con.execute(f"UPDATE jobs SET state='ORACLE_RUNNING', started_at=? WHERE run_id=? AND job_id=? AND mode=? AND state IN {allowed}",
|
||
(time.time(), run_id, job_id, mode))
|
||
con.commit()
|
||
return cur.rowcount == 1
|
||
|
||
|
||
def set_job_state(con, run_id: str, job_id: str, state: str, *, mode: str = "oracle",
|
||
error_class: str | None = None, stage_failed: str | None = None,
|
||
result: dict | None = None) -> None:
|
||
now = time.time()
|
||
if state == "ORACLE_RUNNING":
|
||
con.execute("UPDATE jobs SET state=?, started_at=? WHERE run_id=? AND job_id=? AND mode=?",
|
||
(state, now, run_id, job_id, mode))
|
||
else:
|
||
con.execute("UPDATE jobs SET state=?, finished_at=?, error_class=?, stage_failed=?, result=?"
|
||
" WHERE run_id=? AND job_id=? AND mode=?",
|
||
(state, now, error_class, stage_failed,
|
||
json.dumps(result, ensure_ascii=False) if result else None, run_id, job_id, mode))
|
||
|
||
|
||
def get_job(con, run_id: str, job_id: str, mode: str = "oracle") -> dict | None:
|
||
r = con.execute("SELECT * FROM jobs WHERE run_id=? AND job_id=? AND mode=?", (run_id, job_id, mode)).fetchone()
|
||
return dict(r) if r else None
|
||
|
||
|
||
def list_jobs(con, run_id: str, mode: str | None = None, state: str | None = None) -> list[dict]:
|
||
q, args = "SELECT * FROM jobs WHERE run_id=?", [run_id]
|
||
if mode:
|
||
q += " AND mode=?"; args.append(mode)
|
||
if state:
|
||
q += " AND state=?"; args.append(state)
|
||
q += " ORDER BY CASE priority WHEN 'A' THEN 0 WHEN 'B' THEN 1 ELSE 2 END, estimate_s, job_id"
|
||
return [dict(r) for r in con.execute(q, args)]
|
||
|
||
|
||
# ---------------------------------------------------------------- stages
|
||
|
||
def stage_start(con, run_id, job_id, stage, attempt=0, rollout=-1, lane="normal") -> None:
|
||
con.execute("INSERT OR REPLACE INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, started_at)"
|
||
" VALUES (?,?,?,?,?,?,'RUNNING',?)", (run_id, job_id, rollout, stage, attempt, lane, time.time()))
|
||
|
||
|
||
def stage_finish(con, run_id, job_id, stage, attempt, state, exit_code=None, error_class=None, rollout=-1) -> None:
|
||
now = time.time()
|
||
r = con.execute("SELECT started_at FROM stages WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?",
|
||
(run_id, job_id, rollout, stage, attempt)).fetchone()
|
||
started = r[0] if r and r[0] else now
|
||
con.execute("UPDATE stages SET state=?, exit_code=?, error_class=?, finished_at=?, duration_s=?"
|
||
" WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?",
|
||
(state, exit_code, error_class, now, now - started, run_id, job_id, rollout, stage, attempt))
|
||
|
||
|
||
def stage_history(con, job_id: str, stage: str) -> list[float]:
|
||
"""同 job_id(跨 run)同阶段的历史成功耗时,用于估算。"""
|
||
return [r[0] for r in con.execute(
|
||
"SELECT duration_s FROM stages WHERE job_id=? AND stage=? AND state='OK' AND rollout=-1 AND duration_s IS NOT NULL",
|
||
(job_id, stage))]
|
||
|
||
|
||
def stage_history_by_model(con, model: str, stage: str) -> list[float]:
|
||
return [r[0] for r in con.execute(
|
||
"SELECT s.duration_s FROM stages s JOIN jobs j ON s.run_id=j.run_id AND s.job_id=j.job_id"
|
||
" WHERE j.model=? AND s.stage=? AND s.state='OK' AND s.rollout=-1 AND s.duration_s IS NOT NULL",
|
||
(model, stage))]
|
||
|
||
|
||
# ---------------------------------------------------------------- rollouts
|
||
|
||
def rollout_runner(con, run_id, job_id, kk) -> str:
|
||
# A read-only old database may not yet contain the additive metadata table.
|
||
if not con.execute("SELECT 1 FROM sqlite_master WHERE name='rollout_context'").fetchone():
|
||
return "pi"
|
||
row = con.execute("SELECT runner FROM rollout_context WHERE run_id=? AND job_id=? AND kk=?",
|
||
(run_id, job_id, kk)).fetchone()
|
||
return row[0] if row else "pi"
|
||
|
||
|
||
def rollout_upsert(con, run_id, job_id, kk, mode, policy, state="QUEUED", *, runner="pi") -> None:
|
||
existing = con.execute("SELECT mode,policy FROM rollouts WHERE run_id=? AND job_id=? AND kk=?",
|
||
(run_id, job_id, kk)).fetchone()
|
||
if existing and (existing[0], existing[1], rollout_runner(con, run_id, job_id, kk)) != (mode, policy, runner):
|
||
raise ValueError(f"rollout identity conflict: {run_id}/{job_id}/{kk}")
|
||
inserted = con.execute("INSERT OR IGNORE INTO rollouts (run_id, job_id, kk, mode, policy, state) VALUES (?,?,?,?,?,?)",
|
||
(run_id, job_id, kk, mode, policy, state)).rowcount
|
||
if not inserted:
|
||
actual = con.execute("SELECT mode,policy FROM rollouts WHERE run_id=? AND job_id=? AND kk=?",
|
||
(run_id, job_id, kk)).fetchone()
|
||
if (actual[0], actual[1], rollout_runner(con, run_id, job_id, kk)) != (mode, policy, runner):
|
||
raise ValueError(f"rollout identity conflict: {run_id}/{job_id}/{kk}")
|
||
con.execute("INSERT OR IGNORE INTO rollout_context(run_id,job_id,kk,runner) VALUES (?,?,?,?)",
|
||
(run_id, job_id, kk, runner))
|
||
|
||
|
||
def next_rollout_kk(con, run_id, job_id) -> int:
|
||
return con.execute("SELECT COALESCE(MAX(kk),-1)+1 FROM rollouts WHERE run_id=? AND job_id=?",
|
||
(run_id, job_id)).fetchone()[0]
|
||
|
||
|
||
def ensure_rollout_plan(con, run_id, job_id, mode, policy, k, *, runner="pi") -> int:
|
||
"""Caller holds BEGIN IMMEDIATE; all modes/policies/runners share kk space."""
|
||
if not con.in_transaction:
|
||
raise RuntimeError("rollout allocation requires a transaction")
|
||
if not isinstance(k, int) or k < 0:
|
||
raise ValueError("rollout k must be a nonnegative integer")
|
||
rows = list_rollouts(con, run_id, job_id)
|
||
count = sum((r["mode"], r["policy"], r["runner"]) == (mode, policy, runner) for r in rows)
|
||
added = max(0, k - count)
|
||
kk = next_rollout_kk(con, run_id, job_id)
|
||
for i in range(added):
|
||
rollout_upsert(con, run_id, job_id, kk + i, mode, policy, runner=runner)
|
||
return added
|
||
|
||
|
||
def claim_rollout(con, run_id: str, job_id: str, kk: int, max_active: int | None = None) -> bool:
|
||
"""原子认领一条 QUEUED;同一 job 只允许一条运行,跨 collector 也不重复。"""
|
||
con.execute("BEGIN IMMEDIATE")
|
||
assert_run_open(con, run_id)
|
||
if max_active is not None:
|
||
active = con.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0]
|
||
if active >= max_active:
|
||
con.commit()
|
||
return False
|
||
cur = con.execute(
|
||
"UPDATE rollouts SET state='RUNNING', started_at=?"
|
||
" WHERE run_id=? AND job_id=? AND kk=? AND state='QUEUED'"
|
||
" AND NOT EXISTS (SELECT 1 FROM rollouts active WHERE active.run_id=? AND active.job_id=?"
|
||
" AND active.state IN ('RUNNING','VERIFYING'))",
|
||
(time.time(), run_id, job_id, kk, run_id, job_id))
|
||
if cur.rowcount:
|
||
con.execute("INSERT OR IGNORE INTO rollout_context(run_id,job_id,kk,runner) VALUES (?,?,?,'pi')",
|
||
(run_id, job_id, kk))
|
||
con.execute("UPDATE rollout_context SET owner_host=?,owner_pid=? WHERE run_id=? AND job_id=? AND kk=?",
|
||
(socket.gethostname(), os.getpid(), run_id, job_id, kk))
|
||
con.commit()
|
||
return cur.rowcount == 1
|
||
|
||
|
||
def rollout_set(con, run_id, job_id, kk, **fields) -> None:
|
||
if not fields:
|
||
return
|
||
cols = ", ".join(f"{k}=?" for k in fields)
|
||
con.execute(f"UPDATE rollouts SET {cols} WHERE run_id=? AND job_id=? AND kk=?",
|
||
(*fields.values(), run_id, job_id, kk))
|
||
|
||
|
||
def list_rollouts(con, run_id, job_id) -> list[dict]:
|
||
rows = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=? AND job_id=? ORDER BY kk", (run_id, job_id))]
|
||
for row in rows:
|
||
row["runner"] = rollout_runner(con, run_id, job_id, row["kk"])
|
||
return rows
|
||
|
||
|
||
# ---------------------------------------------------------------- annotations / status
|
||
|
||
def annotate(con, run_id, job_id, note) -> None:
|
||
con.execute("INSERT INTO annotations (run_id, job_id, ts, note) VALUES (?,?,?,?)", (run_id, job_id, time.time(), note))
|
||
|
||
|
||
def status_counts(con, run_id: str, mode: str = "oracle") -> list[dict]:
|
||
return [dict(r) for r in con.execute(
|
||
"SELECT state, task, march, COUNT(*) n FROM jobs WHERE run_id=? AND mode=? GROUP BY state, task, march ORDER BY state, task, march",
|
||
(run_id, mode))]
|
||
|
||
|
||
def error_class_counts(con, run_id: str) -> list[dict]:
|
||
return [dict(r) for r in con.execute(
|
||
"SELECT COALESCE(error_class,'(none)') error_class, stage_failed, COUNT(*) n FROM jobs WHERE run_id=? AND state='FAIL'"
|
||
" GROUP BY error_class, stage_failed ORDER BY n DESC", (run_id,))]
|