138 lines
5.2 KiB
Python

#!/usr/bin/env python3
"""serve.py — jev r41 sidecar 推理服务 (Laya DecisionModel, mmBERT-base)
打分协议 (与训练/评测严格一致):
1. 文档按 win_tokens=800 切窗 (≤4 窗, 开头起)
2. 每窗 build_sequence(max_len=1024, head_len=160) → marker logit
3. p_win = sigmoid(z / T_WIN) # per-window 温度 (ID 校准)
4. p_doc_mean = mean(p_win) # r41 证明 mean 优于 max
5. p_final = sigmoid(logit(p_doc_mean) / T_AGG) # 聚合后二次校准
POST /score {query, doc_text} → {p_win[], p_mean, p_max, p_final}
POST /score_batch [{query, doc_text}...] → [...]
GET /health → {status, windows_protocol, T_WIN, T_AGG}
"""
import json
import math
import os
import sys
sys.path.insert(0, "/srv/laya") # vendored laya.common (build_sequence/DecisionModel)
import torch
from fastapi import FastAPI
from pydantic import BaseModel
from transformers import AutoConfig, AutoModel, AutoTokenizer
from peft import LoraConfig, get_peft_model
from laya.common import DecisionModel, build_sequence
MODEL_DIR = os.environ.get("MODEL_DIR", "/model")
T_WIN = float(os.environ.get("T_WIN", "1.35"))
T_AGG = float(os.environ.get("T_AGG", "1.3"))
MAX_LEN = int(os.environ.get("MAX_LEN", "1024"))
HEAD_LEN = int(os.environ.get("HEAD_LEN", "160"))
WIN_TOKENS = int(os.environ.get("WIN_TOKENS", "800"))
MAX_WIN = int(os.environ.get("MAX_WIN", "4"))
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
Q = {"t": "noul", "ins": INSTR, "crit": None}
app = FastAPI(title="jev-r41-sidecar")
state = {}
def load_model():
tok = AutoTokenizer.from_pretrained(os.path.join(MODEL_DIR, "mmbert"))
cfg = AutoConfig.from_pretrained(os.path.join(MODEL_DIR, "mmbert"))
enc = AutoModel.from_config(cfg)
sd = torch.load(os.path.join(MODEL_DIR, "mmbert", "pytorch_model.bin"),
map_location="cpu", weights_only=False)
sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()}
enc.load_state_dict(sd2, strict=False)
dm = DecisionModel(enc, head_layers=2, n_act=2)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
target_modules=["Wqkv", "Wo"], task_type=None)
dm.encoder = get_peft_model(dm.encoder, lora)
ad = torch.load(os.path.join(MODEL_DIR, "adapter.pt"), map_location="cpu", weights_only=False)
missing, unexpected = dm.load_state_dict(ad, strict=False)
assert len(missing) == 134 and not unexpected, f"adapter mismatch missing={len(missing)} unexpected={len(unexpected)}"
dm.eval()
state["tok"] = tok
state["dm"] = dm
state["ready"] = True
def doc_windows(text):
ids = state["tok"](text[:12000], add_special_tokens=False)["input_ids"]
if len(ids) <= WIN_TOKENS:
return [text]
wins = []
for s in range(0, min(len(ids), WIN_TOKENS * MAX_WIN), WIN_TOKENS):
wins.append(state["tok"].decode(ids[s:s + WIN_TOKENS]))
if len(wins) >= MAX_WIN:
break
return wins or [text[:6000]]
@torch.no_grad()
def score_windows(query, wins):
tok = state["tok"]
dm = state["dm"]
ids_l, mm_l, mp_l, lens = [], [], [], []
for w in wins:
st = f"用户问题: {query}\n\n文档内容:\n{w}"
ids, markers = build_sequence(tok, st, Q, MAX_LEN, HEAD_LEN)
lens.append(len(ids))
ids_l.append(ids + [0] * (MAX_LEN - len(ids)))
mm_l.append([True] * len(markers) + [False] * (8 - len(markers)))
mp_l.append((markers + [0] * (8 - len(markers)))[:8])
b = {
"input_ids": torch.tensor(ids_l),
"attention_mask": torch.tensor(
[[1] * n + [0] * (MAX_LEN - n) for n in lens]),
"marker_pos": torch.tensor(mp_l),
"marker_mask": torch.tensor(mm_l).bool(),
"qtype": torch.tensor([2] * len(ids_l)),
}
logits, _ = dm(**b)
z = logits[..., 1].float().tolist()
p_win = [1 / (1 + math.exp(-v / T_WIN)) for v in z]
return p_win
def score_one(query, doc_text):
wins = doc_windows(doc_text)
p_win = score_windows(query, wins)
p_mean = sum(p_win) / len(p_win)
p_max = max(p_win)
lg_mean = math.log(max(p_mean, 1e-6) / max(1 - p_mean, 1e-6))
p_final = 1 / (1 + math.exp(-lg_mean / T_AGG))
return {"p_win": [round(p, 5) for p in p_win],
"p_mean": round(p_mean, 5),
"p_max": round(p_max, 5),
"p_final": round(p_final, 5),
"n_win": len(wins)}
class ScoreReq(BaseModel):
query: str
doc_text: str
class BatchReq(BaseModel):
items: list
@app.get("/health")
def health():
return {"status": "ok" if state.get("ready") else "loading",
"protocol": {"max_len": MAX_LEN, "head_len": HEAD_LEN,
"win_tokens": WIN_TOKENS, "max_win": MAX_WIN},
"T_WIN": T_WIN, "T_AGG": T_AGG}
@app.post("/score")
def score(req: ScoreReq):
return score_one(req.query, req.doc_text)
@app.post("/score_batch")
def score_batch(req: BatchReq):
return [score_one(it["query"], it["doc_text"]) for it in req.items]
if __name__ == "__main__":
import uvicorn
load_model()
uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8000")), log_level="warning")