serve.py: r41 sidecar FastAPI (win800/mean/T_WIN1.35/T_AGG1.3, smoke-tested)
This commit is contained in:
parent
9b1a8286bf
commit
e12fed76c9
137
serve.py
Normal file
137
serve.py
Normal file
@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""serve.py — jev r41 sidecar 推理服务 (Laya DecisionModel, mmBERT-base)
|
||||
|
||||
打分协议 (与训练/评测严格一致):
|
||||
1. 文档按 win_tokens=800 切窗 (≤4 窗, 开头起)
|
||||
2. 每窗 build_sequence(max_len=1024, head_len=160) → marker logit
|
||||
3. p_win = sigmoid(z / T_WIN) # per-window 温度 (ID 校准)
|
||||
4. p_doc_mean = mean(p_win) # r41 证明 mean 优于 max
|
||||
5. p_final = sigmoid(logit(p_doc_mean) / T_AGG) # 聚合后二次校准
|
||||
|
||||
POST /score {query, doc_text} → {p_win[], p_mean, p_max, p_final}
|
||||
POST /score_batch [{query, doc_text}...] → [...]
|
||||
GET /health → {status, windows_protocol, T_WIN, T_AGG}
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/srv/laya") # vendored laya.common (build_sequence/DecisionModel)
|
||||
|
||||
import torch
|
||||
from fastapi import FastAPI
|
||||
from pydantic import BaseModel
|
||||
from transformers import AutoConfig, AutoModel, AutoTokenizer
|
||||
from peft import LoraConfig, get_peft_model
|
||||
from laya.common import DecisionModel, build_sequence
|
||||
|
||||
MODEL_DIR = os.environ.get("MODEL_DIR", "/model")
|
||||
T_WIN = float(os.environ.get("T_WIN", "1.35"))
|
||||
T_AGG = float(os.environ.get("T_AGG", "1.3"))
|
||||
MAX_LEN = int(os.environ.get("MAX_LEN", "1024"))
|
||||
HEAD_LEN = int(os.environ.get("HEAD_LEN", "160"))
|
||||
WIN_TOKENS = int(os.environ.get("WIN_TOKENS", "800"))
|
||||
MAX_WIN = int(os.environ.get("MAX_WIN", "4"))
|
||||
|
||||
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
|
||||
Q = {"t": "noul", "ins": INSTR, "crit": None}
|
||||
|
||||
app = FastAPI(title="jev-r41-sidecar")
|
||||
state = {}
|
||||
|
||||
def load_model():
|
||||
tok = AutoTokenizer.from_pretrained(os.path.join(MODEL_DIR, "mmbert"))
|
||||
cfg = AutoConfig.from_pretrained(os.path.join(MODEL_DIR, "mmbert"))
|
||||
enc = AutoModel.from_config(cfg)
|
||||
sd = torch.load(os.path.join(MODEL_DIR, "mmbert", "pytorch_model.bin"),
|
||||
map_location="cpu", weights_only=False)
|
||||
sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()}
|
||||
enc.load_state_dict(sd2, strict=False)
|
||||
dm = DecisionModel(enc, head_layers=2, n_act=2)
|
||||
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
|
||||
target_modules=["Wqkv", "Wo"], task_type=None)
|
||||
dm.encoder = get_peft_model(dm.encoder, lora)
|
||||
ad = torch.load(os.path.join(MODEL_DIR, "adapter.pt"), map_location="cpu", weights_only=False)
|
||||
missing, unexpected = dm.load_state_dict(ad, strict=False)
|
||||
assert len(missing) == 134 and not unexpected, f"adapter mismatch missing={len(missing)} unexpected={len(unexpected)}"
|
||||
dm.eval()
|
||||
state["tok"] = tok
|
||||
state["dm"] = dm
|
||||
state["ready"] = True
|
||||
|
||||
def doc_windows(text):
|
||||
ids = state["tok"](text[:12000], add_special_tokens=False)["input_ids"]
|
||||
if len(ids) <= WIN_TOKENS:
|
||||
return [text]
|
||||
wins = []
|
||||
for s in range(0, min(len(ids), WIN_TOKENS * MAX_WIN), WIN_TOKENS):
|
||||
wins.append(state["tok"].decode(ids[s:s + WIN_TOKENS]))
|
||||
if len(wins) >= MAX_WIN:
|
||||
break
|
||||
return wins or [text[:6000]]
|
||||
|
||||
@torch.no_grad()
|
||||
def score_windows(query, wins):
|
||||
tok = state["tok"]
|
||||
dm = state["dm"]
|
||||
ids_l, mm_l, mp_l, lens = [], [], [], []
|
||||
for w in wins:
|
||||
st = f"用户问题: {query}\n\n文档内容:\n{w}"
|
||||
ids, markers = build_sequence(tok, st, Q, MAX_LEN, HEAD_LEN)
|
||||
lens.append(len(ids))
|
||||
ids_l.append(ids + [0] * (MAX_LEN - len(ids)))
|
||||
mm_l.append([True] * len(markers) + [False] * (8 - len(markers)))
|
||||
mp_l.append((markers + [0] * (8 - len(markers)))[:8])
|
||||
b = {
|
||||
"input_ids": torch.tensor(ids_l),
|
||||
"attention_mask": torch.tensor(
|
||||
[[1] * n + [0] * (MAX_LEN - n) for n in lens]),
|
||||
"marker_pos": torch.tensor(mp_l),
|
||||
"marker_mask": torch.tensor(mm_l).bool(),
|
||||
"qtype": torch.tensor([2] * len(ids_l)),
|
||||
}
|
||||
logits, _ = dm(**b)
|
||||
z = logits[..., 1].float().tolist()
|
||||
p_win = [1 / (1 + math.exp(-v / T_WIN)) for v in z]
|
||||
return p_win
|
||||
|
||||
def score_one(query, doc_text):
|
||||
wins = doc_windows(doc_text)
|
||||
p_win = score_windows(query, wins)
|
||||
p_mean = sum(p_win) / len(p_win)
|
||||
p_max = max(p_win)
|
||||
lg_mean = math.log(max(p_mean, 1e-6) / max(1 - p_mean, 1e-6))
|
||||
p_final = 1 / (1 + math.exp(-lg_mean / T_AGG))
|
||||
return {"p_win": [round(p, 5) for p in p_win],
|
||||
"p_mean": round(p_mean, 5),
|
||||
"p_max": round(p_max, 5),
|
||||
"p_final": round(p_final, 5),
|
||||
"n_win": len(wins)}
|
||||
|
||||
class ScoreReq(BaseModel):
|
||||
query: str
|
||||
doc_text: str
|
||||
|
||||
class BatchReq(BaseModel):
|
||||
items: list
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok" if state.get("ready") else "loading",
|
||||
"protocol": {"max_len": MAX_LEN, "head_len": HEAD_LEN,
|
||||
"win_tokens": WIN_TOKENS, "max_win": MAX_WIN},
|
||||
"T_WIN": T_WIN, "T_AGG": T_AGG}
|
||||
|
||||
@app.post("/score")
|
||||
def score(req: ScoreReq):
|
||||
return score_one(req.query, req.doc_text)
|
||||
|
||||
@app.post("/score_batch")
|
||||
def score_batch(req: BatchReq):
|
||||
return [score_one(it["query"], it["doc_text"]) for it in req.items]
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
load_model()
|
||||
uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8000")), log_level="warning")
|
||||
Loading…
x
Reference in New Issue
Block a user