138 lines
5.2 KiB
Python
138 lines
5.2 KiB
Python
#!/usr/bin/env python3
|
|
"""serve.py — jev r41 sidecar 推理服务 (Laya DecisionModel, mmBERT-base)
|
|
|
|
打分协议 (与训练/评测严格一致):
|
|
1. 文档按 win_tokens=800 切窗 (≤4 窗, 开头起)
|
|
2. 每窗 build_sequence(max_len=1024, head_len=160) → marker logit
|
|
3. p_win = sigmoid(z / T_WIN) # per-window 温度 (ID 校准)
|
|
4. p_doc_mean = mean(p_win) # r41 证明 mean 优于 max
|
|
5. p_final = sigmoid(logit(p_doc_mean) / T_AGG) # 聚合后二次校准
|
|
|
|
POST /score {query, doc_text} → {p_win[], p_mean, p_max, p_final}
|
|
POST /score_batch [{query, doc_text}...] → [...]
|
|
GET /health → {status, windows_protocol, T_WIN, T_AGG}
|
|
"""
|
|
import json
|
|
import math
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, "/srv/laya") # vendored laya.common (build_sequence/DecisionModel)
|
|
|
|
import torch
|
|
from fastapi import FastAPI
|
|
from pydantic import BaseModel
|
|
from transformers import AutoConfig, AutoModel, AutoTokenizer
|
|
from peft import LoraConfig, get_peft_model
|
|
from laya.common import DecisionModel, build_sequence
|
|
|
|
MODEL_DIR = os.environ.get("MODEL_DIR", "/model")
|
|
T_WIN = float(os.environ.get("T_WIN", "1.35"))
|
|
T_AGG = float(os.environ.get("T_AGG", "1.3"))
|
|
MAX_LEN = int(os.environ.get("MAX_LEN", "1024"))
|
|
HEAD_LEN = int(os.environ.get("HEAD_LEN", "160"))
|
|
WIN_TOKENS = int(os.environ.get("WIN_TOKENS", "800"))
|
|
MAX_WIN = int(os.environ.get("MAX_WIN", "4"))
|
|
|
|
INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content."
|
|
Q = {"t": "noul", "ins": INSTR, "crit": None}
|
|
|
|
app = FastAPI(title="jev-r41-sidecar")
|
|
state = {}
|
|
|
|
def load_model():
|
|
tok = AutoTokenizer.from_pretrained(os.path.join(MODEL_DIR, "mmbert"))
|
|
cfg = AutoConfig.from_pretrained(os.path.join(MODEL_DIR, "mmbert"))
|
|
enc = AutoModel.from_config(cfg)
|
|
sd = torch.load(os.path.join(MODEL_DIR, "mmbert", "pytorch_model.bin"),
|
|
map_location="cpu", weights_only=False)
|
|
sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()}
|
|
enc.load_state_dict(sd2, strict=False)
|
|
dm = DecisionModel(enc, head_layers=2, n_act=2)
|
|
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
|
|
target_modules=["Wqkv", "Wo"], task_type=None)
|
|
dm.encoder = get_peft_model(dm.encoder, lora)
|
|
ad = torch.load(os.path.join(MODEL_DIR, "adapter.pt"), map_location="cpu", weights_only=False)
|
|
missing, unexpected = dm.load_state_dict(ad, strict=False)
|
|
assert len(missing) == 134 and not unexpected, f"adapter mismatch missing={len(missing)} unexpected={len(unexpected)}"
|
|
dm.eval()
|
|
state["tok"] = tok
|
|
state["dm"] = dm
|
|
state["ready"] = True
|
|
|
|
def doc_windows(text):
|
|
ids = state["tok"](text[:12000], add_special_tokens=False)["input_ids"]
|
|
if len(ids) <= WIN_TOKENS:
|
|
return [text]
|
|
wins = []
|
|
for s in range(0, min(len(ids), WIN_TOKENS * MAX_WIN), WIN_TOKENS):
|
|
wins.append(state["tok"].decode(ids[s:s + WIN_TOKENS]))
|
|
if len(wins) >= MAX_WIN:
|
|
break
|
|
return wins or [text[:6000]]
|
|
|
|
@torch.no_grad()
|
|
def score_windows(query, wins):
|
|
tok = state["tok"]
|
|
dm = state["dm"]
|
|
ids_l, mm_l, mp_l, lens = [], [], [], []
|
|
for w in wins:
|
|
st = f"用户问题: {query}\n\n文档内容:\n{w}"
|
|
ids, markers = build_sequence(tok, st, Q, MAX_LEN, HEAD_LEN)
|
|
lens.append(len(ids))
|
|
ids_l.append(ids + [0] * (MAX_LEN - len(ids)))
|
|
mm_l.append([True] * len(markers) + [False] * (8 - len(markers)))
|
|
mp_l.append((markers + [0] * (8 - len(markers)))[:8])
|
|
b = {
|
|
"input_ids": torch.tensor(ids_l),
|
|
"attention_mask": torch.tensor(
|
|
[[1] * n + [0] * (MAX_LEN - n) for n in lens]),
|
|
"marker_pos": torch.tensor(mp_l),
|
|
"marker_mask": torch.tensor(mm_l).bool(),
|
|
"qtype": torch.tensor([2] * len(ids_l)),
|
|
}
|
|
logits, _ = dm(**b)
|
|
z = logits[..., 1].float().tolist()
|
|
p_win = [1 / (1 + math.exp(-v / T_WIN)) for v in z]
|
|
return p_win
|
|
|
|
def score_one(query, doc_text):
|
|
wins = doc_windows(doc_text)
|
|
p_win = score_windows(query, wins)
|
|
p_mean = sum(p_win) / len(p_win)
|
|
p_max = max(p_win)
|
|
lg_mean = math.log(max(p_mean, 1e-6) / max(1 - p_mean, 1e-6))
|
|
p_final = 1 / (1 + math.exp(-lg_mean / T_AGG))
|
|
return {"p_win": [round(p, 5) for p in p_win],
|
|
"p_mean": round(p_mean, 5),
|
|
"p_max": round(p_max, 5),
|
|
"p_final": round(p_final, 5),
|
|
"n_win": len(wins)}
|
|
|
|
class ScoreReq(BaseModel):
|
|
query: str
|
|
doc_text: str
|
|
|
|
class BatchReq(BaseModel):
|
|
items: list
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {"status": "ok" if state.get("ready") else "loading",
|
|
"protocol": {"max_len": MAX_LEN, "head_len": HEAD_LEN,
|
|
"win_tokens": WIN_TOKENS, "max_win": MAX_WIN},
|
|
"T_WIN": T_WIN, "T_AGG": T_AGG}
|
|
|
|
@app.post("/score")
|
|
def score(req: ScoreReq):
|
|
return score_one(req.query, req.doc_text)
|
|
|
|
@app.post("/score_batch")
|
|
def score_batch(req: BatchReq):
|
|
return [score_one(it["query"], it["doc_text"]) for it in req.items]
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
load_model()
|
|
uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8000")), log_level="warning")
|