#!/usr/bin/env python3 """serve.py — jev r41 sidecar 推理服务 (Laya DecisionModel, mmBERT-base) 打分协议 (与训练/评测严格一致): 1. 文档按 win_tokens=800 切窗 (≤4 窗, 开头起) 2. 每窗 build_sequence(max_len=1024, head_len=160) → marker logit 3. p_win = sigmoid(z / T_WIN) # per-window 温度 (ID 校准) 4. p_doc_mean = mean(p_win) # r41 证明 mean 优于 max 5. p_final = sigmoid(logit(p_doc_mean) / T_AGG) # 聚合后二次校准 POST /score {query, doc_text} → {p_win[], p_mean, p_max, p_final} POST /score_batch [{query, doc_text}...] → [...] GET /health → {status, windows_protocol, T_WIN, T_AGG} """ import json import math import os import sys sys.path.insert(0, "/srv/laya") # vendored laya.common (build_sequence/DecisionModel) import torch from fastapi import FastAPI from pydantic import BaseModel from transformers import AutoConfig, AutoModel, AutoTokenizer from peft import LoraConfig, get_peft_model from laya.common import DecisionModel, build_sequence MODEL_DIR = os.environ.get("MODEL_DIR", "/model") T_WIN = float(os.environ.get("T_WIN", "1.35")) T_AGG = float(os.environ.get("T_AGG", "1.3")) MAX_LEN = int(os.environ.get("MAX_LEN", "1024")) HEAD_LEN = int(os.environ.get("HEAD_LEN", "160")) WIN_TOKENS = int(os.environ.get("WIN_TOKENS", "800")) MAX_WIN = int(os.environ.get("MAX_WIN", "4")) INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content." Q = {"t": "noul", "ins": INSTR, "crit": None} app = FastAPI(title="jev-r41-sidecar") state = {} def load_model(): tok = AutoTokenizer.from_pretrained(os.path.join(MODEL_DIR, "mmbert")) cfg = AutoConfig.from_pretrained(os.path.join(MODEL_DIR, "mmbert")) enc = AutoModel.from_config(cfg) sd = torch.load(os.path.join(MODEL_DIR, "mmbert", "pytorch_model.bin"), map_location="cpu", weights_only=False) sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()} enc.load_state_dict(sd2, strict=False) dm = DecisionModel(enc, head_layers=2, n_act=2) lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, target_modules=["Wqkv", "Wo"], task_type=None) dm.encoder = get_peft_model(dm.encoder, lora) ad = torch.load(os.path.join(MODEL_DIR, "adapter.pt"), map_location="cpu", weights_only=False) missing, unexpected = dm.load_state_dict(ad, strict=False) assert len(missing) == 134 and not unexpected, f"adapter mismatch missing={len(missing)} unexpected={len(unexpected)}" dm.eval() state["tok"] = tok state["dm"] = dm state["ready"] = True def doc_windows(text): ids = state["tok"](text[:12000], add_special_tokens=False)["input_ids"] if len(ids) <= WIN_TOKENS: return [text] wins = [] for s in range(0, min(len(ids), WIN_TOKENS * MAX_WIN), WIN_TOKENS): wins.append(state["tok"].decode(ids[s:s + WIN_TOKENS])) if len(wins) >= MAX_WIN: break return wins or [text[:6000]] @torch.no_grad() def score_windows(query, wins): tok = state["tok"] dm = state["dm"] ids_l, mm_l, mp_l, lens = [], [], [], [] for w in wins: st = f"用户问题: {query}\n\n文档内容:\n{w}" ids, markers = build_sequence(tok, st, Q, MAX_LEN, HEAD_LEN) lens.append(len(ids)) ids_l.append(ids + [0] * (MAX_LEN - len(ids))) mm_l.append([True] * len(markers) + [False] * (8 - len(markers))) mp_l.append((markers + [0] * (8 - len(markers)))[:8]) b = { "input_ids": torch.tensor(ids_l), "attention_mask": torch.tensor( [[1] * n + [0] * (MAX_LEN - n) for n in lens]), "marker_pos": torch.tensor(mp_l), "marker_mask": torch.tensor(mm_l).bool(), "qtype": torch.tensor([2] * len(ids_l)), } logits, _ = dm(**b) z = logits[..., 1].float().tolist() p_win = [1 / (1 + math.exp(-v / T_WIN)) for v in z] return p_win def score_one(query, doc_text): wins = doc_windows(doc_text) p_win = score_windows(query, wins) p_mean = sum(p_win) / len(p_win) p_max = max(p_win) lg_mean = math.log(max(p_mean, 1e-6) / max(1 - p_mean, 1e-6)) p_final = 1 / (1 + math.exp(-lg_mean / T_AGG)) return {"p_win": [round(p, 5) for p in p_win], "p_mean": round(p_mean, 5), "p_max": round(p_max, 5), "p_final": round(p_final, 5), "n_win": len(wins)} class ScoreReq(BaseModel): query: str doc_text: str class BatchReq(BaseModel): items: list @app.get("/health") def health(): return {"status": "ok" if state.get("ready") else "loading", "protocol": {"max_len": MAX_LEN, "head_len": HEAD_LEN, "win_tokens": WIN_TOKENS, "max_win": MAX_WIN}, "T_WIN": T_WIN, "T_AGG": T_AGG} @app.post("/score") def score(req: ScoreReq): return score_one(req.query, req.doc_text) @app.post("/score_batch") def score_batch(req: BatchReq): return [score_one(it["query"], it["doc_text"]) for it in req.items] if __name__ == "__main__": import uvicorn load_model() uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8000")), log_level="warning")