2157 lines
121 KiB
Python
2157 lines
121 KiB
Python
"""High-level inference runtime for laya System 1 decision models."""
|
|
import copy
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import warnings
|
|
from contextlib import ExitStack, contextmanager, nullcontext
|
|
from contextvars import ContextVar
|
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from ._compile import compile_model, independent_dims, configure_cache, cuda_graph_step
|
|
from .calibrate import (
|
|
_install_temperatures,
|
|
apply_binning_map,
|
|
apply_calibration_payload,
|
|
MIN_BINNING_BUCKET_N,
|
|
fit_binning_map,
|
|
calibration_payload,
|
|
fit_temperature_map,
|
|
)
|
|
from .common import (
|
|
QTYPES,
|
|
TEMP_MAX,
|
|
TEMP_MIN,
|
|
amp_dtype,
|
|
build_model,
|
|
build_sequence,
|
|
collapsed_options,
|
|
clamp_temperature,
|
|
collate_items,
|
|
answer_confidence,
|
|
confidence_from_probs,
|
|
_resolve_noul_labels,
|
|
_reuse_question_tokens,
|
|
_disable_question_token_reuse,
|
|
encode_text,
|
|
render_criterion,
|
|
resolve_lang_temperatures,
|
|
serialize_state,
|
|
temp_bucket,
|
|
temperature_rejected,
|
|
unpermute_probs,
|
|
uses_parallel_layout,
|
|
window_batch_cap,
|
|
state_room,
|
|
window_budget,
|
|
)
|
|
from .confidence import apply_confidence_gate, check_min_confidence
|
|
from .hooks import (
|
|
HookRegistry, PredictContext, _as_sequence, aggregate_usage, compose_hooks, dispatch,
|
|
normalise_hooks, validate_timeout,
|
|
)
|
|
from .revisions import resolve_revision, snapshot_revision, verify_digests
|
|
|
|
|
|
def _fix_tokenizer_config(path: str):
|
|
"""Ensure tokenizer_config.json can be loaded across all transformers versions."""
|
|
cfg_file = os.path.join(path, "tokenizer", "tokenizer_config.json")
|
|
if not os.path.exists(cfg_file):
|
|
return
|
|
try:
|
|
with open(cfg_file) as f:
|
|
tcfg = json.load(f)
|
|
changed = False
|
|
if tcfg.get("tokenizer_class") in (None, "TokenizersBackend"):
|
|
tcfg["tokenizer_class"] = "PreTrainedTokenizerFast"
|
|
tcfg.pop("backend", None)
|
|
tcfg.pop("is_local", None)
|
|
changed = True
|
|
# Checkpoints built on the mmBERT/Gemma tokenizer store extra_special_tokens as a list;
|
|
# transformers expects a mapping and raises "'list' object has no attribute 'keys'",
|
|
# which makes AutoTokenizer -- and so the whole model -- fail to load.
|
|
extra = tcfg.get("extra_special_tokens")
|
|
if isinstance(extra, list):
|
|
tcfg["extra_special_tokens"] = {"extra_%d" % i: t for i, t in enumerate(extra)}
|
|
changed = True
|
|
if changed:
|
|
# HuggingFace snapshots are symlinks into a shared blob store, so writing through the
|
|
# link would truncate a file shared with other revisions and processes, race
|
|
# concurrent loads, and desync the hub's cache metadata. Write to a temporary file in
|
|
# the same directory and `os.replace` it into place: the snapshot entry becomes a
|
|
# regular file and is swapped atomically, so a concurrent load never sees a missing
|
|
# or half-written config.
|
|
cfg_dir = os.path.dirname(cfg_file)
|
|
fd, tmp_file = tempfile.mkstemp(dir=cfg_dir, prefix=".tokenizer_config.", suffix=".tmp")
|
|
try:
|
|
with os.fdopen(fd, "w") as f:
|
|
json.dump(tcfg, f, indent=2)
|
|
# mkstemp creates the file 0600; keep the mode the cache file had so a shared
|
|
# cache stays readable to the same users as before.
|
|
try:
|
|
os.chmod(tmp_file, os.stat(cfg_file).st_mode & 0o777)
|
|
except OSError:
|
|
pass
|
|
os.replace(tmp_file, cfg_file)
|
|
except BaseException:
|
|
try:
|
|
os.unlink(tmp_file)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
except Exception as e:
|
|
# Do not swallow this silently: if the patch did not apply, AutoTokenizer may fail later
|
|
# with a confusing error and no hint that the config was the cause.
|
|
warnings.warn(
|
|
"laya: could not patch %s (%s); the tokenizer may fail to load with this "
|
|
"transformers version." % (cfg_file, e),
|
|
RuntimeWarning, stacklevel=2)
|
|
|
|
|
|
def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str, torch.Tensor], model_id: str):
|
|
"""Verify that the loaded checkpoint weights and config strictly match the expected architecture."""
|
|
# 1. Verify required configuration attributes
|
|
required_cfg = ["encoder", "head_layers"]
|
|
missing_cfg = [k for k in required_cfg if k not in cfg]
|
|
if missing_cfg:
|
|
raise ValueError(
|
|
f"Incompatible model config for {model_id!r}: missing configuration keys {missing_cfg}. "
|
|
f"Ensure this is a valid RL Agent decision model."
|
|
)
|
|
|
|
# 2. Check for required component prefixes
|
|
required_prefixes = ("encoder.", "type_emb.", "scorer.", "act_head.")
|
|
for prefix in required_prefixes:
|
|
if not any(k.startswith(prefix) for k in weights.keys()):
|
|
raise ValueError(
|
|
f"Incompatible model weights for {model_id!r}: checkpoint is missing '{prefix}' parameters. "
|
|
f"Expected an RL Agent decision model with encoder and decision heads."
|
|
)
|
|
|
|
# 3. Check for parameter shape mismatches
|
|
model_sd = model.state_dict()
|
|
shape_mismatches = []
|
|
missing_keys = []
|
|
|
|
for name, param in model.named_parameters():
|
|
if name not in weights:
|
|
missing_keys.append(name)
|
|
elif tuple(weights[name].shape) != tuple(param.shape):
|
|
shape_mismatches.append(f" - {name}: expected {tuple(param.shape)}, found {tuple(weights[name].shape)}")
|
|
|
|
if shape_mismatches:
|
|
err_details = "\n".join(shape_mismatches[:5])
|
|
if len(shape_mismatches) > 5:
|
|
err_details += f"\n ... and {len(shape_mismatches) - 5} more mismatched layers."
|
|
raise ValueError(
|
|
f"Model architecture mismatch for {model_id!r}:\n{err_details}\n"
|
|
f"The checkpoint weights do not match the configured model architecture."
|
|
)
|
|
|
|
if missing_keys:
|
|
raise ValueError(
|
|
f"Model weights incomplete for {model_id!r}: missing {len(missing_keys)} parameter tensors "
|
|
f"(e.g. {missing_keys[:3]})."
|
|
)
|
|
|
|
|
|
_TOKENIZERS: Dict[tuple, Any] = {}
|
|
_TOKENIZERS_LOCK = threading.Lock()
|
|
|
|
class _InferenceGate:
|
|
"""Read-write synchronization gate protecting shared model device placement (#649).
|
|
|
|
Normal inference acquires the read lock so multiple threads can evaluate the model
|
|
concurrently without serialization. When a thread encounters GPU OOM and needs
|
|
to temporarily migrate the model to CPU for a scoped fallback, it acquires the write
|
|
lock. This ensures all in-flight requests finish safely before the model is moved,
|
|
and blocks new requests until the model and device state are restored.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._cond = threading.Condition(threading.Lock())
|
|
self._readers = 0
|
|
self._writers = 0
|
|
self._waiting_writers = 0
|
|
|
|
@contextmanager
|
|
def read_lock(self):
|
|
with self._cond:
|
|
while self._writers > 0 or self._waiting_writers > 0:
|
|
self._cond.wait()
|
|
self._readers += 1
|
|
try:
|
|
yield
|
|
finally:
|
|
with self._cond:
|
|
self._readers -= 1
|
|
if self._readers == 0:
|
|
self._cond.notify_all()
|
|
|
|
@contextmanager
|
|
def write_lock(self):
|
|
with self._cond:
|
|
self._waiting_writers += 1
|
|
try:
|
|
while self._readers > 0 or self._writers > 0:
|
|
self._cond.wait()
|
|
self._writers = 1
|
|
finally:
|
|
self._waiting_writers -= 1
|
|
try:
|
|
yield
|
|
finally:
|
|
with self._cond:
|
|
self._writers = 0
|
|
self._cond.notify_all()
|
|
|
|
|
|
# The per-inference CPU fallback rewrites shared runtime state (device, dtype, amp) and moves the
|
|
# model while other threads may be running their own forward, so the demotion and the restore are
|
|
# serialised. A second request that hits OOM waits here and re-demotes only if it needs to.
|
|
_OOM_FALLBACK_LOCK = threading.Lock()
|
|
_BATCH_AUTOCAST_CACHE = ContextVar("laya_batch_autocast_cache", default=False)
|
|
|
|
|
|
def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:
|
|
"""Tokenizer for a checkpoint, parsed once per process.
|
|
|
|
Parsing `tokenizer.json` is not free and `huggingface_hub` caches only the download, not
|
|
the parsed object: the multilingual checkpoint ships a 34 MB, 256k-vocabulary file that
|
|
costs seconds to load, several times the cost of applying its weights. A tokenizer is
|
|
read-only during inference, so one instance is shared by every Agent that wants the same
|
|
directory -- including an Agent the Router has rebuilt after eviction.
|
|
|
|
Keyed on the tokenizer directory and its mtime, so a re-download or a config rewritten by
|
|
`_fix_tokenizer_config()` still produces a fresh parse. Non-directory sources (a hub id)
|
|
are not cached, so a caller cannot pin a stale remote revision.
|
|
"""
|
|
from transformers import AutoTokenizer
|
|
|
|
if not os.path.isdir(tok_dir):
|
|
return AutoTokenizer.from_pretrained(cfg.get("encoder"))
|
|
|
|
try:
|
|
stamp = (os.path.abspath(tok_dir), os.path.getmtime(os.path.join(tok_dir, "tokenizer_config.json")))
|
|
except OSError:
|
|
return AutoTokenizer.from_pretrained(tok_dir)
|
|
|
|
with _TOKENIZERS_LOCK:
|
|
cached = _TOKENIZERS.get(stamp)
|
|
if cached is not None:
|
|
return cached
|
|
tokenizer = AutoTokenizer.from_pretrained(tok_dir)
|
|
_TOKENIZERS[stamp] = tokenizer
|
|
return tokenizer
|
|
|
|
|
|
def _amp_context(device, dtype, enabled: bool):
|
|
"""Autocast context for the forward pass, or a no-op when mixed precision is not in use.
|
|
|
|
Entering `torch.autocast` on a device torch has no autocast backend for raises even with
|
|
`enabled=False` ("User specified an unsupported autocast device_type mps"), which broke
|
|
every `predict()` on Apple Silicon on some torch builds. Only enter it when we use it;
|
|
the `enabled` flag is set per device by the load-time policy, which skips XPU on builds
|
|
without an XPU autocast backend.
|
|
"""
|
|
if not enabled:
|
|
return nullcontext()
|
|
return torch.autocast(device_type=device.type, dtype=dtype)
|
|
|
|
|
|
def _pad_cuda_compile_batch(batch: Dict, pad_token_id: int) -> Dict:
|
|
"""Keep compiled CUDA attention on the same sequence-length guard family.
|
|
|
|
Some Inductor SDPA versions specialise on the token length modulo eight even with
|
|
dynamic shapes. Pad only the masked tail; marker positions and caller-visible token
|
|
accounting continue to refer to the original sequence.
|
|
"""
|
|
missing = -batch["input_ids"].shape[1] % 8
|
|
if not missing:
|
|
return batch
|
|
padded = dict(batch)
|
|
padded["input_ids"] = torch.nn.functional.pad(batch["input_ids"], (0, missing), value=pad_token_id)
|
|
padded["attention_mask"] = torch.nn.functional.pad(batch["attention_mask"], (0, missing), value=0)
|
|
return padded
|
|
|
|
|
|
MPS_AMP_MIN_ROWS_DEFAULT = 5
|
|
# An unsupported autocast op fails the same way on every request. Retry that request in
|
|
# full precision, and only turn AMP off after this many failures in a row (#351).
|
|
_AMP_FAIL_LIMIT = 3
|
|
|
|
|
|
def _mps_amp_min_rows() -> int:
|
|
"""Rows at which MPS fp16 autocast starts to pay off. Override with LAYA_MPS_AMP_MIN_ROWS."""
|
|
raw = os.environ.get("LAYA_MPS_AMP_MIN_ROWS", "")
|
|
try:
|
|
return max(1, int(raw))
|
|
except (TypeError, ValueError):
|
|
return MPS_AMP_MIN_ROWS_DEFAULT
|
|
|
|
|
|
def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:
|
|
"""Autocast dtype on CUDA at compute capability >= 8: the checkpoint's `amp_dtype` (bf16 for the
|
|
shipped checkpoints), or LAYA_CUDA_AMP=fp16|float16|bf16|bfloat16 when set. fp16 stays 2-10x
|
|
closer to the fp32 forward than bf16 on every shipped checkpoint at the same speed; anything
|
|
else is ignored. The long spellings count because they are what `agent.dtype` reports back, so
|
|
the precision a user reads off one response is one they can ask for on the next."""
|
|
raw = os.environ.get("LAYA_CUDA_AMP", "").lower()
|
|
if raw in ("fp16", "float16"):
|
|
return torch.float16
|
|
if raw in ("bf16", "bfloat16"):
|
|
return torch.bfloat16
|
|
return amp_dtype(checkpoint_default)
|
|
|
|
|
|
def _cpu_amp_dtype() -> Optional[torch.dtype]:
|
|
"""Autocast dtype on CPU: bf16 when LAYA_CPU_AMP=bf16|bfloat16, None to leave the forward in
|
|
fp32. No fp16 spelling is offered here, because CPU autocast has no fp16 fast path that beats
|
|
fp32 -- bf16 is the only reduced precision worth asking for, and only on hardware with native
|
|
BF16 instructions, which is why it stays opt-in. Anything else is ignored.
|
|
|
|
On hardware without those instructions the failure is a native fault, not an exception:
|
|
oneDNN picks a kernel the CPU cannot execute and the process dies with SIGILL, or
|
|
`0xC000001D` on Windows. `_infer`'s autocast retry catches `RuntimeError`, so it cannot
|
|
see that and cannot fall back. Qualify the host first; `docs/reference/agent.md` has the
|
|
one-line child-process probe and the `get_cpu_capability()` reading to check."""
|
|
if os.environ.get("LAYA_CPU_AMP", "").lower() in ("bf16", "bfloat16"):
|
|
return torch.bfloat16
|
|
return None
|
|
|
|
def _option_count(qdef: Dict) -> int:
|
|
"""How many options a validated question definition renders to.
|
|
|
|
Mirrors `render_options`: a choice has one option per criterion, a score one per level,
|
|
and a noul is always the pair [false, true].
|
|
"""
|
|
if qdef.get("type") == "noul":
|
|
return 2
|
|
crit = qdef.get("criteria")
|
|
return len(crit) if isinstance(crit, (dict, list, tuple)) else 0
|
|
|
|
|
|
def _start_evidence():
|
|
"""A recorder for `predict_long`: what the start-hook chain left for inference to run on.
|
|
|
|
Why a recorder at all: `ctx.skip()` only assigns `ctx.results` (see `laya.hooks.PredictContext`),
|
|
so outside `predict_batch` the only account of what the hooks did is the context they were
|
|
handed. The result count cannot tell a hook that answered the document from one that replaced
|
|
the window list -- both return fewer results than there were windows, and the two need opposite
|
|
readings: the first scored no window, the second scored the ones it left behind and booked
|
|
tokens for them.
|
|
|
|
The probe belongs last in the chain because `compose_hooks` orders defaults, then installed,
|
|
then per-call hooks, and `dispatch` calls them in order, so a probe appended after the caller's
|
|
own start hooks observes exactly what `predict_batch` is about to act on.
|
|
|
|
Returns the probe and the dict it fills: `answered` is whether a hook replaced the call before
|
|
inference, `states` is a snapshot of the states that reached it (`None` if the probe never ran,
|
|
which means `predict_batch` was replaced and no hook chain was dispatched).
|
|
"""
|
|
evidence = {"answered": False, "states": None, "question_types": None,
|
|
"max_len": None, "head_max_len": None, "questions": None}
|
|
|
|
def probe(ctx):
|
|
evidence["answered"] = ctx.results is not None
|
|
evidence["states"] = list(ctx.states)
|
|
# Snapshot only the inference schema; leave malformed questions to predict_batch's
|
|
# validator, and do not inspect questions when a hook has already answered the call.
|
|
if ctx.results is None and isinstance(ctx.questions, dict):
|
|
evidence["question_types"] = {
|
|
qid: qdef.get("type") if isinstance(qdef, dict) else None
|
|
for qid, qdef in ctx.questions.items()
|
|
}
|
|
# Also what the budget and the questions ACTUALLY are once the chain has run, for
|
|
# `_check_scan_budget`. Recorded, never judged here: a hook that raises is governed by the
|
|
# caller's `hooks_raise`/`hooks_timeout`, so a correctness check that raises from inside the
|
|
# chain can be switched off by a policy meant for third-party telemetry. Measured: with
|
|
# `hooks_raise=False` -- what `docs/hooks/tracing.md` recommends -- a refusal became a
|
|
# RuntimeWarning blaming `_StartAdapter`, and the scan proceeded.
|
|
evidence["max_len"] = ctx.max_len
|
|
evidence["head_max_len"] = ctx.head_max_len
|
|
evidence["questions"] = dict(ctx.questions) if ctx.questions else {}
|
|
|
|
return probe, evidence
|
|
|
|
|
|
def _check_scan_budget(agent, evidence, sized, cfg_max_len, cfg_head_max_len, asked=None):
|
|
"""Raise when the budget or questions in force leave less room than the scan was sized for.
|
|
|
|
`predict_long` sizes its windows from the agent's config, before any hook has run. A start hook
|
|
may then set `ctx.max_len`/`ctx.head_max_len`, or rewrite `ctx.questions` -- both documented
|
|
powers -- and `build_sequence` uses whatever it finds. When the head widens faster than
|
|
`max_len`, or the questions get more options, the real room SHRINKS: every window is
|
|
re-truncated on the way in, and once the stride exceeds the real room consecutive windows stop
|
|
touching. Measured on the English checkpoint with `widen_for_high_cardinality` from
|
|
`docs/hooks/patterns.md` at 50 options, windows sized 303 against a room of 253; at 100 options
|
|
a reviewer measured 43.4% of a document reaching no model, against the 37.6% this change exists
|
|
to remove. Silently worse than not windowing at all.
|
|
|
|
Called by `predict_long` AFTER the chain has run, not raised from inside it. A hook that raises
|
|
is subject to the caller's `hooks_raise` and `hooks_timeout`, so the first version of this check
|
|
could be switched off by `hooks_raise=False` -- which `docs/hooks/tracing.md` recommends -- and
|
|
could blow a `hooks_timeout` the caller set for their own hooks. The cost is that the forward
|
|
pass has already happened when this fires: a misconfigured scan is refused rather than answered
|
|
wrongly, but it is not refused for free. Sizing the windows after the chain instead would change
|
|
when hooks run and what they see, which is a larger change than this one.
|
|
"""
|
|
if evidence.get("answered"):
|
|
return # a hook answered the document; no window was scored
|
|
questions = evidence.get("questions")
|
|
if not questions:
|
|
return # nothing to fit, as `window_budget` also concludes
|
|
eff_max_len = evidence.get("max_len")
|
|
eff_head = evidence.get("head_max_len")
|
|
eff_max_len = cfg_max_len if eff_max_len is None else eff_max_len
|
|
eff_head = cfg_head_max_len if eff_head is None else eff_head
|
|
if eff_max_len == cfg_max_len and eff_head == cfg_head_max_len and (
|
|
asked is None or questions == asked):
|
|
# Nothing the scan was sized against moved, so there is nothing to recompute -- and
|
|
# recomputing anyway costs a tokenization of every question head, which `window_budget`
|
|
# already paid and `_encode_state` will pay again. That showed up as a third head
|
|
# tokenization in `test_question_token_reuse`, which asserts two.
|
|
return
|
|
try:
|
|
internal = [agent._to_internal(q) for q in questions.values()]
|
|
except Exception:
|
|
return # malformed questions are the validator's to report
|
|
if not internal:
|
|
return
|
|
room = min(state_room(agent.tok, q, eff_max_len, eff_head) for q in internal)
|
|
if room >= sized:
|
|
return
|
|
raise ValueError(
|
|
"predict_long: after the start hooks ran, the questions and token budget (max_len=%d "
|
|
"head_max_len=%d) leave %d state tokens per window, but the scan was sized for %d from the "
|
|
"agent's config (max_len=%d head_max_len=%d). Every window would be re-truncated and parts "
|
|
"of the document would reach no model. Pass window=/stride= explicitly, or call "
|
|
"predict_batch directly, if you need a start hook to change either."
|
|
% (eff_max_len, eff_head, room, sized, cfg_max_len, cfg_head_max_len))
|
|
|
|
|
|
def _with_start_probe(hook_kwargs, probe):
|
|
"""`hook_kwargs` with `probe` appended after the caller's own start hooks.
|
|
|
|
`probe` may be one hook or a list of them, appended in order.
|
|
"""
|
|
kwargs = dict(hook_kwargs)
|
|
extra = list(probe) if isinstance(probe, (list, tuple)) else [probe]
|
|
kwargs["on_predict_start"] = list(_as_sequence(hook_kwargs.get("on_predict_start"))) + extra
|
|
return kwargs
|
|
def _option_logits(logits, items, offset):
|
|
"""Raw per-option logits, the rows `_decode_answers` divides by temperature.
|
|
|
|
Calibration record collection slices with this same helper, so a fitted map sees the
|
|
option width the decoder scales and not a second tokenization of the state.
|
|
"""
|
|
return [logits[offset + j, : len(item["markers"])] for j, item in enumerate(items)]
|
|
|
|
|
|
class Agent(HookRegistry):
|
|
"""System 1 decision model runtime: fast, non-autoregressive, calibrated decisions.
|
|
|
|
`dtype` is the autocast target, not the precision of every call. On MPS a call
|
|
autocasts only at or above `mps_amp_min_rows` rows, so `dtype` can say float16 while
|
|
a call runs in float32. `dtype_for(rows)` returns the precision of a call with `rows` rows.
|
|
"""
|
|
|
|
# Hooks are opt-in. `hooks`/`_hooks_mutex` defaults come from HookRegistry; the rest keep a
|
|
# hand-built instance (`Agent.__new__` in tests) working and make an unset hook a no-op.
|
|
hooks_raise = True
|
|
hooks_concurrent = True
|
|
hooks_timeout = None
|
|
_hooks_lock = None
|
|
model_id = None
|
|
# Autocast is chosen per device in __init__; this default covers instances built
|
|
# without it (for example a hand-constructed runtime in tests).
|
|
amp_enabled = False
|
|
mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT
|
|
_amp_failures = 0
|
|
# The stock forward is also used by lightweight runtimes built with __new__ in tests.
|
|
_fast = None
|
|
_backend = None
|
|
# The checkpoint's `option_layout` (see `uses_parallel_layout`); runtimes built with __new__ are sequential.
|
|
parallel_options = False
|
|
# Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired
|
|
# and what the last failure was, so an operator sees a slow lane in /health instead of
|
|
# discovering it by accident. Class defaults cover instances built without __init__.
|
|
cpu_fallback_count = 0
|
|
last_fallback_reason = None
|
|
# Set when `compile=True` wrapped the model in torch.compile.
|
|
_compiled = False
|
|
_reduce_overhead = False
|
|
_gate = _InferenceGate()
|
|
|
|
def __init__(
|
|
self,
|
|
model_id_or_path: str = "convaiinnovations/laya",
|
|
device: Optional[str] = None,
|
|
token: Optional[str] = None,
|
|
subfolder: Optional[str] = None,
|
|
fast: bool = False,
|
|
compile: bool = False,
|
|
revision: Optional[str] = None,
|
|
expected_sha256: Optional[Dict[str, str]] = None,
|
|
lang_temperatures: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
hooks=None,
|
|
on_predict_start=None,
|
|
on_predict_end=None,
|
|
hooks_raise: bool = True,
|
|
hooks_concurrent: bool = True,
|
|
hooks_timeout: Optional[float] = None,
|
|
calibration: Optional[str] = None,
|
|
backend: Optional[str] = None,
|
|
compile_warmup: bool = True,
|
|
compile_cache: bool = False,
|
|
compile_mode: str = "default",
|
|
):
|
|
"""Load a Laya checkpoint.
|
|
|
|
`backend` selects "eager", "auto", "compile" or "tilelang"; see `laya.backends`.
|
|
It takes precedence over `fast` and `compile`. Omit it to keep those legacy flags.
|
|
For ONNX use `load(backend="onnx")` instead.
|
|
|
|
`revision` optionally pins the Hub download to an explicit commit SHA/branch/tag;
|
|
when omitted, huggingface_hub's normal default and existing offline cache are used.
|
|
`expected_sha256` ({path relative to the checkpoint dir: hexdigest})
|
|
verifies artifact integrity before any weight is parsed or executed; it is opt-in
|
|
and applies to local directories too. A missing artifact raises `FileNotFoundError`
|
|
and a digest mismatch raises `ValueError`; either error refuses the load.
|
|
|
|
`fast=True` swaps the encoder/head forward for the TileLang fast path (CUDA only, needs
|
|
`pip install laya[fast]`); see `Agent.accelerate`.
|
|
|
|
`compile=True` runs the model under `torch.compile` and turns ModernBERT's encoder
|
|
`reference_compile` on. `torch.compile` specializes per input shape and Laya sees a
|
|
new one on almost every request, so those graphs usually cost more than they return;
|
|
use it when the traffic is repetitive. `fast=True` takes precedence, because the
|
|
TileLang path replaces the forward that would be compiled.
|
|
Compiled agents run `warmup()` before returning; `compile_warmup=False` defers that
|
|
work to requests or a manual `warmup()` call. Eager and fast agents are unchanged.
|
|
`compile_cache=True` opts into a persistent Laya Inductor directory (process-wide),
|
|
respecting any existing `TORCHINDUCTOR_CACHE_DIR`; see the compile engineering notes.
|
|
`compile_mode="reduce-overhead"` opts into CUDA graphs. It can retain more GPU memory
|
|
and records each new shape separately. CUDA outputs are copied before the next replay;
|
|
compiled CUDA graph forwards are serialized. The default mode remains "default".
|
|
|
|
`subfolder` selects one checkpoint from a repo that bundles several, e.g.
|
|
`Agent("convaiinnovations/laya", subfolder="multilingual")`. Only that subfolder is
|
|
downloaded, so bundling does not cost every user the whole family.
|
|
|
|
`calibration` is an optional JSON path with `temperature` and `temperature_by_options`.
|
|
It is applied after the checkpoint config, so a fitted map overrides shipped scalars
|
|
without rewriting `model.safetensors`.
|
|
|
|
`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see
|
|
`laya.hooks`. `hooks_raise=False` warns and continues when a hook fails,
|
|
`hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and
|
|
`hooks_timeout` bounds each hook call in seconds (None means no limit).
|
|
"""
|
|
if backend is not None:
|
|
from .backends import normalise
|
|
backend = normalise(backend)
|
|
if backend == "onnx":
|
|
raise ValueError("use laya.load(..., backend='onnx') or ONNXAgent")
|
|
fast = compile = False
|
|
|
|
# Model loading can import Dynamo, which populates Inductor's default directory.
|
|
if compile and not fast and compile_cache:
|
|
configure_cache()
|
|
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
|
|
self.hooks_raise = bool(hooks_raise)
|
|
self.hooks_concurrent = bool(hooks_concurrent)
|
|
self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout)
|
|
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
|
|
self._hooks_mutex = threading.Lock()
|
|
self.model_id = model_id_or_path
|
|
# Retained so `save_calibration` can record which checkpoint the map was fitted for.
|
|
self.model_id_or_path = model_id_or_path
|
|
self.subfolder = subfolder
|
|
|
|
from safetensors.torch import load_file
|
|
|
|
model_dir = model_id_or_path
|
|
self.revision: Optional[str] = None
|
|
if not os.path.exists(model_dir):
|
|
if model_id_or_path.startswith(("/", "./", "../")) or os.path.isabs(model_id_or_path):
|
|
raise FileNotFoundError(
|
|
f"Local model path not found: {model_id_or_path!r}. "
|
|
f"Check that the directory exists and that training saved the model successfully."
|
|
)
|
|
from huggingface_hub import snapshot_download
|
|
|
|
# Restrict root checkpoints too: the default repo also contains sibling
|
|
# checkpoints, which an unfiltered snapshot would unnecessarily download.
|
|
revision = resolve_revision(model_id_or_path, revision)
|
|
prefix = f"{subfolder}/" if subfolder else ""
|
|
kw = {
|
|
"token": token or os.environ.get("HF_TOKEN") or None,
|
|
"allow_patterns": [prefix + name for name in (
|
|
"rl_agent_config.json", "model.safetensors", "tokenizer/*", "encoder/*",
|
|
)],
|
|
}
|
|
if revision:
|
|
kw["revision"] = revision
|
|
model_dir = snapshot_download(model_id_or_path, **kw)
|
|
# The cache layout records which commit the snapshot points at.
|
|
self.revision = snapshot_revision(model_dir) or revision
|
|
|
|
if subfolder:
|
|
model_dir = os.path.join(model_dir, subfolder)
|
|
if not os.path.isdir(model_dir):
|
|
raise FileNotFoundError(
|
|
f"Subfolder {subfolder!r} not found in {model_id_or_path!r}."
|
|
)
|
|
|
|
# Verify integrity before any file in the checkpoint is parsed or executed.
|
|
verify_digests(model_dir, expected_sha256)
|
|
|
|
_fix_tokenizer_config(model_dir)
|
|
|
|
cfg_path = os.path.join(model_dir, "rl_agent_config.json")
|
|
if not os.path.exists(cfg_path):
|
|
raise FileNotFoundError(
|
|
f"Incompatible model: {model_id_or_path!r} does not contain 'rl_agent_config.json'. "
|
|
f"That file ships with the weights of a Laya checkpoint, so load one of those "
|
|
f"(e.g. 'convaiinnovations/laya') or a directory your own training run wrote."
|
|
)
|
|
|
|
with open(cfg_path) as f:
|
|
self.cfg = json.load(f)
|
|
self.parallel_options = uses_parallel_layout(self.cfg)
|
|
|
|
weights_path = os.path.join(model_dir, "model.safetensors")
|
|
if not os.path.exists(weights_path):
|
|
raise FileNotFoundError(
|
|
f"Incompatible model: 'model.safetensors' not found in {model_id_or_path!r}."
|
|
)
|
|
|
|
# 1. Device resolution with automatic fallback
|
|
if isinstance(device, str) and device.strip().lower() == "auto":
|
|
device = None
|
|
if device is not None:
|
|
target_device = torch.device(device)
|
|
if target_device.type == "cuda" and not torch.cuda.is_available():
|
|
warnings.warn("Warning: CUDA requested but not available. Falling back to CPU.",
|
|
RuntimeWarning)
|
|
self.device = torch.device("cpu")
|
|
elif target_device.type == "cuda" and target_device.index is not None:
|
|
# `is_available` only says a GPU exists; a bad ordinal sails through it
|
|
# and dies later in `.to()` with a bare CUDA error. Check up front.
|
|
try:
|
|
count = torch.cuda.device_count()
|
|
except (RuntimeError, AttributeError):
|
|
count = 0
|
|
if target_device.index < 0 or target_device.index >= max(count, 1):
|
|
warnings.warn("Warning: CUDA device %s not found (%d visible). Falling back to CPU."
|
|
% (target_device, count),
|
|
RuntimeWarning)
|
|
self.device = torch.device("cpu")
|
|
else:
|
|
self.device = target_device
|
|
elif target_device.type == "mps" and not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()):
|
|
warnings.warn("Warning: MPS requested but not available. Falling back to CPU.",
|
|
RuntimeWarning)
|
|
self.device = torch.device("cpu")
|
|
elif target_device.type == "xpu" and not (hasattr(torch, "xpu") and torch.xpu.is_available()):
|
|
warnings.warn("Warning: XPU requested but not available. Falling back to CPU.",
|
|
RuntimeWarning)
|
|
self.device = torch.device("cpu")
|
|
else:
|
|
self.device = target_device
|
|
else:
|
|
if torch.cuda.is_available():
|
|
self.device = torch.device("cuda")
|
|
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
|
self.device = torch.device("mps")
|
|
elif hasattr(torch, "xpu") and torch.xpu.is_available():
|
|
self.device = torch.device("xpu")
|
|
else:
|
|
self.device = torch.device("cpu")
|
|
|
|
tok_dir = os.path.join(model_dir, "tokenizer")
|
|
self.tok = _load_tokenizer(tok_dir, self.cfg)
|
|
|
|
enc_dir = os.path.join(model_dir, "encoder")
|
|
# build_model skips initialisation of every parameter; the checkpoint supplies them all.
|
|
self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None,
|
|
pretrained=False)
|
|
|
|
# Load weights and verify architectural compatibility
|
|
weights = load_file(weights_path)
|
|
_verify_compatibility(self.model, self.cfg, weights, model_id_or_path)
|
|
|
|
self.model.load_state_dict(weights, strict=True)
|
|
|
|
# ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder.
|
|
# That is a loss for the batch sizes Laya runs (a handful of questions per call) and can
|
|
# hang on some platforms, so keep the eager path unless explicitly requested.
|
|
try:
|
|
self.model.encoder.config.reference_compile = compile
|
|
except Exception:
|
|
pass
|
|
|
|
# The TileLang fast path replaces the forward itself, so it takes precedence over compile.
|
|
# dynamic=True plus independent dimensions (laya/_compile.py) so a new request shape
|
|
# does not recompile.
|
|
if compile and not fast:
|
|
if compile_mode not in ("default", "reduce-overhead"):
|
|
raise ValueError("compile_mode must be 'default' or 'reduce-overhead'")
|
|
self._reduce_overhead = compile_mode == "reduce-overhead"
|
|
compile_kwargs = {"mode": compile_mode} if self._reduce_overhead else {}
|
|
self.model = compile_model(self.model, **compile_kwargs)
|
|
self._compiled = True
|
|
|
|
# Keep what the checkpoint shipped for inspection, but only ever apply clamped values:
|
|
# some buckets are fitted to sharpen rather than soften (see clamp_temperature).
|
|
self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])
|
|
# `_decode_answers` indexes this by question type, so a list of the wrong length would
|
|
# load cleanly and then raise a bare `IndexError` on the first `score`/`noul` question;
|
|
# refuse the shape here, the way the language-override check below does.
|
|
if not isinstance(self.temperature_raw, (list, tuple)) or len(self.temperature_raw) != 3:
|
|
raise ValueError(
|
|
"Incompatible model: %r temperature must be a list of 3 floats, got %r"
|
|
% (model_id_or_path, self.temperature_raw))
|
|
self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})
|
|
self.temperature = [clamp_temperature(t) for t in self.temperature_raw]
|
|
self.temperature_by_options = {k: clamp_temperature(v)
|
|
for k, v in self.temperature_by_options_raw.items()}
|
|
# Optional histogram-binning map installed by `load_calibration` or fit via
|
|
# `fit_binning`; absent/None means answer_confidence is the temperature-scaled one.
|
|
self.binning_map = None
|
|
|
|
# Shared with `ONNXAgent` so both backends accept the same option and produce the same
|
|
# confidences; see `common.resolve_lang_temperatures` for why the shape is checked before
|
|
# it is read.
|
|
self.lang_temperatures = resolve_lang_temperatures(lang_temperatures, self.temperature_raw)
|
|
entries = [(k, v, self.temperature_by_options[k]) for k, v in self.temperature_by_options_raw.items()]
|
|
entries += [("temperature[%d]" % i, t, self.temperature[i]) for i, t in enumerate(self.temperature_raw)]
|
|
rejected = []
|
|
for name, raw, applied in entries:
|
|
if not temperature_rejected(raw, applied):
|
|
continue
|
|
rejected.append("%s=%r -> %g" % (name, raw, applied))
|
|
if rejected:
|
|
warnings.warn(
|
|
"laya: this checkpoint ships invalid temperatures or values outside [%g, %g]; "
|
|
"using %s. Treat confidence from the affected entries as uncalibrated."
|
|
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
|
|
RuntimeWarning, stacklevel=2)
|
|
if calibration:
|
|
self.load_calibration(calibration)
|
|
# Autocast policy. CUDA, MPS and XPU all support fp16/bf16 autocast and the shipped
|
|
# checkpoints are trained in reduced precision; on CUDA the checkpoint's `amp_dtype`
|
|
# (bf16) is the default and _cuda_amp_dtype's LAYA_CUDA_AMP override wins over it. CPU
|
|
# bf16 is only a win on hardware with native BF16, so it stays opt-in through
|
|
# _cpu_amp_dtype. MPS fp16 is slower than fp32 on
|
|
# a single small row (autocast overhead dominates) and only wins once the batch has
|
|
# several rows, so it is gated per call by `mps_amp_min_rows` (default 5, override with
|
|
# LAYA_MPS_AMP_MIN_ROWS) rather than enabled unconditionally. XPU autocast supports
|
|
# bf16/fp16 only, and entering it on a torch build without an XPU autocast backend
|
|
# raises on every predict (the failure #273 fixed for MPS), so it is only enabled on
|
|
# builds that have one.
|
|
self.dtype = torch.float32
|
|
self.amp_enabled = False
|
|
self.mps_amp_min_rows = _mps_amp_min_rows()
|
|
if self.device.type == "cuda":
|
|
self.amp_enabled = True
|
|
try:
|
|
major = torch.cuda.get_device_capability(self.device)[0]
|
|
except (RuntimeError, AttributeError):
|
|
# A driver that reports a GPU but cannot answer the capability query
|
|
# cannot be trusted with bf16 either; fp16 is the safe default and
|
|
# placement below still decides whether the device is usable at all.
|
|
warnings.warn("Warning: could not query CUDA capability; assuming fp16 defaults.",
|
|
RuntimeWarning)
|
|
self.dtype = torch.float16
|
|
else:
|
|
if major < 8:
|
|
self.dtype = torch.float16
|
|
else:
|
|
self.dtype = _cuda_amp_dtype(self.cfg.get("amp_dtype", "fp16"))
|
|
elif self.device.type == "mps":
|
|
self.amp_enabled = True
|
|
self.dtype = torch.float16
|
|
elif self.device.type == "xpu":
|
|
# Device selection above already requires torch.xpu for an xpu device; the probe
|
|
# also guards hand-built agents on builds without an XPU autocast backend.
|
|
if getattr(torch, "xpu", None) is not None and torch.xpu.is_available():
|
|
self.amp_enabled = True
|
|
self.dtype = torch.bfloat16
|
|
elif self.device.type == "cpu":
|
|
cpu_amp = _cpu_amp_dtype()
|
|
if cpu_amp is not None:
|
|
self.amp_enabled = True
|
|
self.dtype = cpu_amp
|
|
|
|
self._fast = None
|
|
self._gate = _InferenceGate()
|
|
|
|
# 2. Place on device with graceful fallback to CPU on memory error
|
|
fell_back_from = fell_back_why = None
|
|
try:
|
|
self.model.to(self.device).eval()
|
|
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
|
|
if self.device.type != "cpu":
|
|
# Record what actually went wrong: the reason matters more than the symptom,
|
|
# and it is the only place the underlying exception is ever surfaced.
|
|
fell_back_from, fell_back_why = self.device, e
|
|
self.device = torch.device("cpu")
|
|
self.dtype = torch.float32
|
|
self.amp_enabled = False
|
|
self.model.to(self.device).eval()
|
|
else:
|
|
raise e
|
|
|
|
if backend is not None:
|
|
self.set_backend(backend)
|
|
elif fast:
|
|
self.accelerate()
|
|
|
|
if fell_back_from is not None:
|
|
warnings.warn(
|
|
"\n[laya] Warning: could not place the model on %s, so it is running on CPU.\n"
|
|
" Reason: %s\n"
|
|
" Inference will be roughly 10-15x slower (~200-500 ms rather than ~35 ms).\n"
|
|
" If this is a newer NVIDIA GPU (Blackwell / RTX 50-series), your PyTorch build\n"
|
|
" may predate stable Blackwell support (stable since torch 2.6, CUDA 12.8+).\n"
|
|
" Upgrade to a current stable build with a CUDA 13.x wheel, e.g.:\n"
|
|
" pip install --upgrade torch --index-url https://download.pytorch.org/whl/cu130\n"
|
|
" See https://pytorch.org/get-started/locally/\n"
|
|
% (fell_back_from, fell_back_why), RuntimeWarning)
|
|
|
|
if self._compiled and compile_warmup:
|
|
try:
|
|
self.warmup()
|
|
except Exception as e:
|
|
# Compilation is lazy: unavailable toolchains fail on the first forward.
|
|
# Only automatic warm-up is best effort; explicit warmup() still raises.
|
|
warnings.warn("Warning: laya compile warm-up failed (%s: %s); keeping the compiled model. "
|
|
"Later requests and explicit warmup() calls may still raise."
|
|
% (type(e).__name__, e), RuntimeWarning)
|
|
|
|
@property
|
|
def backend(self) -> str:
|
|
"""The active inference backend, including the legacy compile and fast flags."""
|
|
if self._backend is not None:
|
|
return self._backend.name
|
|
return "tilelang" if self._fast is not None else "compile" if self._compiled else "eager"
|
|
|
|
@property
|
|
def backend_object(self):
|
|
"""The installed Backend object, or None for a legacy runtime."""
|
|
return self._backend
|
|
|
|
def set_backend(self, name: str = "auto", strict: bool = False, **options) -> str:
|
|
"""Switch backends; unavailable backends warn and use eager unless `strict=True`.
|
|
|
|
Options go to the backend constructor, e.g. `warmup=False` for compile or
|
|
`use_graphs=False` for tilelang. Switching waits for in-flight inference.
|
|
"""
|
|
from . import backends
|
|
name = backends.normalise(name)
|
|
with self._gate.write_lock():
|
|
self.deaccelerate()
|
|
if self._compiled:
|
|
self.model = self.model._orig_mod
|
|
self._compiled = False
|
|
self.model.encoder.config.reference_compile = False
|
|
return backends.install(self, name, strict=strict, **options).name
|
|
|
|
def accelerate(self, use_graphs: bool = True, strict: bool = False):
|
|
"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,
|
|
sliding-window flash attention, 16-bit resident weights, CUDA graphs per shape bucket).
|
|
|
|
The fast path runs in the agent's autocast dtype at the time of the call (bf16 or fp16), so it
|
|
matches the stock forward it replaces within rounding (see benchmarks/parity_fast.py). After
|
|
changing `agent.dtype`, call `deaccelerate()` then `accelerate()` to rebuild it. Returns True if
|
|
enabled. With `strict=False` any failure (no CUDA, tilelang missing) leaves the stock path in place.
|
|
"""
|
|
if self._fast is not None:
|
|
return True
|
|
if self.device.type != "cuda":
|
|
if strict:
|
|
raise RuntimeError("laya fast path needs a CUDA device")
|
|
return False
|
|
if self._backend is not None:
|
|
self.deaccelerate()
|
|
last = None
|
|
for _attempt in range(2): # tilelang's JIT cache has been seen to fail once, then succeed
|
|
try:
|
|
from .fast import FastLaya
|
|
fast_dtype = self.dtype if self.dtype in (torch.bfloat16, torch.float16) else torch.bfloat16
|
|
self._fast = FastLaya(self.model, max_len=self.cfg.get("max_len", 512), use_graphs=use_graphs,
|
|
dtype=fast_dtype)
|
|
break
|
|
except Exception as e: # tilelang missing / unsupported arch
|
|
last = e
|
|
if self._fast is None:
|
|
if strict:
|
|
raise last
|
|
warnings.warn("Warning: laya fast path unavailable (%s); using the stock forward." % last,
|
|
RuntimeWarning)
|
|
return False
|
|
self._stock_forward = self.model.forward
|
|
self.model.forward = self._fast.forward
|
|
return True
|
|
|
|
# (rows, tokens, markers) run by `warmup()`: one row, which torch specialises into a graph of
|
|
# its own, and a small batch. The three sizes differ within each shape, so none are tied.
|
|
WARMUP_SHAPES = ((1, 64, 3), (4, 128, 5))
|
|
|
|
@torch.no_grad()
|
|
def warmup(self, shapes=None) -> float:
|
|
"""Run the forward on synthetic input of each shape now and return the seconds it took.
|
|
|
|
`compile=True` calls this at load unless `compile_warmup=False`. Extra shapes can still
|
|
be warmed manually. `fast=True` builds its kernels and CUDA graphs per shape bucket on
|
|
first use; calling this before serving moves that cost out of the first requests.
|
|
With the stock forward it is a few ordinary forward passes. `shapes` is a list
|
|
of (rows, tokens, markers); tokens are capped at the agent's `max_len`. Nothing is
|
|
returned to or recorded for any caller, and hooks do not run.
|
|
"""
|
|
max_len = int(self.cfg.get("max_len", 512))
|
|
fill = self.tok.cls_token_id or 0
|
|
t0 = time.perf_counter()
|
|
for rows, tokens, markers in (self.WARMUP_SHAPES if shapes is None else shapes):
|
|
tokens = max(markers + 2, min(int(tokens), max_len))
|
|
self._infer({
|
|
"input_ids": torch.full((rows, tokens), fill, dtype=torch.long),
|
|
"attention_mask": torch.ones((rows, tokens), dtype=torch.long),
|
|
"marker_pos": torch.arange(1, markers + 1, dtype=torch.long).repeat(rows, 1),
|
|
"marker_mask": torch.ones((rows, markers), dtype=torch.bool),
|
|
"qtype": torch.zeros(rows, dtype=torch.long),
|
|
})
|
|
if self.device.type == "cuda":
|
|
torch.cuda.synchronize(self.device)
|
|
return time.perf_counter() - t0
|
|
|
|
def deaccelerate(self):
|
|
"""Restore the stock forward."""
|
|
if self._backend is not None:
|
|
self._backend.uninstall()
|
|
self._backend = None
|
|
self._fast = None
|
|
elif self._fast is not None:
|
|
self.model.forward = self._stock_forward
|
|
self._fast = None
|
|
|
|
def _restore_runtime(self, device, dtype, amp_enabled: bool, had_fast: bool) -> None:
|
|
"""Undo the scoped CPU fallback of `_infer`, best effort.
|
|
|
|
A model that no longer fits `device` after the CPU retry stays demoted: raising out of a
|
|
request that already succeeded would trade a silent slowdown for a crash, the worse
|
|
failure of the two.
|
|
"""
|
|
try:
|
|
self.model.to(device)
|
|
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
|
|
# The move can fail partway through, leaving parameters split between devices, so
|
|
# finish the demotion before giving up: every later call must find one device,
|
|
# not a mix of both.
|
|
self.model.to(torch.device("cpu"))
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
warnings.warn("Warning: could not move the model back to %s after the CPU retry (%s); "
|
|
"staying on CPU." % (device, e), RuntimeWarning)
|
|
return
|
|
self.device, self.dtype, self.amp_enabled = device, dtype, amp_enabled
|
|
if had_fast:
|
|
self.accelerate()
|
|
|
|
@staticmethod
|
|
def _check_question(qid: str, qdef: Any) -> None:
|
|
"""Reject a question that cannot be answered, naming it and what to fix.
|
|
|
|
`render_options` reads `criteria` in the shape the question's type expects and the decision
|
|
head needs at least one option, so a malformed definition used to surface from three frames
|
|
down as something that names neither the question nor the problem: `AttributeError:
|
|
'NoneType' object has no attribute 'items'`, `KeyError: 'bool'`, or a `selected index k out
|
|
of range` raised inside the model for a question that ended up with no options at all.
|
|
"""
|
|
if qid is None:
|
|
raise ValueError("question id must not be None")
|
|
if not isinstance(qid, (str, int)) or (isinstance(qid, str) and not qid.strip()):
|
|
raise ValueError("question id must be a non-empty string, got %r" % (qid,))
|
|
if not isinstance(qdef, dict):
|
|
raise ValueError("question %r: definition must be a dict, got %s"
|
|
% (qid, type(qdef).__name__))
|
|
t = qdef.get("type")
|
|
if not isinstance(t, str) or t not in QTYPES:
|
|
raise ValueError("question %r: unknown type %r; use one of %s" % (qid, t, sorted(QTYPES)))
|
|
if "instructions" not in qdef:
|
|
raise ValueError("question %r: no 'instructions'; add the text the model should answer" % (qid,))
|
|
ins = qdef["instructions"]
|
|
if ins is None:
|
|
raise ValueError("question %r: 'instructions' must not be None; add the text the model should answer" % (qid,))
|
|
if isinstance(ins, str) and not ins.strip():
|
|
raise ValueError("question %r: 'instructions' must not be empty; add the text the model should answer" % (qid,))
|
|
if isinstance(ins, (list, dict)) and not ins:
|
|
raise ValueError("question %r: 'instructions' must not be empty; add the text the model should answer" % (qid,))
|
|
if not isinstance(ins, (str, dict, list, int, float)):
|
|
raise ValueError("question %r: 'instructions' must be a string, dict, or list, got %s"
|
|
% (qid, type(ins).__name__))
|
|
crit = qdef.get("criteria")
|
|
if t == "choice":
|
|
if not isinstance(crit, (dict, list)):
|
|
raise ValueError("question %r: a choice question takes 'criteria' as a dict of "
|
|
"label -> description, or a list of labels" % (qid,))
|
|
if not crit:
|
|
raise ValueError("question %r: a choice question needs at least one criterion" % (qid,))
|
|
# A label is used as a dict key when a list of labels is normalised, so an unhashable
|
|
# label raised `TypeError` from three frames down -- which names neither the question nor
|
|
# the label, and which `serve` cannot classify as a caller error, so over HTTP it became
|
|
# a 500 "inference failed" instead of a 422. Requiring the documented scalars exactly
|
|
# also catches the hashable-but-not-scalar shapes (`tuple`, `frozenset`, `bytes`,
|
|
# `complex`) that the old deny-list let through to the same 500. Labels are rendered as
|
|
# option text, so a nested structure has no meaning here.
|
|
for i, label in enumerate(crit if isinstance(crit, list) else crit.keys()):
|
|
if label is not None and not isinstance(label, (str, int, float, bool)):
|
|
raise ValueError(
|
|
"question %r: choice label %d is a %s; a label is rendered as option text "
|
|
"and used as the answer key, so it must be a scalar (a string, number or "
|
|
"bool), got %r" % (qid, i, type(label).__name__, label))
|
|
if label is None:
|
|
# The score path rejects a null level with the same argument one question type
|
|
# over: a label is option text AND the answer key, and a null one can be
|
|
# neither. `_to_internal` normalises a list of labels to `{label: None}`, so a
|
|
# null label became the option text "None" (`str(None)`) while the answer key
|
|
# and the `probabilities` key for that same option were the JSON string "null",
|
|
# because a dict key has to be a str. A client then cannot tell whether that
|
|
# option was the *string* `"null"` or JSON `null`, and the answer key is
|
|
# unreachable: `criteria[answer["choice"]]` yields `None`, and
|
|
# `answer["choice"] == "null"` is False for it. `""` stays accepted, because
|
|
# unlike a null it does round-trip.
|
|
raise ValueError(
|
|
"question %r: choice label %d is null; a label is rendered as option text "
|
|
"and used as the answer key, so it must be a string, number or bool -- a "
|
|
"null label renders as the text \"None\" while its answer key is \"null\""
|
|
% (qid, i))
|
|
# `_to_internal` normalises the list form to `{label: None}`, so those labels become
|
|
# the answer keys. Two entries that land on one key made the model score fewer options
|
|
# than the caller wrote, and the response carry fewer probabilities than their list,
|
|
# without a word -- the same silent-shape class as the two checks above. Python
|
|
# treats values as one key whenever they compare equal, so `[1, 1.0]` and `[True, 1]`
|
|
# collapse as well as an exact repeat. An unhashable label raised
|
|
# `TypeError: cannot use 'tuple' as a dict key` from `_to_internal`, three frames down,
|
|
# which names neither the question nor the label -- and which `serve` cannot classify
|
|
# as a caller error, so over HTTP it became a 500 "inference failed" instead of a 422.
|
|
if isinstance(crit, list):
|
|
keys: Dict[Any, int] = {}
|
|
for i, label in enumerate(crit):
|
|
try:
|
|
first = keys[label]
|
|
except TypeError as exc:
|
|
raise ValueError(
|
|
"question %r: choice label %d (%r) cannot be an answer key because it is "
|
|
"unhashable; a label is rendered as option text and used as the answer "
|
|
"key" % (qid, i, label)) from exc
|
|
except KeyError:
|
|
keys[label] = i
|
|
else:
|
|
raise ValueError(
|
|
"question %r: choice label %d (%r) repeats label %d; the labels are the "
|
|
"answer keys, so every option needs its own (1, 1.0 and True are one "
|
|
"key)" % (qid, i, label, first))
|
|
elif t == "score":
|
|
if not isinstance(crit, list):
|
|
raise ValueError("question %r: a score question takes 'criteria' as a list of level "
|
|
"descriptions, index 0 first" % (qid,))
|
|
if not crit:
|
|
raise ValueError("question %r: a score question needs at least one level" % (qid,))
|
|
if None in crit:
|
|
raise ValueError("question %r: score level %d is null; give every level a description, "
|
|
"index 0 first" % (qid, crit.index(None)))
|
|
elif crit is not None and not isinstance(crit, dict):
|
|
raise ValueError("question %r: a noul question takes 'criteria' as a dict with optional "
|
|
"'true'/'false' descriptions, or omits it" % (qid,))
|
|
elif isinstance(crit, dict):
|
|
# `render_options` reads these two descriptions out by name -- `crit.get("false")` and
|
|
# `crit.get("true")` -- so a dict keyed any other way is not a noul description at all.
|
|
# It used to be substituted with the default pair without a word, so a caller saw their
|
|
# descriptions accepted and never reach the model (#156). `labels` just below has
|
|
# rejected the same mistake since #163; this is the same rule on the other parameter,
|
|
# and a noul is a boolean question either way, so those are the only two keys it can have.
|
|
keys = {str(k).lower() for k in crit}
|
|
if not keys <= {"true", "false"}:
|
|
raise ValueError(
|
|
"question %r: a noul question takes 'criteria' keyed only 'true'/'false' (either "
|
|
"or both, and omitted is fine), got %s. Those keys are the option texts the model "
|
|
"reads; any other key was silently dropped and replaced with the defaults. If you "
|
|
"want the answer worded differently, keep 'criteria' keyed 'true'/'false' and set "
|
|
"'labels' instead." % (qid, sorted(keys)))
|
|
if "option_order" in qdef:
|
|
# Slot s shows option `order[s]`. Anything other than a permutation of the option
|
|
# indices would either drop an option or show one twice, so reject it here rather
|
|
# than let it reach the encoder.
|
|
order = qdef["option_order"]
|
|
n = _option_count(qdef)
|
|
if (not isinstance(order, (list, tuple))
|
|
or len(order) != n
|
|
or sorted(int(i) for i in order if isinstance(i, int) and not isinstance(i, bool)) != list(range(n))):
|
|
raise ValueError(
|
|
"question %r: 'option_order' must be a permutation of range(%d) -- one slot per "
|
|
"option, each option once -- got %r" % (qid, n, order))
|
|
if "labels" in qdef:
|
|
if t != "noul":
|
|
raise ValueError("question %r: 'labels' is only supported for noul questions" % (qid,))
|
|
try:
|
|
_resolve_noul_labels(qdef["labels"])
|
|
except ValueError as e:
|
|
raise ValueError("question %r: %s" % (qid, e)) from e
|
|
|
|
@staticmethod
|
|
def _to_internal(qdef: Dict) -> Dict:
|
|
t = qdef["type"]
|
|
crit = qdef.get("criteria")
|
|
if t == "choice" and isinstance(crit, list):
|
|
crit = {c: None for c in crit}
|
|
elif t == "noul" and isinstance(crit, dict):
|
|
# Normalize boolean literal keys to string keys ("true"/"false")
|
|
crit = {str(k).lower(): v for k, v in crit.items()}
|
|
ins = qdef["instructions"]
|
|
if not isinstance(ins, str):
|
|
# `ensure_ascii=False`, matching `serialize_state` and `render_criterion` in
|
|
# common.py and the instructions path in shortlist.py. The default escaped
|
|
# non-ASCII to literal `\uXXXX`, which the tokenizer then read as escape text:
|
|
# on the English checkpoint one German question answered noul=0.1652 as a dict
|
|
# and noul=0.2650 as the identical plain string.
|
|
ins = json.dumps(ins, ensure_ascii=False)
|
|
q = {"t": t, "ins": ins, "crit": crit}
|
|
if "labels" in qdef:
|
|
q["labels"] = qdef["labels"]
|
|
if "option_order" in qdef:
|
|
q["option_order"] = [int(i) for i in qdef["option_order"]]
|
|
return q
|
|
|
|
def _encode_state(self, state: Union[str, dict, list], ids: List[str], internal: Dict[str, Dict],
|
|
max_len: Optional[int] = None, head_max_len: Optional[int] = None) -> List[Dict]:
|
|
"""Tokenize one state against every (already validated + normalized) question.
|
|
|
|
`max_len` / `head_max_len` override the agent config for this call (a start hook may set
|
|
`ctx.max_len` / `ctx.head_max_len`).
|
|
"""
|
|
max_len = self.cfg.get("max_len", 512) if max_len is None else max_len
|
|
head_max_len = self.cfg.get("head_max_len", 192) if head_max_len is None else head_max_len
|
|
# A chronological conversation list is serialized newest-last, so the default
|
|
# right-truncation (st[:room]) would silently drop the newest turn. Truncate
|
|
# from the left for lists so the most recent intent is preserved.
|
|
truncate_left = isinstance(state, list)
|
|
# Tokenize the shared state once. The ids are identical for every question, so
|
|
# re-serializing and re-tokenizing it inside build_sequence per question was pure
|
|
# duplicated work. Tokenize in full and let build_sequence slice per question, so
|
|
# left-truncation for conversation lists keeps its meaning.
|
|
state_ids = encode_text(
|
|
self.tok,
|
|
serialize_state(state).replace(self.tok.mask_token, " "),
|
|
add_special_tokens=False,
|
|
)["input_ids"]
|
|
items = []
|
|
for qid in ids:
|
|
q = internal[qid]
|
|
seq, markers, stats, state_stats, *layout = build_sequence(
|
|
self.tok, state, q, max_len, head_max_len, option_order=q.get("option_order"),
|
|
truncate_left=truncate_left, state_ids=state_ids, return_stats=True,
|
|
return_truncation_stats=True, return_layout=self.parallel_options)
|
|
n_opts = stats["options"]
|
|
if len(markers) != n_opts:
|
|
# The markers are placed at absolute positions and `build_sequence` then drops the
|
|
# ones past `max_len`, so this is about the question fitting in the sequence --
|
|
# `head_max_len` is how much of it the options were given, and `max_len` is the
|
|
# ceiling that dropped them. Naming only `head_max_len` pointed at the wrong knob
|
|
# in both directions: lowering it shortens the option block and can make the
|
|
# call succeed, while raising it makes the overflow worse.
|
|
#
|
|
# The count reported is the markers that SURVIVED, not `len(seq)`: `build_sequence`
|
|
# truncates to `max_len` first, so `len(seq)` is always exactly `max_len` here and
|
|
# would state the ceiling as though it were the requirement.
|
|
raise ValueError(
|
|
"question %r: only %d of its %d option markers fit in max_len=%d with "
|
|
"head_max_len=%d spent on the question; lower head_max_len, raise max_len, "
|
|
"or use fewer options"
|
|
% (qid, len(markers), n_opts, max_len, head_max_len))
|
|
item = {"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,
|
|
"state_stats": state_stats}
|
|
if layout:
|
|
item["layout"] = layout[0]
|
|
items.append(item)
|
|
return items
|
|
|
|
def _amp_enabled_for(self, rows: int) -> bool:
|
|
"""Whether to autocast a forward with `rows` question rows.
|
|
|
|
MPS fp16 loses to fp32 on a single small row and wins once the batch grows, so it is
|
|
only enabled at or above `mps_amp_min_rows`. Other devices are unaffected.
|
|
"""
|
|
if not self.amp_enabled:
|
|
return False
|
|
if self.device.type == "mps" and rows < self.mps_amp_min_rows:
|
|
return False
|
|
return True
|
|
|
|
def dtype_for(self, rows: int) -> torch.dtype:
|
|
"""Precision that a forward pass with `rows` question rows runs in.
|
|
|
|
`dtype` is the autocast target, set once at load time. Whether a forward autocasts is
|
|
decided per call: on MPS only at or above `mps_amp_min_rows` rows. This returns `dtype`
|
|
when a forward with `rows` rows autocasts, and `torch.float32` when it does not. A
|
|
`predict` call runs one row per question.
|
|
"""
|
|
return self.dtype if self._amp_enabled_for(rows) else torch.float32
|
|
|
|
def _infer(self, b: Dict):
|
|
"""Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""
|
|
if "option_ids" in b and (self._fast is not None or self._compiled
|
|
or getattr(self._backend, "name", "eager") != "eager"):
|
|
# Those forwards take only the five sequential-layout inputs and would run this
|
|
# checkpoint's batch without its masks.
|
|
raise ValueError("this checkpoint uses option_layout='parallel', which only the eager "
|
|
"backend runs; load it without fast=/compile= or call set_backend('eager')")
|
|
if self._compiled and self._fast is None and self.device.type == "cuda":
|
|
b = _pad_cuda_compile_batch(b, self.tok.pad_token_id)
|
|
use_amp = self._amp_enabled_for(b["input_ids"].shape[0])
|
|
|
|
limit = getattr(self._backend, "max_len", None)
|
|
if limit is not None and b["input_ids"].shape[1] > limit:
|
|
raise ValueError("the %s backend was built for max_len=%d; this request needs %d tokens"
|
|
% (self.backend, limit, b["input_ids"].shape[1]))
|
|
if self._fast is not None and b["input_ids"].shape[1] > self._fast.max_len:
|
|
raise ValueError(
|
|
"the CUDA fast path was built for max_len=%d; this request needs %d tokens. "
|
|
"Use max_len <= %d or load without fast=True."
|
|
% (self._fast.max_len, b["input_ids"].shape[1], self._fast.max_len)
|
|
)
|
|
|
|
def run(enabled=None):
|
|
# Recomputed inside run() so a fallback that moves to CPU takes effect on the
|
|
# retry. A disabled gate never enters torch.autocast at all. `enabled=False`
|
|
# retries one request in full precision without reading the agent's AMP flag.
|
|
if enabled is None:
|
|
enabled = self._amp_enabled_for(b["input_ids"].shape[0])
|
|
dims = independent_dims() if self._compiled else nullcontext()
|
|
graph_outputs = self._reduce_overhead and self._fast is None and self.device.type == "cuda"
|
|
step = cuda_graph_step() if graph_outputs else nullcontext()
|
|
with _amp_context(self.device, self.dtype, enabled), dims, step:
|
|
out = self.model(
|
|
b["input_ids"].to(self.device),
|
|
b["attention_mask"].to(self.device),
|
|
b["marker_pos"].to(self.device),
|
|
b["marker_mask"].to(self.device),
|
|
b["qtype"].to(self.device),
|
|
**({"position_ids": b["position_ids"].to(self.device),
|
|
"option_ids": b["option_ids"].to(self.device)} if "option_ids" in b else {}),
|
|
)
|
|
# CUDA graph outputs belong to a reusable pool; callers can retain our copies.
|
|
return tuple(t.clone() for t in out) if graph_outputs else out
|
|
|
|
try:
|
|
with self._gate.read_lock():
|
|
out = run()
|
|
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
|
|
low = str(e).lower()
|
|
# An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare
|
|
# "cuda" substring used to be accepted too, so any CUDA-shaped RuntimeError (a
|
|
# shape, assert or kernel error) silently and permanently demoted the agent.
|
|
if self.device.type != "cpu" and (isinstance(e, torch.cuda.OutOfMemoryError) or "memory" in low):
|
|
warnings.warn("Warning: GPU memory exceeded during inference. "
|
|
"Retrying this request on CPU...", RuntimeWarning)
|
|
# Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move
|
|
# the model for the life of the process, so one oversized request left every
|
|
# later call ~10-15x slower on CPU. Demote under the write lock (#649) so any
|
|
# concurrent in-flight GPU requests finish before the model is moved to CPU,
|
|
# and put the runtime back before allowing new requests to start.
|
|
with self._gate.write_lock(), _OOM_FALLBACK_LOCK:
|
|
# Recorded on entry, under the same lock as the demotion: the count and
|
|
# reason must be readable by /health without racing a concurrent fallback.
|
|
self.cpu_fallback_count += 1
|
|
self.last_fallback_reason = str(e)
|
|
if self.device.type == "cpu":
|
|
return run()
|
|
held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled
|
|
held_backend = self._backend
|
|
had_fast = self._fast is not None and held_backend is None
|
|
# Only our batch scope can retain BF16 copies after this failed forward.
|
|
# Release them before moving the model and retrying on CPU.
|
|
if self.device.type == "cuda" and _BATCH_AUTOCAST_CACHE.get():
|
|
torch.clear_autocast_cache()
|
|
# FastLaya keeps copied CUDA weights and replaces model.forward. Move
|
|
# the model first without that replacement, or the retry would still
|
|
# execute on the failed CUDA fast path.
|
|
self.deaccelerate()
|
|
self.device = torch.device("cpu")
|
|
self.dtype = torch.float32
|
|
self.amp_enabled = False
|
|
self.model.to(self.device)
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
try:
|
|
return run()
|
|
finally:
|
|
self._restore_runtime(held_device, held_dtype, held_amp, had_fast)
|
|
if held_backend is not None and self.device == held_device:
|
|
from .backends import BackendUnavailable, warn_fallback
|
|
try:
|
|
held_backend.install()
|
|
except BackendUnavailable as exc:
|
|
warn_fallback(held_backend.name, exc)
|
|
else:
|
|
self._backend = held_backend
|
|
self._fast = getattr(held_backend, "fast", None)
|
|
if use_amp and self.device.type in ("mps", "cpu"):
|
|
# Not every MPS/CPU build implements autocast for every op. Retry this
|
|
# request in full precision. One miss must not turn AMP off; a build that
|
|
# lacks the op fails the same way every time, so after a short streak the
|
|
# process drops to full precision instead of paying for two forwards (#351).
|
|
with self._gate.read_lock():
|
|
try:
|
|
return run(enabled=False)
|
|
finally:
|
|
self._amp_failures += 1
|
|
if self._amp_failures >= _AMP_FAIL_LIMIT:
|
|
warnings.warn("Warning: autocast failed %d times in a row. "
|
|
"Disabling mixed precision." % self._amp_failures,
|
|
RuntimeWarning)
|
|
self.amp_enabled = False
|
|
self.dtype = torch.float32
|
|
raise
|
|
else:
|
|
self._amp_failures = 0
|
|
return out
|
|
|
|
def _forward(self, b: Dict):
|
|
"""Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
|
|
logits, act = self._infer(b)
|
|
return logits.float().cpu().numpy(), torch.softmax(act.float(), -1).cpu().numpy()
|
|
|
|
def _decode_answers(self, logits, act, items: List[Dict], ids: List[str],
|
|
internal: Dict[str, Dict], offset: int, lang: Optional[str] = None) -> Dict[str, Any]:
|
|
"""Turn one state's logit rows (starting at `offset`) into typed answers."""
|
|
answers = {}
|
|
raw_rows = _option_logits(logits, items, offset)
|
|
for j, qid in enumerate(ids):
|
|
r = offset + j
|
|
q = internal[qid]
|
|
k = len(items[j]["markers"])
|
|
qt = QTYPES[q["t"]]
|
|
t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])
|
|
if lang and lang.split("-")[0].lower() in self.lang_temperatures:
|
|
l_cfg = self.lang_temperatures[lang.split("-")[0].lower()]
|
|
t_scale = l_cfg["temperature_by_options"].get(temp_bucket(qt, k), l_cfg["temperature"][qt])
|
|
z = raw_rows[j] / t_scale
|
|
p = np.exp(z - z.max())
|
|
p = p / p.sum()
|
|
|
|
# The row comes back in slot order; everything below indexes by option.
|
|
p = unpermute_probs(p, q.get("option_order"))
|
|
|
|
# `confidence` means one thing for `noul` (max(p)) and another for `choice` and
|
|
# `score` (normalized entropy), and only the first is the quantity temperature
|
|
# scaling fits and ECE measures. Rather than change one underneath existing
|
|
# callers, report both: `answer_confidence` is the calibrated one, on every
|
|
# question type, so a caller can gate across types on a single number.
|
|
ans_raw = answer_confidence(p, k)
|
|
lang_override = bool(lang and lang.split("-")[0].lower() in self.lang_temperatures)
|
|
if getattr(self, "binning_map", None) and not lang_override:
|
|
ans_conf = round(
|
|
apply_binning_map(ans_raw, temp_bucket(qt, k), self.binning_map), 4
|
|
)
|
|
else:
|
|
ans_conf = round(ans_raw, 4)
|
|
ext = {"act_probability": round(float(act[r, 0]), 4)}
|
|
|
|
if q["t"] == "choice":
|
|
keys = list(q["crit"].keys())
|
|
answers[qid] = {
|
|
"type": "choice",
|
|
"choice": keys[int(p.argmax())],
|
|
"probabilities": {kk: round(float(v), 4) for kk, v in zip(keys, p)},
|
|
"confidence": round(confidence_from_probs(p, k), 4),
|
|
"answer_confidence": ans_conf,
|
|
"action": ext,
|
|
}
|
|
elif q["t"] == "score":
|
|
exp_score = float((np.arange(k) * p).sum())
|
|
answers[qid] = {
|
|
"type": "score",
|
|
"score": round(exp_score, 4),
|
|
# A legend maps an index to the text of a level, and the keys are already
|
|
# strings. `render_criterion` rather than `str`: a dict or list level then
|
|
# comes back as the same JSON text the model was shown, where `str` produced
|
|
# a Python repr. A numeric scale passed as `[1, 2, 3]` used to come back as
|
|
# `{"0": 1, "1": 2, "2": 3}`, so the response's JSON types depended on what
|
|
# the caller happened to pass; `structured` stringifies every level it builds
|
|
# and `probabilities` stringifies its keys right below, so this was the one
|
|
# path that did not.
|
|
"legend": {str(i): render_criterion(c) for i, c in enumerate(q["crit"])},
|
|
"probabilities": {str(i): round(float(v), 4) for i, v in enumerate(p)},
|
|
"confidence": round(confidence_from_probs(p, k), 4),
|
|
"answer_confidence": ans_conf,
|
|
"action": ext,
|
|
}
|
|
else:
|
|
answers[qid] = {
|
|
"type": "noul",
|
|
"noul": round(float(p[1]), 4),
|
|
"confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),
|
|
# identical here: over two options max(p_true, 1 - p_true) is max(p)
|
|
"answer_confidence": ans_conf,
|
|
"action": ext,
|
|
}
|
|
return answers
|
|
|
|
@torch.no_grad()
|
|
@_reuse_question_tokens
|
|
def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],
|
|
batch_size: Optional[int] = None, lang: Optional[str] = None,
|
|
hooks=None,
|
|
on_predict_start=None, on_predict_end=None,
|
|
hooks_raise: Optional[bool] = None,
|
|
hooks_timeout: Optional[float] = None,
|
|
max_len: Optional[int] = None,
|
|
head_max_len: Optional[int] = None,
|
|
sort_by_length: bool = False,
|
|
min_confidence: Optional[float] = None) -> List[Dict[str, Any]]:
|
|
"""Evaluate the same questions over many states, packing them into shared forward passes.
|
|
|
|
This is the throughput path. `system_one`/`predict` handle one state per forward pass; on a
|
|
GPU that leaves most of the batch dimension idle. `predict_batch` collates several states'
|
|
question rows into one tensor, so a call that would take N sequential forward passes takes
|
|
one (or `ceil(len(states) / batch_size)`), which is several times faster per decision on GPU.
|
|
|
|
Args:
|
|
states: A list of states (each a text string, JSON dict, or conversation turn list).
|
|
The same `questions` are evaluated against every state.
|
|
questions: Question definitions, exactly as accepted by `system_one`.
|
|
batch_size: Optional cap on states per forward pass. `None` sends them all in one pass;
|
|
set it to bound peak memory when batching many or long states.
|
|
hooks (HookArg): Per-call hooks, appended after any installed on the Agent.
|
|
See `laya.hooks`.
|
|
on_predict_start (PredictHookArg): A per-call start hook. It may rewrite the
|
|
state/questions or call `ctx.skip(...)` to short-circuit inference.
|
|
on_predict_end (PredictHookArg): A per-call end hook. It may rewrite the results.
|
|
hooks_raise: Override the Agent's `hooks_raise` for this call.
|
|
hooks_timeout: Override the Agent's `hooks_timeout` for this call.
|
|
max_len: Override the agent config's `max_len` for this call. A start hook may also
|
|
set `ctx.max_len` to shape the token budget.
|
|
head_max_len: Override the agent config's `head_max_len` for this call. A start hook
|
|
may also set `ctx.head_max_len`.
|
|
sort_by_length: Group similarly sized encoded states within windows of eight batches
|
|
to reduce padding. Requires an explicit `batch_size` greater than one and
|
|
smaller than the number of states; otherwise it has no effect. Results retain
|
|
input order. This buffers up to eight batches of tokenized states instead of
|
|
one. Changing batch shapes can slightly change floating-point predictions.
|
|
|
|
Returns:
|
|
A list of per-state result dicts, each identical in shape to `system_one`'s output and
|
|
aligned with `states` by index.
|
|
"""
|
|
mc = check_min_confidence(min_confidence) if min_confidence is not None else None
|
|
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
|
|
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
|
|
timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
|
|
ctx = PredictContext(states=states, questions=questions, model=self.model_id, agent=self,
|
|
max_len=max_len, head_max_len=head_max_len)
|
|
amp_stack = ExitStack()
|
|
try:
|
|
dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
|
|
states, questions = ctx.states, ctx.questions
|
|
if ctx.results is None:
|
|
# A start hook may have normalised a bare string/dict into a list; only the value
|
|
# that survives the hook is validated.
|
|
if isinstance(states, (str, bytes, dict)):
|
|
raise TypeError(
|
|
"predict_batch expects a list of states; pass a single state to predict()/system_one()."
|
|
)
|
|
if not isinstance(questions, dict):
|
|
raise TypeError(
|
|
"questions must be a dict of question id -> definition, got %s"
|
|
% type(questions).__name__
|
|
)
|
|
states = list(states)
|
|
if any(state is None for state in states):
|
|
raise TypeError(
|
|
"state must not be None; pass a string, dict, or list"
|
|
)
|
|
if len(states) == 1:
|
|
_disable_question_token_reuse()
|
|
if not states:
|
|
ctx.results = []
|
|
else:
|
|
ids = list(questions.keys())
|
|
# Empty questions: empty answers, zero usage, no tokenization or forward.
|
|
if not ids:
|
|
ctx.results = [
|
|
{"model": "laya-rl-agent", "answers": {},
|
|
"usage": {"input_tokens": 0, "output_tokens": 0}} for _ in states
|
|
]
|
|
else:
|
|
# Validate + normalize each question once (state-independent).
|
|
for qid in ids:
|
|
self._check_question(qid, questions[qid])
|
|
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
|
|
chunk = batch_size if (batch_size and batch_size > 0) else len(states)
|
|
|
|
# Keep inner _infer weight casts cached across CUDA eager forwards.
|
|
# A disabled outer scope leaves encoding/decoding in their usual
|
|
# precision and inherits the caller's cache_enabled setting. If the
|
|
# caller already owns a CUDA autocast scope, its cache is sufficient.
|
|
if (chunk < len(states) and getattr(self, "device", None) is not None
|
|
and self.device.type == "cuda" and self.amp_enabled
|
|
and self._fast is None and not self._compiled and self.backend != "compile"
|
|
and not torch.is_autocast_enabled()):
|
|
amp_stack.enter_context(torch.autocast(device_type="cuda", dtype=self.dtype,
|
|
enabled=False))
|
|
amp_stack.callback(_BATCH_AUTOCAST_CACHE.reset, _BATCH_AUTOCAST_CACHE.set(True))
|
|
|
|
# Per-call token-budget overrides (a start hook may have set them).
|
|
overrides: Dict[str, int] = {}
|
|
if ctx.max_len is not None:
|
|
overrides["max_len"] = ctx.max_len
|
|
if ctx.head_max_len is not None:
|
|
overrides["head_max_len"] = ctx.head_max_len
|
|
|
|
results: List[Dict[str, Any]] = []
|
|
reorder = sort_by_length and 1 < chunk < len(states)
|
|
# Bound tokenized lookahead independently of the full input size. Use
|
|
# actual post-truncation lengths, with each state's questions kept together.
|
|
window = chunk * 8 if reorder else chunk
|
|
for start in range(0, len(states), window):
|
|
part = states[start:start + window]
|
|
encoded = [self._encode_state(st, ids, internal, **overrides) for st in part]
|
|
order = list(range(len(encoded)))
|
|
if reorder:
|
|
order.sort(key=lambda i: max(len(item["ids"]) for item in encoded[i]))
|
|
window_results = [None] * len(encoded)
|
|
for offset in range(0, len(order), chunk):
|
|
indices = order[offset:offset + chunk]
|
|
per_state_items = [encoded[i] for i in indices]
|
|
|
|
b = collate_items(per_state_items, self.tok.pad_token_id)
|
|
logits, act = self._forward(b)
|
|
att = b["attention_mask"]
|
|
|
|
row = 0
|
|
for index, items in zip(indices, per_state_items):
|
|
nrows = len(items)
|
|
n_tokens = int(att[row:row + nrows].sum())
|
|
answers = self._decode_answers(logits, act, items, ids, internal, row,
|
|
**({"lang": lang} if lang else {}))
|
|
# Truncation is a token budget that moves with max_len, head_max_len
|
|
# and each question's head, so only build_sequence knows it (#174).
|
|
stats = [item["state_stats"] for item in items]
|
|
dropped = max(s["state_tokens_dropped"] for s in stats)
|
|
usage = {
|
|
"input_tokens": n_tokens,
|
|
"output_tokens": 0,
|
|
"state_tokens": stats[0]["state_tokens"],
|
|
# worst case: the questions share one state, not one head budget
|
|
"state_tokens_dropped": dropped,
|
|
"truncated": dropped > 0,
|
|
"truncated_questions": [
|
|
qid for qid, s in zip(ids, stats) if s["truncated"]
|
|
],
|
|
}
|
|
# Only when a question actually lost options to the head
|
|
# budget: an answer chosen from 42 distinguishable spans of
|
|
# 58 has a ceiling the caller cannot otherwise see, and a
|
|
# key that is always present would be noise on the
|
|
# overwhelming majority of requests that never collapse.
|
|
collapsed = collapsed_options(ids, items)
|
|
if collapsed:
|
|
usage["options"] = collapsed
|
|
window_results[index] = {
|
|
"model": "laya-rl-agent",
|
|
"answers": answers,
|
|
"usage": usage,
|
|
}
|
|
row += nrows
|
|
results.extend(window_results)
|
|
ctx.results = results
|
|
except BaseException as exc:
|
|
amp_stack.close()
|
|
ctx.error = exc
|
|
try:
|
|
dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
|
|
except BaseException as hook_exc:
|
|
# A failing on_error hook must not hide the failure that triggered it.
|
|
exc.__context__ = hook_exc
|
|
raise
|
|
finally:
|
|
amp_stack.close()
|
|
ctx.elapsed_ms = (time.perf_counter() - ctx.started_at) * 1000.0
|
|
if ctx.results is not None:
|
|
ctx.usage = aggregate_usage(ctx.results)
|
|
apply_confidence_gate(ctx.results, mc)
|
|
try:
|
|
dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
|
|
except BaseException as hook_exc:
|
|
# End hooks run on the failure path too; do not let one mask the real error.
|
|
if ctx.error is not None:
|
|
ctx.error.__context__ = hook_exc
|
|
else:
|
|
raise
|
|
return ctx.results
|
|
|
|
def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
|
|
window: Optional[int] = None, stride: Optional[int] = None,
|
|
aggregate: str = "auto", batch_size: Optional[int] = None,
|
|
lang: Optional[str] = None,
|
|
hooks=None, on_predict_start=None, on_predict_end=None,
|
|
hooks_raise: Optional[bool] = None,
|
|
hooks_timeout: Optional[float] = None) -> Dict[str, Any]:
|
|
"""Evaluate questions over a state longer than the context window, scanning it in
|
|
overlapping windows and aggregating per question.
|
|
|
|
`system_one`/`predict` truncate a state that exceeds `max_len` to a single window (the
|
|
first, or for a conversation list the last), silently dropping the rest. `predict_long`
|
|
tokenizes the state once, splits it into overlapping token windows, scores every window in
|
|
shared forward passes (via `predict_batch`), and combines the per-window answers:
|
|
|
|
* noul -> P(true) is the max over windows (the statement holds if any window supports it)
|
|
* choice-> the answer from the single most-confident window, so a localized signal isn't
|
|
out-voted by the many neutral windows a long document is mostly made of
|
|
(averaging drowns it -- the neutral majority dominates)
|
|
* score -> the level from the most-confident window, likewise
|
|
|
|
The returned probability/confidence is the deciding window's, **not a calibrated number for
|
|
the whole document**: a `noul` max over many windows drifts up with the window count even
|
|
with no signal, and `choice` can land on a confidently-neutral window when nothing in the
|
|
document is decisive. Each answer therefore carries `answer["window"]` — the deciding
|
|
window's `index`, `token_start`/`token_end` into the tokenized state, and the window `count`
|
|
— so a caller can inspect the span the answer came from rather than trust the raw number.
|
|
That span is the one the model read, not merely the one asked for: the window is capped at
|
|
the room the questions leave, so what is handed to `predict_batch` is not cut short again.
|
|
|
|
A state the questions leave room for is passed straight to `system_one` (identical output),
|
|
since `system_one` reads it whole; with an explicit `window`, a state that fits that window.
|
|
|
|
The hooks wrap the inference that answers the state, which for a document needing several
|
|
windows is the one shared `predict_batch` over them: `on_predict_start` fires once, and
|
|
`ctx.states` holds the decoded window texts in scan order -- not the caller's `state`, which
|
|
was tokenized to produce them. Three outcomes follow from what the chain leaves behind:
|
|
|
|
* `ctx.skip([result])` answers the document: the payload comes back with no window
|
|
attribution and `usage["windows"]` at 0, because nothing was scored
|
|
* a scan left as this method built it: every window is scored, each answer carries
|
|
`answer["window"]`, and `usage["windows"]` is the window count
|
|
* a rewritten scan (`ctx.states` replaced, in any way): the answers are aggregated over
|
|
the states that were scored, but no answer carries `answer["window"]` -- the offsets
|
|
above describe this method's windows, not the text the model read
|
|
|
|
Args:
|
|
window: state tokens per window. Defaults to the checkpoint's state budget,
|
|
`max(64, max_len - head_max_len - 8)` -- the 64 is a floor, so widening
|
|
`head_max_len` stops shrinking the default once the budget reaches it -- and
|
|
either way is capped at the room the questions leave for the state inside
|
|
`max_len` -- the smallest room of them,
|
|
because one list of windows is scored for every question. A wider window is
|
|
re-truncated on the way to the model, so it is clamped instead, with a
|
|
`RuntimeWarning` when the caller is the one who asked for it. Options are what
|
|
make the room small: on the English checkpoint a 2-option question leaves 483
|
|
tokens for the state and a 100-option one leaves 100.
|
|
A smaller window isolates a localized signal better (a short deciding span is a
|
|
larger fraction of its window, so that window classifies it clearly), at the
|
|
cost of more windows; the large default favors context and throughput. `noul`
|
|
is robust to this, `choice`/`score` benefit from a smaller window when the
|
|
deciding span is a small part of a long, otherwise-neutral document.
|
|
stride: token step between windows. Defaults to half the *effective* window (50%
|
|
overlap), so a span near a boundary still lands whole inside some window. A
|
|
stride past the effective window is refused rather than clamped: the tokens
|
|
between each pair of windows would be read by no window at all, which is the
|
|
failure this method exists to prevent.
|
|
aggregate: "auto" (the per-type rules above) is the only mode for now.
|
|
batch_size: cap on windows per forward pass, to bound memory on very long states.
|
|
lang: per-language temperature selection, as in `system_one`.
|
|
hooks (HookArg): Per-call hooks, appended after any installed on the Agent.
|
|
See `laya.hooks`.
|
|
on_predict_start (PredictHookArg): A per-call start hook, as in `system_one`.
|
|
on_predict_end (PredictHookArg): A per-call end hook, as in `system_one`.
|
|
hooks_raise: Override the Agent's `hooks_raise` for this call.
|
|
hooks_timeout: Override the Agent's `hooks_timeout` for this call.
|
|
|
|
Raises:
|
|
ValueError: `aggregate` is anything but "auto"; the questions' options fill the whole
|
|
sequence, leaving no room for the state; or `stride` steps past the effective
|
|
window, so tokens between two windows would be read by nothing.
|
|
|
|
Returns a single result dict, the same shape as `system_one`, with `usage["windows"]` added.
|
|
The key is always present and counts the windows the model scored to produce the answer: `1`
|
|
for a state that fit one window, `N` for a document scanned in `N` overlapping windows (or
|
|
the `N` a start hook rewrote them to), and `0` when a start hook answered the document, or
|
|
left no states to score, before any window was read -- on either path, so a cached answer
|
|
never reads as a window the model read.
|
|
|
|
Across several windows the truncation keys are combined like every other `usage` field:
|
|
`truncated`, `state_tokens` and `state_tokens_dropped` are summed (so `truncated` is the
|
|
number of windows that were cut, and the token counts include the overlap), and
|
|
`truncated_questions` is the last window's list. The two can disagree: when only an
|
|
earlier window was cut, `truncated` is above 0 and `truncated_questions` is empty. A
|
|
window is cut when it is larger than the room a question's head leaves. Test
|
|
`usage["truncated"] > 0` here, not `is True`.
|
|
"""
|
|
if state is None:
|
|
raise TypeError("state must not be None; pass a string, dict, or list")
|
|
if not isinstance(questions, dict):
|
|
raise TypeError("questions must be a dict of question id -> definition, got %s"
|
|
% type(questions).__name__)
|
|
if aggregate != "auto":
|
|
raise ValueError("predict_long: only aggregate='auto' is supported")
|
|
hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start,
|
|
"on_predict_end": on_predict_end, "hooks_raise": hooks_raise,
|
|
"hooks_timeout": hooks_timeout}
|
|
max_len = self.cfg.get("max_len", 512)
|
|
head_max_len = self.cfg.get("head_max_len", 192)
|
|
# Size the window against the room these questions actually leave, not the config alone:
|
|
# every window is decoded and scored as a normal state, so a window wider than the room
|
|
# was re-truncated by `build_sequence` on the way in and the tail of it reached no model
|
|
# -- while `answer["window"]` reported the whole span. Measured on the English checkpoint
|
|
# (max_len=512, head_max_len=192, config budget 312): a 2-option question leaves room for
|
|
# 483 tokens, 48 options leave 308, and 100 -- `serve`'s documented maximum -- leave 100.
|
|
# At 87 options the room (152) fell below the 156-token default stride, so the windows
|
|
# stopped overlapping and part of the document was read by no window at all: measured on a
|
|
# 920-token document at 100 options, 420 of its tokens reached no window.
|
|
ids = list(questions.keys())
|
|
for qid in ids:
|
|
self._check_question(qid, questions[qid])
|
|
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
|
|
budget, step, room = window_budget(self.tok, [internal[qid] for qid in ids], max_len,
|
|
head_max_len, window=window, stride=stride)
|
|
# Snapshot the questions the scan was just sized against, BEFORE any start hook can
|
|
# rewrite them, and compare against this instead of `questions` itself. `==` over the
|
|
# mapping cannot see an in-place rewrite: a hook that adds options to
|
|
# `ctx.questions[q]["criteria"]` -- the `widen_for_high_cardinality` pattern in
|
|
# `docs/hooks/patterns.md`, and the case this guard exists for -- mutates the same nested
|
|
# dict the caller's mapping holds, so both sides change together and compare equal. A deep
|
|
# copy is independent, so `_check_scan_budget` sees the difference; a shallow one would
|
|
# share the nested dicts and miss it exactly as before.
|
|
asked = copy.deepcopy(questions)
|
|
|
|
state_ids = encode_text(self.tok, serialize_state(state).replace(self.tok.mask_token, " "),
|
|
add_special_tokens=False)["input_ids"]
|
|
# Fits in one window: identical to a plain call, no windowing overhead. `windows` is still
|
|
# written, so the key is total over the three paths this method can take and a caller can
|
|
# ask "how much of the document did the model read?" without handling a KeyError on the
|
|
# shortest, most common inputs. The result is copied first: a start hook that answers with
|
|
# `ctx.skip(...)` hands back its own payload dict, and it may be a cached object.
|
|
# "Fits" is the room the questions leave, not the default window: `system_one` reads a
|
|
# state up to that room whole, so windowing one between the two only re-reads it in pieces
|
|
# and lets the per-window max inflate the answer. An explicit `window` still scans.
|
|
if len(state_ids) <= (budget if window and window > 0 else room):
|
|
probe, evidence = _start_evidence()
|
|
single = dict(self.system_one(state, questions, lang=lang,
|
|
**_with_start_probe(hook_kwargs, probe)))
|
|
# Why 0 for a hook answer here: the state did fit one window, but no window was scored,
|
|
# which is the same fact the multi-window path reports as 0. Reading 1 would make a
|
|
# cached answer and a served answer agree on how much of the input the model saw.
|
|
# The same budget check as the multi-window path. Without it a document short enough to
|
|
# fit one window was silently truncated by a re-budgeting hook and still reported
|
|
# `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens
|
|
# never reached the model, while a longer document on the identical input hard-failed.
|
|
# Sized for the whole state when it is longer than the window, so a hook that narrows
|
|
# the room below it is refused rather than cutting its tail.
|
|
_check_scan_budget(self, evidence, max(budget, len(state_ids)), max_len, head_max_len,
|
|
asked)
|
|
single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
|
|
return single
|
|
|
|
windows, starts = [], []
|
|
i, n = 0, len(state_ids)
|
|
while i < n:
|
|
# Decode each token window back to text so predict_batch re-tokenizes it as a normal
|
|
# state. For BPE tokenizers the re-tokenized boundaries can shift by a token or two vs
|
|
# this split; harmless for aggregation since the 50% default overlap absorbs it.
|
|
windows.append(self.tok.decode(state_ids[i:i + budget]))
|
|
starts.append(i)
|
|
if i + budget >= n:
|
|
break
|
|
i += step
|
|
|
|
probe, evidence = _start_evidence()
|
|
# `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that
|
|
# mutates it in place (`append`, `sort`) would otherwise also grow the split this call
|
|
# attributes answers to, and the counts would agree while `starts` no longer lined up.
|
|
# Bound one forward pass to what the un-capped scan would have used; see
|
|
# `window_batch_cap`. An explicit batch_size is honoured untouched.
|
|
cap = window_batch_cap(len(windows), budget, max(64, max_len - head_max_len - 8),
|
|
batch_size)
|
|
results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,
|
|
**_with_start_probe(hook_kwargs, probe))
|
|
_check_scan_budget(self, evidence, budget, max_len, head_max_len, asked)
|
|
|
|
if evidence["answered"]:
|
|
# The hook replaced the call before any window was scored. Aggregating over its payload
|
|
# would pick between answers that were never scored and name a deciding window that
|
|
# decided nothing, so pass the document answer through unattributed and report the
|
|
# count as what it is: none of them.
|
|
if len(results) != 1:
|
|
raise ValueError(
|
|
"predict_long: a start hook answered this state with %d results; ctx.skip()"
|
|
" takes one result for the document, not one per window" % len(results))
|
|
warnings.warn("laya: predict_long: a hook answered the state before it was scanned, so "
|
|
"no window decided the result and none is reported", RuntimeWarning,
|
|
stacklevel=2)
|
|
document = dict(results[0])
|
|
document["usage"] = dict(document.get("usage") or {})
|
|
document["usage"]["windows"] = 0
|
|
return document
|
|
|
|
# The contract on `ctx.states` is that a start hook may replace it (see `docs/hooks/api.md`),
|
|
# so a scan that comes back different from the split above is a supported outcome, not a
|
|
# failure to report: the states that were scored are the hook's, while `starts` describes
|
|
# this method's windows. Aggregate what came back and name nothing.
|
|
rewritten = evidence["states"] is not None and evidence["states"] != windows
|
|
|
|
if len(results) != len(windows) and not rewritten:
|
|
# The observer saw the scan leave the hook chain and it is the one computed above, so
|
|
# nothing here explains a count that is not the split. That is `predict_batch`
|
|
# disagreeing with its own input -- a bug, or a replacement that never dispatched hooks.
|
|
seen = ("the scan that reached inference was the split made here"
|
|
if evidence["states"] is not None
|
|
else "no start hook chain ran, so nothing rewrote the scan")
|
|
raise ValueError(
|
|
"predict_long: the state was split into %d windows and the call returned %d"
|
|
" results, and %s" % (len(windows), len(results), seen))
|
|
|
|
if not results:
|
|
# A hook that left no states scored nothing, which is what 0 already means here; the
|
|
# questions go unanswered rather than being aggregated over an empty list.
|
|
warnings.warn("laya: predict_long: a start hook left no states to score, so the call"
|
|
" aggregated nothing and returns no answers", RuntimeWarning, stacklevel=2)
|
|
return {"model": "laya-rl-agent", "answers": {},
|
|
"usage": {**aggregate_usage(results), "windows": 0}}
|
|
|
|
question_types = evidence["question_types"]
|
|
if question_types is None: # a replacement predict_batch may not dispatch hooks
|
|
question_types = {qid: self._to_internal(qdef)["t"] for qid, qdef in questions.items()}
|
|
answers = {}
|
|
for qid, qtype in question_types.items():
|
|
per = [r["answers"][qid] for r in results]
|
|
if qtype == "noul":
|
|
# Evidence anywhere: the strongest window decides. Its own P(true) and confidence
|
|
# (and act) are carried through, so the fields stay mutually consistent.
|
|
best = max(range(len(per)), key=lambda j: float(per[j]["noul"]))
|
|
else:
|
|
# choice / score: the most-confident window wins. Averaging over a long, mostly
|
|
# neutral document lets the neutral majority out-vote the one window that saw the
|
|
# deciding span; the single most-confident window preserves a localized signal.
|
|
best = max(range(len(per)), key=lambda j: float(per[j]["answer_confidence"]))
|
|
ans = per[best]
|
|
# Name the window that decided, so a caller can check the deciding span itself. The
|
|
# probability here is that window's, NOT a document-level calibrated number.
|
|
if not rewritten:
|
|
ans["window"] = {"index": best, "token_start": starts[best],
|
|
"token_end": min(starts[best] + budget, len(state_ids)),
|
|
"count": len(results)}
|
|
answers[qid] = ans
|
|
# Aggregate usage generically so fields predict_batch may grow later (e.g. the
|
|
# fallback counters from #351) are propagated, not silently dropped: sum numeric
|
|
# fields across windows, merge the per-question records, then record the window
|
|
# count.
|
|
# A per-question field has to be merged rather than replaced. `usage["options"]` is a
|
|
# dict keyed by question id, set only on the windows where option spans actually
|
|
# collapsed, so replacing it left the caller holding whichever collapsing window came
|
|
# last. The deciding window is the most confident one, not the last one, so that could
|
|
# report a collapse for a window that did not decide while the deciding window's own
|
|
# record was gone.
|
|
usage: Dict[str, Any] = {}
|
|
for r in results:
|
|
for key, val in r["usage"].items():
|
|
prev = usage.get(key)
|
|
if isinstance(val, (int, float)):
|
|
usage[key] = (prev if isinstance(prev, (int, float)) else 0) + val
|
|
elif isinstance(val, dict) and isinstance(prev, dict):
|
|
usage[key] = {**prev, **val}
|
|
else:
|
|
usage[key] = val
|
|
usage["output_tokens"] = 0
|
|
usage["windows"] = len(results)
|
|
result = {"model": "laya-rl-agent", "answers": answers, "usage": usage}
|
|
# `predict_long` accepts no `min_confidence` -- the window loop has no single confidence to
|
|
# gate on. The gate is still called, per its contract in laya/confidence.py: with
|
|
# `min_confidence` None it writes nothing, so the payload comes back exactly as the
|
|
# windows built it -- no `abstention`, no `abstention_threshold`, no flag.
|
|
apply_confidence_gate([result], None)
|
|
return result
|
|
|
|
@torch.no_grad()
|
|
def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]], lang: Optional[str] = None,
|
|
hooks=None, on_predict_start=None, on_predict_end=None,
|
|
hooks_raise: Optional[bool] = None,
|
|
hooks_timeout: Optional[float] = None,
|
|
max_len: Optional[int] = None,
|
|
head_max_len: Optional[int] = None,
|
|
min_confidence: Optional[float] = None) -> Dict[str, Any]:
|
|
"""Evaluate typed questions across state in a single, parallel forward pass.
|
|
|
|
Args:
|
|
state: Text string, JSON dict, or conversation turn list.
|
|
questions: Dictionary mapping question_id -> question definition.
|
|
- choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}
|
|
- score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}
|
|
- noul: {"type": "noul", "instructions": "...",
|
|
"criteria": {"false": "...", "true": "..."},
|
|
"labels": {"false": "B", "true": "A"}}
|
|
|
|
Noul criteria and labels are optional. Labels only control the text shown to the
|
|
model; their keys retain false/true semantics, and the returned `noul` value is
|
|
always P(true). Labels default to false/true for compatibility.
|
|
|
|
Returns:
|
|
Dictionary with answers, probabilities, calibrated confidence, and token usage.
|
|
Empty questions return empty answers and zero token usage without tokenization
|
|
or a model forward pass.
|
|
|
|
When the head budget leaves two options with the same token span, `usage` carries
|
|
an `options` entry for each question it happened to -- `total`, `distinct` and
|
|
`tokens_per_option` -- because an answer chosen among 42 distinguishable spans of
|
|
58 has a ceiling that is the budget's and not the model's. Questions whose options
|
|
all survive are absent, so a request that collapses nothing is unchanged.
|
|
|
|
`usage` also reports whether the state fit: `truncated`, `state_tokens`,
|
|
`state_tokens_dropped`, and `truncated_questions` (the questions whose head left
|
|
too little room). A caller that cares whether the answer saw the whole state should
|
|
read `usage["truncated"]` rather than estimate from the length of what it sent.
|
|
|
|
To score many states at once, see `predict_batch`, which shares forward passes across them.
|
|
"""
|
|
return self.predict_batch([state], questions, lang=lang, hooks=hooks,
|
|
on_predict_start=on_predict_start,
|
|
on_predict_end=on_predict_end, hooks_raise=hooks_raise,
|
|
hooks_timeout=hooks_timeout,
|
|
max_len=max_len, head_max_len=head_max_len,
|
|
min_confidence=min_confidence)[0]
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
if hasattr(self, "model") and self.model is not None:
|
|
del self.model
|
|
self.model = None
|
|
try:
|
|
import gc
|
|
gc.collect()
|
|
import torch
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
if hasattr(torch, "xpu") and torch.xpu.is_available():
|
|
torch.xpu.empty_cache()
|
|
except Exception:
|
|
pass
|
|
return False
|
|
|
|
def decide(self, state: Union[str, dict, list], schema: Any = None, *,
|
|
questions: Optional[Dict[str, Any]] = None, return_details: bool = False,
|
|
min_confidence: Optional[float] = None,
|
|
**predict_kwargs) -> Any:
|
|
"""Answer `state` against a schema (JSON schema or pydantic model) and return typed values.
|
|
|
|
See `laya.structured`. Pass exactly one of `schema` or `questions`; extra keyword arguments
|
|
are forwarded to `predict` / `system_one`.
|
|
"""
|
|
from .structured import decide as _decide
|
|
return _decide(self, state, schema, questions=questions,
|
|
return_details=return_details, min_confidence=min_confidence, **predict_kwargs)
|
|
|
|
def decide_batch(self, states: List[Union[str, dict, list]], schema: Any = None, *,
|
|
questions: Optional[Dict[str, Any]] = None,
|
|
return_details: bool = False, min_confidence: Optional[float] = None,
|
|
**predict_kwargs) -> List[Any]:
|
|
"""Answer many states against one schema (JSON schema or pydantic model) in one batched call.
|
|
|
|
The throughput form of :meth:`decide`: the schema is planned once and its questions
|
|
run over every state through :meth:`predict_batch` (shared forward passes, results
|
|
in input order), then each state's answers are projected as ``decide`` does. Extra
|
|
keyword arguments (``batch_size=``, ``lang=``, ``hooks=``, ...) are forwarded to
|
|
``predict_batch``. See `laya.structured`.
|
|
"""
|
|
from .structured import decide_batch as _decide_batch
|
|
return _decide_batch(self, states, schema, questions=questions,
|
|
return_details=return_details, min_confidence=min_confidence,
|
|
**predict_kwargs)
|
|
|
|
def __repr__(self) -> str:
|
|
return "Agent(model_id=%r, device=%s)" % (self.model_id, getattr(self, "device", None))
|
|
|
|
predict = system_one
|
|
|
|
def fit_temperatures(self, records, compute_ece: bool = False, seed: int = 0) -> Dict[str, Any]:
|
|
"""Fit per-bucket temperatures from CPU records and store them on this agent.
|
|
|
|
`records` are `(qtype, logits, target, k)`. Build them with
|
|
`laya.calibrate.records_from_labeled` when you have labeled forwards; this method
|
|
does not download weights or write `model.safetensors`. `seed` only affects the
|
|
held-out ECE split when `compute_ece` is true. The checkpoint `cfg` is left as loaded.
|
|
"""
|
|
result = fit_temperature_map(records, compute_ece=compute_ece, seed=seed)
|
|
# Already clamped inside the fitter; don't report that as a bad calibration file.
|
|
_install_temperatures(self, result["temperature"], result["temperature_by_options"], warn=False)
|
|
return result
|
|
|
|
def fit_binning(self, records, min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Any]:
|
|
"""Fit a histogram-binning map on top of this agent's fitted temperatures and store it.
|
|
|
|
`records` are the same `(qtype, logits, target[, k])` tuples as `fit_temperatures`
|
|
consumed. The map is keyed exactly like `temperature_by_options`, composes on top of
|
|
the current temperatures, and is written out by `save_calibration` as `binning_map`.
|
|
"""
|
|
self.binning_map = fit_binning_map(
|
|
records, self.temperature, self.temperature_by_options, min_bucket_n=min_bucket_n
|
|
)
|
|
return self.binning_map
|
|
|
|
def save_calibration(self, path: str) -> None:
|
|
"""Write `temperature`, `temperature_by_options`, `binning_map` when the agent has one, and
|
|
the checkpoint they were fitted for. Does not write weights."""
|
|
payload = calibration_payload(
|
|
self.temperature,
|
|
self.temperature_by_options,
|
|
model_id_or_path=getattr(self, "model_id_or_path", None),
|
|
subfolder=getattr(self, "subfolder", None),
|
|
config=getattr(self, "cfg", None),
|
|
binning_map=getattr(self, "binning_map", None),
|
|
)
|
|
destination = os.path.realpath(path)
|
|
fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination), prefix=".calibration.", suffix=".tmp")
|
|
try:
|
|
with os.fdopen(fd, "w") as f:
|
|
json.dump(payload, f, indent=2)
|
|
f.write("\n")
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
try:
|
|
mode = os.stat(destination).st_mode & 0o777
|
|
except FileNotFoundError:
|
|
pass
|
|
else:
|
|
os.chmod(temporary, mode)
|
|
os.replace(temporary, destination)
|
|
except BaseException:
|
|
try:
|
|
os.unlink(temporary)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
def load_calibration(self, path: str) -> None:
|
|
"""Read a JSON map written by `save_calibration` onto this agent.
|
|
|
|
The file carries three fields and all three are installed: `temperature`,
|
|
`temperature_by_options` and `binning_map`. A file with no `binning_map` key installs
|
|
`None`, so loading one clears a map this agent's `fit_binning` fitted -- the file is the
|
|
whole calibration state, not a patch onto the current one.
|
|
|
|
A file with no `version` is treated as version 1 and still loads. A newer file
|
|
whose recorded checkpoint does not match this agent warns and still loads.
|
|
Temperatures that are not numbers, or that sit outside `[TEMP_MIN, TEMP_MAX]`, are
|
|
clamped with `clamp_temperature` the same way checkpoint load is; binning values are
|
|
not clamped, they are refused with a `ValueError` naming the field, because an
|
|
out-of-range binning value would move a confidence with nothing to fall back to.
|
|
"""
|
|
with open(path) as f:
|
|
payload = json.load(f)
|
|
apply_calibration_payload(self, payload)
|
|
|
|
|
|
RLAgent = Agent
|
|
|
|
|
|
def _is_local_checkpoint_arg(model_id_or_path: str) -> bool:
|
|
"""True when `model_id_or_path` should be treated as a local path, not a registry name.
|
|
|
|
A bare word with no path separator reads as a name/alias. Only treat it as a path when
|
|
it looks like one (contains a separator, or starts with `.` / `~`) or when the named
|
|
directory actually holds a Laya checkpoint (`rl_agent_config.json`). That way
|
|
`load("laya")` resolves the alias from the repo root, while `load("./laya")` and a
|
|
real checkpoint directory still load from disk.
|
|
"""
|
|
if not model_id_or_path:
|
|
return False
|
|
if model_id_or_path.startswith((".", "~")):
|
|
return True
|
|
if os.sep in model_id_or_path or "/" in model_id_or_path or "\\" in model_id_or_path:
|
|
return True
|
|
return os.path.isdir(model_id_or_path) and os.path.isfile(
|
|
os.path.join(model_id_or_path, "rl_agent_config.json")
|
|
)
|
|
|
|
|
|
if TYPE_CHECKING: # pragma: no cover - the annotation below is the only use
|
|
from .onnx_agent import ONNXAgent
|
|
|
|
|
|
def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str] = None,
|
|
token: Optional[str] = None, subfolder: Optional[str] = None, fast: bool = False,
|
|
compile: bool = False,
|
|
revision: Optional[str] = None, expected_sha256: Optional[Dict[str, str]] = None,
|
|
lang_temperatures: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
hooks=None, on_predict_start=None, on_predict_end=None,
|
|
hooks_raise: bool = True, hooks_concurrent: bool = True,
|
|
hooks_timeout: Optional[float] = None,
|
|
calibration: Optional[str] = None, backend: Optional[str] = None,
|
|
onnx_path: Optional[str] = None, compile_warmup: bool = True,
|
|
compile_cache: bool = False, compile_mode: str = "default") -> Union[Agent, "ONNXAgent"]:
|
|
"""Load a Laya agent.
|
|
|
|
`subfolder` picks one checkpoint out of a repo that bundles several:
|
|
|
|
laya.load("convaiinnovations/laya") # English (repo root)
|
|
laya.load("convaiinnovations/laya", subfolder="multilingual")
|
|
laya.load("convaiinnovations/laya", fast=True) # TileLang GPU fast path
|
|
laya.load("convaiinnovations/laya", compile=True) # torch.compile the model
|
|
|
|
`model_id_or_path` also accepts a checkpoint name or alias -- the same ones
|
|
`Router` resolves, so both entry points read one table:
|
|
|
|
laya.load("typed-decisions")
|
|
laya.load("ml") # multilingual
|
|
|
|
Anything else (a Hub repo id, a local directory) is passed to `Agent` unchanged.
|
|
|
|
`backend` selects "auto", "eager", "compile", "tilelang" or "onnx". ONNX returns
|
|
the existing `ONNXAgent`, with `onnx_path` (default "laya.onnx").
|
|
Other backends use `Agent`; an explicit backend takes precedence over the legacy flags.
|
|
|
|
`revision`/`expected_sha256` pin and verify the downloaded artifacts; see `Agent`.
|
|
`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see
|
|
`laya.hooks`. `calibration` is the same optional JSON path accepted by `Agent`.
|
|
"""
|
|
# A registry name or alias resolves to the same (repo, subfolder) the Router would
|
|
# pick, instead of being handed to the Hub as a repo id. A caller who spelled out a
|
|
# subfolder is loading that subfolder of whatever repo they named, so the name is only
|
|
# resolved when it is the whole argument. Treat the argument as a local path only when
|
|
# it looks like one (separator, or starts with `.`/`~`) or when the directory actually
|
|
# holds a checkpoint (`rl_agent_config.json`); a bare word like `laya` therefore
|
|
# resolves the alias even when a same-named package directory sits in the cwd.
|
|
if subfolder is None and not _is_local_checkpoint_arg(model_id_or_path):
|
|
from .router import resolve_model_spec
|
|
|
|
spec = resolve_model_spec(model_id_or_path)
|
|
if spec is not None:
|
|
model_id_or_path, subfolder = spec
|
|
if backend is not None:
|
|
from .backends import normalise
|
|
backend = normalise(backend)
|
|
if backend == "onnx":
|
|
from .onnx_agent import ONNXAgent
|
|
return ONNXAgent(model_id_or_path, onnx_path=onnx_path or "laya.onnx",
|
|
token=token, subfolder=subfolder, revision=revision, expected_sha256=expected_sha256,
|
|
lang_temperatures=lang_temperatures, calibration=calibration,
|
|
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
|
|
hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent, hooks_timeout=hooks_timeout)
|
|
options = {"backend": backend} if backend is not None else {}
|
|
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,
|
|
compile=compile, compile_warmup=compile_warmup, compile_cache=compile_cache,
|
|
compile_mode=compile_mode,
|
|
revision=revision, expected_sha256=expected_sha256,
|
|
lang_temperatures=lang_temperatures,
|
|
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
|
|
hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent,
|
|
hooks_timeout=hooks_timeout, calibration=calibration, **options)
|