explicit model classes + exc tail to termination-log

This commit is contained in:
dingbin.wang 2026-10-15 16:35:55 +08:00
parent 695ddc1466
commit 128ff308a5

View File

@ -9,7 +9,12 @@ import json, os, argparse, math, random
import torch
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_cosine_schedule_with_warmup
from transformers import AutoTokenizer, get_cosine_schedule_with_warmup
try:
from transformers import XLMRobertaTokenizerFast as XTok
except Exception:
from transformers import XLMRobertaTokenizer as XTok
from transformers import XLMRobertaForSequenceClassification
def load_jsonl(p):
return [json.loads(l) for l in open(p, encoding="utf-8")]
@ -76,8 +81,18 @@ def main():
dev = "cuda" if torch.cuda.is_available() else "cpu"
print(f"[jev] device={dev} base={args.base}", flush=True)
tok = AutoTokenizer.from_pretrained(args.base)
model = AutoModelForSequenceClassification.from_pretrained(args.base, num_labels=1).to(dev)
try:
tok = XTok.from_pretrained(args.base)
model = XLMRobertaForSequenceClassification.from_pretrained(args.base, num_labels=1).to(dev)
except Exception:
import traceback
tail = traceback.format_exc()[-1200:]
try:
with open("/dev/termination-log", "a") as f:
f.write("\nEXC-TAIL:\n" + tail)
except Exception:
pass
raise
train = [r for r in load_jsonl(os.path.join(args.data, "train.jsonl")) if r.get("hard_label") in (0, 1)]
calib = load_jsonl(os.path.join(args.data, "calibration.jsonl"))