diff --git a/train_jev.py b/train_jev.py index a1c71d7..0afe0b1 100644 --- a/train_jev.py +++ b/train_jev.py @@ -15,6 +15,7 @@ try: except Exception: from transformers import XLMRobertaTokenizer as XTok from transformers import XLMRobertaForSequenceClassification +from peft import LoraConfig, get_peft_model def load_jsonl(p): return [json.loads(l) for l in open(p, encoding="utf-8")] @@ -105,6 +106,10 @@ def main(): try: tok = XTok.from_pretrained(args.base) model = XLMRobertaForSequenceClassification.from_pretrained(args.base, num_labels=1).to(dev) + lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, + target_modules=["query", "key", "value"], task_type="SEQ_CLS") + model = get_peft_model(model, lora) + model.print_trainable_parameters() except Exception: import traceback tail = traceback.format_exc()[-1200:]