LoRA training: artifact 1.1GB -> ~20MB adapter
This commit is contained in:
parent
366f90d3af
commit
a5e747c807
@ -15,6 +15,7 @@ try:
|
||||
except Exception:
|
||||
from transformers import XLMRobertaTokenizer as XTok
|
||||
from transformers import XLMRobertaForSequenceClassification
|
||||
from peft import LoraConfig, get_peft_model
|
||||
|
||||
def load_jsonl(p):
|
||||
return [json.loads(l) for l in open(p, encoding="utf-8")]
|
||||
@ -105,6 +106,10 @@ def main():
|
||||
try:
|
||||
tok = XTok.from_pretrained(args.base)
|
||||
model = XLMRobertaForSequenceClassification.from_pretrained(args.base, num_labels=1).to(dev)
|
||||
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
|
||||
target_modules=["query", "key", "value"], task_type="SEQ_CLS")
|
||||
model = get_peft_model(model, lora)
|
||||
model.print_trainable_parameters()
|
||||
except Exception:
|
||||
import traceback
|
||||
tail = traceback.format_exc()[-1200:]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user