mmBERT fresh-head config + full 4K decision data
This commit is contained in:
parent
d9cb063aa4
commit
e94dbb638b
84
laya_cfg_min/encoder/config.json
Normal file
84
laya_cfg_min/encoder/config.json
Normal file
@ -0,0 +1,84 @@
|
||||
{
|
||||
"architectures": [
|
||||
"ModernBertForMaskedLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 50281,
|
||||
"classifier_activation": "gelu",
|
||||
"classifier_bias": false,
|
||||
"classifier_dropout": 0.0,
|
||||
"classifier_pooling": "mean",
|
||||
"cls_token_id": 50281,
|
||||
"decoder_bias": true,
|
||||
"deterministic_flash_attn": false,
|
||||
"dtype": "float32",
|
||||
"embedding_dropout": 0.0,
|
||||
"eos_token_id": 50282,
|
||||
"global_attn_every_n_layers": 3,
|
||||
"gradient_checkpointing": false,
|
||||
"hidden_activation": "gelu",
|
||||
"hidden_size": 1024,
|
||||
"initializer_cutoff_factor": 2.0,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 2624,
|
||||
"layer_norm_eps": 1e-05,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"local_attention": 128,
|
||||
"max_position_embeddings": 8192,
|
||||
"mlp_bias": false,
|
||||
"mlp_dropout": 0.0,
|
||||
"model_type": "modernbert",
|
||||
"norm_bias": false,
|
||||
"norm_eps": 1e-05,
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 28,
|
||||
"pad_token_id": 50283,
|
||||
"position_embedding_type": "absolute",
|
||||
"repad_logits_with_grad": false,
|
||||
"rope_parameters": {
|
||||
"full_attention": {
|
||||
"rope_theta": 160000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_attention": {
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"sep_token_id": 50282,
|
||||
"sparse_pred_ignore_index": -100,
|
||||
"sparse_prediction": false,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.0.0",
|
||||
"vocab_size": 50368
|
||||
}
|
||||
11
laya_cfg_min/rl_agent_config.json
Normal file
11
laya_cfg_min/rl_agent_config.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"encoder": "/tmp/base",
|
||||
"head_layers": 2,
|
||||
"max_len": 512,
|
||||
"head_max_len": 192,
|
||||
"max_prefixes": 6,
|
||||
"act_costs": {},
|
||||
"cost_wrong_act": 3.0,
|
||||
"amp_dtype": "bf16",
|
||||
"model_name": "laya-mmbert-fromscratch"
|
||||
}
|
||||
251498
laya_cfg_min/tokenizer/tokenizer.json
Normal file
251498
laya_cfg_min/tokenizer/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
14
laya_cfg_min/tokenizer/tokenizer_config.json
Normal file
14
laya_cfg_min/tokenizer/tokenizer_config.json
Normal file
@ -0,0 +1,14 @@
|
||||
{
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"cls_token": "[CLS]",
|
||||
"mask_token": "[MASK]",
|
||||
"model_input_names": [
|
||||
"input_ids",
|
||||
"attention_mask"
|
||||
],
|
||||
"model_max_length": 8192,
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"tokenizer_class": "PreTrainedTokenizerFast",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user