File size: 663 Bytes
3301578 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
model_name: "google/gemma-2-2b-it"
new_model_name: "gemma-2-2b-ft"
lora_r: 4
lora_alpha: 16
lora_dropout: 0.1
use_4bit: True
bnb_4bit_compute_dtype: "float16"
bnb_4bit_quant_type: "nf4"
use_nested_quant: False
output_dir: "./results"
num_train_epochs: 1
fp16: False
bf16: False
per_device_train_batch_size: 2
per_device_eval_batch_size: 2
gradient_accumulation_steps: 1
gradient_checkpointing: True
max_grad_norm: 0.3
learning_rate: 2e-4
weight_decay: 0.001
optimizer: "paged_adamw_32bit"
lr_scheduler_type: "constant"
max_steps: -1
warmup_ratio: 0.03
group_by_length: True
save_steps: 25
logging_steps: 25
max_seq_length: 40
packing: True
device_map: "auto"
|