iLLuMinator-4.9B/config_2.8b.json at main · Anipaleja/iLLuMinator-4.9B · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
{
  "comments":{
    "//":"config for 2.8B param model"
  },
  "model": {
    "vocab_size": 50000,
    "d_model": 2560,
    "n_layers": 24,
    "n_heads": 20,
    "n_kv_heads": 5,
    "d_ff": 10240,
    "max_seq_length": 3072,
    "dropout": 0.0,
    "tie_embeddings": true,
    "use_rope": true,
    "use_swiglu": true,
    "use_rmsnorm": true,
    "rope_theta": 10000.0
  },
  "training": {
    "learning_rate": 0.00035,
    "weight_decay": 0.1,
    "beta1": 0.9,
    "beta2": 0.95,
    "eps": 1e-08,
    "max_grad_norm": 1.0,
    "warmup_steps": 2000,
    "max_steps": 80000,
    "batch_size": 2,
    "gradient_accumulation_steps": 32,
    "eval_every": 500,
    "save_every": 1000,
    "log_every": 100,
    "num_epochs": 5,
    "mixed_precision": true,
    "gradient_checkpointing": true
  },
  "data": {
    "max_seq_length": 2048,
    "dataset_size": 20000,
    "train_split": 0.9,
    "val_split": 0.1,
    "use_external_data": true,
    "quality_threshold": 0.8,
    "data_format": "alpaca",
    "cache_dir": "datasets"
  },
  "system": {
    "device": "auto",
    "num_workers": 4,
    "pin_memory": true,
    "compile_model": false,
    "use_wandb": false,
    "checkpoint_dir": "checkpoints",
    "log_dir": "logs"
  }
}