config.json

{
  "bias": false,
  "capacity_factor": 0.12,
  "d_model": 1024,
  "dropout": 0.2,
  "ffn": "swiglu",
  "hidden_dim": 4096,
  "mixture_of_depth": true,
  "mixture_of_expert": false,
  "model_type": {
    "mixture_of_depth": true,
    "name": "mixture of depth"
  },
  "moe_num_experts": 4,
  "moe_num_experts_per_tok": 2,
  "multiple_of": 4,
  "num_heads": 16,
  "num_kv_heads": 0,
  "num_layers": 16,
  "seq_len": 512,
  "vocab_size": 50257,
  "weight_tying": true,
  "window_size": 128
}