|
window_size
|
2048
|
|
vocab_size
|
49152
|
|
eos_token_id
|
49131
|
|
num_layers
|
26
|
|
num_heads
|
14
|
|
model_dim
|
1792
|
|
head_dim
|
128
|
|
max_seq_len
|
131072
|
|
model_spec
|
daisy_milli
|
|
model_class
|
models.daisy.daisy_core.DaisyCore
|
|
target_tokens
|
3442039096
|
|
full_window_target_tokens
|
3442039096
|
|
torch_coordinate_descent_tuning
|
False
|
|
torch_inductor_config_max_autotune
|
False
|
|
overfit
|
False
|
|
full_windows
|
True
|
|
wandb_log
|
True
|
|
wandb_project
|
milli
|
|
wandb_run_name
|
milli_v18_mid_v12
|
|
wandb_group
|
mid
|
|
init_model
|
JonathanMiddleton/daisy-milli-base-v18
|
|
use_value_embeddings
|
True
|
|
use_tied_embeddings
|
False
|
|
seed
|
1337
|
|
task_val_debug_log_samples
|
False
|
|
log_interval
|
16384
|
|
muon_warmup_steps
|
300
|
|
lr_scale
|
1.0
|
|
cooldown_fraction
|
0.0
|
|
lr_schedule
|
{"name": "n_phase_linear", "config": {"cooldown_fraction": 0.0, "phases": [{"progress": 0.0, "scale": 0}, {"progress": 0.183, "scale": 0.23237}, {"progress": 1.0, "scale": 0.1}], "floor": 0.0, "warmup_fraction": 0.0, "begin_after_fraction": 0.0}}
|
|
grad_acc_steps
|
4
|
|
val_loss_every_tokens
|
209715200
|
|
checkpoint_warmup_tokens
|
1000000000
|
|
checkpoint_per_n_tokens
|
0
|
|
save_checkpoint
|
True
|
|
benchmarks_frequency
|
2
|
|
mmlu_cache_bin_path
|
data/mmlu_cache/mmlu_cache.bin
|
|
mmlu_cache_bin_rebuild
|
False
|
|
task_training
|
False
|
|
track_last_n_layers
|
0
|