andrew-healey/context-compression

1{"hellaswag": true, "attention_kind": "selective", "log_dir": "protection_none_torch_compile", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 2500, "group": "debugging_head_two_and_baselines", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1337, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": false, "seq_len": 1024, "batch_size": 8}