Modelpublic

andrew-healey/context-compression

sourceHugging Faceupdated 2y agoView on Hugging Face

0likes

dir12_head_baseline_lr_10e-4_head_dim_22_fixed/

dir12_head_baseline_lr_11e-4/

dir12_head_baseline_lr_12e-4_head_dim_22_fixed/

dir12_head_baseline_lr_14e-4_head_dim_22_fixed/

dir12_head_baseline_lr_16e-4/

dir12_head_baseline_lr_16e-4_head_dim_22/

dir12_head_baseline_lr_16e-4_head_dim_22_fixed/

dir12_head_baseline_lr_20e-4_head_dim_22/

dir12_head_baseline_lr_20e-4_head_dim_22_fixed/

dir12_head_baseline_lr_25e-4_head_dim_22_fixed/

dir12_head_baseline_lr_30e-4_head_dim_22/

dir12_head_baseline_lr_30e-4_head_dim_22_fixed/

dir12_head_baseline_lr_35e-4_head_dim_22/

dir12_head_baseline_lr_40e-4_head_dim_22/

dir12_head_baseline_lr_45e-4_head_dim_22/

dir12_head_baseline_lr_50e-4_head_dim_22/

dir12_head_baseline_lr_6e-4/

dir12_head_baseline_lr_8e-4_head_dim_22_fixed/

dir12_head_one_mask_per_head_4_latent_masks/

dir12_mini_head_one_mask_per_head_1_latent_mask/

dir12_mini_head_one_mask_per_head_1_latent_mask_halved_lr/

dir12_mini_head_one_mask_per_head_2_latent_masks/

dir12_mini_head_two_masks_4_heads/

dir4x_smaller_bs_half_lr_half_seq_len/

dirallowing_more_selection_patterns/

diratt_conv_playground/

dirattention_kindselective_n_heads2_seed1338/

dirattention_kindselective_n_heads2_seed1339/

dirattention_kindselective_n_heads2_seed1340/

dirattention_kindselective_n_heads2_seed1341/

dirattention_kindselective_n_heads4_seed1338/

dirattention_kindselective_n_heads4_seed1339/

dirattention_kindselective_n_heads4_seed1340/

dirattention_kindselective_n_heads4_seed1341/

dirattention_kindselective_n_heads4_seed1342/

dirattention_kindselective_n_heads4_seed1343/

dirattention_kindselective_n_heads4_seed1344/

dirattention_kindselective_n_heads4_seed1345/

dirattention_kindselective_n_heads8_seed1338/

dirattention_kindselective_n_heads8_seed1339/

dirattention_kindselective_n_heads8_seed1340/

dirattention_kindselective_n_heads8_seed1341/

dirattention_kindself_n_heads2_seed1338/

dirattention_kindself_n_heads2_seed1339/

dirattention_kindself_n_heads2_seed1340/

dirattention_kindself_n_heads2_seed1341/

dirattention_kindself_n_heads4_seed1338/

dirattention_kindself_n_heads4_seed1339/

dirattention_kindself_n_heads4_seed1340/

dirattention_kindself_n_heads4_seed1341/

dirbaseline_lr_30e-4_seed_1339/

dirdense_attention_repro_old_codepath/

dirfix_128_heads_divergence/

dirhalf_total_bs_sqrt_lr/

dirlatent_masks_degenerate/

dirlatent_masks_learnable/

dirlr0.75e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective/

dirlr1.5e-3_total_batch_size20480_seq_len128/

dirlr1.5e-3_total_batch_size20480_seq_len512/

dirlr1.5e-3_total_batch_size40960_seq_len128/

dirlr1.5e-3_total_batch_size40960_seq_len512/

dirlr1.5e-3_total_batch_size5120_seq_len128/

dirlr1.5e-3_total_batch_size5120_seq_len512/

dirlr1.5e-4_total_batch_size10240_n_heads2_seed1338/

dirlr1.5e-4_total_batch_size10240_n_heads2_seed1339/

dirlr1.5e-4_total_batch_size10240_n_heads2_seed1340/

dirlr1.5e-4_total_batch_size10240_n_heads4_seed1338/

dirlr1.5e-4_total_batch_size10240_n_heads4_seed1340/

dirlr1.5e-4_total_batch_size5120_n_heads2_seed1338/

dirlr1.5e-4_total_batch_size5120_n_heads2_seed1339/

dirlr1.5e-4_total_batch_size5120_n_heads2_seed1340/

dirlr1.5e-4_total_batch_size5120_n_heads4_seed1338/

dirlr1.5e-4_total_batch_size5120_n_heads4_seed1339/

dirlr1.5e-4_total_batch_size5120_n_heads4_seed1340/

dirlr1.5e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective/

dirlr1.5e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps1/

dirlr1.75e-4_total_batch_size10240_n_heads2_seed1339/

dirlr1.75e-4_total_batch_size10240_n_heads2_seed1340/

dirlr1.75e-4_total_batch_size10240_n_heads4_seed1338/

dirlr1.75e-4_total_batch_size10240_n_heads4_seed1339/

dirlr1.75e-4_total_batch_size10240_n_heads4_seed1340/

dirlr1.75e-4_total_batch_size5120_n_heads2_seed1338/

dirlr1.75e-4_total_batch_size5120_n_heads2_seed1339/

dirlr1.75e-4_total_batch_size5120_n_heads2_seed1340/

dirlr1.75e-4_total_batch_size5120_n_heads4_seed1338/

dirlr1.75e-4_total_batch_size5120_n_heads4_seed1339/

dirlr1.75e-4_total_batch_size5120_n_heads4_seed1340/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps1_seed1338/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps1_seed1339/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps200_seed1338/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps200_seed1339/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps1/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps1_seed1338/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps1_seed1339/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps200_seed1338/

dirlr1.75e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps200_seed1339/

dirlr10e-4_total_batch_size61440_baseline_seed1338/

dirlr10e-4_total_batch_size61440_baseline_seed1339/

dirlr11e-4_total_batch_size122880_baseline_seed1339/

dirlr11e-4_total_batch_size122880_baseline_seed1340/

dirlr12e-4_total_batch_size61440_baseline_seed1338/

dirlr12e-4_total_batch_size61440_baseline_seed1339/

dirlr13e-4_total_batch_size122880_baseline_seed1339/

dirlr16e-4_total_batch_size61440_baseline_seed1338/

dirlr16e-4_total_batch_size61440_baseline_seed1339/

dirlr1e-4_total_batch_size10240_seq_len1024/

dirlr1e-4_total_batch_size10240_seq_len128/

dirlr1e-4_total_batch_size10240_seq_len256/

dirlr1e-4_total_batch_size10240_seq_len512/

dirlr1e-4_total_batch_size10240_seq_len64/

dirlr1e-4_total_batch_size20480_seq_len1024/

dirlr1e-4_total_batch_size20480_seq_len128/

dirlr1e-4_total_batch_size20480_seq_len256/

dirlr1e-4_total_batch_size20480_seq_len512/

dirlr1e-4_total_batch_size40960_seq_len1024/

dirlr1e-4_total_batch_size40960_seq_len128/

dirlr1e-4_total_batch_size40960_seq_len256/

dirlr1e-4_total_batch_size40960_seq_len512/

dirlr1e-4_total_batch_size5120_seq_len128/

dirlr1e-4_total_batch_size5120_seq_len256/

dirlr1e-4_total_batch_size5120_seq_len256_decay_lrfalse/

dirlr1e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective/

dirlr1e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps200/

dirlr1e-4_total_batch_size5120_seq_len512/

dirlr1e-4_total_batch_size5120_seq_len64/

dirlr2e-4_total_batch_size10240_n_heads2_seed1338/

dirlr2e-4_total_batch_size10240_n_heads2_seed1339/

dirlr2e-4_total_batch_size10240_n_heads2_seed1340/

dirlr2e-4_total_batch_size10240_n_heads4_seed1338/

dirlr2e-4_total_batch_size10240_n_heads4_seed1339/

dirlr2e-4_total_batch_size10240_n_heads4_seed1340/

dirlr2e-4_total_batch_size10240_seq_len1024/

dirlr2e-4_total_batch_size10240_seq_len128/

dirlr2e-4_total_batch_size10240_seq_len256/

dirlr2e-4_total_batch_size10240_seq_len512/

dirlr2e-4_total_batch_size5120_n_heads2_seed1338/

dirlr2e-4_total_batch_size5120_n_heads2_seed1339/

dirlr2e-4_total_batch_size5120_n_heads2_seed1340/

dirlr2e-4_total_batch_size5120_n_heads4_seed1338/

dirlr2e-4_total_batch_size5120_n_heads4_seed1339/

dirlr2e-4_total_batch_size5120_n_heads4_seed1340/

dirlr2e-4_total_batch_size5120_seq_len128/

dirlr2e-4_total_batch_size5120_seq_len256/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps1_seed1338/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps1_seed1339/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps200_seed1338/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrfalse_attention_kindselective_warmup_steps200_seed1339/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps1_seed1338/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps1_seed1339/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps200_seed1338/

dirlr2e-4_total_batch_size5120_seq_len256_decay_lrtrue_attention_kindselective_warmup_steps200_seed1339/

dirlr2e-4_total_batch_size5120_seq_len512/

dirlr2e-4_total_batch_size61440_baseline_seed1338/

dirlr2e-4_total_batch_size61440_baseline_seed1339/

dirlr2e-4_total_batch_size61440_baseline_seed1340/

dirlr2e-4_total_batch_size61440_baseline_seed1341/

dirlr3e-3_total_batch_size20480_seq_len128/

dirlr3e-3_total_batch_size20480_seq_len512/

dirlr3e-3_total_batch_size40960_seq_len128/

dirlr3e-3_total_batch_size40960_seq_len512/

dirlr3e-3_total_batch_size5120_seq_len128/

dirlr3e-3_total_batch_size5120_seq_len512/

dirlr3e-4_total_batch_size20480_seq_len128/

dirlr4e-4_total_batch_size10240_seq_len128/

dirlr4e-4_total_batch_size20480_seq_len128/

dirlr4e-4_total_batch_size20480_seq_len512/

dirlr4e-4_total_batch_size40960_seq_len128/

dirlr4e-4_total_batch_size40960_seq_len512/

dirlr4e-4_total_batch_size5120_seq_len128/

dirlr4e-4_total_batch_size5120_seq_len512/

dirlr5.5e-5_total_batch_size61440_baseline_seed1338/

dirlr5.5e-5_total_batch_size61440_baseline_seed1339/

dirlr5.5e-5_total_batch_size61440_baseline_seed1340/

dirlr5.5e-5_total_batch_size61440_baseline_seed1341/

dirlr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339/

dirlr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/

dirlr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1338/

dirlr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1339/

dirlr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/

dirlr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1341/

dirlr5e-4_total_batch_size20480_seq_len128/

dirlr6.5e-5_total_batch_size61440_baseline_seed1338/

dirlr6.5e-5_total_batch_size61440_baseline_seed1339/

dirlr6.5e-5_total_batch_size61440_baseline_seed1341/

dirlr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/

dirlr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/

dirlr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/

dirlr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1338/

dirlr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1339/

dirlr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/

dirlr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1341/

dirlr6e-4_total_batch_size10240_seq_len128/

dirlr6e-4_total_batch_size20480_seq_len128/

dirlr6e-4_total_batch_size5120_seq_len128/

dirlr6e-4_total_batch_size61440_baseline_seed1338/

dirlr6e-4_total_batch_size61440_baseline_seed1339/

dirlr6e-4_total_batch_size61440_baseline_seed1340/

dirlr6e-4_total_batch_size61440_baseline_seed1341/

dirlr6e-5_total_batch_size61440_baseline_seed1338/

dirlr6e-5_total_batch_size61440_baseline_seed1339/

dirlr6e-5_total_batch_size61440_baseline_seed1340/

dirlr6e-5_total_batch_size61440_baseline_seed1341/

dirlr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/

dirlr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339/

dirlr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/

dirlr6e-5_total_batch_size61440_two_masks_4_heads_seed1338/

dirlr6e-5_total_batch_size61440_two_masks_4_heads_seed1339/

dirlr6e-5_total_batch_size61440_two_masks_4_heads_seed1340/

dirlr6e-5_total_batch_size61440_two_masks_4_heads_seed1341/

dirlr8e-4_total_batch_size20480_seq_len128/

dirlr8e-4_total_batch_size20480_seq_len512/

dirlr8e-4_total_batch_size40960_seq_len128/

dirlr8e-4_total_batch_size40960_seq_len512/

dirlr8e-4_total_batch_size5120_seq_len128/

dirlr8e-4_total_batch_size5120_seq_len512/

dirlr8e-4_total_batch_size61440_baseline_seed1338/

dirlr8e-4_total_batch_size61440_baseline_seed1339/

dirlr8e-4_total_batch_size61440_baseline_seed1340/

dirlr8e-4_total_batch_size61440_baseline_seed1341/

dirlr9e-4_total_batch_size61440_baseline_seed1338/

dirlr9e-4_total_batch_size61440_baseline_seed1339/

dirmemory_loss_run_0/

dirmha_baseline_lr_tuning/

dirmha_const_hd_scale_nh/

dirmha_const_width_scale_hd/

dirmha_conv_coord_check_4/

dirmha_coord_check/

dirmha_numerics/

dirn_heads12_lr10e-4_total_batch_size61440_seed1338/

dirn_heads12_lr10e-4_total_batch_size61440_seed1339/

dirn_heads12_lr14e-4_total_batch_size61440_seed1338/

dirn_heads12_lr14e-4_total_batch_size61440_seed1339/

dirnew_mha_const_hd_scale_nh/

dirone_mask_per_head/

dirone_mask_per_head_1_latent_mask/

dirone_mask_per_head_1_latent_vector/

dirone_mask_per_head_2_latent_masks/

dirone_mask_per_head_2_latent_vectors/

dirone_mask_per_head_4_latent_masks/

dirone_mask_per_head_4_latent_vectors/

dirprotection_none_torch_compile/

dirsame_total_bs_half_seq_len/

dirscaling_mha_conv_granularity_2/

dirscaling_mha_conv_granularity_3/

dirscaling_mha_granularity_2/

dirscaling_mha_nh/

dirscaling_mha_nh_2/

dirscaling_mha_nh_3/

dirscaling_sdpa_granularity_2/

dirscaling_sdpa_nh/

dirscaling_sdpa_nh_2/

dirsdpa_16_maybe_fixed/

dirsdpa_16_spooky/

dirselective_run_0/

dirselective_run_0_continued/

dirself_to_selective_run_0_restarted/

dirself_to_selective_run_1_restarted_with_ko_zero/

dirself_to_selective_run_1_restarted_with_memory_penalty/

dirself_to_selective_run_1_restarted_with_o_zero/

dirtrying_new_latent_masks/

dirtwo_latent_masks/

dirtwo_latent_masks_compile_comparison/

dirtwo_masks_12_heads/

dirtwo_masks_4_heads/

dirtwo_masks_shared_1_head/

dirunselective_run_0/

dirunselective_run_0_continued/

dirunselective_run_0_restarted/

dirunselective_run_0_restarted_with_head/

dirunselective_run_1_restarted/

dirunselective_run_1_restarted_with_ko_zero/

dirunselective_run_1_restarted_with_o_rescaled/

dirunselective_run_1_restarted_with_o_zero/

file.gitattributes2 KBdownload

andrew-healey/context-compression · main · files are served by the source, never re-hosted here