AbstractPhil/captionbert-8192-v2-b

1{2  "profile": "_applied",3  "run_name": "captionbert-8192-b",4  "hf_repo": "AbstractPhil/captionbert-8192-v2-B",5  "sources": [6    {7      "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",8      "n_chunks": 66,9      "chunk_rows": 500000,10      "missing": {}11    }12  ],13  "experts": [14    "bert",15    "modern",16    "roberta",17    "albert",18    "distil"19  ],20  "ref_expert": "bert",21  "ref_hf_name": "google-bert/bert-base-uncased",22  "require_all_experts": true,23  "caption_field": "caption_llava",24  "caption_field_candidates": [25    "caption_llava",26    "caption",27    "caption_llava_short"28  ],29  "work_dir": "/content/cbv2",30  "keep_expert_shards": false,31  "disk_floor_gb": 25.0,32  "ram_resident": true,33  "preflight": true,34  "artifact_repo": "AbstractPhil/captionbert-8192-v2",35  "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",36  "push_targets": true,37  "hf_push": true,38  "push_every_min": 30.0,39  "keep_local_ckpts": 3,40  "parity_chunk": 0,41  "parity_n": 64,42  "parity_min_cos": 0.999,43  "fit_chunks": [44    0,45    11,46    22,47    33,48    44,49    5550  ],51  "fit_rows_per_chunk": 4000,52  "holdout_chunks": [53    60,54    6155  ],56  "fit_seed": 0,57  "d_model": 512,58  "n_heads": 8,59  "n_layers": 12,60  "d_ff": 2048,61  "max_len": 8192,62  "output_dim": 768,63  "dropout": 0.1,64  "pooling": "mean",65  "max_tokens": 256,66  "length_bucketing": true,67  "bucket_window": 64,68  "grad_checkpointing": true,69  "vram_probe": true,70  "epochs": 4,71  "batch_size": 2048,72  "lr": 0.0006,73  "min_lr": 1e-06,74  "warmup_steps": 2000,75  "grad_clip": 1.0,76  "seed": 42,77  "amp": true,78  "num_workers": 0,79  "log_every": 50,80  "eval_every": 1000,81  "ckpt_every_min": 20.0,82  "fresh_start": true,83  "allow_mismatched_resume": false,84  "nce_weight": 1.0,85  "mse_weight": 1.0,86  "nce_temperature": 0.07,87  "cv_weight": 0.0,88  "cv_target": 0.084,89  "run_stage0": false,90  "run_stage1": false,91  "run_stage2": false,92  "run_stage3": true,93  "resume": true94}