RLLab/safe-alignment-dynamic
safe-alignment-dynamic Training prompts for score-conditioned SFT / RL and separate reward-model pair sets; nothing here is scored. sft-prompts/train and rl-prompts/train: the same prompt pool, deduplicated across sources with responses merged and HH/PKU test prompts removed. rl-prompts additionally marks selection=pku_label_conflict where PKU's better and safer labels disagree with opposite safety flags; preference_pairs indexes those responses. This is an annotation, not a… See the full description on the dataset page: https://huggingface.co/datasets/RLLab/safe-alignment-dynamic.
safe-alignment-dynamic
Training prompts for score-conditioned SFT / RL and separate reward-model pair sets; nothing here is scored. sft-prompts/train and rl-prompts/train: the same prompt pool, deduplicated across sources with responses merged and HH/PKU test prompts removed. rl-prompts additionally marks selection=pku_label_conflict where PKU's better and safer labels disagree with opposite safety flags; preference_pairs indexes those responses. This is an annotation, not a filter or a sampling weight. Both configs are train-only; evaluation prompts are prepared by the evaluation processor. Strata are source proxies, not content labels: benign (HH helpful-*), harmful (HH harmless-base, PKU with an unsafe response), unlabeled (PKU with two safe responses).
helpful-rm train: HH helpful pairs; PKU pairs whose two replies are both safe (better label, safety held constant); a small answer > refusal augmentation on Guard-Safe HH helpful-base prompts, negatives alternating fixed templates and transplanted HH harmless-base refusals. Test keeps every PKU pair. harmless-rm: HH harmless-base pairs and PKU pairs with one safe and one unsafe reply (safe > unsafe). Both heads receive the same benign answer > refusal pairs, at most 15% of the smaller final training split. Standard chosen/rejected pairs only: no judge labels, no ties, no label flips, no manufactured conflict. Harmless therefore includes appropriate responses to benign requests, rather than pure content safety. Guard screens requests only; it does not certify answer correctness. Native pairs are length-balanced (chosen longer by >20 words subsampled to match rejected longer), then shared augmentation is added. Augmentation can retain length bias; its length counts and realized shares are reported. Sources: Anthropic/hh-rlhf, PKU-Alignment/PKU-SafeRLHF.
Composition
{
"source_revisions": {
"Anthropic/hh-rlhf": "09be8c5bbc57cb3887f3a9732ad6aa7ec602a1fa",
"PKU-Alignment/PKU-SafeRLHF": "9421ffafec3fa40a1f1a7d567b4d525079477ecb"
},
"hh_skipped": {
"hh-helpful-base/train/invalid": 187,
"hh-helpful-base/train/unusable": 111,
"hh-helpful-base/test/unusable": 6,
"hh-helpful-base/test/invalid": 15,
"hh-helpful-online/train/unusable": 105,
"hh-helpful-online/train/invalid": 19,
"hh-helpful-online/test/unusable": 5,
"hh-helpful-online/test/invalid": 2,
"hh-helpful-rs/train/unusable": 655,
"hh-helpful-rs/train/invalid": 43,
"hh-helpful-rs/test/unusable": 32,
"hh-helpful-rs/test/invalid": 3,
"hh-harmless-base/train/invalid": 222,
"hh-harmless-base/train/unusable": 53,
"hh-harmless-base/test/unusable": 4,
"hh-harmless-base/test/invalid": 9
},
"augmentation": {
"refusal_donors": 466,
"shared_pairs": 8443,
"duplicate_or_existing_pairs_removed": 0,
"chosen_longer": 0,
"rejected_longer": 0,
"skipped_no_length_matched_negative": 51,
"negative_kinds": {
"template": 2915,
"transplanted_refusal": 5029,
"clarification": 499
}
},
"guard": {
"model": "Qwen/Qwen3Guard-Gen-4B",
"candidates": 42595,
"labels": {
"Safe": 41934,
"Controversial": 577,
"Unsafe": 84
},
"safe_prompts": 41934,
"harmless_base_veto": {
"pairs": 42262,
"dropped_contradicted": 2330,
"level_matrix": {
"chosen=0,rejected=0": 18399,
"chosen=2,rejected=2": 11507,
"chosen=2,rejected=0": 4911,
"chosen=0,rejected=1": 661,
"chosen=1,rejected=1": 861,
"chosen=1,rejected=2": 915,
"chosen=1,rejected=0": 2110,
"chosen=2,rejected=1": 2144,
"chosen=0,rejected=2": 754
}
}
},
"clarity": {
"model": "Qwen/Qwen3-4B-Instruct-2507",
"question_like": 17127,
"self_contained": 14840
},
"rl_pool": {
"unique_prompts": 185020,
"conflict_prompts": 1454,
"conflict_pairs": 1512,
"duplicate_prompts_merged": 4853
},
"prompts_train_removed_for_test_overlap": 7520,
"splits": [
{
"config": "sft-prompts",
"split": "train",
"rows": 185020,
"by_source": {
"hh-helpful-base": 43455,
"pku": 32206,
"hh-helpful-online": 21847,
"hh-helpful-rs": 51484,
"hh-harmless-base": 36028
},
"by_stratum": {
"benign": 116786,
"unlabeled": 13504,
"harmful": 54730
}
},
{
"config": "rl-prompts",
"split": "train",
"rows": 185020,
"by_source": {
"hh-helpful-base": 43455,
"pku": 32206,
"hh-helpful-online": 21847,
"hh-helpful-rs": 51484,
"hh-harmless-base": 36028
},
"by_stratum": {
"benign": 116786,
"unlabeled": 13504,
"harmful": 54730
}
},
{
"config": "helpful-rm",
"split": "train",
"rows": 131490,
"by_source": {
"hh-helpful-online": 19228,
"hh-helpful-rs": 43951,
"hh-helpful-base": 37743,
"pku": 20918,
"augmentation-benign": 8443,
"pku-conflict-original": 1207
},
"by_stratum": {
"benign": 109365,
"unlabeled": 20918,
"harmful": 1207
}
},
{
"config": "helpful-rm",
"split": "test",
"rows": 14385,
"by_source": {
"hh-helpful-base": 2333,
"hh-helpful-online": 1130,
"hh-helpful-rs": 2714,
"pku": 8208
},
"by_stratum": {
"benign": 6177,
"harmful": 4905,
"unlabeled": 3303
}
},
{
"config": "harmless-rm",
"split": "train",
"rows": 56292,
"by_source": {
"pku": 8971,
"hh-harmless-base": 38878,
"augmentation-benign": 8443
},
"by_stratum": {
"harmful": 47849,
"benign": 8443
}
},
{
"config": "harmless-rm",
"split": "test",
"rows": 3520,
"by_source": {
"hh-harmless-base": 2299,
"pku": 1221
},
"by_stratum": {
"harmful": 3520
}
}
],
"rm_processing": {
"helpful-rm": {
"train_removed_for_test_overlap": 4325,
"train_duplicate_or_conflicting_rows": 2,
"test_duplicate_or_conflicting_rows": 0,
"length_balance": {
"chosen_longer_before": 51550,
"rejected_longer": 29549,
"dropped": 22001
},
"benign_augmentation_share_train": 0.06421020609932314,
"template_share_train": 0.022,
"benign_augmentation_negatives": {
"transplanted_refusal": 5029,
"template": 2915,
"clarification": 499
}
},
"harmless-rm": {
"train_removed_for_test_overlap": 2879,
"train_duplicate_or_conflicting_rows": 0,
"test_duplicate_or_conflicting_rows": 0,
"length_balance": {
"chosen_longer_before": 7449,
"rejected_longer": 14132,
"dropped": 0
},
"benign_augmentation_share_train": 0.14998578838911392,
"template_share_train": 0.052,
"benign_augmentation_negatives": {
"clarification": 499,
"transplanted_refusal": 5029,
"template": 2915
}
}
},
"pku_test_conflicts": {
"shared_mixed_safety_pairs": 1221,
"better_ne_safer": 196
}
}