mlfoundations-dev/train_fasttext_classifier_seed_code_best_codegolf

1operators:2- id: load_hf_python3  config:4    type: hf_source5    dataset: bigcode/commitpackft6    split: train7    subset: python8    trust_remote_code: True9- id: load_hf_c++10  config:11    type: hf_source12    dataset: bigcode/commitpackft13    split: train14    subset: c++15    trust_remote_code: True16- id: load_hf_java17  config:18    type: hf_source19    dataset: bigcode/commitpackft20    split: train21    subset: java22    trust_remote_code: True23- id: load_hf_c24  config:25    trust_remote_code: True26    type: hf_source27    dataset: bigcode/commitpackft28    split: train29    subset: c30- id: load_hf_c#31  config:32    trust_remote_code: True33    type: hf_source34    dataset: bigcode/commitpackft35    split: train36    subset: c#37- id: load_hf_css38  config:39    trust_remote_code: True40    type: hf_source41    dataset: bigcode/commitpackft42    split: train43    subset: css44- id: load_hf_javascript45  config:46    trust_remote_code: True47    type: hf_source48    dataset: bigcode/commitpackft49    split: train50    subset: javascript51- id: load_hf_ruby52  config:53    trust_remote_code: True54    type: hf_source55    dataset: bigcode/commitpackft56    split: train57    subset: ruby58- id: load_hf_shell59  config:60    trust_remote_code: True61    type: hf_source62    dataset: bigcode/commitpackft63    split: train64    subset: shell65- id: mix_operator66  config:67    type: mix68  input_ids:69  - load_hf_python70  - load_hf_shell71  - load_hf_c++72  - load_hf_java73  - load_hf_c74  - load_hf_c#75  - load_hf_css76  - load_hf_javascript77  - load_hf_ruby78- id: sample_dataset79  config:80    type: function81    function: data_strategies.commons.uniform_sample_limited82    function_config:83      num_samples: 50000  # 1.25% of 10000 target from stage 184  input_ids:85    - mix_operator86- id: generate_instructions87  config:88    type: completions89    map: chat90    map_config:91      system_message: "You are a helpful assistant."92      user_message: |93        You are to generate a question or task for a language model based on the following instruction and code pairs. 94 95        Instruction: {{message}}96        Code: {{old_contents}}97        98        Include only the new question and task. Do not include anything like "Here is the instruction". Include99        the code in your question and make the task sound like what a human would ask a language model. 100      output_column: instruction_seed101    model: gpt-4o-mini102    temperature: 1.0103    batch: True104  input_ids:105    - sample_dataset106- id: add_constant_column107  config:108    type: function109    function: data_strategies.commons.add_constant_columns110    function_config:111      response_seed: ""112  input_ids:113  - generate_instructions114- id: rename_task_column115  config:116    type: function117    function: data_strategies.commons.keep_only_columns118    function_config:119      columns_to_keep:120      - response_seed121      - instruction_seed122  input_ids:123  - add_constant_column124- id: annotate125  config:126    type: completions127    map: chat128    map_config:129      system_message: "You are a helpful assistant."130      user_message_column: instruction_seed131      output_column: response132    model: gpt-4o-mini133    temperature: 1.0134    batch: True135  input_ids:136  - rename_task_column137- id: convert_to_sharegpt138  config:139    type: function140    function: data_strategies.commons.convert_instruction_response_to_sharegpt141    function_config:142      input_instruction_column: instruction_seed143      input_response_column: response144      output_sharegpt_column: conversations145  input_ids:146    - annotate