cwenzi/neuroflow-cpp

1#!/bin/bash2T=/tmp/neuroflow_tok13S=/mnt/d/neuroflow-C++/scripts/preprocess_corpus.py4K=/mnt/d/neuroflow-C++/configs/tokenizer_cn_013.json5M=/mnt/d/neuroflow-C++/scripts/mix_training_data.py6 7mkdir -p "$T"8 9echo ">>> 大学 (后台)"10python3 "$S" process --input "/mnt/d/语料/大学" --tokenizer "$K" --output "$T/university.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 4 > /tmp/uni_prep.log 2>&1 &11UPID=$!12 13echo ">>> 初中"14python3 "$S" process --input "/mnt/d/语料/初中" --tokenizer "$K" --output "$T/junior.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 415echo "初中 done"16 17echo ">>> 高中"18python3 "$S" process --input "/mnt/d/语料/高中" --tokenizer "$K" --output "$T/senior.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 419echo "高中 done"20 21echo ">>> 等待大学..."22wait $UPID23echo "大学 done"24 25echo ">>> 混合"26python3 "$M" --inputs "$T/primary.tok1" "$T/junior.tok1" "$T/senior.tok1" "$T/university.tok1" --weights 1 1.5 2 2.5 --output "$T/mixed.tok1" --max-samples 50000027 28echo ">>> 全部就绪"29ls -lh "$T/"*.tok130