cwenzi/neuroflow-cpp

1#!/bin/bash2TOK=/mnt/d/neuroflow-C++/configs/tokenizer_128k.json3S=/mnt/d/neuroflow-C++/scripts/preprocess_corpus.py4M=/mnt/d/neuroflow-C++/scripts/mix_training_data.py5T=/tmp/neuroflow_tok1_v26mkdir -p "$T"7 8echo ">>> 大学 (后台)"9python3 "$S" process --input "/mnt/d/语料/大学" --tokenizer "$TOK" --output "$T/university.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 4 > /tmp/prep_uni.log 2>&1 &10UPID=$!11 12echo ">>> 初中"13python3 "$S" process --input "/mnt/d/语料/初中" --tokenizer "$TOK" --output "$T/junior.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 414echo ">>> 高中"15python3 "$S" process --input "/mnt/d/语料/高中" --tokenizer "$TOK" --output "$T/senior.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 416echo ">>> 小学"17python3 "$S" process --input "/mnt/d/语料/小学" --tokenizer "$TOK" --output "$T/primary.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 418 19echo ">>> 等待大学..."20wait $UPID21 22echo ">>> 混合"23python3 "$M" --inputs "$T/primary.tok1" "$T/junior.tok1" "$T/senior.tok1" "$T/university.tok1" --weights 1 1.5 2 2.5 --output "$T/mixed_v2.tok1" --max-samples 50000024 25echo ">>> TOK1就绪"26ls -lh "$T/"*.tok1 | awk '{print $5, $9}'27