cwenzi/neuroflow-cpp

1#!/bin/bash2# NeuroFlow 课程训练: 预处理→小学→初中→高中→大学3set -e4B=/mnt/d/neuroflow-C++5O=$B/output6T=/tmp/neuroflow_tok17CKPT=8export LD_LIBRARY_PATH=$HOME/local/openblas/usr/lib/x86_64-linux-gnu/openblas-openmp:$LD_LIBRARY_PATH9N=/tmp/numactl-extracted/usr/bin/numactl10 11mkdir -p $O $T12 13echo "========================================"14echo "STEP 0: 预处理好 (TOK1)"15echo "  大学数据量最大，后台预处理"16echo "========================================"17python3 $B/scripts/preprocess_corpus.py process \18  --input /mnt/d/语料/大学 --tokenizer $B/configs/tokenizer_cn_013.json \19  --output $T/university.tok1 --max-seq-len 512 --max-samples 500000 --min-tokens 4 \20  > /tmp/tok1_university.log 2>&1 &21UNI_PID=$!22 23for DS in 小学 初中 高中; do24  echo "预处理 $DS..."25  python3 $B/scripts/preprocess_corpus.py process \26    --input "/mnt/d/语料/$DS" --tokenizer $B/configs/tokenizer_cn_013.json \27    --output "$T/$DS.tok1" --max-seq-len 512 --max-samples 500000 --min-tokens 4 \28    2>&1 | tail -129done30 31train() {32  local DATA=$1 EPOCHS=$2 MSG=$333  local FLAGS=""34  [ -n "$CKPT" ] && FLAGS="--resume $CKPT"35  echo; echo "========================================"36  echo ">>> $MSG"37  echo "========================================"38  OPENBLAS_NUM_THREADS=10 OMP_NUM_THREADS=20 \39  $N --cpunodebind=0 --membind=0 \40  $B/neuroflow_train_v2 --config $B/configs/config.json \41    --tokenizer $B/configs/tokenizer_cn_013.json \42    --output $O --data "$DATA" $FLAGS \43    --epochs $EPOCHS --lr 1e-4 --batch-size 64 \44    --log-interval 10 --save-interval 1000 --grad-clip 1.045  CKPT=$(ls -td $O/checkpoint_step*/checkpoint_step* 2>/dev/null | head -1)46  echo "  → checkpoint: $CKPT"47}48 49train "$T/小学.tok1" 2 "小学 2 epochs (从头)"50train "$T/初中.tok1" 3 "初中 3 epochs (续小学)"51 52echo "等待大学 TOK1预处理完成..."53wait $UNI_PID54 55train "$T/高中.tok1" 3 "高中 3 epochs (续初中)"56train "$T/university.tok1" 5 "大学 5 epochs (续高中)"57 58echo; echo "========================================"59echo "✅ 全部课程训练完成!"60ls -la $O/model_final.nfv1 2>/dev/null61