cwenzi/neuroflow-cpp

1#!/usr/bin/env python32"""本地扫描提取所有 prompts，存为便携文件 → 上传云端"""3import json, os, random, sys4from pathlib import Path5 6SUBJECT_MAP = {7    '语文':'chinese','外语':'english','数学':'math','科学':'science',8    '物理':'physics','化学':'chemistry','生物':'biology',9    '地理':'geography','历史':'history','政治':'politics',10    '哲学':'philosophy','经济学':'economics','法学':'law',11    '教育学':'education','文学':'literature','历史学':'history',12    '理学':'science','工学':'engineering','农学':'agriculture',13    '医学':'medicine','军事学':'military','管理学':'management',14    '艺术学':'arts','交叉学科':'interdisciplinary',15}16 17random.seed(42)18 19def extract_prompts(filepath, max_prompts=10):20    prompts = []21    try:22        ext = filepath.suffix.lower()23        if ext == '.jsonl':24            with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:25                for line in f:26                    try:27                        r = json.loads(line.strip())28                        for k in ('text','content','title','question'):29                            if k in r and isinstance(r[k],str) and len(r[k])>=10:30                                prompts.append(r[k][:200])31                                break32                    except: pass33                    if len(prompts)>=max_prompts: break34        elif ext == '.json':35            with open(filepath,'r',encoding='utf-8',errors='ignore') as f:36                content = f.read(10*1024*1024)37            snippets = [content[i:i+200].strip() for i in range(0,min(len(content),100000),500)]38            prompts = [s for s in snippets if len(s)>=20 and not s.startswith('{')]39            random.shuffle(prompts)40            prompts = prompts[:max_prompts]41        elif ext in ('.txt','.md'):42            with open(filepath,'r',encoding='utf-8',errors='ignore') as f:43                content = f.read(1024*1024)44            paragraphs = [p.strip() for p in content.split('\n') if len(p.strip())>=20]45            random.shuffle(paragraphs)46            prompts = paragraphs[:max_prompts]47        elif ext in ('.csv','.tsv'):48            delim = '\t' if ext=='.tsv' else ','49            with open(filepath,'r',encoding='utf-8',errors='ignore') as f:50                lines = f.readlines(10000)51            for line in lines:52                for fld in line.strip().split(delim):53                    if len(fld)>=20: prompts.append(fld[:200])54            prompts = prompts[:max_prompts]55    except: pass56    return prompts57 58def scan_corpus(root):59    root = Path(root)60    subjects = {}61    for fp in root.rglob('*'):62        if not fp.is_file(): continue63        if fp.suffix.lower() not in ('.txt','.json','.jsonl','.csv','.tsv','.md'): continue64        subj = '通用'65        for part in fp.parts:66            for k in SUBJECT_MAP:67                if k in part: subj = k; break68        fname = fp.stem69        for k in SUBJECT_MAP:70            if k in fname: subj = k; break71        subjects.setdefault(subj, []).append(fp)72    return subjects73 74def main():75    corpus = sys.argv[1] if len(sys.argv)>1 else os.path.expanduser('~/corpus')76    output = sys.argv[2] if len(sys.argv)>2 else os.path.expanduser('~/prompts.json')77 78    print(f'🔍 扫描: {corpus}')79    subjects = scan_corpus(corpus)80    total = sum(len(v) for v in subjects.values())81    print(f'   {len(subjects)} 学科, {total:,} 文件')82 83    all_prompts = []84    for subj, files in sorted(subjects.items()):85        limit = len(files) // 1086        prompts = []87        random.shuffle(files)88        for fp in files[:min(len(files), max(100, limit))]:89            prompts.extend(extract_prompts(fp, 5))90            if len(prompts) >= limit: break91        for p in prompts[:limit]:92            all_prompts.append({'subject': subj, 'prompt': p})93        print(f'   {subj}: {len(prompts[:limit])} prompts')94 95    with open(output, 'w', encoding='utf-8') as f:96        json.dump(all_prompts, f, ensure_ascii=False)97    size_mb = os.path.getsize(output)/1e698    print(f'\n✅ {len(all_prompts)} prompts → {output} ({size_mb:.1f}MB)')99    print(f'   上传此文件到云服务器，运行 gen_distill_from_prompts.py')100 101if __name__ == '__main__':102    main()103