cwenzi/neuroflow-cpp

1import json
2
3path = r'D:\neuroflow-C++\configs\tokenizer_128k.json'
4with open(path, 'r', encoding='utf-8') as f:
5    content = f.read()
6
7print('Lines:', content.count('\n'))
8print('Size:', len(content))
9
10# Find vocab section
11idx = content.find('"vocab"')
12print('vocab key at position:', idx)
13if idx > 0:
14    print('Context:', repr(content[idx:idx+50]))
15
16# The C++ parse_config looks for "\"vocab\":{" or "\"vocab\": {"
17# With indent=2, it's likely "\"vocab\": {"
18pattern1 = '"vocab": {'
19pattern2 = '"vocab":{'
20p1 = content.find(pattern1)
21p2 = content.find(pattern2)
22print(f'Pattern "{pattern1}" found at:', p1)
23print(f'Pattern "{pattern2}" found at:', p2)