Accio-Lab/occamy-1.0-FP8

1import json,time,requests,base64,re,subprocess,sys,unicodedata,concurrent.futures,statistics2from pathlib import Path3from decimal import Decimal4R=Path(__file__).parent5TOOLS=[{'type':'function','function':{'name':'add','description':'Add two integers','parameters':{'type':'object','properties':{'a':{'type':'integer'},'b':{'type':'integer'}},'required':['a','b']}}}]6def request(base,payload):7 payload=dict(payload,stream=True,stream_options={'include_usage':True});start=time.perf_counter();first=None;text='';usage={};finish=None;calls={}8 with requests.post(base+'/v1/chat/completions',json=payload,stream=True,timeout=(15,360)) as r:9  r.raise_for_status()10  for line in r.iter_lines():11   if not line.startswith(b'data: '):continue12   data=line[6:]13   if data==b'[DONE]':break14   v=json.loads(data)15   if v.get('usage'):usage=v['usage']16   for c in v.get('choices',[]):17    delta=c.get('delta',{});content=delta.get('content') or ''18    if (content or delta.get('tool_calls')) and first is None:first=time.perf_counter()19    text+=content;finish=c.get('finish_reason') or finish20    for tc in delta.get('tool_calls') or []:21     idx=tc.get('index',0);dst=calls.setdefault(idx,{'id':'','type':'function','function':{'name':'','arguments':''}})22     if tc.get('id'):dst['id']=tc['id']23     for k in ['name','arguments']:dst['function'][k]+=tc.get('function',{}).get(k) or ''24 elapsed=time.perf_counter()-start25 return {'output':text,'tool_calls':list(calls.values()),'finish_reason':finish,'usage':usage,'elapsed_s':elapsed,'ttft_s':None if first is None else first-start,'tokens_per_s':usage.get('completion_tokens',0)/elapsed}26def run_case(base,c):27 payload={'model':'occamy','messages':json.loads(json.dumps(c['messages'])),'temperature':0,'seed':42,'max_tokens':c['max_tokens'],'chat_template_kwargs':{'enable_thinking':False}}28 for msg in payload['messages']:29  if isinstance(msg['content'],str):msg['content']=unicodedata.normalize('NFC',msg['content'])30 if c.get('image'):31  p=payload['messages'][-1]['content'];data=base64.b64encode((R/c['image']).read_bytes()).decode();payload['messages'][-1]['content']=[{'type':'text','text':p},{'type':'image_url','image_url':{'url':'data:image/png;base64,'+data}}]32 if c['category']=='tool':payload.update(tools=TOOLS,tool_choice='auto')33 out={'id':c['id'],'category':c['category'],'expected':c['expect']}34 try:35  out.update(request(base,payload));s=out['output'].strip();e=c['expect'];cat=c['category'];passed=False36  if cat=='gsm8k':37   nums=re.findall(r'[-+]?\d[\d,]*(?:\.\d+)?',s.split('FINAL:')[-1]);out['parsed_answer']=nums[-1].replace(',','') if nums else None38   passed=bool(nums) and Decimal(out['parsed_answer'])==Decimal(e.replace(',',''))39  elif cat=='arc':40   out['strict_label_format']=s==e;match=re.match(r'^(?:Answer\s*:\s*)?([A-D1-4])(?:[.:)\s]|$)',s,re.I);passed=bool(match) and match.group(1).upper()==e41  elif cat=='json':42   try:passed=json.loads(s)==e43   except ValueError:pass44   try:out['json_content_after_fence_removal']=json.loads(re.sub(r'^```(?:json)?\s*|\s*```$','',s))==e45   except ValueError:out['json_content_after_fence_removal']=False46  elif cat=='code':47   res=subprocess.run([sys.executable,'-I',str(R/'code_check.py')],input=json.dumps({'code':s,'tests':e}),text=True,capture_output=True,timeout=6);out['code_check']={'returncode':res.returncode,'stdout':res.stdout,'stderr':res.stderr[-700:]};passed=res.returncode==0 and json.loads(res.stdout)['pass']48  elif cat=='tool':49   calls=out['tool_calls'];valid=len(calls)==1 and calls[0]['function']['name']=='add' and json.loads(calls[0]['function']['arguments'])=={'a':e['a'],'b':e['b']};out['call_pass']=valid50   if valid:51    actual=json.loads(calls[0]['function']['arguments']);value=actual['a']+actual['b'];payload['messages'] += [{'role':'assistant','content':out['output'] or None,'tool_calls':calls},{'role':'tool','tool_call_id':calls[0]['id'],'content':str(value)}];payload['tool_choice']='none';follow=request(base,payload);out['followup']=follow52    answer_line=next((line for line in follow['output'].splitlines() if line.strip()),'');nums=re.findall(r'(?<![\d])[-+]?\d+',answer_line);out['followup_answer_line']=answer_line;passed=bool(nums) and int(nums[-1])==value and follow['finish_reason']=='stop'53  elif cat=='vision':passed=s.lower().strip('.! ')==str(e).lower()54  else:passed=s==e55  out['pass']=bool(passed);out['truncated']=out['finish_reason']=='length'56 except Exception as exc:out.update(error=repr(exc),pass_=False);out['pass']=False57 return out58def evaluate(base,name):59 cases=json.loads((R/'cases.json').read_text());path=R/(name+'.jsonl');done={}60 if path.exists():done={x['id']:x for x in map(json.loads,path.read_text().splitlines())}61 request(base,{'model':'occamy','messages':[{'role':'user','content':'Say hello.'}],'max_tokens':16,'temperature':0,'chat_template_kwargs':{'enable_thinking':False}})62 with path.open('a') as f:63  with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:64   futures=[pool.submit(run_case,base,c) for c in cases if c['id'] not in done]65   for fut in concurrent.futures.as_completed(futures):66    x=fut.result();f.write(json.dumps(x,ensure_ascii=False)+'\n');f.flush();done[x['id']]=x67    if len(done)%8==0:print(name,len(done),'/',len(cases),'passed',sum(v['pass'] for v in done.values()),flush=True)68 # Separate short sequential timing probes; report shared hardware caveat.69 perf=[]70 for _ in range(3):perf.append(request(base,{'model':'occamy','messages':[{'role':'user','content':'Explain in about 80 words how binary search works.'}],'max_tokens':128,'temperature':0,'chat_template_kwargs':{'enable_thinking':False}}))71 (R/(name+'-performance.json')).write_text(json.dumps(perf,indent=2));print(name,'DONE',flush=True)72if __name__=='__main__':evaluate(sys.argv[1],sys.argv[2])73