LAB 04 · 固定参数的 ICL 对照：192 请求离线链路

import json, hashlib, collections, pathlib, datetime
LABELS=['增加','减少','相同']
MAPS=[dict(zip(LABELS,labels)) for labels in [('K','M','R'),('M','R','K'),('R','K','M')]]
DEMO_PAIRS=[(10,14),(10,6),(10,10),(8,11),(8,5),(8,8)]
SYSTEM='你根据本次输入完成记录比较任务。仅输出要求的单个标签，不补充解释。'

def relation(control,treatment):
    return '增加' if treatment>control else ('减少' if treatment<control else '相同')

def make_data():
    demos=[]
    for i,(c,t) in enumerate(DEMO_PAIRS):
        demos.append({'id':f'E{i+1:02d}','control':c,'treatment':t,'text':f'样本标记值：对照组 {c}，处理组 {t}。','semantic':relation(c,t)})
    test=[]
    objects=['菌落计数','荧光读数','叶片数量','吸光读数','分子数量','信号强度','斑点数量','颗粒数量']
    # Every treatment value has one row from each class; wording/object stay identical within each triple.
    for j,t in enumerate(range(9,31,3)):
        for c in (t-3,t,t+3):
            wording=(f'{objects[j]}记录中，对照为 {c}，处理为 {t}。' if j%2==0 else f'{objects[j]}记录中，处理为 {t}，对照为 {c}。')
            test.append({'id':f'T{len(test)+1:02d}','control':c,'treatment':t,'object':objects[j],'text':wording,'semantic':relation(c,t)})
    return demos,test

def build_prompt(condition,row,demos,mapping=None):
    intro='请比较记录中的处理读数相对于对照读数的方向。'
    if condition in ('A','B'): intro+='只输出增加、减少或相同之一。'
    else: intro+='按本轮约定只输出 K、M 或 R 之一。'
    if condition=='C': intro+='本轮规则：'+'；'.join(f'{x}→{mapping[x]}' for x in LABELS)+'。'
    lines=[intro]
    if condition in ('B','D'):
        lines.append('示例：')
        for ex in demos:
            label=ex['semantic'] if condition=='B' else mapping[ex['semantic']]
            lines.append(ex['text']+' 输出：'+label)
    lines+=['当前记录：'+row['text'],'输出：']
    return '\n'.join(lines)

def make_requests():
    demos,test=make_data(); requests=[]
    for condition in ('A','B','C','D'):
        maps=[(None,None)] if condition in ('A','B') else list(enumerate(MAPS))
        for mapping_id,mapping in maps:
            for row in test:
                expected=row['semantic'] if mapping is None else mapping[row['semantic']]
                prompt=build_prompt(condition,row,demos,mapping)
                identity=f"{condition}-m{mapping_id if mapping_id is not None else 'na'}-{row['id']}"
                requests.append({'request_id':identity,'condition':condition,'record_id':row['id'],
                    'mapping_id':mapping_id,'semantic':row['semantic'],'expected':expected,
                    'system':SYSTEM,'prompt':prompt,'messages':[{'role':'system','content':SYSTEM},{'role':'user','content':prompt}],
                    'input_sha256':hashlib.sha256((SYSTEM+'\n'+prompt).encode()).hexdigest(),
                    'max_visible_output_tokens':8})
    return requests

def grade(raw,request):
    normalized=raw.strip()
    allowed=LABELS if request['condition'] in ('A','B') else ['K','M','R']
    return {'normalized':normalized,'correct':normalized==request['expected'],
            'format_error':normalized not in allowed,'expected':request['expected']}

def synthetic_fixture(requests):
    # This is an intentionally mixed synthetic grading fixture, never a model.
    records=[]
    for i,r in enumerate(requests):
        allowed=LABELS if r['condition'] in ('A','B') else ['K','M','R']
        if i%13==0: raw='答案是 '+r['expected'] # format error
        elif i%17==0: raw=''                   # empty error
        elif i%11==0: raw=next(x for x in allowed if x!=r['expected']) # valid wrong label
        else: raw=' '+r['expected']+'\n'       # only outer whitespace may be removed
        records.append({**r,'raw_output':raw,'score':grade(raw,r),
                        'evidence_type':'SYNTHETIC_FIXTURE_NOT_MODEL_OUTPUT','model_id':None,
                        'model_revision':None,'tokenizer_revision':None,'decoding':None,
                        'request_started_at':None,'request_finished_at':None})
    return records

def aggregate(records):
    groups={}
    for r in records:
        group=(r['condition'],r['mapping_id'],r['semantic'])
        a=groups.setdefault(group,{'n':0,'correct':0,'format_errors':0})
        a['n']+=1; a['correct']+=int(r['score']['correct']); a['format_errors']+=int(r['score']['format_error'])
    return [{'condition':k[0],'mapping_id':k[1],'semantic':k[2],**v} for k,v in groups.items()]

def check_design():
    demos,test=make_data(); requests=make_requests()
    assert len(demos)==6 and len(test)==24 and len(requests)==192
    assert collections.Counter(x['semantic'] for x in demos)==dict.fromkeys(LABELS,2)
    assert collections.Counter(x['semantic'] for x in test)==dict.fromkeys(LABELS,8)
    assert not ({(r['control'],r['treatment']) for r in demos}&{(r['control'],r['treatment']) for r in test})
    assert collections.Counter(r['condition'] for r in requests)=={'A':24,'B':24,'C':72,'D':72}
    for t in range(9,31,3):
        rows=[r for r in test if r['treatment']==t]
        assert set(r['semantic'] for r in rows)==set(LABELS)
    assert {(9,12),(12,9)}.issubset({(r['control'],r['treatment']) for r in test})
    assert len({r['request_id'] for r in requests})==192
    assert all(not any(label in r['text'] for label in LABELS) for r in test)
    return {'demos':len(demos),'heldout':len(test),'independent_requests':len(requests),'requests_per_condition':dict(collections.Counter(r['condition'] for r in requests))}

def run_with_generator(generate,model_manifest,out_dir,max_requests=3):
    # Future extension: explicit model adapter; never called in R1 dry-run.
    required=['model_id','model_revision','tokenizer_revision','software_versions','decoding','system_prompt']
    if any(not model_manifest.get(k) for k in required): raise ValueError('先冻结完整模型与解码身份')
    if model_manifest['system_prompt']!=SYSTEM: raise ValueError('system prompt 与冻结设计不一致')
    if not 1<=max_requests<=192: raise ValueError('max_requests 须明确设为1..192')
    out=pathlib.Path(out_dir); out.mkdir(parents=True,exist_ok=True)
    path=out/'model-responses.jsonl'
    if path.exists(): raise FileExistsError('为保护教师基线，请选新输出目录')
    records=[]
    with path.open('x') as f:
        for request in make_requests()[:max_requests]:
            started=datetime.datetime.now(datetime.timezone.utc).isoformat()
            # Adapter must perform one stateless call with exactly these messages and fixed settings.
            raw=generate(request['messages'],dict(model_manifest['decoding']),request['max_visible_output_tokens'])
            if not isinstance(raw,str): raise TypeError('模型适配器必须返回原始字符串')
            record={**request,'raw_output':raw,'score':grade(raw,request),'model_manifest':model_manifest,
                    'evidence_type':'MODEL_ADAPTER_OUTPUT','request_started_at':started,
                    'request_finished_at':datetime.datetime.now(datetime.timezone.utc).isoformat()}
            f.write(json.dumps(record,ensure_ascii=False)+'\n'); f.flush(); records.append(record)
    return records

def write_artifacts(out_dir,records=None):
    out=pathlib.Path(out_dir); out.mkdir(parents=True,exist_ok=True)
    demos,test=make_data(); requests=make_requests()
    for name,data in [('demonstrations.json',demos),('heldout24.json',test),('mappings.json',MAPS),('requests192.json',requests)]:
        (out/name).write_text(json.dumps(data,ensure_ascii=False,indent=2))
    if records is not None:
        (out/'SYNTHETIC-fixture192.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records))
        (out/'SYNTHETIC-grading-summary.json').write_text(json.dumps(aggregate(records),ensure_ascii=False,indent=2))
    return out

def evaluate_responses(requests,responses,model_manifest):
    required=['model_id','model_revision','tokenizer_revision','software_versions','decoding','system_prompt']
    missing=[k for k in required if not model_manifest.get(k)]
    if missing: raise ValueError('缺少真实运行身份字段: '+','.join(missing))
    by_id={r['request_id']:r for r in responses}
    if len(by_id)!=len(responses): raise ValueError('重复request_id')
    if set(by_id)!=set(r['request_id'] for r in requests): raise ValueError('响应必须恰好覆盖这192个request_id')
    result=[]
    for r in requests:
        answer=by_id[r['request_id']]
        if answer.get('input_sha256')!=r['input_sha256']: raise ValueError('请求哈希不符: '+r['request_id'])
        if not isinstance(answer.get('raw_output'),str): raise ValueError('raw_output必须是字符串')
        result.append({**r,**answer,'score':grade(answer['raw_output'],r),'evidence_type':'USER_SUPPLIED_MODEL_OUTPUT_NOT_EXECUTED_BY_THIS_SCRIPT','model_manifest':model_manifest})
    return result

print(check_design())
demos,test=make_data()
print('完整示例：')
for row in demos: print(row['id'],row['text'],'→',row['semantic'])
print('完整留出：')
for row in test: print(row['id'],row['text'],'参考答案=',row['semantic'])
print('三轮映射:',MAPS)

requests=make_requests()
row_id=next(x['id'] for x in test if (x['control'],x['treatment'])==(12,9))
for condition in ('A','B','C','D'):
    r=next(r for r in requests if r['condition']==condition and r['record_id']==row_id and r['mapping_id'] in (None,0))
    print('\n---',r['request_id'],'---\nSYSTEM:',r['system'],'\nUSER:\n'+r['prompt'])

fixture=synthetic_fixture(requests)
for raw in [' M ','答案是 M','K','']:
    r=next(r for r in requests if r['expected']=='M')
    print('人工构造原始输出=',repr(raw),grade(raw,r))
summary=aggregate(fixture)
print('SYNTHETIC FIXTURE grading summary — NOT MODEL ACCURACY')
for row in summary: print(row)
assert len(fixture)==192
assert sum(row['n'] for row in summary)==192
assert grade(' M ',r)['correct'] and not grade('答案是 M',r)['correct']
assert grade('',r)['format_error']

out=write_artifacts('icl192-student-output',fixture)
print('生成文件：')
for p in sorted(out.iterdir()): print(p.name,p.stat().st_size,'bytes')
print('首条记录证据身份:',fixture[0]['evidence_type'],'model_id=',fixture[0]['model_id'])