GEAK-agent_debug/get_result.py

import json
import numpy as np


index = {
        'call_candidate': None,
        'exe_candidate': None,
        'perf_candidates': [],
        }

paths = [
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_0.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_1.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_2.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_3.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_0.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_1.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_2.json',

    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_0.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_1.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_2.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_3.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_4.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_5.json',

    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_0.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_1.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_2.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_3.json',

    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_0.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_1.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_2.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_3.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_4.json',

    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v4_mem_0.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v4_mem_1.json',

    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_0.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_1.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_2.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_3.json',

    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_code_mem_0.json',

    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_0.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_1.json',
    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_2.json',
    ]

for path in paths:
    result = {i:[] for i in index}
    data = json.load(open(path))
    for k, v in data.items():
        for i in index:
            result[i].append(v[i])
    
    print(f'[path]:')
    print(path)
    print('\n')
    for k, v in index.items():
        print(f'[{k}]:')
        print(np.array([str(i) != str(v) for i in result[k]]).mean())
        if k == 'perf_candidates':
            acc = []
            for i in result[k]:
                if len(i) == 0:
                    acc.append(1)
                else:
                    acc.append(max(1, i[-1][1]))
            print(np.array(acc).mean())

    print('\n')
    print('-'*100)