File size: 4,848 Bytes
14e4843
 
 
 
 
1c22d8d
14e4843
 
 
 
 
 
 
 
4736a54
14e4843
d6d7ec6
 
 
 
 
 
 
 
14e4843
 
 
 
d6d7ec6
 
 
14e4843
88d1c0e
 
14e4843
 
 
 
 
 
 
 
 
 
 
 
 
 
88d1c0e
4736a54
 
 
88d1c0e
14e4843
 
 
 
 
 
 
 
88d1c0e
 
 
14e4843
 
 
 
 
 
 
 
 
 
1c22d8d
 
 
14e4843
21309a8
88d1c0e
14e4843
21309a8
88d1c0e
14e4843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86b14ca
14e4843
 
 
 
 
 
 
 
 
 
 
 
86b14ca
14e4843
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
import json
import os
from tqdm import tqdm
import copy
import pandas as pd
import numpy as np

from src.display.formatting import has_no_nan_values, make_clickable_model
from src.display.utils import AutoEvalColumn, EvalQueueColumn
from src.leaderboard.filter_models import filter_models
from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, update_model_type_with_open_llm_request_file

from src.backend.envs import Tasks as BackendTasks
from src.display.utils import Tasks
from src.display.utils import E2Es, PREs, TS

def get_leaderboard_df(
    results_path: str,
    requests_path: str,
    requests_path_open_llm: str,
    cols: list,
    benchmark_cols: list,
    is_backend: bool = False,
) -> tuple[list[EvalResult], pd.DataFrame]:
    # Returns a list of EvalResult
    raw_data: list[EvalResult] = get_raw_eval_results(results_path, requests_path, requests_path_open_llm)
    if requests_path_open_llm != "":
        for result_idx in tqdm(range(len(raw_data)), desc="updating model type with open llm leaderboard"):
            raw_data[result_idx] = update_model_type_with_open_llm_request_file(
                raw_data[result_idx], requests_path_open_llm
            )

    # all_data_json_ = [v.to_dict() for v in raw_data if v.is_complete()]
    all_data_json_ = [v.to_dict() for v in raw_data] # include incomplete evals

    name_to_bm_map = {}

    task_iterator = Tasks
    if is_backend is True:
        task_iterator = BackendTasks

    for task in task_iterator:
        task = task.value
        name = task.col_name
        bm = (task.benchmark, task.metric)
        name_to_bm_map[name] = bm

    # bm_to_name_map = {bm: name for name, bm in name_to_bm_map.items()}
    system_metrics_to_name_map = {
        "end_to_end_time": f"{E2Es}",
        "prefilling_time": f"{PREs}",
        "decoding_throughput": f"{TS}",
    }

    all_data_json = []
    for entry in all_data_json_:
        new_entry = copy.deepcopy(entry)
        for k, v in entry.items():
            if k in name_to_bm_map:
                benchmark, metric = name_to_bm_map[k]
                new_entry[k] = entry[k][metric]
                for sys_metric, metric_namne in system_metrics_to_name_map.items():
                    if sys_metric in entry[k]:
                        new_entry[f"{k} {metric_namne}"] = entry[k][sys_metric]

        all_data_json += [new_entry]

    # all_data_json.append(baseline_row)
    filter_models(all_data_json)

    df = pd.DataFrame.from_records(all_data_json)

    # if AutoEvalColumn.average.name in df:
    #     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
    for col in cols:
        if col not in df.columns:
            df[col] = np.nan

    if not df.empty:
        df = df.round(decimals=2)

        # filter out if any of the benchmarks have not been produced
        # df = df[has_no_nan_values(df, benchmark_cols)]

    return raw_data, df


def get_evaluation_queue_df(save_path: str, cols: list) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
    all_evals = []

    for entry in entries:
        if ".json" in entry:
            file_path = os.path.join(save_path, entry)
            with open(file_path) as fp:
                data = json.load(fp)

            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
            data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")

            all_evals.append(data)
        elif ".md" not in entry:
            # this is a folder
            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
            for sub_entry in sub_entries:
                file_path = os.path.join(save_path, entry, sub_entry)
                with open(file_path) as fp:
                    data = json.load(fp)

                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")
                all_evals.append(data)

    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
    df_running = pd.DataFrame.from_records(running_list, columns=cols)
    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
    return df_finished[cols], df_running[cols], df_pending[cols]