Evaluation time metric and plot

app.py

@@ -38,6 +38,7 @@ from src.tools.plots import (
     create_metric_plot_obj,
     create_plot_df,
     create_scores_df,
+    create_lat_score_mem_plot_obj
 )
 
 # Start ephemeral Spaces on PRs (see config in README.md)
@@ -344,7 +345,7 @@ with demo:
                     queue=True,
                 )
 
-        with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4):
+        with gr.TabItem("📈 Metrics", elem_id="llm-benchmark-tab-table", id=4):
             with gr.Row():
                 with gr.Column():
                     chart = create_metric_plot_obj(
@@ -359,7 +360,17 @@ with demo:
                         BENCHMARK_COLS,
                         title="Top Scores and Human Baseline Over Time (from last update)",
                     )
-                    gr.Plot(value=chart, min_width=500) 
+                    gr.Plot(value=chart, min_width=500)
+            with gr.Row():
+                with gr.Column():
+                    fig = create_lat_score_mem_plot_obj(leaderboard_df)
+                    plot = gr.components.Plot(
+                        value=fig,
+                        elem_id="plot",
+                        show_label=False,
+                    )
+                    gr.HTML("👆 Hover over the points 👆 for additional information. ",elem_id="text")
+                    gr.HTML('This plot the Evaluation Time from our backend GPU (Nvdia A100-80G) to run all the benchmarks, it\'s not a very precise performance benchmark of the models, for that look for the <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard" target="_blank">🤗 LLM-Perf Leaderboard</a>',elem_id="text")
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
             gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")

src/display/utils.py

@@ -109,8 +109,11 @@ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Avai
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
+auto_eval_column_dict.append(["eval_time", ColumnContent, ColumnContent("Evaluation Time (s)", "number", False)])
 # Dummy column for the search bar (hidden by the custom CSS)
-auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("Model Name", "str", False, dummy=True)])
+
+
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -147,7 +150,8 @@ baseline_row = {
     AutoEvalColumn.likes.name: 0,
     AutoEvalColumn.license.name: "",
     AutoEvalColumn.still_on_hub.name: False,
-    AutoEvalColumn.moe.name: False
+    AutoEvalColumn.moe.name: False,
+    AutoEvalColumn.eval_time.name: 0.0
 }
 
 baseline_list = []
@@ -187,7 +191,8 @@ human_baseline_row = {
     AutoEvalColumn.likes.name: 0,
     AutoEvalColumn.license.name: "",
     AutoEvalColumn.still_on_hub.name: False,
-    AutoEvalColumn.moe.name: False
+    AutoEvalColumn.moe.name: False,
+    AutoEvalColumn.eval_time.name: 0.0
 }
 
 baseline_list = []

src/leaderboard/filter_models.py

@@ -99,7 +99,7 @@ def flag_models(leaderboard_data: list[dict]):
         if model_data[AutoEvalColumn.flagged.name] == True:
             flag_key = "merged"
         else:
-            flag_key = model_data["model_name_for_query"]
+            flag_key = model_data[AutoEvalColumn.dummy.name]
 
         if flag_key in FLAGGED_MODELS:
             issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
@@ -118,7 +118,7 @@ def flag_models(leaderboard_data: list[dict]):
 def remove_forbidden_models(leaderboard_data: list[dict]):
     indices_to_remove = []
     for ix, model in enumerate(leaderboard_data):
-        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
+        if model[AutoEvalColumn.dummy.name] in DO_NOT_SUBMIT_MODELS:
             indices_to_remove.append(ix)
 
     for ix in reversed(indices_to_remove):

src/leaderboard/read_evals.py

@@ -36,6 +36,7 @@ class EvalResult:
     status: str = "FINISHED"
     tags: list = None
     json_filename: str = None
+    eval_time: float = 0.0
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -103,7 +104,8 @@ class EvalResult:
             results=results,
             precision=precision,  
             revision= config.get("model_sha", ""),
-            json_filename=json_filename
+            json_filename=json_filename,
+            eval_time=config.get("total_evaluation_time_seconds", 0.0)
         )
 
     def update_with_request_file(self, requests_path):
@@ -151,7 +153,8 @@ class EvalResult:
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
             AutoEvalColumn.merged.name: "merge" in self.tags if self.tags else False,
             AutoEvalColumn.moe.name: ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower(),
-            AutoEvalColumn.flagged.name: self.flagged
+            AutoEvalColumn.flagged.name: self.flagged,
+            AutoEvalColumn.eval_time.name: self.eval_time,
         }
 
         for task in Tasks:

src/tools/plots.py

@@ -151,6 +151,61 @@ def create_metric_plot_obj(
 
     return fig
 
+def create_lat_score_mem_plot_obj(leaderboard_df):
+    copy_df = leaderboard_df.copy()
+    copy_df = copy_df[~(copy_df[AutoEvalColumn.dummy.name].isin(["baseline", "human_baseline"]))]
+    # plot
+    SCORE_MEMORY_LATENCY_DATA = [
+        AutoEvalColumn.dummy.name,
+        AutoEvalColumn.average.name,
+        AutoEvalColumn.params.name,
+        AutoEvalColumn.architecture.name,
+        "Evaluation Time (min)"
+    ]
+
+    copy_df["LLM Average Score"] = copy_df[AutoEvalColumn.average.name]
+    copy_df["Evaluation Time (min)"] = copy_df[AutoEvalColumn.eval_time.name] / 60
+
+    #copy_df["size"] = copy_df[AutoEvalColumn.params.name]
+    copy_df["size"] = copy_df[AutoEvalColumn.params.name].apply(lambda x: 0.5 if 0 <= x < 0.8 else x)
+    copy_df["size"] = copy_df["size"].apply(lambda x: 0.8 if 0.8 <= x < 2 else x)
+    copy_df["size"] = copy_df["size"].apply(lambda x: 1.5 if 2 <= x < 5 else x)
+    copy_df["size"] = copy_df["size"].apply(lambda x: 2.0 if 5 <= x < 10 else x)
+    copy_df["size"] = copy_df["size"].apply(lambda x: 3.0 if 10 <= x < 20 else x)
+    copy_df["size"] = copy_df["size"].apply(lambda x: 4.5 if 20 <= x < 40 else x)
+    copy_df["size"] = copy_df["size"].apply(lambda x: 7.0 if x > 40 else x)
+
+    fig = px.scatter(
+        copy_df,
+        x="Evaluation Time (min)",
+        y="LLM Average Score",
+        size="size",
+        color=AutoEvalColumn.architecture.name,
+        custom_data=SCORE_MEMORY_LATENCY_DATA,
+        color_discrete_sequence=px.colors.qualitative.Light24,
+        log_x=True
+    )
+    fig.update_traces(
+        hovertemplate="<br>".join(
+            [f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(SCORE_MEMORY_LATENCY_DATA)]
+        )
+    )
+    fig.update_layout(
+        title={
+            "text": "Eval Time vs. Score vs. #Params",
+            "y": 0.95,
+            "x": 0.5,
+            "xanchor": "center",
+            "yanchor": "top",
+        },
+        xaxis_title="Time To Evaluate (min)",
+        yaxis_title="LLM Average Score",
+        legend_title="LLM Architecture",
+        width=1200,
+        height=600,
+    )
+
+    return fig
 
 # Example Usage:
 # human_baselines dictionary is defined.