Create app.py

app.py

@@ -0,0 +1,258 @@
+# some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main
+import json
+import os
+from datetime import datetime, timezone
+
+import gradio as gr
+import pandas as pd
+from huggingface_hub import HfApi
+
+from src.css_html import custom_css
+from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT_3
+from src.utils import (
+    AutoEvalColumn,
+    fields,
+    is_model_on_hub,
+    make_clickable_names,
+    plot_throughput,
+    styled_error,
+    styled_message,
+)
+
+TOKEN = os.environ.get("HF_TOKEN", None)
+api = HfApi(TOKEN)
+df = pd.read_csv("data/code_eval_board.csv")
+
+QUEUE_REPO = "deepcode-ai/evaluation-requests"
+EVAL_REQUESTS_PATH = "eval-queue"
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [
+    c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
+]
+TYPES_LITE = [
+    c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
+]
+
+
+def add_new_eval(
+    model: str,
+    revision: str,
+    precision: str,
+    model_type: str,
+):
+    precision = precision
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # check the model actually exists before adding the eval
+    if revision == "":
+        revision = "main"
+
+    model_on_hub, error = is_model_on_hub(model, revision)
+    if not model_on_hub:
+        return styled_error(f'Model "{model}" {error}')
+
+    print("adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "revision": revision,
+        "precision": precision,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type.split(" ")[1],
+    }
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{precision}.json"
+    print(f"Saving eval request to {out_path}")
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    api.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # remove the local file
+    os.remove(out_path)
+
+    return styled_message("Your request has been submitted to the evaluation queue!\n")
+
+
+def select_columns(df, columns):
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns]
+    ]
+    return filtered_df
+
+
+def filter_items(df, leaderboard_table, query):
+    if query == "all":
+        return df[leaderboard_table.columns]
+    else:
+        query = query[0]
+    filtered_df = df[df["T"].str.contains(query, na=False)]
+    return filtered_df[leaderboard_table.columns]
+
+
+def search_table(df, leaderboard_table, query):
+    filtered_df = df[(df["Model"].str.contains(query, case=False))]
+    return filtered_df[leaderboard_table.columns]
+
+
+df = make_clickable_names(df)
+
+#            <div style='background-color: #F5F1CB; text-align: center; padding: 10px;'>
+#                <p><b>Warning</b>: This leaderboard is not regularily updated with the latest instruction-tuned code models, check the <b>Submit Results</b> section for submitting new evaluation results.
+#            You can also check other code leaderboards like <a href="https://evalplus.github.io/leaderboard.html">EvalPlus</a> & <a href="https://huggingface.co/spaces/mike-ravkine/can-ai-code-results">Can-AI-Code</a> .</p>
+#            </div>
+demo = gr.Blocks(css=custom_css)
+with demo:
+    with gr.Row():
+        gr.Markdown(
+            """<div style="text-align: center;"><h1> ⭐ Deep <span style='color: #e6b800;'>Code</span> Models <span style='color: #e6b800;'>Leaderboard</span></h1></div>\
+            <br>\
+            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">🤗 Open LLM-Perf Leaderboard 🏋️</a>, we compare performance of base multilingual code generation models on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>. We also measure throughput and provide\
+            information about the models. We only compare open pre-trained multilingual code models, that people can start from as base models for their trainings.</p>
+""",
+            elem_classes="markdown-text",
+        )
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.Column():
+            with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
+                with gr.TabItem("🔍 Evaluation table", id=0):
+                    with gr.Column():
+                        with gr.Accordion("➡️ See All Columns", open=False):
+                            shown_columns = gr.CheckboxGroup(
+                                choices=[
+                                    c
+                                    for c in COLS
+                                    if c
+                                    not in [
+                                        AutoEvalColumn.dummy.name,
+                                        AutoEvalColumn.model.name,
+                                        AutoEvalColumn.model_type_symbol.name,
+                                    ]
+                                ],
+                                value=[
+                                    c
+                                    for c in COLS_LITE
+                                    if c
+                                    not in [
+                                        AutoEvalColumn.dummy.name,
+                                        AutoEvalColumn.model.name,
+                                        AutoEvalColumn.model_type_symbol.name,
+                                    ]
+                                ],
+                                label="",
+                                elem_id="column-select",
+                                interactive=True,
+                            )
+                        # with gr.Column(min_width=780):
+                        with gr.Row():
+                            search_bar = gr.Textbox(
+                                placeholder="🔍 Search for your model and press ENTER...",
+                                show_label=False,
+                                elem_id="search-bar",
+                            )
+                            filter_columns = gr.Radio(
+                                label="⏚ Filter model types",
+                                choices=["all", "🟢 base", "🔶 instruction-tuned", "EXT external-evaluation"],
+                                value="all",
+                                elem_id="filter-columns",
+                            )
+
+                    leaderboard_df = gr.components.Dataframe(
+                        value=df[
+                            [
+                                AutoEvalColumn.model_type_symbol.name,
+                                AutoEvalColumn.model.name,
+                            ]
+                            + shown_columns.value
+                        ],
+                        headers=[
+                            AutoEvalColumn.model_type_symbol.name,
+                            AutoEvalColumn.model.name,
+                        ]
+                        + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                    )
+
+                    hidden_leaderboard_df = gr.components.Dataframe(
+                        value=df,
+                        headers=COLS,
+                        datatype=["str" for _ in range(len(COLS))],
+                        visible=False,
+                    )
+                    search_bar.submit(
+                        search_table,
+                        [hidden_leaderboard_df, leaderboard_df, search_bar],
+                        leaderboard_df,
+                    )
+                    filter_columns.change(
+                        filter_items,
+                        [hidden_leaderboard_df, leaderboard_df, filter_columns],
+                        leaderboard_df,
+                    )
+                    shown_columns.change(
+                        select_columns,
+                        [hidden_leaderboard_df, shown_columns],
+                        leaderboard_df,
+                    )
+                    gr.Markdown(
+                        """
+                    **Notes:**
+                    - Win Rate represents how often a model outperforms other models in each language, averaged across all languages.
+                    - The scores of instruction-tuned models might be significantly higher on humaneval-python than other languages. We use the instruction format of HumanEval. For other languages, we use base MultiPL-E prompts.
+                    - For more details check the 📝 About section.
+                    - Models with a 🔴 symbol represent external evaluation submission, this means that we didn't verify the results, you can find the author's submission under `Submission PR` field from `See All Columns` tab.
+                    """,
+                        elem_classes="markdown-text",
+                    )
+
+                with gr.TabItem("📊 Performance Plot", id=1):
+                    with gr.Row():
+                        bs_1_plot = gr.components.Plot(
+                            value=plot_throughput(df, bs=1),
+                            elem_id="bs1-plot",
+                            show_label=False,
+                        )
+                        bs_50_plt = gr.components.Plot(
+                            value=plot_throughput(df, bs=50),
+                            elem_id="bs50-plot",
+                            show_label=False,
+                        )
+                    gr.Markdown(
+                        "**Note:** The throughputs for some models are missing and might appear as zero.",
+                        elem_classes="markdown-text",
+                    )
+                with gr.TabItem("📝 About", id=2):
+                    gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
+                with gr.TabItem("Submit results 🚀", id=3):
+                    gr.Markdown(SUBMISSION_TEXT_3)
+
+
+demo.launch()