Update ssh

model/matchmaker.py

@@ -76,6 +76,9 @@ def load_json_via_sftp():
     return ratings, comparison_counts, total_comparisons
 
 
+class RunningPivot(object):
+    running_pivot = []
+
 def matchmaker(num_players, k_group=4):
     trueskill_env = TrueSkill()
     
@@ -86,14 +89,18 @@ def matchmaker(num_players, k_group=4):
 
     # Randomly select a player
     # selected_player = np.random.randint(0, num_players)
+    comparison_counts[:, RunningPivot.running_pivot] = float('inf')
     selected_player = np.argmin(comparison_counts.sum(axis=1))
 
+    RunningPivot.running_pivot.append(selected_player)
+    RunningPivot.running_pivot = RunningPivot.running_pivot[-5:]
+
     selected_trueskill_score = trueskill_env.expose(ratings[selected_player])
     trueskill_scores = np.array([trueskill_env.expose(p) for p in ratings])
     trueskill_diff = np.abs(trueskill_scores - selected_trueskill_score)
     n = comparison_counts[selected_player]
     ucb_scores = ucb_score(trueskill_diff, total_comparisons, n)
-        
+
     # Exclude self, select opponent with highest UCB score
     ucb_scores[selected_player] = -float('inf')  # minimize the score for the selected player to exclude it
     opponents = np.argsort(ucb_scores)[-k_group + 1:].tolist()

serve/gradio_web.py

@@ -222,7 +222,6 @@ def build_side_by_side_ui_anony(models):
         # share_btn = gr.Button(value="📷  Share")
 
     #gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
-
     dummy_img_output = gr.Image(width=512, visible=False)
     gr.Examples(
         examples=[["A train crossing a bridge that is going over a body of water.", os.path.join("./examples", "example1.jpg")],
@@ -231,12 +230,10 @@ def build_side_by_side_ui_anony(models):
                   ["The bathroom with green tile and a red shower curtain.", os.path.join("./examples", "example4.jpg")]],
         inputs = [textbox, dummy_img_output])
 
-    # vote_list = [Top1_btn, Top2_btn, Top3_btn, Top4_btn, Revote_btn, Submit_btn]
     order_btn_list = [textbox, send_btn, draw_btn, clear_btn]
     vote_order_list = [leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
                        A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn, \
                        vote_textbox, vote_submit_btn, vote_mode_btn]
-    # vote_rank_list = [A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn]
 
     generate_ig0 = gr.Image(width=512, label = "generate A", visible=False, interactive=False)
     generate_ig1 = gr.Image(width=512, label = "generate B", visible=False, interactive=False)

serve/update_skill.py

@@ -104,4 +104,8 @@ def update_skill(rank, model_names, k_group=4):
         
     total_comparisons += 1
 
-    save_json_via_sftp(ratings, comparison_counts, total_comparisons)
+    save_json_via_sftp(ratings, comparison_counts, total_comparisons)
+
+    from model.matchmaker import RunningPivot
+    if group[0] in RunningPivot.running_pivot:
+        RunningPivot.running_pivot.remove(group[0])

sorted_score_list.json

@@ -1,180 +1,180 @@
 {
     "total_models": 27,
-    "total_votes": 595,
+    "total_votes": 667,
     "sorted_score_list": [
         {
             "Rank": 0,
             "\ud83e\udd16 Model": "Midjourney-v6.0",
-            "\u2b50 Score (\u03bc/\u03c3)": "31.37 (34.23/0.95)",
-            "\ud83d\uddf3\ufe0f Votes": 105.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "32.64 (35.04/0.8)",
+            "\ud83d\uddf3\ufe0f Votes": 352.0,
             "Organization": "Midjourney"
         },
         {
             "Rank": 1,
             "\ud83e\udd16 Model": "Midjourney-v5.0",
-            "\u2b50 Score (\u03bc/\u03c3)": "31.06 (33.85/0.93)",
-            "\ud83d\uddf3\ufe0f Votes": 108.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "31.61 (33.99/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 385.0,
             "Organization": "Midjourney"
         },
         {
             "Rank": 2,
             "\ud83e\udd16 Model": "SD-v3.0",
-            "\u2b50 Score (\u03bc/\u03c3)": "29.34 (31.72/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 354.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "29.07 (31.44/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 423.0,
             "Organization": "Stability AI"
         },
         {
             "Rank": 3,
             "\ud83e\udd16 Model": "Dalle-3",
-            "\u2b50 Score (\u03bc/\u03c3)": "29.11 (31.51/0.8)",
-            "\ud83d\uddf3\ufe0f Votes": 225.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "28.26 (30.65/0.8)",
+            "\ud83d\uddf3\ufe0f Votes": 324.0,
             "Organization": "OpenAI"
         },
         {
             "Rank": 4,
-            "\ud83e\udd16 Model": "Pixart-Sigma",
-            "\u2b50 Score (\u03bc/\u03c3)": "26.26 (28.63/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 405.0,
-            "Organization": "PixArt-Alpha"
+            "\ud83e\udd16 Model": "Proteus-v0.2",
+            "\u2b50 Score (\u03bc/\u03c3)": "26.23 (28.59/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 357.0,
+            "Organization": "DataAutoGPT3"
         },
         {
             "Rank": 5,
+            "\ud83e\udd16 Model": "Dreamshaper-xl",
+            "\u2b50 Score (\u03bc/\u03c3)": "25.96 (28.31/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 342.0,
+            "Organization": "Lykon"
+        },
+        {
+            "Rank": 6,
             "\ud83e\udd16 Model": "Open-Dalle-v1.1",
             "\u2b50 Score (\u03bc/\u03c3)": "25.84 (28.19/0.78)",
             "\ud83d\uddf3\ufe0f Votes": 351.0,
             "Organization": "DataAutoGPT3"
         },
         {
-            "Rank": 6,
-            "\ud83e\udd16 Model": "Proteus-v0.2",
-            "\u2b50 Score (\u03bc/\u03c3)": "25.83 (28.19/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 327.0,
-            "Organization": "DataAutoGPT3"
+            "Rank": 7,
+            "\ud83e\udd16 Model": "Pixart-Sigma",
+            "\u2b50 Score (\u03bc/\u03c3)": "25.53 (27.9/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 423.0,
+            "Organization": "PixArt-Alpha"
         },
         {
-            "Rank": 7,
+            "Rank": 8,
             "\ud83e\udd16 Model": "Deepfloyd-IF",
-            "\u2b50 Score (\u03bc/\u03c3)": "25.49 (27.85/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 294.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "25.29 (27.66/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 306.0,
             "Organization": "DeepFloyd"
         },
         {
-            "Rank": 8,
+            "Rank": 9,
             "\ud83e\udd16 Model": "Realvisxl-v2.0",
-            "\u2b50 Score (\u03bc/\u03c3)": "25.03 (27.39/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 312.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "25.13 (27.49/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 327.0,
             "Organization": "Realistic Vision"
         },
-        {
-            "Rank": 9,
-            "\ud83e\udd16 Model": "Kandinsky-v2.2",
-            "\u2b50 Score (\u03bc/\u03c3)": "24.97 (27.35/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 282.0,
-            "Organization": "AI-Forever"
-        },
         {
             "Rank": 10,
-            "\ud83e\udd16 Model": "Dreamshaper-xl",
-            "\u2b50 Score (\u03bc/\u03c3)": "24.73 (27.09/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 300.0,
-            "Organization": "Lykon"
+            "\ud83e\udd16 Model": "Realvisxl-v3.0",
+            "\u2b50 Score (\u03bc/\u03c3)": "24.26 (26.62/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 309.0,
+            "Organization": "Realistic Vision"
         },
         {
             "Rank": 11,
-            "\ud83e\udd16 Model": "Dalle-2",
-            "\u2b50 Score (\u03bc/\u03c3)": "23.84 (26.23/0.8)",
-            "\ud83d\uddf3\ufe0f Votes": 207.0,
-            "Organization": "OpenAI"
+            "\ud83e\udd16 Model": "Kandinsky-v2.2",
+            "\u2b50 Score (\u03bc/\u03c3)": "24.2 (26.56/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 321.0,
+            "Organization": "AI-Forever"
         },
         {
             "Rank": 12,
-            "\ud83e\udd16 Model": "Realvisxl-v3.0",
-            "\u2b50 Score (\u03bc/\u03c3)": "22.71 (25.09/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 261.0,
-            "Organization": "Realistic Vision"
+            "\ud83e\udd16 Model": "Dalle-2",
+            "\u2b50 Score (\u03bc/\u03c3)": "23.27 (25.64/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 249.0,
+            "Organization": "OpenAI"
         },
         {
             "Rank": 13,
-            "\ud83e\udd16 Model": "SDXL-turbo",
-            "\u2b50 Score (\u03bc/\u03c3)": "21.4 (23.79/0.8)",
-            "\ud83d\uddf3\ufe0f Votes": 243.0,
-            "Organization": "Stability AI"
+            "\ud83e\udd16 Model": "Kandinsky-v2.0",
+            "\u2b50 Score (\u03bc/\u03c3)": "21.19 (23.57/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 252.0,
+            "Organization": "AI-Forever"
         },
         {
             "Rank": 14,
-            "\ud83e\udd16 Model": "LCM-v1.5",
-            "\u2b50 Score (\u03bc/\u03c3)": "20.74 (23.12/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 243.0,
-            "Organization": "Tsinghua"
+            "\ud83e\udd16 Model": "SDXL-turbo",
+            "\u2b50 Score (\u03bc/\u03c3)": "21.05 (23.42/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 273.0,
+            "Organization": "Stability AI"
         },
         {
             "Rank": 15,
-            "\ud83e\udd16 Model": "Kandinsky-v2.0",
-            "\u2b50 Score (\u03bc/\u03c3)": "20.49 (22.89/0.8)",
-            "\ud83d\uddf3\ufe0f Votes": 225.0,
-            "Organization": "AI-Forever"
+            "\ud83e\udd16 Model": "LCM-v1.5",
+            "\u2b50 Score (\u03bc/\u03c3)": "19.94 (22.32/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 273.0,
+            "Organization": "Tsinghua"
         },
         {
             "Rank": 16,
             "\ud83e\udd16 Model": "Openjourney-v4",
-            "\u2b50 Score (\u03bc/\u03c3)": "19.22 (21.64/0.81)",
-            "\ud83d\uddf3\ufe0f Votes": 225.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "19.87 (22.24/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 264.0,
             "Organization": "Prompthero"
         },
         {
             "Rank": 17,
-            "\ud83e\udd16 Model": "SD-v2.1",
-            "\u2b50 Score (\u03bc/\u03c3)": "18.8 (21.24/0.81)",
-            "\ud83d\uddf3\ufe0f Votes": 225.0,
+            "\ud83e\udd16 Model": "SD-turbo",
+            "\u2b50 Score (\u03bc/\u03c3)": "17.98 (20.36/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 252.0,
             "Organization": "Stability AI"
         },
         {
             "Rank": 18,
             "\ud83e\udd16 Model": "Playground-v2.5",
-            "\u2b50 Score (\u03bc/\u03c3)": "18.54 (20.99/0.81)",
-            "\ud83d\uddf3\ufe0f Votes": 213.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "17.85 (20.23/0.8)",
+            "\ud83d\uddf3\ufe0f Votes": 252.0,
             "Organization": "Playground AI"
         },
         {
             "Rank": 19,
-            "\ud83e\udd16 Model": "SD-turbo",
-            "\u2b50 Score (\u03bc/\u03c3)": "17.8 (20.26/0.82)",
-            "\ud83d\uddf3\ufe0f Votes": 204.0,
-            "Organization": "Stability AI"
+            "\ud83e\udd16 Model": "SSD-1b",
+            "\u2b50 Score (\u03bc/\u03c3)": "17.48 (19.89/0.8)",
+            "\ud83d\uddf3\ufe0f Votes": 252.0,
+            "Organization": "Segmind"
         },
         {
             "Rank": 20,
-            "\ud83e\udd16 Model": "SD-v1.5",
-            "\u2b50 Score (\u03bc/\u03c3)": "17.47 (19.9/0.81)",
-            "\ud83d\uddf3\ufe0f Votes": 228.0,
+            "\ud83e\udd16 Model": "SD-v2.1",
+            "\u2b50 Score (\u03bc/\u03c3)": "17.35 (19.74/0.8)",
+            "\ud83d\uddf3\ufe0f Votes": 255.0,
             "Organization": "Stability AI"
         },
         {
             "Rank": 21,
-            "\ud83e\udd16 Model": "SDXL",
-            "\u2b50 Score (\u03bc/\u03c3)": "16.83 (19.19/0.79)",
-            "\ud83d\uddf3\ufe0f Votes": 366.0,
+            "\ud83e\udd16 Model": "SD-v1.5",
+            "\u2b50 Score (\u03bc/\u03c3)": "17.02 (19.39/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 270.0,
             "Organization": "Stability AI"
         },
         {
             "Rank": 22,
-            "\ud83e\udd16 Model": "SSD-1b",
-            "\u2b50 Score (\u03bc/\u03c3)": "16.76 (19.23/0.82)",
-            "\ud83d\uddf3\ufe0f Votes": 216.0,
-            "Organization": "Segmind"
+            "\ud83e\udd16 Model": "Playground-v2.0",
+            "\u2b50 Score (\u03bc/\u03c3)": "16.84 (19.21/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 315.0,
+            "Organization": "Playground AI"
         },
         {
             "Rank": 23,
-            "\ud83e\udd16 Model": "Playground-v2.0",
-            "\u2b50 Score (\u03bc/\u03c3)": "16.37 (18.81/0.81)",
-            "\ud83d\uddf3\ufe0f Votes": 267.0,
-            "Organization": "Playground AI"
+            "\ud83e\udd16 Model": "SDXL",
+            "\u2b50 Score (\u03bc/\u03c3)": "16.74 (19.09/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 378.0,
+            "Organization": "Stability AI"
         },
         {
             "Rank": 24,
             "\ud83e\udd16 Model": "Stable-cascade",
-            "\u2b50 Score (\u03bc/\u03c3)": "15.34 (17.77/0.81)",
-            "\ud83d\uddf3\ufe0f Votes": 204.0,
+            "\u2b50 Score (\u03bc/\u03c3)": "16.32 (18.69/0.79)",
+            "\ud83d\uddf3\ufe0f Votes": 249.0,
             "Organization": "Stability AI"
         },
         {