keep gsm8k and XSum/ROUGE

src/backend/envs.py

@@ -17,45 +17,46 @@ class Task:
 
 
 class Tasks(Enum):
-    task0 = Task("nq_open", "em", "NQ Open", 64)  # 64, as in the ATLAS paper
-    task1 = Task("triviaqa", "em", "TriviaQA", 64)  # 64, as in the ATLAS paper
+    # task0 = Task("nq_open", "em", "NQ Open", 64)  # 64, as in the ATLAS paper
+    # task1 = Task("triviaqa", "em", "TriviaQA", 64)  # 64, as in the ATLAS paper
 
-    task11 = Task("nq8", "em", "NQ Open 8", 8)
-    task12 = Task("tqa8", "em", "TriviaQA 8", 8)
+    # task11 = Task("nq8", "em", "NQ Open 8", 8)
+    # task12 = Task("tqa8", "em", "TriviaQA 8", 8)
 
     # TruthfulQA is intended as a zero-shot benchmark [5, 47]. https://owainevans.github.io/pdfs/truthfulQA_lin_evans.pdf
-    task2 = Task("truthfulqa_gen", "rougeL_acc", "TruthfulQA Gen", 0)
-    task3 = Task("truthfulqa_mc1", "acc", "TruthfulQA MC1", 0)
-    task4 = Task("truthfulqa_mc2", "acc", "TruthfulQA MC2", 0)
+    # task2 = Task("truthfulqa_gen", "rougeL_acc", "TruthfulQA Gen", 0)
+    # task3 = Task("truthfulqa_mc1", "acc", "TruthfulQA MC1", 0)
+    # task4 = Task("truthfulqa_mc2", "acc", "TruthfulQA MC2", 0)
 
-    task5 = Task("halueval_qa", "acc", "HaluEval QA", 0)
-    task6 = Task("halueval_dialogue", "acc", "HaluEval Dialogue", 0)
-    task7 = Task("halueval_summarization", "acc", "HaluEval Summarization", 0)
+    # task5 = Task("halueval_qa", "acc", "HaluEval QA", 0)
+    # task6 = Task("halueval_dialogue", "acc", "HaluEval Dialogue", 0)
+    # task7 = Task("halueval_summarization", "acc", "HaluEval Summarization", 0)
 
     # task8 = Task("xsum", "rougeL", "XSum", 2)
     # task9 = Task("cnndm", "rougeL", "CNN/DM", 2)
 
     task8_1 = Task("xsum_v2", "rougeL", "XSum", 0)
-    task9_1 = Task("cnndm_v2", "rougeL", "CNN/DM", 0)
+    # task9_1 = Task("cnndm_v2", "rougeL", "CNN/DM", 0)
 
-    task10 = Task("memo-trap", "acc", "memo-trap", 0)
-    task10_2 = Task("memo-trap_v2", "acc", "memo-trap", 0)
+    # task10 = Task("memo-trap", "acc", "memo-trap", 0)
+    # task10_2 = Task("memo-trap_v2", "acc", "memo-trap", 0)
 
-    task13 = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
+    # task13 = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
 
-    task14 = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT", 0)
+    # task14 = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT", 0)
 
     # task15 = Task("fever10", "acc", "FEVER", 16)
     # task15_1 = Task("fever11", "acc", "FEVER", 8)
 
-    task16 = Task("squadv2", "exact", "SQuADv2", 4)
+    # task16 = Task("squadv2", "exact", "SQuADv2", 4)
 
-    task17 = Task("truefalse_cieacf", "acc", "TrueFalse", 8)
+    # task17 = Task("truefalse_cieacf", "acc", "TrueFalse", 8)
 
     # task18 = Task("faithdial_hallu", "acc", "FaithDial", 8)
-    task19 = Task("faithdial_hallu_v2", "acc", "FaithDial", 8)
+    # task19 = Task("faithdial_hallu_v2", "acc", "FaithDial", 8)
 
-    task20 = Task("race", "acc", "RACE", 0)
+    # task20 = Task("race", "acc", "RACE", 0)
+    task21 = Task("gsm8k", "acc", "GSM8K", 0)
 
 
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")

src/display/utils.py

@@ -20,32 +20,33 @@ class Tasks(Enum):
     # nqopen = Task("nq8", "em", "NQ Open/EM")
     # triviaqa = Task("tqa8", "em", "TriviaQA/EM")
 
-    truthfulqa_mc1 = Task("truthfulqa_mc1", "acc", "TruthQA MC1/Acc")
-    truthfulqa_mc2 = Task("truthfulqa_mc2", "acc", "TruthQA MC2/Acc")
-    truthfulqa_gen = Task("truthfulqa_gen", "rougeL_acc", "TruthQA Gen/ROUGE")
+    # truthfulqa_mc1 = Task("truthfulqa_mc1", "acc", "TruthQA MC1/Acc")
+    # truthfulqa_mc2 = Task("truthfulqa_mc2", "acc", "TruthQA MC2/Acc")
+    # truthfulqa_gen = Task("truthfulqa_gen", "rougeL_acc", "TruthQA Gen/ROUGE")
 
     xsum_r = Task("xsum_v2", "rougeL", "XSum/ROUGE")
-    xsum_f = Task("xsum_v2", "factKB", "XSum/factKB")
-    xsum_b = Task("xsum_v2", "bertscore_precision", "XSum/BERT-P")
+    # xsum_f = Task("xsum_v2", "factKB", "XSum/factKB")
+    # xsum_b = Task("xsum_v2", "bertscore_precision", "XSum/BERT-P")
 
-    cnndm_r = Task("cnndm_v2", "rougeL", "CNN-DM/ROUGE")
-    cnndm_f = Task("cnndm_v2", "factKB", "CNN-DM/factKB")
-    cnndm_b = Task("cnndm_v2", "bertscore_precision", "CNN-DM/BERT-P")
+    # cnndm_r = Task("cnndm_v2", "rougeL", "CNN-DM/ROUGE")
+    # cnndm_f = Task("cnndm_v2", "factKB", "CNN-DM/factKB")
+    # cnndm_b = Task("cnndm_v2", "bertscore_precision", "CNN-DM/BERT-P")
 
-    race = Task("race", "acc", "RACE/Acc")
-    squadv2 = Task("squadv2", "exact", "SQUaDv2/EM")
+    # race = Task("race", "acc", "RACE/Acc")
+    # squadv2 = Task("squadv2", "exact", "SQUaDv2/EM")
 
-    memotrap = Task("memo-trap_v2", "acc", "MemoTrap/Acc")
-    ifeval = Task("ifeval", "prompt_level_strict_acc", "IFEval/Acc")
+    # memotrap = Task("memo-trap_v2", "acc", "MemoTrap/Acc")
+    # ifeval = Task("ifeval", "prompt_level_strict_acc", "IFEval/Acc")
 
-    faithdial = Task("faithdial_hallu_v2", "acc", "FaithDial/Acc")
+    # faithdial = Task("faithdial_hallu_v2", "acc", "FaithDial/Acc")
 
-    halueval_qa = Task("halueval_qa", "acc", "HaluQA/Acc")
-    halueval_summ = Task("halueval_summarization", "acc", "HaluSumm/Acc")
-    halueval_dial = Task("halueval_dialogue", "acc", "HaluDial/Acc")
+    # halueval_qa = Task("halueval_qa", "acc", "HaluQA/Acc")
+    # halueval_summ = Task("halueval_summarization", "acc", "HaluSumm/Acc")
+    # halueval_dial = Task("halueval_dialogue", "acc", "HaluDial/Acc")
 
-    # XXX include me back at some point
-    selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
+    # # XXX include me back at some point
+    # selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
+    gsm8k = Task("gsm8k", "acc", "GSM8K")
 
 
 # These classes are for user facing column names,