add base class

backend-cli.py

@@ -406,8 +406,8 @@ if __name__ == "__main__":
     if local_debug:
         debug_model_names = ["mistralai/Mixtral-8x7B-Instruct-v0.1"]
         # debug_model_names = ["TheBloke/Mixtral-8x7B-v0.1-GPTQ"]
-        # debug_task_name = 'selfcheck'
-        debug_task_name = "mmlu"
+        debug_task_name = 'selfcheckgpt'
+        # debug_task_name = "mmlu"
         task_lst = TASKS_HARNESS.copy()
         for task in task_lst:
             for debug_model_name in debug_model_names:

src/backend/huggingface_generate_until.py

@@ -2,12 +2,13 @@ from typing import List, Literal, Optional, Tuple, Union
 import torch
 import transformers
 
-from lm_eval.models.huggingface import HFLM
 from lm_eval.api.registry import register_model
 
+from src.backend.hflm_with_measurement import HFLMWithMeasurement
+
 
 @register_model("hf-chat")
-class HFLMwithChatTemplate(HFLM):
+class HFLMwithChatTemplate(HFLMWithMeasurement):
     def __init__(self, use_chat_template=True, **kwargs):
         super().__init__(**kwargs)
         self.use_chat_template = use_chat_template

src/backend/moe_infinity.py

@@ -5,12 +5,13 @@ from transformers import AutoModelForCausalLM
 from moe_infinity import MoE
 from typing import List, Tuple, Optional, Union
 
-from lm_eval.models.huggingface import HFLM
 from lm_eval.api.registry import register_model
 
+from src.backend.hflm_with_measurement import HFLMWithMeasurement
+
 
 @register_model("moe-infinity")
-class MoEHFLM(HFLM):
+class MoEHFLM(HFLMWithMeasurement):
     def __init__(
         self,
         pretrained: str = "mistralai/Mixtral-8x7B-Instruct-v0.1",