add gpu info

backend-cli.py

@@ -166,8 +166,13 @@ def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[in
     gpu_info = analyze_gpu_stats(gpu_stats_list)
     for task_name in results['results'].keys():
         for key, value in gpu_info.items():
-            results['results'][task_name][f"{key},none"] = int(value)
+            if "GPU" not in key:
+                results['results'][task_name][f"{key},none"] = int(value)
+            else:
+                results['results'][task_name][f"{key},none"] = value
 
+        results['results'][task_name]['batch_size,none'] = batch_size
+    print(f"gpu_stats_list: {gpu_stats_list}")
     print("GPU Usage:", gpu_info)
 
     dumped = json.dumps(results, indent=2, default=lambda o: "<not serializable>")
@@ -430,25 +435,31 @@ if __name__ == "__main__":
     if local_debug:
         # debug_model_names = [args.model]  # Use model from arguments
         # debug_task_name = [args.task]  # Use task from arguments
-        debug_model_names = ["mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x7B-v0.1"]  # Use model from arguments
+        debug_model_names = ["microsoft/phi-2", "mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x7B-v0.1",
+                            "databricks/dbrx-instruct", "databricks/dbrx-base",
+                            "mistralai/Mixtral-8x22B-v0.1", "mistralai/Mixtral-8x22B-Instruct-v0.1", "alpindale/WizardLM-2-8x22B",
+                            "CohereForAI/c4ai-command-r-plus"]  # Use model from arguments
         debug_task_name = ['mmlu', 'selfcheckgpt']  # Use task from arguments
-        precisions = ['float16', 'float16', '8bit']
+        precisions = ['4bit', 'float16', 'float32', '8bit']
         task_lst = TASKS_HARNESS.copy()
         for precision in precisions:
-            for task in task_lst:
-                for debug_model_name in debug_model_names:
+            for debug_model_name in debug_model_names:
+                for task in task_lst:
                     task_name = task.benchmark
                     if task_name not in debug_task_name:
                         continue
-                    eval_request = EvalRequest(
-                        model=debug_model_name,
-                        private=False,
-                        status="",
-                        json_filepath="",
-                        precision=args.precision,  # Use precision from arguments
-                        inference_framework=args.inference_framework  # Use inference framework from arguments
-                    )
-                    results = process_evaluation(task, eval_request, limit=args.limit)
+                    try:
+                        eval_request = EvalRequest(
+                            model=debug_model_name,
+                            private=False,
+                            status="",
+                            json_filepath="",
+                            precision=precision,  # Use precision from arguments
+                            inference_framework=args.inference_framework  # Use inference framework from arguments
+                        )
+                        results = process_evaluation(task, eval_request, limit=args.limit)
+                    except Exception as e:
+                        print(f"debug running error: {e}")
     else:
         while True:
             res = False

src/backend/manage_requests.py

@@ -37,12 +37,11 @@ class EvalRequest:
             # Quantized models need some added config, the install of bits and bytes, etc
             # elif self.precision == "8bit":
             #    model_args += ",load_in_8bit=True"
-            # elif self.precision == "4bit":
-            #    model_args += ",load_in_4bit=True"
+        elif self.precision == "4bit":
+           model_args += ",load_in_4bit=True"
             # elif self.precision == "GPTQ":
             # A GPTQ model does not need dtype to be specified,
             # it will be inferred from the config
-            pass
         elif self.precision == "8bit":
             model_args += ",load_in_8bit=True"
         else:

src/display/utils.py

@@ -16,7 +16,24 @@ MULTIPLE_CHOICEs = ["mmlu"]
 GPU_TEMP = 'Temp(C)'
 GPU_Power = 'Power(W)'
 GPU_Mem = 'Mem(M)'
+GPU_Name = "GPU"
 GPU_Util = 'Util(%)'
+BATCH_SIZE = 'bs'
+
+system_metrics_to_name_map = {
+    "end_to_end_time": f"{E2Es}",
+    "prefilling_time": f"{PREs}",
+    "decoding_throughput": f"{TS}",
+}
+
+gpu_metrics_to_name_map = {
+    GPU_Util: GPU_Util,
+    GPU_TEMP: GPU_TEMP,
+    GPU_Power: GPU_Power,
+    GPU_Mem: GPU_Mem,
+    "batch_size": BATCH_SIZE,
+    GPU_Name: GPU_Name,
+}
 
 @dataclass
 class Task:
@@ -87,14 +104,16 @@ for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
     # System performance metrics
     auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_batch_size", ColumnContent, ColumnContent(f"{task.value.col_name} {BATCH_SIZE}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Name}", "str", True)])
     if task.value.benchmark in MULTIPLE_CHOICEs:
         continue
-    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", False)])
     auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} {TS}", "number", True)])
 
     auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu_power", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Power}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu_temp", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_TEMP}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_power", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Power}", "number", False)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_temp", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_TEMP}", "number", False)])
     auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True)])
 
 # Model information

src/leaderboard/read_evals.py

@@ -103,6 +103,10 @@ class EvalResult:
 
                 if to_add is True:
                     multiplier = 100.0
+                    if "GPU" in metric:
+                        results[benchmark][metric] = value
+                        continue
+
                     if "rouge" in metric and "truthful" not in benchmark:
                         multiplier = 1.0
                     if "squad" in benchmark:
@@ -111,6 +115,10 @@ class EvalResult:
                         multiplier = 1.0
                     if "throughput" in metric:
                         multiplier = 1.0
+                    if "batch_" in metric or "Mem" in metric or "Util" in metric:
+                        multiplier = 1
+
+
                     # print('RESULTS', data['results'])
                     # print('XXX', benchmark, metric, value, multiplier)
                     results[benchmark][metric] = value * multiplier

src/populate.py

@@ -12,7 +12,7 @@ from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, update_
 
 from src.backend.envs import Tasks as BackendTasks
 from src.display.utils import Tasks
-from src.display.utils import E2Es, PREs, TS, GPU_Mem, GPU_Power, GPU_TEMP, GPU_Util
+from src.display.utils import system_metrics_to_name_map, gpu_metrics_to_name_map
 
 def get_leaderboard_df(
     results_path: str,
@@ -45,19 +45,7 @@ def get_leaderboard_df(
         bm = (task.benchmark, task.metric)
         name_to_bm_map[name] = bm
 
-    # bm_to_name_map = {bm: name for name, bm in name_to_bm_map.items()}
-    system_metrics_to_name_map = {
-        "end_to_end_time": f"{E2Es}",
-        "prefilling_time": f"{PREs}",
-        "decoding_throughput": f"{TS}",
-    }
 
-    gpu_metrics_to_name_map = {
-        GPU_Util: GPU_Util,
-        GPU_TEMP: GPU_TEMP,
-        GPU_Power: GPU_Power,
-        GPU_Mem: GPU_Mem
-    }
 
     all_data_json = []
     for entry in all_data_json_:
@@ -73,7 +61,6 @@ def get_leaderboard_df(
                 for gpu_metric, metric_namne in gpu_metrics_to_name_map.items():
                     if gpu_metric in entry[k]:
                         new_entry[f"{k} {metric_namne}"] = entry[k][gpu_metric]
-
         all_data_json += [new_entry]
 
     # all_data_json.append(baseline_row)

src/utils.py

@@ -3,10 +3,10 @@ from huggingface_hub import snapshot_download
 import subprocess
 import re
 try:
-    from src.display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util
+    from src.display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util, GPU_Name
 except:
     print("local debug: from display.utils")
-    from display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util
+    from display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util, GPU_Name
 
 def my_snapshot_download(repo_id, revision, local_dir, repo_type, max_workers):
     for i in range(10):
@@ -49,24 +49,36 @@ def parse_nvidia_smi():
 
     # Regex to extract the relevant data for each GPU
     gpu_info_pattern = re.compile(r'(\d+)C\s+P\d+\s+(\d+)W / \d+W\s+\|\s+(\d+)MiB / \d+MiB\s+\|\s+(\d+)%')
+    gpu_name_pattern = re.compile(r'NVIDIA\s+([\w\s]+?\d+GB)')
     lines = output.split('\n')
-
+    gpu_name = ""
     for line in lines:
         match = gpu_info_pattern.search(line)
+        name_match = gpu_name_pattern.search(line)
+
+        gpu_info = {}
+
+        if name_match:
+            # print(name_match)
+            gpu_name = name_match.group(1).strip()
         if match:
             temp, power_usage, mem_usage, gpu_util = map(int, match.groups())
-            gpu_stats.append({
+            gpu_info.update({
                 GPU_TEMP: temp,
                 GPU_Power: power_usage,
                 GPU_Mem: mem_usage,
                 GPU_Util: gpu_util
             })
-
+        # print(f"gpu_info: {gpu_info}")
+        if len(gpu_info) >= 4:
+            gpu_stats.append(gpu_info)
+    gpu_name = f"{len(gpu_stats)}x{gpu_name}"
     gpu_stats_total = {
                         GPU_TEMP: 0,
                         GPU_Power: 0,
                         GPU_Mem: 0,
-                        GPU_Util: 0
+                        GPU_Util: 0,
+                        GPU_Name: gpu_name
                     }
     for gpu_stat in gpu_stats:
         gpu_stats_total[GPU_TEMP] += gpu_stat[GPU_TEMP]
@@ -77,7 +89,6 @@ def parse_nvidia_smi():
     gpu_stats_total[GPU_TEMP] /= len(gpu_stats)
     gpu_stats_total[GPU_Power] /= len(gpu_stats)
     gpu_stats_total[GPU_Util] /= len(gpu_stats)
-
     return [gpu_stats_total]
 
 def monitor_gpus(stop_event, interval, stats_list):
@@ -88,11 +99,28 @@ def monitor_gpus(stop_event, interval, stats_list):
         stop_event.wait(interval)
 
 def analyze_gpu_stats(stats_list):
+    # Check if the stats_list is empty, and return None if it is
     if not stats_list:
         return None
-    avg_stats = {key: sum(d[key] for d in stats_list) / len(stats_list) for key in stats_list[0]}
-    return avg_stats
 
+    # Initialize dictionaries to store the stats
+    avg_stats = {}
+    max_stats = {}
+
+    # Calculate average stats, excluding 'GPU_Mem'
+    for key in stats_list[0].keys():
+        if key != GPU_Mem and key != GPU_Name:
+            total = sum(d[key] for d in stats_list)
+            avg_stats[key] = total / len(stats_list)
+
+    # Calculate max stats for 'GPU_Mem'
+    max_stats[GPU_Mem] = max(d[GPU_Mem] for d in stats_list)
+    if GPU_Name in stats_list[0]:
+        avg_stats[GPU_Name] = stats_list[0][GPU_Name]
+    # Update average stats with max GPU memory usage
+    avg_stats.update(max_stats)
+
+    return avg_stats
 
 if __name__ == "__main__":
     print(analyze_gpu_stats(parse_nvidia_smi()))