uncertainty benchmark

app.py

@@ -19,6 +19,7 @@ from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
     ASSET_BENCHMARK_COLS,
+    UNCERTAINTY_BENCHMARK_COLS,
     COLS,
     ASSET_COLS,
     UNCERTAINTY_COLS,
@@ -65,8 +66,10 @@ print(ASSET_COLS)
 
 ASSET_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ASSET_COLS, ASSET_BENCHMARK_COLS, AssetTasks)
 
-UNCERTAINTY_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, UNCERTAINTY_COLS, ASSET_BENCHMARK_COLS, UncertaintyTasks)
-
+UNCERTAINTY_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, UNCERTAINTY_COLS, UNCERTAINTY_BENCHMARK_COLS, UncertaintyTasks)
+missing_uncertainties = (UNCERTAINTY_LEADERBOARD_DF[UNCERTAINTY_BENCHMARK_COLS] == 0).all(axis=1)
+UNCERTAINTY_LEADERBOARD_DF = UNCERTAINTY_LEADERBOARD_DF[~missing_uncertainties]
+UNCERTAINTY_LEADERBOARD_DF = UNCERTAINTY_LEADERBOARD_DF.loc[:,~UNCERTAINTY_LEADERBOARD_DF.columns.duplicated()]
 
 (
     finished_eval_queue_df,

src/display/utils.py

@@ -53,7 +53,7 @@ AutoEvalColumnAsset = make_dataclass("AutoEvalColumnAsset", auto_eval_column_ass
 auto_eval_column_uncertainty_dict = get_auto_eval_column_dict(UncertaintyTasks)
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumnUncertainty = make_dataclass("AutoEvalColumnUncertainty", auto_eval_column_uncertainty_dict, frozen=True)
-
+AutoEvalColumnUncertainty.average.name = 'fmsr_uacc'
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -122,5 +122,5 @@ EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
 ASSET_BENCHMARK_COLS = [t.value.col_name for t in AssetTasks]
-ASSET_BENCHMARK_COLS = [t.value.col_name for t in UncertaintyTasks]
+UNCERTAINTY_BENCHMARK_COLS = [t.value.col_name for t in UncertaintyTasks]
 

src/leaderboard/read_evals.py

@@ -135,8 +135,8 @@ class EvalResult:
 
         for task in task_class:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
-            if task.value.col_name == 'Uncertainty_Score' and self.results[task.value.benchmark] == 0:
-                data_dict[task.value.col_name] = None
+            # if task.value.col_name == 'Uncertainty_Score' and self.results[task.value.benchmark] == 0:
+            #     data_dict[task.value.col_name] = None
         return data_dict