[Add] load models and datasets from hub, compute similarities

app.py

@@ -59,9 +59,31 @@ def validate_inputs(selected_models, selected_dataset):
         raise gr.Error("Please select at least one model!")
     if not selected_dataset:
         raise gr.Error("Please select a dataset!")
+    
+
+def update_datasets_based_on_models(selected_models, current_dataset):
+    # Get available datasets for selected models
+    available_datasets = get_leaderboard_datasets(selected_models) if selected_models else []
+    
+    # Check if current dataset is still valid
+    valid_dataset = current_dataset if current_dataset in available_datasets else None
+    
+    return gr.Dropdown.update(
+        choices=available_datasets,
+        value=valid_dataset
+    )
 
 with gr.Blocks(title="LLM Similarity Analyzer") as demo:
     gr.Markdown("## Model Similarity Comparison Tool")
+
+    model_dropdown = gr.Dropdown(
+        choices=get_leaderboard_models_cached(),
+        label="Select Models",
+        multiselect=True,
+        filterable=True,
+        allow_custom_value=False,
+        info="Search and select multiple models"
+    )
     
     with gr.Row():
         dataset_dropdown = gr.Dropdown(
@@ -76,14 +98,11 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
             label="Select Metric",
             info="Select a similarity metric to compute"
         )
-        
-    model_dropdown = gr.Dropdown(
-        choices=get_leaderboard_models_cached(),
-        label="Select Models",
-        multiselect=True,
-        filterable=True,
-        allow_custom_value=False,
-        info="Search and select multiple models"
+
+    model_dropdown.change(
+        fn=update_datasets_based_on_models,
+        inputs=[model_dropdown, dataset_dropdown],
+        outputs=dataset_dropdown
     )
     
     generate_btn = gr.Button("Generate Heatmap", variant="primary")

src/dataloading.py

@@ -1,33 +1,27 @@
 import datasets
 import numpy as np
-
 from huggingface_hub import HfApi
+
 from functools import lru_cache
 
 
 def get_leaderboard_models():
-    #api = HfApi()
+    api = HfApi()
     
     # List all datasets in the open-llm-leaderboard organization
-    #datasets = api.list_datasets(author="open-llm-leaderboard")
+    datasets = api.list_datasets(author="open-llm-leaderboard")
     
     models = []
-    #for dataset in datasets:
-    #    if dataset.id.endswith("-details"):
-    #        # Format: "open-llm-leaderboard/<provider>__<model_name>-details"
-    #        model_part = dataset.id.split("/")[-1].replace("-details", "")
-    #        provider, model = model_part.split("__", 1)
-    #        models.append(f"{provider}/{model}")
+    for dataset in datasets:
+        if dataset.id.endswith("-details"):
+            # Format: "open-llm-leaderboard/<provider>__<model_name>-details"
+            model_part = dataset.id.split("/")[-1].replace("-details", "")
+            if "__" in model_part:
+                provider, model = model_part.split("__", 1)
+                models.append(f"{provider}/{model}")
+            else:
+                models.append(model_part)
     
-    # Example models
-    models = [
-        "meta_llama/Llama-3.2-1B-Instruct",
-        "meta_llama/Llama-3.2-3B-Instruct",
-        "meta_llama/Llama-3.1-8B-Instruct",
-        "meta_llama/Llama-3.1-70B-Instruct",
-        "meta_llama/Llama-3.3-70B-Instruct",
-        ]
-
     return sorted(models)
 
 
@@ -37,15 +31,27 @@ def get_leaderboard_models_cached():
     return get_leaderboard_models()
 
 
-def get_leaderboard_datasets():
-    return [
-        "ai2_arc",
-        "hellaswag",
-        "mmlu_pro",
-        "truthful_qa",
-        "winogrande",
-        "gsm8k"
-    ]
+def get_leaderboard_datasets(model_ids):
+    if model_ids is None:
+        return ['bbh_boolean_expressions', 'bbh_causal_judgement', 'bbh_date_understanding', 'bbh_disambiguation_qa', 'bbh_formal_fallacies', 'bbh_geometric_shapes', 'bbh_hyperbaton', 'bbh_logical_deduction_five_objects', 'bbh_logical_deduction_seven_objects', 'bbh_logical_deduction_three_objects', 'bbh_movie_recommendation', 'bbh_navigate', 'bbh_object_counting', 'bbh_penguins_in_a_table', 'bbh_reasoning_about_colored_objects', 'bbh_ruin_names', 'bbh_salient_translation_error_detection', 'bbh_snarks', 'bbh_sports_understanding', 'bbh_temporal_sequences', 'bbh_tracking_shuffled_objects_five_objects', 'bbh_tracking_shuffled_objects_seven_objects', 'bbh_tracking_shuffled_objects_three_objects', 'bbh_web_of_lies', 'gpqa_diamond', 'gpqa_extended', 'gpqa_main', 'ifeval', 'math_algebra_hard', 'math_counting_and_prob_hard', 'math_geometry_hard', 'math_intermediate_algebra_hard', 'math_num_theory_hard', 'math_prealgebra_hard', 'math_precalculus_hard', 'mmlu_pro', 'musr_murder_mysteries', 'musr_object_placements', 'musr_team_allocation']
+
+    # Map each model to its corresponding leaderboard version
+    leaderboard_model_ids = [f"open-llm-leaderboard/{model_id.replace('/', '__')}-details" for model_id in model_ids]
+
+    model_datasets = {}
+
+    for model_id in leaderboard_model_ids:
+        # Retrieve the list of available configuration names
+        config_names = datasets.get_dataset_config_names(model_id)
+        dataset_names = [name.split("__leaderboard_")[-1] for name in config_names]
+        model_datasets[model_id] = set(dataset_names)
+
+    # Compute the intersection of datasets across all models
+    if model_datasets:
+        common_datasets = set.intersection(*model_datasets.values())
+
+    return sorted(common_datasets)
+    
 
 def filter_labels(doc):
     labels = []
@@ -85,4 +91,13 @@ def load_run_data(model_name, dataset_name):
         log_probs = []
         labels = []
 
-    return log_probs, labels
+    return log_probs, labels
+
+
+if __name__ == "__main__":
+    model_ids = [
+        'Qwen/Qwen2.5-7B-Instruct'
+    ]
+    
+    datasets = get_leaderboard_datasets(model_ids)
+    print(datasets)   

src/similarity.py

@@ -1,15 +1,28 @@
+import numpy as np
+
 from src.dataloading import load_run_data
-from lmsim.metrics import Kappa_p
-import random
+from lmsim.metrics import Metric, Kappa_p, EC
 
 
-def compute_similarity(selected_model_a, selected_model_b, selected_dataset):
-    """
-    probs_a, gt_a = load_run_data(selected_model_a, selected_dataset)
-    probs_b, gt_b = load_run_data(selected_model_b, selected_dataset)
 
-    assert len(probs_a) == len(probs_b), f"Models must have the same number of responses: {len(probs_a)} != {len(probs_b)}"
+def load_data_and_compute_similarities(models, dataset, metric_name):
+    # Load data
+    probs = []
+    gts = []
+    for model in models:
+        model_probs, model_gt = load_run_data(model, dataset)
+        probs.append(model_probs)
+        gts.append(model_gt)
     
+    # Compute pairwise similarities
+    similarities = compute_pairwise_similarities(probs, gts, metric_name)
+    return similarities
+
+
+def compute_similarity(metric: Metric, probs_a: list[np.array], gt_a: list[int], probs_b: list[np.array], gt_b: list[int]) -> float:
+    # Check that the models have the same number of responses
+    assert len(probs_a) == len(probs_b), f"Models must have the same number of responses: {len(probs_a)} != {len(probs_b)}"
+
     # Only keep responses where the ground truth is the same
     output_a = []
     output_b = []
@@ -21,10 +34,26 @@ def compute_similarity(selected_model_a, selected_model_b, selected_dataset):
             gt.append(gt_a[i])
 
     # Placeholder similarity value
-    kappa_p = Kappa_p()
-    similarity = kappa_p.compute_k(output_a, output_b, gt)
-    """
+    similarity = metric.compute_k(output_a, output_b, gt)
     
-    similarity = random.random()
+    return similarity
+
+
+def compute_pairwise_similarities(metric_name: str, probs: list[list[np.array]], gts: list[list[int]]) -> np.array:
+    # Select chosen metric
+    if metric_name == "Kappa_p (prob.)":
+        metric = Kappa_p()
+    elif metric_name == "Kappa_p (det.)":
+        metric = Kappa_p()
+    elif metric_name == "Error Consistency":
+        metric = EC()
+    else:
+        raise ValueError(f"Invalid metric: {metric_name}")
     
-    return similarity
+
+    similarities = np.zeros((len(probs), len(probs)))
+    for i in range(len(probs)):
+        for j in range(i, len(probs)):
+            similarities[i, j] = compute_similarity(metric, probs[i], gts[i], probs[j], gts[j])
+            similarities[j, i] = similarities[i, j]
+    return similarities