refactor: streamline leaderboard functionality and enhance submission process

app.py

@@ -1,12 +1,14 @@
 import gradio as gr
-import pandas as pd
-from datasets import load_dataset
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+from gradio_leaderboard import Leaderboard, SelectColumns
+
+from utils import (
+    LLM_BENCHMARKS_ABOUT_TEXT, 
+    LLM_BENCHMARKS_SUBMIT_TEXT, 
+    custom_css, 
+    prepare_leaderboard_data, 
+    submit
+)
 
-# HF Variables
-USERNAME = "alinf"
-DATASET_NAME = "crcis_multiple-choice_results"
-SPLIT_NAME = "results"
 
 # Dataset Variables
 overall_score_column = "overall"
@@ -15,300 +17,53 @@ precision_column = "precision"
 num_params_column = "#parameters (B)"
 num_params_range = [0, 100]  # in billions
 
-# Leaderboard title and description
-TITLE = """
-<h1 align="center">🏆 {} Leaderboard</h1>
-""".format(USERNAME)
+# Prepare the data
+leaderboard_df = prepare_leaderboard_data(overall_score_column, model_name_column)
 
-DESCRIPTION = """
-<p align="center">Evaluating Large Language Models on {} Benchmark</p>
-<p align="center">
-    <a href="https://huggingface.co/datasets/{}/{}">Dataset</a> •
-    <a href="https://arxiv.org/abs/xxxx.xxxxx">Paper</a> •
-    <a href="https://github.com/your-username/your-benchmark">GitHub</a>
-</p>
-""".format(USERNAME, USERNAME, DATASET_NAME)
+# Determine columns data type (markdown for all to support clickable links)
+columns_datatype = ["markdown" for _ in range(len(leaderboard_df.columns))]
 
-def load_results():
-    """Load results from HuggingFace dataset."""
-    try:
-        dataset = load_dataset(
-            f"{USERNAME}/{DATASET_NAME}", 
-            split=SPLIT_NAME,
-            download_mode="force_redownload"
-        )
-        print(f"Loaded {len(dataset)} entries from the dataset.")
-        return dataset
-    except Exception as e:
-        print(f"Error loading dataset: {e}")
-        # Return sample data if loading fails
-        return pd.DataFrame([
-            {
-                "rank": 1, 
-                "model": "Sample Model", 
-                "overall": 75.5,
-                "precision": "fp16",
-                "#parameters (B)": 7.0
-            }
-        ])
+NUM_MODELS = len(leaderboard_df)
 
-def prepare_leaderboard_data():
-    """Prepare and format the leaderboard DataFrame."""
-    data = load_results()
-    df = pd.DataFrame(data)
-    
-    # Remove columns with parentheses (except those with #)
-    df = df[[col for col in df.columns if ("(" not in col and ")" not in col) or "#" in col]]
-    
-    # Sort by overall score (descending)
-    df = df.sort_values(by=overall_score_column, ascending=False)
+with gr.Blocks(
+    css=custom_css,
+    theme=gr.themes.Default(
+        font=["sans-serif", "ui-sans-serif", "system-ui"], 
+        font_mono=["monospace", "ui-monospace", "Consolas"]
+    )
+) as demo:
+    gr.Markdown("""
+    # 🏆 CRCIS LLM Leaderboard
+    """)
     
-    # Add rank column
-    df.insert(0, "Rank", range(1, len(df) + 1))
+    gr.Markdown(f"""
+    - **Total Models**: {NUM_MODELS}
+    """)
     
-    # Add a hidden column for model links (useful for search)
-    df['model_link'] = df[model_name_column].apply(
-        lambda x: f"https://huggingface.co/{x}" if pd.notna(x) else ""
-    )
+    with gr.Tab("🏆 Leaderboard"):
+        Leaderboard(
+            value=leaderboard_df,
+            datatype=columns_datatype,
+            select_columns=SelectColumns(
+                default_selection=leaderboard_df.columns.tolist(),
+                cant_deselect=[model_name_column],
+                label="Select Columns to Show",
+            ),
+            search_columns=[model_name_column],
+            filter_columns=[precision_column, num_params_column],
+        )
     
-    return df
-
-# Custom CSS for styling
-custom_css = """
-.gradio-container {
-    font-family: 'IBM Plex Sans', sans-serif;
-}
-
-.leaderboard-table {
-    margin-top: 20px;
-}
-
-h1 {
-    color: #2c3e50;
-    margin-bottom: 10px;
-}
-
-.description {
-    color: #7f8c8d;
-    margin-bottom: 30px;
-}
-
-.tab-nav button {
-    font-size: 16px;
-    font-weight: 600;
-}
-"""
-
-# Prepare the data
-df = prepare_leaderboard_data()
-
-print(df)
-
-# Determine filterable columns
-filterable_columns = []
-filterable_columns.append(precision_column)
-filterable_columns.append(
-    ColumnFilter(num_params_column, type="slider", min=num_params_range[0], max=num_params_range[1])
-)
-
-# Determine search columns
-search_columns = []
-search_columns.append(model_name_column)
-
-# Columns to hide
-hidden_columns = ["model_link"]
-
-# Default visible columns (all except hidden ones)
-visible_columns = [col for col in df.columns if col not in hidden_columns]
-
-columns_datatype = df[visible_columns].dtypes.to_list()
-
-# Create Gradio interface
-with gr.Blocks(title=f"{USERNAME} Benchmark Leaderboard", css=custom_css) as demo:
-    gr.HTML(TITLE)
-    gr.Markdown(DESCRIPTION, elem_classes="description")
+    with gr.TabItem("📖 About"):
+        gr.Markdown(LLM_BENCHMARKS_ABOUT_TEXT)
     
-    with gr.Tabs():
-        # Main Leaderboard Tab
-        with gr.TabItem("🏆 Leaderboard"):
-            gr.Markdown("""
-            **💡 Tips:** 
-            - Models are ranked by overall score by default
-            - Click column headers to sort by different metrics
-            - Use filters to find models that match your requirements
-            - Search supports multiple queries separated by semicolons (;)
-            ---
-            """)
-            
-            # Create the leaderboard with gradio_leaderboard
-            Leaderboard(
-                value=df,
-                datatype=columns_datatype,
-                select_columns=SelectColumns(
-                    default_selection=visible_columns,
-                    cant_deselect=[model_name_column, "Rank"],
-                    label="📊 Select Columns to Display",
-                    info="Choose which metrics to show in the leaderboard"
-                ),
-                search_columns=search_columns if search_columns else None,
-                hide_columns=hidden_columns,
-                filter_columns=filterable_columns if filterable_columns else None,
-                interactive=False,
-                elem_classes="leaderboard-table"
-            )
-        
-        # About Tab
-        with gr.TabItem("📖 About"):
-            gr.Markdown(f"""
-            ## About This Benchmark
-            
-            This leaderboard tracks the performance of Large Language Models on the **{USERNAME} Benchmark**.
-            
-            ### Evaluation Details
-            - **Tasks**: Multiple-choice questions across multiple subjects
-            - **Evaluation Method**: 5-shot evaluation using lm-evaluation-harness
-            - **Metric**: Accuracy (%)
-            - **Dataset**: [{USERNAME}/{DATASET_NAME}](https://huggingface.co/datasets/{USERNAME}/{DATASET_NAME})
-            
-            ### How to Submit Your Model
-            
-            1. **Evaluate your model** using lm-evaluation-harness:
-            ```bash
-            lm_eval --model hf \\
-                --model_args pretrained=your-org/your-model \\
-                --tasks {DATASET_NAME} \\
-                --num_fewshot 5 \\
-                --output_path ./results
-            ```
-            
-            2. **Submit results** via [GitHub Issues](https://github.com/your-username/your-benchmark/issues)
-            
-            3. **Include the following information**:
-               - Model name and organization
-               - All evaluation scores
-               - Evaluation logs or result files
-               - Model precision and parameter count
-            
-            ### Evaluation Criteria
-            
-            We verify all submissions to ensure:
-            - Results are reproducible
-            - Evaluation was performed correctly
-            - Model information is accurate
-            
-            Results are typically added within 48 hours of submission.
-            
-            ### Citation
-            
-            If you use this benchmark in your research, please cite:
-            
-            ```bibtex
-            @article{{yourbenchmark2024,
-                title={{Your Benchmark Title}},
-                author={{Your Name}},
-                journal={{arXiv preprint arXiv:xxxx.xxxxx}},
-                year={{2024}}
-            }}
-            ```
-            
-            ### Contact
-            
-            For questions or issues, please:
-            - Open an issue on [GitHub](https://github.com/your-username/your-benchmark/issues)
-            - Contact us at your-email@example.com
-            """)
+    with gr.Tab("📤 Submit"):
+        gr.Markdown(LLM_BENCHMARKS_SUBMIT_TEXT)
+        model_name = gr.Textbox(label="Model name")
+        model_id = gr.Textbox(label="Model ID (e.g., username/model-name)")
+        contact_email = gr.Textbox(label="Contact E-Mail")
+        submit_btn = gr.Button("Submit")
         
-        # Submit Tab
-        with gr.TabItem("📤 Submit"):
-            gr.Markdown("""
-            ## Submit Your Model Results
-            
-            Ready to add your model to the leaderboard? Follow these steps:
-            """)
-            
-            with gr.Accordion("📝 Submission Form", open=True):
-                gr.Markdown("""
-                Please fill out the form below with your model information.
-                Note: This is for display purposes. Actual submissions should be made via GitHub Issues.
-                """)
-                
-                with gr.Row():
-                    model_name = gr.Textbox(
-                        label="Model Name",
-                        placeholder="e.g., gpt-4",
-                        info="The name of your model"
-                    )
-                    organization = gr.Textbox(
-                        label="Organization",
-                        placeholder="e.g., OpenAI",
-                        info="Your organization or username"
-                    )
-                
-                with gr.Row():
-                    precision = gr.Dropdown(
-                        label="Precision",
-                        choices=["fp32", "fp16", "bf16", "int8", "int4"],
-                        value="fp16",
-                        info="Model precision used for evaluation"
-                    )
-                    param_count = gr.Number(
-                        label="Parameters (Billions)",
-                        info="Number of parameters in billions (e.g., 7.0)"
-                    )
-                
-                with gr.Row():
-                    overall_score = gr.Number(
-                        label="Overall Score",
-                        info="Average score across all tasks (e.g., 75.5)"
-                    )
-                
-                results_file = gr.File(
-                    label="Upload Results JSON",
-                    file_types=[".json", ".jsonl"],
-                )
-                
-                contact_email = gr.Textbox(
-                    label="Contact Email",
-                    placeholder="your-email@example.com",
-                    info="We'll contact you about your submission"
-                )
-                
-                submit_btn = gr.Button("📤 Submit for Review", variant="primary", size="lg")
-                
-                output_message = gr.Markdown(visible=False)
-            
-            def handle_submission(model, org, prec, params, score, file, email):
-                """Handle submission form."""
-                if not all([model, org, score, email]):
-                    return gr.Markdown(
-                        "⚠️ Please fill in all required fields.",
-                        visible=True
-                    )
-                
-                return gr.Markdown(
-                    f"""
-                    ✅ **Thank you for your submission!**
-                    
-                    We've received your submission for **{model}** from **{org}**.
-                    
-                    **Next steps:**
-                    1. Create an issue on our [GitHub repository](https://github.com/your-username/your-benchmark/issues)
-                    2. Include all the information you provided here
-                    3. Attach your results file
-                    
-                    We'll review your submission and add it to the leaderboard within 48 hours.
-                    
-                    You'll receive a notification at **{email}** when your model is added.
-                    """,
-                    visible=True
-                )
-            
-            submit_btn.click(
-                fn=handle_submission,
-                inputs=[model_name, organization, precision, param_count, 
-                       overall_score, results_file, contact_email],
-                outputs=output_message
-            )
+        submit_btn.click(submit, inputs=[model_name, model_id, contact_email], outputs=[])
 
 if __name__ == "__main__":
     demo.launch()

envs.py

@@ -0,0 +1,21 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN")
+
+OWNER = "alinf"  # Change to your HF username
+RESULT_DATASET_NAME = "crcis_multiple-choice_results"
+# ----------------------------------
+
+REQUEST_QUEUE_REPO = f"{OWNER}/crcis_request_queue"
+
+# If you setup a cache later, just set/change 'HF_HOME' env variable
+CACHE_PATH = os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+
+API = HfApi(token=TOKEN)

utils.py

@@ -0,0 +1,195 @@
+import json
+import os
+from datetime import datetime
+
+import gradio as gr
+import pandas as pd
+from datasets import load_dataset
+
+from envs import API, EVAL_REQUESTS_PATH, REQUEST_QUEUE_REPO, OWNER, RESULT_DATASET_NAME
+
+custom_css = """
+.gradio-container {
+    font-family: 'IBM Plex Sans', sans-serif;
+}
+
+.leaderboard-table {
+    margin-top: 20px;
+}
+
+h1 {
+    color: #2c3e50;
+    margin-bottom: 10px;
+}
+
+.description {
+    color: #7f8c8d;
+    margin-bottom: 30px;
+}
+
+.tab-nav button {
+    font-size: 16px;
+    font-weight: 600;
+}
+
+#leaderboard-table {
+    margin-top: 15px;
+    text-align: center;
+}
+
+#leaderboard-table th,
+#leaderboard-table td {
+    text-align: center;
+    vertical-align: middle;
+}
+
+#leaderboard-table td:first-child,
+#leaderboard-table th:first-child {
+    text-align: left;
+    max-width: 500px;
+}
+"""
+
+LLM_BENCHMARKS_ABOUT_TEXT = f"""
+# CRCIS LLM Leaderboard
+
+This leaderboard tracks the performance of Large Language Models on the **CRCIS Benchmark**.
+
+## Evaluation Details
+- **Tasks**: Multiple-choice questions across multiple subjects including:
+  - Humanities - History
+  - Quran - General Information
+  - Quran - Tafsir
+- **Evaluation Method**: 5-shot evaluation using lm-evaluation-harness
+- **Metric**: Accuracy (%)
+- **Dataset**: [{OWNER}/{RESULT_DATASET_NAME}](https://huggingface.co/datasets/{OWNER}/{RESULT_DATASET_NAME})
+
+## Background and Goals
+
+This leaderboard provides a comprehensive benchmarking system for evaluating LLMs on CRCIS-specific tasks. 
+The evaluation framework is based on the open-source [LM Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness), 
+offering a reliable platform for assessing model performance on specialized domain knowledge.
+
+## Data Integrity
+
+To maintain evaluation integrity and prevent overfitting, the full benchmark dataset is used for evaluation.
+This approach ensures that results genuinely represent each model's capabilities.
+
+"""
+
+LLM_BENCHMARKS_SUBMIT_TEXT = """## Submitting a Model for Evaluation
+
+To submit your model for evaluation, follow these steps:
+
+1. **Ensure your model is on Hugging Face**: Your model must be publicly available on [Hugging Face](https://huggingface.co/).
+
+2. **Submit Request**: Fill out the form below with your model's information and Hugging Face identifier.
+
+3. **Evaluation Queue**: Submissions will be queued and processed. The evaluation may take some time depending on the queue.
+
+4. **Results**: Once the evaluation is complete, your model's results will be E-mailed to you.
+
+We appreciate your contributions to the LLM ecosystem!
+"""
+
+
+def load_results_from_hf_dataset():
+    """Load results from HuggingFace dataset."""
+    try:
+        dataset = load_dataset(
+            f"{OWNER}/{RESULT_DATASET_NAME}", 
+            split="results",
+            download_mode="force_redownload"
+        )
+        print(f"Loaded {len(dataset)} entries from the dataset.")
+        return pd.DataFrame(dataset)
+    except Exception as e:
+        print(f"Error loading dataset: {e}")
+        # Return sample data if loading fails
+        return pd.DataFrame([
+            {
+                "model": "Sample Model", 
+                "overall": 75.5,
+                "precision": "fp16",
+                "#parameters (B)": 7.0
+            }
+        ])
+
+
+def sort_dataframe_by_column(df, column_name):
+    """Sort dataframe by specified column in descending order."""
+    if column_name not in df.columns:
+        raise ValueError(f"Column '{column_name}' does not exist in the DataFrame.")
+    return df.sort_values(by=column_name, ascending=False).reset_index(drop=True)
+
+
+def make_clickable_model(model_name):
+    """Make model name clickable with link to HuggingFace."""
+    link = f"https://huggingface.co/{model_name}"
+    style = "color: var(--link-text-color); text-decoration: underline; text-decoration-style: dotted;"
+    return f'<a target="_blank" href="{link}" style="{style}">{model_name}</a>'
+
+
+def prepare_leaderboard_data(overall_score_column="overall", model_name_column="model"):
+    """Prepare and format the leaderboard DataFrame."""
+    df = load_results_from_hf_dataset()
+    
+    # Remove columns with parentheses (except those with #)
+    df = df[[col for col in df.columns if ("(" not in col and ")" not in col) or "#" in col]]
+    
+    # Sort by overall score (descending)
+    df = sort_dataframe_by_column(df, overall_score_column)
+    
+    # Make model names clickable
+    df[model_name_column] = df[model_name_column].apply(make_clickable_model)
+    
+    return df
+
+
+def submit(model_name, model_id, contact_email):
+    """Handle model submission to evaluation queue."""
+    if model_name == "" or model_id == "" or contact_email == "":
+        gr.Info("Please fill all the fields")
+        return
+
+    try:
+        user_name = ""
+        if "/" in model_id:
+            user_name = model_id.split("/")[0]
+            model_path = model_id.split("/")[1]
+        else:
+            gr.Error("Model ID must be in the format 'username/model-name'")
+            return
+
+        eval_entry = {
+            "model_name": model_name,
+            "model_id": model_id,
+            "contact_email": contact_email,
+        }
+
+        # Get the current timestamp to add to the filename
+        timestamp = datetime.now().strftime("%Y%m%d")
+
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        os.makedirs(OUT_DIR, exist_ok=True)
+
+        # Add the timestamp to the filename
+        out_path = f"{OUT_DIR}/{user_name}_{model_path}_{timestamp}.json"
+
+        with open(out_path, "w") as f:
+            f.write(json.dumps(eval_entry))
+
+        print("Uploading eval file")
+        API.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split("eval-queue/")[1],
+            repo_id=REQUEST_QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model_name} to eval queue",
+        )
+
+        gr.Info("Successfully submitted", duration=10)
+        # Remove the local file
+        os.remove(out_path)
+    except Exception as e:
+        gr.Error(f"Error submitting the model: {e}")