1st

app.py

@@ -0,0 +1,321 @@
+import gradio as gr
+import pandas as pd
+from datasets import load_dataset
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+
+# HF Variables
+USERNAME = "alinf"
+DATASET_NAME = "crcis_multiple-choice_results"
+SPLIT_NAME = "results"
+
+# Dataset Variables
+overall_score_column = "overall"
+model_name_column = "model"
+precision_column = "precision"
+num_params_column = "#parameters (B)"
+num_params_range = [0, 100]  # in billions
+
+# Leaderboard title and description
+TITLE = """
+<h1 align="center">🏆 {} Leaderboard</h1>
+""".format(USERNAME)
+
+DESCRIPTION = """
+<p align="center">Evaluating Large Language Models on {} Benchmark</p>
+<p align="center">
+    <a href="https://huggingface.co/datasets/{}/{}">Dataset</a> •
+    <a href="https://arxiv.org/abs/xxxx.xxxxx">Paper</a> •
+    <a href="https://github.com/your-username/your-benchmark">GitHub</a>
+</p>
+""".format(USERNAME, USERNAME, DATASET_NAME)
+
+def load_results():
+    """Load results from HuggingFace dataset."""
+    try:
+        dataset = load_dataset(
+            f"{USERNAME}/{DATASET_NAME}", 
+            split=SPLIT_NAME,
+            download_mode="force_redownload"
+        )
+        print(f"Loaded {len(dataset)} entries from the dataset.")
+        return dataset
+    except Exception as e:
+        print(f"Error loading dataset: {e}")
+        # Return sample data if loading fails
+        return pd.DataFrame([
+            {
+                "rank": 1, 
+                "model": "Sample Model", 
+                "overall": 75.5,
+                "precision": "fp16",
+                "#parameters (B)": 7.0
+            }
+        ])
+
+def prepare_leaderboard_data():
+    """Prepare and format the leaderboard DataFrame."""
+    data = load_results()
+    df = pd.DataFrame(data)
+    
+    # Remove columns with parentheses (except those with #)
+    df = df[[col for col in df.columns if ("(" not in col and ")" not in col) or "#" in col]]
+    
+    # Sort by overall score (descending)
+    df = df.sort_values(by=overall_score_column, ascending=False)
+    
+    # Add rank column
+    df.insert(0, "Rank", range(1, len(df) + 1))
+    
+    # Add a hidden column for model links (useful for search)
+    df['model_link'] = df[model_name_column].apply(
+        lambda x: f"https://huggingface.co/{x}" if pd.notna(x) else ""
+    )
+    
+    return df
+
+# Custom CSS for styling
+custom_css = """
+.gradio-container {
+    font-family: 'IBM Plex Sans', sans-serif;
+}
+
+.leaderboard-table {
+    margin-top: 20px;
+}
+
+h1 {
+    color: #2c3e50;
+    margin-bottom: 10px;
+}
+
+.description {
+    color: #7f8c8d;
+    margin-bottom: 30px;
+}
+
+.tab-nav button {
+    font-size: 16px;
+    font-weight: 600;
+}
+"""
+
+# Prepare the data
+df = prepare_leaderboard_data()
+
+# Determine filterable columns
+filterable_columns = []
+filterable_columns.append(precision_column)
+filterable_columns.append(
+    ColumnFilter(num_params_column, type="slider", min=num_params_range[0], max=num_params_range[1])
+)
+
+# Determine search columns
+search_columns = []
+search_columns.append(model_name_column)
+
+# Columns to hide
+hidden_columns = ["model_link"]
+
+# Default visible columns (all except hidden ones)
+visible_columns = [col for col in df.columns if col not in hidden_columns]
+
+columns_datatype = df[visible_columns].dtypes.to_list()
+
+# Create Gradio interface
+with gr.Blocks(title=f"{USERNAME} Benchmark Leaderboard", css=custom_css) as demo:
+    gr.HTML(TITLE)
+    gr.Markdown(DESCRIPTION, elem_classes="description")
+    
+    with gr.Tabs():
+        # Main Leaderboard Tab
+        with gr.TabItem("🏆 Leaderboard"):
+            gr.Markdown("""
+            ### How to use this leaderboard:
+            - **Search**: Use the search box to find models by name
+            - **Filter**: Use the filters below to narrow down results by precision, parameters, etc.
+            - **Sort**: Click on column headers to sort by that metric
+            - **Select Columns**: Choose which columns to display using the column selector
+            """)
+            
+            # Create the leaderboard with gradio_leaderboard
+            leaderboard = Leaderboard(
+                value=df,
+                datatype=columns_datatype,
+                select_columns=SelectColumns(
+                    default_selection=visible_columns,
+                    cant_deselect=[model_name_column, "Rank"],
+                    label="📊 Select Columns to Display",
+                    info="Choose which metrics to show in the leaderboard"
+                ),
+                search_columns=search_columns if search_columns else None,
+                hide_columns=hidden_columns,
+                filter_columns=filterable_columns if filterable_columns else None,
+                interactive=False,
+                elem_classes="leaderboard-table"
+            )
+            
+            gr.Markdown("""
+            ---
+            **💡 Tips:** 
+            - Models are ranked by overall score by default
+            - Click column headers to sort by different metrics
+            - Use filters to find models that match your requirements
+            - Search supports multiple queries separated by semicolons (;)
+            - To search in specific columns, use `column_name: query`
+            """)
+        
+        # About Tab
+        with gr.TabItem("📖 About"):
+            gr.Markdown(f"""
+            ## About This Benchmark
+            
+            This leaderboard tracks the performance of Large Language Models on the **{USERNAME} Benchmark**.
+            
+            ### Evaluation Details
+            - **Tasks**: Multiple-choice questions across multiple subjects
+            - **Evaluation Method**: 5-shot evaluation using lm-evaluation-harness
+            - **Metric**: Accuracy (%)
+            - **Dataset**: [{USERNAME}/{DATASET_NAME}](https://huggingface.co/datasets/{USERNAME}/{DATASET_NAME})
+            
+            ### How to Submit Your Model
+            
+            1. **Evaluate your model** using lm-evaluation-harness:
+            ```bash
+            lm_eval --model hf \\
+                --model_args pretrained=your-org/your-model \\
+                --tasks {DATASET_NAME} \\
+                --num_fewshot 5 \\
+                --output_path ./results
+            ```
+            
+            2. **Submit results** via [GitHub Issues](https://github.com/your-username/your-benchmark/issues)
+            
+            3. **Include the following information**:
+               - Model name and organization
+               - All evaluation scores
+               - Evaluation logs or result files
+               - Model precision and parameter count
+            
+            ### Evaluation Criteria
+            
+            We verify all submissions to ensure:
+            - Results are reproducible
+            - Evaluation was performed correctly
+            - Model information is accurate
+            
+            Results are typically added within 48 hours of submission.
+            
+            ### Citation
+            
+            If you use this benchmark in your research, please cite:
+            
+            ```bibtex
+            @article{{yourbenchmark2024,
+                title={{Your Benchmark Title}},
+                author={{Your Name}},
+                journal={{arXiv preprint arXiv:xxxx.xxxxx}},
+                year={{2024}}
+            }}
+            ```
+            
+            ### Contact
+            
+            For questions or issues, please:
+            - Open an issue on [GitHub](https://github.com/your-username/your-benchmark/issues)
+            - Contact us at your-email@example.com
+            """)
+        
+        # Submit Tab
+        with gr.TabItem("📤 Submit"):
+            gr.Markdown("""
+            ## Submit Your Model Results
+            
+            Ready to add your model to the leaderboard? Follow these steps:
+            """)
+            
+            with gr.Accordion("📝 Submission Form", open=True):
+                gr.Markdown("""
+                Please fill out the form below with your model information.
+                Note: This is for display purposes. Actual submissions should be made via GitHub Issues.
+                """)
+                
+                with gr.Row():
+                    model_name = gr.Textbox(
+                        label="Model Name",
+                        placeholder="e.g., gpt-4",
+                        info="The name of your model"
+                    )
+                    organization = gr.Textbox(
+                        label="Organization",
+                        placeholder="e.g., OpenAI",
+                        info="Your organization or username"
+                    )
+                
+                with gr.Row():
+                    precision = gr.Dropdown(
+                        label="Precision",
+                        choices=["fp32", "fp16", "bf16", "int8", "int4"],
+                        value="fp16",
+                        info="Model precision used for evaluation"
+                    )
+                    param_count = gr.Number(
+                        label="Parameters (Billions)",
+                        info="Number of parameters in billions (e.g., 7.0)"
+                    )
+                
+                with gr.Row():
+                    overall_score = gr.Number(
+                        label="Overall Score",
+                        info="Average score across all tasks (e.g., 75.5)"
+                    )
+                
+                results_file = gr.File(
+                    label="Upload Results JSON",
+                    file_types=[".json", ".jsonl"],
+                )
+                
+                contact_email = gr.Textbox(
+                    label="Contact Email",
+                    placeholder="your-email@example.com",
+                    info="We'll contact you about your submission"
+                )
+                
+                submit_btn = gr.Button("📤 Submit for Review", variant="primary", size="lg")
+                
+                output_message = gr.Markdown(visible=False)
+            
+            def handle_submission(model, org, prec, params, score, file, email):
+                """Handle submission form."""
+                if not all([model, org, score, email]):
+                    return gr.Markdown(
+                        "⚠️ Please fill in all required fields.",
+                        visible=True
+                    )
+                
+                return gr.Markdown(
+                    f"""
+                    ✅ **Thank you for your submission!**
+                    
+                    We've received your submission for **{model}** from **{org}**.
+                    
+                    **Next steps:**
+                    1. Create an issue on our [GitHub repository](https://github.com/your-username/your-benchmark/issues)
+                    2. Include all the information you provided here
+                    3. Attach your results file
+                    
+                    We'll review your submission and add it to the leaderboard within 48 hours.
+                    
+                    You'll receive a notification at **{email}** when your model is added.
+                    """,
+                    visible=True
+                )
+            
+            submit_btn.click(
+                fn=handle_submission,
+                inputs=[model_name, organization, precision, param_count, 
+                       overall_score, results_file, contact_email],
+                outputs=output_message
+            )
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,2 @@
+gradio_leaderboard
+datasets