| from dataclasses import dataclass |
| from enum import Enum |
|
|
| @dataclass |
| class EvalDimension: |
| metric: str |
| col_name: str |
|
|
|
|
| |
| |
| class EvalDimensions(Enum): |
| d0 = EvalDimension("speed", "Speed (words/sec)") |
| d1 = EvalDimension("contamination_score", "Contamination Score") |
| d2 = EvalDimension("paraphrasing", "Paraphrasing") |
| d3 = EvalDimension("sentiment analysis", "Sentiment Analysis") |
| d4 = EvalDimension("coding", "Coding") |
| d5 = EvalDimension("function calling", "Function Calling") |
| d6 = EvalDimension("rag qa", "RAG QA") |
| d7 = EvalDimension("reading comprehension", "Reading Comprehension") |
| d8 = EvalDimension("entity extraction", "Entity Extraction") |
| d9 = EvalDimension("summarization", "Summarization") |
| d10 = EvalDimension("long context", "Long Context") |
| d11 = EvalDimension("mmlu", "MMLU") |
| d12 = EvalDimension("arabic language & grammar", "Arabic Language & Grammar") |
| d13 = EvalDimension("general knowledge", "General Knowledge") |
| d14 = EvalDimension("translation (incl dialects)", "Translation (incl Dialects)") |
| d15 = EvalDimension("trust & safety","Trust & Safety") |
| d16 = EvalDimension("writing (incl dialects)", "Writing (incl Dialects)") |
| d17 = EvalDimension("dialect detection", "Dialect Detection") |
| d18 = EvalDimension("reasoning & math", "Reasoning & Math") |
| d19 = EvalDimension("diacritization", "Diacritization") |
| d20 = EvalDimension("instruction following", "Instruction Following") |
| d21 = EvalDimension("transliteration", "Transliteration") |
| d22 = EvalDimension("structuring", "Structuring") |
| d23 = EvalDimension("hallucination", "Hallucination") |
|
|
|
|
|
|
|
|
|
|
| |
|
|
|
|
|
|
|
|
|
|
|
|
| NUM_FEWSHOT = 0 |
| |
|
|
|
|
|
|
| |
| TITLE = """<div ><img class='abl_header_image' src='https://huggingface.co/spaces/silma-ai/Arabic-LLM-Broad-Leaderboard/resolve/main/src/images/abl_logo.png' ></div>""" |
|
|
| |
| INTRODUCTION_TEXT = """ |
| <h1 style='width: 100%;text-align: center;' id="space-title">Arabic Broad Leaderboard (ABL) - The first comprehensive Leaderboard for Arabic LLMs</h1> |
| ABL is the official Leaderboard of <a href='https://huggingface.co/datasets/silma-ai/arabic-broad-benchmark' target='_blank'>Arabic Broad Benchmark (ABB)</a>. |
| With advanced features and innovative visualizations, we provide the community with a comprehensive view of the capabilities of Arabic models, showcasing their speed, diverse skills while also defending against benchmarking contamination. |
| The benchmark consists of <b>450 high quality human-validated questions</b> sampled from <b>63 Arabic benchmarking datasets</b>, evaluating <b>22 categories and skills</b>. |
| Find more details in the about Tab. |
| |
| |
| """ |
|
|
| |
| LLM_BENCHMARKS_TEXT = f""" |
| ## How it works |
| |
| ## Reproducibility |
| To reproduce our results, here is the commands you can run: |
| |
| """ |
|
|
| EVALUATION_QUEUE_TEXT = """ |
| ## Some good practices before submitting a model |
| |
| ### 1) Make sure you can load your model and tokenizer using AutoClasses: |
| ```python |
| from transformers import AutoConfig, AutoModel, AutoTokenizer |
| config = AutoConfig.from_pretrained("your model name", revision=revision) |
| model = AutoModel.from_pretrained("your model name", revision=revision) |
| tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision) |
| ``` |
| If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. |
| |
| Note: make sure your model is public! |
| Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted! |
| |
| ### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index) |
| It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`! |
| |
| ### 3) Make sure your model has an open license! |
| This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗 |
| |
| ### 4) Fill up your model card |
| When we add extra information about models to the leaderboard, it will be automatically taken from the model card |
| |
| ## In case of model failure |
| If your model is displayed in the `FAILED` category, its execution stopped. |
| Make sure you have followed the above steps first. |
| If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task). |
| """ |
|
|
| CITATION_BUTTON_LABEL = "Copy the following snippet to cite the Leaderboard" |
| CITATION_BUTTON_TEXT = r""" |
| |
| @misc{ABL, |
| author = {SILMA AI Team}, |
| title = {Arabic Broad Leaderboard}, |
| year = {2025}, |
| publisher = {SILMA.AI}, |
| howpublished = "{\url{https://huggingface.co/spaces/silma-ai/Arabic-LLM-Broad-Leaderboard}}" |
| } |
| |
| """ |
|
|
| FOOTER_TEXT = """<div style='display:flex;justify-content:center;align-items:center;'> |
| <span style='font-size:36px;font-weight:bold;margin-right:20px;'>Sponsored By</span> |
| <a href='https://silma.ai/?ref=abl' target='_blank'> |
| <img style='height:60px' src='https://huggingface.co/spaces/silma-ai/Arabic-LLM-Broad-Leaderboard/resolve/main/src/images/silma-logo-wide.png' > |
| </a> |
| </div>""" |
|
|
