Spaces:
Running
Running
| import os | |
| import json | |
| import gradio as gr | |
| import pandas as pd | |
| import numpy as np | |
| from collections import defaultdict | |
| LENGTHS = ["dataset_total_score", "4k", "8k", "16k", "32k", "64k", "128k"] | |
| datasets_params = json.load(open("datasets_config.json", "r")) | |
| TASKS = datasets_params.keys() | |
| def task_keys_star_first(): | |
| """LIBRA Mini datasets (display name ends with ' *') first, then others; order within each group follows datasets_config.""" | |
| keys = list(datasets_params.keys()) | |
| star = [k for k in keys if datasets_params[k]["name"].endswith(" *")] | |
| rest = [k for k in keys if k not in star] | |
| return star + rest | |
| TASK_TAB_ORDER = task_keys_star_first() | |
| LIBRA_MINI_TASK_KEYS = frozenset( | |
| k for k in datasets_params if datasets_params[k]["name"].endswith(" *") | |
| ) | |
| def make_default_md(): | |
| leaderboard_md = f""" | |
| 🏅 LIBRA LeaderBoard | |
| | [GitHub](https://github.com/ai-forever/LIBRA) | [Datasets](https://huggingface.co/datasets/ai-forever/LIBRA) | | |
| """ | |
| return leaderboard_md | |
| def make_model_desc_md(): | |
| with open("docs/description.md", "r") as f: | |
| description = f.read() | |
| return description | |
| def make_overall_table_by_tasks(files): | |
| results = defaultdict(list) | |
| result_dct = {} | |
| for file in files: | |
| if not file.endswith("json"): continue | |
| path = "results/" + file | |
| data = json.load(open(path)) | |
| model_name = file.split('/')[-1].split(".json")[0] | |
| result_dct[model_name] = {} | |
| for dataset in data.keys(): | |
| if dataset == "total_score": | |
| result_dct[model_name][dataset] = round(data[dataset] * 100, 1) | |
| continue | |
| result_dct[model_name][dataset] = round(data[dataset]["dataset_total_score"] * 100, 1) | |
| for file in files: | |
| if not file.endswith("json"): continue | |
| model_name = file.split('/')[-1].split(".json")[0] | |
| results['Model'].append(model_name) | |
| for key in result_dct[model_name].keys(): | |
| if key == "total_score": | |
| results["Total Score"].append(result_dct[model_name][key]) | |
| else: | |
| results[datasets_params[key]["name"]].append(result_dct[model_name][key]) | |
| table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False) | |
| front = ["Model", "Total Score"] | |
| rest = [datasets_params[t]["name"] for t in TASK_TAB_ORDER if datasets_params[t]["name"] in table.columns] | |
| rest += [c for c in table.columns if c not in front and c not in rest] | |
| return table[front + rest] | |
| def make_overall_table_by_tasks_mini(files): | |
| """Per-task scores for LIBRA Mini datasets only; Total Score = mean over mini tasks.""" | |
| results = defaultdict(list) | |
| result_dct = {} | |
| for file in files: | |
| if not file.endswith("json"): | |
| continue | |
| path = "results/" + file | |
| data = json.load(open(path)) | |
| model_name = file.split("/")[-1].split(".json")[0] | |
| result_dct[model_name] = {} | |
| mini_raw = [] | |
| for dataset in data.keys(): | |
| if dataset == "total_score": | |
| continue | |
| if dataset not in LIBRA_MINI_TASK_KEYS: | |
| continue | |
| result_dct[model_name][dataset] = round(data[dataset]["dataset_total_score"] * 100, 1) | |
| mini_raw.append(data[dataset]["dataset_total_score"]) | |
| if mini_raw: | |
| result_dct[model_name]["total_score"] = round(float(np.mean(mini_raw)) * 100, 1) | |
| else: | |
| result_dct[model_name]["total_score"] = float("nan") | |
| for file in files: | |
| if not file.endswith("json"): | |
| continue | |
| model_name = file.split("/")[-1].split(".json")[0] | |
| results["Model"].append(model_name) | |
| for key in result_dct[model_name].keys(): | |
| if key == "total_score": | |
| results["Total Score"].append(result_dct[model_name][key]) | |
| else: | |
| results[datasets_params[key]["name"]].append(result_dct[model_name][key]) | |
| table = pd.DataFrame(results).sort_values(["Total Score"], ascending=False) | |
| front = ["Model", "Total Score"] | |
| mini_tab_order = [t for t in TASK_TAB_ORDER if t in LIBRA_MINI_TASK_KEYS] | |
| rest = [datasets_params[t]["name"] for t in mini_tab_order if datasets_params[t]["name"] in table.columns] | |
| rest += [c for c in table.columns if c not in front and c not in rest] | |
| return table[front + rest] | |
| def make_overall_table_by_lengths(files): | |
| results = defaultdict(list) | |
| result_dct = {} | |
| for file in files: | |
| if not file.endswith("json"): continue | |
| path = "results/" + file | |
| data = json.load(open(path)) | |
| model_name = file.split('/')[-1].split(".json")[0] | |
| result_dct[model_name] = {} | |
| for dataset in data.keys(): | |
| if dataset == "total_score": | |
| result_dct[model_name][dataset] = data[dataset] | |
| continue | |
| for length in data[dataset].keys(): | |
| if length == "dataset_total_score": continue | |
| if length not in result_dct[model_name]: | |
| result_dct[model_name][length] = [] | |
| result_dct[model_name][length].append(data[dataset][length]) | |
| for model_name in result_dct.keys(): | |
| for length in result_dct[model_name].keys(): | |
| result_dct[model_name][length] = round(np.mean(result_dct[model_name][length]) * 100, 1) | |
| for file in files: | |
| if not file.endswith("json"): continue | |
| model_name = file.split('/')[-1].split(".json")[0] | |
| results['Model'].append(model_name) | |
| for key in result_dct[model_name].keys(): | |
| if key == "total_score": | |
| results["Total Score"].append(result_dct[model_name][key]) | |
| else: | |
| results[key].append(result_dct[model_name][key]) | |
| table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False) | |
| front = ["Model", "Total Score"] | |
| length_cols = [c for c in LENGTHS if c in table.columns] | |
| return table[front + length_cols] | |
| def make_overall_table_by_lengths_mini(files): | |
| """Aggregate by context length using only LIBRA Mini datasets (names ending with ' *').""" | |
| results = defaultdict(list) | |
| result_dct = {} | |
| for file in files: | |
| if not file.endswith("json"): | |
| continue | |
| path = "results/" + file | |
| data = json.load(open(path)) | |
| model_name = file.split("/")[-1].split(".json")[0] | |
| result_dct[model_name] = {} | |
| mini_scores = [] | |
| for dataset in data.keys(): | |
| if dataset == "total_score": | |
| continue | |
| if dataset not in LIBRA_MINI_TASK_KEYS: | |
| continue | |
| block = data[dataset] | |
| if "dataset_total_score" in block: | |
| mini_scores.append(block["dataset_total_score"]) | |
| for length in block.keys(): | |
| if length == "dataset_total_score": | |
| continue | |
| if length not in result_dct[model_name]: | |
| result_dct[model_name][length] = [] | |
| result_dct[model_name][length].append(block[length]) | |
| if mini_scores: | |
| result_dct[model_name]["total_score"] = float(np.mean(mini_scores)) | |
| else: | |
| result_dct[model_name]["total_score"] = float("nan") | |
| for model_name in result_dct.keys(): | |
| for length in list(result_dct[model_name].keys()): | |
| if length == "total_score": | |
| continue | |
| result_dct[model_name][length] = round(np.mean(result_dct[model_name][length]) * 100, 1) | |
| for file in files: | |
| if not file.endswith("json"): | |
| continue | |
| model_name = file.split("/")[-1].split(".json")[0] | |
| results["Model"].append(model_name) | |
| for key in result_dct[model_name].keys(): | |
| if key == "total_score": | |
| results["Total Score"].append(round(result_dct[model_name][key] * 100, 1)) | |
| else: | |
| results[key].append(result_dct[model_name][key]) | |
| table = pd.DataFrame(results).sort_values(["Total Score"], ascending=False) | |
| front = ["Model", "Total Score"] | |
| length_cols = [c for c in LENGTHS if c in table.columns] | |
| return table[front + length_cols] | |
| def load_model(files, tab_name): | |
| results = defaultdict(list) | |
| for file in files: | |
| if not file.endswith("json"): continue | |
| model_name = file.split('/')[-1].split(".json")[0] | |
| results['Model'].append(model_name) | |
| result = json.load(open("results/" + file, "r")) | |
| task_block = result.get(tab_name) | |
| if not task_block: | |
| results["Dataset Total Score"].append("-") | |
| for length in LENGTHS: | |
| if length == "dataset_total_score": | |
| continue | |
| results[length].append("-") | |
| continue | |
| for length in LENGTHS: | |
| if length in task_block.keys(): | |
| if length == "dataset_total_score": | |
| results["Dataset Total Score"].append(round(task_block[length] * 100, 1)) | |
| continue | |
| results[length].append(round(task_block[length] * 100, 1)) | |
| else: | |
| results[length].append("-") | |
| df = pd.DataFrame(results) | |
| df["_sort_key"] = pd.to_numeric(df["Dataset Total Score"], errors="coerce").fillna(-1) | |
| return df.sort_values("_sort_key", ascending=False).drop(columns=["_sort_key"]) | |
| def build_leaderboard_tab(files): | |
| default_md = make_default_md() | |
| md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown") | |
| with gr.Tabs() as tabs: | |
| with gr.Tab("Results by length (LIBRA-mini)", id=0): | |
| df = make_overall_table_by_lengths_mini(files) | |
| gr.Dataframe( | |
| headers=[ | |
| "Model", | |
| ] + LENGTHS, | |
| datatype=[ | |
| "markdown", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| ], | |
| value=df, | |
| elem_id="arena_leaderboard_dataframe", | |
| max_height=700, | |
| wrap=True, | |
| ) | |
| with gr.Tab("Results by length (LIBRA)", id=1): | |
| df = make_overall_table_by_lengths(files) | |
| gr.Dataframe( | |
| headers=[ | |
| "Model", | |
| ] + LENGTHS, | |
| datatype=[ | |
| "markdown", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| ], | |
| value=df, | |
| elem_id="arena_leaderboard_dataframe", | |
| max_height=700, | |
| wrap=True, | |
| ) | |
| with gr.Tab("Results by task (LIBRA-mini)", id=2): | |
| df = make_overall_table_by_tasks_mini(files) | |
| gr.Dataframe( | |
| headers=[ | |
| "Model", | |
| ] + LENGTHS, | |
| datatype=[ | |
| "markdown", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str" | |
| ], | |
| value=df, | |
| elem_id="arena_leaderboard_dataframe", | |
| max_height=700, | |
| wrap=False, | |
| ) | |
| with gr.Tab("Results by task (LIBRA)", id=3): | |
| df = make_overall_table_by_tasks(files) | |
| gr.Dataframe( | |
| headers=[ | |
| "Model", | |
| ] + LENGTHS, | |
| datatype=[ | |
| "markdown", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str" | |
| ], | |
| value=df, | |
| elem_id="arena_leaderboard_dataframe", | |
| max_height=700, | |
| wrap=False, | |
| ) | |
| for tab_id, tab_name in enumerate(TASK_TAB_ORDER): | |
| df = load_model(files, tab_name) | |
| with gr.Tab(datasets_params[tab_name]["name"], id=tab_id + 4): | |
| gr.Dataframe( | |
| headers=[ | |
| "Model", | |
| ] + LENGTHS, | |
| datatype=[ | |
| "markdown", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| "str", | |
| ], | |
| value=df, | |
| elem_id="arena_leaderboard_dataframe", | |
| max_height=700, | |
| wrap=True, | |
| ) | |
| with gr.Tab("Description", id=len(TASK_TAB_ORDER) + 4): | |
| desc_md = make_model_desc_md() | |
| gr.Markdown(desc_md, elem_id="leaderboard_markdown") | |
| return [md_1] | |
| def build_demo(files): | |
| with gr.Blocks(title="LIBRA leaderboard") as demo: | |
| build_leaderboard_tab(files) | |
| return demo | |
| if __name__ == "__main__": | |
| files = os.listdir("results") | |
| demo = build_demo(files) | |
| text_size = gr.themes.sizes.text_lg | |
| demo.launch(theme=gr.themes.Base(text_size=text_size), share=False) | |