import os import json import gradio as gr import pandas as pd import numpy as np from collections import defaultdict LENGTHS = ["dataset_total_score", "4k", "8k", "16k", "32k", "64k", "128k"] datasets_params = json.load(open("datasets_config.json", "r")) TASKS = datasets_params.keys() def task_keys_star_first(): """LIBRA Mini datasets (display name ends with ' *') first, then others; order within each group follows datasets_config.""" keys = list(datasets_params.keys()) star = [k for k in keys if datasets_params[k]["name"].endswith(" *")] rest = [k for k in keys if k not in star] return star + rest TASK_TAB_ORDER = task_keys_star_first() LIBRA_MINI_TASK_KEYS = frozenset( k for k in datasets_params if datasets_params[k]["name"].endswith(" *") ) def make_default_md(): leaderboard_md = f""" 🏅 LIBRA LeaderBoard | [GitHub](https://github.com/ai-forever/LIBRA) | [Datasets](https://huggingface.co/datasets/ai-forever/LIBRA) | """ return leaderboard_md def make_model_desc_md(): with open("docs/description.md", "r") as f: description = f.read() return description def make_overall_table_by_tasks(files): results = defaultdict(list) result_dct = {} for file in files: if not file.endswith("json"): continue path = "results/" + file data = json.load(open(path)) model_name = file.split('/')[-1].split(".json")[0] result_dct[model_name] = {} for dataset in data.keys(): if dataset == "total_score": result_dct[model_name][dataset] = round(data[dataset] * 100, 1) continue result_dct[model_name][dataset] = round(data[dataset]["dataset_total_score"] * 100, 1) for file in files: if not file.endswith("json"): continue model_name = file.split('/')[-1].split(".json")[0] results['Model'].append(model_name) for key in result_dct[model_name].keys(): if key == "total_score": results["Total Score"].append(result_dct[model_name][key]) else: results[datasets_params[key]["name"]].append(result_dct[model_name][key]) table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False) front = ["Model", "Total Score"] rest = [datasets_params[t]["name"] for t in TASK_TAB_ORDER if datasets_params[t]["name"] in table.columns] rest += [c for c in table.columns if c not in front and c not in rest] return table[front + rest] def make_overall_table_by_tasks_mini(files): """Per-task scores for LIBRA Mini datasets only; Total Score = mean over mini tasks.""" results = defaultdict(list) result_dct = {} for file in files: if not file.endswith("json"): continue path = "results/" + file data = json.load(open(path)) model_name = file.split("/")[-1].split(".json")[0] result_dct[model_name] = {} mini_raw = [] for dataset in data.keys(): if dataset == "total_score": continue if dataset not in LIBRA_MINI_TASK_KEYS: continue result_dct[model_name][dataset] = round(data[dataset]["dataset_total_score"] * 100, 1) mini_raw.append(data[dataset]["dataset_total_score"]) if mini_raw: result_dct[model_name]["total_score"] = round(float(np.mean(mini_raw)) * 100, 1) else: result_dct[model_name]["total_score"] = float("nan") for file in files: if not file.endswith("json"): continue model_name = file.split("/")[-1].split(".json")[0] results["Model"].append(model_name) for key in result_dct[model_name].keys(): if key == "total_score": results["Total Score"].append(result_dct[model_name][key]) else: results[datasets_params[key]["name"]].append(result_dct[model_name][key]) table = pd.DataFrame(results).sort_values(["Total Score"], ascending=False) front = ["Model", "Total Score"] mini_tab_order = [t for t in TASK_TAB_ORDER if t in LIBRA_MINI_TASK_KEYS] rest = [datasets_params[t]["name"] for t in mini_tab_order if datasets_params[t]["name"] in table.columns] rest += [c for c in table.columns if c not in front and c not in rest] return table[front + rest] def make_overall_table_by_lengths(files): results = defaultdict(list) result_dct = {} for file in files: if not file.endswith("json"): continue path = "results/" + file data = json.load(open(path)) model_name = file.split('/')[-1].split(".json")[0] result_dct[model_name] = {} for dataset in data.keys(): if dataset == "total_score": result_dct[model_name][dataset] = data[dataset] continue for length in data[dataset].keys(): if length == "dataset_total_score": continue if length not in result_dct[model_name]: result_dct[model_name][length] = [] result_dct[model_name][length].append(data[dataset][length]) for model_name in result_dct.keys(): for length in result_dct[model_name].keys(): result_dct[model_name][length] = round(np.mean(result_dct[model_name][length]) * 100, 1) for file in files: if not file.endswith("json"): continue model_name = file.split('/')[-1].split(".json")[0] results['Model'].append(model_name) for key in result_dct[model_name].keys(): if key == "total_score": results["Total Score"].append(result_dct[model_name][key]) else: results[key].append(result_dct[model_name][key]) table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False) front = ["Model", "Total Score"] length_cols = [c for c in LENGTHS if c in table.columns] return table[front + length_cols] def make_overall_table_by_lengths_mini(files): """Aggregate by context length using only LIBRA Mini datasets (names ending with ' *').""" results = defaultdict(list) result_dct = {} for file in files: if not file.endswith("json"): continue path = "results/" + file data = json.load(open(path)) model_name = file.split("/")[-1].split(".json")[0] result_dct[model_name] = {} mini_scores = [] for dataset in data.keys(): if dataset == "total_score": continue if dataset not in LIBRA_MINI_TASK_KEYS: continue block = data[dataset] if "dataset_total_score" in block: mini_scores.append(block["dataset_total_score"]) for length in block.keys(): if length == "dataset_total_score": continue if length not in result_dct[model_name]: result_dct[model_name][length] = [] result_dct[model_name][length].append(block[length]) if mini_scores: result_dct[model_name]["total_score"] = float(np.mean(mini_scores)) else: result_dct[model_name]["total_score"] = float("nan") for model_name in result_dct.keys(): for length in list(result_dct[model_name].keys()): if length == "total_score": continue result_dct[model_name][length] = round(np.mean(result_dct[model_name][length]) * 100, 1) for file in files: if not file.endswith("json"): continue model_name = file.split("/")[-1].split(".json")[0] results["Model"].append(model_name) for key in result_dct[model_name].keys(): if key == "total_score": results["Total Score"].append(round(result_dct[model_name][key] * 100, 1)) else: results[key].append(result_dct[model_name][key]) table = pd.DataFrame(results).sort_values(["Total Score"], ascending=False) front = ["Model", "Total Score"] length_cols = [c for c in LENGTHS if c in table.columns] return table[front + length_cols] def load_model(files, tab_name): results = defaultdict(list) for file in files: if not file.endswith("json"): continue model_name = file.split('/')[-1].split(".json")[0] results['Model'].append(model_name) result = json.load(open("results/" + file, "r")) task_block = result.get(tab_name) if not task_block: results["Dataset Total Score"].append("-") for length in LENGTHS: if length == "dataset_total_score": continue results[length].append("-") continue for length in LENGTHS: if length in task_block.keys(): if length == "dataset_total_score": results["Dataset Total Score"].append(round(task_block[length] * 100, 1)) continue results[length].append(round(task_block[length] * 100, 1)) else: results[length].append("-") df = pd.DataFrame(results) df["_sort_key"] = pd.to_numeric(df["Dataset Total Score"], errors="coerce").fillna(-1) return df.sort_values("_sort_key", ascending=False).drop(columns=["_sort_key"]) def build_leaderboard_tab(files): default_md = make_default_md() md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown") with gr.Tabs() as tabs: with gr.Tab("Results by length (LIBRA-mini)", id=0): df = make_overall_table_by_lengths_mini(files) gr.Dataframe( headers=[ "Model", ] + LENGTHS, datatype=[ "markdown", "str", "str", "str", "str", "str", "str", "str", ], value=df, elem_id="arena_leaderboard_dataframe", max_height=700, wrap=True, ) with gr.Tab("Results by length (LIBRA)", id=1): df = make_overall_table_by_lengths(files) gr.Dataframe( headers=[ "Model", ] + LENGTHS, datatype=[ "markdown", "str", "str", "str", "str", "str", "str", "str", ], value=df, elem_id="arena_leaderboard_dataframe", max_height=700, wrap=True, ) with gr.Tab("Results by task (LIBRA-mini)", id=2): df = make_overall_table_by_tasks_mini(files) gr.Dataframe( headers=[ "Model", ] + LENGTHS, datatype=[ "markdown", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str" ], value=df, elem_id="arena_leaderboard_dataframe", max_height=700, wrap=False, ) with gr.Tab("Results by task (LIBRA)", id=3): df = make_overall_table_by_tasks(files) gr.Dataframe( headers=[ "Model", ] + LENGTHS, datatype=[ "markdown", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str" ], value=df, elem_id="arena_leaderboard_dataframe", max_height=700, wrap=False, ) for tab_id, tab_name in enumerate(TASK_TAB_ORDER): df = load_model(files, tab_name) with gr.Tab(datasets_params[tab_name]["name"], id=tab_id + 4): gr.Dataframe( headers=[ "Model", ] + LENGTHS, datatype=[ "markdown", "str", "str", "str", "str", "str", "str", "str", ], value=df, elem_id="arena_leaderboard_dataframe", max_height=700, wrap=True, ) with gr.Tab("Description", id=len(TASK_TAB_ORDER) + 4): desc_md = make_model_desc_md() gr.Markdown(desc_md, elem_id="leaderboard_markdown") return [md_1] def build_demo(files): with gr.Blocks(title="LIBRA leaderboard") as demo: build_leaderboard_tab(files) return demo if __name__ == "__main__": files = os.listdir("results") demo = build_demo(files) text_size = gr.themes.sizes.text_lg demo.launch(theme=gr.themes.Base(text_size=text_size), share=False)