diff --git a/__pycache__/data_manager.cpython-311.pyc b/__pycache__/data_manager.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b82e6dc91eec060c907fc8e867150a02adaee1fd Binary files /dev/null and b/__pycache__/data_manager.cpython-311.pyc differ diff --git a/__pycache__/title.cpython-311.pyc b/__pycache__/title.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9bb52ceb1f4e6406ba6740683a5306a54f6e61c0 Binary files /dev/null and b/__pycache__/title.cpython-311.pyc differ diff --git a/about.md b/about.md index a5ed5334cb319f3e6c0789c8be9c4bc7c3f26ec1..a2fbe44fd9aaad7e168bdffee087b25d5607b20a 100644 --- a/about.md +++ b/about.md @@ -24,13 +24,3 @@ Therefore, the compression rate of a model can be directly calculated through th ### Can Models Using Different Tokenizers Be Directly Compared? Yes. When calculating the sum of negative log probabilities, we essentially treat the model + tokenizer as a single entity or system. As long as this system has a high probability of generating real text, we consider it better. From the perspective of compression, you can choose any tokenizer. From the compression rate perspective, we don't care; we only care about whether your system can compress the text more effectively. - -### Is It Really Uncheatable? Can't I train my model on a large number of arXiv papers to improve its test performance on arXiv papers? -Uncheatable Eval's data sources currently include new arXiv papers, new GitHub projects, BBC news, AO3 fanfictions, and new Wikipedia entries, with more sources to be added in the future. If you genuinely achieve excellent results across these data by training extensively on these sources, I would consider you to have developed a genuinely good language model rather than cheating. - -From my test results, accurately modeling these data is very challenging. I believe Uncheatable Eval more accurately reflects the value of every bit of data and computing you invest compared to other benchmarks. Models trained with more data and computing are almost always better, and there are no shortcuts. This is a key strength of Uncheatable Eval. - -### Is This Too "Random"? Why Consider Random Texts from the Internet as Ground Truth? -This is why we choose rigorous and verified texts such as arXiv papers and news reports, which typically have better quality. Additionally, a round of Uncheatable Eval evaluates a model over millions of tokens, increasing the reliability of the results. - -In fact, the model rankings obtained through Uncheatable Eval are very stable. For instance, the model ranked first in January's data is highly likely to remain first in February, March, April, May, and June, indicating that the data obtained through this method is sufficiently representative. \ No newline at end of file diff --git a/app.py b/app.py index cb438bb7b794b2db75d5cae3c18eb59551c86672..c851b7079bc2d4523206663ca773f4febab1ae59 100644 --- a/app.py +++ b/app.py @@ -1,36 +1,22 @@ +from operator import is_ import pandas as pd import gradio as gr import os -import re import requests from dotenv import load_dotenv from matplotlib.colors import LinearSegmentedColormap -import plotly.express as px import plotly.graph_objects as go -# from sklearn.linear_model import LinearRegression import numpy as np from huggingface_hub import HfApi from huggingface_hub.hf_api import HTTPError from huggingface_hub.utils import GatedRepoError from gradio_rangeslider import RangeSlider import datetime -from gradio.themes.utils.colors import slate +from title import css, TITLE_HTML, SUBTITLE_HTML +from data_manager import DataManager load_dotenv() webhook_url = os.environ.get("WEBHOOK_URL") -file_name_list = [ - "14b", - "9b", - "7b", - "3b", - "1b5", - "other", -] -sheet_name_list = [ - "cr", - "bpc", - "bpb", -] metric_list = [ "Compression Rate (%)", "Bits Per Character (BPC)", @@ -58,92 +44,54 @@ model_size_to_file_name = { "Other": "other", } + def read_about_md(): - with open('about.md', 'r', encoding='utf-8') as f: + with open("about.md", "r", encoding="utf-8") as f: return f.read() -def rename_columns(df): - df.columns = [col.rsplit("_", maxsplit=1)[0] for col in df.columns] - return df - -def get_folders_matching_format(directory): - pattern = re.compile(r"^\d{4}-\d{2}$") - folders = [] - if not os.path.exists(directory): - return folders - for item in os.listdir(directory): - full_path = os.path.join(directory, item) - if os.path.isdir(full_path) and pattern.match(item): - folders.append(full_path) - return folders - -def get_unique_column_names(data=None): - return [ - "ao3_\u200benglish", - "bbc_\u200bnews", - "wikipedia_\u200benglish", - "arxiv_\u200bcomputer_\u200bscience", - "arxiv_\u200bphysics", - "github_\u200bcpp", - "github_\u200bpython", - ] - -def color_cell(value): - return "background-color: #fffdd0" if pd.notna(value) else "default" - -# def color_cell_themed(value): -# return "background-color: rgba(255, 253, 208, 1.0)" if pd.notna(value) else "default" - -# --- 核心改动点 1: 修改 update_table 函数 --- -# 添加 request: gr.Request = None 参数来接收主题模式信息 -# 默认值为 None 是为了处理初始加载 -def update_table(period: str, models_size: list, metric: str, visible_columns: list, color_columns: list, size_range: list, midpoint: float = 0.5, sort_by: str = "Average (lower=better)", ascending: bool = True, request: gr.Request = None): - # 打印日志并检查当前模式 + +def update_table( + data_manager: DataManager, + period: str, + models_size: list, + metric: str, + visible_columns: list, + color_columns: list, + size_range: list, + midpoint: float = 0.5, + ascending: bool = True, + request: gr.Request = None, +): is_dark_mode = request.is_dark if request else False - print(f"Updating - time: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}, period: {period}, models: {models_size}, metric: {metric}, visible_columns: {visible_columns}, color_columns: {color_columns}, size_range: {size_range}, sort_by: {sort_by}, ascending: {ascending}, is_dark: {is_dark_mode}\n") + print( + f"Updating - time: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}, period: {period}, models: {models_size}, metric: {metric}, visible_columns: {visible_columns}, color_columns: {color_columns}, size_range: {size_range}, ascending: {ascending}, is_dark: {is_dark_mode}\n" + ) - if not models_size: - return "No data available for the selected models and period." - - target_period_data = all_data[period] target_file_name = [model_size_to_file_name[model] for model in models_size] - sheet_name = metric_to_sheet[metric] - combined_data = pd.concat([df.dropna(axis=1, how="all") for df in [target_period_data[file_name][sheet_name] for file_name in target_file_name]], axis=0) - - if len(combined_data) == 0: - return "No data available for the selected models and period." - - combined_data = combined_data[combined_data["Parameters Count (B)"].between(size_range[0], size_range[1])] - combined_data.reset_index(drop=True, inplace=True) - - if len(combined_data) == 0: + metric_code = metric_to_sheet[metric] + + # 过滤掉不在当前 period 可用列中的列名，避免错误 + if visible_columns: + available_columns = data_manager.get_available_columns(period) + visible_columns = [col for col in visible_columns if col in available_columns] + + filtered_data = data_manager.query( + period=period, + metric_code=metric_code, + param_range=(size_range[0], size_range[1]), + model_groups=target_file_name, + visible_columns=visible_columns, + ) + + if len(filtered_data) == 0: return "No data available for the selected models and period." - - combined_data["Name"] = combined_data["Name"].apply(lambda x: x.replace(".pth", "")) - ordered_columns = get_unique_column_names() - relevant_columns = [col for col in ordered_columns if col in visible_columns and col not in ["Name", "Parameters Count (B)", "Average (The lower the better)"]] - - if len(combined_data) > 0 and relevant_columns: - combined_data["Average (The lower the better)"] = round(combined_data[relevant_columns].mean(axis=1), 3) - - combined_data = combined_data.rename(columns={"Parameters Count (B)": "Params (B)", "Average (The lower the better)": "Average (lower=better)"}) - sorted_data = combined_data.sort_values(by=sort_by, ascending=ascending) - visible_columns_final = ["Name", "Params (B)", "Average (lower=better)"] + relevant_columns - filtered_data = sorted_data[visible_columns_final] - filtered_data.columns = [col.replace("_", " ") for col in filtered_data.columns] - formatter = {col: "{:.3f}" for col in filtered_data.columns if filtered_data[col].dtype in ["float64", "float32"]} - - # --- 核心改动点 2: 根据主题模式选择不同的配色方案 --- - if is_dark_mode: - # 夜间模式配色 (绿 -> 深灰 -> 红) - colors = ["#2ca02c", "#2b2b2b", "#d62728"] - else: - # 日间模式配色 (绿 -> 白 -> 红) - colors = ["#63be7b", "#ffffff", "#f8696b"] - + + colors = ["#2ca02c", "#2b2b2b", "#d62728"] if is_dark_mode else ["#63be7b", "#ffffff", "#f8696b"] + vmin, vmax, vmid = {}, {}, {} for column in filtered_data.columns: - if column in ["Name", "Params (B)"]: continue + if column in ["Name", "Params (B)"]: + continue col_values = filtered_data[column].dropna() if len(col_values) > 1: sorted_values = np.sort(col_values) @@ -152,93 +100,84 @@ def update_table(period: str, models_size: list, metric: str, visible_columns: l idx = int(len(sorted_values) * midpoint) vmid[column] = sorted_values[idx] - # --- 核心改动点 3: 修改样式函数以包含固定的黑色字体 --- def custom_background_gradient(series, cmap, vmin_val, vmax_val, vmid_val): - if len(series) == 0: return series + if len(series) == 0: + return series + def normalize(x): - if pd.isna(x): return 0.5 # Neutral for NaN - if vmid_val == vmin_val and x <= vmid_val: return 0.0 - if vmid_val == vmax_val and x >= vmid_val: return 1.0 - if vmid_val == vmin_val or vmid_val == vmax_val: return 0.5 + if pd.isna(x): + return 0.5 # Neutral for NaN + if vmid_val == vmin_val and x <= vmid_val: + return 0.0 + if vmid_val == vmax_val and x >= vmid_val: + return 1.0 + if vmid_val == vmin_val or vmid_val == vmax_val: + return 0.5 if x <= vmid_val: return 0.5 * (x - vmin_val) / (vmid_val - vmin_val) else: return 0.5 + 0.5 * (x - vmid_val) / (vmax_val - vmid_val) + normed = series.apply(normalize) cmap_colors = [cmap(x) for x in normed] - # 在返回的CSS中同时设置 background-color 和 color - return [ - "background-color: rgba({}, {}, {}, {}); color: black;".format(*[int(255 * c) for c in color[:3]], color[3]) - for color in cmap_colors - ] + return ["background-color: rgba({}, {}, {}, {}); color: black;".format(*[int(255 * c) for c in color[:3]], color[3]) for color in cmap_colors] target_color_columns = [] - if "Average" in color_columns: target_color_columns.append("Average (lower=better)") - if "Individual Tests" in color_columns: target_color_columns.extend([col for col in filtered_data.columns if col not in ["Name", "Params (B)", "Average (lower=better)"]]) - + if "Average" in color_columns: + target_color_columns.append("Average (lower=better)") + if "Individual Tests" in color_columns: + target_color_columns.extend([col for col in filtered_data.columns if col not in ["Name", "Params (B)", "Average (lower=better)"]]) + def color_params_column_dynamic(value): if not pd.notna(value): return "default" - - # 2. 根据 is_dark_mode 返回不同的颜色 + if is_dark_mode: - # 为夜间模式选择一个柔和、不刺眼的暗金色 - # 字体颜色也设置为浅色以保证对比度 return "background-color: #4b4936; color: #f0f0f0;" else: - # 为日间模式使用明亮的奶油色，字体为黑色 return "background-color: #fffdd0; color: black;" - - styler = filtered_data.style.format(formatter).map(color_params_column_dynamic, subset=["Params (B)"]) + + formatter = {col: "{:.3f}" for col in filtered_data.columns if filtered_data[col].dtype in ["float64", "float32"]} + styler = filtered_data.style.format(formatter) + styler = styler.map(color_params_column_dynamic, subset=["Params (B)"]) for column in target_color_columns: if column in vmin: custom_cmap = LinearSegmentedColormap.from_list("custom_cmap", colors) - styler = styler.apply(custom_background_gradient, cmap=custom_cmap, vmin_val=vmin[column], vmax_val=vmax[column], vmid_val=vmid[column], subset=[column]) - + styler = styler.apply( + custom_background_gradient, cmap=custom_cmap, vmin_val=vmin[column], vmax_val=vmax[column], vmid_val=vmid[column], subset=[column] + ) + styler = styler.hide(axis="index") - widths = [300, 150, 150, 100, 100, 100, 100, 100, 100, 100, 100] - + widths = [250, 80, 80, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70] + table_styles = [] - table_styles.append({"selector": "th", "props": [("background-color", "var(--background-fill-secondary)"), ("color", "var(--body-text-color)"), ("padding", "8px"), ("font-weight", "bold")]}) + table_styles.append( + { + "selector": "th", + "props": [ + ("background-color", "var(--background-fill-secondary)"), + ("color", "var(--body-text-color)"), + ("padding", "8px"), + ("font-weight", "bold"), + ], + } + ) table_styles.append({"selector": "table", "props": [("border-collapse", "collapse"), ("border", f"1px solid var(--border-color-primary)")]}) for i, w in enumerate(widths): - table_styles.append({"selector": f"th.col{i}, td.col{i}", "props": [("min-width", f"{w}px"), ("max-width", f"{w}px"), ("text-align", "center"), ("border", f"1px solid var(--border-color-primary)")]}) + table_styles.append( + { + "selector": f"th.col{i}, td.col{i}", + "props": [ + ("min-width", f"{w}px"), + ("max-width", f"{w}px"), + ("text-align", "center"), + ("border", f"1px solid var(--border-color-primary)"), + ], + } + ) styler = styler.set_table_styles(table_styles) return styler.to_html() -def create_world_languages_gdp_chart(): - languages = ["English", "Chinese", "Spanish", "Japanese", "German", "French", "Arabic", "Italian", "Portuguese", "Korean", "Other"] - shares = [27, 18, 8, 6, 5, 4, 3, 2, 2, 2, 23] - colors = ["#FF7F7F", "#FFA07A", "#FFDB58", "#90EE90", "#98FB98", "#87CEFA", "#B0C4DE", "#DDA0DD", "#D8BFD8", "#F0E68C", "#E0FFFF"] - fig = go.Figure( - data=[ - go.Pie( - labels=languages, - values=shares, - hole=0.3, - marker=dict(colors=colors, line=dict(color="#FFFFFF", width=2)), - textinfo="label+percent", - textposition="outside", - insidetextorientation="radial", - textfont=dict(size=12), - ) - ] - ) - fig.update_layout( - title={ - "text": "World Languages by Share of Global GDP", - "y": 0.95, - "x": 0.5, - "xanchor": "center", - "yanchor": "top", - "font": dict(size=20, color="black"), - }, - showlegend=False, - width=700, - height=500, - margin=dict(t=80, b=20, l=20, r=20), - ) - return fig def check_model_exists(model_id): api = HfApi() @@ -253,6 +192,7 @@ def check_model_exists(model_id): else: return "Error: " + str(e) + def submit_model(name): if "Exists" not in check_model_exists(name): return f"# ERROR: Model {name} does not exist on Hugging Face!" @@ -271,14 +211,24 @@ def submit_model(name): except Exception as e: print(e) return "ERROR: Unexpected error. Please try again later." -def create_scaling_plot(all_data, period): - selected_columns = ["Name", "Parameters Count (B)", "Average (The lower the better)"] - target_data = all_data[period] - new_df = pd.DataFrame() - for size in target_data.keys(): - new_df = pd.concat([new_df, target_data[size]["cr"].loc[:, selected_columns].dropna(axis=1, how="all")], axis=0) - x_values = new_df["Parameters Count (B)"].astype(float).tolist() - y_values = new_df["Average (The lower the better)"].astype(float).tolist() + + +def create_scaling_plot(data_manager: DataManager, period: str): + new_df = data_manager.query( + period=period, + metric_code="cr", + param_range=(0, 40), + model_groups=None, + visible_columns=None, + ) + + if len(new_df) == 0: + fig = go.Figure() + fig.update_layout(title={"text": "Compression Rate Scaling Law", "x": 0.5}, width=800, height=600) + return fig + + x_values = new_df["Params (B)"].astype(float).tolist() + y_values = new_df["Average (lower=better)"].astype(float).tolist() names = new_df["Name"].tolist() x_min, x_max = np.log10(min(x_values)), np.log10(max(x_values)) y_min, y_max = np.log10(min(y_values)), np.log10(max(y_values)) @@ -326,100 +276,88 @@ def create_scaling_plot(all_data, period): ) return fig -def read_all_data(folder_name): - all_data = {} - time_list = [] - for folder in get_folders_matching_format(folder_name): - folder_name = os.path.basename(folder) - time_list.append(folder_name) - if all_data.get(folder) is None: - all_data[folder_name] = {} - for file_name in file_name_list: - if all_data.get(file_name) is None: - all_data[folder_name][file_name] = {} - for sheet_name in sheet_name_list: - final_file_name = os.path.join(folder, file_name) - all_data[folder_name][file_name][sheet_name] = rename_columns(pd.read_excel(final_file_name + ".xlsx", sheet_name=sheet_name)) - return all_data, time_list - -all_data, time_list = read_all_data("data") -time_list.sort() -last_period = time_list[-1] -initial_fig = create_scaling_plot(all_data, last_period) -initial_metric = metric_list[0] -initial_columns = get_unique_column_names(all_data) -initial_colors = ["Average", "Individual Tests"] -initial_size_range = [0, 40] -# 初始调用 update_table 时，request 参数将为默认的 None -initial_data = update_table(last_period, model_size_list, initial_metric, initial_columns, initial_colors, initial_size_range) -css = """ -.gradio-container { - max-width: 95% !important; - margin: 0 auto; -} -.tab-buttons button { - font-size: 1.3em; -} -.gr-dataframe th { - white-space: normal; - word-break: break-word; -} -table { - margin-left: auto !important; - margin-right: auto !important; - width: 100% !important; -} -""" -TITLE_HTML = '

🏆 LLM Compression Leaderboard

' -SUBTITLE_HTML = "

Welcome to Uncheatable Eval LLM Compression Leaderboard, where fancy fine-tuning and cheating won't work 🚫; only compute 💻, data 📊, and real innovation 🔥 can prevail!

" -# theme = gr.themes.Default(primary_hue=slate, secondary_hue=slate) -theme = gr.themes.Default() -with gr.Blocks(theme=theme, css=css) as demo: - gr.HTML(TITLE_HTML) - gr.HTML(SUBTITLE_HTML) - with gr.Tabs() as tabs: - with gr.Tab("🏆 Leaderboard"): - with gr.Row(): - with gr.Column(): - period_selector = gr.Dropdown(label="Period", choices=time_list, value=last_period) - model_selector = gr.CheckboxGroup(label="Model Size", choices=model_size_list, value=model_size_list) - size_range_slider = RangeSlider(minimum=0, maximum=40, value=[0, 40], step=0.1, label="Model Size Range") - metric_selector = gr.Dropdown(label="Metric", choices=metric_list, value=initial_metric) - with gr.Column(): - midpoint_slider = gr.Slider(minimum=0.1, maximum=0.9, value=0.5, step=0.01, label="Color Gradient Midpoint") - color_selector = gr.CheckboxGroup(label="Colored Columns", choices=["Average", "Individual Tests"], value=initial_colors) - colfilter = gr.CheckboxGroup(label="Data Source", choices=get_unique_column_names(all_data), value=initial_columns) - table = gr.HTML(initial_data) - - # --- 核心改动点 4: 更新所有 .change() 事件，添加 gr.Request() --- - # 定义共享的输入列表，避免重复 - shared_inputs = [period_selector, model_selector, metric_selector, colfilter, color_selector, size_range_slider, midpoint_slider] - - period_selector.change(update_table, inputs=shared_inputs, outputs=table) - model_selector.change(update_table, inputs=shared_inputs, outputs=table) - metric_selector.change(update_table, inputs=shared_inputs, outputs=table) - colfilter.change(update_table, inputs=shared_inputs, outputs=table) - color_selector.change(update_table, inputs=shared_inputs, outputs=table) - size_range_slider.change(update_table, inputs=shared_inputs, outputs=table) - midpoint_slider.change(update_table, inputs=shared_inputs, outputs=table) - - with gr.Tab("🌍 MultiLang"): - gr.Markdown("## Coming soon...") - # world_languages_plot = gr.Plot(create_world_languages_gdp_chart()) - with gr.Tab("📈 Scaling Law"): - period_selector_2 = gr.Dropdown(label="Period", choices=time_list, value=last_period) - def update_plot(period): - new_fig = create_scaling_plot(all_data, period) - return new_fig - plot = gr.Plot(initial_fig) - period_selector_2.change(update_plot, inputs=period_selector_2, outputs=plot) - with gr.Tab("ℹ️ About"): - gr.Markdown(read_about_md()) - with gr.Tab("🚀 Submit"): - with gr.Group(): + +if __name__ == "__main__": + data_manager = DataManager("data") + time_list = data_manager.get_available_periods() + last_period = time_list[-1] + + initial_fig = create_scaling_plot(data_manager, last_period) if last_period else go.Figure() + initial_metric = metric_list[0] + initial_columns = data_manager.get_available_columns(last_period) + initial_colors = ["Average", "Individual Tests"] + initial_size_range = [0, 40] + initial_data = update_table(data_manager, last_period, model_size_list, initial_metric, initial_columns, initial_colors, initial_size_range) + + theme = gr.themes.Default() + with gr.Blocks(theme=theme, css=css) as demo: + gr.HTML(TITLE_HTML) + gr.HTML(SUBTITLE_HTML) + with gr.Tabs() as tabs: + with gr.Tab("🏆 Leaderboard"): with gr.Row(): - model_name = gr.Textbox(max_lines=1, placeholder="Enter model name...", show_label=False, scale=4) - submit = gr.Button("Submit", variant="primary", scale=0) - output = gr.Markdown("# Enter a public HF repo id, then hit Submit to add it to the evaluation queue.") - submit.click(fn=submit_model, inputs=model_name, outputs=output) -demo.launch(share=False) \ No newline at end of file + with gr.Column(): + period_selector = gr.Dropdown(label="Period", choices=time_list, value=last_period) + model_selector = gr.CheckboxGroup(label="Model Size", choices=model_size_list, value=model_size_list) + size_range_slider = RangeSlider(minimum=0, maximum=40, value=[0, 40], step=0.1, label="Model Size Range") + metric_selector = gr.Dropdown(label="Metric", choices=metric_list, value=initial_metric) + with gr.Column(): + midpoint_slider = gr.Slider(minimum=0.1, maximum=0.9, value=0.5, step=0.01, label="Color Gradient Midpoint") + color_selector = gr.CheckboxGroup(label="Colored Columns", choices=["Average", "Individual Tests"], value=initial_colors) + colfilter = gr.CheckboxGroup(label="Data Source", choices=initial_columns, value=initial_columns) + table = gr.HTML(initial_data) + + def update_table_wrapper(period, models_size, metric, visible_columns, color_columns, size_range, midpoint): + return update_table(data_manager, period, models_size, metric, visible_columns, color_columns, size_range, midpoint) + + def update_column_choices(period, current_selected): + if not period: + return gr.update(choices=[], value=[]) + columns = data_manager.get_available_columns(period) + # 只保留在新 choices 中存在的已选择值 + if current_selected: + valid_selected = [col for col in current_selected if col in columns] + # 如果过滤后为空，默认选择所有列（保持默认行为） + if not valid_selected: + valid_selected = columns + else: + # 如果没有当前选择，默认选择所有列（保持默认行为） + valid_selected = columns + return gr.update(choices=columns, value=valid_selected) + + shared_inputs = [period_selector, model_selector, metric_selector, colfilter, color_selector, size_range_slider, midpoint_slider] + + period_selector.change(update_column_choices, inputs=[period_selector, colfilter], outputs=colfilter) + period_selector.change(update_table_wrapper, inputs=shared_inputs, outputs=table) + model_selector.change(update_table_wrapper, inputs=shared_inputs, outputs=table) + metric_selector.change(update_table_wrapper, inputs=shared_inputs, outputs=table) + colfilter.change(update_table_wrapper, inputs=shared_inputs, outputs=table) + color_selector.change(update_table_wrapper, inputs=shared_inputs, outputs=table) + size_range_slider.change(update_table_wrapper, inputs=shared_inputs, outputs=table) + midpoint_slider.change(update_table_wrapper, inputs=shared_inputs, outputs=table) + + with gr.Tab("📚 Long Context"): + gr.Markdown("## Coming soon...") + + with gr.Tab("📈 Scaling Law"): + period_selector_2 = gr.Dropdown(label="Period", choices=time_list, value=last_period) + + def update_plot(period): + new_fig = create_scaling_plot(data_manager, period) + return new_fig + + plot = gr.Plot(initial_fig) + period_selector_2.change(update_plot, inputs=period_selector_2, outputs=plot) + + with gr.Tab("ℹ️ About"): + gr.Markdown(read_about_md()) + + with gr.Tab("🚀 Submit"): + with gr.Group(): + with gr.Row(): + model_name = gr.Textbox(max_lines=1, placeholder="Enter model name...", show_label=False, scale=4) + submit = gr.Button("Submit", variant="primary", scale=0) + output = gr.Markdown("# Enter a public HF repo id, then hit Submit to add it to the evaluation queue.") + submit.click(fn=submit_model, inputs=model_name, outputs=output) + + demo.launch(share=False) diff --git a/data/2024-10/7b.xlsx b/data/2024-10/7b.xlsx index 9d786238459790394d254d7bb27226e43b973ffa..1297d07e7a9cd087f2dda7f1821316fe9197bffb 100644 Binary files a/data/2024-10/7b.xlsx and b/data/2024-10/7b.xlsx differ diff --git a/data/2024-10/xb.xlsx b/data/2024-10/xb.xlsx deleted file mode 100644 index 36f7f0122e757e6afd515057736801b7fa8fc6e4..0000000000000000000000000000000000000000 Binary files a/data/2024-10/xb.xlsx and /dev/null differ diff --git a/data/2025-12/2025-12-21_11-34-39.json b/data/2025-12/2025-12-21_11-34-39.json new file mode 100644 index 0000000000000000000000000000000000000000..e7a8ff245c5255dc97248094109d679bc7e79d9e --- /dev/null +++ b/data/2025-12/2025-12-21_11-34-39.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4649.08, + "avg tokens": 1909.12, + "avg character count": 7857.404, + "parameters count": 1.527404544, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 10.463994754364728, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-35-15.json b/data/2025-12/2025-12-21_11-35-15.json new file mode 100644 index 0000000000000000000000000000000000000000..ba3f4684c9a682c570dc6d6c5a2e2e5cbb1d6d59 --- /dev/null +++ b/data/2025-12/2025-12-21_11-35-15.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4283.474, + "avg tokens": 2095.926, + "avg character count": 9964.74, + "parameters count": 1.527404544, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.729221971112452, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-36-04.json b/data/2025-12/2025-12-21_11-36-04.json new file mode 100644 index 0000000000000000000000000000000000000000..c2a856db580315faf6e35c02ecb06d40a0fdc2ba --- /dev/null +++ b/data/2025-12/2025-12-21_11-36-04.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4036.0446875, + "avg tokens": 2925.354, + "avg character count": 9913.284, + "parameters count": 1.527404544, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.338155351540054, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-36-44.json b/data/2025-12/2025-12-21_11-36-44.json new file mode 100644 index 0000000000000000000000000000000000000000..a6cc6d6c9dfb997f1553eee3100a254513acce19 --- /dev/null +++ b/data/2025-12/2025-12-21_11-36-44.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4376.222, + "avg tokens": 2448.906, + "avg character count": 9946.974, + "parameters count": 1.527404544, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.9293688424729485, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-37-00.json b/data/2025-12/2025-12-21_11-37-00.json new file mode 100644 index 0000000000000000000000000000000000000000..e672c5cb6c8146ca6b40d66513b6345fafed331f --- /dev/null +++ b/data/2025-12/2025-12-21_11-37-00.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1719.608, + "avg tokens": 739.35, + "avg character count": 3394.84, + "parameters count": 1.527404544, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.128911006492899, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-37-31.json b/data/2025-12/2025-12-21_11-37-31.json new file mode 100644 index 0000000000000000000000000000000000000000..9638cc842451dd60dd010da3764cdb32f5fd4b90 --- /dev/null +++ b/data/2025-12/2025-12-21_11-37-31.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1347.243, + "avg tokens": 1773.934, + "avg character count": 5773.33, + "parameters count": 1.527404544, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.150883427491335, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-37-59.json b/data/2025-12/2025-12-21_11-37-59.json new file mode 100644 index 0000000000000000000000000000000000000000..3f1ab553d620f06db5efcb187e2865c767e91703 --- /dev/null +++ b/data/2025-12/2025-12-21_11-37-59.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1377.357875, + "avg tokens": 1654.562, + "avg character count": 5774.754, + "parameters count": 1.527404544, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.231036645040064, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-38-27.json b/data/2025-12/2025-12-21_11-38-27.json new file mode 100644 index 0000000000000000000000000000000000000000..3b0df9e8a52bb9cf88939b2bfa7d857cc104d2d0 --- /dev/null +++ b/data/2025-12/2025-12-21_11-38-27.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 2226.4415625, + "avg tokens": 1598.294, + "avg character count": 5024.17, + "parameters count": 1.527404544, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.270959789757048, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-38-57.json b/data/2025-12/2025-12-21_11-38-57.json new file mode 100644 index 0000000000000000000000000000000000000000..1a7a1e5ee567965cd61e05bf396cfde7d0134edf --- /dev/null +++ b/data/2025-12/2025-12-21_11-38-57.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1621.03725, + "avg tokens": 1791.012, + "avg character count": 6339.622, + "parameters count": 1.527404544, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.49917614458958, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-39-11.json b/data/2025-12/2025-12-21_11-39-11.json new file mode 100644 index 0000000000000000000000000000000000000000..b99b4a013dacda067d95d5aed981c3278dcf4837 --- /dev/null +++ b/data/2025-12/2025-12-21_11-39-11.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1502.122, + "avg tokens": 718.362, + "avg character count": 3043.39, + "parameters count": 1.527404544, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1b-1.5b-20251202-ctx8192.pth", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.845923087226053, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-39-42.json b/data/2025-12/2025-12-21_11-39-42.json new file mode 100644 index 0000000000000000000000000000000000000000..7c9b787ebe28e190bd5a2ca331b2932846051430 --- /dev/null +++ b/data/2025-12/2025-12-21_11-39-42.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 5066.424, + "avg tokens": 1833.724, + "avg character count": 7857.404, + "parameters count": 1.720574976, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.403338759364772, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-40-01.json b/data/2025-12/2025-12-21_11-40-01.json new file mode 100644 index 0000000000000000000000000000000000000000..c260fed192bb81a5dd6955ee1e7a0ec5c654bf42 --- /dev/null +++ b/data/2025-12/2025-12-21_11-40-01.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4186.624, + "avg tokens": 2071.622, + "avg character count": 9964.74, + "parameters count": 1.720574976, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.554463084306498, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-40-26.json b/data/2025-12/2025-12-21_11-40-26.json new file mode 100644 index 0000000000000000000000000000000000000000..82ea4de0d99ce75d38ffc0c419b0616c28a5099f --- /dev/null +++ b/data/2025-12/2025-12-21_11-40-26.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 3646.42, + "avg tokens": 3000.148, + "avg character count": 9913.284, + "parameters count": 1.720574976, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 6.6297572273752685, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-40-48.json b/data/2025-12/2025-12-21_11-40-48.json new file mode 100644 index 0000000000000000000000000000000000000000..970f88c1a8d8bbd0c0e3ce4fa68c502c28c10bc0 --- /dev/null +++ b/data/2025-12/2025-12-21_11-40-48.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4222.864, + "avg tokens": 2501.464, + "avg character count": 9946.974, + "parameters count": 1.720574976, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.651496251241524, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-41-02.json b/data/2025-12/2025-12-21_11-41-02.json new file mode 100644 index 0000000000000000000000000000000000000000..fcbc0f79e275d892371cd755ef1b7dd09f94dd0d --- /dev/null +++ b/data/2025-12/2025-12-21_11-41-02.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1826.096, + "avg tokens": 720.27, + "avg character count": 3394.84, + "parameters count": 1.720574976, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.69422558705976, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-41-20.json b/data/2025-12/2025-12-21_11-41-20.json new file mode 100644 index 0000000000000000000000000000000000000000..c35605349c0afbc089e361ae110b39db4d132f3f --- /dev/null +++ b/data/2025-12/2025-12-21_11-41-20.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1175.4235, + "avg tokens": 1617.712, + "avg character count": 5773.33, + "parameters count": 1.720574976, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 3.6215040096210274, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-41-38.json b/data/2025-12/2025-12-21_11-41-38.json new file mode 100644 index 0000000000000000000000000000000000000000..bf6d3a817e65bb0cc150a353e94b91d477c2932f --- /dev/null +++ b/data/2025-12/2025-12-21_11-41-38.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1212.134, + "avg tokens": 1498.248, + "avg character count": 5774.754, + "parameters count": 1.720574976, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 3.723493701808611, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-41-55.json b/data/2025-12/2025-12-21_11-41-55.json new file mode 100644 index 0000000000000000000000000000000000000000..97571534acbee8106d303223ba551bdb0aa0b381 --- /dev/null +++ b/data/2025-12/2025-12-21_11-41-55.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2129.001, + "avg tokens": 1446.138, + "avg character count": 5024.17, + "parameters count": 1.720574976, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 6.952745099660591, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-42-12.json b/data/2025-12/2025-12-21_11-42-12.json new file mode 100644 index 0000000000000000000000000000000000000000..306836e171ca0c5b287ffd8f825fe274419e8e05 --- /dev/null +++ b/data/2025-12/2025-12-21_11-42-12.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1409.987, + "avg tokens": 1585.12, + "avg character count": 6339.622, + "parameters count": 1.720574976, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 3.9134078347560655, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-42-26.json b/data/2025-12/2025-12-21_11-42-26.json new file mode 100644 index 0000000000000000000000000000000000000000..ce9c994578cdb1f38957bd8b600210cbb861311f --- /dev/null +++ b/data/2025-12/2025-12-21_11-42-26.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1590.208, + "avg tokens": 750.442, + "avg character count": 3043.39, + "parameters count": 1.720574976, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-1.7B-Base", + "tokenizer_name": "Qwen/Qwen3-1.7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.36465723868738, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-42-49.json b/data/2025-12/2025-12-21_11-42-49.json new file mode 100644 index 0000000000000000000000000000000000000000..6d9742df16aeea36000e3c3533eddd76a1a72ec9 --- /dev/null +++ b/data/2025-12/2025-12-21_11-42-49.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4847.352, + "avg tokens": 1949.908, + "avg character count": 7857.404, + "parameters count": 1.711376384, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 10.910258782503071, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-43-05.json b/data/2025-12/2025-12-21_11-43-05.json new file mode 100644 index 0000000000000000000000000000000000000000..a1c78bd140f3debc02d97f3f94eb623c6b1bb0e0 --- /dev/null +++ b/data/2025-12/2025-12-21_11-43-05.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4517.79, + "avg tokens": 2182.888, + "avg character count": 9964.74, + "parameters count": 1.711376384, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.152028407052809, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-43-28.json b/data/2025-12/2025-12-21_11-43-28.json new file mode 100644 index 0000000000000000000000000000000000000000..18524830453345e34d5aa05f74013493eae396cb --- /dev/null +++ b/data/2025-12/2025-12-21_11-43-28.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4149.150625, + "avg tokens": 3143.934, + "avg character count": 9913.284, + "parameters count": 1.711376384, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.543799491984567, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-43-47.json b/data/2025-12/2025-12-21_11-43-47.json new file mode 100644 index 0000000000000000000000000000000000000000..419bce7e74f3efb1e207751f1836c91e5d0c87af --- /dev/null +++ b/data/2025-12/2025-12-21_11-43-47.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4609.628, + "avg tokens": 2602.328, + "avg character count": 9946.974, + "parameters count": 1.711376384, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.352282091400047, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-43-58.json b/data/2025-12/2025-12-21_11-43-58.json new file mode 100644 index 0000000000000000000000000000000000000000..0396d880ef5b3c51683860b3835781b641d3d09c --- /dev/null +++ b/data/2025-12/2025-12-21_11-43-58.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1738.216, + "avg tokens": 755.956, + "avg character count": 3394.84, + "parameters count": 1.711376384, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.227695599265683, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-44-14.json b/data/2025-12/2025-12-21_11-44-14.json new file mode 100644 index 0000000000000000000000000000000000000000..fdc68d4043c08b452bbeb4372712b36c5b77999f --- /dev/null +++ b/data/2025-12/2025-12-21_11-44-14.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1344.355, + "avg tokens": 1998.214, + "avg character count": 5773.33, + "parameters count": 1.711376384, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.141985440017216, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-44-30.json b/data/2025-12/2025-12-21_11-44-30.json new file mode 100644 index 0000000000000000000000000000000000000000..f69c15e0583e88e7defea5803dd6160e87394454 --- /dev/null +++ b/data/2025-12/2025-12-21_11-44-30.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1449.103, + "avg tokens": 1865.214, + "avg character count": 5774.754, + "parameters count": 1.711376384, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.4514268998080775, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-44-45.json b/data/2025-12/2025-12-21_11-44-45.json new file mode 100644 index 0000000000000000000000000000000000000000..bf1f076170dd4907bac89a9799750cd992778c6d --- /dev/null +++ b/data/2025-12/2025-12-21_11-44-45.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2527.254, + "avg tokens": 1888.098, + "avg character count": 5024.17, + "parameters count": 1.711376384, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.25333236766804, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-45-01.json b/data/2025-12/2025-12-21_11-45-01.json new file mode 100644 index 0000000000000000000000000000000000000000..9ef80511924710136a7ad06ac0c71d5470a317e2 --- /dev/null +++ b/data/2025-12/2025-12-21_11-45-01.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1684.25, + "avg tokens": 1931.562, + "avg character count": 6339.622, + "parameters count": 1.711376384, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.674622635306498, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-45-11.json b/data/2025-12/2025-12-21_11-45-11.json new file mode 100644 index 0000000000000000000000000000000000000000..2ee97e1b8305b151851d15f95e781e2b5a91009d --- /dev/null +++ b/data/2025-12/2025-12-21_11-45-11.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1584.196, + "avg tokens": 779.642, + "avg character count": 3043.39, + "parameters count": 1.711376384, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM2-1.7B", + "tokenizer_name": "HuggingFaceTB/SmolLM2-1.7B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.32925286434202, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-45-38.json b/data/2025-12/2025-12-21_11-45-38.json new file mode 100644 index 0000000000000000000000000000000000000000..d287883d1948fdc1498403f307187349714d8b0b --- /dev/null +++ b/data/2025-12/2025-12-21_11-45-38.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 5079.304, + "avg tokens": 1833.724, + "avg character count": 7857.404, + "parameters count": 1.543714304, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.432328635305005, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-45-55.json b/data/2025-12/2025-12-21_11-45-55.json new file mode 100644 index 0000000000000000000000000000000000000000..e58cbcc6e875a9de79e0f44c4b95542168ff48f7 --- /dev/null +++ b/data/2025-12/2025-12-21_11-45-55.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4373.472, + "avg tokens": 2071.622, + "avg character count": 9964.74, + "parameters count": 1.543714304, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.891616914785782, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-46-17.json b/data/2025-12/2025-12-21_11-46-17.json new file mode 100644 index 0000000000000000000000000000000000000000..6cfdd781d1ada0994355bdf4d2dd280077c5fb4f --- /dev/null +++ b/data/2025-12/2025-12-21_11-46-17.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 3793.949, + "avg tokens": 3000.148, + "avg character count": 9913.284, + "parameters count": 1.543714304, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 6.897987835477859, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-46-35.json b/data/2025-12/2025-12-21_11-46-35.json new file mode 100644 index 0000000000000000000000000000000000000000..a94cc99a1f9bfc0f667a2995a8d2c83e038ade51 --- /dev/null +++ b/data/2025-12/2025-12-21_11-46-35.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4389.584, + "avg tokens": 2501.464, + "avg character count": 9946.974, + "parameters count": 1.543714304, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.9535797317909775, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-46-50.json b/data/2025-12/2025-12-21_11-46-50.json new file mode 100644 index 0000000000000000000000000000000000000000..29aeced9a8395d2dd0a12ac042f81b8dcc0a95f6 --- /dev/null +++ b/data/2025-12/2025-12-21_11-46-50.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1785.08, + "avg tokens": 720.27, + "avg character count": 3394.84, + "parameters count": 1.543714304, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.476483279602297, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-47-06.json b/data/2025-12/2025-12-21_11-47-06.json new file mode 100644 index 0000000000000000000000000000000000000000..80f9a97860b882877f69d21194ca8620c65b9c0c --- /dev/null +++ b/data/2025-12/2025-12-21_11-47-06.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1258.625, + "avg tokens": 1617.712, + "avg character count": 5773.33, + "parameters count": 1.543714304, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 3.877849544533749, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-47-21.json b/data/2025-12/2025-12-21_11-47-21.json new file mode 100644 index 0000000000000000000000000000000000000000..50246294a87123bae13ca96fc25855bdf1eacb91 --- /dev/null +++ b/data/2025-12/2025-12-21_11-47-21.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1324.075, + "avg tokens": 1498.248, + "avg character count": 5774.754, + "parameters count": 1.543714304, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.067359651014027, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-47-36.json b/data/2025-12/2025-12-21_11-47-36.json new file mode 100644 index 0000000000000000000000000000000000000000..7d096ad7d22e20078922254920dcceaf139879ab --- /dev/null +++ b/data/2025-12/2025-12-21_11-47-36.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2284.521, + "avg tokens": 1446.14, + "avg character count": 5024.17, + "parameters count": 1.543714304, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.4606316238563135, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-47-52.json b/data/2025-12/2025-12-21_11-47-52.json new file mode 100644 index 0000000000000000000000000000000000000000..615d3ff385272d3f87bde9b96c41999d4943c48a --- /dev/null +++ b/data/2025-12/2025-12-21_11-47-52.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1501.532, + "avg tokens": 1585.12, + "avg character count": 6339.622, + "parameters count": 1.543714304, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.1674902626314605, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-48-04.json b/data/2025-12/2025-12-21_11-48-04.json new file mode 100644 index 0000000000000000000000000000000000000000..17440deea75c9e2f1d3641cdce55d092ac1452ad --- /dev/null +++ b/data/2025-12/2025-12-21_11-48-04.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1596.3, + "avg tokens": 750.442, + "avg character count": 3043.39, + "parameters count": 1.543714304, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-1.5B", + "tokenizer_name": "Qwen/Qwen2.5-1.5B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.400532729125164, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-48-25.json b/data/2025-12/2025-12-21_11-48-25.json new file mode 100644 index 0000000000000000000000000000000000000000..da7ffe3159e2af92d02315dfd8be67088f334665 --- /dev/null +++ b/data/2025-12/2025-12-21_11-48-25.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 5037.112, + "avg tokens": 1832.424, + "avg character count": 7857.404, + "parameters count": 1.2358144, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.337364283933086, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-48-37.json b/data/2025-12/2025-12-21_11-48-37.json new file mode 100644 index 0000000000000000000000000000000000000000..886f2b101990f4ac10e914946ed843658e361286 --- /dev/null +++ b/data/2025-12/2025-12-21_11-48-37.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4519.312, + "avg tokens": 2045.48, + "avg character count": 9964.74, + "parameters count": 1.2358144, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.154774747018926, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-48-52.json b/data/2025-12/2025-12-21_11-48-52.json new file mode 100644 index 0000000000000000000000000000000000000000..35e7425d96515ef832a20852955ee2400f36d0db --- /dev/null +++ b/data/2025-12/2025-12-21_11-48-52.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4072.908, + "avg tokens": 2984.08, + "avg character count": 9913.284, + "parameters count": 1.2358144, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.405178572252937, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-49-05.json b/data/2025-12/2025-12-21_11-49-05.json new file mode 100644 index 0000000000000000000000000000000000000000..13588f33b07f0cede52e592fa9c6019e58ebec07 --- /dev/null +++ b/data/2025-12/2025-12-21_11-49-05.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4462.048, + "avg tokens": 2454.32, + "avg character count": 9946.974, + "parameters count": 1.2358144, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.084878780102732, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-49-14.json b/data/2025-12/2025-12-21_11-49-14.json new file mode 100644 index 0000000000000000000000000000000000000000..e27e8192f2f130bde7f141a5c242990570edef20 --- /dev/null +++ b/data/2025-12/2025-12-21_11-49-14.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1729.884, + "avg tokens": 708.868, + "avg character count": 3394.84, + "parameters count": 1.2358144, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.183463375115702, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-49-24.json b/data/2025-12/2025-12-21_11-49-24.json new file mode 100644 index 0000000000000000000000000000000000000000..38b92c22b8c1e7d412d2035ee350617bff733b64 --- /dev/null +++ b/data/2025-12/2025-12-21_11-49-24.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1425.586, + "avg tokens": 1587.644, + "avg character count": 5773.33, + "parameters count": 1.2358144, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.3922598238503845, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-49-34.json b/data/2025-12/2025-12-21_11-49-34.json new file mode 100644 index 0000000000000000000000000000000000000000..61ab0f2135429c1b871416428aa05446c1b5a33f --- /dev/null +++ b/data/2025-12/2025-12-21_11-49-34.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1492.726, + "avg tokens": 1465.598, + "avg character count": 5774.754, + "parameters count": 1.2358144, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.585430207820226, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-49-44.json b/data/2025-12/2025-12-21_11-49-44.json new file mode 100644 index 0000000000000000000000000000000000000000..d136709d110975aff7b54bbdb03021bd4df9628b --- /dev/null +++ b/data/2025-12/2025-12-21_11-49-44.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2487.342, + "avg tokens": 1423.34, + "avg character count": 5024.17, + "parameters count": 1.2358144, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.122990501967813, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-49-54.json b/data/2025-12/2025-12-21_11-49-54.json new file mode 100644 index 0000000000000000000000000000000000000000..f91731d191a7a4d42344e0aff29d5eb5512b1b2a --- /dev/null +++ b/data/2025-12/2025-12-21_11-49-54.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1718.098, + "avg tokens": 1561.164, + "avg character count": 6339.622, + "parameters count": 1.2358144, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.768567493231305, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-50-02.json b/data/2025-12/2025-12-21_11-50-02.json new file mode 100644 index 0000000000000000000000000000000000000000..f96cb5dfcf1dbe17e94989b74d592228854044a0 --- /dev/null +++ b/data/2025-12/2025-12-21_11-50-02.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1544.044, + "avg tokens": 712.354, + "avg character count": 3043.39, + "parameters count": 1.2358144, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-1B", + "tokenizer_name": "meta-llama/Llama-3.2-1B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.092799697556432, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-51-11.json b/data/2025-12/2025-12-21_11-51-11.json new file mode 100644 index 0000000000000000000000000000000000000000..d9dcde48718bb49f9114ec549b9a52d657c4543b --- /dev/null +++ b/data/2025-12/2025-12-21_11-51-11.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4940.919465942383, + "avg tokens": 2143.444, + "avg character count": 7857.404, + "parameters count": 2.172819456, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.120857325182547, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-52-13.json b/data/2025-12/2025-12-21_11-52-13.json new file mode 100644 index 0000000000000000000000000000000000000000..f9728fe7f068b7f8415d17bb42916ae098d9e3c4 --- /dev/null +++ b/data/2025-12/2025-12-21_11-52-13.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4349.212635742188, + "avg tokens": 2445.186, + "avg character count": 9964.74, + "parameters count": 2.172819456, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.84784262988836, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-53-39.json b/data/2025-12/2025-12-21_11-53-39.json new file mode 100644 index 0000000000000000000000000000000000000000..0c6190312638d54285e7a17db707aeaf7b11e684 --- /dev/null +++ b/data/2025-12/2025-12-21_11-53-39.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 3757.0556427116394, + "avg tokens": 3175.026, + "avg character count": 9913.284, + "parameters count": 2.172819456, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 6.830909988678903, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-54-51.json b/data/2025-12/2025-12-21_11-54-51.json new file mode 100644 index 0000000000000000000000000000000000000000..56ed347415d49cec19ce500d9ffb5e240641dfff --- /dev/null +++ b/data/2025-12/2025-12-21_11-54-51.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4258.790341552734, + "avg tokens": 2790.282, + "avg character count": 9946.974, + "parameters count": 2.172819456, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.716591946417019, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-55-10.json b/data/2025-12/2025-12-21_11-55-10.json new file mode 100644 index 0000000000000000000000000000000000000000..2b39abd9523bc11d2465eaa59d7ba4b46b7ba933 --- /dev/null +++ b/data/2025-12/2025-12-21_11-55-10.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1752.3583596801757, + "avg tokens": 851.568, + "avg character count": 3394.84, + "parameters count": 2.172819456, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.302773374515704, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-56-06.json b/data/2025-12/2025-12-21_11-56-06.json new file mode 100644 index 0000000000000000000000000000000000000000..21511424ea2276815db53d8f2648c811232c87fd --- /dev/null +++ b/data/2025-12/2025-12-21_11-56-06.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1434.8123400726317, + "avg tokens": 2150.434, + "avg character count": 5773.33, + "parameters count": 2.172819456, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.420686367617089, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-56-55.json b/data/2025-12/2025-12-21_11-56-55.json new file mode 100644 index 0000000000000000000000000000000000000000..029456ff70a1337db76a94aabc0f98cbce1b95cb --- /dev/null +++ b/data/2025-12/2025-12-21_11-56-55.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1621.1595363311767, + "avg tokens": 1964.578, + "avg character count": 5774.754, + "parameters count": 2.172819456, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.979958753038942, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-57-39.json b/data/2025-12/2025-12-21_11-57-39.json new file mode 100644 index 0000000000000000000000000000000000000000..0bdf03333c44780a466f5c4ef21ea1ec4cda4a61 --- /dev/null +++ b/data/2025-12/2025-12-21_11-57-39.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 2560.999067321777, + "avg tokens": 1795.488, + "avg character count": 5024.17, + "parameters count": 2.172819456, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.36353468859659, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-58-34.json b/data/2025-12/2025-12-21_11-58-34.json new file mode 100644 index 0000000000000000000000000000000000000000..fa9033db13969556f49926b770955c513b69c3e6 --- /dev/null +++ b/data/2025-12/2025-12-21_11-58-34.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1787.3829472351074, + "avg tokens": 2147.22, + "avg character count": 6339.622, + "parameters count": 2.172819456, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.9608673196414275, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-58-54.json b/data/2025-12/2025-12-21_11-58-54.json new file mode 100644 index 0000000000000000000000000000000000000000..1e1043facc091d5e9fc133d2fdfe806bcd14f26f --- /dev/null +++ b/data/2025-12/2025-12-21_11-58-54.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1537.1381856079101, + "avg tokens": 851.326, + "avg character count": 3043.39, + "parameters count": 2.172819456, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.052131693914195, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-59-23.json b/data/2025-12/2025-12-21_11-59-23.json new file mode 100644 index 0000000000000000000000000000000000000000..eea171ec39405550e0a64d58e33978a4182a5ce9 --- /dev/null +++ b/data/2025-12/2025-12-21_11-59-23.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4946.728, + "avg tokens": 1834.88, + "avg character count": 7857.404, + "parameters count": 1.644515328, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.133930980595974, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-59-37.json b/data/2025-12/2025-12-21_11-59-37.json new file mode 100644 index 0000000000000000000000000000000000000000..f65de05aebeab866319383ec9fedfe6b245742b3 --- /dev/null +++ b/data/2025-12/2025-12-21_11-59-37.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4660.344, + "avg tokens": 2078.53, + "avg character count": 9964.74, + "parameters count": 1.644515328, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.409256887690246, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_11-59-55.json b/data/2025-12/2025-12-21_11-59-55.json new file mode 100644 index 0000000000000000000000000000000000000000..e3bf6c6362a4ba3e8b9de0a4e2d5bd944836f7e3 --- /dev/null +++ b/data/2025-12/2025-12-21_11-59-55.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4274.282, + "avg tokens": 3000.542, + "avg character count": 9913.284, + "parameters count": 1.644515328, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.771307743304398, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-00-11.json b/data/2025-12/2025-12-21_12-00-11.json new file mode 100644 index 0000000000000000000000000000000000000000..c1c36410f4b0022c915431e88b3407754ad8a323 --- /dev/null +++ b/data/2025-12/2025-12-21_12-00-11.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4655.424, + "avg tokens": 2501.924, + "avg character count": 9946.974, + "parameters count": 1.644515328, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.435260828655581, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-00-21.json b/data/2025-12/2025-12-21_12-00-21.json new file mode 100644 index 0000000000000000000000000000000000000000..2dddc909e1c1d3b21166faea68a0c62fbf96e75a --- /dev/null +++ b/data/2025-12/2025-12-21_12-00-21.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1713.328, + "avg tokens": 720.306, + "avg character count": 3394.84, + "parameters count": 1.644515328, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.095572268175346, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-00-35.json b/data/2025-12/2025-12-21_12-00-35.json new file mode 100644 index 0000000000000000000000000000000000000000..a03042a9b1ea1369fdc22ff652583aab9fd05b2e --- /dev/null +++ b/data/2025-12/2025-12-21_12-00-35.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1525.651, + "avg tokens": 1629.506, + "avg character count": 5773.33, + "parameters count": 1.644515328, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.700562149542127, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-00-47.json b/data/2025-12/2025-12-21_12-00-47.json new file mode 100644 index 0000000000000000000000000000000000000000..34ae760bcd471b6b9737900b6a96d62545c8f30d --- /dev/null +++ b/data/2025-12/2025-12-21_12-00-47.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1569.83, + "avg tokens": 1517.18, + "avg character count": 5774.754, + "parameters count": 1.644515328, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.822282122199536, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-01-00.json b/data/2025-12/2025-12-21_12-01-00.json new file mode 100644 index 0000000000000000000000000000000000000000..70b3811b55150189be3988db638f18ea496b23a8 --- /dev/null +++ b/data/2025-12/2025-12-21_12-01-00.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2591.236, + "avg tokens": 1519.574, + "avg character count": 5024.17, + "parameters count": 1.644515328, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.46228038458606, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-01-13.json b/data/2025-12/2025-12-21_12-01-13.json new file mode 100644 index 0000000000000000000000000000000000000000..65a657a91399ec2304d77a084e541f2b4de611da --- /dev/null +++ b/data/2025-12/2025-12-21_12-01-13.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1834.134, + "avg tokens": 1609.206, + "avg character count": 6339.622, + "parameters count": 1.644515328, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 5.090624499085795, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-01-23.json b/data/2025-12/2025-12-21_12-01-23.json new file mode 100644 index 0000000000000000000000000000000000000000..2bff963246e1e82c6b3d99e2741d170de999d0cd --- /dev/null +++ b/data/2025-12/2025-12-21_12-01-23.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1532.04, + "avg tokens": 751.892, + "avg character count": 3043.39, + "parameters count": 1.644515328, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-2-1_6b", + "tokenizer_name": "stabilityai/stablelm-2-1_6b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.022108727888813, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-03-11.json b/data/2025-12/2025-12-21_12-03-11.json new file mode 100644 index 0000000000000000000000000000000000000000..d8456749454df1f985276c5120a08a43b761e20e --- /dev/null +++ b/data/2025-12/2025-12-21_12-03-11.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 5042.128, + "avg tokens": 2079.228, + "avg character count": 7857.404, + "parameters count": 1.554872208, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.348654130029065, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-04-22.json b/data/2025-12/2025-12-21_12-04-22.json new file mode 100644 index 0000000000000000000000000000000000000000..3b0881f46b3c1490b61e461c790b46e9b4538cc8 --- /dev/null +++ b/data/2025-12/2025-12-21_12-04-22.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4286.346, + "avg tokens": 2317.894, + "avg character count": 9964.74, + "parameters count": 1.554872208, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.7344042893665215, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-05-45.json b/data/2025-12/2025-12-21_12-05-45.json new file mode 100644 index 0000000000000000000000000000000000000000..58ee13a16d3e941f81b722fb682f92ca4affbc79 --- /dev/null +++ b/data/2025-12/2025-12-21_12-05-45.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 3753.1995, + "avg tokens": 3293.932, + "avg character count": 9913.284, + "parameters count": 1.554872208, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 6.823898923027586, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-06-56.json b/data/2025-12/2025-12-21_12-06-56.json new file mode 100644 index 0000000000000000000000000000000000000000..ce0037ed80f2ba58957b0ef4dfab36c41b19090a --- /dev/null +++ b/data/2025-12/2025-12-21_12-06-56.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4326.6625, + "avg tokens": 2782.132, + "avg character count": 9946.974, + "parameters count": 1.554872208, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.839570940275909, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-08-06.json b/data/2025-12/2025-12-21_12-08-06.json new file mode 100644 index 0000000000000000000000000000000000000000..95bda5fd3d9171d052d476e242e0bd88bf3ec65f --- /dev/null +++ b/data/2025-12/2025-12-21_12-08-06.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1779.22, + "avg tokens": 822.958, + "avg character count": 3394.84, + "parameters count": 1.554872208, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.445374202127635, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-09-22.json b/data/2025-12/2025-12-21_12-09-22.json new file mode 100644 index 0000000000000000000000000000000000000000..ab9519185cc2175181ea13f159a857ab379ecf14 --- /dev/null +++ b/data/2025-12/2025-12-21_12-09-22.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1412.3945, + "avg tokens": 2142.084, + "avg character count": 5773.33, + "parameters count": 1.554872208, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.351616540690811, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-10-36.json b/data/2025-12/2025-12-21_12-10-36.json new file mode 100644 index 0000000000000000000000000000000000000000..95ad75859dcb7e90e9fc701f080d7ee0a1ae4343 --- /dev/null +++ b/data/2025-12/2025-12-21_12-10-36.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1462.95625, + "avg tokens": 2023.828, + "avg character count": 5774.754, + "parameters count": 1.554872208, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.493982004379503, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-11-47.json b/data/2025-12/2025-12-21_12-11-47.json new file mode 100644 index 0000000000000000000000000000000000000000..5f953e830d5b955650d5fd766dbb5f72753dbe1f --- /dev/null +++ b/data/2025-12/2025-12-21_12-11-47.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2402.947, + "avg tokens": 1759.972, + "avg character count": 5024.17, + "parameters count": 1.554872208, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.847379113017852, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-12-57.json b/data/2025-12/2025-12-21_12-12-57.json new file mode 100644 index 0000000000000000000000000000000000000000..a6ecaae11ac5882f471a3deeaefb56a29bdf8c9b --- /dev/null +++ b/data/2025-12/2025-12-21_12-12-57.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1704.072, + "avg tokens": 2101.646, + "avg character count": 6339.622, + "parameters count": 1.554872208, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.72963844048806, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-14-03.json b/data/2025-12/2025-12-21_12-14-03.json new file mode 100644 index 0000000000000000000000000000000000000000..053fff33e52ed03d4c4901503ebef8a8c242e908 --- /dev/null +++ b/data/2025-12/2025-12-21_12-14-03.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1578.976, + "avg tokens": 843.83, + "avg character count": 3043.39, + "parameters count": 1.554872208, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.298512539311618, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-15-34.json b/data/2025-12/2025-12-21_12-15-34.json new file mode 100644 index 0000000000000000000000000000000000000000..ae88e598c2bd5643f3cdc7e523eaa6ed49bce2f1 --- /dev/null +++ b/data/2025-12/2025-12-21_12-15-34.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4804.152251220703, + "avg tokens": 1909.12, + "avg character count": 7857.404, + "parameters count": 1.599868928, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 10.813026223670692, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-16-12.json b/data/2025-12/2025-12-21_12-16-12.json new file mode 100644 index 0000000000000000000000000000000000000000..143be0a038be9d69fa8554d586b0cbecb3c46175 --- /dev/null +++ b/data/2025-12/2025-12-21_12-16-12.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4701.647844726563, + "avg tokens": 2095.926, + "avg character count": 9964.74, + "parameters count": 1.599868928, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 8.483786716551576, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-17-04.json b/data/2025-12/2025-12-21_12-17-04.json new file mode 100644 index 0000000000000000000000000000000000000000..c1c515b39a4b1fb487b9150a738726a4441a9989 --- /dev/null +++ b/data/2025-12/2025-12-21_12-17-04.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4380.663763916015, + "avg tokens": 2925.354, + "avg character count": 9913.284, + "parameters count": 1.599868928, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 7.9647262930554215, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-17-48.json b/data/2025-12/2025-12-21_12-17-48.json new file mode 100644 index 0000000000000000000000000000000000000000..238a3210bcaf899021edf048e4df75703b4261fe --- /dev/null +++ b/data/2025-12/2025-12-21_12-17-48.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4678.483851806641, + "avg tokens": 2448.906, + "avg character count": 9946.974, + "parameters count": 1.599868928, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 8.477043459981786, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-18-05.json b/data/2025-12/2025-12-21_12-18-05.json new file mode 100644 index 0000000000000000000000000000000000000000..8f92220f4c6f18669c8fc512612d90869d2a7258 --- /dev/null +++ b/data/2025-12/2025-12-21_12-18-05.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1762.4755895996093, + "avg tokens": 739.35, + "avg character count": 3394.84, + "parameters count": 1.599868928, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 9.356482877825025, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-18-38.json b/data/2025-12/2025-12-21_12-18-38.json new file mode 100644 index 0000000000000000000000000000000000000000..50735dbe6d0e1c9fdbbad3bda9d84381b387755f --- /dev/null +++ b/data/2025-12/2025-12-21_12-18-38.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1643.5051528320312, + "avg tokens": 1773.934, + "avg character count": 5773.33, + "parameters count": 1.599868928, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 5.063673221450839, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-19-08.json b/data/2025-12/2025-12-21_12-19-08.json new file mode 100644 index 0000000000000000000000000000000000000000..f9fc90cfbbdadb87ee700234ae5fa6a9b2c6b279 --- /dev/null +++ b/data/2025-12/2025-12-21_12-19-08.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1694.53338470459, + "avg tokens": 1654.562, + "avg character count": 5774.754, + "parameters count": 1.599868928, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 5.20535220153215, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-19-39.json b/data/2025-12/2025-12-21_12-19-39.json new file mode 100644 index 0000000000000000000000000000000000000000..05a25d70143f3839602a0a7c2b9d3539187eab97 --- /dev/null +++ b/data/2025-12/2025-12-21_12-19-39.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2686.577872581482, + "avg tokens": 1598.294, + "avg character count": 5024.17, + "parameters count": 1.599868928, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 8.773641317428913, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-20-12.json b/data/2025-12/2025-12-21_12-20-12.json new file mode 100644 index 0000000000000000000000000000000000000000..5bfb61b5da7c52c02260a932382a079e1bd07b42 --- /dev/null +++ b/data/2025-12/2025-12-21_12-20-12.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2004.1836710205077, + "avg tokens": 1791.012, + "avg character count": 6339.622, + "parameters count": 1.599868928, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 5.562596024262513, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-20-27.json b/data/2025-12/2025-12-21_12-20-27.json new file mode 100644 index 0000000000000000000000000000000000000000..d2b41d6e55d2c2ca8f20435b7f8512e4695cd0f9 --- /dev/null +++ b/data/2025-12/2025-12-21_12-20-27.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1586.8226706542969, + "avg tokens": 718.362, + "avg character count": 3043.39, + "parameters count": 1.599868928, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 9.344721199526102, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-21-34.json b/data/2025-12/2025-12-21_12-21-34.json new file mode 100644 index 0000000000000000000000000000000000000000..86924d5a222a122c4f1adcaa67467cb9162be34f --- /dev/null +++ b/data/2025-12/2025-12-21_12-21-34.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 5113.168, + "avg tokens": 2079.228, + "avg character count": 7857.404, + "parameters count": 1.554859392, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.508548601053453, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-22-03.json b/data/2025-12/2025-12-21_12-22-03.json new file mode 100644 index 0000000000000000000000000000000000000000..ea1271b1c4f32d03ba163c795498d4b899f80078 --- /dev/null +++ b/data/2025-12/2025-12-21_12-22-03.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4357.728, + "avg tokens": 2317.894, + "avg character count": 9964.74, + "parameters count": 1.554859392, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.863207994663192, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-22-37.json b/data/2025-12/2025-12-21_12-22-37.json new file mode 100644 index 0000000000000000000000000000000000000000..496e420e01a5158fdb6badb6b640de4a20ef70cc --- /dev/null +++ b/data/2025-12/2025-12-21_12-22-37.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 3843.735, + "avg tokens": 3293.932, + "avg character count": 9913.284, + "parameters count": 1.554859392, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 6.988506506756019, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-23-07.json b/data/2025-12/2025-12-21_12-23-07.json new file mode 100644 index 0000000000000000000000000000000000000000..d3139e2dcb3f12dd3b3eead441a9ee8294a8226a --- /dev/null +++ b/data/2025-12/2025-12-21_12-23-07.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4405.662, + "avg tokens": 2782.132, + "avg character count": 9946.974, + "parameters count": 1.554859392, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.982711798731203, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-23-34.json b/data/2025-12/2025-12-21_12-23-34.json new file mode 100644 index 0000000000000000000000000000000000000000..e32bce55f91d083b5d896bbe6b0370ab62c8eca2 --- /dev/null +++ b/data/2025-12/2025-12-21_12-23-34.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1803.196, + "avg tokens": 822.958, + "avg character count": 3394.84, + "parameters count": 1.554859392, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.572655983959118, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-24-04.json b/data/2025-12/2025-12-21_12-24-04.json new file mode 100644 index 0000000000000000000000000000000000000000..188dfa136f20949d8d8b0439ecf4d4deac18eb06 --- /dev/null +++ b/data/2025-12/2025-12-21_12-24-04.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1460.103, + "avg tokens": 2142.084, + "avg character count": 5773.33, + "parameters count": 1.554859392, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.498607411677315, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-24-34.json b/data/2025-12/2025-12-21_12-24-34.json new file mode 100644 index 0000000000000000000000000000000000000000..d4b0a2904fc2eaf2ab6f24ed87688c132e149466 --- /dev/null +++ b/data/2025-12/2025-12-21_12-24-34.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1512.8945, + "avg tokens": 2023.828, + "avg character count": 5774.754, + "parameters count": 1.554859392, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.6473848124472115, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-25-02.json b/data/2025-12/2025-12-21_12-25-02.json new file mode 100644 index 0000000000000000000000000000000000000000..85e7f9e528dde2d22a3a52cbde0b498c90c41d7b --- /dev/null +++ b/data/2025-12/2025-12-21_12-25-02.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2469.5905, + "avg tokens": 1759.972, + "avg character count": 5024.17, + "parameters count": 1.554859392, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.065018873661096, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-25-30.json b/data/2025-12/2025-12-21_12-25-30.json new file mode 100644 index 0000000000000000000000000000000000000000..47522cc66ca21d737a732c3a23e7d0b82674b11e --- /dev/null +++ b/data/2025-12/2025-12-21_12-25-30.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1762.199, + "avg tokens": 2101.646, + "avg character count": 6339.622, + "parameters count": 1.554859392, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 4.890969472058469, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-25-56.json b/data/2025-12/2025-12-21_12-25-56.json new file mode 100644 index 0000000000000000000000000000000000000000..cb9bfc6051896bf3428f538b89a2d78881100416 --- /dev/null +++ b/data/2025-12/2025-12-21_12-25-56.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1605.912, + "avg tokens": 843.83, + "avg character count": 3043.39, + "parameters count": 1.554859392, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.457137327629423, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-27-10.json b/data/2025-12/2025-12-21_12-27-10.json new file mode 100644 index 0000000000000000000000000000000000000000..dcc04c658af147d1b74c8d3b23ac729037e941b1 --- /dev/null +++ b/data/2025-12/2025-12-21_12-27-10.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 5037.36, + "avg tokens": 1940.684, + "avg character count": 7857.404, + "parameters count": 1.343757312, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 11.33792247409094, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-27-39.json b/data/2025-12/2025-12-21_12-27-39.json new file mode 100644 index 0000000000000000000000000000000000000000..7b44e02b039dcb828afda2da7e6373da8d8ee131 --- /dev/null +++ b/data/2025-12/2025-12-21_12-27-39.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 4782.966, + "avg tokens": 2120.99, + "avg character count": 9964.74, + "parameters count": 1.343757312, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 8.63051950222736, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-28-09.json b/data/2025-12/2025-12-21_12-28-09.json new file mode 100644 index 0000000000000000000000000000000000000000..495946de9434c628931c63db5e8fe7df796beabb --- /dev/null +++ b/data/2025-12/2025-12-21_12-28-09.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 4204.882375, + "avg tokens": 2919.23, + "avg character count": 9913.284, + "parameters count": 1.343757312, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 7.645128459124055, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-28-37.json b/data/2025-12/2025-12-21_12-28-37.json new file mode 100644 index 0000000000000000000000000000000000000000..c8181f4f832234d259d95cc619640f5ede74e568 --- /dev/null +++ b/data/2025-12/2025-12-21_12-28-37.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 4522.604, + "avg tokens": 2427.174, + "avg character count": 9946.974, + "parameters count": 1.343757312, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 8.194601472330136, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-29-04.json b/data/2025-12/2025-12-21_12-29-04.json new file mode 100644 index 0000000000000000000000000000000000000000..0aa90cc5eb57d417c250f0c4dec63d1ab0d4cf13 --- /dev/null +++ b/data/2025-12/2025-12-21_12-29-04.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 1801.808, + "avg tokens": 742.244, + "avg character count": 3394.84, + "parameters count": 1.343757312, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 9.565287485744983, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-29-33.json b/data/2025-12/2025-12-21_12-29-33.json new file mode 100644 index 0000000000000000000000000000000000000000..912a12ae9c3556ed8821090815f84e2153a511d6 --- /dev/null +++ b/data/2025-12/2025-12-21_12-29-33.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 1612.233875, + "avg tokens": 1949.844, + "avg character count": 5773.33, + "parameters count": 1.343757312, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 4.967325770464302, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-30-02.json b/data/2025-12/2025-12-21_12-30-02.json new file mode 100644 index 0000000000000000000000000000000000000000..efb2514071351044122c52ac12963d8c60e92d90 --- /dev/null +++ b/data/2025-12/2025-12-21_12-30-02.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 1704.8759375, + "avg tokens": 1835.926, + "avg character count": 5774.754, + "parameters count": 1.343757312, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 5.237122971260852, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-30-32.json b/data/2025-12/2025-12-21_12-30-32.json new file mode 100644 index 0000000000000000000000000000000000000000..92f30ad1453b7225300fcf35caf5a2dc3060979a --- /dev/null +++ b/data/2025-12/2025-12-21_12-30-32.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 2833.00590625, + "avg tokens": 1770.196, + "avg character count": 5024.17, + "parameters count": 1.343757312, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 9.251835923040524, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-31-00.json b/data/2025-12/2025-12-21_12-31-00.json new file mode 100644 index 0000000000000000000000000000000000000000..ae9463edb40230dbc709bd599ece091d23e2715f --- /dev/null +++ b/data/2025-12/2025-12-21_12-31-00.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 2087.38575, + "avg tokens": 2019.3, + "avg character count": 6339.622, + "parameters count": 1.343757312, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 5.793522740428221, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-31-27.json b/data/2025-12/2025-12-21_12-31-27.json new file mode 100644 index 0000000000000000000000000000000000000000..2f56842da4d870b4a7310290d2b264cf1a03d502 --- /dev/null +++ b/data/2025-12/2025-12-21_12-31-27.json @@ -0,0 +1,27 @@ +{ + "neg_log_prob_sum": 1613.5975, + "avg tokens": 720.61, + "avg character count": 3043.39, + "parameters count": 1.343757312, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba2-1.3b", + "tokenizer_name": "EleutherAI/gpt-neox-20b", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "mamba-ssm", + "causal-conv1d>=1.2.0" + ], + "batch_size": 1, + "compression_rate": 9.502396861733095, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-32-12.json b/data/2025-12/2025-12-21_12-32-12.json new file mode 100644 index 0000000000000000000000000000000000000000..6cc60e8121ae10f92bcf1ab88c2ac2fb2fb7ada4 --- /dev/null +++ b/data/2025-12/2025-12-21_12-32-12.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4872.714011230469, + "avg tokens": 1909.12, + "avg character count": 7857.404, + "parameters count": 1.577750528, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 10.967342754488017, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-32-46.json b/data/2025-12/2025-12-21_12-32-46.json new file mode 100644 index 0000000000000000000000000000000000000000..7f47f5b26d926b9d789e49bf25bbac1f625e24a4 --- /dev/null +++ b/data/2025-12/2025-12-21_12-32-46.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4792.907945800781, + "avg tokens": 2095.926, + "avg character count": 9964.74, + "parameters count": 1.577750528, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 8.648459031198238, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-33-33.json b/data/2025-12/2025-12-21_12-33-33.json new file mode 100644 index 0000000000000000000000000000000000000000..4ae3acf0ad0076213ed75bad12ea35cf4a52a0c4 --- /dev/null +++ b/data/2025-12/2025-12-21_12-33-33.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4466.362116149902, + "avg tokens": 2925.354, + "avg character count": 9913.284, + "parameters count": 1.577750528, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 8.120539191760662, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-34-12.json b/data/2025-12/2025-12-21_12-34-12.json new file mode 100644 index 0000000000000000000000000000000000000000..1b92ef377f67b6f63f84202875aaaa2f4e9d828a --- /dev/null +++ b/data/2025-12/2025-12-21_12-34-12.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 4748.486298095703, + "avg tokens": 2448.906, + "avg character count": 9946.974, + "parameters count": 1.577750528, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 8.603882367263312, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-34-27.json b/data/2025-12/2025-12-21_12-34-27.json new file mode 100644 index 0000000000000000000000000000000000000000..efcadb0654941a0c2d6ed59427dc10d9a2fbf243 --- /dev/null +++ b/data/2025-12/2025-12-21_12-34-27.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1796.0758248901368, + "avg tokens": 739.35, + "avg character count": 3394.84, + "parameters count": 1.577750528, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 9.53485699434719, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-34-56.json b/data/2025-12/2025-12-21_12-34-56.json new file mode 100644 index 0000000000000000000000000000000000000000..99b6f1fdcf274f2a620e6b0d171cdc61170a254c --- /dev/null +++ b/data/2025-12/2025-12-21_12-34-56.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1687.2764771118163, + "avg tokens": 1773.934, + "avg character count": 5773.33, + "parameters count": 1.577750528, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 5.198533572963009, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-35-23.json b/data/2025-12/2025-12-21_12-35-23.json new file mode 100644 index 0000000000000000000000000000000000000000..73873af8ee0ea098802591bbcaa97384d12df706 --- /dev/null +++ b/data/2025-12/2025-12-21_12-35-23.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1740.2128545532228, + "avg tokens": 1654.562, + "avg character count": 5774.754, + "parameters count": 1.577750528, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 5.345672676234896, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-35-50.json b/data/2025-12/2025-12-21_12-35-50.json new file mode 100644 index 0000000000000000000000000000000000000000..d4a2f29735d2b4b864675c00a80dac7b6f83f0dc --- /dev/null +++ b/data/2025-12/2025-12-21_12-35-50.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2758.5850687446596, + "avg tokens": 1598.294, + "avg character count": 5024.17, + "parameters count": 1.577750528, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 9.008797468254503, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-36-19.json b/data/2025-12/2025-12-21_12-36-19.json new file mode 100644 index 0000000000000000000000000000000000000000..523e9ee3ae3a698a3abf867d746186c39d4fef8b --- /dev/null +++ b/data/2025-12/2025-12-21_12-36-19.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 2055.73394821167, + "avg tokens": 1791.012, + "avg character count": 6339.622, + "parameters count": 1.577750528, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 5.705673413375846, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-36-33.json b/data/2025-12/2025-12-21_12-36-33.json new file mode 100644 index 0000000000000000000000000000000000000000..456308df3850829701047168e6e079a95e5726cf --- /dev/null +++ b/data/2025-12/2025-12-21_12-36-33.json @@ -0,0 +1,26 @@ +{ + "neg_log_prob_sum": 1613.6375427856444, + "avg tokens": 718.362, + "avg character count": 3043.39, + "parameters count": 1.577750528, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/RWKV-5-World-1B5-v2-20231025-ctx4096", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [ + "rwkv" + ], + "batch_size": 1, + "compression_rate": 9.502632671741877, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-37-21.json b/data/2025-12/2025-12-21_12-37-21.json new file mode 100644 index 0000000000000000000000000000000000000000..ad96dcb43ccf14aea857a76f9d297b969123bc79 --- /dev/null +++ b/data/2025-12/2025-12-21_12-37-21.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 5081.348833374023, + "avg tokens": 1940.684, + "avg character count": 7857.404, + "parameters count": 1.372178432, + "avg bytes": 8012.242, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-ao3_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 11.436931078304335, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-38-00.json b/data/2025-12/2025-12-21_12-38-00.json new file mode 100644 index 0000000000000000000000000000000000000000..c9e68d7054541c9b282e5336dd0072df915aaa13 --- /dev/null +++ b/data/2025-12/2025-12-21_12-38-00.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4878.203364257813, + "avg tokens": 2120.99, + "avg character count": 9964.74, + "parameters count": 1.372178432, + "avg bytes": 9994.128, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_cs", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.802368503363429, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-38-50.json b/data/2025-12/2025-12-21_12-38-50.json new file mode 100644 index 0000000000000000000000000000000000000000..eab5b27aaf8f60b42862287e8811d0cbe989e5d3 --- /dev/null +++ b/data/2025-12/2025-12-21_12-38-50.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4331.801428955078, + "avg tokens": 2919.23, + "avg character count": 9913.284, + "parameters count": 1.372178432, + "avg bytes": 9918.674, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_math", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 7.875886988105991, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-39-34.json b/data/2025-12/2025-12-21_12-39-34.json new file mode 100644 index 0000000000000000000000000000000000000000..5086f6cb913ff4a1409ec9a9843d8c9599cdcd4f --- /dev/null +++ b/data/2025-12/2025-12-21_12-39-34.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 4622.299731201172, + "avg tokens": 2427.174, + "avg character count": 9946.974, + "parameters count": 1.372178432, + "avg bytes": 9952.8, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-arxiv_physics", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 8.375242268138514, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-39-54.json b/data/2025-12/2025-12-21_12-39-54.json new file mode 100644 index 0000000000000000000000000000000000000000..caca8bcc486b3465b7cea93200aa729519cef2cc --- /dev/null +++ b/data/2025-12/2025-12-21_12-39-54.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1820.2870346069335, + "avg tokens": 742.244, + "avg character count": 3394.84, + "parameters count": 1.372178432, + "avg bytes": 3396.996, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-bbc_news", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.663387437834412, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-40-31.json b/data/2025-12/2025-12-21_12-40-31.json new file mode 100644 index 0000000000000000000000000000000000000000..2076c6ca2140f68e08b6d3e9a366661cf5dd581f --- /dev/null +++ b/data/2025-12/2025-12-21_12-40-31.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1660.0797366027832, + "avg tokens": 1949.844, + "avg character count": 5773.33, + "parameters count": 1.372178432, + "avg bytes": 5853.154, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_cpp", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 5.114739855377741, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-41-06.json b/data/2025-12/2025-12-21_12-41-06.json new file mode 100644 index 0000000000000000000000000000000000000000..e3dfd693e04a71f2c8de10caf3ece9120c6f46f3 --- /dev/null +++ b/data/2025-12/2025-12-21_12-41-06.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1753.4741184387208, + "avg tokens": 1835.926, + "avg character count": 5774.754, + "parameters count": 1.372178432, + "avg bytes": 5870.628, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_javascript", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 5.38640928832207, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-41-40.json b/data/2025-12/2025-12-21_12-41-40.json new file mode 100644 index 0000000000000000000000000000000000000000..106ea24ed22d71206854e477c7cffac72b50faa7 --- /dev/null +++ b/data/2025-12/2025-12-21_12-41-40.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 2887.201386779785, + "avg tokens": 1770.196, + "avg character count": 5024.17, + "parameters count": 1.372178432, + "avg bytes": 5522.098, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_markdown", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.428823797483615, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-42-17.json b/data/2025-12/2025-12-21_12-42-17.json new file mode 100644 index 0000000000000000000000000000000000000000..9d6f3cbd78031a9c561ced017e6ecd12a2df0234 --- /dev/null +++ b/data/2025-12/2025-12-21_12-42-17.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 2138.387826965332, + "avg tokens": 2019.3, + "avg character count": 6339.622, + "parameters count": 1.372178432, + "avg bytes": 6497.474, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-github_python", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 5.935078604124101, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2025-12/2025-12-21_12-42-37.json b/data/2025-12/2025-12-21_12-42-37.json new file mode 100644 index 0000000000000000000000000000000000000000..d5f57a0a407b9437f16985e8d24493e6682745a7 --- /dev/null +++ b/data/2025-12/2025-12-21_12-42-37.json @@ -0,0 +1,24 @@ +{ + "neg_log_prob_sum": 1631.9096075439454, + "avg tokens": 720.61, + "avg character count": 3043.39, + "parameters count": 1.372178432, + "avg bytes": 3062.292, + "sample_count": 500, + "model_name_or_path": "state-spaces/mamba-1.4b-hf", + "tokenizer_name": "state-spaces/mamba-1.4b-hf", + "data_path": "Jellyfish042/UncheatableEval-2025-12-wikipedia_english", + "chunk_size": 4000, + "ensure_bos_token": true, + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "compression_rate": 9.610235968609071, + "track_byte_wise_data": false +} \ No newline at end of file diff --git a/data_manager.py b/data_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..bb0a541c639d000c6c1c2e28815065df8a9d701a --- /dev/null +++ b/data_manager.py @@ -0,0 +1,289 @@ +import pandas as pd +import re +from pathlib import Path +from typing import List, Tuple, Optional +import json +import math + + +class DataManager: + def __init__(self, data_dir: str): + self.data_dir = Path(data_dir) + self.master_df = self._load_all_data() + + def _load_old_format_folder(self, period_dir: Path) -> pd.DataFrame: + all_xlsx_data = [] + period = period_dir.name + + for file_path in period_dir.iterdir(): + if file_path.suffix != ".xlsx": + continue + + model_group = file_path.stem + + xls = pd.read_excel(file_path, sheet_name=None) + + for sheet_name, df in xls.items(): + if df.empty: + continue + + df = self._clean_dataframe(df) + + df["Period"] = period + df["Metric"] = sheet_name + df["Model Group"] = model_group + + all_xlsx_data.append(df) + + return all_xlsx_data + + def _load_new_format_folder(self, period_dir: Path) -> pd.DataFrame: + """ + 读取新格式 JSON 数据并转换为宽表。 + """ + raw_records = [] + period = period_dir.name + + for file_path in period_dir.rglob("*.json"): + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + + full_path = data["model_name_or_path"] + model_name = full_path.split("/")[-1].replace(".pth", "") + + params = data["parameters count"] + + data_path = data["data_path"] + source_col = data_path.split("-")[-1] + + neg_log_prob = data["neg_log_prob_sum"] + avg_char = data["avg character count"] + avg_bytes = data["avg bytes"] + + metrics = {} + + metrics["cr"] = data["compression_rate"] + metrics["bpc"] = (neg_log_prob / avg_char) * (1 / math.log(2)) + metrics["bpb"] = (neg_log_prob / avg_bytes) * (1 / math.log(2)) + + for metric_type, value in metrics.items(): + if value is not None: + raw_records.append( + { + "Name": model_name, + "Params (B)": params, + "Period": period, + "Metric": metric_type, + "Model Group": "other", + "Source": source_col, + "Value": value, + } + ) + + if not raw_records: + return [] + + df_long = pd.DataFrame(raw_records) + df_wide = df_long.pivot_table( + index=["Name", "Params (B)", "Period", "Metric", "Model Group"], + columns="Source", + values="Value", + ).reset_index() + + df_wide.columns.name = None + + def assign_group(p): + if p >= 13: + return "14b" + if p >= 8: + return "9b" + if p >= 6: + return "7b" + if p >= 2.5: + return "3b" + if p >= 1: + return "1b5" + return "other" + + df_wide["Model Group"] = df_wide["Params (B)"].apply(assign_group) + + metadata_cols = ["Name", "Params (B)", "Period", "Metric", "Model Group"] + new_columns = {} + for col in df_wide.columns: + if col not in metadata_cols: + new_columns[col] = col.replace("_", " ") + + df_wide = df_wide.rename(columns=new_columns) + + return [df_wide] + + def _load_all_data(self) -> pd.DataFrame: + all_records = [] + + if not self.data_dir.exists(): + print(f"Warning: Directory {self.data_dir} does not exist.") + return pd.DataFrame() + + period_dirs = [d for d in self.data_dir.iterdir() if d.is_dir() and re.match(r"^\d{4}-\d{2}$", d.name)] + + for period_dir in period_dirs: + if period_dir.name <= "2025-11": + all_records.extend(self._load_old_format_folder(period_dir)) + else: + all_records.extend(self._load_new_format_folder(period_dir)) + + if not all_records: + return pd.DataFrame() + + final_df = pd.concat(all_records, ignore_index=True) + + exclude_cols = ["Name", "Period", "Metric", "Model Group"] + numeric_cols = [c for c in final_df.columns if c not in exclude_cols] + for col in numeric_cols: + final_df[col] = pd.to_numeric(final_df[col], errors="coerce") + + return final_df + + def _clean_dataframe(self, df: pd.DataFrame) -> pd.DataFrame: + df = df.dropna(axis=1, how="all") + new_columns = [] + for col in df.columns: + col_str = str(col) + if "Parameters" in col_str: + new_columns.append("Params (B)") + elif col_str == "Average (The lower the better)": + new_columns.append("Average (lower=better)") + else: + new_columns.append(col_str.rsplit("_", maxsplit=1)[0].replace("\u200b", "")) # 去除_202xxxxx后缀 + df.columns = new_columns + column_mapping = {col: col.replace("_", " ") for col in df.columns} + df = df.rename(columns=column_mapping) + return df + + def get_available_periods(self) -> List[str]: + """返回所有可用的时间周期，已排序，从旧到新""" + if self.master_df.empty: + return [] + return sorted(self.master_df["Period"].unique().tolist()) + + def get_available_columns(self, period: str) -> List[str]: + """获取特定时间段内的数据列（排除元数据列和全部为NaN的列）""" + if self.master_df.empty: + return [] + subset = self.master_df[self.master_df["Period"] == period] + if subset.empty: + return [] + + metadata_cols = ["Name", "Params (B)", "Period", "Metric", "Model Group", "Average (lower=better)"] + return [c for c in subset.columns if c not in metadata_cols and not subset[c].isna().all()] + + def query( + self, + period: str, + metric_code: str, + param_range: Tuple[float, float], + model_groups: Optional[List[str]] = None, + visible_columns: Optional[List[str]] = None, + ) -> pd.DataFrame: + """ + 统一查询接口。 + + Args: + period: 时间周期 (e.g. "2025-12") + metric_display_name: bpc, bpb, cr + param_range: 参数量范围元组 (min, max) + model_groups: (可选) 文件名列表，如 ['14b', '7b'] + visible_columns: (可选) 需要参与计算平均值的列名列表 + """ + + mask = ( + (self.master_df["Period"] == period) + & (self.master_df["Metric"] == metric_code) + & (self.master_df["Params (B)"].between(param_range[0], param_range[1])) + ) + + if model_groups: + mask = mask & (self.master_df["Model Group"].isin(model_groups)) + + filtered_df = self.master_df.loc[mask].copy() + + if filtered_df.empty: + return filtered_df + + # 始终排除的列 + exclude_cols = ["Period", "Metric", "Model Group"] + # 始终保留的列 + metadata_cols = ["Name", "Params (B)", "Average (lower=better)"] + + if visible_columns is not None: + valid_visible_cols = [c for c in visible_columns if c in filtered_df.columns] + columns_to_keep = metadata_cols + valid_visible_cols + cols_for_average = valid_visible_cols + else: + all_cols = [c for c in filtered_df.columns if c not in exclude_cols] + columns_to_keep = all_cols + cols_for_average = [c for c in all_cols if c not in metadata_cols] + + if not cols_for_average: + return pd.DataFrame() + + filtered_df["Average (lower=better)"] = filtered_df[cols_for_average].mean(axis=1).round(3) + columns_to_keep = [c for c in columns_to_keep if c not in exclude_cols] + columns_to_keep = list(dict.fromkeys(columns_to_keep)) + filtered_df = filtered_df[columns_to_keep] + + if "Name" in filtered_df.columns: + filtered_df["Name"] = filtered_df["Name"].apply(lambda x: x.replace(".pth", "")) + + filtered_df = filtered_df.sort_values(by="Average (lower=better)", ascending=True, kind="mergesort", na_position="last").reset_index( + drop=True + ) + + # 定义列优先级顺序 + fixed_cols = ["Name", "Params (B)", "Average (lower=better)"] + column_priority = [ + "ao3 english", + "bbc news", + "wikipedia english", + "arxiv physics", + "arxiv cs", + "arxiv math", + "github cpp", + "github python", + "github javascript", + "github markdown", + ] + existing_cols = filtered_df.columns.tolist() + ordered_cols = [] + for col in fixed_cols: + if col in existing_cols: + ordered_cols.append(col) + for col in column_priority: + if col in existing_cols and col not in ordered_cols: + ordered_cols.append(col) + for col in existing_cols: + if col not in ordered_cols: + ordered_cols.append(col) + filtered_df = filtered_df[ordered_cols] + + return filtered_df + + +if __name__ == "__main__": + + dm = DataManager("data") + periods = dm.get_available_periods() + + print(f"Total records loaded: {len(dm.master_df)}") + print(f"Available periods: {periods}") + print(f"Available columns: {dm.get_available_columns('2025-11')}") + + result = dm.query( + period="2025-11", + metric_code="cr", + param_range=(0, 20), + model_groups=["7b"], + visible_columns=["wikipedia_english"], + ) + + print(result.head(20)) diff --git a/temp.py b/temp.py new file mode 100644 index 0000000000000000000000000000000000000000..0ce7106b79fba0bc42c347ccedabe4293c149b96 --- /dev/null +++ b/temp.py @@ -0,0 +1,42 @@ +import pandas as pd +from pathlib import Path +import re + + +def find_unnamed_columns(data_dir: str): + path = Path(data_dir) + found_issues = False + + print(f"🔍 Scanning {data_dir} for 'Unnamed' columns...\n") + + # 遍历所有 Excel 文件 + for file_path in path.rglob("*.xls*"): + # 跳过临时文件 + if file_path.name.startswith("~$"): + continue + + try: + xls = pd.read_excel(file_path, sheet_name=None) + for sheet_name, df in xls.items(): + # 检查是否有 Unnamed 列 + unnamed_cols = [c for c in df.columns if "Unnamed" in str(c)] + + if unnamed_cols: + found_issues = True + print(f"⚠️ File: {file_path} | Sheet: {sheet_name}") + print(f" Found columns: {unnamed_cols}") + + # 打印前 3 行内容，帮助判断是不是垃圾数据 + print(f" Sample content:") + print(df[unnamed_cols].head(3).to_string(index=False)) + print("-" * 40) + + except Exception as e: + print(f"❌ Error reading {file_path.name}: {e}") + + if not found_issues: + print("✅ No 'Unnamed' columns found!") + + +if __name__ == "__main__": + find_unnamed_columns("data") diff --git a/title.py b/title.py new file mode 100644 index 0000000000000000000000000000000000000000..5b898e945d57a149b8e84fd9238f801870a1cb97 --- /dev/null +++ b/title.py @@ -0,0 +1,20 @@ +css = """ +.gradio-container { + max-width: 95% !important; + margin: 0 auto; +} +.tab-buttons button { + font-size: 1.3em; +} +.gr-dataframe th { + white-space: normal; + word-break: break-word; +} +table { + margin-left: auto !important; + margin-right: auto !important; + width: 100% !important; +} +""" +TITLE_HTML = '

🏆 LLM Compression Leaderboard

' +SUBTITLE_HTML = "

Welcome to Uncheatable Eval LLM Compression Leaderboard, where fancy fine-tuning and cheating won't work 🚫; only compute 💻, data 📊, and real innovation 🔥 can prevail!

"