diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_12-56-21.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_12-56-21.json new file mode 100644 index 0000000000000000000000000000000000000000..6e7574782ad996529642a1f23a07e53b94cfbdc0 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_12-56-21.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 6625.368, + "avg tokens": 2441.756, + "avg character count": 9041.522, + "parameters count": 0.521411104, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.0571655477545074, + "bpb": 1.0374143915485972, + "compression_rate": 12.967679894357465, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_12-56-55.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_12-56-55.json new file mode 100644 index 0000000000000000000000000000000000000000..e16204b5664e6e161ebd8b2ba0580a55fb875cd7 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_12-56-55.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 7220.192, + "avg tokens": 2549.552, + "avg character count": 2942.368, + "parameters count": 0.521411104, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 3.5401877646392856, + "bpb": 1.9857768023518056, + "compression_rate": 24.82221002939757, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_12-57-31.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_12-57-31.json new file mode 100644 index 0000000000000000000000000000000000000000..63c3d2d8370be5313bbb9bf3647e3c9492400deb --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_12-57-31.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 5811.552, + "avg tokens": 2794.574, + "avg character count": 11865.894, + "parameters count": 0.521411104, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7065879107185971, + "bpb": 0.706090917290103, + "compression_rate": 8.826136466126288, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_12-58-06.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_12-58-06.json new file mode 100644 index 0000000000000000000000000000000000000000..97692d791819aef386eafb8d2b004d93830d530d --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_12-58-06.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 3753.104, + "avg tokens": 2562.956, + "avg character count": 8435.788, + "parameters count": 0.521411104, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6418587722617651, + "bpb": 0.6415354108203909, + "compression_rate": 8.019192635254885, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_12-58-41.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_12-58-41.json new file mode 100644 index 0000000000000000000000000000000000000000..1e3298b0883612b7ec48cb5d90f36e9b0016725a --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_12-58-41.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 4919.824, + "avg tokens": 2680.144, + "avg character count": 10817.836, + "parameters count": 0.521411104, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6561206591453691, + "bpb": 0.6556647537057437, + "compression_rate": 8.195809421321796, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_12-59-16.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_12-59-16.json new file mode 100644 index 0000000000000000000000000000000000000000..1dda91aa1f3f5bee3785e62486b7902db93a208a --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_12-59-16.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 5367.464, + "avg tokens": 3012.596, + "avg character count": 11165.434, + "parameters count": 0.521411104, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6935345007592216, + "bpb": 0.6931919226714106, + "compression_rate": 8.664899033392633, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_12-59-50.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_12-59-50.json new file mode 100644 index 0000000000000000000000000000000000000000..9ae85cd1bc5fe48001ba7e14a8e9830bc79d50c8 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_12-59-50.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1583.032, + "avg tokens": 677.694, + "avg character count": 2745.97, + "parameters count": 0.521411104, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8317033383352832, + "bpb": 0.8310544671613104, + "compression_rate": 10.38818083951638, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_13-00-25.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_13-00-25.json new file mode 100644 index 0000000000000000000000000000000000000000..6e61d058bc3d4ef603a75b21e5f8df2f1313a880 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_13-00-25.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 5648.404, + "avg tokens": 2960.446, + "avg character count": 11560.664, + "parameters count": 0.521411104, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7048837713592735, + "bpb": 0.704549677977209, + "compression_rate": 8.806870974715112, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_13-00-59.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_13-00-59.json new file mode 100644 index 0000000000000000000000000000000000000000..36abcab337a73b2ee95f1c2f01f32ab26930816d --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_13-00-59.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1561.695, + "avg tokens": 1786.494, + "avg character count": 4752.824, + "parameters count": 0.521411104, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4740444064162885, + "bpb": 0.46505835948881646, + "compression_rate": 5.813229493610206, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_13-01-33.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_13-01-33.json new file mode 100644 index 0000000000000000000000000000000000000000..39e361a398526bb1c317bee5dd5de697882854a5 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_13-01-33.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1845.11, + "avg tokens": 1906.502, + "avg character count": 5202.974, + "parameters count": 0.521411104, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5116172110209728, + "bpb": 0.4959198353128354, + "compression_rate": 6.198997941410442, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_13-02-06.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_13-02-06.json new file mode 100644 index 0000000000000000000000000000000000000000..6ebc5c853f6d17eb338291d8a00c1c7fa3da2b44 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_13-02-06.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 3319.7385, + "avg tokens": 1865.53, + "avg character count": 4927.866, + "parameters count": 0.521411104, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9718953946795967, + "bpb": 0.894346811486149, + "compression_rate": 11.179335143576862, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_other-2026-04-18_13-02-40.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_other-2026-04-18_13-02-40.json new file mode 100644 index 0000000000000000000000000000000000000000..91dc7859eb11592975fbbdf37f67b74b864af489 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_other-2026-04-18_13-02-40.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1861.302, + "avg tokens": 1923.26, + "avg character count": 5229.94, + "parameters count": 0.521411104, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5134458836997574, + "bpb": 0.5043396993440932, + "compression_rate": 6.304246241801165, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_python-2026-04-18_13-03-14.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_python-2026-04-18_13-03-14.json new file mode 100644 index 0000000000000000000000000000000000000000..87a6b7ac0913aec9b74e005769a566e355c8a6af --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-github_python-2026-04-18_13-03-14.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1845.834, + "avg tokens": 1785.66, + "avg character count": 5274.818, + "parameters count": 0.521411104, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5048469081026566, + "bpb": 0.49021738719360003, + "compression_rate": 6.12771733992, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_13-03-47.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_13-03-47.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3a173e714ea22444d702d1c84ba26e920322d5 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_13-03-47.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1844.224, + "avg tokens": 873.27, + "avg character count": 3127.092, + "parameters count": 0.521411104, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.850839316236429, + "bpb": 0.8463723770765732, + "compression_rate": 10.579654713457165, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_13-04-19.json b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_13-04-19.json new file mode 100644 index 0000000000000000000000000000000000000000..0390579ec6ccd8986d67e1749cae0893913355e2 --- /dev/null +++ b/data/2026-04/Falcon-H1-0.5B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_13-04-19.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 3608.968, + "avg tokens": 1879.4, + "avg character count": 2498.564, + "parameters count": 0.521411104, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-0.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-0.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.083853059728292, + "bpb": 1.483552496463094, + "compression_rate": 18.544406205788675, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_14-04-00.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_14-04-00.json new file mode 100644 index 0000000000000000000000000000000000000000..8d2b58e47ce7d814f23dfba19cb46fab7762fb15 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_14-04-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5843.832, + "avg tokens": 2377.902, + "avg character count": 9041.522, + "parameters count": 1.554859392, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9324610885410921, + "bpb": 0.9150398013502375, + "compression_rate": 11.437997516877969, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-04-27.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-04-27.json new file mode 100644 index 0000000000000000000000000000000000000000..9d324d047d5e21c9b17d6b9f6dfbf5f44c418d19 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-04-27.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4650.168, + "avg tokens": 1555.08, + "avg character count": 2942.368, + "parameters count": 1.554859392, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.2800595686537335, + "bpb": 1.2789404688183765, + "compression_rate": 15.986755860229707, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-04-58.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-04-58.json new file mode 100644 index 0000000000000000000000000000000000000000..d5280305e4d375bf53cc270c59613955f85a9c9b --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-04-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5254.16, + "avg tokens": 2819.35, + "avg character count": 11865.894, + "parameters count": 1.554859392, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6388183289044345, + "bpb": 0.638369002632854, + "compression_rate": 7.979612532910675, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-05-31.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-05-31.json new file mode 100644 index 0000000000000000000000000000000000000000..97c686d36be5e6c4047108a8a87fe06ccb1d9dfc --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-05-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3345.872, + "avg tokens": 2586.686, + "avg character count": 8435.788, + "parameters count": 1.554859392, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5722136381152818, + "bpb": 0.5719253631320749, + "compression_rate": 7.149067039150935, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-06-00.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-06-00.json new file mode 100644 index 0000000000000000000000000000000000000000..ca8af446b37006b1066bb54edff99730572be038 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-06-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4425.232, + "avg tokens": 2704.822, + "avg character count": 10817.836, + "parameters count": 1.554859392, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5901605701161627, + "bpb": 0.589750497044361, + "compression_rate": 7.371881213054513, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-06-30.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-06-30.json new file mode 100644 index 0000000000000000000000000000000000000000..e58dc58eb2dfe52fbb940ecea3538c23efac8536 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-06-30.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4804.504, + "avg tokens": 3040.754, + "avg character count": 11165.434, + "parameters count": 1.554859392, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6207939695609851, + "bpb": 0.6204873223635003, + "compression_rate": 7.756091529543754, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_14-06-58.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_14-06-58.json new file mode 100644 index 0000000000000000000000000000000000000000..2c9e59ec887ac16542b1efcc9e429db55c0361a6 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_14-06-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1451.58, + "avg tokens": 670.55, + "avg character count": 2745.97, + "parameters count": 1.554859392, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7626402573420691, + "bpb": 0.7620452672100215, + "compression_rate": 9.52556584012527, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-07-27.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-07-27.json new file mode 100644 index 0000000000000000000000000000000000000000..4d57c48b7ffa5d428eb2effb75e69b6207fdcd8c --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-07-27.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5061.156, + "avg tokens": 2976.364, + "avg character count": 11560.664, + "parameters count": 1.554859392, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6315990727146314, + "bpb": 0.6312997140417752, + "compression_rate": 7.89124642552219, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_14-07-55.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_14-07-55.json new file mode 100644 index 0000000000000000000000000000000000000000..75ebbd926e127ff3f57dc29e8fcbe7e546f00db1 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_14-07-55.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1315.011, + "avg tokens": 1788.226, + "avg character count": 4752.824, + "parameters count": 1.554859392, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3991647593966107, + "bpb": 0.39159814071873705, + "compression_rate": 4.894976758984213, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_14-08-24.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_14-08-24.json new file mode 100644 index 0000000000000000000000000000000000000000..a55bb8f243b9bdc5c9d890a79a2d74f10e34f44d --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_14-08-24.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1528.351, + "avg tokens": 1893.87, + "avg character count": 5202.974, + "parameters count": 1.554859392, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4237853982045054, + "bpb": 0.4107828672654787, + "compression_rate": 5.134785840818483, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_14-08-52.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_14-08-52.json new file mode 100644 index 0000000000000000000000000000000000000000..89b641f5ee95e96826b6766a8e9e65c1d08cb2dc --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_14-08-52.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2836.6095, + "avg tokens": 1749.97, + "avg character count": 4927.866, + "parameters count": 1.554859392, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8304532750258473, + "bpb": 0.7641905113177798, + "compression_rate": 9.552381391472247, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_other-2026-04-18_14-09-23.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_other-2026-04-18_14-09-23.json new file mode 100644 index 0000000000000000000000000000000000000000..ad02097b9c4fcf8e6c065ed376810e89dfc486b3 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_other-2026-04-18_14-09-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1545.968, + "avg tokens": 1934.992, + "avg character count": 5229.94, + "parameters count": 1.554859392, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.42646002955541157, + "bpb": 0.41889657686694, + "compression_rate": 5.23620721083675, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_python-2026-04-18_14-09-50.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_python-2026-04-18_14-09-50.json new file mode 100644 index 0000000000000000000000000000000000000000..7b0395b5b4d13522402b48377048e776e5458497 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-github_python-2026-04-18_14-09-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1562.37, + "avg tokens": 1773.106, + "avg character count": 5274.818, + "parameters count": 1.554859392, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4273177673682181, + "bpb": 0.41493489622017193, + "compression_rate": 5.186686202752149, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-10-22.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-10-22.json new file mode 100644 index 0000000000000000000000000000000000000000..eb9354eb3bd579ea8025e8cf315971e8bc021f7d --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-10-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1636.864, + "avg tokens": 852.764, + "avg character count": 3127.092, + "parameters count": 1.554859392, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7551730410904675, + "bpb": 0.7512083535574139, + "compression_rate": 9.390104419467674, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-10-53.json b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-10-53.json new file mode 100644 index 0000000000000000000000000000000000000000..73f571a143de1032c567761c3e796f68f7336160 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-10-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2209.256, + "avg tokens": 1044.362, + "avg character count": 2498.564, + "parameters count": 1.554859392, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.275645801049798, + "bpb": 0.9081674467953358, + "compression_rate": 11.352093084941698, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_13-17-26.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_13-17-26.json new file mode 100644 index 0000000000000000000000000000000000000000..48400a0824f57d668a950f87b8ea33b0dc0ba0b5 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_13-17-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5761.232, + "avg tokens": 2377.902, + "avg character count": 9041.522, + "parameters count": 1.554872208, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9192811603854752, + "bpb": 0.902106115441483, + "compression_rate": 11.276326443018538, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_13-18-40.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_13-18-40.json new file mode 100644 index 0000000000000000000000000000000000000000..b65fe8d6bec33a79d3463d5e2b309ec718529fd4 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_13-18-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4590.432, + "avg tokens": 1555.08, + "avg character count": 2942.368, + "parameters count": 1.554872208, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.2507699519359936, + "bpb": 1.262511215542939, + "compression_rate": 15.781390194286738, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_13-20-09.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_13-20-09.json new file mode 100644 index 0000000000000000000000000000000000000000..85cbf2886253dfde2f9474bdabf93208a042ad21 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_13-20-09.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5155.776, + "avg tokens": 2819.35, + "avg character count": 11865.894, + "parameters count": 1.554872208, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6268564734468668, + "bpb": 0.626415560797236, + "compression_rate": 7.830194509965451, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_13-21-49.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_13-21-49.json new file mode 100644 index 0000000000000000000000000000000000000000..e039e64dd7d09d25dd7b248ffccf73b644456949 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_13-21-49.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3268.288, + "avg tokens": 2586.686, + "avg character count": 8435.788, + "parameters count": 1.554872208, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5589451619453818, + "bpb": 0.5586635714755982, + "compression_rate": 6.983294643444977, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_13-23-19.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_13-23-19.json new file mode 100644 index 0000000000000000000000000000000000000000..6f1abe9b3e49e299f4d02c90a911032e11c29f61 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_13-23-19.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4340.768, + "avg tokens": 2704.822, + "avg character count": 10817.836, + "parameters count": 1.554872208, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5788962290840335, + "bpb": 0.5784939830395912, + "compression_rate": 7.23117478799489, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_13-24-59.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_13-24-59.json new file mode 100644 index 0000000000000000000000000000000000000000..d43dc806cc304b8b4c963f2748044c28bd371e76 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_13-24-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4710.544, + "avg tokens": 3040.754, + "avg character count": 11165.434, + "parameters count": 1.554872208, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6086533195833912, + "bpb": 0.6083526693776198, + "compression_rate": 7.604408367220247, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_13-26-37.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_13-26-37.json new file mode 100644 index 0000000000000000000000000000000000000000..30e8066df99a59f94262b2c2b83f00f664af4c23 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_13-26-37.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1432.772, + "avg tokens": 670.55, + "avg character count": 2745.97, + "parameters count": 1.554872208, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7527587916563407, + "bpb": 0.7521715107614029, + "compression_rate": 9.402143884517535, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_13-28-03.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_13-28-03.json new file mode 100644 index 0000000000000000000000000000000000000000..e29cd81634de5e1326f3843eae4a03c235c65d20 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_13-28-03.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4972.244, + "avg tokens": 2976.364, + "avg character count": 11560.664, + "parameters count": 1.554872208, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6205034382877923, + "bpb": 0.6202093386068188, + "compression_rate": 7.752616732585235, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_13-29-36.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_13-29-36.json new file mode 100644 index 0000000000000000000000000000000000000000..1f20aa388b2698e6d2e84fef33a6fad9d245b3c0 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_13-29-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1272.189, + "avg tokens": 1788.226, + "avg character count": 4752.824, + "parameters count": 1.554872208, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.38616636369734914, + "bpb": 0.37884614428535535, + "compression_rate": 4.735576803566942, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_13-31-12.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_13-31-12.json new file mode 100644 index 0000000000000000000000000000000000000000..c907c63ea494b65858944d35a875440676eec67a --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_13-31-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1474.695, + "avg tokens": 1893.87, + "avg character count": 5202.974, + "parameters count": 1.554872208, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4089075139187222, + "bpb": 0.3963614643770083, + "compression_rate": 4.954518304712604, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_13-32-22.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_13-32-22.json new file mode 100644 index 0000000000000000000000000000000000000000..ddb7ae0360f9639f9c59d267bf0c2eace4a3864c --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_13-32-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2763.2135, + "avg tokens": 1749.97, + "avg character count": 4927.866, + "parameters count": 1.554872208, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8089656685809711, + "bpb": 0.7444174241978645, + "compression_rate": 9.305217802473306, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_other-2026-04-18_13-33-38.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_other-2026-04-18_13-33-38.json new file mode 100644 index 0000000000000000000000000000000000000000..c56e3593b3b5390260caf1e634c6aed5e23efa82 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_other-2026-04-18_13-33-38.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1492.682, + "avg tokens": 1934.992, + "avg character count": 5229.94, + "parameters count": 1.554872208, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4117609225008738, + "bpb": 0.40445816482029234, + "compression_rate": 5.055727060253655, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_python-2026-04-18_13-35-03.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_python-2026-04-18_13-35-03.json new file mode 100644 index 0000000000000000000000000000000000000000..a591ca8a28d39cd66cdf6c2282717f8909470fa9 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-github_python-2026-04-18_13-35-03.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1510.662, + "avg tokens": 1773.106, + "avg character count": 5274.818, + "parameters count": 1.554872208, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.41317531256232976, + "bpb": 0.4012022633523156, + "compression_rate": 5.0150282919039455, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_13-36-12.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_13-36-12.json new file mode 100644 index 0000000000000000000000000000000000000000..13489c6ee4d7818f5e3165adcce31a039fb06bdb --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_13-36-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1608.864, + "avg tokens": 852.764, + "avg character count": 3127.092, + "parameters count": 1.554872208, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7422551412829496, + "bpb": 0.7383582732211077, + "compression_rate": 9.229478415263847, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_13-37-21.json b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_13-37-21.json new file mode 100644 index 0000000000000000000000000000000000000000..eea8510cdbd725315f8405b9df2d064dccd1f778 --- /dev/null +++ b/data/2026-04/Falcon-H1-1.5B-Deep-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_13-37-21.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2168.62, + "avg tokens": 1044.362, + "avg character count": 2498.564, + "parameters count": 1.554872208, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "tokenizer_name": "tiiuae/Falcon-H1-1.5B-Deep-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.2521821812739733, + "bpb": 0.891463048406025, + "compression_rate": 11.143288105075312, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-57-07.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-57-07.json new file mode 100644 index 0000000000000000000000000000000000000000..f02e200193e5e1a64f350e86acbac46171f1fa6a --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-57-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5538.752, + "avg tokens": 2377.902, + "avg character count": 9041.522, + "parameters count": 3.149387264, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8837815185445357, + "bpb": 0.8672697178509292, + "compression_rate": 10.840871473136616, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-57-54.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-57-54.json new file mode 100644 index 0000000000000000000000000000000000000000..adb428d60399454ae651efe2c89032b94aac28ce --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-57-54.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4324.176, + "avg tokens": 1555.08, + "avg character count": 2942.368, + "parameters count": 3.149387264, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.1202199286870558, + "bpb": 1.1892825551019173, + "compression_rate": 14.866031938773967, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-58-44.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-58-44.json new file mode 100644 index 0000000000000000000000000000000000000000..056089743e850b50b088f6a0e827d6933b605ddd --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-58-44.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4838.416, + "avg tokens": 2819.35, + "avg character count": 11865.894, + "parameters count": 3.149387264, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5882707842289687, + "bpb": 0.5878570116332283, + "compression_rate": 7.348212645415354, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-59-36.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-59-36.json new file mode 100644 index 0000000000000000000000000000000000000000..5368741591c0d32a051d851c93f9d0c427d1705c --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-59-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3062.376, + "avg tokens": 2586.686, + "avg character count": 8435.788, + "parameters count": 3.149387264, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5237299311620183, + "bpb": 0.5234660817410084, + "compression_rate": 6.543326021762605, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-07-33.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-07-33.json new file mode 100644 index 0000000000000000000000000000000000000000..67d2f0894b40ac9461a89fc6c2a9a1ad6cf37b70 --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-07-33.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4080.776, + "avg tokens": 2704.822, + "avg character count": 10817.836, + "parameters count": 3.149387264, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5442230126412252, + "bpb": 0.5438448592812081, + "compression_rate": 6.798060741015101, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-08-23.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-08-23.json new file mode 100644 index 0000000000000000000000000000000000000000..6d8813c25c2f6d6316b9ab01bee2e1ade938ab7c --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-08-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4341.216, + "avg tokens": 3040.754, + "avg character count": 11165.434, + "parameters count": 3.149387264, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5609321406250597, + "bpb": 0.560655062758109, + "compression_rate": 7.008188284476362, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-09-00.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-09-00.json new file mode 100644 index 0000000000000000000000000000000000000000..aedff4cc84a7b709dfa913a8d3e2e492353028fd --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-09-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1359.304, + "avg tokens": 670.55, + "avg character count": 2745.97, + "parameters count": 3.149387264, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7141597103611954, + "bpb": 0.7136025433662984, + "compression_rate": 8.92003179207873, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-09-46.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-09-46.json new file mode 100644 index 0000000000000000000000000000000000000000..cc4d907f681e7cc30e1a20e229f0e32223cf1878 --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-09-46.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4689.756, + "avg tokens": 2976.364, + "avg character count": 11560.664, + "parameters count": 3.149387264, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5852507887245284, + "bpb": 0.5849733977229115, + "compression_rate": 7.312167471536394, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-10-27.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-10-27.json new file mode 100644 index 0000000000000000000000000000000000000000..04e674883c32152c70519a7e048873e77563f3c7 --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-10-27.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1138.289, + "avg tokens": 1788.226, + "avg character count": 4752.824, + "parameters count": 3.149387264, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.34552171412163746, + "bpb": 0.33897195993082224, + "compression_rate": 4.237149499135278, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-11-05.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-11-05.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3a617f31a76773770c07c3113b592084982bed --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-11-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1318.184, + "avg tokens": 1893.87, + "avg character count": 5202.974, + "parameters count": 3.149387264, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3655097103654904, + "bpb": 0.3542951868409009, + "compression_rate": 4.428689835511261, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-11-43.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-11-43.json new file mode 100644 index 0000000000000000000000000000000000000000..b0999120c68cf6da1946675797466c797f9e1f05 --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-11-43.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2548.363, + "avg tokens": 1749.97, + "avg character count": 4927.866, + "parameters count": 3.149387264, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7460654698169392, + "bpb": 0.6865361002257488, + "compression_rate": 8.58170125282186, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-12-27.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-12-27.json new file mode 100644 index 0000000000000000000000000000000000000000..75ed42a7f6bae9ee5dc8217bd766216fce47aee9 --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-12-27.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1325.29, + "avg tokens": 1934.992, + "avg character count": 5229.94, + "parameters count": 3.149387264, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3655853242560592, + "bpb": 0.3591015107401879, + "compression_rate": 4.488768884252349, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-13-12.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-13-12.json new file mode 100644 index 0000000000000000000000000000000000000000..006c368df56ef8d5cc8cc2c68bcad87df26f4784 --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-13-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1375.67, + "avg tokens": 1773.106, + "avg character count": 5274.818, + "parameters count": 3.149387264, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3762541734899138, + "bpb": 0.36535102996294344, + "compression_rate": 4.566887874536793, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-13-47.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-13-47.json new file mode 100644 index 0000000000000000000000000000000000000000..56cfd8fbf8190ad6efdb5bfb4bf57016fd54d88a --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-13-47.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1516.412, + "avg tokens": 852.764, + "avg character count": 3127.092, + "parameters count": 3.149387264, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6996020815327835, + "bpb": 0.6959291436763868, + "compression_rate": 8.699114295954836, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-14-22.json b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-14-22.json new file mode 100644 index 0000000000000000000000000000000000000000..9b2f6245b99ec0a75c8de2a8785db68abc5f021b --- /dev/null +++ b/data/2026-04/Falcon-H1-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-14-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1968.792, + "avg tokens": 1044.362, + "avg character count": 2498.564, + "parameters count": 3.149387264, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-3B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.136799559643805, + "bpb": 0.8093189761218631, + "compression_rate": 10.116487201523288, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_10-29-14.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_10-29-14.json new file mode 100644 index 0000000000000000000000000000000000000000..b24832bf31b6c47fe12223cfccc8faf06ba46303 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_10-29-14.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 5191.48, + "avg tokens": 2277.202, + "avg character count": 9041.522, + "parameters count": 7.585648736, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8283696540111537, + "bpb": 0.8128931201160011, + "compression_rate": 10.161164001450015, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_10-29-59.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_10-29-59.json new file mode 100644 index 0000000000000000000000000000000000000000..a8d4b749a185a95b37d40ba02ca7a4e3b40a0caa --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_10-29-59.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 3993.272, + "avg tokens": 1420.144, + "avg character count": 2942.368, + "parameters count": 7.585648736, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.957971848293875, + "bpb": 1.0982736889934506, + "compression_rate": 13.728421112418133, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_10-31-00.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_10-31-00.json new file mode 100644 index 0000000000000000000000000000000000000000..5b41e384e90c082730ab14eb6a19091f92aaf1b0 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_10-31-00.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 4536.224, + "avg tokens": 2680.07, + "avg character count": 11865.894, + "parameters count": 7.585648736, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.551529271133005, + "bpb": 0.551141341451196, + "compression_rate": 6.88926676813995, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_10-31-58.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_10-31-58.json new file mode 100644 index 0000000000000000000000000000000000000000..008016570632c23ac4f55a0c3258e06e53320476 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_10-31-58.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 2860.384, + "avg tokens": 2507.906, + "avg character count": 8435.788, + "parameters count": 7.585648736, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.48918510183496033, + "bpb": 0.48893865572178996, + "compression_rate": 6.1117331965223745, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_10-32-58.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_10-32-58.json new file mode 100644 index 0000000000000000000000000000000000000000..16a8758abb3178e34cbe18bffb33221bcf937764 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_10-32-58.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 3819.072, + "avg tokens": 2585.504, + "avg character count": 10817.836, + "parameters count": 7.585648736, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5093214793788606, + "bpb": 0.5089675773492105, + "compression_rate": 6.362094716865132, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_10-34-03.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_10-34-03.json new file mode 100644 index 0000000000000000000000000000000000000000..befbe1f7ad5eb07833f43e470eb52af80886dd3c --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_10-34-03.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 4043.08, + "avg tokens": 2908.606, + "avg character count": 11165.434, + "parameters count": 7.585648736, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5224097393721865, + "bpb": 0.5221516900186618, + "compression_rate": 6.5268961252332724, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_10-34-46.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_10-34-46.json new file mode 100644 index 0000000000000000000000000000000000000000..97b4edf6e6bf6dcf7b6b6b13a02178bfb704aa95 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_10-34-46.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1276.464, + "avg tokens": 640.794, + "avg character count": 2745.97, + "parameters count": 7.585648736, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.67063670858505, + "bpb": 0.6701134969922243, + "compression_rate": 8.376418712402804, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_10-35-50.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_10-35-50.json new file mode 100644 index 0000000000000000000000000000000000000000..70e80803031d059b5530c6ffa4a2969e10523014 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_10-35-50.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 4341.76, + "avg tokens": 2815.974, + "avg character count": 11560.664, + "parameters count": 7.585648736, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5418231704277596, + "bpb": 0.5415663627910339, + "compression_rate": 6.7695795348879235, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_10-36-40.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_10-36-40.json new file mode 100644 index 0000000000000000000000000000000000000000..f3d10baf302d978f0aa4c30e693c67f7c712f950 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_10-36-40.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 998.398, + "avg tokens": 1739.02, + "avg character count": 4752.824, + "parameters count": 7.585648736, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.30305852761083923, + "bpb": 0.29731371106196497, + "compression_rate": 3.7164213882745623, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_10-37-31.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_10-37-31.json new file mode 100644 index 0000000000000000000000000000000000000000..b34f63883452f10ae83ccf2172f1900e197f0cfe --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_10-37-31.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1165.105, + "avg tokens": 1839.594, + "avg character count": 5202.974, + "parameters count": 7.585648736, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3230635412775339, + "bpb": 0.3131513458396307, + "compression_rate": 3.914391822995384, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_10-38-24.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_10-38-24.json new file mode 100644 index 0000000000000000000000000000000000000000..b05fb7a22bb7ab356976bbca33f047e4c50cb77b --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_10-38-24.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 2328.6785, + "avg tokens": 1677.064, + "avg character count": 4927.866, + "parameters count": 7.585648736, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6817500564696259, + "bpb": 0.6273524831703908, + "compression_rate": 7.841906039629885, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_other-2026-04-19_10-39-15.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_other-2026-04-19_10-39-15.json new file mode 100644 index 0000000000000000000000000000000000000000..bc656a9072cefe8e9f5d6e362b1628607d5d9387 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_other-2026-04-19_10-39-15.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1167.157, + "avg tokens": 1878.942, + "avg character count": 5229.94, + "parameters count": 7.585648736, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.321963849650061, + "bpb": 0.3162536818137807, + "compression_rate": 3.953171022672259, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_python-2026-04-19_10-40-05.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_python-2026-04-19_10-40-05.json new file mode 100644 index 0000000000000000000000000000000000000000..3ddc6a547c19fbed8a16c708e766a05937ba40fc --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-github_python-2026-04-19_10-40-05.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1235.476, + "avg tokens": 1718.548, + "avg character count": 5274.818, + "parameters count": 7.585648736, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3379102555457521, + "bpb": 0.32811824717737365, + "compression_rate": 4.101478089717171, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_10-40-48.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_10-40-48.json new file mode 100644 index 0000000000000000000000000000000000000000..8bcff5d6273390d3b71f0a201398be9a0b14b359 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_10-40-48.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1402.14, + "avg tokens": 802.672, + "avg character count": 3127.092, + "parameters count": 7.585648736, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6468822870040444, + "bpb": 0.6434861300981587, + "compression_rate": 8.043576626226983, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_10-41-30.json b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_10-41-30.json new file mode 100644 index 0000000000000000000000000000000000000000..c499f9d6ad9c152c06fd83125e179014da3e81c9 --- /dev/null +++ b/data/2026-04/Falcon-H1-7B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_10-41-30.json @@ -0,0 +1,29 @@ +{ + "neg_log_prob_sum": 1747.552, + "avg tokens": 953.084, + "avg character count": 2498.564, + "parameters count": 7.585648736, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "tiiuae/Falcon-H1-7B-Base", + "tokenizer_name": "tiiuae/Falcon-H1-7B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.0090534419352837, + "bpb": 0.7183729898128974, + "compression_rate": 8.979662372661219, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-ao3_english-2026-04-18_14-16-42.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-ao3_english-2026-04-18_14-16-42.json new file mode 100644 index 0000000000000000000000000000000000000000..cc3336e775d0e373508bb8441ca45faf2afd65d0 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-ao3_english-2026-04-18_14-16-42.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 5651.224, + "avg tokens": 2454.446, + "avg character count": 9041.522, + "parameters count": 2.172819456, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9017279214442758, + "bpb": 0.8848808258597602, + "compression_rate": 11.061010323247004, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-18-10.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-18-10.json new file mode 100644 index 0000000000000000000000000000000000000000..e018d4f1d9b2f80fa3e887a8264e4b64b8e09d99 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-18-10.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 3936.392, + "avg tokens": 1386.946, + "avg character count": 2942.368, + "parameters count": 2.172819456, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.930082578859948, + "bpb": 1.0826299243237893, + "compression_rate": 13.532874054047367, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-21-38.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-21-38.json new file mode 100644 index 0000000000000000000000000000000000000000..c50600b0da7f833cb4d40a97c0eb23ff6de94bcb --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-21-38.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 5224.928, + "avg tokens": 2936.558, + "avg character count": 11865.894, + "parameters count": 2.172819456, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6352642046694408, + "bpb": 0.6348173782656928, + "compression_rate": 7.935217228321161, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-24-30.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-24-30.json new file mode 100644 index 0000000000000000000000000000000000000000..18a21557bc091c9a36246fa5cee4695cc0bbcb2f --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-24-30.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 3268.224, + "avg tokens": 2471.02, + "avg character count": 8435.788, + "parameters count": 2.172819456, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5589342166154829, + "bpb": 0.558652631659837, + "compression_rate": 6.983157895747963, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-27-44.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-27-44.json new file mode 100644 index 0000000000000000000000000000000000000000..6d2c7e3753f366209340afe8c58709ade95fa7a5 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-27-44.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4400.2, + "avg tokens": 2740.496, + "avg character count": 10817.836, + "parameters count": 2.172819456, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5868222368059209, + "bpb": 0.5864144833750179, + "compression_rate": 7.330181042187724, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-31-20.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-31-20.json new file mode 100644 index 0000000000000000000000000000000000000000..109da48e3ce78ffa34f9ba56f614728286543872 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-31-20.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4652.704, + "avg tokens": 3016.894, + "avg character count": 11165.434, + "parameters count": 2.172819456, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6011797649356257, + "bpb": 0.6008828063646002, + "compression_rate": 7.5110350795575025, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-bbc_news-2026-04-18_14-32-04.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-bbc_news-2026-04-18_14-32-04.json new file mode 100644 index 0000000000000000000000000000000000000000..0cba854c20faac97995890951c28dfc85a1eca5f --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-bbc_news-2026-04-18_14-32-04.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1407.924, + "avg tokens": 693.58, + "avg character count": 2745.97, + "parameters count": 2.172819456, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7397039926687302, + "bpb": 0.7391268967548481, + "compression_rate": 9.239086209435602, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-35-48.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-35-48.json new file mode 100644 index 0000000000000000000000000000000000000000..020ab6dc88ca8518ce89da5e402b97af45f5339d --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-35-48.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4946.96, + "avg tokens": 3113.818, + "avg character count": 11560.664, + "parameters count": 2.172819456, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6173481609253644, + "bpb": 0.6170555567495055, + "compression_rate": 7.713194459368819, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_cpp-2026-04-18_14-37-49.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_cpp-2026-04-18_14-37-49.json new file mode 100644 index 0000000000000000000000000000000000000000..03ddafae1aa48fb55bb7f358b3a5c3115b152484 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_cpp-2026-04-18_14-37-49.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1261.262, + "avg tokens": 1795.864, + "avg character count": 4752.824, + "parameters count": 2.172819456, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3828495295979182, + "bpb": 0.37559218452103876, + "compression_rate": 4.694902306512985, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_javascript-2026-04-18_14-39-53.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_javascript-2026-04-18_14-39-53.json new file mode 100644 index 0000000000000000000000000000000000000000..f6889ee8b9c8e1263c758447376e918db65b1105 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_javascript-2026-04-18_14-39-53.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1607.608, + "avg tokens": 1817.806, + "avg character count": 5202.974, + "parameters count": 2.172819456, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.445761998674878, + "bpb": 0.43208518441046695, + "compression_rate": 5.4010648051308365, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_markdown-2026-04-18_14-41-50.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_markdown-2026-04-18_14-41-50.json new file mode 100644 index 0000000000000000000000000000000000000000..11dc276c5d2fdfbb5f9d3c904710d9e93d7a4cb7 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_markdown-2026-04-18_14-41-50.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 2897.1875, + "avg tokens": 1741.934, + "avg character count": 4927.866, + "parameters count": 2.172819456, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8481882500002017, + "bpb": 0.7805103934850675, + "compression_rate": 9.756379918563344, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_other-2026-04-18_14-43-59.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_other-2026-04-18_14-43-59.json new file mode 100644 index 0000000000000000000000000000000000000000..69e08c4a80350712da3e167eae3eceb86559842d --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_other-2026-04-18_14-43-59.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1578.52, + "avg tokens": 1918.494, + "avg character count": 5229.94, + "parameters count": 2.172819456, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.43543959891395434, + "bpb": 0.4277168896872393, + "compression_rate": 5.346461121090491, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_python-2026-04-18_14-45-59.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_python-2026-04-18_14-45-59.json new file mode 100644 index 0000000000000000000000000000000000000000..4c296c1e18eca4bf5c72e1310faf971577d121e6 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-github_python-2026-04-18_14-45-59.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1587.556, + "avg tokens": 1786.844, + "avg character count": 5274.818, + "parameters count": 2.172819456, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4342062926784429, + "bpb": 0.4216238049269452, + "compression_rate": 5.270297561586815, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-46-55.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-46-55.json new file mode 100644 index 0000000000000000000000000000000000000000..30954428e51e3c5fc0fff948effb8335aa7ec1ab --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-46-55.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1571.288, + "avg tokens": 860.406, + "avg character count": 3127.092, + "parameters count": 2.172819456, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7249193197412603, + "bpb": 0.7211134654097846, + "compression_rate": 9.013918317622307, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Index-1.9B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-48-15.json b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-48-15.json new file mode 100644 index 0000000000000000000000000000000000000000..66a4f2400f08fbdd80972d846523f32911b23d59 --- /dev/null +++ b/data/2026-04/Index-1.9B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-48-15.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1677.256, + "avg tokens": 1268.326, + "avg character count": 2498.564, + "parameters count": 2.172819456, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "IndexTeam/Index-1.9B", + "tokenizer_name": "IndexTeam/Index-1.9B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9684638510365392, + "bpb": 0.6894761399956175, + "compression_rate": 8.61845174994522, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_10-01-14.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_10-01-14.json new file mode 100644 index 0000000000000000000000000000000000000000..0d3733f6c0beaa06107f8c475069f042c2a5b23d --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_10-01-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5134.032, + "avg tokens": 2128.638, + "avg character count": 9041.522, + "parameters count": 48.942532608, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8192030618479108, + "bpb": 0.8038977885411085, + "compression_rate": 10.048722356763856, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_10-14-07.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_10-14-07.json new file mode 100644 index 0000000000000000000000000000000000000000..65160dfa8eec14c61a8e341f2d14b629ffb7c3aa --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_10-14-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3783.472, + "avg tokens": 1264.472, + "avg character count": 2942.368, + "parameters count": 48.942532608, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.855103199784068, + "bpb": 1.040572180067731, + "compression_rate": 13.007152250846637, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_10-28-48.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_10-28-48.json new file mode 100644 index 0000000000000000000000000000000000000000..6e0e7fd38d49b80e7d3c0b9aaf9ec22a8febb36b --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_10-28-48.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4154.992, + "avg tokens": 2406.12, + "avg character count": 11865.894, + "parameters count": 48.942532608, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5051778107349784, + "bpb": 0.5048224833251153, + "compression_rate": 6.3102810415639405, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_10-43-18.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_10-43-18.json new file mode 100644 index 0000000000000000000000000000000000000000..f5d1f82ea5fa70c49c2638eab65a4a99c6949cd8 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_10-43-18.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2740.92, + "avg tokens": 2156.88, + "avg character count": 8435.788, + "parameters count": 48.942532608, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.46875427541248993, + "bpb": 0.4685181221265986, + "compression_rate": 5.856476526582482, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_10-57-50.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_10-57-50.json new file mode 100644 index 0000000000000000000000000000000000000000..7a40396f17e0f07984a7d0e2e2a06dba9c8d0a3d --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_10-57-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3563.968, + "avg tokens": 2278.39, + "avg character count": 10817.836, + "parameters count": 48.942532608, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4753001394629163, + "bpb": 0.47496987716128714, + "compression_rate": 5.937123464516089, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_11-12-23.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_11-12-23.json new file mode 100644 index 0000000000000000000000000000000000000000..40ec3e91ac012f7ba93b5ee48abcf327ea4700e9 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_11-12-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3801.928, + "avg tokens": 2526.488, + "avg character count": 11165.434, + "parameters count": 48.942532608, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4912502882930386, + "bpb": 0.4910076304523459, + "compression_rate": 6.137595380654324, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_11-24-51.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_11-24-51.json new file mode 100644 index 0000000000000000000000000000000000000000..6a301d08246ee6b0471c3b469ff522fb9f4404af --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_11-24-51.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1271.82, + "avg tokens": 578.582, + "avg character count": 2745.97, + "parameters count": 48.942532608, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6681968145694969, + "bpb": 0.6676755065122485, + "compression_rate": 8.345943831403106, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_11-39-11.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_11-39-11.json new file mode 100644 index 0000000000000000000000000000000000000000..29940916660156630f9eb3aec653e70380f8f0e0 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_11-39-11.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4028.352, + "avg tokens": 2448.832, + "avg character count": 11560.664, + "parameters count": 48.942532608, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5027119076685506, + "bpb": 0.502473637576003, + "compression_rate": 6.280920469700038, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_11-53-24.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_11-53-24.json new file mode 100644 index 0000000000000000000000000000000000000000..025bf6d8d62e2f3101f29148cfdfd8df48e17300 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_11-53-24.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 781.9155, + "avg tokens": 1435.332, + "avg character count": 4752.824, + "parameters count": 48.942532608, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.23734638906136946, + "bpb": 0.2328472202887745, + "compression_rate": 2.9105902536096813, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_12-07-54.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_12-07-54.json new file mode 100644 index 0000000000000000000000000000000000000000..280a95d9cfdb54495cc48f11c9b909f7a0a9d709 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_12-07-54.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 878.0765, + "avg tokens": 1472.278, + "avg character count": 5202.974, + "parameters count": 48.942532608, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.24347548384272874, + "bpb": 0.23600519929547342, + "compression_rate": 2.950064991193418, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_12-21-50.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_12-21-50.json new file mode 100644 index 0000000000000000000000000000000000000000..4df94a5b2afb196b03aa20478ce4b633afea3b42 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_12-21-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1916.7875, + "avg tokens": 1413.92, + "avg character count": 4927.866, + "parameters count": 48.942532608, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5611637614918819, + "bpb": 0.5163878989027318, + "compression_rate": 6.4548487362841485, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_other-2026-04-19_12-35-56.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_other-2026-04-19_12-35-56.json new file mode 100644 index 0000000000000000000000000000000000000000..a887a1580592ffc8d127f27dab1317675e8c3fcc --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_other-2026-04-19_12-35-56.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 876.0505, + "avg tokens": 1542.868, + "avg character count": 5229.94, + "parameters count": 48.942532608, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.24166122592578443, + "bpb": 0.23737525978064952, + "compression_rate": 2.967190747258119, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_python-2026-04-19_12-50-00.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_python-2026-04-19_12-50-00.json new file mode 100644 index 0000000000000000000000000000000000000000..7aafcc315c0eb8ca8c210f8e97cf7c107e83db92 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-github_python-2026-04-19_12-50-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 922.5965, + "avg tokens": 1418.036, + "avg character count": 5274.818, + "parameters count": 48.942532608, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.25233579533768075, + "bpb": 0.24502357506902583, + "compression_rate": 3.062794688362823, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_13-02-10.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_13-02-10.json new file mode 100644 index 0000000000000000000000000000000000000000..bf4c7ea3c23f7683c2ca6c1b4168a02809104c17 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_13-02-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1314.852, + "avg tokens": 715.93, + "avg character count": 3127.092, + "parameters count": 48.942532608, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6066116570612362, + "bpb": 0.6034269225126051, + "compression_rate": 7.542836531407564, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_13-14-36.json b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_13-14-36.json new file mode 100644 index 0000000000000000000000000000000000000000..e5dd886a0581d5a8cefebc42fff3901226548a94 --- /dev/null +++ b/data/2026-04/JoyAI-LLM-Flash-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_13-14-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1790.44, + "avg tokens": 927.532, + "avg character count": 2498.564, + "parameters count": 48.942532608, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "jdopensource/JoyAI-LLM-Flash-Base", + "tokenizer_name": "jdopensource/JoyAI-LLM-Flash-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.0338173883115405, + "bpb": 0.7360031265911426, + "compression_rate": 9.200039082389283, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_17-53-42.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_17-53-42.json new file mode 100644 index 0000000000000000000000000000000000000000..a175fa868200e518f3580dedd527157e1bb6cba9 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_17-53-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5535.776, + "avg tokens": 2097.61, + "avg character count": 9041.522, + "parameters count": 4.540469248, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8833066581790256, + "bpb": 0.8668037293610448, + "compression_rate": 10.83504661701306, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-54-01.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-54-01.json new file mode 100644 index 0000000000000000000000000000000000000000..fe805fb60f7ee563dbca324386bd87defea48663 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-54-01.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4721.424, + "avg tokens": 1533.964, + "avg character count": 2942.368, + "parameters count": 4.540469248, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.3149976450036616, + "bpb": 1.2985380794952643, + "compression_rate": 16.231725993690805, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-54-30.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-54-30.json new file mode 100644 index 0000000000000000000000000000000000000000..3d36e27d2002f4dcdcc36c478f7882a1d583ea31 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-54-30.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5289.408, + "avg tokens": 2452.278, + "avg character count": 11865.894, + "parameters count": 4.540469248, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6431038985211237, + "bpb": 0.6426515579042587, + "compression_rate": 8.033144473803233, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-54-57.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-54-57.json new file mode 100644 index 0000000000000000000000000000000000000000..b7058d45dfa24de7d82f7a26c3e72315ff98a5c0 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-54-57.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3675.408, + "avg tokens": 2283.974, + "avg character count": 8435.788, + "parameters count": 4.540469248, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6285711417645421, + "bpb": 0.6282544744863321, + "compression_rate": 7.853180931079152, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-55-24.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-55-24.json new file mode 100644 index 0000000000000000000000000000000000000000..4a3efdae6dad9e798675196a8ffaa7074b8f9609 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-55-24.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4586.688, + "avg tokens": 2359.912, + "avg character count": 10817.836, + "parameters count": 4.540469248, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6116927666221709, + "bpb": 0.6112677319036394, + "compression_rate": 7.640846648795492, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-55-53.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-55-53.json new file mode 100644 index 0000000000000000000000000000000000000000..788b2a75e3a18724c1ca2a6f852005ad36556a91 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-55-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4948.696, + "avg tokens": 2641.25, + "avg character count": 11165.434, + "parameters count": 4.540469248, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6394251381600616, + "bpb": 0.6391092879162895, + "compression_rate": 7.988866098953619, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-56-06.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-56-06.json new file mode 100644 index 0000000000000000000000000000000000000000..0bc8bd93cc436d22ab375d7a3b53c189b82b8b33 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-56-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1381.08, + "avg tokens": 577.996, + "avg character count": 2745.97, + "parameters count": 4.540469248, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7256005226098354, + "bpb": 0.7250344298202075, + "compression_rate": 9.062930372752593, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-56-35.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-56-35.json new file mode 100644 index 0000000000000000000000000000000000000000..ae4f4e1ec39070df670d3914fadf0fbfa56161a9 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-56-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4948.744, + "avg tokens": 2554.072, + "avg character count": 11560.664, + "parameters count": 4.540469248, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6175707924241213, + "bpb": 0.6172780827277307, + "compression_rate": 7.7159760340966335, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-56-53.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-56-53.json new file mode 100644 index 0000000000000000000000000000000000000000..6f49f281ffde8ba330fc4b4ec7468dd183a14930 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-56-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1166.731, + "avg tokens": 1310.738, + "avg character count": 4752.824, + "parameters count": 4.540469248, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3541551355050011, + "bpb": 0.347441725064591, + "compression_rate": 4.343021563307388, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-57-13.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-57-13.json new file mode 100644 index 0000000000000000000000000000000000000000..9db25bf2ab3a8962a3aa63a4acaedf4190e1dc6e --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-57-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1446.453, + "avg tokens": 1371.522, + "avg character count": 5202.974, + "parameters count": 4.540469248, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4010764939396129, + "bpb": 0.3887707147800168, + "compression_rate": 4.85963393475021, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-57-31.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-57-31.json new file mode 100644 index 0000000000000000000000000000000000000000..20c61a078c7a14be6519a302dbfa284ca3a8bf21 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-57-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2767.1615, + "avg tokens": 1400.102, + "avg character count": 4927.866, + "parameters count": 4.540469248, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8101214954685995, + "bpb": 0.74548102640983, + "compression_rate": 9.318512830122875, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-57-50.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-57-50.json new file mode 100644 index 0000000000000000000000000000000000000000..f711c1fbfb680e502184eb6cc03b97c62df4f95b --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-57-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1450.346, + "avg tokens": 1433.536, + "avg character count": 5229.94, + "parameters count": 4.540469248, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.40008240663815353, + "bpb": 0.3929867724769587, + "compression_rate": 4.912334655961984, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-58-07.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-58-07.json new file mode 100644 index 0000000000000000000000000000000000000000..14648557297ea9ba4699d93d8a6403acd2e2880f --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-58-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1467.512, + "avg tokens": 1309.164, + "avg character count": 5274.818, + "parameters count": 4.540469248, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4013735231898133, + "bpb": 0.3897424678033097, + "compression_rate": 4.871780847541371, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-58-20.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-58-20.json new file mode 100644 index 0000000000000000000000000000000000000000..7059492a1f38004f0f5565941bcd33a89b0ae12f --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-58-20.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1536.044, + "avg tokens": 723.416, + "avg character count": 3127.092, + "parameters count": 4.540469248, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7086593747121117, + "bpb": 0.7049388857178999, + "compression_rate": 8.811736071473748, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-58-34.json b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-58-34.json new file mode 100644 index 0000000000000000000000000000000000000000..dc25763d091f9cd4867a901275472bd5a3eddf67 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Depth-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-58-34.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2158.936, + "avg tokens": 921.834, + "avg character count": 2498.564, + "parameters count": 4.540469248, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.2465905459282434, + "bpb": 0.8874822089040544, + "compression_rate": 11.09352761130068, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-44-38.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-44-38.json new file mode 100644 index 0000000000000000000000000000000000000000..0ba50e58df2bfde37744cc8e4da277b875ef1ee2 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-44-38.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5354.024, + "avg tokens": 2097.61, + "avg character count": 9041.522, + "parameters count": 4.512746496, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8543057102112334, + "bpb": 0.8383446097328611, + "compression_rate": 10.479307621660764, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-44-59.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-44-59.json new file mode 100644 index 0000000000000000000000000000000000000000..722b0888fd608eb1c6569a5eb587ddc5190ae175 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-44-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4503.216, + "avg tokens": 1533.964, + "avg character count": 2942.368, + "parameters count": 4.512746496, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.2080064054706394, + "bpb": 1.2385241097161253, + "compression_rate": 15.481551371451566, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-45-32.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-45-32.json new file mode 100644 index 0000000000000000000000000000000000000000..cd9f1177a42c9e468652ac9d26d987fd0a8c5d84 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-45-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5052.88, + "avg tokens": 2452.278, + "avg character count": 11865.894, + "parameters count": 4.512746496, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6143460339530276, + "bpb": 0.6139139207834355, + "compression_rate": 7.673924009792945, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-46-02.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-46-02.json new file mode 100644 index 0000000000000000000000000000000000000000..9468b6202b320df4486a8682ad13f300933a3552 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-46-02.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3483.952, + "avg tokens": 2283.974, + "avg character count": 8435.788, + "parameters count": 4.512746496, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.595828187372085, + "bpb": 0.5955280156367962, + "compression_rate": 7.444100195459953, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-46-34.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-46-34.json new file mode 100644 index 0000000000000000000000000000000000000000..1bde1a4164b3104fec2b7a60845ec443ed8e369a --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-46-34.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4362.632, + "avg tokens": 2359.912, + "avg character count": 10817.836, + "parameters count": 4.512746496, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5818120695879934, + "bpb": 0.5814077974717787, + "compression_rate": 7.267597468397234, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-47-09.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-47-09.json new file mode 100644 index 0000000000000000000000000000000000000000..094945c8b5875e686b974713fb44ecfec4c58fdc --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-47-09.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4697.944, + "avg tokens": 2641.25, + "avg character count": 11165.434, + "parameters count": 4.512746496, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6070252630729859, + "bpb": 0.6067254170615057, + "compression_rate": 7.584067713268822, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-47-23.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-47-23.json new file mode 100644 index 0000000000000000000000000000000000000000..3f7ae1c8b35abe5cce7530e1cb6173b20b21ae12 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-47-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1327.924, + "avg tokens": 577.996, + "avg character count": 2745.97, + "parameters count": 4.512746496, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6976730880080394, + "bpb": 0.6971287834047044, + "compression_rate": 8.714109792558805, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-47-57.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-47-57.json new file mode 100644 index 0000000000000000000000000000000000000000..9bf5cb3eeed9abf03523f617df80c6585c149a63 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-47-57.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4713.576, + "avg tokens": 2554.072, + "avg character count": 11560.664, + "parameters count": 4.512746496, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5882233684893218, + "bpb": 0.5879445685756722, + "compression_rate": 7.349307107195903, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-48-19.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-48-19.json new file mode 100644 index 0000000000000000000000000000000000000000..caa6b1731a6b5d2f9d50c8a411869fe17355b847 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-48-19.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1101.156, + "avg tokens": 1310.738, + "avg character count": 4752.824, + "parameters count": 4.512746496, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33425018482593244, + "bpb": 0.32791409519865733, + "compression_rate": 4.098926189983216, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-48-40.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-48-40.json new file mode 100644 index 0000000000000000000000000000000000000000..371e8b094db1e1dac8236e5b84a32fc58ea71e38 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-48-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1358.362, + "avg tokens": 1371.522, + "avg character count": 5202.974, + "parameters count": 4.512746496, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3766503774825733, + "bpb": 0.3650940373935504, + "compression_rate": 4.56367546741938, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-49-01.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-49-01.json new file mode 100644 index 0000000000000000000000000000000000000000..018a458aacdcb17a228b51c574060b4b5460d0d6 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-49-01.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2621.747, + "avg tokens": 1400.102, + "avg character count": 4927.866, + "parameters count": 4.512746496, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7675495631101814, + "bpb": 0.7063059545121934, + "compression_rate": 8.828824431402417, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-49-23.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-49-23.json new file mode 100644 index 0000000000000000000000000000000000000000..bcfec595c8a3ef7da3553b3a34cb8d61779f28b4 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-49-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1356.572, + "avg tokens": 1433.536, + "avg character count": 5229.94, + "parameters count": 4.512746496, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3742145602069666, + "bpb": 0.3675777034670436, + "compression_rate": 4.594721293338045, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-49-44.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-49-44.json new file mode 100644 index 0000000000000000000000000000000000000000..6fd88e08f74b81407d11db4d079e59aafc166d47 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-49-44.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1382.446, + "avg tokens": 1309.164, + "avg character count": 5274.818, + "parameters count": 4.512746496, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.37810745100528276, + "bpb": 0.3671506029557607, + "compression_rate": 4.589382536947008, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-49-59.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-49-59.json new file mode 100644 index 0000000000000000000000000000000000000000..2b133829d2d3fe2a85c25d570bb8a59cd696f4a4 --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-49-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1469.356, + "avg tokens": 723.416, + "avg character count": 3127.092, + "parameters count": 4.512746496, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6778926281991203, + "bpb": 0.674333665808343, + "compression_rate": 8.429170822604288, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-50-15.json b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-50-15.json new file mode 100644 index 0000000000000000000000000000000000000000..2455513a554199e13a9c23403dac4c5889883b5f --- /dev/null +++ b/data/2026-04/Llama-3.1-Minitron-4B-Width-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-50-15.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1975.148, + "avg tokens": 921.834, + "avg character count": 2498.564, + "parameters count": 4.512746496, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "tokenizer_name": "nvidia/Llama-3.1-Minitron-4B-Width-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.1404695755729108, + "bpb": 0.8119317617346807, + "compression_rate": 10.14914702168351, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-ao3_english-2026-04-18_16-27-32.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-ao3_english-2026-04-18_16-27-32.json new file mode 100644 index 0000000000000000000000000000000000000000..da633cb308d72593e194165b06267dc562063719 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-ao3_english-2026-04-18_16-27-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5341.112, + "avg tokens": 2097.61, + "avg character count": 9041.522, + "parameters count": 3.212749824, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8522454289479728, + "bpb": 0.8363228209622333, + "compression_rate": 10.454035262027917, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-27-51.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-27-51.json new file mode 100644 index 0000000000000000000000000000000000000000..784b72714bfe066cd4cd76d0b952d4253144772f --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-27-51.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4225.96, + "avg tokens": 1533.964, + "avg character count": 2942.368, + "parameters count": 3.212749824, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.0720628877812444, + "bpb": 1.1622701080063573, + "compression_rate": 14.528376350079467, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-28-18.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-28-18.json new file mode 100644 index 0000000000000000000000000000000000000000..4bc7ce2904f7615928f3baf4c9d99ed21ca781d6 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-28-18.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4962.4, + "avg tokens": 2452.278, + "avg character count": 11865.894, + "parameters count": 3.212749824, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.603345173225666, + "bpb": 0.6029207977422223, + "compression_rate": 7.536509971777779, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-28-43.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-28-43.json new file mode 100644 index 0000000000000000000000000000000000000000..a53c385fb47491e27bb692eb9760b3e82373b13f --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-28-43.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3204.464, + "avg tokens": 2283.974, + "avg character count": 8435.788, + "parameters count": 3.212749824, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5480299317037378, + "bpb": 0.5477538402077727, + "compression_rate": 6.846923002597159, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-29-09.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-29-09.json new file mode 100644 index 0000000000000000000000000000000000000000..43e9111112a89552267b701872cb35e2c13db360 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-29-09.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4181.968, + "avg tokens": 2359.912, + "avg character count": 10817.836, + "parameters count": 3.212749824, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5577182437186455, + "bpb": 0.5573307131973221, + "compression_rate": 6.966633914966526, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-29-39.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-29-39.json new file mode 100644 index 0000000000000000000000000000000000000000..90dd59f1e1dc4041275e489db4207059d20c82b0 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-29-39.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4433.792, + "avg tokens": 2641.25, + "avg character count": 11165.434, + "parameters count": 3.212749824, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5728939628081774, + "bpb": 0.5726109762832352, + "compression_rate": 7.15763720354044, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-bbc_news-2026-04-18_16-29-57.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-bbc_news-2026-04-18_16-29-57.json new file mode 100644 index 0000000000000000000000000000000000000000..3acbc124d6e03508bf0421b3cf66f427946fe9e6 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-bbc_news-2026-04-18_16-29-57.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1296.42, + "avg tokens": 577.996, + "avg character count": 2745.97, + "parameters count": 3.212749824, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6811213177526595, + "bpb": 0.6805899263674178, + "compression_rate": 8.507374079592722, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-30-26.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-30-26.json new file mode 100644 index 0000000000000000000000000000000000000000..ea7140a6b3907e02635c5b25ce12ab5111a6fd27 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-30-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4644.068, + "avg tokens": 2554.072, + "avg character count": 11560.664, + "parameters count": 3.212749824, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5795492259917878, + "bpb": 0.5792745373567934, + "compression_rate": 7.240931716959917, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_cpp-2026-04-18_16-30-47.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_cpp-2026-04-18_16-30-47.json new file mode 100644 index 0000000000000000000000000000000000000000..3373a531331a6632329de8b169eae7822093c6df --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_cpp-2026-04-18_16-30-47.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1123.373, + "avg tokens": 1310.738, + "avg character count": 4752.824, + "parameters count": 3.212749824, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.34099403978951415, + "bpb": 0.33453011277748235, + "compression_rate": 4.181626409718529, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_javascript-2026-04-18_16-31-06.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_javascript-2026-04-18_16-31-06.json new file mode 100644 index 0000000000000000000000000000000000000000..d16efcec0d895f8b5c9075f53d7713601b8ac750 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_javascript-2026-04-18_16-31-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1326.487, + "avg tokens": 1371.522, + "avg character count": 5202.974, + "parameters count": 3.212749824, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3678119892014987, + "bpb": 0.3565268274436847, + "compression_rate": 4.456585343046059, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_markdown-2026-04-18_16-31-24.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_markdown-2026-04-18_16-31-24.json new file mode 100644 index 0000000000000000000000000000000000000000..6eacd2c2cdb2fcc482e0babc4dca27a0ec32c852 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_markdown-2026-04-18_16-31-24.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2561.287, + "avg tokens": 1400.102, + "avg character count": 4927.866, + "parameters count": 3.212749824, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.749849134126896, + "bpb": 0.6900178618740375, + "compression_rate": 8.625223273425469, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_other-2026-04-18_16-31-43.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_other-2026-04-18_16-31-43.json new file mode 100644 index 0000000000000000000000000000000000000000..2a27e37e154fcfe5381fd0db44ae544ba45489f8 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_other-2026-04-18_16-31-43.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1303.666, + "avg tokens": 1433.536, + "avg character count": 5229.94, + "parameters count": 3.212749824, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3596202773216426, + "bpb": 0.35324225648772556, + "compression_rate": 4.415528206096569, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_python-2026-04-18_16-31-59.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_python-2026-04-18_16-31-59.json new file mode 100644 index 0000000000000000000000000000000000000000..4da71b3454742cac5fc8ab08e72144c1972daf5e --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-github_python-2026-04-18_16-31-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1357.268, + "avg tokens": 1309.164, + "avg character count": 5274.818, + "parameters count": 3.212749824, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3712211137440726, + "bpb": 0.3604638188924265, + "compression_rate": 4.505797736155332, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-32-12.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-32-12.json new file mode 100644 index 0000000000000000000000000000000000000000..df0af00603d1b876aae900259696f4b38fdaab06 --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-32-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1445.62, + "avg tokens": 723.416, + "avg character count": 3127.092, + "parameters count": 3.212749824, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6669419399908615, + "bpb": 0.6634404691346799, + "compression_rate": 8.293005864183499, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-32-26.json b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-32-26.json new file mode 100644 index 0000000000000000000000000000000000000000..e0416e5031f5f58499c0297018dc091bed9d155f --- /dev/null +++ b/data/2026-04/Llama-3.2-3B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-32-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1586.896, + "avg tokens": 921.834, + "avg character count": 2498.564, + "parameters count": 3.212749824, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "meta-llama/Llama-3.2-3B", + "tokenizer_name": "meta-llama/Llama-3.2-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9162891123087232, + "bpb": 0.6523315037504622, + "compression_rate": 8.154143796880778, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-ao3_english-2026-04-18_14-25-34.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-ao3_english-2026-04-18_14-25-34.json new file mode 100644 index 0000000000000000000000000000000000000000..be4f50ea1c8a3724decc0853bc1e98585871a1bb --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-ao3_english-2026-04-18_14-25-34.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5562.264, + "avg tokens": 2132.32, + "avg character count": 9041.522, + "parameters count": 3.849090048, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8875331707333354, + "bpb": 0.8709512774524623, + "compression_rate": 10.886890968155779, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-25-53.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-25-53.json new file mode 100644 index 0000000000000000000000000000000000000000..d8f15e0f17284fbbaf28950bb3f67d102d7e7d4d --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-25-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4057.256, + "avg tokens": 1588.088, + "avg character count": 2942.368, + "parameters count": 3.849090048, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.9893443345009842, + "bpb": 1.1158712740606729, + "compression_rate": 13.94839092575841, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-26-20.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-26-20.json new file mode 100644 index 0000000000000000000000000000000000000000..0f947e9d2b011ce277c934d98383f2417a529be9 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-26-20.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4645.936, + "avg tokens": 2489.996, + "avg character count": 11865.894, + "parameters count": 3.849090048, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5648684226816375, + "bpb": 0.5644711106277828, + "compression_rate": 7.0558888828472845, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-26-44.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-26-44.json new file mode 100644 index 0000000000000000000000000000000000000000..0136b7b97296c7f35c700362823d535c82bdc227 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-26-44.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3074.096, + "avg tokens": 2188.618, + "avg character count": 8435.788, + "parameters count": 3.849090048, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5257342946997481, + "bpb": 0.5254694355022723, + "compression_rate": 6.568367943778404, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-27-09.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-27-09.json new file mode 100644 index 0000000000000000000000000000000000000000..58c0eb61627b7d96585d85097215797fd025d0c0 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-27-09.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3985.792, + "avg tokens": 2338.076, + "avg character count": 10817.836, + "parameters count": 3.849090048, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5315556967599531, + "bpb": 0.5311863452843687, + "compression_rate": 6.639829316054609, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-27-36.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-27-36.json new file mode 100644 index 0000000000000000000000000000000000000000..54223b2fcbb41e5bab04b3f92786c762325aac52 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-27-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4213.2, + "avg tokens": 2599.718, + "avg character count": 11165.434, + "parameters count": 3.849090048, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5443910864793415, + "bpb": 0.5441221792263883, + "compression_rate": 6.801527240329854, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-bbc_news-2026-04-18_14-27-52.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-bbc_news-2026-04-18_14-27-52.json new file mode 100644 index 0000000000000000000000000000000000000000..81117eb2b12665306bbdfda6c5d99d51aa1be612 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-bbc_news-2026-04-18_14-27-52.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1362.472, + "avg tokens": 600.428, + "avg character count": 2745.97, + "parameters count": 3.849090048, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7158241341857586, + "bpb": 0.715265668654964, + "compression_rate": 8.94082085818705, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-28-19.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-28-19.json new file mode 100644 index 0000000000000000000000000000000000000000..6d4b04d0eb39e0652f09d0466b93f6c6ef33331d --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-28-19.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4391.552, + "avg tokens": 2543.524, + "avg character count": 11560.664, + "parameters count": 3.849090048, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5480368854423939, + "bpb": 0.5477771326945041, + "compression_rate": 6.847214158681301, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_cpp-2026-04-18_14-28-37.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_cpp-2026-04-18_14-28-37.json new file mode 100644 index 0000000000000000000000000000000000000000..de9da430e912d6d8bc412b3b882ed8307f9278a7 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_cpp-2026-04-18_14-28-37.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1023.888, + "avg tokens": 1468.266, + "avg character count": 4752.824, + "parameters count": 3.849090048, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3107958847257376, + "bpb": 0.30490439783714834, + "compression_rate": 3.811304972964354, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_javascript-2026-04-18_14-28-55.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_javascript-2026-04-18_14-28-55.json new file mode 100644 index 0000000000000000000000000000000000000000..d6fcf2c42d9ea034de3da697b7d14406dfed78be --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_javascript-2026-04-18_14-28-55.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1211.433, + "avg tokens": 1527.154, + "avg character count": 5202.974, + "parameters count": 3.849090048, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3359094974276711, + "bpb": 0.32560316395907785, + "compression_rate": 4.070039549488473, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_markdown-2026-04-18_14-29-14.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_markdown-2026-04-18_14-29-14.json new file mode 100644 index 0000000000000000000000000000000000000000..b847a2ec6eeba544952f391accaa569be6522824 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_markdown-2026-04-18_14-29-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2353.8485, + "avg tokens": 1508.46, + "avg character count": 4927.866, + "parameters count": 3.849090048, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6891188920222111, + "bpb": 0.6341333513758554, + "compression_rate": 7.926666892198192, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_other-2026-04-18_14-29-34.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_other-2026-04-18_14-29-34.json new file mode 100644 index 0000000000000000000000000000000000000000..fbc9e7fa38363005e925e1314f1bb30e8db5e69e --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_other-2026-04-18_14-29-34.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1196.814, + "avg tokens": 1576.382, + "avg character count": 5229.94, + "parameters count": 3.849090048, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33014482435104114, + "bpb": 0.3242895633974506, + "compression_rate": 4.053619542468132, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_python-2026-04-18_14-29-51.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_python-2026-04-18_14-29-51.json new file mode 100644 index 0000000000000000000000000000000000000000..bd882e7257479023a504d98f95d09c46a250735c --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-github_python-2026-04-18_14-29-51.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1252.17, + "avg tokens": 1439.878, + "avg character count": 5274.818, + "parameters count": 3.849090048, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3424761668269755, + "bpb": 0.33255184687366807, + "compression_rate": 4.156898085920851, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-30-03.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-30-03.json new file mode 100644 index 0000000000000000000000000000000000000000..3ee9ea53bf4ec79cb6be3f316bdd68b07100fc44 --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-30-03.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1471.82, + "avg tokens": 752.304, + "avg character count": 3127.092, + "parameters count": 3.849090048, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6790294033821819, + "bpb": 0.6754644728779379, + "compression_rate": 8.443305910974223, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-30-15.json b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-30-15.json new file mode 100644 index 0000000000000000000000000000000000000000..809f055f84bbc0efbdfffc98c794cecab7be914f --- /dev/null +++ b/data/2026-04/Ministral-3-3B-Base-2512-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-30-15.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1541.768, + "avg tokens": 875.59, + "avg character count": 2498.564, + "parameters count": 3.849090048, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "mistralai/Ministral-3-3B-Base-2512", + "tokenizer_name": "mistralai/Ministral-3-3B-Base-2512", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8902317682481999, + "bpb": 0.6337805614699027, + "compression_rate": 7.9222570183737835, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_17-47-22.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_17-47-22.json new file mode 100644 index 0000000000000000000000000000000000000000..912f53e9ee1206e049cf7dbe888168a6a3d88228 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_17-47-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5455.344, + "avg tokens": 2161.144, + "avg character count": 9041.522, + "parameters count": 4.190509056, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8704726632466703, + "bpb": 0.8542095135618565, + "compression_rate": 10.677618919523207, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-47-42.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-47-42.json new file mode 100644 index 0000000000000000000000000000000000000000..d9702f72eec7263e7b6d29fea21b74225c3f3f37 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-47-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4495.656, + "avg tokens": 1544.108, + "avg character count": 2942.368, + "parameters count": 4.190509056, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.204299603837016, + "bpb": 1.2364448751714232, + "compression_rate": 15.45556093964279, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-48-11.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-48-11.json new file mode 100644 index 0000000000000000000000000000000000000000..1d49e190a0f5e9e7445039d0600fb40118cbbaea --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-48-11.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5155.392, + "avg tokens": 2531.178, + "avg character count": 11865.894, + "parameters count": 4.190509056, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6268097854437799, + "bpb": 0.6263689056331354, + "compression_rate": 7.829611320414193, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-48-37.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-48-37.json new file mode 100644 index 0000000000000000000000000000000000000000..b8174f4283cef4ed284810319e8e67b452ae55cc --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-48-37.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3635.04, + "avg tokens": 2282.834, + "avg character count": 8435.788, + "parameters count": 4.190509056, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6216673749308325, + "bpb": 0.621354185694975, + "compression_rate": 7.766927321187188, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-49-05.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-49-05.json new file mode 100644 index 0000000000000000000000000000000000000000..2ab882b942793d90ece27a8301930afc3ad4ffe5 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-49-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4480.416, + "avg tokens": 2398.092, + "avg character count": 10817.836, + "parameters count": 4.190509056, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5975200533932633, + "bpb": 0.5971048665845108, + "compression_rate": 7.463810832306385, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-49-34.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-49-34.json new file mode 100644 index 0000000000000000000000000000000000000000..0f76a37ce7efa0d1692e2b3dd45b23b2258f01d5 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-49-34.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4847.52, + "avg tokens": 2662.022, + "avg character count": 11165.434, + "parameters count": 4.190509056, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6263521028031752, + "bpb": 0.6260427101119107, + "compression_rate": 7.825533876398884, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-49-53.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-49-53.json new file mode 100644 index 0000000000000000000000000000000000000000..1444359bba5ec3c40a37d664fd31b91935d4dba8 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_17-49-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1346.008, + "avg tokens": 605.77, + "avg character count": 2745.97, + "parameters count": 4.190509056, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7071741740065886, + "bpb": 0.7066224569275045, + "compression_rate": 8.832780711593806, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-50-22.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-50-22.json new file mode 100644 index 0000000000000000000000000000000000000000..170debbd4318ec0a815b91d52708b62ce2abee29 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-50-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4793.712, + "avg tokens": 2560.364, + "avg character count": 11560.664, + "parameters count": 4.190509056, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5982238156778811, + "bpb": 0.5979402758576552, + "compression_rate": 7.47425344822069, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-50-44.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-50-44.json new file mode 100644 index 0000000000000000000000000000000000000000..0505af8c690a4a56479736c9551993b5ad5e9139 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_17-50-44.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1136.618, + "avg tokens": 1690.002, + "avg character count": 4752.824, + "parameters count": 4.190509056, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3450144907501586, + "bpb": 0.33847435155101324, + "compression_rate": 4.230929394387665, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-51-06.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-51-06.json new file mode 100644 index 0000000000000000000000000000000000000000..936ac977bd8d925846779b60ca753725c8ae3efc --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_17-51-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1416.726, + "avg tokens": 1709.552, + "avg character count": 5202.974, + "parameters count": 4.190509056, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3928337090476442, + "bpb": 0.3807808339900668, + "compression_rate": 4.759760424875835, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-51-28.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-51-28.json new file mode 100644 index 0000000000000000000000000000000000000000..fc099576687b0c25b7bd1bc43adbfadfe7193d8d --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_17-51-28.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2714.662, + "avg tokens": 1590.274, + "avg character count": 4927.866, + "parameters count": 4.190509056, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7947516034506041, + "bpb": 0.731337514675512, + "compression_rate": 9.1417189334439, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-51-51.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-51-51.json new file mode 100644 index 0000000000000000000000000000000000000000..d9d4d93c7df6590f23a6de99d186b5f4547bac8f --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_17-51-51.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1422.926, + "avg tokens": 1816.258, + "avg character count": 5229.94, + "parameters count": 4.190509056, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.39251851527015014, + "bpb": 0.38555702998701613, + "compression_rate": 4.8194628748377015, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-52-13.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-52-13.json new file mode 100644 index 0000000000000000000000000000000000000000..e0a9dcbae8e5117c9cef77ea4186dfb708fa906f --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_17-52-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1440.704, + "avg tokens": 1673.746, + "avg character count": 5274.818, + "parameters count": 4.190509056, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.39404137094187763, + "bpb": 0.382622787639283, + "compression_rate": 4.7827848454910376, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-52-33.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-52-33.json new file mode 100644 index 0000000000000000000000000000000000000000..459d47044c85f166a892eba328324933de2372f5 --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-52-33.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1508.704, + "avg tokens": 740.452, + "avg character count": 3127.092, + "parameters count": 4.190509056, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6960459682571994, + "bpb": 0.6923917001323778, + "compression_rate": 8.654896251654723, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-52-52.json b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-52-52.json new file mode 100644 index 0000000000000000000000000000000000000000..61108102d8f139f152ef41dd396766ab039a98ff --- /dev/null +++ b/data/2026-04/Minitron-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-52-52.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1800.26, + "avg tokens": 834.666, + "avg character count": 2498.564, + "parameters count": 4.190509056, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "nvidia/Minitron-4B-Base", + "tokenizer_name": "nvidia/Minitron-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.039487551373815, + "bpb": 0.740039872141468, + "compression_rate": 9.25049840176835, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-ao3_english-2026-04-18_21-21-48.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-ao3_english-2026-04-18_21-21-48.json new file mode 100644 index 0000000000000000000000000000000000000000..5757f6fcf6805ad07d06eeb27852422c0c03d0d9 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-ao3_english-2026-04-18_21-21-48.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4773.368, + "avg tokens": 2132.32, + "avg character count": 9041.522, + "parameters count": 24.01136128, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7616543256697346, + "bpb": 0.7474242426017007, + "compression_rate": 9.342803032521259, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_21-23-01.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_21-23-01.json new file mode 100644 index 0000000000000000000000000000000000000000..a507c4128bd060f8437bb8affccc5a20acd2d5bf --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_21-23-01.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3388.128, + "avg tokens": 1588.088, + "avg character count": 2942.368, + "parameters count": 24.01136128, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.6612590483233376, + "bpb": 0.9318403147449014, + "compression_rate": 11.648003934311268, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_cs-2026-04-18_21-24-53.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_cs-2026-04-18_21-24-53.json new file mode 100644 index 0000000000000000000000000000000000000000..394d448455c1021122c59ef3f20dafb32aa0c259 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_cs-2026-04-18_21-24-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4258.704, + "avg tokens": 2489.996, + "avg character count": 11865.894, + "parameters count": 24.01136128, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5177874622353774, + "bpb": 0.5174232655626296, + "compression_rate": 6.467790819532871, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_math-2026-04-18_21-26-32.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_math-2026-04-18_21-26-32.json new file mode 100644 index 0000000000000000000000000000000000000000..493f37f3b4d2326928afa5c05e66ce83ec30d970 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_math-2026-04-18_21-26-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2797.208, + "avg tokens": 2188.618, + "avg character count": 8435.788, + "parameters count": 24.01136128, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4783806930585424, + "bpb": 0.47813969008854645, + "compression_rate": 5.976746126106831, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-28-18.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-28-18.json new file mode 100644 index 0000000000000000000000000000000000000000..60b8f6f8a5012367d2a6dfec82969953e4c40c89 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-28-18.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3630.76, + "avg tokens": 2338.076, + "avg character count": 10817.836, + "parameters count": 24.01136128, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4842076961286909, + "bpb": 0.4838712444113176, + "compression_rate": 6.04839055514147, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-30-14.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-30-14.json new file mode 100644 index 0000000000000000000000000000000000000000..d0ba59ad5dd4a54535160911af95268e29b1fd47 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-30-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3817.864, + "avg tokens": 2599.718, + "avg character count": 11165.434, + "parameters count": 24.01136128, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4933093921461989, + "bpb": 0.4930657171912028, + "compression_rate": 6.163321464890035, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-bbc_news-2026-04-18_21-30-47.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-bbc_news-2026-04-18_21-30-47.json new file mode 100644 index 0000000000000000000000000000000000000000..2af808d5e5226e8dc577e156b62363fea6435eb7 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-bbc_news-2026-04-18_21-30-47.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1205.196, + "avg tokens": 600.428, + "avg character count": 2745.97, + "parameters count": 24.01136128, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6331934771680736, + "bpb": 0.6326994777142487, + "compression_rate": 7.908743471428109, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-32-42.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-32-42.json new file mode 100644 index 0000000000000000000000000000000000000000..4483b672c571e247d925f6093971e624b521818d --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-32-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4071.456, + "avg tokens": 2543.524, + "avg character count": 11560.664, + "parameters count": 24.01136128, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5080910041497284, + "bpb": 0.5078501845297141, + "compression_rate": 6.348127306621426, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_cpp-2026-04-18_21-33-50.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_cpp-2026-04-18_21-33-50.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0d0234dbff1d1431514b66386dfc749ca561b5 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_cpp-2026-04-18_21-33-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 853.646, + "avg tokens": 1468.266, + "avg character count": 4752.824, + "parameters count": 24.01136128, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.25911980979617594, + "bpb": 0.2542079012510063, + "compression_rate": 3.1775987656375784, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_javascript-2026-04-18_21-35-01.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_javascript-2026-04-18_21-35-01.json new file mode 100644 index 0000000000000000000000000000000000000000..c61fef7654b0931605c0ef87691b5d373f02d98c --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_javascript-2026-04-18_21-35-01.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1003.1095, + "avg tokens": 1527.154, + "avg character count": 5202.974, + "parameters count": 24.01136128, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2781449803744181, + "bpb": 0.26961097064171824, + "compression_rate": 3.370137133021478, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_markdown-2026-04-18_21-36-12.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_markdown-2026-04-18_21-36-12.json new file mode 100644 index 0000000000000000000000000000000000000000..dbe830da1dec7d1e848a56105f86b2922f219621 --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_markdown-2026-04-18_21-36-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2044.2245, + "avg tokens": 1508.46, + "avg character count": 4927.866, + "parameters count": 24.01136128, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5984725535584208, + "bpb": 0.5507197821565968, + "compression_rate": 6.8839972769574596, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_other-2026-04-18_21-37-26.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_other-2026-04-18_21-37-26.json new file mode 100644 index 0000000000000000000000000000000000000000..5ec4d0d620ef04a0627005c1f497d8898b8d3ced --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_other-2026-04-18_21-37-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1004.673, + "avg tokens": 1576.382, + "avg character count": 5229.94, + "parameters count": 24.01136128, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2771421383065652, + "bpb": 0.27222690286644946, + "compression_rate": 3.4028362858306185, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_python-2026-04-18_21-38-33.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_python-2026-04-18_21-38-33.json new file mode 100644 index 0000000000000000000000000000000000000000..ebc8bc0f674d9bf09846e876afec7de9cd8d7f5a --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-github_python-2026-04-18_21-38-33.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1064.714, + "avg tokens": 1439.878, + "avg character count": 5274.818, + "parameters count": 24.01136128, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2912058023167911, + "bpb": 0.28276720181145576, + "compression_rate": 3.534590022643197, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-wikipedia_english-2026-04-18_21-39-13.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-wikipedia_english-2026-04-18_21-39-13.json new file mode 100644 index 0000000000000000000000000000000000000000..2b8a75244d58c44cec9b091c0f2ac625c9e990da --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-wikipedia_english-2026-04-18_21-39-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1283.068, + "avg tokens": 752.304, + "avg character count": 3127.092, + "parameters count": 24.01136128, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5919479953654451, + "bpb": 0.5888402456051351, + "compression_rate": 7.36050307006419, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_21-39-56.json b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_21-39-56.json new file mode 100644 index 0000000000000000000000000000000000000000..22dbfbf6a30a9afcf48570e5de08b047c1a7e2ae --- /dev/null +++ b/data/2026-04/Mistral-Small-3.1-24B-Base-2503-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_21-39-56.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1297.978, + "avg tokens": 875.59, + "avg character count": 2498.564, + "parameters count": 24.01136128, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "tokenizer_name": "mistralai/Mistral-Small-3.1-24B-Base-2503", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "cuda:0", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7494650622449435, + "bpb": 0.5335648590550469, + "compression_rate": 6.669560738188086, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-ao3_english-2026-04-19_04-47-47.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-ao3_english-2026-04-19_04-47-47.json new file mode 100644 index 0000000000000000000000000000000000000000..04ad5d2a9fe29e17be2e02f457a85c8519cca970 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-ao3_english-2026-04-19_04-47-47.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4948.112, + "avg tokens": 2132.32, + "avg character count": 9041.522, + "parameters count": 31.577937344, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7895370540671326, + "bpb": 0.7747860344956403, + "compression_rate": 9.684825431195504, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_04-51-40.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_04-51-40.json new file mode 100644 index 0000000000000000000000000000000000000000..3faef4572c2a7df8dcefdbb4b76fb3388af15ec1 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_04-51-40.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 3689.048, + "avg tokens": 1588.088, + "avg character count": 2942.368, + "parameters count": 31.577937344, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.808805443507185, + "bpb": 1.014602650616815, + "compression_rate": 12.682533132710189, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_cs-2026-04-19_04-55-35.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_cs-2026-04-19_04-55-35.json new file mode 100644 index 0000000000000000000000000000000000000000..8f0ed385fdb48a42f73281853d66b6c5ec0bffbf --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_cs-2026-04-19_04-55-35.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4249.216, + "avg tokens": 2489.996, + "avg character count": 11865.894, + "parameters count": 31.577937344, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5166338794924376, + "bpb": 0.5162704942163097, + "compression_rate": 6.4533811777038705, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_math-2026-04-19_04-59-23.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_math-2026-04-19_04-59-23.json new file mode 100644 index 0000000000000000000000000000000000000000..543b7fe0ee6dc2614e1862ee2c54e82300957a8d --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_math-2026-04-19_04-59-23.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 2855.904, + "avg tokens": 2188.618, + "avg character count": 8435.788, + "parameters count": 31.577937344, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.48841892874203974, + "bpb": 0.4881728686185081, + "compression_rate": 6.102160857731351, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_other-2026-04-19_05-03-15.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_other-2026-04-19_05-03-15.json new file mode 100644 index 0000000000000000000000000000000000000000..33d8d8e202a2b9a500af90d8fe94fca7d08bf157 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_other-2026-04-19_05-03-15.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 3687.648, + "avg tokens": 2338.076, + "avg character count": 10817.836, + "parameters count": 31.577937344, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4917944293243218, + "bpb": 0.49145270596539203, + "compression_rate": 6.143158824567401, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_physics-2026-04-19_05-07-10.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_physics-2026-04-19_05-07-10.json new file mode 100644 index 0000000000000000000000000000000000000000..fcf1fd37957286412ba4475e4ba20a5d0881a191 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-arxiv_physics-2026-04-19_05-07-10.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 3913.248, + "avg tokens": 2599.718, + "avg character count": 11165.434, + "parameters count": 31.577937344, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5056340383516355, + "bpb": 0.505384275518206, + "compression_rate": 6.317303443977575, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-bbc_news-2026-04-19_05-10-55.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-bbc_news-2026-04-19_05-10-55.json new file mode 100644 index 0000000000000000000000000000000000000000..9aa7a4d6f17655e1de78b74d514765cd23849609 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-bbc_news-2026-04-19_05-10-55.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1218.184, + "avg tokens": 600.428, + "avg character count": 2745.97, + "parameters count": 31.577937344, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6400171945397368, + "bpb": 0.6395178714166446, + "compression_rate": 7.993973392708058, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-biorxiv_all-2026-04-19_05-14-51.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-biorxiv_all-2026-04-19_05-14-51.json new file mode 100644 index 0000000000000000000000000000000000000000..8ccb2083b293d4e52216aa2126416ca192ef7ea2 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-biorxiv_all-2026-04-19_05-14-51.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4154.32, + "avg tokens": 2543.524, + "avg character count": 11560.664, + "parameters count": 31.577937344, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5184318878453554, + "bpb": 0.5181861669622567, + "compression_rate": 6.477327087028209, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_cpp-2026-04-19_05-18-40.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_cpp-2026-04-19_05-18-40.json new file mode 100644 index 0000000000000000000000000000000000000000..b82cbd4e87cf34302c9344a79eafa791e868e9da --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_cpp-2026-04-19_05-18-40.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 850.015, + "avg tokens": 1468.258, + "avg character count": 4752.824, + "parameters count": 31.577937344, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2580176386041714, + "bpb": 0.2531266229583154, + "compression_rate": 3.1640827869789425, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_javascript-2026-04-19_05-23-09.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_javascript-2026-04-19_05-23-09.json new file mode 100644 index 0000000000000000000000000000000000000000..7df14008e65ec63c2db9676fad08172d41fca4db --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_javascript-2026-04-19_05-23-09.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1006.023, + "avg tokens": 1527.154, + "avg character count": 5202.974, + "parameters count": 31.577937344, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.27895284372365453, + "bpb": 0.27039404722803767, + "compression_rate": 3.379925590350471, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_markdown-2026-04-19_05-27-21.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_markdown-2026-04-19_05-27-21.json new file mode 100644 index 0000000000000000000000000000000000000000..bcb047ece1f83c11a4dfc01c269a4fe12a92ad42 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_markdown-2026-04-19_05-27-21.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 2019.7555, + "avg tokens": 1508.454, + "avg character count": 4927.866, + "parameters count": 31.577937344, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.591308944613796, + "bpb": 0.5441277653063977, + "compression_rate": 6.801597066329972, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_other-2026-04-19_05-31-06.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_other-2026-04-19_05-31-06.json new file mode 100644 index 0000000000000000000000000000000000000000..41284b903d7ddf6a9647ff2db2f7955902868a8e --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_other-2026-04-19_05-31-06.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1011.608, + "avg tokens": 1576.382, + "avg character count": 5229.94, + "parameters count": 31.577937344, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2790551793947163, + "bpb": 0.274106015345215, + "compression_rate": 3.4263251918151876, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_python-2026-04-19_05-34-53.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_python-2026-04-19_05-34-53.json new file mode 100644 index 0000000000000000000000000000000000000000..28733d7a7aacb1ac707a7ec9e55cf7b8a0b11617 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-github_python-2026-04-19_05-34-53.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1030.353, + "avg tokens": 1439.86, + "avg character count": 5274.818, + "parameters count": 31.577937344, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.28180785829294325, + "bpb": 0.2736415926606008, + "compression_rate": 3.4205199082575097, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-wikipedia_english-2026-04-19_05-38-35.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-wikipedia_english-2026-04-19_05-38-35.json new file mode 100644 index 0000000000000000000000000000000000000000..4d8478cec5038d8fdb3492fd5b0dd7d176f2f89f --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-wikipedia_english-2026-04-19_05-38-35.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1286.904, + "avg tokens": 752.304, + "avg character count": 3127.092, + "parameters count": 31.577937344, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.593717747639075, + "bpb": 0.5906007066112091, + "compression_rate": 7.382508832640114, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_05-42-17.json b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_05-42-17.json new file mode 100644 index 0000000000000000000000000000000000000000..9e0b0cc2adbf9d0bc94c843f291c9dc16d5e0431 --- /dev/null +++ b/data/2026-04/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_05-42-17.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1407.064, + "avg tokens": 875.59, + "avg character count": 2498.564, + "parameters count": 31.577937344, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "tokenizer_name": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8124523746493539, + "bpb": 0.5784072648699982, + "compression_rate": 7.230090810874977, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_15-27-45.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_15-27-45.json new file mode 100644 index 0000000000000000000000000000000000000000..8b9621f3e18fb8597f5b426f3114d0ac56f66bdd --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_15-27-45.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5842.824, + "avg tokens": 2198.884, + "avg character count": 9041.522, + "parameters count": 3.93363712, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9323002487398709, + "bpb": 0.9148819665391476, + "compression_rate": 11.436024581739344, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_15-28-07.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_15-28-07.json new file mode 100644 index 0000000000000000000000000000000000000000..43140b328e1400ec2b039dfe75b41a3e24b8260d --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_15-28-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4219.264, + "avg tokens": 1331.99, + "avg character count": 2942.368, + "parameters count": 3.93363712, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.0687797206200353, + "bpb": 1.1604285002667643, + "compression_rate": 14.505356253334552, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_15-28-39.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_15-28-39.json new file mode 100644 index 0000000000000000000000000000000000000000..663a1f5748eb8fecd016971d3585ec7f35bb8513 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_15-28-39.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4675.264, + "avg tokens": 2515.748, + "avg character count": 11865.894, + "parameters count": 3.93363712, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.568434218917403, + "bpb": 0.5680343987859692, + "compression_rate": 7.100429984824615, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_15-29-06.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_15-29-06.json new file mode 100644 index 0000000000000000000000000000000000000000..a5897625294a2dc906c3bcd6b14cb27018843f19 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_15-29-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3113.376, + "avg tokens": 2240.856, + "avg character count": 8435.788, + "parameters count": 3.93363712, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.532451990925177, + "bpb": 0.5321837474256896, + "compression_rate": 6.65229684282112, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_15-29-36.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_15-29-36.json new file mode 100644 index 0000000000000000000000000000000000000000..454bcafd86977f1f4159962408699c21893a4a70 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_15-29-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4030.536, + "avg tokens": 2374.546, + "avg character count": 10817.836, + "parameters count": 3.93363712, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5375228741981705, + "bpb": 0.5371493764293466, + "compression_rate": 6.714367205366832, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_15-30-07.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_15-30-07.json new file mode 100644 index 0000000000000000000000000000000000000000..9e0c3724d138a9ba34e77aa5a503c0ee0a74c0d6 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_15-30-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4321.192, + "avg tokens": 2614.854, + "avg character count": 11165.434, + "parameters count": 3.93363712, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5583448228818568, + "bpb": 0.5580690230455795, + "compression_rate": 6.975862788069744, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_15-30-22.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_15-30-22.json new file mode 100644 index 0000000000000000000000000000000000000000..05f5d8463edf593c65ba828846f61009dfeda246 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_15-30-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1403.892, + "avg tokens": 614.136, + "avg character count": 2745.97, + "parameters count": 3.93363712, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7375856350738313, + "bpb": 0.7370101918420009, + "compression_rate": 9.212627398025012, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_15-30-53.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_15-30-53.json new file mode 100644 index 0000000000000000000000000000000000000000..b501c08151791e13d80ad2bdd22f7f7388da7244 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_15-30-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4565.1, + "avg tokens": 2531.644, + "avg character count": 11560.664, + "parameters count": 3.93363712, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5696945375423251, + "bpb": 0.5694245197287158, + "compression_rate": 7.117806496608948, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_15-31-13.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_15-31-13.json new file mode 100644 index 0000000000000000000000000000000000000000..e021c9a85421f69c7101d99c1c9d9246408d5b3a --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_15-31-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1054.378, + "avg tokens": 1640.434, + "avg character count": 4752.824, + "parameters count": 3.93363712, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3200509658725893, + "bpb": 0.3139840384717242, + "compression_rate": 3.924800480896552, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_15-31-34.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_15-31-34.json new file mode 100644 index 0000000000000000000000000000000000000000..1061c668921c2336d62406281492b02d22c57c41 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_15-31-34.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1212.478, + "avg tokens": 1676.42, + "avg character count": 5202.974, + "parameters count": 3.93363712, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3361992579219055, + "bpb": 0.3258840340578264, + "compression_rate": 4.07355042572283, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_15-31-54.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_15-31-54.json new file mode 100644 index 0000000000000000000000000000000000000000..b538775c0357b0fb1e283bb43f2caee406e1b0dc --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_15-31-54.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2287.9055, + "avg tokens": 1552.914, + "avg character count": 4927.866, + "parameters count": 3.93363712, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.669813245504765, + "bpb": 0.6163681232442325, + "compression_rate": 7.704601540552907, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_15-32-15.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_15-32-15.json new file mode 100644 index 0000000000000000000000000000000000000000..7420b1b914df6fa30617b22963b1d59673ba3064 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_15-32-15.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1222.816, + "avg tokens": 1773.904, + "avg character count": 5229.94, + "parameters count": 3.93363712, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.337317556056031, + "bpb": 0.33133508360983155, + "compression_rate": 4.141688545122895, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_15-32-35.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_15-32-35.json new file mode 100644 index 0000000000000000000000000000000000000000..8da7b4f137d7043854d52eac84eca4809b5dd668 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_15-32-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1231.562, + "avg tokens": 1630.29, + "avg character count": 5274.818, + "parameters count": 3.93363712, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33683975256535736, + "bpb": 0.3270787653748519, + "compression_rate": 4.088484567185649, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_15-32-50.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_15-32-50.json new file mode 100644 index 0000000000000000000000000000000000000000..f8f2905b4647faed5c9e9f4e417af2067f78609e --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_15-32-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1540.456, + "avg tokens": 759.786, + "avg character count": 3127.092, + "parameters count": 3.93363712, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7106948666389249, + "bpb": 0.7069636912337491, + "compression_rate": 8.837046140421863, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_15-33-06.json b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_15-33-06.json new file mode 100644 index 0000000000000000000000000000000000000000..d47ca06abafe33f5dc76492ffdf6a0ccab327502 --- /dev/null +++ b/data/2026-04/Nanbeige4-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_15-33-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2059.616, + "avg tokens": 1185.3, + "avg character count": 2498.564, + "parameters count": 3.93363712, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "Nanbeige/Nanbeige4-3B-Base", + "tokenizer_name": "Nanbeige/Nanbeige4-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.1892422164633618, + "bpb": 0.8466543506496407, + "compression_rate": 10.583179383120509, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-ao3_english-2026-04-18_18-46-59.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-ao3_english-2026-04-18_18-46-59.json new file mode 100644 index 0000000000000000000000000000000000000000..4c17cbe25d693cf9be66d2f328e84be101e25499 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-ao3_english-2026-04-18_18-46-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4999.584, + "avg tokens": 2098.9, + "avg character count": 9041.522, + "parameters count": 32.233522176, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7977500959802792, + "bpb": 0.7828456311190715, + "compression_rate": 9.785570388988393, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_18-49-24.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_18-49-24.json new file mode 100644 index 0000000000000000000000000000000000000000..20148972c3641ef929472b776a5caba7e534cf99 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_18-49-24.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4179.512, + "avg tokens": 2032.472, + "avg character count": 2942.368, + "parameters count": 32.233522176, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.049288612347576, + "bpb": 1.1494954669835649, + "compression_rate": 14.36869333729456, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_18-52-17.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_18-52-17.json new file mode 100644 index 0000000000000000000000000000000000000000..8259300d99c3bfbd6e6d53b7d63bea91a8e8204f --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_18-52-17.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4416.88, + "avg tokens": 2452.726, + "avg character count": 11865.894, + "parameters count": 32.233522176, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5370190288402749, + "bpb": 0.5366413052417514, + "compression_rate": 6.708016315521892, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_math-2026-04-18_18-54-58.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_math-2026-04-18_18-54-58.json new file mode 100644 index 0000000000000000000000000000000000000000..df5b9904f24a58e6495d937094444ddb227e69e4 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_math-2026-04-18_18-54-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2930.032, + "avg tokens": 2284.334, + "avg character count": 8435.788, + "parameters count": 32.233522176, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5010963570974011, + "bpb": 0.5008439102238817, + "compression_rate": 6.260548877798522, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_other-2026-04-18_18-57-46.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_other-2026-04-18_18-57-46.json new file mode 100644 index 0000000000000000000000000000000000000000..b7a0b353c2421076b5351331be6d1b4257d898c3 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_other-2026-04-18_18-57-46.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3802.856, + "avg tokens": 2360.336, + "avg character count": 10817.836, + "parameters count": 32.233522176, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5071588709992313, + "bpb": 0.5068064716580126, + "compression_rate": 6.335080895725158, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_19-00-52.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_19-00-52.json new file mode 100644 index 0000000000000000000000000000000000000000..1fdebdbe20b5c08a212db423421177904a350864 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_19-00-52.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4103.872, + "avg tokens": 2642.216, + "avg character count": 11165.434, + "parameters count": 32.233522176, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5302647244023898, + "bpb": 0.5300027950028854, + "compression_rate": 6.625034937536068, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-bbc_news-2026-04-18_19-01-46.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-bbc_news-2026-04-18_19-01-46.json new file mode 100644 index 0000000000000000000000000000000000000000..addbe1ec4d884575c35b8325c85c043415bed152 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-bbc_news-2026-04-18_19-01-46.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1246.296, + "avg tokens": 578.15, + "avg character count": 2745.97, + "parameters count": 32.233522176, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6547868544375036, + "bpb": 0.6542760084478851, + "compression_rate": 8.178450105598564, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_19-04-45.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_19-04-45.json new file mode 100644 index 0000000000000000000000000000000000000000..c1ed60a660e5d2e0651b26eabc1617488e763567 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_19-04-45.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4259.672, + "avg tokens": 2555.634, + "avg character count": 11560.664, + "parameters count": 32.233522176, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5315791264423542, + "bpb": 0.5313271741696475, + "compression_rate": 6.641589677120593, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_cpp-2026-04-18_19-06-23.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_cpp-2026-04-18_19-06-23.json new file mode 100644 index 0000000000000000000000000000000000000000..659ad307e1dfa0e057205270a3e0624897ec9712 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_cpp-2026-04-18_19-06-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1065.674, + "avg tokens": 1318.424, + "avg character count": 4752.824, + "parameters count": 32.233522176, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3234798080055784, + "bpb": 0.3173478830308639, + "compression_rate": 3.966848537885799, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_javascript-2026-04-18_19-08-06.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_javascript-2026-04-18_19-08-06.json new file mode 100644 index 0000000000000000000000000000000000000000..19dc322f0bc19d0a0eab7950aced2f9f450ac1a4 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_javascript-2026-04-18_19-08-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1255.992, + "avg tokens": 1386.714, + "avg character count": 5202.974, + "parameters count": 32.233522176, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.34826494035838174, + "bpb": 0.3375795187247583, + "compression_rate": 4.219743984059479, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_markdown-2026-04-18_19-09-53.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_markdown-2026-04-18_19-09-53.json new file mode 100644 index 0000000000000000000000000000000000000000..80e5359819997d11d7c4d1c50746e93f5647aff8 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_markdown-2026-04-18_19-09-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2220.4425, + "avg tokens": 1456.428, + "avg character count": 4927.866, + "parameters count": 32.233522176, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6500625997803293, + "bpb": 0.5981934028729472, + "compression_rate": 7.47741753591184, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_other-2026-04-18_19-11-39.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_other-2026-04-18_19-11-39.json new file mode 100644 index 0000000000000000000000000000000000000000..dddcaac4dcd572e14fd69f174414f23a84ad2821 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_other-2026-04-18_19-11-39.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1249.924, + "avg tokens": 1441.496, + "avg character count": 5229.94, + "parameters count": 32.233522176, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3447953812640483, + "bpb": 0.33868028636028236, + "compression_rate": 4.233503579503529, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_python-2026-04-18_19-13-17.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_python-2026-04-18_19-13-17.json new file mode 100644 index 0000000000000000000000000000000000000000..c58e47d0cbf2716533cdc9c2488b90c4546aa24e --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-github_python-2026-04-18_19-13-17.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1208.81, + "avg tokens": 1323.066, + "avg character count": 5274.818, + "parameters count": 32.233522176, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33061694116782564, + "bpb": 0.32103627943438884, + "compression_rate": 4.01295349292986, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_19-14-18.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_19-14-18.json new file mode 100644 index 0000000000000000000000000000000000000000..dc0fe330873ca53253969c70b8f9a39d2ae307ad --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_19-14-18.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1365.46, + "avg tokens": 725.774, + "avg character count": 3127.092, + "parameters count": 32.233522176, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6299598382561958, + "bpb": 0.6266525248575975, + "compression_rate": 7.833156560719969, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_19-15-49.json b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_19-15-49.json new file mode 100644 index 0000000000000000000000000000000000000000..64d99fb9b1a0a2f961375f0e28678e4ce89a95f3 --- /dev/null +++ b/data/2026-04/Olmo-3-1125-32B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_19-15-49.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1785.02, + "avg tokens": 1214.888, + "avg character count": 2498.564, + "parameters count": 32.233522176, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "allenai/Olmo-3-1125-32B", + "tokenizer_name": "allenai/Olmo-3-1125-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.0306878278433602, + "bpb": 0.7337751061346491, + "compression_rate": 9.172188826683115, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-ao3_english-2026-04-18_21-43-10.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-ao3_english-2026-04-18_21-43-10.json new file mode 100644 index 0000000000000000000000000000000000000000..784e6d20104c15c0b42eddcef2ec5fd170672b4f --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-ao3_english-2026-04-18_21-43-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4953.192, + "avg tokens": 2099.312, + "avg character count": 9041.522, + "parameters count": 32.763876352, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7903476356050326, + "bpb": 0.7755814718372441, + "compression_rate": 9.694768397965552, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_21-44-42.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_21-44-42.json new file mode 100644 index 0000000000000000000000000000000000000000..1e1e602a6e91c9578da222f897f2ca7fab8cf785 --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_21-44-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3452.136, + "avg tokens": 1344.516, + "avg character count": 2942.368, + "parameters count": 32.763876352, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.6926433021546803, + "bpb": 0.9494445005567099, + "compression_rate": 11.868056256958875, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_21-47-25.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_21-47-25.json new file mode 100644 index 0000000000000000000000000000000000000000..f78c41e4abc9c78b277d94c7f35095f102f14c46 --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_21-47-25.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4383.888, + "avg tokens": 2492.906, + "avg character count": 11865.894, + "parameters count": 32.763876352, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5330077512417215, + "bpb": 0.5326328490594381, + "compression_rate": 6.657910613242976, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_math-2026-04-18_21-49-56.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_math-2026-04-18_21-49-56.json new file mode 100644 index 0000000000000000000000000000000000000000..43412165899fef05a962f38ac0180699af9be69f --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_math-2026-04-18_21-49-56.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2716.544, + "avg tokens": 2299.142, + "avg character count": 8435.788, + "parameters count": 32.763876352, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4645854728872594, + "bpb": 0.4643514197985634, + "compression_rate": 5.8043927474820425, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-52-32.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-52-32.json new file mode 100644 index 0000000000000000000000000000000000000000..29043825c65efc5c865cc276c1059951e0f74eae --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-52-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3685.784, + "avg tokens": 2392.214, + "avg character count": 10817.836, + "parameters count": 32.763876352, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4915458413852722, + "bpb": 0.49120429075767164, + "compression_rate": 6.140053634470895, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-55-27.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-55-27.json new file mode 100644 index 0000000000000000000000000000000000000000..ebc3e45438b2ca1c6df4e46cd19baa8d37c4dcfb --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-55-27.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3933.584, + "avg tokens": 2685.326, + "avg character count": 11165.434, + "parameters count": 32.763876352, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5082616698750959, + "bpb": 0.5080106090976106, + "compression_rate": 6.350132613720133, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-bbc_news-2026-04-18_21-56-13.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-bbc_news-2026-04-18_21-56-13.json new file mode 100644 index 0000000000000000000000000000000000000000..f38e1dba6c292d4832e1cec74dbf606654c0fa8c --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-bbc_news-2026-04-18_21-56-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1241.776, + "avg tokens": 589.862, + "avg character count": 2745.97, + "parameters count": 32.763876352, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6524121083241745, + "bpb": 0.6519031150436021, + "compression_rate": 8.148788938045026, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-59-05.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-59-05.json new file mode 100644 index 0000000000000000000000000000000000000000..c512d327e60b61bb23628bd9551a6066c1cf128c --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-59-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4170.352, + "avg tokens": 2635.91, + "avg character count": 11560.664, + "parameters count": 32.763876352, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5204325762915841, + "bpb": 0.5201859071432584, + "compression_rate": 6.50232383929073, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_cpp-2026-04-18_22-00-36.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_cpp-2026-04-18_22-00-36.json new file mode 100644 index 0000000000000000000000000000000000000000..21ec9bb4d5ab6dba6c65485074a6960b265b192e --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_cpp-2026-04-18_22-00-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 861.1945, + "avg tokens": 1340.222, + "avg character count": 4752.824, + "parameters count": 32.763876352, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.26141111776721593, + "bpb": 0.25645577489253124, + "compression_rate": 3.2056971861566406, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_javascript-2026-04-18_22-02-12.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_javascript-2026-04-18_22-02-12.json new file mode 100644 index 0000000000000000000000000000000000000000..b02b8d46b1b9a446692212d170fd2601ce500abd --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_javascript-2026-04-18_22-02-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1022.2945, + "avg tokens": 1404.042, + "avg character count": 5202.974, + "parameters count": 32.763876352, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.28346465030923895, + "bpb": 0.2747674231244844, + "compression_rate": 3.4345927890560546, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_markdown-2026-04-18_22-03-48.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_markdown-2026-04-18_22-03-48.json new file mode 100644 index 0000000000000000000000000000000000000000..c65bee9e1600845f445dc17f87f7562622cd6152 --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_markdown-2026-04-18_22-03-48.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2099.7425, + "avg tokens": 1426.42, + "avg character count": 4927.866, + "parameters count": 32.763876352, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6147261495937174, + "bpb": 0.5656764862102708, + "compression_rate": 7.070956077628384, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_other-2026-04-18_22-05-29.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_other-2026-04-18_22-05-29.json new file mode 100644 index 0000000000000000000000000000000000000000..ff29c722d956e716c4f5295e5c65e283d59bf067 --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_other-2026-04-18_22-05-29.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1007.649, + "avg tokens": 1478.744, + "avg character count": 5229.94, + "parameters count": 32.763876352, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.27796307706335505, + "bpb": 0.2730332819200625, + "compression_rate": 3.412916024000781, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_python-2026-04-18_22-07-00.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_python-2026-04-18_22-07-00.json new file mode 100644 index 0000000000000000000000000000000000000000..ddb0f2238ef71f7f9f69f10989f962a430d91bd1 --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-github_python-2026-04-18_22-07-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1060.267, + "avg tokens": 1326.794, + "avg character count": 5274.818, + "parameters count": 32.763876352, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2899895205707986, + "bpb": 0.281586165639812, + "compression_rate": 3.5198270704976498, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_22-07-59.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_22-07-59.json new file mode 100644 index 0000000000000000000000000000000000000000..29187194d8af32094211a4323b0b6eb1844a38a9 --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_22-07-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1354.932, + "avg tokens": 759.072, + "avg character count": 3127.092, + "parameters count": 32.763876352, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.625102707928569, + "bpb": 0.6218208946511463, + "compression_rate": 7.772761183139329, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_22-09-11.json b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_22-09-11.json new file mode 100644 index 0000000000000000000000000000000000000000..47ea5bd6712ce156777b44c0547505b4f132dbee --- /dev/null +++ b/data/2026-04/Qwen2.5-32B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_22-09-11.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1443.132, + "avg tokens": 1022.712, + "avg character count": 2498.564, + "parameters count": 32.763876352, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-32B", + "tokenizer_name": "Qwen/Qwen2.5-32B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8332783870047642, + "bpb": 0.5932338777528031, + "compression_rate": 7.415423471910039, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-ao3_english-2026-04-18_16-33-07.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-ao3_english-2026-04-18_16-33-07.json new file mode 100644 index 0000000000000000000000000000000000000000..a53bedc2ddd8022c8c56479ea1fcb0a7bbfbbd55 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-ao3_english-2026-04-18_16-33-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5548.944, + "avg tokens": 2099.312, + "avg character count": 9041.522, + "parameters count": 3.085938688, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8854077876457711, + "bpb": 0.8688656031630603, + "compression_rate": 10.860820039538254, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-33-27.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-33-27.json new file mode 100644 index 0000000000000000000000000000000000000000..3c90c791c6be3cad570e1b116e0b6214e64305b4 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-33-27.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3943.68, + "avg tokens": 1344.516, + "avg character count": 2942.368, + "parameters count": 3.085938688, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.9336560140855894, + "bpb": 1.0846343504298457, + "compression_rate": 13.557929380373071, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-33-56.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-33-56.json new file mode 100644 index 0000000000000000000000000000000000000000..09bc4b6ecc89159cdcc4102d61f65c0611d019f1 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-33-56.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5018.928, + "avg tokens": 2492.906, + "avg character count": 11865.894, + "parameters count": 3.085938688, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6102180363467568, + "bpb": 0.6097888266908706, + "compression_rate": 7.622360333635883, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-34-23.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-34-23.json new file mode 100644 index 0000000000000000000000000000000000000000..3098bc30d311c575ba445358f02a9e35f0382284 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-34-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3141.2, + "avg tokens": 2299.142, + "avg character count": 8435.788, + "parameters count": 3.085938688, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5372104730987088, + "bpb": 0.5369398323278576, + "compression_rate": 6.711747904098219, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-34-49.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-34-49.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ca3f5595d9947080448174eaca2908e7ecc69e --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-34-49.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4224.728, + "avg tokens": 2392.214, + "avg character count": 10817.836, + "parameters count": 3.085938688, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5634208297024237, + "bpb": 0.5630293367392328, + "compression_rate": 7.037866709240411, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-35-19.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-35-19.json new file mode 100644 index 0000000000000000000000000000000000000000..6c39c108666c0b7c8422f7d15557fd66071f83da --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-35-19.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4561.408, + "avg tokens": 2685.326, + "avg character count": 11165.434, + "parameters count": 3.085938688, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5893833326202318, + "bpb": 0.5890922010112697, + "compression_rate": 7.363652512640871, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-bbc_news-2026-04-18_16-35-36.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-bbc_news-2026-04-18_16-35-36.json new file mode 100644 index 0000000000000000000000000000000000000000..865656644df1f0986a850115547b804456971ee6 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-bbc_news-2026-04-18_16-35-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1379.012, + "avg tokens": 589.862, + "avg character count": 2745.97, + "parameters count": 3.085938688, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7245140237243565, + "bpb": 0.7239487785901062, + "compression_rate": 9.049359732376328, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-36-05.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-36-05.json new file mode 100644 index 0000000000000000000000000000000000000000..adf118e9d827b936dbe13c354c4163834d211a5d --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-36-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4782.552, + "avg tokens": 2635.91, + "avg character count": 11560.664, + "parameters count": 3.085938688, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5968311208762397, + "bpb": 0.5965482411508201, + "compression_rate": 7.456853014385251, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_cpp-2026-04-18_16-36-26.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_cpp-2026-04-18_16-36-26.json new file mode 100644 index 0000000000000000000000000000000000000000..46ed638571ddca128b85e0239d5b0abbf056e148 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_cpp-2026-04-18_16-36-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1067.352, + "avg tokens": 1340.222, + "avg character count": 4752.824, + "parameters count": 3.085938688, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.323989156190702, + "bpb": 0.3178475759460761, + "compression_rate": 3.973094699325951, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_javascript-2026-04-18_16-36-46.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_javascript-2026-04-18_16-36-46.json new file mode 100644 index 0000000000000000000000000000000000000000..9d89b3c29996f6f3ba6f0104cb7bd147c486220d --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_javascript-2026-04-18_16-36-46.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1271.431, + "avg tokens": 1404.042, + "avg character count": 5202.974, + "parameters count": 3.085938688, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.35254590903827226, + "bpb": 0.3417291392554556, + "compression_rate": 4.271614240693196, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_markdown-2026-04-18_16-37-06.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_markdown-2026-04-18_16-37-06.json new file mode 100644 index 0000000000000000000000000000000000000000..514fc131a701bfe64f4ae77022f382ad4fb77653 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_markdown-2026-04-18_16-37-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2513.631, + "avg tokens": 1426.42, + "avg character count": 4927.866, + "parameters count": 3.085938688, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.735897237937226, + "bpb": 0.67717920254946, + "compression_rate": 8.46474003186825, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_other-2026-04-18_16-37-25.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_other-2026-04-18_16-37-25.json new file mode 100644 index 0000000000000000000000000000000000000000..2de45de7d7cd3bb6d6fe16ae13389ce4bc32eb7f --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_other-2026-04-18_16-37-25.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1243.13, + "avg tokens": 1478.744, + "avg character count": 5229.94, + "parameters count": 3.085938688, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3429212354597371, + "bpb": 0.33683937934071023, + "compression_rate": 4.210492241758878, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_python-2026-04-18_16-37-44.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_python-2026-04-18_16-37-44.json new file mode 100644 index 0000000000000000000000000000000000000000..3136cefd52f7f86d2834bb4a748a241d1bb69854 --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-github_python-2026-04-18_16-37-44.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1279.92, + "avg tokens": 1326.794, + "avg character count": 5274.818, + "parameters count": 3.085938688, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.35006596184638067, + "bpb": 0.3399217038026348, + "compression_rate": 4.249021297532935, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-38-04.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-38-04.json new file mode 100644 index 0000000000000000000000000000000000000000..e186f0b02831cb5161dada57f1de5bc0b0c708ff --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-38-04.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1543.14, + "avg tokens": 759.072, + "avg character count": 3127.092, + "parameters count": 3.085938688, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7119331396061884, + "bpb": 0.7081954632202723, + "compression_rate": 8.852443290253403, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-38-22.json b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-38-22.json new file mode 100644 index 0000000000000000000000000000000000000000..2e5e8e4d5d46a81409a95a6f97f4b964e4f6359a --- /dev/null +++ b/data/2026-04/Qwen2.5-3B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-38-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1785.344, + "avg tokens": 1022.712, + "avg character count": 2498.564, + "parameters count": 3.085938688, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen2.5-3B", + "tokenizer_name": "Qwen/Qwen2.5-3B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.0308749085798345, + "bpb": 0.7339082940733769, + "compression_rate": 9.173853675917211, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_18-31-08.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_18-31-08.json new file mode 100644 index 0000000000000000000000000000000000000000..ef18a5717c24bd423613671591bcfc2a97907eaa --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_18-31-08.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5015.936, + "avg tokens": 2099.312, + "avg character count": 9041.522, + "parameters count": 30.532122624, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8003592749778657, + "bpb": 0.785406062498974, + "compression_rate": 9.817575781237176, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_18-31-56.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_18-31-56.json new file mode 100644 index 0000000000000000000000000000000000000000..e272e90004480211fa374f69fe7937519095eea3 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_18-31-56.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3472.072, + "avg tokens": 1344.516, + "avg character count": 2942.368, + "parameters count": 30.532122624, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.702418275351494, + "bpb": 0.9549275190597755, + "compression_rate": 11.936593988247193, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_18-33-02.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_18-33-02.json new file mode 100644 index 0000000000000000000000000000000000000000..592e37fdd9aeafc63e0daacb234c29bf5e0aef1c --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_18-33-02.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4398.656, + "avg tokens": 2492.906, + "avg character count": 11865.894, + "parameters count": 30.532122624, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5348032940271069, + "bpb": 0.5344271289121418, + "compression_rate": 6.680339111401773, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_18-34-04.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_18-34-04.json new file mode 100644 index 0000000000000000000000000000000000000000..90046c01c2fad07839a884c587d6a74fe4e9e467 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_18-34-04.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2731.656, + "avg tokens": 2299.142, + "avg character count": 8435.788, + "parameters count": 30.532122624, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4671699389096291, + "bpb": 0.4669345837951694, + "compression_rate": 5.8366822974396175, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_18-35-07.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_18-35-07.json new file mode 100644 index 0000000000000000000000000000000000000000..a1192400189a2235b0a7c195211fea666612302c --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_18-35-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3700.824, + "avg tokens": 2392.214, + "avg character count": 10817.836, + "parameters count": 30.532122624, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.49355161531408476, + "bpb": 0.49320867097447096, + "compression_rate": 6.165108387180887, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_18-36-15.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_18-36-15.json new file mode 100644 index 0000000000000000000000000000000000000000..e712fb63d69962d49c0a24092505bae33d8c175c --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_18-36-15.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3938.056, + "avg tokens": 2685.326, + "avg character count": 11165.434, + "parameters count": 30.532122624, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5088395007254558, + "bpb": 0.508588154522822, + "compression_rate": 6.357351931535275, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_18-36-55.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_18-36-55.json new file mode 100644 index 0000000000000000000000000000000000000000..fb386fb897dd0432990b95bb1960d9ee7ed24190 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_18-36-55.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1277.176, + "avg tokens": 589.862, + "avg character count": 2745.97, + "parameters count": 30.532122624, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6710107836365302, + "bpb": 0.6704872802010406, + "compression_rate": 8.381091002513008, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_18-38-02.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_18-38-02.json new file mode 100644 index 0000000000000000000000000000000000000000..bd7a066a2d996bef78ec86c2355ec89a19baeb3d --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_18-38-02.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4209.688, + "avg tokens": 2635.91, + "avg character count": 11560.664, + "parameters count": 30.532122624, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5253414510870464, + "bpb": 0.5250924552819737, + "compression_rate": 6.563655691024671, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_18-38-52.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_18-38-52.json new file mode 100644 index 0000000000000000000000000000000000000000..afedea14a2cb0dd621000fcbaf7aaee60e26a184 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_18-38-52.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 839.9885, + "avg tokens": 1340.222, + "avg character count": 4752.824, + "parameters count": 30.532122624, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2549741466028953, + "bpb": 0.2501408237840755, + "compression_rate": 3.1267602973009434, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_18-39-42.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_18-39-42.json new file mode 100644 index 0000000000000000000000000000000000000000..577a633b4803230c77cc0da82020c3b911cd5c20 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_18-39-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 985.2885, + "avg tokens": 1404.042, + "avg character count": 5202.974, + "parameters count": 30.532122624, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2732035241373348, + "bpb": 0.26482112755100273, + "compression_rate": 3.3102640943875343, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_18-40-32.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_18-40-32.json new file mode 100644 index 0000000000000000000000000000000000000000..ce25d215f227760d6d9dfcee838fd37bf5c7df35 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_18-40-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2111.19, + "avg tokens": 1426.42, + "avg character count": 4927.866, + "parameters count": 30.532122624, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6180775498713583, + "bpb": 0.5687604746402294, + "compression_rate": 7.109505933002867, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_18-41-23.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_18-41-23.json new file mode 100644 index 0000000000000000000000000000000000000000..ea4170dea1aa873a71f2189b94404db6a0085710 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_18-41-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1003.238, + "avg tokens": 1478.744, + "avg character count": 5229.94, + "parameters count": 30.532122624, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.27674628914124483, + "bpb": 0.2718380742569284, + "compression_rate": 3.397975928211605, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_18-42-13.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_18-42-13.json new file mode 100644 index 0000000000000000000000000000000000000000..d18a41728daf214f7806784ccbaa72b9ec03e6eb --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_18-42-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1049.914, + "avg tokens": 1326.794, + "avg character count": 5274.818, + "parameters count": 30.532122624, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.28715791163977505, + "bpb": 0.2788366114493402, + "compression_rate": 3.4854576431167525, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_18-42-52.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_18-42-52.json new file mode 100644 index 0000000000000000000000000000000000000000..5e1a015a286f6f0acb0e447c36bd780880e31bca --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_18-42-52.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1371.8, + "avg tokens": 759.072, + "avg character count": 3127.092, + "parameters count": 30.532122624, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6328848198554695, + "bpb": 0.6295621501908897, + "compression_rate": 7.869526877386121, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_18-43-36.json b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_18-43-36.json new file mode 100644 index 0000000000000000000000000000000000000000..ee7f7783c23ce1c41d96896d749d620f3a4417b1 --- /dev/null +++ b/data/2026-04/Qwen3-30B-A3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_18-43-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1436.652, + "avg tokens": 1022.712, + "avg character count": 2498.564, + "parameters count": 30.532122624, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-30B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3-30B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8295367722752793, + "bpb": 0.5905701189782501, + "compression_rate": 7.382126487228126, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-07-51.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-07-51.json new file mode 100644 index 0000000000000000000000000000000000000000..44d92724b9c41811bc5201cb171bd65a40c4a52d --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-07-51.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5447.12, + "avg tokens": 2099.312, + "avg character count": 9041.522, + "parameters count": 4.022468096, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8691604147097235, + "bpb": 0.8529217819285199, + "compression_rate": 10.661522274106499, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-08-15.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-08-15.json new file mode 100644 index 0000000000000000000000000000000000000000..9f1897a4fc16e93993d4dc62bfae88b356d49b24 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-08-15.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3810.304, + "avg tokens": 1344.516, + "avg character count": 2942.368, + "parameters count": 4.022468096, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.8682594036773716, + "bpb": 1.047951812515276, + "compression_rate": 13.099397656440951, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-08-54.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-08-54.json new file mode 100644 index 0000000000000000000000000000000000000000..44dbc666a45b2b30f099e23c6fcb16ecc69c4ca9 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-08-54.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4699.76, + "avg tokens": 2492.906, + "avg character count": 11865.894, + "parameters count": 4.022468096, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5714125244476577, + "bpb": 0.5710106094625558, + "compression_rate": 7.137632618281947, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-09-30.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-09-30.json new file mode 100644 index 0000000000000000000000000000000000000000..e4a38ea1470607fe9469075911dc9bf556d9ad26 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-09-30.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2959.088, + "avg tokens": 2299.142, + "avg character count": 8435.788, + "parameters count": 4.022468096, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5060655368714863, + "bpb": 0.5058105865794523, + "compression_rate": 6.322632332243154, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-10-08.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-10-08.json new file mode 100644 index 0000000000000000000000000000000000000000..935b1d1bed50600661cd34c4352e3c74941bdaf1 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-10-08.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3982.888, + "avg tokens": 2392.214, + "avg character count": 10817.836, + "parameters count": 4.022468096, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.531168411687528, + "bpb": 0.530799329316976, + "compression_rate": 6.634991616462201, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-10-49.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-10-49.json new file mode 100644 index 0000000000000000000000000000000000000000..1732a9188502d80230e3727cf588e621aa4daf30 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-10-49.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4278.408, + "avg tokens": 2685.326, + "avg character count": 11165.434, + "parameters count": 4.022468096, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5528166665531916, + "bpb": 0.5525435974033072, + "compression_rate": 6.906794967541339, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-11-08.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-11-08.json new file mode 100644 index 0000000000000000000000000000000000000000..a9092b92b84e68dd00bef9da1a8c418dc0665829 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-11-08.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1384.512, + "avg tokens": 589.862, + "avg character count": 2745.97, + "parameters count": 4.022468096, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.727403648419779, + "bpb": 0.7268361488829286, + "compression_rate": 9.085451861036606, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-11-49.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-11-49.json new file mode 100644 index 0000000000000000000000000000000000000000..da3c0e8db2efc9e34d1010320a5a7f5152a863ad --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-11-49.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4550.504, + "avg tokens": 2635.91, + "avg character count": 11560.664, + "parameters count": 4.022468096, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5678730524773828, + "bpb": 0.5676038979920702, + "compression_rate": 7.095048724900878, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-12-14.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-12-14.json new file mode 100644 index 0000000000000000000000000000000000000000..43453d03009bcdf773a911ccf4982794a17eb556 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-12-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 962.8595, + "avg tokens": 1340.222, + "avg character count": 4752.824, + "parameters count": 4.022468096, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2922710005089242, + "bpb": 0.2867306737155604, + "compression_rate": 3.5841334214445046, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-12-40.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-12-40.json new file mode 100644 index 0000000000000000000000000000000000000000..31f81ac7a3b5bb043adf7a74ada132638419149b --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-12-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1129.5675, + "avg tokens": 1404.042, + "avg character count": 5202.974, + "parameters count": 4.022468096, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.31320960485279076, + "bpb": 0.30359974666807465, + "compression_rate": 3.7949968333509334, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-13-06.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-13-06.json new file mode 100644 index 0000000000000000000000000000000000000000..58dc9cb6157877fcbca62cc7a0c9fa94c9c65d65 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-13-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2296.257, + "avg tokens": 1426.42, + "avg character count": 4927.866, + "parameters count": 4.022468096, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6722582526608005, + "bpb": 0.6186180406386679, + "compression_rate": 7.732725507983348, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-13-32.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-13-32.json new file mode 100644 index 0000000000000000000000000000000000000000..04bedda38d08399f7b19e7c7f9df325a61449c7e --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-13-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1128.354, + "avg tokens": 1478.744, + "avg character count": 5229.94, + "parameters count": 4.022468096, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.31125992270795183, + "bpb": 0.30573959363590913, + "compression_rate": 3.821744920448864, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-13-57.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-13-57.json new file mode 100644 index 0000000000000000000000000000000000000000..753d9f021059dcd863cffff7c6ae424a898b50e3 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-13-57.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1168.183, + "avg tokens": 1326.794, + "avg character count": 5274.818, + "parameters count": 4.022468096, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.31950520775328967, + "bpb": 0.3102465433099516, + "compression_rate": 3.8780817913743952, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-14-16.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-14-16.json new file mode 100644 index 0000000000000000000000000000000000000000..79654391a660bb72ce9b922c014594ae09bba906 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-14-16.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1516.248, + "avg tokens": 759.072, + "avg character count": 3127.092, + "parameters count": 4.022468096, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6995264195481965, + "bpb": 0.6958538789201313, + "compression_rate": 8.698173486501641, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-14-37.json b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-14-37.json new file mode 100644 index 0000000000000000000000000000000000000000..b07a36f7f0d6f941cb6ae6eee48d9c2662a47b14 --- /dev/null +++ b/data/2026-04/Qwen3-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-14-37.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1667.864, + "avg tokens": 1022.712, + "avg character count": 2498.564, + "parameters count": 4.022468096, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3-4B-Base", + "tokenizer_name": "Qwen/Qwen3-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9630408193175082, + "bpb": 0.6856153340680555, + "compression_rate": 8.570191675850694, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_09-31-13.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_09-31-13.json new file mode 100644 index 0000000000000000000000000000000000000000..6f4fcd4c7af0f3bc7927ffc20f1659102fd9a893 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_09-31-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4964.112, + "avg tokens": 2143.744, + "avg character count": 9041.522, + "parameters count": 34.660610688, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7920900667849278, + "bpb": 0.7772913489573846, + "compression_rate": 9.716141861967307, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_09-32-00.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_09-32-00.json new file mode 100644 index 0000000000000000000000000000000000000000..22558eff7a40cbdd5213e3566f4e0243a2bbf3ea --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_09-32-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3365.016, + "avg tokens": 1215.72, + "avg character count": 2942.368, + "parameters count": 34.660610688, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.6499268261862607, + "bpb": 0.9254837977082415, + "compression_rate": 11.56854747135302, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_09-33-01.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_09-33-01.json new file mode 100644 index 0000000000000000000000000000000000000000..c51aee6cc9789aa04fc7b8be684dbba92ed1dab0 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_09-33-01.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4079.072, + "avg tokens": 2506.012, + "avg character count": 11865.894, + "parameters count": 34.660610688, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4959472034579969, + "bpb": 0.49559836858938466, + "compression_rate": 6.1949796073673085, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-33-58.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-33-58.json new file mode 100644 index 0000000000000000000000000000000000000000..527bf038c76c1f7ff98a2237b332e280cb62e4c3 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-33-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2684.384, + "avg tokens": 2311.45, + "avg character count": 8435.788, + "parameters count": 34.660610688, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.45908544461307926, + "bpb": 0.4588541623785763, + "compression_rate": 5.735677029732203, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-34-56.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-34-56.json new file mode 100644 index 0000000000000000000000000000000000000000..c05602ad3c14eb19c4777f3f67c1881814bf7632 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-34-56.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3517.072, + "avg tokens": 2403.796, + "avg character count": 10817.836, + "parameters count": 34.660610688, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.46904596564871465, + "bpb": 0.46872004905975656, + "compression_rate": 5.859000613246957, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-36-00.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-36-00.json new file mode 100644 index 0000000000000000000000000000000000000000..0e9f45dbe610fe270db758c22aa0af2a53b33515 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-36-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3768.176, + "avg tokens": 2692.576, + "avg character count": 11165.434, + "parameters count": 34.660610688, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.48688916421849887, + "bpb": 0.48664866059730727, + "compression_rate": 6.083108257466341, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_09-36-45.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_09-36-45.json new file mode 100644 index 0000000000000000000000000000000000000000..e520a67cfecebd0904c5ea69b421b4b67bfcb5d1 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_09-36-45.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1266.472, + "avg tokens": 603.514, + "avg character count": 2745.97, + "parameters count": 34.660610688, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6653870485929297, + "bpb": 0.6648679326347914, + "compression_rate": 8.310849157934893, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-37-47.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-37-47.json new file mode 100644 index 0000000000000000000000000000000000000000..e057c8b66dd6110d7195742ec043bdf898976892 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-37-47.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4067.374, + "avg tokens": 2637.476, + "avg character count": 11560.664, + "parameters count": 34.660610688, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5075815973235366, + "bpb": 0.5073410191468019, + "compression_rate": 6.3417627393350235, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_09-38-40.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_09-38-40.json new file mode 100644 index 0000000000000000000000000000000000000000..f9b2b6cb9eef3efa71a81f5f9fdccc4758b421d1 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_09-38-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 816.3365, + "avg tokens": 1450.036, + "avg character count": 4752.824, + "parameters count": 34.660610688, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.24779470484214297, + "bpb": 0.24309747644760485, + "compression_rate": 3.0387184555950606, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_09-39-30.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_09-39-30.json new file mode 100644 index 0000000000000000000000000000000000000000..09664517ffc843fbbced644156da844416f4c5ff --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_09-39-30.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 885.77375, + "avg tokens": 1516.922, + "avg character count": 5202.974, + "parameters count": 34.660610688, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.24560979864105034, + "bpb": 0.238074029312308, + "compression_rate": 2.97592536640385, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_09-40-19.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_09-40-19.json new file mode 100644 index 0000000000000000000000000000000000000000..764528e5cdc453a188dff89b9757c518d47a7aa9 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_09-40-19.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1869.91225, + "avg tokens": 1457.148, + "avg character count": 4927.866, + "parameters count": 34.660610688, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5474404397304073, + "bpb": 0.5037595758058627, + "compression_rate": 6.296994697573284, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_other-2026-04-19_09-41-10.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_other-2026-04-19_09-41-10.json new file mode 100644 index 0000000000000000000000000000000000000000..81c068318f41dff7078064538b720c7f2210b5ca --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_other-2026-04-19_09-41-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 915.2975, + "avg tokens": 1590.85, + "avg character count": 5229.94, + "parameters count": 34.660610688, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2524876316340276, + "bpb": 0.24800965451087473, + "compression_rate": 3.100120681385934, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_python-2026-04-19_09-41-58.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_python-2026-04-19_09-41-58.json new file mode 100644 index 0000000000000000000000000000000000000000..926ac6a1e9959056cb4d5c18e8709e3f73971b56 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-github_python-2026-04-19_09-41-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 968.314, + "avg tokens": 1418.482, + "avg character count": 5274.818, + "parameters count": 34.660610688, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2648398116908215, + "bpb": 0.2571652483717298, + "compression_rate": 3.214565604646623, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-42-41.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-42-41.json new file mode 100644 index 0000000000000000000000000000000000000000..49ee939e7baff04438c570487b0709a372f6d721 --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-42-41.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1325.65, + "avg tokens": 759.516, + "avg character count": 3127.092, + "parameters count": 34.660610688, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6115933528512927, + "bpb": 0.6083824642080136, + "compression_rate": 7.604780802600169, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-43-29.json b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-43-29.json new file mode 100644 index 0000000000000000000000000000000000000000..4332ce2367d2c817c1136a1fdb6e547d519b950f --- /dev/null +++ b/data/2026-04/Qwen3.5-35B-A3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-43-29.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1308.028, + "avg tokens": 835.802, + "avg character count": 2498.564, + "parameters count": 34.660610688, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-35B-A3B-Base", + "tokenizer_name": "Qwen/Qwen3.5-35B-A3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7552680295337278, + "bpb": 0.5376961515989137, + "compression_rate": 6.721201894986422, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_15-59-05.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_15-59-05.json new file mode 100644 index 0000000000000000000000000000000000000000..168303c905948452806d1a53999ef7f454876e8b --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_15-59-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5432.624, + "avg tokens": 2143.744, + "avg character count": 9041.522, + "parameters count": 4.205751296, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8668473851874012, + "bpb": 0.8506519670261795, + "compression_rate": 10.633149587827244, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_15-59-42.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_15-59-42.json new file mode 100644 index 0000000000000000000000000000000000000000..70b08edfccb38029fcf4323ee6350462b7d62984 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_15-59-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3718.56, + "avg tokens": 1215.72, + "avg character count": 2942.368, + "parameters count": 4.205751296, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.8232756987732546, + "bpb": 1.0227193662098362, + "compression_rate": 12.783992077622951, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-00-26.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-00-26.json new file mode 100644 index 0000000000000000000000000000000000000000..66bd583af46f5457b6c8ddbe87bc755b3ca7d72f --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-00-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4418.64, + "avg tokens": 2506.012, + "avg character count": 11865.894, + "parameters count": 4.205751296, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5372330155210902, + "bpb": 0.536855141410546, + "compression_rate": 6.7106892676318255, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-01-05.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-01-05.json new file mode 100644 index 0000000000000000000000000000000000000000..b6872eda7292d549197357b2572313f922edf685 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-01-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2958.264, + "avg tokens": 2311.45, + "avg character count": 8435.788, + "parameters count": 4.205751296, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5059246157490382, + "bpb": 0.5056697364515272, + "compression_rate": 6.32087170564409, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-01-42.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-01-42.json new file mode 100644 index 0000000000000000000000000000000000000000..0269525fd620e99f1f39c7962bc0ba8519881094 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-01-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3831.096, + "avg tokens": 2403.796, + "avg character count": 10817.836, + "parameters count": 4.205751296, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5109250316208847, + "bpb": 0.5105700153629602, + "compression_rate": 6.382125192037003, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-02-21.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-02-21.json new file mode 100644 index 0000000000000000000000000000000000000000..5e6c8778d15bea318aa23d34c34a2b2183575706 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-02-21.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4121.64, + "avg tokens": 2692.576, + "avg character count": 11165.434, + "parameters count": 4.205751296, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5325605425037296, + "bpb": 0.5322974790626249, + "compression_rate": 6.653718488282811, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-02-58.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-02-58.json new file mode 100644 index 0000000000000000000000000000000000000000..40ca1d20123224dcef83c59afbc0fc2b8e8f68ff --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-02-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1387.388, + "avg tokens": 603.514, + "avg character count": 2745.97, + "parameters count": 4.205751296, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7289146594423307, + "bpb": 0.7283459810578662, + "compression_rate": 9.104324763223328, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-03-36.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-03-36.json new file mode 100644 index 0000000000000000000000000000000000000000..76612c4f9b946d415d8371689c169e996bc0eab7 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-03-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4412.048, + "avg tokens": 2637.476, + "avg character count": 11560.664, + "parameters count": 4.205751296, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5505946518092791, + "bpb": 0.5503336867582398, + "compression_rate": 6.879171084477997, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-04-05.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-04-05.json new file mode 100644 index 0000000000000000000000000000000000000000..1fc9518453ceecc1a8ecaee0fb95d5c91765be0d --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-04-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 989.1585, + "avg tokens": 1450.036, + "avg character count": 4752.824, + "parameters count": 4.205751296, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3002539253721926, + "bpb": 0.29456227322519346, + "compression_rate": 3.6820284153149183, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-04-37.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-04-37.json new file mode 100644 index 0000000000000000000000000000000000000000..842321efa8f56faa10ede0b1010e66ca63f4e728 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-04-37.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1059.1625, + "avg tokens": 1516.922, + "avg character count": 5202.974, + "parameters count": 4.205751296, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.293687511458938, + "bpb": 0.2846766277184184, + "compression_rate": 3.55845784648023, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-05-06.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-05-06.json new file mode 100644 index 0000000000000000000000000000000000000000..0aafb811b16a58cfee9fb2bf80a77dfd09338e7f --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-05-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2094.467, + "avg tokens": 1457.148, + "avg character count": 4927.866, + "parameters count": 4.205751296, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6131816803065636, + "bpb": 0.5642552517955738, + "compression_rate": 7.053190647444672, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-05-40.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-05-40.json new file mode 100644 index 0000000000000000000000000000000000000000..6b7f6027bad6607dab57c02b33ad3aaec5b94a5b --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-05-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1072.867, + "avg tokens": 1590.85, + "avg character count": 5229.94, + "parameters count": 4.205751296, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2959536630312049, + "bpb": 0.29070479708086017, + "compression_rate": 3.633809963510752, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-06-07.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-06-07.json new file mode 100644 index 0000000000000000000000000000000000000000..00d21687491a893417415fcff9d36efcc4559124 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-06-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1127.196, + "avg tokens": 1418.482, + "avg character count": 5274.818, + "parameters count": 4.205751296, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3082950121330965, + "bpb": 0.2993611982307603, + "compression_rate": 3.7420149778845038, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-06-33.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-06-33.json new file mode 100644 index 0000000000000000000000000000000000000000..bc70815c1948721d9ed8f7b399e782dd243d2364 --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-06-33.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1486.204, + "avg tokens": 759.516, + "avg character count": 3127.092, + "parameters count": 4.205751296, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6856655130547297, + "bpb": 0.6820657427192746, + "compression_rate": 8.525821783990931, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-06-58.json b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-06-58.json new file mode 100644 index 0000000000000000000000000000000000000000..83938973288e2006d7fac34277fa4c52eb8c591f --- /dev/null +++ b/data/2026-04/Qwen3.5-4B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-06-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1543.992, + "avg tokens": 835.802, + "avg character count": 2498.564, + "parameters count": 4.205751296, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "tokenizer_name": "Qwen/Qwen3.5-4B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8915159273775787, + "bpb": 0.6346947897900579, + "compression_rate": 7.933684872375723, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_04-11-35.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_04-11-35.json new file mode 100644 index 0000000000000000000000000000000000000000..f1ae0eef7d3bbb503e4d31f99f170382240db501 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-ao3_english-2026-04-19_04-11-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4907.472, + "avg tokens": 2146.384, + "avg character count": 9041.522, + "parameters count": 36.151104512, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7830524017639333, + "bpb": 0.7684225357628099, + "compression_rate": 9.605281697035124, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_04-13-18.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_04-13-18.json new file mode 100644 index 0000000000000000000000000000000000000000..73a002274501bd3cfbea69200178316177466dab --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_04-13-18.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3501.752, + "avg tokens": 1318.22, + "avg character count": 2942.368, + "parameters count": 36.151104512, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.7169709039871999, + "bpb": 0.9630904398649011, + "compression_rate": 12.038630498311264, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_04-16-28.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_04-16-28.json new file mode 100644 index 0000000000000000000000000000000000000000..9321558fb8fd4631ace8e1f10c8e8b0234edf916 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_04-16-28.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4281.856, + "avg tokens": 2493.368, + "avg character count": 11865.894, + "parameters count": 36.151104512, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5206023597548278, + "bpb": 0.520236183164864, + "compression_rate": 6.5029522895608, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_04-19-16.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_04-19-16.json new file mode 100644 index 0000000000000000000000000000000000000000..07b74dbeec2f1d3ff3e4794872244afa4addc83c --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-19_04-19-16.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2770.888, + "avg tokens": 2221.302, + "avg character count": 8435.788, + "parameters count": 36.151104512, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4738794261376338, + "bpb": 0.4736406908567658, + "compression_rate": 5.920508635709573, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_04-22-15.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_04-22-15.json new file mode 100644 index 0000000000000000000000000000000000000000..42f2c47644bab13cb1bd3bf4772fefbcae05ccd3 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-19_04-22-15.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3666.104, + "avg tokens": 2358.242, + "avg character count": 10817.836, + "parameters count": 36.151104512, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4889212648614004, + "bpb": 0.48858153792079595, + "compression_rate": 6.10726922400995, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_04-25-32.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_04-25-32.json new file mode 100644 index 0000000000000000000000000000000000000000..596bc2e9f0a9a649d4c3726ba49f2f17cf127f50 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_04-25-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3959.296, + "avg tokens": 2620.358, + "avg character count": 11165.434, + "parameters count": 36.151104512, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5115839388429961, + "bpb": 0.5113312369985574, + "compression_rate": 6.391640462481968, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_04-26-24.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_04-26-24.json new file mode 100644 index 0000000000000000000000000000000000000000..1b58942085350b163ea6c5c3e94cd3117230ab02 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-bbc_news-2026-04-19_04-26-24.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1229.988, + "avg tokens": 606.982, + "avg character count": 2745.97, + "parameters count": 36.151104512, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6462188545224218, + "bpb": 0.6457146930414585, + "compression_rate": 8.071433663018231, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_04-29-38.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_04-29-38.json new file mode 100644 index 0000000000000000000000000000000000000000..048e151c59aa21ca5ca06c0c9df0390d6de12804 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_04-29-38.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4113.538, + "avg tokens": 2579.482, + "avg character count": 11560.664, + "parameters count": 36.151104512, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5133425617341966, + "bpb": 0.513099253036258, + "compression_rate": 6.413740662953225, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_04-31-35.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_04-31-35.json new file mode 100644 index 0000000000000000000000000000000000000000..6c74b0127eb53850e77a32156a7f803e62a76582 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_cpp-2026-04-19_04-31-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 810.85575, + "avg tokens": 1519.406, + "avg character count": 4752.824, + "parameters count": 36.151104512, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.24613105164451723, + "bpb": 0.24146535967463165, + "compression_rate": 3.0183169959328957, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_04-33-36.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_04-33-36.json new file mode 100644 index 0000000000000000000000000000000000000000..558f0f341a5165f7880ebe307f3120cbb6097fee --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_javascript-2026-04-19_04-33-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 948.0775, + "avg tokens": 1581.52, + "avg character count": 5202.974, + "parameters count": 36.151104512, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.26288555499766214, + "bpb": 0.25481973305862776, + "compression_rate": 3.185246663232847, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_04-35-31.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_04-35-31.json new file mode 100644 index 0000000000000000000000000000000000000000..63c933d302a25ea5193ced97004c7adf30e5e0fa --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_markdown-2026-04-19_04-35-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1951.04875, + "avg tokens": 1499.832, + "avg character count": 4927.866, + "parameters count": 36.151104512, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5711941753606147, + "bpb": 0.5256179752159806, + "compression_rate": 6.570224690199758, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_other-2026-04-19_04-37-38.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_other-2026-04-19_04-37-38.json new file mode 100644 index 0000000000000000000000000000000000000000..1bac53659dee631707f3222c655bcb709120fed9 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_other-2026-04-19_04-37-38.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 933.182, + "avg tokens": 1663.01, + "avg character count": 5229.94, + "parameters count": 36.151104512, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.25742112598745776, + "bpb": 0.25285565121260256, + "compression_rate": 3.160695640157532, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_python-2026-04-19_04-39-31.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_python-2026-04-19_04-39-31.json new file mode 100644 index 0000000000000000000000000000000000000000..59823f55e3753e407781e5c842ab8fd1a9ae0e04 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-github_python-2026-04-19_04-39-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 994.601, + "avg tokens": 1497.148, + "avg character count": 5274.818, + "parameters count": 36.151104512, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2720294672466811, + "bpb": 0.2641465611317929, + "compression_rate": 3.3018320141474113, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_04-40-35.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_04-40-35.json new file mode 100644 index 0000000000000000000000000000000000000000..9855edaacc51f4407c0b2682fc87eaafc14757f4 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_04-40-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1308.484, + "avg tokens": 761.826, + "avg character count": 3127.092, + "parameters count": 36.151104512, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6036737575621549, + "bpb": 0.6005044470989765, + "compression_rate": 7.506305588737207, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_04-41-50.json b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_04-41-50.json new file mode 100644 index 0000000000000000000000000000000000000000..4ba2d526566d0cbdada3c59cb937e3bf47739256 --- /dev/null +++ b/data/2026-04/Seed-OSS-36B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_04-41-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1630.176, + "avg tokens": 950.962, + "avg character count": 2498.564, + "parameters count": 36.151104512, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "ByteDance-Seed/Seed-OSS-36B-Base", + "tokenizer_name": "ByteDance-Seed/Seed-OSS-36B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_eos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9412794032797266, + "bpb": 0.6701227814916123, + "compression_rate": 8.376534768645154, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-39-04.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-39-04.json new file mode 100644 index 0000000000000000000000000000000000000000..e66ed3ced4be82f56324a91868ddfb485fb50f81 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_16-39-04.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5355.296, + "avg tokens": 2097.61, + "avg character count": 9041.522, + "parameters count": 3.075098624, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8545086747222981, + "bpb": 0.8385437822325696, + "compression_rate": 10.48179727790712, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-39-23.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-39-23.json new file mode 100644 index 0000000000000000000000000000000000000000..d3b730fc0077a23c4bf295ec2b745efd9ce9d377 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-39-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4079.008, + "avg tokens": 1533.964, + "avg character count": 2942.368, + "parameters count": 3.075098624, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.00000972459815, + "bpb": 1.1218537489041058, + "compression_rate": 14.023171861301323, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-39-49.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-39-49.json new file mode 100644 index 0000000000000000000000000000000000000000..c701acf6b56bcbf2bd9b8da1e01c4035114ab472 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-39-49.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5121.936, + "avg tokens": 2452.278, + "avg character count": 11865.894, + "parameters count": 3.075098624, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6227420931748298, + "bpb": 0.6223040744608671, + "compression_rate": 7.778800930760839, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-40-14.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-40-14.json new file mode 100644 index 0000000000000000000000000000000000000000..e41b6bcec850d1385625d1f0a922a3ff43468036 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-40-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3470.896, + "avg tokens": 2283.974, + "avg character count": 8435.788, + "parameters count": 3.075098624, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5935953400727163, + "bpb": 0.5932962932215178, + "compression_rate": 7.416203665268972, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-40-40.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-40-40.json new file mode 100644 index 0000000000000000000000000000000000000000..ea09f83013861143bb32dc2eb1ebdb0208a5bbd0 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-40-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4417.288, + "avg tokens": 2359.912, + "avg character count": 10817.836, + "parameters count": 3.075098624, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5891011373973803, + "bpb": 0.5886918004724025, + "compression_rate": 7.358647505905032, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-41-07.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-41-07.json new file mode 100644 index 0000000000000000000000000000000000000000..1cc18647ebb3980aa01faada8fe0ea3cf0409729 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-41-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4805.656, + "avg tokens": 2641.25, + "avg character count": 11165.434, + "parameters count": 3.075098624, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6209428204419365, + "bpb": 0.6206360997181164, + "compression_rate": 7.757951246476455, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-41-22.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-41-22.json new file mode 100644 index 0000000000000000000000000000000000000000..c098d83735dac68987fafd490395275864a9fdb3 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_16-41-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1347.84, + "avg tokens": 577.996, + "avg character count": 2745.97, + "parameters count": 3.075098624, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7081366817233183, + "bpb": 0.7075842137232227, + "compression_rate": 8.844802671540283, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-41-50.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-41-50.json new file mode 100644 index 0000000000000000000000000000000000000000..0392dcbb4b659d7c490f4e8a2995bccbf5912b9f --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-41-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4844.348, + "avg tokens": 2554.072, + "avg character count": 11560.664, + "parameters count": 3.075098624, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6045428563567254, + "bpb": 0.6042563215041871, + "compression_rate": 7.553204018802338, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-42-08.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-42-08.json new file mode 100644 index 0000000000000000000000000000000000000000..7320b81f384aac42da7e442c2426635e9e687386 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_16-42-08.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1096.286, + "avg tokens": 1310.73, + "avg character count": 4752.824, + "parameters count": 3.075098624, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33277192161880986, + "bpb": 0.32646385413960904, + "compression_rate": 4.080798176745113, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-42-26.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-42-26.json new file mode 100644 index 0000000000000000000000000000000000000000..898adb8658e4f9a691fbb5e96ef3b44de60a0d0d --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_16-42-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1355.096, + "avg tokens": 1371.522, + "avg character count": 5202.974, + "parameters count": 3.075098624, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.375744771957052, + "bpb": 0.36421621754425587, + "compression_rate": 4.552702719303198, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-42-45.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-42-45.json new file mode 100644 index 0000000000000000000000000000000000000000..d8a778c1ba1fb895f3b10ae51899f848b811a345 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_16-42-45.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2601.486, + "avg tokens": 1400.098, + "avg character count": 4927.866, + "parameters count": 3.075098624, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7616178993385911, + "bpb": 0.7008475845991653, + "compression_rate": 8.760594807489566, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-43-03.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-43-03.json new file mode 100644 index 0000000000000000000000000000000000000000..ea0c15467d37b183e0f78a4539dc2b36d64a242f --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_other-2026-04-18_16-43-03.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1359.446, + "avg tokens": 1433.536, + "avg character count": 5229.94, + "parameters count": 3.075098624, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.37500736194991485, + "bpb": 0.3683564445288997, + "compression_rate": 4.604455556611247, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-43-21.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-43-21.json new file mode 100644 index 0000000000000000000000000000000000000000..e06fbe5d1b3834e911b3b8e99d3b052819fa3578 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-github_python-2026-04-18_16-43-21.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1362.198, + "avg tokens": 1309.164, + "avg character count": 5274.818, + "parameters count": 3.075098624, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.37256949894932184, + "bpb": 0.3617731304116988, + "compression_rate": 4.522164130146235, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-43-36.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-43-36.json new file mode 100644 index 0000000000000000000000000000000000000000..f45b443a74ae745023d0d1d50c1ba0d98d1776f7 --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-43-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1492.068, + "avg tokens": 723.416, + "avg character count": 3127.092, + "parameters count": 3.075098624, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6883708903572756, + "bpb": 0.6847569166868497, + "compression_rate": 8.559461458585622, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-43-51.json b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-43-51.json new file mode 100644 index 0000000000000000000000000000000000000000..33527441064ab5903248c9f09d8dac92e25235de --- /dev/null +++ b/data/2026-04/SmolLM3-3B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-43-51.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1734.204, + "avg tokens": 921.834, + "avg character count": 2498.564, + "parameters count": 3.075098624, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "HuggingFaceTB/SmolLM3-3B-Base", + "tokenizer_name": "HuggingFaceTB/SmolLM3-3B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.0013461775202892, + "bpb": 0.7128859755964264, + "compression_rate": 8.91107469495533, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_14-12-23.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_14-12-23.json new file mode 100644 index 0000000000000000000000000000000000000000..f9052b4e92de239fd8d79bd315d218804935df1e --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-ao3_english-2026-04-18_14-12-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 6198.664, + "avg tokens": 2178.396, + "avg character count": 9041.522, + "parameters count": 1.961560064, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.9890792515836321, + "bpb": 0.9706001601683396, + "compression_rate": 12.132502002104244, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-12-44.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-12-44.json new file mode 100644 index 0000000000000000000000000000000000000000..2ec9da6c796354c241bb9b6b5c632970454a8c3b --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_14-12-44.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4662.144, + "avg tokens": 1393.502, + "avg character count": 2942.368, + "parameters count": 1.961560064, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.2859316131463623, + "bpb": 1.2822342403669678, + "compression_rate": 16.027928004587096, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-13-13.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-13-13.json new file mode 100644 index 0000000000000000000000000000000000000000..52384106397af1903a5288800543d8a476e256d9 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_cs-2026-04-18_14-13-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4929.168, + "avg tokens": 2455.014, + "avg character count": 11865.894, + "parameters count": 1.961560064, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5993047156251833, + "bpb": 0.5988831820823461, + "compression_rate": 7.486039776029327, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-13-39.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-13-39.json new file mode 100644 index 0000000000000000000000000000000000000000..ba5cbb2d6c7dafff17e431d2655fd9033197cd4e --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_math-2026-04-18_14-13-39.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3396.08, + "avg tokens": 2152.026, + "avg character count": 8435.788, + "parameters count": 1.961560064, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5808002494209421, + "bpb": 0.5805076485967117, + "compression_rate": 7.256345607458896, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-14-07.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-14-07.json new file mode 100644 index 0000000000000000000000000000000000000000..e18e0b89a66393563739572e321830c9cca9b594 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_other-2026-04-18_14-14-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4321.048, + "avg tokens": 2315.488, + "avg character count": 10817.836, + "parameters count": 1.961560064, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5762663180550319, + "bpb": 0.5758658994042666, + "compression_rate": 7.198323742553332, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-14-35.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-14-35.json new file mode 100644 index 0000000000000000000000000000000000000000..12725a80e20c1ded1529fdec69f8a9c8d67d1b94 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-arxiv_physics-2026-04-18_14-14-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4676.048, + "avg tokens": 2577.858, + "avg character count": 11165.434, + "parameters count": 1.961560064, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.604196062648237, + "bpb": 0.6038976141477249, + "compression_rate": 7.548720176846562, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_14-14-59.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_14-14-59.json new file mode 100644 index 0000000000000000000000000000000000000000..1bdd2fc661496e7c9b92478a33866b7e430c0f01 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-bbc_news-2026-04-18_14-14-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1500.296, + "avg tokens": 590.42, + "avg character count": 2745.97, + "parameters count": 1.961560064, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7882349767351968, + "bpb": 0.7876200183345918, + "compression_rate": 9.845250229182398, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-15-29.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-15-29.json new file mode 100644 index 0000000000000000000000000000000000000000..bf2c121c27dc70144d1a3c37b01ebc7c2bfdc20e --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-biorxiv_all-2026-04-18_14-15-29.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4920.908, + "avg tokens": 2606.928, + "avg character count": 11560.664, + "parameters count": 1.961560064, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6140970422002427, + "bpb": 0.6138059789553778, + "compression_rate": 7.672574736942223, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_14-15-50.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_14-15-50.json new file mode 100644 index 0000000000000000000000000000000000000000..3f1abe0b228162c0f7b4c2d9026e5fa0629db08a --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_cpp-2026-04-18_14-15-50.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1107.57025, + "avg tokens": 1356.156, + "avg character count": 4752.824, + "parameters count": 1.961560064, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33619719710032386, + "bpb": 0.32982419965717913, + "compression_rate": 4.122802495714739, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_14-16-11.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_14-16-11.json new file mode 100644 index 0000000000000000000000000000000000000000..85ff9b07b0dce73d8d636c20b9015c0892590670 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_javascript-2026-04-18_14-16-11.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1276.302, + "avg tokens": 1441.212, + "avg character count": 5202.974, + "parameters count": 1.961560064, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3538965534090052, + "bpb": 0.3430383433234021, + "compression_rate": 4.287979291542526, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_14-16-32.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_14-16-32.json new file mode 100644 index 0000000000000000000000000000000000000000..ba1e416180bce1e88daf19695604dc8b31803135 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_markdown-2026-04-18_14-16-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2364.63625, + "avg tokens": 1418.782, + "avg character count": 4927.866, + "parameters count": 1.961560064, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6922771421506337, + "bpb": 0.637039601315605, + "compression_rate": 7.9629950164450625, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_other-2026-04-18_14-16-54.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_other-2026-04-18_14-16-54.json new file mode 100644 index 0000000000000000000000000000000000000000..a3047a6def5859a94ac6af61d74678af4c81b302 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_other-2026-04-18_14-16-54.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1231.02, + "avg tokens": 1499.21, + "avg character count": 5229.94, + "parameters count": 1.961560064, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33958065469874066, + "bpb": 0.3335580452213373, + "compression_rate": 4.169475565266716, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_python-2026-04-18_14-17-14.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_python-2026-04-18_14-17-14.json new file mode 100644 index 0000000000000000000000000000000000000000..b69efbe5e18715fbf283a376085fe2cb772776a2 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-github_python-2026-04-18_14-17-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1228.182, + "avg tokens": 1340.802, + "avg character count": 5274.818, + "parameters count": 1.961560064, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3359153018566875, + "bpb": 0.3261811035218823, + "compression_rate": 4.077263794023529, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-17-36.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-17-36.json new file mode 100644 index 0000000000000000000000000000000000000000..d707a45b43a644929830b5d7773d82a45ad700ba --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-wikipedia_english-2026-04-18_14-17-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1603.372, + "avg tokens": 760.196, + "avg character count": 3127.092, + "parameters count": 1.961560064, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7397213875064178, + "bpb": 0.7358378217494292, + "compression_rate": 9.197972771867866, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-17-59.json b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-17-59.json new file mode 100644 index 0000000000000000000000000000000000000000..218c79ad4b8183b3fedafe46bd11db2771531786 --- /dev/null +++ b/data/2026-04/Youtu-LLM-2B-Base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_14-17-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2447.76, + "avg tokens": 1198.794, + "avg character count": 2498.564, + "parameters count": 1.961560064, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "tencent/Youtu-LLM-2B-Base", + "tokenizer_name": "tencent/Youtu-LLM-2B-Base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_newline_token", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.4133603194820583, + "bpb": 1.0062102126542833, + "compression_rate": 12.577627658178542, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-ao3_english-2026-04-18_20-51-26.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-ao3_english-2026-04-18_20-51-26.json new file mode 100644 index 0000000000000000000000000000000000000000..f630a13c790ca17de521a94c3602d59647c9e85f --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-ao3_english-2026-04-18_20-51-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4874.824, + "avg tokens": 2159.186, + "avg character count": 9041.522, + "parameters count": 27.43240664, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7778429793132726, + "bpb": 0.7633104416036207, + "compression_rate": 9.541380520045259, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_20-53-02.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_20-53-02.json new file mode 100644 index 0000000000000000000000000000000000000000..ea9e8d9d2bf53ffb23544678dc9331c2c9b30396 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_20-53-02.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3426.576, + "avg tokens": 1327.272, + "avg character count": 2942.368, + "parameters count": 27.43240664, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.680110782345764, + "bpb": 0.942414707572242, + "compression_rate": 11.780183844653024, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_cs-2026-04-18_20-55-46.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_cs-2026-04-18_20-55-46.json new file mode 100644 index 0000000000000000000000000000000000000000..4ec00bc7c3e55bac9b594a67b96e746b7e1e1145 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_cs-2026-04-18_20-55-46.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4327.648, + "avg tokens": 2449.01, + "avg character count": 11865.894, + "parameters count": 27.43240664, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5261699041229461, + "bpb": 0.5257998114838652, + "compression_rate": 6.572497643548315, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_math-2026-04-18_20-58-14.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_math-2026-04-18_20-58-14.json new file mode 100644 index 0000000000000000000000000000000000000000..407a374dfadca1d9886d32aeb6f949fa2afafbae --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_math-2026-04-18_20-58-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2809.184, + "avg tokens": 2198.142, + "avg character count": 8435.788, + "parameters count": 27.43240664, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4804288379158677, + "bpb": 0.4801868031128551, + "compression_rate": 6.002335038910688, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-00-51.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-00-51.json new file mode 100644 index 0000000000000000000000000000000000000000..f8f64e5f39768ba7ca10089f3956ff4f0402deeb --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_other-2026-04-18_21-00-51.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3680.888, + "avg tokens": 2327.854, + "avg character count": 10817.836, + "parameters count": 27.43240664, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4908928979573821, + "bpb": 0.4905518010275221, + "compression_rate": 6.131897512844026, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-03-44.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-03-44.json new file mode 100644 index 0000000000000000000000000000000000000000..4433a4ddad55ff194fae2ef7bd69a35541fcce44 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-arxiv_physics-2026-04-18_21-03-44.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3886.248, + "avg tokens": 2574.342, + "avg character count": 11165.434, + "parameters count": 27.43240664, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5021453458293383, + "bpb": 0.5018973062693899, + "compression_rate": 6.273716328367374, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-bbc_news-2026-04-18_21-04-48.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-bbc_news-2026-04-18_21-04-48.json new file mode 100644 index 0000000000000000000000000000000000000000..9b7753c99957f5b8588fe69aae4296c6ff0ca708 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-bbc_news-2026-04-18_21-04-48.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1168.62, + "avg tokens": 605.97, + "avg character count": 2745.97, + "parameters count": 27.43240664, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6139769475572058, + "bpb": 0.6134979402905629, + "compression_rate": 7.668724253632036, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-07-39.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-07-39.json new file mode 100644 index 0000000000000000000000000000000000000000..813983c18b65c051355ad051ea3ada8b832a92fe --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-biorxiv_all-2026-04-18_21-07-39.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4170.852, + "avg tokens": 2507.224, + "avg character count": 11560.664, + "parameters count": 27.43240664, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5204949730120877, + "bpb": 0.520248274289622, + "compression_rate": 6.503103428620275, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_cpp-2026-04-18_21-09-35.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_cpp-2026-04-18_21-09-35.json new file mode 100644 index 0000000000000000000000000000000000000000..d6d2301b88541264c81ff90299d2ee716b659f3a --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_cpp-2026-04-18_21-09-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 853.203, + "avg tokens": 1597.758, + "avg character count": 4752.824, + "parameters count": 27.43240664, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.25898533944694485, + "bpb": 0.2540759799390641, + "compression_rate": 3.1759497492383013, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_javascript-2026-04-18_21-11-31.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_javascript-2026-04-18_21-11-31.json new file mode 100644 index 0000000000000000000000000000000000000000..a99451f536672c2fca884a5dc386d27e07734570 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_javascript-2026-04-18_21-11-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1007.2865, + "avg tokens": 1626.518, + "avg character count": 5202.974, + "parameters count": 27.43240664, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2793031905030471, + "bpb": 0.27073364471107003, + "compression_rate": 3.3841705588883753, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_markdown-2026-04-18_21-13-19.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_markdown-2026-04-18_21-13-19.json new file mode 100644 index 0000000000000000000000000000000000000000..847f2028280c15bc4c3cb111d889b19e8c87862a --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_markdown-2026-04-18_21-13-19.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2096.55, + "avg tokens": 1498.294, + "avg character count": 4927.866, + "parameters count": 27.43240664, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6137915048777212, + "bpb": 0.564816417805585, + "compression_rate": 7.060205222569812, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_other-2026-04-18_21-15-19.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_other-2026-04-18_21-15-19.json new file mode 100644 index 0000000000000000000000000000000000000000..cceeb138b4f8be267a22fc4bd445fe584e5ae81b --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_other-2026-04-18_21-15-19.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1029.935, + "avg tokens": 1719.35, + "avg character count": 5229.94, + "parameters count": 27.43240664, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.28411073873466514, + "bpb": 0.2790719121582412, + "compression_rate": 3.4883989019780146, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_python-2026-04-18_21-17-13.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_python-2026-04-18_21-17-13.json new file mode 100644 index 0000000000000000000000000000000000000000..b220feae751b84ced63b77cb394b1395a1ff4a15 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-github_python-2026-04-18_21-17-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1091.241, + "avg tokens": 1591.81, + "avg character count": 5274.818, + "parameters count": 27.43240664, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.29846109934308884, + "bpb": 0.28981225387468823, + "compression_rate": 3.6226531734336027, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-wikipedia_english-2026-04-18_21-18-22.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-wikipedia_english-2026-04-18_21-18-22.json new file mode 100644 index 0000000000000000000000000000000000000000..7f53339ca818426e059e9f41e765ead91c9e9282 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-wikipedia_english-2026-04-18_21-18-22.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1249.552, + "avg tokens": 743.918, + "avg character count": 3127.092, + "parameters count": 27.43240664, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.576485269295846, + "bpb": 0.5734586994425765, + "compression_rate": 7.168233743032207, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_21-19-29.json b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_21-19-29.json new file mode 100644 index 0000000000000000000000000000000000000000..054d6b378b697fd184cfc40db93a85de3c9bb085 --- /dev/null +++ b/data/2026-04/gemma-3-27b-pt-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_21-19-29.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1208.712, + "avg tokens": 833.256, + "avg character count": 2498.564, + "parameters count": 27.43240664, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-27b-pt", + "tokenizer_name": "google/gemma-3-27b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6979220097075683, + "bpb": 0.49686993764003995, + "compression_rate": 6.210874220500499, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-ao3_english-2026-04-18_17-41-03.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-ao3_english-2026-04-18_17-41-03.json new file mode 100644 index 0000000000000000000000000000000000000000..41590b7510677553a2040b869fe9707054c547aa --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-ao3_english-2026-04-18_17-41-03.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5398.896, + "avg tokens": 2159.186, + "avg character count": 9041.522, + "parameters count": 4.300079472, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8614656343782894, + "bpb": 0.8453707641408227, + "compression_rate": 10.567134551760283, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-41-30.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-41-30.json new file mode 100644 index 0000000000000000000000000000000000000000..73016116f6c340e84cbeed8e97bfd082aa13579c --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_17-41-30.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3904.984, + "avg tokens": 1327.272, + "avg character count": 2942.368, + "parameters count": 4.300079472, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.9146826812794144, + "bpb": 1.0739917499084461, + "compression_rate": 13.424896873855577, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-42-10.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-42-10.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb4c9ff75b13943ecc7172320b48aa17e945c5a --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_cs-2026-04-18_17-42-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4830.016, + "avg tokens": 2449.01, + "avg character count": 11865.894, + "parameters count": 4.300079472, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5872494841614417, + "bpb": 0.5868364299185268, + "compression_rate": 7.335455373981585, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-42-46.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-42-46.json new file mode 100644 index 0000000000000000000000000000000000000000..ab0903ea8e9d0f2db8aef50a2accc4b1286e4514 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_math-2026-04-18_17-42-46.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3209.024, + "avg tokens": 2198.142, + "avg character count": 8435.788, + "parameters count": 4.300079472, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.548809786459032, + "bpb": 0.548533302080756, + "compression_rate": 6.8566662760094506, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-43-23.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-43-23.json new file mode 100644 index 0000000000000000000000000000000000000000..87806689f326ee83e283901d421e9b3af2e90f74 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_other-2026-04-18_17-43-23.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4135.36, + "avg tokens": 2327.854, + "avg character count": 10817.836, + "parameters count": 4.300079472, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5515024783413784, + "bpb": 0.5511192668446239, + "compression_rate": 6.888990835557799, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-44-04.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-44-04.json new file mode 100644 index 0000000000000000000000000000000000000000..25b3a27711fdc30489ad843a88fa5eac19fd0d74 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-arxiv_physics-2026-04-18_17-44-04.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4390.36, + "avg tokens": 2574.342, + "avg character count": 11165.434, + "parameters count": 4.300079472, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.567282077858977, + "bpb": 0.5670018633789913, + "compression_rate": 7.087523292237391, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-bbc_news-2026-04-18_17-44-28.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-bbc_news-2026-04-18_17-44-28.json new file mode 100644 index 0000000000000000000000000000000000000000..310c6bd6f285c4053344148650bb5f9ec82d2630 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-bbc_news-2026-04-18_17-44-28.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1267.32, + "avg tokens": 605.97, + "avg character count": 2745.97, + "parameters count": 4.300079472, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.665832576182333, + "bpb": 0.6653131126363029, + "compression_rate": 8.316413907953786, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-45-08.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-45-08.json new file mode 100644 index 0000000000000000000000000000000000000000..16495ca2643fb97c1b06faecadd50817332979ca --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-biorxiv_all-2026-04-18_17-45-08.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4676.044, + "avg tokens": 2507.224, + "avg character count": 11560.664, + "parameters count": 4.300079472, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5835396210614365, + "bpb": 0.5832630411010368, + "compression_rate": 7.29078801376296, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_cpp-2026-04-18_17-45-39.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_cpp-2026-04-18_17-45-39.json new file mode 100644 index 0000000000000000000000000000000000000000..8e075375f75eb86130ff271f4ced757b10bfc7b0 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_cpp-2026-04-18_17-45-39.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1057.192, + "avg tokens": 1597.758, + "avg character count": 4752.824, + "parameters count": 4.300079472, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3209051409577727, + "bpb": 0.31482202170379037, + "compression_rate": 3.9352752712973795, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_javascript-2026-04-18_17-46-10.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_javascript-2026-04-18_17-46-10.json new file mode 100644 index 0000000000000000000000000000000000000000..71f7ba90535d1002ba729681218bb7373352a018 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_javascript-2026-04-18_17-46-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1224.839, + "avg tokens": 1626.518, + "avg character count": 5202.974, + "parameters count": 4.300079472, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33962675023696004, + "bpb": 0.32920636447948254, + "compression_rate": 4.115079555993532, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_markdown-2026-04-18_17-46-40.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_markdown-2026-04-18_17-46-40.json new file mode 100644 index 0000000000000000000000000000000000000000..27be2b6bf6a445d8c64b97e3ff73911f9c5ce42f --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_markdown-2026-04-18_17-46-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2451.253, + "avg tokens": 1498.294, + "avg character count": 4927.866, + "parameters count": 4.300079472, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7176352902177524, + "bpb": 0.6603743953615194, + "compression_rate": 8.254679942018992, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_other-2026-04-18_17-47-12.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_other-2026-04-18_17-47-12.json new file mode 100644 index 0000000000000000000000000000000000000000..b0e92dc8a5280e53a0bd70f2b10d44a2563ef41d --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_other-2026-04-18_17-47-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1244.036, + "avg tokens": 1719.35, + "avg character count": 5229.94, + "parameters count": 4.300079472, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3431711583473888, + "bpb": 0.3370848697380803, + "compression_rate": 4.213560871726004, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_python-2026-04-18_17-47-42.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_python-2026-04-18_17-47-42.json new file mode 100644 index 0000000000000000000000000000000000000000..247b7b236a79709ccc138164661f299a3f224694 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-github_python-2026-04-18_17-47-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1293.511, + "avg tokens": 1591.81, + "avg character count": 5274.818, + "parameters count": 4.300079472, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3537831836160648, + "bpb": 0.3435312074250343, + "compression_rate": 4.2941400928129285, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-48-06.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-48-06.json new file mode 100644 index 0000000000000000000000000000000000000000..547db0fe088775a27e26a326e09c41780ea5c175 --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-wikipedia_english-2026-04-18_17-48-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1423.684, + "avg tokens": 743.918, + "avg character count": 3127.092, + "parameters count": 4.300079472, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6568216881988004, + "bpb": 0.6533733490540651, + "compression_rate": 8.167166863175813, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-48-31.json b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-48-31.json new file mode 100644 index 0000000000000000000000000000000000000000..217197b0260066fdc8a6b6bc0094af2bcb4daf5a --- /dev/null +++ b/data/2026-04/gemma-3-4b-pt-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_17-48-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1484.248, + "avg tokens": 833.256, + "avg character count": 2498.564, + "parameters count": 4.300079472, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "google/gemma-3-4b-pt", + "tokenizer_name": "google/gemma-3-4b-pt", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8570191634272175, + "bpb": 0.6101355916068956, + "compression_rate": 7.626694895086194, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-ao3_english-2026-04-19_09-19-36.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-ao3_english-2026-04-19_09-19-36.json new file mode 100644 index 0000000000000000000000000000000000000000..af171924d232fa9cfff663087349ef2310dcb59f --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-ao3_english-2026-04-19_09-19-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4861.384, + "avg tokens": 2159.186, + "avg character count": 9041.522, + "parameters count": 25.805933872, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7756984486303248, + "bpb": 0.7612059774557557, + "compression_rate": 9.515074718196946, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_09-20-28.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_09-20-28.json new file mode 100644 index 0000000000000000000000000000000000000000..1872cddc00d031ded9c523478486afb4d36e1d7d --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_09-20-28.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3318.6, + "avg tokens": 1327.272, + "avg character count": 2942.368, + "parameters count": 25.805933872, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.627168240918238, + "bpb": 0.912717957678231, + "compression_rate": 11.408974470977888, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_cs-2026-04-19_09-22-00.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_cs-2026-04-19_09-22-00.json new file mode 100644 index 0000000000000000000000000000000000000000..8844049e2517029f6c477141899eaa0f519802ec --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_cs-2026-04-19_09-22-00.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4318.336, + "avg tokens": 2481.97, + "avg character count": 11865.894, + "parameters count": 25.805933872, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5250377200480876, + "bpb": 0.5246684237544248, + "compression_rate": 6.55835529693031, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-23-21.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-23-21.json new file mode 100644 index 0000000000000000000000000000000000000000..b8bb0ea824ae68204e36a346eb6d522d9fb91e2a --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-23-21.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2866.232, + "avg tokens": 2202.728, + "avg character count": 8435.788, + "parameters count": 25.805933872, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.49018523135446923, + "bpb": 0.48993828138696677, + "compression_rate": 6.124228517337085, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-24-48.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-24-48.json new file mode 100644 index 0000000000000000000000000000000000000000..a103850a0b5da8de1274b6592c2454fb802ea354 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-24-48.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3704.704, + "avg tokens": 2346.49, + "avg character count": 10817.836, + "parameters count": 25.805933872, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.49406906231167735, + "bpb": 0.49372575842401756, + "compression_rate": 6.17157198030022, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-26-24.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-26-24.json new file mode 100644 index 0000000000000000000000000000000000000000..dfd4f4561ba603128ea76ebd1032a03639e81f8e --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-26-24.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3954.184, + "avg tokens": 2585.158, + "avg character count": 11165.434, + "parameters count": 25.805933872, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5109234130587745, + "bpb": 0.5106710374874482, + "compression_rate": 6.383387968593103, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-bbc_news-2026-04-19_09-26-58.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-bbc_news-2026-04-19_09-26-58.json new file mode 100644 index 0000000000000000000000000000000000000000..f0db0815b390c7c708493fe82d7de357a636e0c5 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-bbc_news-2026-04-19_09-26-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1174.028, + "avg tokens": 605.97, + "avg character count": 2745.97, + "parameters count": 25.805933872, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6168182367122685, + "bpb": 0.6163370127530328, + "compression_rate": 7.704212659412909, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-28-31.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-28-31.json new file mode 100644 index 0000000000000000000000000000000000000000..69bd877bd0396a8383c97ac85f9639f61e4d924f --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-28-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4171.148, + "avg tokens": 2520.374, + "avg character count": 11560.664, + "parameters count": 25.805933872, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5205319118706259, + "bpb": 0.5202851956402693, + "compression_rate": 6.503564945503366, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_cpp-2026-04-19_09-29-33.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_cpp-2026-04-19_09-29-33.json new file mode 100644 index 0000000000000000000000000000000000000000..5d25020b2bddfaf93943737620d122f0f1f04e37 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_cpp-2026-04-19_09-29-33.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 810.612, + "avg tokens": 1597.848, + "avg character count": 4752.824, + "parameters count": 25.805933872, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.24605706259795954, + "bpb": 0.24139277317398627, + "compression_rate": 3.0174096646748283, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_javascript-2026-04-19_09-30-36.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_javascript-2026-04-19_09-30-36.json new file mode 100644 index 0000000000000000000000000000000000000000..bac3c85c9c7dec3a5035c16e18c286ee924616a7 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_javascript-2026-04-19_09-30-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 936.212, + "avg tokens": 1629.85, + "avg character count": 5202.974, + "parameters count": 25.805933872, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.25959545629494557, + "bpb": 0.2516305807555648, + "compression_rate": 3.14538225944456, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_markdown-2026-04-19_09-31-35.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_markdown-2026-04-19_09-31-35.json new file mode 100644 index 0000000000000000000000000000000000000000..9569c84c25a311c11b1c8d325c1de8ac354b89f5 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_markdown-2026-04-19_09-31-35.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1994.9305, + "avg tokens": 1501.42, + "avg character count": 4927.866, + "parameters count": 25.805933872, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5840411121706921, + "bpb": 0.5374398410632251, + "compression_rate": 6.717998013290313, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_other-2026-04-19_09-32-41.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_other-2026-04-19_09-32-41.json new file mode 100644 index 0000000000000000000000000000000000000000..428456b99105a3debe5985c0259301150041264e --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_other-2026-04-19_09-32-41.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 954.404, + "avg tokens": 1724.14, + "avg character count": 5229.94, + "parameters count": 25.805933872, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2632752799849693, + "bpb": 0.2586059792622583, + "compression_rate": 3.232574740778229, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_python-2026-04-19_09-33-42.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_python-2026-04-19_09-33-42.json new file mode 100644 index 0000000000000000000000000000000000000000..445e5740751fa70e5034c3bcd85ab3c4e8376d90 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-github_python-2026-04-19_09-33-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1010.866, + "avg tokens": 1592.52, + "avg character count": 5274.818, + "parameters count": 25.805933872, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2764780443994964, + "bpb": 0.26846622682367194, + "compression_rate": 3.3558278352958992, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-34-18.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-34-18.json new file mode 100644 index 0000000000000000000000000000000000000000..b17eac3a6f666c0566a3a63b3979048b02238120 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-34-18.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1263.974, + "avg tokens": 743.918, + "avg character count": 3127.092, + "parameters count": 25.805933872, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5831389104038469, + "bpb": 0.5800774086786554, + "compression_rate": 7.250967608483193, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-34-55.json b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-34-55.json new file mode 100644 index 0000000000000000000000000000000000000000..b6dcbffe24944bb5906828b47902b91ae9ad26e9 --- /dev/null +++ b/data/2026-04/gemma-4-26B-A4B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-34-55.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1220.448, + "avg tokens": 833.256, + "avg character count": 2498.564, + "parameters count": 25.805933872, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-26B-A4B", + "tokenizer_name": "google/gemma-4-26B-A4B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7046984897176354, + "bpb": 0.5016943007539525, + "compression_rate": 6.271178759424406, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-ao3_english-2026-04-19_08-49-12.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-ao3_english-2026-04-19_08-49-12.json new file mode 100644 index 0000000000000000000000000000000000000000..aa6c2d8226bf981663c706fdcbfa8862fab41a7a --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-ao3_english-2026-04-19_08-49-12.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4680.904, + "avg tokens": 2159.186, + "avg character count": 9041.522, + "parameters count": 31.273086512, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7469004651735972, + "bpb": 0.7329460303272806, + "compression_rate": 9.161825379091008, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_08-51-28.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_08-51-28.json new file mode 100644 index 0000000000000000000000000000000000000000..a96e2ee96d6c9126a59cb26d032fcb67977465b0 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_08-51-28.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3197.04, + "avg tokens": 1327.272, + "avg character count": 2942.368, + "parameters count": 31.273086512, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.567565224174424, + "bpb": 0.8792851863483432, + "compression_rate": 10.99106482935429, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_cs-2026-04-19_08-56-20.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_cs-2026-04-19_08-56-20.json new file mode 100644 index 0000000000000000000000000000000000000000..b60badaa25f6e253d22eccc7d35b7fd61c589adb --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_cs-2026-04-19_08-56-20.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4123.728, + "avg tokens": 2481.97, + "avg character count": 11865.894, + "parameters count": 31.273086512, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5013766291503163, + "bpb": 0.5010239753812548, + "compression_rate": 6.262799692265684, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-00-30.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-00-30.json new file mode 100644 index 0000000000000000000000000000000000000000..31fe74aea28ee579e572945f5fe0060ceef41f41 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_math-2026-04-19_09-00-30.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2761.456, + "avg tokens": 2202.728, + "avg character count": 8435.788, + "parameters count": 31.273086512, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.47226635814378853, + "bpb": 0.4720284355089636, + "compression_rate": 5.900355443862045, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-05-02.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-05-02.json new file mode 100644 index 0000000000000000000000000000000000000000..2fe5fe68e1d6e69c649b2cfffcd9bedf862a258e --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_other-2026-04-19_09-05-02.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3553.896, + "avg tokens": 2346.49, + "avg character count": 10817.836, + "parameters count": 31.273086512, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.47395691107122756, + "bpb": 0.4736275821118455, + "compression_rate": 5.920344776398069, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-10-10.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-10-10.json new file mode 100644 index 0000000000000000000000000000000000000000..e05a82c984082552defb213760271683841a3c31 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-arxiv_physics-2026-04-19_09-10-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3804.808, + "avg tokens": 2585.158, + "avg character count": 11165.434, + "parameters count": 31.273086512, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.49162241549541696, + "bpb": 0.49137957383888636, + "compression_rate": 6.142244672986079, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-bbc_news-2026-04-19_09-11-10.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-bbc_news-2026-04-19_09-11-10.json new file mode 100644 index 0000000000000000000000000000000000000000..a4962ee214abff4f529686dcac402a7f764506b8 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-bbc_news-2026-04-19_09-11-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1141.128, + "avg tokens": 605.97, + "avg character count": 2745.97, + "parameters count": 31.273086512, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5995330271705593, + "bpb": 0.5990652886377861, + "compression_rate": 7.4883161079723255, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-16-10.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-16-10.json new file mode 100644 index 0000000000000000000000000000000000000000..f264d19023d7f4ec8809a4a34c3df2abeb6f803b --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-biorxiv_all-2026-04-19_09-16-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4044.54, + "avg tokens": 2520.374, + "avg character count": 11560.664, + "parameters count": 31.273086512, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5047320638915764, + "bpb": 0.5044928363066702, + "compression_rate": 6.306160453833377, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_cpp-2026-04-19_09-19-07.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_cpp-2026-04-19_09-19-07.json new file mode 100644 index 0000000000000000000000000000000000000000..d9ab1ad3b0b0ce34649b0b6b571c33823654b249 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_cpp-2026-04-19_09-19-07.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 782.401, + "avg tokens": 1597.848, + "avg character count": 4752.824, + "parameters count": 31.273086512, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.23749376006487216, + "bpb": 0.23299179770852152, + "compression_rate": 2.912397471356519, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_javascript-2026-04-19_09-22-09.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_javascript-2026-04-19_09-22-09.json new file mode 100644 index 0000000000000000000000000000000000000000..d23822c8a6e4a025f3365d3e4cfa8b9efa4c6361 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_javascript-2026-04-19_09-22-09.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 900.603, + "avg tokens": 1629.85, + "avg character count": 5202.974, + "parameters count": 31.273086512, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.24972169415217585, + "bpb": 0.24205976415619962, + "compression_rate": 3.025747051952495, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_markdown-2026-04-19_09-24-53.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_markdown-2026-04-19_09-24-53.json new file mode 100644 index 0000000000000000000000000000000000000000..4693ec10f46973e9b6a1b211d8d9b4a162a2804b --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_markdown-2026-04-19_09-24-53.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1877.3545, + "avg tokens": 1501.42, + "avg character count": 4927.866, + "parameters count": 31.273086512, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5496192524594986, + "bpb": 0.5057645387141708, + "compression_rate": 6.322056733927135, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_other-2026-04-19_09-28-06.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_other-2026-04-19_09-28-06.json new file mode 100644 index 0000000000000000000000000000000000000000..ea68ba174a9c0e3a8fa1b5a2de86865c899b474f --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_other-2026-04-19_09-28-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 912.462, + "avg tokens": 1724.14, + "avg character count": 5229.94, + "parameters count": 31.273086512, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.2517054502345391, + "bpb": 0.2472413454361033, + "compression_rate": 3.090516817951291, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_python-2026-04-19_09-30-59.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_python-2026-04-19_09-30-59.json new file mode 100644 index 0000000000000000000000000000000000000000..9a2abeb6d67d989bc7998241108b1b3a70b981b4 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-github_python-2026-04-19_09-30-59.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 963.335, + "avg tokens": 1592.52, + "avg character count": 5274.818, + "parameters count": 31.273086512, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.26347802468535775, + "bpb": 0.2558429234113938, + "compression_rate": 3.1980365426424227, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-32-16.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-32-16.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f4d5b3a607bcdab16d22910dd8454bcdfb8492 --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-wikipedia_english-2026-04-19_09-32-16.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1219.944, + "avg tokens": 743.918, + "avg character count": 3127.092, + "parameters count": 31.273086512, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5628255129565249, + "bpb": 0.5598706573498139, + "compression_rate": 6.998383216872674, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-33-37.json b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-33-37.json new file mode 100644 index 0000000000000000000000000000000000000000..21e265e2e528f126b6e70218b2c9d22902a4b65a --- /dev/null +++ b/data/2026-04/gemma-4-31B-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_09-33-37.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1161.32, + "avg tokens": 833.256, + "avg character count": 2498.564, + "parameters count": 31.273086512, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "google/gemma-4-31B", + "tokenizer_name": "google/gemma-4-31B", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "sdpa", + "dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6705574101304473, + "bpb": 0.47738832408392656, + "compression_rate": 5.967354051049082, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-ao3_english-2026-04-19_01-44-26.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-ao3_english-2026-04-19_01-44-26.json new file mode 100644 index 0000000000000000000000000000000000000000..99a412c1d12d4388548d42cf561b3c35956971f8 --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-ao3_english-2026-04-19_01-44-26.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4836.208, + "avg tokens": 2097.61, + "avg character count": 9041.522, + "parameters count": 32.519640064, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.7716812831188743, + "bpb": 0.7572638651502009, + "compression_rate": 9.465798314377512, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_01-46-13.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_01-46-13.json new file mode 100644 index 0000000000000000000000000000000000000000..425e4eb3eda920e103cb3bd23153588c03cd42ef --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-ao3_nonenglish-2026-04-19_01-46-13.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4387.752, + "avg tokens": 1533.964, + "avg character count": 2942.368, + "parameters count": 32.519640064, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.1513923652821916, + "bpb": 1.2067679275111716, + "compression_rate": 15.084599093889645, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_01-49-01.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_01-49-01.json new file mode 100644 index 0000000000000000000000000000000000000000..38e29aaf422bf2d352eee3ae4bcc1ba6e3a0dec2 --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_cs-2026-04-19_01-49-01.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4544.144, + "avg tokens": 2452.278, + "avg character count": 11865.894, + "parameters count": 32.519640064, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5524922111966732, + "bpb": 0.5521036042107716, + "compression_rate": 6.901295052634645, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_math-2026-04-19_01-51-37.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_math-2026-04-19_01-51-37.json new file mode 100644 index 0000000000000000000000000000000000000000..3c100c54c723fea55bb38e72aaf3edd0c9f9b5a9 --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_math-2026-04-19_01-51-37.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2920.696, + "avg tokens": 2283.974, + "avg character count": 8435.788, + "parameters count": 32.519640064, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.49949970709840397, + "bpb": 0.49924806459972126, + "compression_rate": 6.240600807496516, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_other-2026-04-19_01-54-18.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_other-2026-04-19_01-54-18.json new file mode 100644 index 0000000000000000000000000000000000000000..e130b057e8777df57c310fe9ad33ec29f0687e49 --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_other-2026-04-19_01-54-18.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3822.808, + "avg tokens": 2359.912, + "avg character count": 10817.836, + "parameters count": 32.519640064, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5098197221579858, + "bpb": 0.509465473924341, + "compression_rate": 6.368318424054263, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_01-57-17.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_01-57-17.json new file mode 100644 index 0000000000000000000000000000000000000000..6a6174ae5f9adbc63e15427fea5b16425f25ba70 --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-arxiv_physics-2026-04-19_01-57-17.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4060.248, + "avg tokens": 2641.25, + "avg character count": 11165.434, + "parameters count": 32.519640064, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5246280309730309, + "bpb": 0.5243688858728721, + "compression_rate": 6.554611073410902, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-bbc_news-2026-04-19_01-58-05.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-bbc_news-2026-04-19_01-58-05.json new file mode 100644 index 0000000000000000000000000000000000000000..6f61a01a5a897c99aaa13139c3b927c6605562cd --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-bbc_news-2026-04-19_01-58-05.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1183.036, + "avg tokens": 577.996, + "avg character count": 2745.97, + "parameters count": 32.519640064, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6215509165770623, + "bpb": 0.621066000316259, + "compression_rate": 7.763325003953238, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_02-00-58.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_02-00-58.json new file mode 100644 index 0000000000000000000000000000000000000000..01789990725e25646bd1cff256308f620b760fae --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-biorxiv_all-2026-04-19_02-00-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4297.104, + "avg tokens": 2554.072, + "avg character count": 11560.664, + "parameters count": 32.519640064, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5362503945261387, + "bpb": 0.5359962282150103, + "compression_rate": 6.699952852687628, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_cpp-2026-04-19_02-02-31.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_cpp-2026-04-19_02-02-31.json new file mode 100644 index 0000000000000000000000000000000000000000..9c029401347714daa63c92bc0fa7a45455687ecd --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_cpp-2026-04-19_02-02-31.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1045.575, + "avg tokens": 1310.738, + "avg character count": 4752.824, + "parameters count": 32.519640064, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.31737886094193224, + "bpb": 0.3113625863068777, + "compression_rate": 3.8920323288359713, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_javascript-2026-04-19_02-04-08.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_javascript-2026-04-19_02-04-08.json new file mode 100644 index 0000000000000000000000000000000000000000..1a063b2f1e6b9dddf059f87c13dcb01cba5fa8f2 --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_javascript-2026-04-19_02-04-08.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1236.781, + "avg tokens": 1371.522, + "avg character count": 5202.974, + "parameters count": 32.519640064, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.34293806107155117, + "bpb": 0.33241607808642515, + "compression_rate": 4.155200976080314, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_markdown-2026-04-19_02-05-47.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_markdown-2026-04-19_02-05-47.json new file mode 100644 index 0000000000000000000000000000000000000000..389e7e812486eb7459258b21745d7ab7a70beccf --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_markdown-2026-04-19_02-05-47.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2263.0685, + "avg tokens": 1400.102, + "avg character count": 4927.866, + "parameters count": 32.519640064, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.662541899910027, + "bpb": 0.6096769661675888, + "compression_rate": 7.62096207709486, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_other-2026-04-19_02-07-28.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_other-2026-04-19_02-07-28.json new file mode 100644 index 0000000000000000000000000000000000000000..9fcb4df6791f266b8e1c695d93db25946e7706ad --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_other-2026-04-19_02-07-28.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1209.396, + "avg tokens": 1433.536, + "avg character count": 5229.94, + "parameters count": 32.519640064, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33361560776432403, + "bpb": 0.32769879096887494, + "compression_rate": 4.096234887110937, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_python-2026-04-19_02-09-02.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_python-2026-04-19_02-09-02.json new file mode 100644 index 0000000000000000000000000000000000000000..f696b0de3b22ed663ef28145456633dc1688423f --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-github_python-2026-04-19_02-09-02.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1239.882, + "avg tokens": 1309.164, + "avg character count": 5274.818, + "parameters count": 32.519640064, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.33911532354054486, + "bpb": 0.3292883945513926, + "compression_rate": 4.116104931892408, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_02-10-01.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_02-10-01.json new file mode 100644 index 0000000000000000000000000000000000000000..772f0a6bc698c4d5d22b571714a927c95e744dd2 --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-wikipedia_english-2026-04-19_02-10-01.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1295.688, + "avg tokens": 723.416, + "avg character count": 3127.092, + "parameters count": 32.519640064, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5977702773501192, + "bpb": 0.5946319603852847, + "compression_rate": 7.432899504816058, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/marin-32b-base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_02-11-10.json b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_02-11-10.json new file mode 100644 index 0000000000000000000000000000000000000000..eae19cc4e3dbd74a89bf5416ac040dcad9d2af9b --- /dev/null +++ b/data/2026-04/marin-32b-base-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-19_02-11-10.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1821.532, + "avg tokens": 921.834, + "avg character count": 2498.564, + "parameters count": 32.519640064, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "marin-community/marin-32b-base", + "tokenizer_name": "marin-community/marin-32b-base", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.0517702100969017, + "bpb": 0.7487842358223771, + "compression_rate": 9.359802947779713, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-ao3_english-2026-04-18_16-15-58.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-ao3_english-2026-04-18_16-15-58.json new file mode 100644 index 0000000000000000000000000000000000000000..460b8d0095131595506ce868e6a14e3ab860c3e3 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-ao3_english-2026-04-18_16-15-58.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 5072.232, + "avg tokens": 2185.918, + "avg character count": 9041.522, + "parameters count": 2.94773504, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8093420502254276, + "bpb": 0.7942210114326212, + "compression_rate": 9.927762642907766, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-16-42.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-16-42.json new file mode 100644 index 0000000000000000000000000000000000000000..e8b8b2ce19b43bd95fc9f21d7f70afaaa740b3ef --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-16-42.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 3514.184, + "avg tokens": 1657.484, + "avg character count": 2942.368, + "parameters count": 2.94773504, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.7230665333402693, + "bpb": 0.966509625560633, + "compression_rate": 12.081370319507913, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-17-47.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-17-47.json new file mode 100644 index 0000000000000000000000000000000000000000..e1618f1ca2325248198d3e62614ee9097377ad33 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-17-47.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4768.096, + "avg tokens": 2505.628, + "avg character count": 11865.894, + "parameters count": 2.94773504, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5797210436636718, + "bpb": 0.5793132847072987, + "compression_rate": 7.241416058841234, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-18-45.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-18-45.json new file mode 100644 index 0000000000000000000000000000000000000000..d1e2c2e670634e31c1789453c77d5fc7fc521551 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-18-45.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 3245.04, + "avg tokens": 2245.17, + "avg character count": 8435.788, + "parameters count": 2.94773504, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5549692708596188, + "bpb": 0.5546896834003537, + "compression_rate": 6.933621042504422, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-19-46.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-19-46.json new file mode 100644 index 0000000000000000000000000000000000000000..cacd588c8e9bda574bb8da608549297aa9965e8d --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-19-46.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4152.736, + "avg tokens": 2386.452, + "avg character count": 10817.836, + "parameters count": 2.94773504, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5538197873697726, + "bpb": 0.5534349656908409, + "compression_rate": 6.917937071135512, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-20-52.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-20-52.json new file mode 100644 index 0000000000000000000000000000000000000000..dafa7daecbdb6555485a5327a426f6b1a0e57365 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-20-52.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4454.416, + "avg tokens": 2638.652, + "avg character count": 11165.434, + "parameters count": 2.94773504, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5755588070518758, + "bpb": 0.5752745042012939, + "compression_rate": 7.190931302516174, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-bbc_news-2026-04-18_16-21-11.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-bbc_news-2026-04-18_16-21-11.json new file mode 100644 index 0000000000000000000000000000000000000000..2ebdee3765ae37d22203815d74c9808f2edce5fc --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-bbc_news-2026-04-18_16-21-11.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1320.092, + "avg tokens": 602.084, + "avg character count": 2745.97, + "parameters count": 2.94773504, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6935582624417578, + "bpb": 0.6930171681077253, + "compression_rate": 8.662714601346567, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-22-14.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-22-14.json new file mode 100644 index 0000000000000000000000000000000000000000..b2d08bcdf97c7a0ed79517bb3704d94cbb1287d6 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-22-14.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 4655.676, + "avg tokens": 2562.3, + "avg character count": 11560.664, + "parameters count": 2.94773504, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5809978282550003, + "bpb": 0.5807224530267702, + "compression_rate": 7.259030662834627, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_cpp-2026-04-18_16-22-52.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_cpp-2026-04-18_16-22-52.json new file mode 100644 index 0000000000000000000000000000000000000000..c1379dc89aad2d24c9483ebf48ddcd0677e4f76c --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_cpp-2026-04-18_16-22-52.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1079.54, + "avg tokens": 1486.492, + "avg character count": 4752.824, + "parameters count": 2.94773504, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3276887602909916, + "bpb": 0.3214770498737314, + "compression_rate": 4.018463123421642, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_javascript-2026-04-18_16-23-33.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_javascript-2026-04-18_16-23-33.json new file mode 100644 index 0000000000000000000000000000000000000000..8fe05f2e7068b6f33e0effa710dd95278aa994e2 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_javascript-2026-04-18_16-23-33.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1244.3075, + "avg tokens": 1541.046, + "avg character count": 5202.974, + "parameters count": 2.94773504, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.34502502983696315, + "bpb": 0.33443901473544985, + "compression_rate": 4.180487684193123, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_markdown-2026-04-18_16-24-13.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_markdown-2026-04-18_16-24-13.json new file mode 100644 index 0000000000000000000000000000000000000000..baea6b3a4c9f1903d5a207470e5d8a542cab9bfc --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_markdown-2026-04-18_16-24-13.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 2341.6195, + "avg tokens": 1565.416, + "avg character count": 4927.866, + "parameters count": 2.94773504, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6855386977443976, + "bpb": 0.6308388246660965, + "compression_rate": 7.885485308326206, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_other-2026-04-18_16-24-55.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_other-2026-04-18_16-24-55.json new file mode 100644 index 0000000000000000000000000000000000000000..11954a0c364585273d56485eabfd883c08f33121 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_other-2026-04-18_16-24-55.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1248.076, + "avg tokens": 1614.406, + "avg character count": 5229.94, + "parameters count": 2.94773504, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.3442856047779772, + "bpb": 0.33817955098021624, + "compression_rate": 4.227244387252703, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_python-2026-04-18_16-25-35.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_python-2026-04-18_16-25-35.json new file mode 100644 index 0000000000000000000000000000000000000000..9f4e838d36203413047b0d7eebaad9d58d384580 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-github_python-2026-04-18_16-25-35.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1268.104, + "avg tokens": 1506.072, + "avg character count": 5274.818, + "parameters count": 2.94773504, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.34683421345181165, + "bpb": 0.33678360544325925, + "compression_rate": 4.209795068040741, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-25-56.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-25-56.json new file mode 100644 index 0000000000000000000000000000000000000000..84312d91bee3df83a6f2012729bf0c5cacc9ede9 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-25-56.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1443.768, + "avg tokens": 727.702, + "avg character count": 3127.092, + "parameters count": 2.94773504, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6660875131893071, + "bpb": 0.6625905281067214, + "compression_rate": 8.282381601334016, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-26-25.json b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-26-25.json new file mode 100644 index 0000000000000000000000000000000000000000..0a45cdb74c9a92c6d739de4873a945b2ce010c68 --- /dev/null +++ b/data/2026-04/rwkv7-g1e-2.9b-20260312-ctx8192-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-26-25.json @@ -0,0 +1,28 @@ +{ + "neg_log_prob_sum": 1513.404, + "avg tokens": 1112.008, + "avg character count": 2498.564, + "parameters count": 2.94773504, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "/mnt/Public/rwkv_models/rwkv7-g1e-2.9b-20260312-ctx8192", + "tokenizer_name": "rwkv_vocab_v20230424", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8738541200711772, + "bpb": 0.622120861796844, + "compression_rate": 7.77651077246055, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-ao3_english-2026-04-18_16-50-56.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-ao3_english-2026-04-18_16-50-56.json new file mode 100644 index 0000000000000000000000000000000000000000..e2b9e9f37da1494f6dd06fa813dafa787a2d7857 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-ao3_english-2026-04-18_16-50-56.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5346.848, + "avg tokens": 2220.424, + "avg character count": 9041.522, + "parameters count": 2.7954432, + "avg bytes": 9213.662, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8531606840073023, + "bpb": 0.8372209761967686, + "compression_rate": 10.465262202459607, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-51-17.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-51-17.json new file mode 100644 index 0000000000000000000000000000000000000000..6e8c66cbf97464f9092c27045bfdc553ba926bbe --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-ao3_nonenglish-2026-04-18_16-51-17.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4931.672, + "avg tokens": 2204.548, + "avg character count": 2942.368, + "parameters count": 2.7954432, + "avg bytes": 5245.572, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-ao3_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 2.418085955832498, + "bpb": 1.356362802319929, + "compression_rate": 16.954535028999114, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-51-42.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-51-42.json new file mode 100644 index 0000000000000000000000000000000000000000..156355d9e432cdb17cff4fae1c0522fc50ace2c7 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_cs-2026-04-18_16-51-42.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 5252.24, + "avg tokens": 2532.726, + "avg character count": 11865.894, + "parameters count": 2.7954432, + "avg bytes": 11874.246, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_cs", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6385848888889997, + "bpb": 0.6381357268123508, + "compression_rate": 7.976696585154384, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-52-06.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-52-06.json new file mode 100644 index 0000000000000000000000000000000000000000..e838d070d692c0aeb69bda061fe0950aed983351 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_math-2026-04-18_16-52-06.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 3520.208, + "avg tokens": 2239.788, + "avg character count": 8435.788, + "parameters count": 2.7954432, + "avg bytes": 8440.04, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_math", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6020287167597924, + "bpb": 0.6017254212654982, + "compression_rate": 7.521567765818728, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-52-30.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-52-30.json new file mode 100644 index 0000000000000000000000000000000000000000..059023bdcef118b3d13827e5d5eb9e397aa53847 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_other-2026-04-18_16-52-30.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4486.76, + "avg tokens": 2388.578, + "avg character count": 10817.836, + "parameters count": 2.7954432, + "avg bytes": 10825.358, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.5983661059068529, + "bpb": 0.5979503312185117, + "compression_rate": 7.474379140231396, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-52-57.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-52-57.json new file mode 100644 index 0000000000000000000000000000000000000000..07f19bd84002341b522b6976b3ec2cfd58f87743 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-arxiv_physics-2026-04-18_16-52-57.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4770.112, + "avg tokens": 2614.73, + "avg character count": 11165.434, + "parameters count": 2.7954432, + "avg bytes": 11170.952, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-arxiv_physics", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6163501505525836, + "bpb": 0.6160456984225637, + "compression_rate": 7.700571230282047, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-bbc_news-2026-04-18_16-53-14.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-bbc_news-2026-04-18_16-53-14.json new file mode 100644 index 0000000000000000000000000000000000000000..12933dc575b66db2d2410f51283aab514977374a --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-bbc_news-2026-04-18_16-53-14.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1311.012, + "avg tokens": 604.752, + "avg character count": 2745.97, + "parameters count": 2.7954432, + "avg bytes": 2748.114, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-bbc_news", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6887877547627693, + "bpb": 0.6882503822424839, + "compression_rate": 8.603129778031049, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-53-40.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-53-40.json new file mode 100644 index 0000000000000000000000000000000000000000..05dd9219d013984f8cfb7b4d497ab35c6ba2de08 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-biorxiv_all-2026-04-18_16-53-40.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 4825.728, + "avg tokens": 2499.498, + "avg character count": 11560.664, + "parameters count": 2.7954432, + "avg bytes": 11566.146, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-biorxiv_all", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6022192024851699, + "bpb": 0.6019337689736075, + "compression_rate": 7.524172112170094, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_cpp-2026-04-18_16-53-58.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_cpp-2026-04-18_16-53-58.json new file mode 100644 index 0000000000000000000000000000000000000000..7df5baefebd22d6d52ce8c358897bcac584cc118 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_cpp-2026-04-18_16-53-58.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1166.979, + "avg tokens": 1626.922, + "avg character count": 4752.824, + "parameters count": 2.7954432, + "avg bytes": 4844.66, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_cpp", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.354230414616986, + "bpb": 0.34751557717601683, + "compression_rate": 4.343944714700211, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_javascript-2026-04-18_16-54-16.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_javascript-2026-04-18_16-54-16.json new file mode 100644 index 0000000000000000000000000000000000000000..12b77c2044dbb1113930a018f59b0f40be03db6b --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_javascript-2026-04-18_16-54-16.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1427.726, + "avg tokens": 1709.168, + "avg character count": 5202.974, + "parameters count": 2.7954432, + "avg bytes": 5367.664, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_javascript", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.39588381951326995, + "bpb": 0.3837373613453145, + "compression_rate": 4.796717016816431, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_markdown-2026-04-18_16-54-36.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_markdown-2026-04-18_16-54-36.json new file mode 100644 index 0000000000000000000000000000000000000000..c2508c11246181dea430fc5d157f00b78bc36b94 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_markdown-2026-04-18_16-54-36.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 2808.677, + "avg tokens": 1715.716, + "avg character count": 4927.866, + "parameters count": 2.7954432, + "avg bytes": 5355.16, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_markdown", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.8222756826908222, + "bpb": 0.7566654179070076, + "compression_rate": 9.458317723837595, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_other-2026-04-18_16-54-55.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_other-2026-04-18_16-54-55.json new file mode 100644 index 0000000000000000000000000000000000000000..75182f6f79e5c5479463aa01e11ef0b91609ff30 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_other-2026-04-18_16-54-55.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1481.68, + "avg tokens": 1763.352, + "avg character count": 5229.94, + "parameters count": 2.7954432, + "avg bytes": 5324.37, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_other", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.4087259869490586, + "bpb": 0.40147705516039633, + "compression_rate": 5.0184631895049545, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_python-2026-04-18_16-55-15.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_python-2026-04-18_16-55-15.json new file mode 100644 index 0000000000000000000000000000000000000000..63c9f584d5c43b75e6c5b0745cb61aa723471dc5 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-github_python-2026-04-18_16-55-15.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1476.33, + "avg tokens": 1686.474, + "avg character count": 5274.818, + "parameters count": 2.7954432, + "avg bytes": 5432.234, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-github_python", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.40378530021615966, + "bpb": 0.392084356033927, + "compression_rate": 4.901054450424088, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-55-32.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-55-32.json new file mode 100644 index 0000000000000000000000000000000000000000..49b9c55f542f98c824ccaa74a6166508b1a76c33 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-wikipedia_english-2026-04-18_16-55-32.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1466.652, + "avg tokens": 729.084, + "avg character count": 3127.092, + "parameters count": 2.7954432, + "avg bytes": 3143.596, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_english", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 0.6766451281605658, + "bpb": 0.6730927151930083, + "compression_rate": 8.413658939912603, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file diff --git a/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-55-48.json b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-55-48.json new file mode 100644 index 0000000000000000000000000000000000000000..209e2e44ee67d3cf8a2f6c7a548f14a461a76ae2 --- /dev/null +++ b/data/2026-04/stablelm-3b-4e1t-UncheatableEval-2026-04-wikipedia_nonenglish-2026-04-18_16-55-48.json @@ -0,0 +1,30 @@ +{ + "neg_log_prob_sum": 1916.82, + "avg tokens": 1350.794, + "avg character count": 2498.564, + "parameters count": 2.7954432, + "avg bytes": 3509.576, + "sample_count": 500, + "model_name_or_path": "stabilityai/stablelm-3b-4e1t", + "tokenizer_name": "stabilityai/stablelm-3b-4e1t", + "data_path": "Jellyfish042/UncheatableEval-2026-04-wikipedia_nonenglish", + "chunk_size": 4000, + "bos_mode": "add_default_bos", + "model_args": { + "device_map": "auto", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "torch_dtype": "torch.bfloat16" + }, + "tokenizer_args": { + "trust_remote_code": true + }, + "requirements": [], + "batch_size": 1, + "enable_chunking": true, + "bpc": 1.1067904237301036, + "bpb": 0.7879546441726244, + "compression_rate": 9.849433052157805, + "track_byte_wise_data": false, + "track_single_sample_byte_wise_data": false +} \ No newline at end of file