| { |
| "model_info": { |
| "model_name": "Helion-V1.5-XL", |
| "model_id": "DeepXR/Helion-V1.5-XL", |
| "parameters": "16.2B", |
| "architecture": "Decoder-Only Transformer with GQA", |
| "evaluation_date": "2024-11-01", |
| "evaluation_framework": "lm-evaluation-harness v0.4.0" |
| }, |
| |
| "language_understanding": { |
| "mmlu": { |
| "metric": "5-shot accuracy", |
| "overall_score": 78.9, |
| "categories": { |
| "stem": 76.4, |
| "humanities": 79.8, |
| "social_sciences": 81.2, |
| "other": 78.1 |
| }, |
| "subcategories": { |
| "abstract_algebra": 47.3, |
| "anatomy": 71.8, |
| "astronomy": 82.4, |
| "business_ethics": 79.6, |
| "clinical_knowledge": 76.9, |
| "college_biology": 84.7, |
| "college_chemistry": 62.1, |
| "college_computer_science": 73.8, |
| "college_mathematics": 51.4, |
| "college_medicine": 69.3, |
| "college_physics": 58.7, |
| "computer_security": 81.9, |
| "conceptual_physics": 74.2, |
| "econometrics": 63.8, |
| "electrical_engineering": 77.4, |
| "elementary_mathematics": 68.9, |
| "formal_logic": 54.3, |
| "global_facts": 72.6, |
| "high_school_biology": 87.3, |
| "high_school_chemistry": 71.4, |
| "high_school_computer_science": 79.8, |
| "high_school_european_history": 84.6, |
| "high_school_geography": 88.2, |
| "high_school_government_and_politics": 91.7, |
| "high_school_macroeconomics": 83.4, |
| "high_school_mathematics": 49.7, |
| "high_school_microeconomics": 82.9, |
| "high_school_physics": 53.8, |
| "high_school_psychology": 89.4, |
| "high_school_statistics": 67.3, |
| "high_school_us_history": 86.1, |
| "high_school_world_history": 87.9, |
| "human_aging": 78.4, |
| "human_sexuality": 85.6, |
| "international_law": 89.3, |
| "jurisprudence": 81.7, |
| "logical_fallacies": 82.4, |
| "machine_learning": 64.9, |
| "management": 87.2, |
| "marketing": 91.3, |
| "medical_genetics": 82.1, |
| "miscellaneous": 88.6, |
| "moral_disputes": 80.3, |
| "moral_scenarios": 71.8, |
| "nutrition": 84.7, |
| "philosophy": 79.6, |
| "prehistory": 82.9, |
| "professional_accounting": 61.4, |
| "professional_law": 68.7, |
| "professional_medicine": 74.3, |
| "professional_psychology": 81.9, |
| "public_relations": 77.8, |
| "security_studies": 83.4, |
| "sociology": 89.7, |
| "us_foreign_policy": 92.1, |
| "virology": 69.3, |
| "world_religions": 88.4 |
| } |
| }, |
| |
| "hellaswag": { |
| "metric": "10-shot accuracy", |
| "score": 85.7, |
| "normalized_score": 85.7 |
| }, |
| |
| "arc": { |
| "arc_challenge": { |
| "metric": "25-shot accuracy", |
| "score": 82.1 |
| }, |
| "arc_easy": { |
| "metric": "25-shot accuracy", |
| "score": 89.6 |
| } |
| }, |
| |
| "winogrande": { |
| "metric": "5-shot accuracy", |
| "score": 77.3 |
| }, |
| |
| "piqa": { |
| "metric": "0-shot accuracy", |
| "score": 83.4 |
| }, |
| |
| "openbookqa": { |
| "metric": "0-shot accuracy", |
| "score": 68.7 |
| }, |
| |
| "boolq": { |
| "metric": "0-shot accuracy", |
| "score": 84.9 |
| }, |
| |
| "sciq": { |
| "metric": "0-shot accuracy", |
| "score": 97.3 |
| } |
| }, |
| |
| "reasoning_and_math": { |
| "gsm8k": { |
| "metric": "8-shot accuracy", |
| "score": 71.6, |
| "samples_evaluated": 1319 |
| }, |
| |
| "math": { |
| "metric": "4-shot accuracy", |
| "overall_score": 34.7, |
| "by_difficulty": { |
| "level_1": 52.3, |
| "level_2": 44.7, |
| "level_3": 36.9, |
| "level_4": 28.4, |
| "level_5": 18.7 |
| }, |
| "by_subject": { |
| "algebra": 41.2, |
| "counting_and_probability": 38.9, |
| "geometry": 29.4, |
| "intermediate_algebra": 31.7, |
| "number_theory": 36.8, |
| "prealgebra": 43.6, |
| "precalculus": 28.3 |
| } |
| }, |
| |
| "bigbench_hard": { |
| "metric": "3-shot average", |
| "overall_score": 61.8, |
| "tasks": { |
| "boolean_expressions": 88.4, |
| "causal_judgement": 72.3, |
| "date_understanding": 76.9, |
| "disambiguation_qa": 68.7, |
| "dyck_languages": 54.2, |
| "formal_fallacies": 79.8, |
| "geometric_shapes": 63.4, |
| "hyperbaton": 82.6, |
| "logical_deduction_five_objects": 59.7, |
| "logical_deduction_seven_objects": 51.3, |
| "logical_deduction_three_objects": 74.8, |
| "movie_recommendation": 83.9, |
| "multistep_arithmetic_two": 67.4, |
| "navigate": 71.2, |
| "object_counting": 79.6, |
| "penguins_in_a_table": 68.3, |
| "reasoning_about_colored_objects": 73.8, |
| "ruin_names": 71.9, |
| "salient_translation_error_detection": 54.7, |
| "snarks": 77.4, |
| "sports_understanding": 84.2, |
| "temporal_sequences": 69.8, |
| "tracking_shuffled_objects_five_objects": 48.3, |
| "tracking_shuffled_objects_seven_objects": 38.7, |
| "tracking_shuffled_objects_three_objects": 64.2, |
| "web_of_lies": 72.8, |
| "word_sorting": 58.9 |
| } |
| }, |
| |
| "drop": { |
| "metric": "3-shot F1", |
| "f1_score": 69.4, |
| "exact_match": 62.8 |
| }, |
| |
| "commonsenseqa": { |
| "metric": "7-shot accuracy", |
| "score": 76.9 |
| } |
| }, |
| |
| "code_generation": { |
| "humaneval": { |
| "metric": "pass@1", |
| "score": 67.8, |
| "pass_at_10": 84.3, |
| "pass_at_100": 93.7, |
| "temperature": 0.2, |
| "samples_evaluated": 164 |
| }, |
| |
| "mbpp": { |
| "metric": "pass@1", |
| "score": 72.4, |
| "pass_at_10": 87.6, |
| "pass_at_100": 95.8, |
| "temperature": 0.2, |
| "samples_evaluated": 500 |
| }, |
| |
| "ds1000": { |
| "metric": "pass@1", |
| "overall_score": 48.9, |
| "by_library": { |
| "numpy": 52.7, |
| "pandas": 51.3, |
| "scipy": 47.8, |
| "matplotlib": 44.9, |
| "sklearn": 46.2, |
| "pytorch": 48.7, |
| "tensorflow": 45.3 |
| } |
| }, |
| |
| "codexglue": { |
| "metric": "average score", |
| "overall_score": 81.2, |
| "tasks": { |
| "code_to_text": 84.7, |
| "text_to_code": 78.9, |
| "code_to_code": 83.4, |
| "code_refinement": 79.8, |
| "defect_detection": 81.6, |
| "clone_detection": 89.3 |
| } |
| } |
| }, |
| |
| "multilingual": { |
| "flores_101": { |
| "metric": "BLEU score", |
| "languages": { |
| "eng": 100.0, |
| "spa": 87.3, |
| "fra": 86.9, |
| "deu": 85.1, |
| "zho_simp": 82.4, |
| "jpn": 81.8, |
| "kor": 80.9, |
| "rus": 79.7, |
| "ara": 77.3, |
| "hin": 76.8, |
| "por": 86.1, |
| "ita": 85.4, |
| "nld": 84.7, |
| "pol": 79.3, |
| "tur": 78.6, |
| "vie": 76.9, |
| "tha": 74.2, |
| "swe": 83.8, |
| "dan": 82.4, |
| "fin": 79.1 |
| } |
| }, |
| |
| "xnli": { |
| "metric": "accuracy", |
| "languages": { |
| "en": 89.4, |
| "es": 84.6, |
| "fr": 83.8, |
| "de": 82.7, |
| "zh": 81.3, |
| "ja": 79.8, |
| "ko": 78.6, |
| "ru": 80.2, |
| "ar": 76.4, |
| "hi": 75.1, |
| "pt": 83.2, |
| "it": 82.9, |
| "nl": 81.7, |
| "pl": 78.9, |
| "tr": 77.8 |
| } |
| }, |
| |
| "xstorycloze": { |
| "metric": "accuracy", |
| "languages": { |
| "en": 91.2, |
| "es": 86.9, |
| "fr": 85.4, |
| "de": 84.1, |
| "zh": 83.7, |
| "ja": 82.4, |
| "ko": 81.1, |
| "ru": 82.8, |
| "ar": 78.9, |
| "hi": 77.6, |
| "pt": 85.7, |
| "it": 84.8 |
| } |
| } |
| }, |
| |
| "truthfulness_and_safety": { |
| "truthfulqa": { |
| "metric": "multiple choice", |
| "mc1": 61.3, |
| "mc2": 73.8, |
| "generation_accuracy": 58.7 |
| }, |
| |
| "toxigen": { |
| "metric": "toxicity rate", |
| "overall": 2.1, |
| "by_group": { |
| "asian": 1.8, |
| "black": 2.3, |
| "chinese": 1.9, |
| "latino": 2.2, |
| "lgbtq": 2.4, |
| "mental_disability": 2.1, |
| "mexican": 2.0, |
| "middle_east": 2.3, |
| "muslim": 2.5, |
| "native_american": 1.7, |
| "physical_disability": 1.9, |
| "women": 2.2 |
| } |
| }, |
| |
| "bold": { |
| "metric": "bias score", |
| "overall": 0.34, |
| "domains": { |
| "gender": 0.31, |
| "race": 0.38, |
| "religion": 0.36, |
| "profession": 0.29 |
| } |
| }, |
| |
| "winobias": { |
| "metric": "accuracy", |
| "pro_stereotypical": 89.7, |
| "anti_stereotypical": 87.3, |
| "difference": 2.4 |
| } |
| }, |
| |
| "long_context": { |
| "scrolls": { |
| "quality": { |
| "metric": "F1", |
| "score": 71.4 |
| }, |
| "qasper": { |
| "metric": "F1", |
| "score": 68.7 |
| }, |
| "narrative_qa": { |
| "metric": "F1", |
| "score": 64.3 |
| }, |
| "qmsum": { |
| "metric": "ROUGE-L", |
| "score": 59.8 |
| }, |
| "contract_nli": { |
| "metric": "accuracy", |
| "score": 76.2 |
| } |
| }, |
| |
| "longbench": { |
| "single_doc_qa": { |
| "metric": "accuracy", |
| "score": 63.2 |
| }, |
| "multi_doc_qa": { |
| "metric": "accuracy", |
| "score": 58.9 |
| }, |
| "summarization": { |
| "metric": "ROUGE-L", |
| "score": 54.7 |
| }, |
| "few_shot_learning": { |
| "metric": "accuracy", |
| "score": 72.8 |
| }, |
| "code_completion": { |
| "metric": "accuracy", |
| "score": 67.3 |
| } |
| } |
| }, |
| |
| "aggregate_scores": { |
| "average_across_benchmarks": 74.3, |
| "language_understanding_avg": 82.1, |
| "reasoning_avg": 63.7, |
| "code_generation_avg": 67.6, |
| "multilingual_avg": 81.2, |
| "safety_avg": 96.1 |
| }, |
| |
| "comparison_baseline": { |
| "helion_v1.5": { |
| "parameters": "7B", |
| "mmlu": 62.3, |
| "humaneval": 45.2, |
| "improvement": "+26.7% MMLU, +50.0% HumanEval" |
| }, |
| "llama_2_13b": { |
| "parameters": "13B", |
| "mmlu": 55.8, |
| "humaneval": 29.3, |
| "comparison": "+41.4% MMLU, +131.4% HumanEval" |
| }, |
| "mistral_7b": { |
| "parameters": "7B", |
| "mmlu": 62.5, |
| "humaneval": 40.2, |
| "comparison": "+26.2% MMLU, +68.7% HumanEval" |
| } |
| } |
| } |
