{
  "model_info": {
    "model_name": "Helion-V1.5-XL",
    "model_id": "DeepXR/Helion-V1.5-XL",
    "parameters": "16.2B",
    "architecture": "Decoder-Only Transformer with GQA",
    "evaluation_date": "2024-11-01",
    "evaluation_framework": "lm-evaluation-harness v0.4.0"
  },
  
  "language_understanding": {
    "mmlu": {
      "metric": "5-shot accuracy",
      "overall_score": 78.9,
      "categories": {
        "stem": 76.4,
        "humanities": 79.8,
        "social_sciences": 81.2,
        "other": 78.1
      },
      "subcategories": {
        "abstract_algebra": 47.3,
        "anatomy": 71.8,
        "astronomy": 82.4,
        "business_ethics": 79.6,
        "clinical_knowledge": 76.9,
        "college_biology": 84.7,
        "college_chemistry": 62.1,
        "college_computer_science": 73.8,
        "college_mathematics": 51.4,
        "college_medicine": 69.3,
        "college_physics": 58.7,
        "computer_security": 81.9,
        "conceptual_physics": 74.2,
        "econometrics": 63.8,
        "electrical_engineering": 77.4,
        "elementary_mathematics": 68.9,
        "formal_logic": 54.3,
        "global_facts": 72.6,
        "high_school_biology": 87.3,
        "high_school_chemistry": 71.4,
        "high_school_computer_science": 79.8,
        "high_school_european_history": 84.6,
        "high_school_geography": 88.2,
        "high_school_government_and_politics": 91.7,
        "high_school_macroeconomics": 83.4,
        "high_school_mathematics": 49.7,
        "high_school_microeconomics": 82.9,
        "high_school_physics": 53.8,
        "high_school_psychology": 89.4,
        "high_school_statistics": 67.3,
        "high_school_us_history": 86.1,
        "high_school_world_history": 87.9,
        "human_aging": 78.4,
        "human_sexuality": 85.6,
        "international_law": 89.3,
        "jurisprudence": 81.7,
        "logical_fallacies": 82.4,
        "machine_learning": 64.9,
        "management": 87.2,
        "marketing": 91.3,
        "medical_genetics": 82.1,
        "miscellaneous": 88.6,
        "moral_disputes": 80.3,
        "moral_scenarios": 71.8,
        "nutrition": 84.7,
        "philosophy": 79.6,
        "prehistory": 82.9,
        "professional_accounting": 61.4,
        "professional_law": 68.7,
        "professional_medicine": 74.3,
        "professional_psychology": 81.9,
        "public_relations": 77.8,
        "security_studies": 83.4,
        "sociology": 89.7,
        "us_foreign_policy": 92.1,
        "virology": 69.3,
        "world_religions": 88.4
      }
    },
    
    "hellaswag": {
      "metric": "10-shot accuracy",
      "score": 85.7,
      "normalized_score": 85.7
    },
    
    "arc": {
      "arc_challenge": {
        "metric": "25-shot accuracy",
        "score": 82.1
      },
      "arc_easy": {
        "metric": "25-shot accuracy",
        "score": 89.6
      }
    },
    
    "winogrande": {
      "metric": "5-shot accuracy",
      "score": 77.3
    },
    
    "piqa": {
      "metric": "0-shot accuracy",
      "score": 83.4
    },
    
    "openbookqa": {
      "metric": "0-shot accuracy",
      "score": 68.7
    },
    
    "boolq": {
      "metric": "0-shot accuracy",
      "score": 84.9
    },
    
    "sciq": {
      "metric": "0-shot accuracy",
      "score": 97.3
    }
  },
  
  "reasoning_and_math": {
    "gsm8k": {
      "metric": "8-shot accuracy",
      "score": 71.6,
      "samples_evaluated": 1319
    },
    
    "math": {
      "metric": "4-shot accuracy",
      "overall_score": 34.7,
      "by_difficulty": {
        "level_1": 52.3,
        "level_2": 44.7,
        "level_3": 36.9,
        "level_4": 28.4,
        "level_5": 18.7
      },
      "by_subject": {
        "algebra": 41.2,
        "counting_and_probability": 38.9,
        "geometry": 29.4,
        "intermediate_algebra": 31.7,
        "number_theory": 36.8,
        "prealgebra": 43.6,
        "precalculus": 28.3
      }
    },
    
    "bigbench_hard": {
      "metric": "3-shot average",
      "overall_score": 61.8,
      "tasks": {
        "boolean_expressions": 88.4,
        "causal_judgement": 72.3,
        "date_understanding": 76.9,
        "disambiguation_qa": 68.7,
        "dyck_languages": 54.2,
        "formal_fallacies": 79.8,
        "geometric_shapes": 63.4,
        "hyperbaton": 82.6,
        "logical_deduction_five_objects": 59.7,
        "logical_deduction_seven_objects": 51.3,
        "logical_deduction_three_objects": 74.8,
        "movie_recommendation": 83.9,
        "multistep_arithmetic_two": 67.4,
        "navigate": 71.2,
        "object_counting": 79.6,
        "penguins_in_a_table": 68.3,
        "reasoning_about_colored_objects": 73.8,
        "ruin_names": 71.9,
        "salient_translation_error_detection": 54.7,
        "snarks": 77.4,
        "sports_understanding": 84.2,
        "temporal_sequences": 69.8,
        "tracking_shuffled_objects_five_objects": 48.3,
        "tracking_shuffled_objects_seven_objects": 38.7,
        "tracking_shuffled_objects_three_objects": 64.2,
        "web_of_lies": 72.8,
        "word_sorting": 58.9
      }
    },
    
    "drop": {
      "metric": "3-shot F1",
      "f1_score": 69.4,
      "exact_match": 62.8
    },
    
    "commonsenseqa": {
      "metric": "7-shot accuracy",
      "score": 76.9
    }
  },
  
  "code_generation": {
    "humaneval": {
      "metric": "pass@1",
      "score": 67.8,
      "pass_at_10": 84.3,
      "pass_at_100": 93.7,
      "temperature": 0.2,
      "samples_evaluated": 164
    },
    
    "mbpp": {
      "metric": "pass@1",
      "score": 72.4,
      "pass_at_10": 87.6,
      "pass_at_100": 95.8,
      "temperature": 0.2,
      "samples_evaluated": 500
    },
    
    "ds1000": {
      "metric": "pass@1",
      "overall_score": 48.9,
      "by_library": {
        "numpy": 52.7,
        "pandas": 51.3,
        "scipy": 47.8,
        "matplotlib": 44.9,
        "sklearn": 46.2,
        "pytorch": 48.7,
        "tensorflow": 45.3
      }
    },
    
    "codexglue": {
      "metric": "average score",
      "overall_score": 81.2,
      "tasks": {
        "code_to_text": 84.7,
        "text_to_code": 78.9,
        "code_to_code": 83.4,
        "code_refinement": 79.8,
        "defect_detection": 81.6,
        "clone_detection": 89.3
      }
    }
  },
  
  "multilingual": {
    "flores_101": {
      "metric": "BLEU score",
      "languages": {
        "eng": 100.0,
        "spa": 87.3,
        "fra": 86.9,
        "deu": 85.1,
        "zho_simp": 82.4,
        "jpn": 81.8,
        "kor": 80.9,
        "rus": 79.7,
        "ara": 77.3,
        "hin": 76.8,
        "por": 86.1,
        "ita": 85.4,
        "nld": 84.7,
        "pol": 79.3,
        "tur": 78.6,
        "vie": 76.9,
        "tha": 74.2,
        "swe": 83.8,
        "dan": 82.4,
        "fin": 79.1
      }
    },
    
    "xnli": {
      "metric": "accuracy",
      "languages": {
        "en": 89.4,
        "es": 84.6,
        "fr": 83.8,
        "de": 82.7,
        "zh": 81.3,
        "ja": 79.8,
        "ko": 78.6,
        "ru": 80.2,
        "ar": 76.4,
        "hi": 75.1,
        "pt": 83.2,
        "it": 82.9,
        "nl": 81.7,
        "pl": 78.9,
        "tr": 77.8
      }
    },
    
    "xstorycloze": {
      "metric": "accuracy",
      "languages": {
        "en": 91.2,
        "es": 86.9,
        "fr": 85.4,
        "de": 84.1,
        "zh": 83.7,
        "ja": 82.4,
        "ko": 81.1,
        "ru": 82.8,
        "ar": 78.9,
        "hi": 77.6,
        "pt": 85.7,
        "it": 84.8
      }
    }
  },
  
  "truthfulness_and_safety": {
    "truthfulqa": {
      "metric": "multiple choice",
      "mc1": 61.3,
      "mc2": 73.8,
      "generation_accuracy": 58.7
    },
    
    "toxigen": {
      "metric": "toxicity rate",
      "overall": 2.1,
      "by_group": {
        "asian": 1.8,
        "black": 2.3,
        "chinese": 1.9,
        "latino": 2.2,
        "lgbtq": 2.4,
        "mental_disability": 2.1,
        "mexican": 2.0,
        "middle_east": 2.3,
        "muslim": 2.5,
        "native_american": 1.7,
        "physical_disability": 1.9,
        "women": 2.2
      }
    },
    
    "bold": {
      "metric": "bias score",
      "overall": 0.34,
      "domains": {
        "gender": 0.31,
        "race": 0.38,
        "religion": 0.36,
        "profession": 0.29
      }
    },
    
    "winobias": {
      "metric": "accuracy",
      "pro_stereotypical": 89.7,
      "anti_stereotypical": 87.3,
      "difference": 2.4
    }
  },
  
  "long_context": {
    "scrolls": {
      "quality": {
        "metric": "F1",
        "score": 71.4
      },
      "qasper": {
        "metric": "F1",
        "score": 68.7
      },
      "narrative_qa": {
        "metric": "F1",
        "score": 64.3
      },
      "qmsum": {
        "metric": "ROUGE-L",
        "score": 59.8
      },
      "contract_nli": {
        "metric": "accuracy",
        "score": 76.2
      }
    },
    
    "longbench": {
      "single_doc_qa": {
        "metric": "accuracy",
        "score": 63.2
      },
      "multi_doc_qa": {
        "metric": "accuracy",
        "score": 58.9
      },
      "summarization": {
        "metric": "ROUGE-L",
        "score": 54.7
      },
      "few_shot_learning": {
        "metric": "accuracy",
        "score": 72.8
      },
      "code_completion": {
        "metric": "accuracy",
        "score": 67.3
      }
    }
  },
  
  "aggregate_scores": {
    "average_across_benchmarks": 74.3,
    "language_understanding_avg": 82.1,
    "reasoning_avg": 63.7,
    "code_generation_avg": 67.6,
    "multilingual_avg": 81.2,
    "safety_avg": 96.1
  },
  
  "comparison_baseline": {
    "helion_v1.5": {
      "parameters": "7B",
      "mmlu": 62.3,
      "humaneval": 45.2,
      "improvement": "+26.7% MMLU, +50.0% HumanEval"
    },
    "llama_2_13b": {
      "parameters": "13B",
      "mmlu": 55.8,
      "humaneval": 29.3,
      "comparison": "+41.4% MMLU, +131.4% HumanEval"
    },
    "mistral_7b": {
      "parameters": "7B",
      "mmlu": 62.5,
      "humaneval": 40.2,
      "comparison": "+26.2% MMLU, +68.7% HumanEval"
    }
  }
}