{
  "artifact": {
    "architecture": "Qwen3_5ForCausalLM",
    "formats": [
      "bf16",
      "fp32",
      "fp8_e4m3fn_row_bf16",
      "nvfp4"
    ],
    "name": "qwen3.8-27b",
    "path": "/home/jay/ninfer/models/qwen3_8_27b_quasar.ninfer",
    "prefill_signature": "87cbdc4c4af2a96250b3e3581b74d33aac0977f0903689741f56501a7f94b603"
  },
  "corpus": {
    "id": "ninfer-ppl-1m-v1",
    "mode": "quick",
    "source": "/home/jay/ninfer/eval/corpora/perplexity-1m/manifest.json",
    "stream_count": 4
  },
  "domains": [
    {
      "domain": "chinese_reference",
      "mean_nll": 1.6247369674902634,
      "perplexity": 5.077083423537842,
      "scored_tokens": 65510,
      "total_nll": 106436.51874028715
    },
    {
      "domain": "english_long_form",
      "mean_nll": 1.9387747800008068,
      "perplexity": 6.950230190759524,
      "scored_tokens": 65455,
      "total_nll": 126902.50322495282
    },
    {
      "domain": "english_reference",
      "mean_nll": 1.842452174084933,
      "perplexity": 6.311997415497592,
      "scored_tokens": 65304,
      "total_nll": 120319.49677644245
    },
    {
      "domain": "ninfer_code",
      "mean_nll": 0.5087736366130513,
      "perplexity": 1.6632501945556721,
      "scored_tokens": 64898,
      "total_nll": 33018.391468913804
    }
  ],
  "execution": {
    "context_tokens": 4096,
    "device": 0,
    "kv_dtype": "fp8-e4m3-r256",
    "prefill_chunk_tokens": 1024,
    "purpose": "causal_scoring",
    "score_tile_tokens": 1024,
    "stride_tokens": 2048
  },
  "metric": {
    "log_base": "natural",
    "name": "fixed-window truncated-context causal perplexity"
  },
  "overall": {
    "mean_nll": 1.4805733887152526,
    "perplexity": 4.395465268681477,
    "scored_tokens": 261167,
    "total_nll": 386676.91021059634
  },
  "schema_version": 2,
  "streams": [
    {
      "domain": "english_reference",
      "id": "wikitext-00",
      "input_tokens": 65305,
      "mean_nll": 1.842452174084933,
      "path": "data/wikitext/00.txt",
      "perplexity": 6.311997415497592,
      "scored_tokens": 65304,
      "seconds": 8.984852312,
      "total_nll": 120319.49677644245,
      "unscored_tokens": 1,
      "windows": [
        {
          "first_target": 1,
          "index": 0,
          "input_begin": 0,
          "input_end": 4096,
          "mean_nll": 1.8457662800121928,
          "perplexity": 6.332950745144078,
          "scored_tokens": 4095,
          "seconds": 0.914292607,
          "target_begin": 1,
          "target_end": 4096,
          "total_nll": 7558.41291664993
        },
        {
          "first_target": 2048,
          "index": 1,
          "input_begin": 2048,
          "input_end": 6144,
          "mean_nll": 2.0775022294130023,
          "perplexity": 7.984500535873488,
          "scored_tokens": 2048,
          "seconds": 0.265332387,
          "target_begin": 4096,
          "target_end": 6144,
          "total_nll": 4254.724565837829
        },
        {
          "first_target": 2048,
          "index": 2,
          "input_begin": 4096,
          "input_end": 8192,
          "mean_nll": 1.9088845194657602,
          "perplexity": 6.745560059292232,
          "scored_tokens": 2048,
          "seconds": 0.266712713,
          "target_begin": 6144,
          "target_end": 8192,
          "total_nll": 3909.395495865877
        },
        {
          "first_target": 2048,
          "index": 3,
          "input_begin": 6144,
          "input_end": 10240,
          "mean_nll": 1.5835539476978997,
          "perplexity": 4.872240766987337,
          "scored_tokens": 2048,
          "seconds": 0.266983841,
          "target_begin": 8192,
          "target_end": 10240,
          "total_nll": 3243.1184848852986
        },
        {
          "first_target": 2048,
          "index": 4,
          "input_begin": 8192,
          "input_end": 12288,
          "mean_nll": 1.2794115898952838,
          "perplexity": 3.5945240489155355,
          "scored_tokens": 2048,
          "seconds": 0.267186239,
          "target_begin": 10240,
          "target_end": 12288,
          "total_nll": 2620.2349361055412
        },
        {
          "first_target": 2048,
          "index": 5,
          "input_begin": 10240,
          "input_end": 14336,
          "mean_nll": 1.0159086055225974,
          "perplexity": 2.761871709558577,
          "scored_tokens": 2048,
          "seconds": 0.267247769,
          "target_begin": 12288,
          "target_end": 14336,
          "total_nll": 2080.5808241102795
        },
        {
          "first_target": 2048,
          "index": 6,
          "input_begin": 12288,
          "input_end": 16384,
          "mean_nll": 1.5985023702131222,
          "perplexity": 4.94562016729489,
          "scored_tokens": 2048,
          "seconds": 0.267461947,
          "target_begin": 14336,
          "target_end": 16384,
          "total_nll": 3273.7328541964744
        },
        {
          "first_target": 2048,
          "index": 7,
          "input_begin": 14336,
          "input_end": 18432,
          "mean_nll": 2.0979548808973316,
          "perplexity": 8.14948618840031,
          "scored_tokens": 2048,
          "seconds": 0.267805293,
          "target_begin": 16384,
          "target_end": 18432,
          "total_nll": 4296.611596077735
        },
        {
          "first_target": 2048,
          "index": 8,
          "input_begin": 16384,
          "input_end": 20480,
          "mean_nll": 2.0286089733409534,
          "perplexity": 7.603502324714037,
          "scored_tokens": 2048,
          "seconds": 0.269815775,
          "target_begin": 18432,
          "target_end": 20480,
          "total_nll": 4154.5911774022725
        },
        {
          "first_target": 2048,
          "index": 9,
          "input_begin": 18432,
          "input_end": 22528,
          "mean_nll": 1.8033923492445896,
          "perplexity": 6.070204830587119,
          "scored_tokens": 2048,
          "seconds": 0.270131363,
          "target_begin": 20480,
          "target_end": 22528,
          "total_nll": 3693.3475312529195
        },
        {
          "first_target": 2048,
          "index": 10,
          "input_begin": 20480,
          "input_end": 24576,
          "mean_nll": 1.8572085089047388,
          "perplexity": 6.40582997239652,
          "scored_tokens": 2048,
          "seconds": 0.268834624,
          "target_begin": 22528,
          "target_end": 24576,
          "total_nll": 3803.563026236905
        },
        {
          "first_target": 2048,
          "index": 11,
          "input_begin": 22528,
          "input_end": 26624,
          "mean_nll": 1.9486400697081359,
          "perplexity": 7.01913555195508,
          "scored_tokens": 2048,
          "seconds": 0.269076242,
          "target_begin": 24576,
          "target_end": 26624,
          "total_nll": 3990.8148627622622
        },
        {
          "first_target": 2048,
          "index": 12,
          "input_begin": 24576,
          "input_end": 28672,
          "mean_nll": 1.9843051996337937,
          "perplexity": 7.273991657228664,
          "scored_tokens": 2048,
          "seconds": 0.269023862,
          "target_begin": 26624,
          "target_end": 28672,
          "total_nll": 4063.8570488500095
        },
        {
          "first_target": 2048,
          "index": 13,
          "input_begin": 26624,
          "input_end": 30720,
          "mean_nll": 1.8548890402843052,
          "perplexity": 6.39098906894623,
          "scored_tokens": 2048,
          "seconds": 0.26925305,
          "target_begin": 28672,
          "target_end": 30720,
          "total_nll": 3798.812754502257
        },
        {
          "first_target": 2048,
          "index": 14,
          "input_begin": 28672,
          "input_end": 32768,
          "mean_nll": 2.0077196482264767,
          "perplexity": 7.446317748269288,
          "scored_tokens": 2048,
          "seconds": 0.269129682,
          "target_begin": 30720,
          "target_end": 32768,
          "total_nll": 4111.809839567824
        },
        {
          "first_target": 2048,
          "index": 15,
          "input_begin": 30720,
          "input_end": 34816,
          "mean_nll": 2.173236410614949,
          "perplexity": 8.78667536238467,
          "scored_tokens": 2048,
          "seconds": 0.269165981,
          "target_begin": 32768,
          "target_end": 34816,
          "total_nll": 4450.788168939415
        },
        {
          "first_target": 2048,
          "index": 16,
          "input_begin": 32768,
          "input_end": 36864,
          "mean_nll": 1.8943258565094285,
          "perplexity": 6.64806514378053,
          "scored_tokens": 2048,
          "seconds": 0.26930061,
          "target_begin": 34816,
          "target_end": 36864,
          "total_nll": 3879.5793541313096
        },
        {
          "first_target": 2048,
          "index": 17,
          "input_begin": 34816,
          "input_end": 38912,
          "mean_nll": 1.91056196188172,
          "perplexity": 6.756884843536329,
          "scored_tokens": 2048,
          "seconds": 0.269444519,
          "target_begin": 36864,
          "target_end": 38912,
          "total_nll": 3912.8308979337626
        },
        {
          "first_target": 2048,
          "index": 18,
          "input_begin": 36864,
          "input_end": 40960,
          "mean_nll": 1.5073480183870096,
          "perplexity": 4.514741891681945,
          "scored_tokens": 2048,
          "seconds": 0.269618777,
          "target_begin": 38912,
          "target_end": 40960,
          "total_nll": 3087.0487416565957
        },
        {
          "first_target": 2048,
          "index": 19,
          "input_begin": 38912,
          "input_end": 43008,
          "mean_nll": 1.840387708570229,
          "perplexity": 6.298979956175713,
          "scored_tokens": 2048,
          "seconds": 0.269685436,
          "target_begin": 40960,
          "target_end": 43008,
          "total_nll": 3769.114027151829
        },
        {
          "first_target": 2048,
          "index": 20,
          "input_begin": 40960,
          "input_end": 45056,
          "mean_nll": 2.0028997818420042,
          "perplexity": 7.41051384599249,
          "scored_tokens": 2048,
          "seconds": 0.269882165,
          "target_begin": 43008,
          "target_end": 45056,
          "total_nll": 4101.938753212425
        },
        {
          "first_target": 2048,
          "index": 21,
          "input_begin": 43008,
          "input_end": 47104,
          "mean_nll": 1.7652919990283564,
          "perplexity": 5.8432783405991255,
          "scored_tokens": 2048,
          "seconds": 0.269860854,
          "target_begin": 45056,
          "target_end": 47104,
          "total_nll": 3615.318014010074
        },
        {
          "first_target": 2048,
          "index": 22,
          "input_begin": 45056,
          "input_end": 49152,
          "mean_nll": 1.9092456020167115,
          "perplexity": 6.747996203124082,
          "scored_tokens": 2048,
          "seconds": 0.269925385,
          "target_begin": 47104,
          "target_end": 49152,
          "total_nll": 3910.134992930225
        },
        {
          "first_target": 2048,
          "index": 23,
          "input_begin": 47104,
          "input_end": 51200,
          "mean_nll": 1.839433740539163,
          "perplexity": 6.292973795967105,
          "scored_tokens": 2048,
          "seconds": 0.269944083,
          "target_begin": 49152,
          "target_end": 51200,
          "total_nll": 3767.160300624206
        },
        {
          "first_target": 2048,
          "index": 24,
          "input_begin": 49152,
          "input_end": 53248,
          "mean_nll": 1.9033467464030873,
          "perplexity": 6.708307920648875,
          "scored_tokens": 2048,
          "seconds": 0.270171562,
          "target_begin": 51200,
          "target_end": 53248,
          "total_nll": 3898.054136633523
        },
        {
          "first_target": 2048,
          "index": 25,
          "input_begin": 51200,
          "input_end": 55296,
          "mean_nll": 2.1935342931635153,
          "perplexity": 8.96684864902037,
          "scored_tokens": 2048,
          "seconds": 0.270129443,
          "target_begin": 53248,
          "target_end": 55296,
          "total_nll": 4492.358232398879
        },
        {
          "first_target": 2048,
          "index": 26,
          "input_begin": 53248,
          "input_end": 57344,
          "mean_nll": 2.305697312911463,
          "perplexity": 10.03117067901851,
          "scored_tokens": 2048,
          "seconds": 0.270112102,
          "target_begin": 55296,
          "target_end": 57344,
          "total_nll": 4722.068096842676
        },
        {
          "first_target": 2048,
          "index": 27,
          "input_begin": 55296,
          "input_end": 59392,
          "mean_nll": 2.0198363912121433,
          "perplexity": 7.5370916983438745,
          "scored_tokens": 2048,
          "seconds": 0.270202692,
          "target_begin": 57344,
          "target_end": 59392,
          "total_nll": 4136.6249292024695
        },
        {
          "first_target": 2048,
          "index": 28,
          "input_begin": 57344,
          "input_end": 61440,
          "mean_nll": 1.7693895649548357,
          "perplexity": 5.867270680354071,
          "scored_tokens": 2048,
          "seconds": 0.27041005,
          "target_begin": 59392,
          "target_end": 61440,
          "total_nll": 3623.7098290275035
        },
        {
          "first_target": 2048,
          "index": 29,
          "input_begin": 59392,
          "input_end": 63488,
          "mean_nll": 1.5752266707514222,
          "perplexity": 4.831836730053017,
          "scored_tokens": 2048,
          "seconds": 0.270684657,
          "target_begin": 61440,
          "target_end": 63488,
          "total_nll": 3226.0642216989127
        },
        {
          "first_target": 2279,
          "index": 30,
          "input_begin": 61209,
          "input_end": 65305,
          "mean_nll": 1.5812306911090939,
          "perplexity": 4.860934440357616,
          "scored_tokens": 1817,
          "seconds": 0.267955772,
          "target_begin": 63488,
          "target_end": 65305,
          "total_nll": 2873.0961657452235
        }
      ]
    },
    {
      "domain": "english_long_form",
      "id": "pg19-00",
      "input_tokens": 65456,
      "mean_nll": 1.9387747800008068,
      "path": "data/pg19/00.txt",
      "perplexity": 6.950230190759524,
      "scored_tokens": 65455,
      "seconds": 8.453844183,
      "total_nll": 126902.50322495282,
      "unscored_tokens": 1,
      "windows": [
        {
          "first_target": 1,
          "index": 0,
          "input_begin": 0,
          "input_end": 4096,
          "mean_nll": 1.879641575502501,
          "perplexity": 6.5511563464119975,
          "scored_tokens": 4095,
          "seconds": 0.299330815,
          "target_begin": 1,
          "target_end": 4096,
          "total_nll": 7697.132251682742
        },
        {
          "first_target": 2048,
          "index": 1,
          "input_begin": 2048,
          "input_end": 6144,
          "mean_nll": 1.9061542456360785,
          "perplexity": 6.727167952365076,
          "scored_tokens": 2048,
          "seconds": 0.270735377,
          "target_begin": 4096,
          "target_end": 6144,
          "total_nll": 3903.8038950626888
        },
        {
          "first_target": 2048,
          "index": 2,
          "input_begin": 4096,
          "input_end": 8192,
          "mean_nll": 1.7274914740778984,
          "perplexity": 5.626521914528641,
          "scored_tokens": 2048,
          "seconds": 0.270844316,
          "target_begin": 6144,
          "target_end": 8192,
          "total_nll": 3537.902538911536
        },
        {
          "first_target": 2048,
          "index": 3,
          "input_begin": 6144,
          "input_end": 10240,
          "mean_nll": 1.798679677661368,
          "perplexity": 6.041665250432132,
          "scored_tokens": 2048,
          "seconds": 0.270969304,
          "target_begin": 8192,
          "target_end": 10240,
          "total_nll": 3683.6959798504818
        },
        {
          "first_target": 2048,
          "index": 4,
          "input_begin": 8192,
          "input_end": 12288,
          "mean_nll": 1.7587103529387613,
          "perplexity": 5.804946233113602,
          "scored_tokens": 2048,
          "seconds": 0.270963365,
          "target_begin": 10240,
          "target_end": 12288,
          "total_nll": 3601.838802818583
        },
        {
          "first_target": 2048,
          "index": 5,
          "input_begin": 10240,
          "input_end": 14336,
          "mean_nll": 1.8493010394660154,
          "perplexity": 6.355375812922027,
          "scored_tokens": 2048,
          "seconds": 0.271131603,
          "target_begin": 12288,
          "target_end": 14336,
          "total_nll": 3787.3685288263996
        },
        {
          "first_target": 2048,
          "index": 6,
          "input_begin": 12288,
          "input_end": 16384,
          "mean_nll": 1.880314715946937,
          "perplexity": 6.555567679263298,
          "scored_tokens": 2048,
          "seconds": 0.271045824,
          "target_begin": 14336,
          "target_end": 16384,
          "total_nll": 3850.884538259327
        },
        {
          "first_target": 2048,
          "index": 7,
          "input_begin": 14336,
          "input_end": 18432,
          "mean_nll": 1.7651189638422513,
          "perplexity": 5.842267335316262,
          "scored_tokens": 2048,
          "seconds": 0.271129303,
          "target_begin": 16384,
          "target_end": 18432,
          "total_nll": 3614.9636379489307
        },
        {
          "first_target": 2048,
          "index": 8,
          "input_begin": 16384,
          "input_end": 20480,
          "mean_nll": 2.0822866432852045,
          "perplexity": 8.022793221984006,
          "scored_tokens": 2048,
          "seconds": 0.27142666,
          "target_begin": 18432,
          "target_end": 20480,
          "total_nll": 4264.523045448099
        },
        {
          "first_target": 2048,
          "index": 9,
          "input_begin": 18432,
          "input_end": 22528,
          "mean_nll": 1.941861613371015,
          "perplexity": 6.9717175401145814,
          "scored_tokens": 2048,
          "seconds": 0.27150011,
          "target_begin": 20480,
          "target_end": 22528,
          "total_nll": 3976.9325841838386
        },
        {
          "first_target": 2048,
          "index": 10,
          "input_begin": 20480,
          "input_end": 24576,
          "mean_nll": 1.7912304518355247,
          "perplexity": 5.9968267350749915,
          "scored_tokens": 2048,
          "seconds": 0.27150875,
          "target_begin": 22528,
          "target_end": 24576,
          "total_nll": 3668.4399653591545
        },
        {
          "first_target": 2048,
          "index": 11,
          "input_begin": 22528,
          "input_end": 26624,
          "mean_nll": 1.9588476394744156,
          "perplexity": 7.091150792800085,
          "scored_tokens": 2048,
          "seconds": 0.27149598,
          "target_begin": 24576,
          "target_end": 26624,
          "total_nll": 4011.7199656436032
        },
        {
          "first_target": 2048,
          "index": 12,
          "input_begin": 24576,
          "input_end": 28672,
          "mean_nll": 1.807070305301557,
          "perplexity": 6.092571884517622,
          "scored_tokens": 2048,
          "seconds": 0.271532809,
          "target_begin": 26624,
          "target_end": 28672,
          "total_nll": 3700.879985257589
        },
        {
          "first_target": 2048,
          "index": 13,
          "input_begin": 26624,
          "input_end": 30720,
          "mean_nll": 1.936253996126692,
          "perplexity": 6.932732226132486,
          "scored_tokens": 2048,
          "seconds": 0.271587479,
          "target_begin": 28672,
          "target_end": 30720,
          "total_nll": 3965.4481840674653
        },
        {
          "first_target": 2048,
          "index": 14,
          "input_begin": 28672,
          "input_end": 32768,
          "mean_nll": 1.9177435839457164,
          "perplexity": 6.8055849002018585,
          "scored_tokens": 2048,
          "seconds": 0.271727928,
          "target_begin": 30720,
          "target_end": 32768,
          "total_nll": 3927.5388599208272
        },
        {
          "first_target": 2048,
          "index": 15,
          "input_begin": 30720,
          "input_end": 34816,
          "mean_nll": 1.945710913934679,
          "perplexity": 6.998605493077551,
          "scored_tokens": 2048,
          "seconds": 0.271740538,
          "target_begin": 32768,
          "target_end": 34816,
          "total_nll": 3984.815951738223
        },
        {
          "first_target": 2048,
          "index": 16,
          "input_begin": 32768,
          "input_end": 36864,
          "mean_nll": 2.0489736632038853,
          "perplexity": 7.7599327133996825,
          "scored_tokens": 2048,
          "seconds": 0.271774527,
          "target_begin": 34816,
          "target_end": 36864,
          "total_nll": 4196.298062241557
        },
        {
          "first_target": 2048,
          "index": 17,
          "input_begin": 34816,
          "input_end": 38912,
          "mean_nll": 2.0511355649722747,
          "perplexity": 7.776727072992065,
          "scored_tokens": 2048,
          "seconds": 0.271776717,
          "target_begin": 36864,
          "target_end": 38912,
          "total_nll": 4200.7256370632185
        },
        {
          "first_target": 2048,
          "index": 18,
          "input_begin": 36864,
          "input_end": 40960,
          "mean_nll": 1.890148473520421,
          "perplexity": 6.620351554977687,
          "scored_tokens": 2048,
          "seconds": 0.272027435,
          "target_begin": 38912,
          "target_end": 40960,
          "total_nll": 3871.0240737698223
        },
        {
          "first_target": 2048,
          "index": 19,
          "input_begin": 38912,
          "input_end": 43008,
          "mean_nll": 2.08802111915693,
          "perplexity": 8.068931900271904,
          "scored_tokens": 2048,
          "seconds": 0.272278823,
          "target_begin": 40960,
          "target_end": 43008,
          "total_nll": 4276.267252033393
        },
        {
          "first_target": 2048,
          "index": 20,
          "input_begin": 40960,
          "input_end": 45056,
          "mean_nll": 1.8258198215283103,
          "perplexity": 6.207882289038997,
          "scored_tokens": 2048,
          "seconds": 0.272158914,
          "target_begin": 43008,
          "target_end": 45056,
          "total_nll": 3739.2789944899796
        },
        {
          "first_target": 2048,
          "index": 21,
          "input_begin": 43008,
          "input_end": 47104,
          "mean_nll": 1.78787010694306,
          "perplexity": 5.9767091489292214,
          "scored_tokens": 2048,
          "seconds": 0.272232403,
          "target_begin": 45056,
          "target_end": 47104,
          "total_nll": 3661.557979019387
        },
        {
          "first_target": 2048,
          "index": 22,
          "input_begin": 45056,
          "input_end": 49152,
          "mean_nll": 1.8529249662100007,
          "perplexity": 6.378449011838473,
          "scored_tokens": 2048,
          "seconds": 0.272231973,
          "target_begin": 47104,
          "target_end": 49152,
          "total_nll": 3794.7903307980814
        },
        {
          "first_target": 2048,
          "index": 23,
          "input_begin": 47104,
          "input_end": 51200,
          "mean_nll": 1.948944653639023,
          "perplexity": 7.021273793473396,
          "scored_tokens": 2048,
          "seconds": 0.272352352,
          "target_begin": 49152,
          "target_end": 51200,
          "total_nll": 3991.438650652719
        },
        {
          "first_target": 2048,
          "index": 24,
          "input_begin": 49152,
          "input_end": 53248,
          "mean_nll": 2.05624976388566,
          "perplexity": 7.816600675999136,
          "scored_tokens": 2048,
          "seconds": 0.27256076,
          "target_begin": 51200,
          "target_end": 53248,
          "total_nll": 4211.199516437831
        },
        {
          "first_target": 2048,
          "index": 25,
          "input_begin": 51200,
          "input_end": 55296,
          "mean_nll": 2.0779339250703863,
          "perplexity": 7.987948154188591,
          "scored_tokens": 2048,
          "seconds": 0.272408342,
          "target_begin": 53248,
          "target_end": 55296,
          "total_nll": 4255.608678544151
        },
        {
          "first_target": 2048,
          "index": 26,
          "input_begin": 53248,
          "input_end": 57344,
          "mean_nll": 2.1253865680483077,
          "perplexity": 8.376134808449915,
          "scored_tokens": 2048,
          "seconds": 0.272615169,
          "target_begin": 55296,
          "target_end": 57344,
          "total_nll": 4352.791691362934
        },
        {
          "first_target": 2048,
          "index": 27,
          "input_begin": 55296,
          "input_end": 59392,
          "mean_nll": 2.206638266732696,
          "perplexity": 9.085123237812091,
          "scored_tokens": 2048,
          "seconds": 0.272707729,
          "target_begin": 57344,
          "target_end": 59392,
          "total_nll": 4519.195170268561
        },
        {
          "first_target": 2048,
          "index": 28,
          "input_begin": 57344,
          "input_end": 61440,
          "mean_nll": 2.097771597767034,
          "perplexity": 8.147992661934667,
          "scored_tokens": 2048,
          "seconds": 0.272664949,
          "target_begin": 59392,
          "target_end": 61440,
          "total_nll": 4296.2362322268855
        },
        {
          "first_target": 2048,
          "index": 29,
          "input_begin": 59392,
          "input_end": 63488,
          "mean_nll": 2.007324002542988,
          "perplexity": 7.443372227524008,
          "scored_tokens": 2048,
          "seconds": 0.272868467,
          "target_begin": 61440,
          "target_end": 63488,
          "total_nll": 4110.99955720804
        },
        {
          "first_target": 2128,
          "index": 30,
          "input_begin": 61360,
          "input_end": 65456,
          "mean_nll": 2.1581314450491638,
          "perplexity": 8.654950289466438,
          "scored_tokens": 1968,
          "seconds": 0.272397122,
          "target_begin": 63488,
          "target_end": 65456,
          "total_nll": 4247.202683856754
        }
      ]
    },
    {
      "domain": "chinese_reference",
      "id": "zhwiki-00",
      "input_tokens": 65511,
      "mean_nll": 1.6247369674902634,
      "path": "data/zhwiki/00.txt",
      "perplexity": 5.077083423537842,
      "scored_tokens": 65510,
      "seconds": 8.516838268,
      "total_nll": 106436.51874028715,
      "unscored_tokens": 1,
      "windows": [
        {
          "first_target": 1,
          "index": 0,
          "input_begin": 0,
          "input_end": 4096,
          "mean_nll": 2.079994668933899,
          "perplexity": 8.004426242057212,
          "scored_tokens": 4095,
          "seconds": 0.301726294,
          "target_begin": 1,
          "target_end": 4096,
          "total_nll": 8517.578169284317
        },
        {
          "first_target": 2048,
          "index": 1,
          "input_begin": 2048,
          "input_end": 6144,
          "mean_nll": 2.100261183766019,
          "perplexity": 8.168303062139259,
          "scored_tokens": 2048,
          "seconds": 0.272914656,
          "target_begin": 4096,
          "target_end": 6144,
          "total_nll": 4301.334904352807
        },
        {
          "first_target": 2048,
          "index": 2,
          "input_begin": 4096,
          "input_end": 8192,
          "mean_nll": 2.2140471487873676,
          "perplexity": 9.152683809444921,
          "scored_tokens": 2048,
          "seconds": 0.273104956,
          "target_begin": 6144,
          "target_end": 8192,
          "total_nll": 4534.368560716529
        },
        {
          "first_target": 2048,
          "index": 3,
          "input_begin": 6144,
          "input_end": 10240,
          "mean_nll": 1.793296437256718,
          "perplexity": 6.009228898616271,
          "scored_tokens": 2048,
          "seconds": 0.273359132,
          "target_begin": 8192,
          "target_end": 10240,
          "total_nll": 3672.6711035017584
        },
        {
          "first_target": 2048,
          "index": 4,
          "input_begin": 8192,
          "input_end": 12288,
          "mean_nll": 0.8543262269661565,
          "perplexity": 2.3497906215317825,
          "scored_tokens": 2048,
          "seconds": 0.273314584,
          "target_begin": 10240,
          "target_end": 12288,
          "total_nll": 1749.6601128266884
        },
        {
          "first_target": 2048,
          "index": 5,
          "input_begin": 10240,
          "input_end": 14336,
          "mean_nll": 1.7433476454714523,
          "perplexity": 5.716448068529086,
          "scored_tokens": 2048,
          "seconds": 0.272841687,
          "target_begin": 12288,
          "target_end": 14336,
          "total_nll": 3570.3759779255342
        },
        {
          "first_target": 2048,
          "index": 6,
          "input_begin": 12288,
          "input_end": 16384,
          "mean_nll": 1.9950002442092716,
          "perplexity": 7.3522048233671615,
          "scored_tokens": 2048,
          "seconds": 0.272764239,
          "target_begin": 14336,
          "target_end": 16384,
          "total_nll": 4085.760500140588
        },
        {
          "first_target": 2048,
          "index": 7,
          "input_begin": 14336,
          "input_end": 18432,
          "mean_nll": 1.8089285654726808,
          "perplexity": 6.103903993933182,
          "scored_tokens": 2048,
          "seconds": 0.272761448,
          "target_begin": 16384,
          "target_end": 18432,
          "total_nll": 3704.68570208805
        },
        {
          "first_target": 2048,
          "index": 8,
          "input_begin": 16384,
          "input_end": 20480,
          "mean_nll": 1.9959139674388386,
          "perplexity": 7.35892577377946,
          "scored_tokens": 2048,
          "seconds": 0.273331623,
          "target_begin": 18432,
          "target_end": 20480,
          "total_nll": 4087.6318053147415
        },
        {
          "first_target": 2048,
          "index": 9,
          "input_begin": 18432,
          "input_end": 22528,
          "mean_nll": 1.8495006914388354,
          "perplexity": 6.356644802915052,
          "scored_tokens": 2048,
          "seconds": 0.273765919,
          "target_begin": 20480,
          "target_end": 22528,
          "total_nll": 3787.777416066735
        },
        {
          "first_target": 2048,
          "index": 10,
          "input_begin": 20480,
          "input_end": 24576,
          "mean_nll": 2.136289845802298,
          "perplexity": 8.467961830827347,
          "scored_tokens": 2048,
          "seconds": 0.273547521,
          "target_begin": 22528,
          "target_end": 24576,
          "total_nll": 4375.1216042031065
        },
        {
          "first_target": 2048,
          "index": 11,
          "input_begin": 22528,
          "input_end": 26624,
          "mean_nll": 1.4811802622150467,
          "perplexity": 4.398133569650986,
          "scored_tokens": 2048,
          "seconds": 0.273783499,
          "target_begin": 24576,
          "target_end": 26624,
          "total_nll": 3033.4571770164157
        },
        {
          "first_target": 2048,
          "index": 12,
          "input_begin": 24576,
          "input_end": 28672,
          "mean_nll": 1.8457695716915343,
          "perplexity": 6.3329715912215265,
          "scored_tokens": 2048,
          "seconds": 0.273754759,
          "target_begin": 26624,
          "target_end": 28672,
          "total_nll": 3780.1360828242623
        },
        {
          "first_target": 2048,
          "index": 13,
          "input_begin": 26624,
          "input_end": 30720,
          "mean_nll": 1.4523532171740772,
          "perplexity": 4.2731583625030645,
          "scored_tokens": 2048,
          "seconds": 0.273655351,
          "target_begin": 28672,
          "target_end": 30720,
          "total_nll": 2974.41938877251
        },
        {
          "first_target": 2048,
          "index": 14,
          "input_begin": 28672,
          "input_end": 32768,
          "mean_nll": 1.3210484131903795,
          "perplexity": 3.7473480876577416,
          "scored_tokens": 2048,
          "seconds": 0.273514631,
          "target_begin": 30720,
          "target_end": 32768,
          "total_nll": 2705.507150213897
        },
        {
          "first_target": 2048,
          "index": 15,
          "input_begin": 30720,
          "input_end": 34816,
          "mean_nll": 0.14677550084458243,
          "perplexity": 1.1580939427317412,
          "scored_tokens": 2048,
          "seconds": 0.273512071,
          "target_begin": 32768,
          "target_end": 34816,
          "total_nll": 300.5962257297048
        },
        {
          "first_target": 2048,
          "index": 16,
          "input_begin": 32768,
          "input_end": 36864,
          "mean_nll": 0.09175024266247583,
          "perplexity": 1.0960910311139735,
          "scored_tokens": 2048,
          "seconds": 0.273538052,
          "target_begin": 34816,
          "target_end": 36864,
          "total_nll": 187.9044969727505
        },
        {
          "first_target": 2048,
          "index": 17,
          "input_begin": 34816,
          "input_end": 38912,
          "mean_nll": 2.030121356334072,
          "perplexity": 7.615010432457746,
          "scored_tokens": 2048,
          "seconds": 0.27359025,
          "target_begin": 36864,
          "target_end": 38912,
          "total_nll": 4157.688537772179
        },
        {
          "first_target": 2048,
          "index": 18,
          "input_begin": 36864,
          "input_end": 40960,
          "mean_nll": 2.392360850189517,
          "perplexity": 10.939289505033198,
          "scored_tokens": 2048,
          "seconds": 0.273785069,
          "target_begin": 38912,
          "target_end": 40960,
          "total_nll": 4899.555021188131
        },
        {
          "first_target": 2048,
          "index": 19,
          "input_begin": 38912,
          "input_end": 43008,
          "mean_nll": 2.3396868835621323,
          "perplexity": 10.377986535763647,
          "scored_tokens": 2048,
          "seconds": 0.274404564,
          "target_begin": 40960,
          "target_end": 43008,
          "total_nll": 4791.678737535247
        },
        {
          "first_target": 2048,
          "index": 20,
          "input_begin": 40960,
          "input_end": 45056,
          "mean_nll": 1.4742965458320285,
          "perplexity": 4.367962030842032,
          "scored_tokens": 2048,
          "seconds": 0.274483762,
          "target_begin": 43008,
          "target_end": 45056,
          "total_nll": 3019.3593258639944
        },
        {
          "first_target": 2048,
          "index": 21,
          "input_begin": 43008,
          "input_end": 47104,
          "mean_nll": 0.6221426616326895,
          "perplexity": 1.8629153658571411,
          "scored_tokens": 2048,
          "seconds": 0.274545592,
          "target_begin": 45056,
          "target_end": 47104,
          "total_nll": 1274.148171023748
        },
        {
          "first_target": 2048,
          "index": 22,
          "input_begin": 45056,
          "input_end": 49152,
          "mean_nll": 0.15728924756193305,
          "perplexity": 1.1703340812277716,
          "scored_tokens": 2048,
          "seconds": 0.274495943,
          "target_begin": 47104,
          "target_end": 49152,
          "total_nll": 322.1283790068389
        },
        {
          "first_target": 2048,
          "index": 23,
          "input_begin": 47104,
          "input_end": 51200,
          "mean_nll": 1.713273456526548,
          "perplexity": 5.54708994784956,
          "scored_tokens": 2048,
          "seconds": 0.274526392,
          "target_begin": 49152,
          "target_end": 51200,
          "total_nll": 3508.78403896637
        },
        {
          "first_target": 2048,
          "index": 24,
          "input_begin": 49152,
          "input_end": 53248,
          "mean_nll": 1.699062856621343,
          "perplexity": 5.4688199211367445,
          "scored_tokens": 2048,
          "seconds": 0.274652161,
          "target_begin": 51200,
          "target_end": 53248,
          "total_nll": 3479.6807303605106
        },
        {
          "first_target": 2048,
          "index": 25,
          "input_begin": 51200,
          "input_end": 55296,
          "mean_nll": 1.2721061145397219,
          "perplexity": 3.568360028727861,
          "scored_tokens": 2048,
          "seconds": 0.274305364,
          "target_begin": 53248,
          "target_end": 55296,
          "total_nll": 2605.2733225773504
        },
        {
          "first_target": 2048,
          "index": 26,
          "input_begin": 53248,
          "input_end": 57344,
          "mean_nll": 1.3956022591373392,
          "perplexity": 4.037405404859487,
          "scored_tokens": 2048,
          "seconds": 0.274351764,
          "target_begin": 55296,
          "target_end": 57344,
          "total_nll": 2858.1934267132706
        },
        {
          "first_target": 2048,
          "index": 27,
          "input_begin": 55296,
          "input_end": 59392,
          "mean_nll": 1.5793114079873085,
          "perplexity": 4.851613878190097,
          "scored_tokens": 2048,
          "seconds": 0.274340913,
          "target_begin": 57344,
          "target_end": 59392,
          "total_nll": 3234.429763558008
        },
        {
          "first_target": 2048,
          "index": 28,
          "input_begin": 57344,
          "input_end": 61440,
          "mean_nll": 1.703135644278513,
          "perplexity": 5.491138682347212,
          "scored_tokens": 2048,
          "seconds": 0.274410544,
          "target_begin": 59392,
          "target_end": 61440,
          "total_nll": 3488.0217994823947
        },
        {
          "first_target": 2048,
          "index": 29,
          "input_begin": 59392,
          "input_end": 63488,
          "mean_nll": 2.0750669797910657,
          "perplexity": 7.965079940558572,
          "scored_tokens": 2048,
          "seconds": 0.274478802,
          "target_begin": 61440,
          "target_end": 63488,
          "total_nll": 4249.7371746121025
        },
        {
          "first_target": 2073,
          "index": 30,
          "input_begin": 61415,
          "input_end": 65511,
          "mean_nll": 2.5599871150156215,
          "perplexity": 12.935650638812858,
          "scored_tokens": 2023,
          "seconds": 0.275128327,
          "target_begin": 63488,
          "target_end": 65511,
          "total_nll": 5178.8539336766025
        }
      ]
    },
    {
      "domain": "ninfer_code",
      "id": "ninfer-00",
      "input_tokens": 64899,
      "mean_nll": 0.5087736366130513,
      "path": "data/ninfer/00.txt",
      "perplexity": 1.6632501945556721,
      "scored_tokens": 64898,
      "seconds": 8.577218327,
      "total_nll": 33018.391468913804,
      "unscored_tokens": 1,
      "windows": [
        {
          "first_target": 1,
          "index": 0,
          "input_begin": 0,
          "input_end": 4096,
          "mean_nll": 0.6189394225847403,
          "perplexity": 1.856957549862919,
          "scored_tokens": 4095,
          "seconds": 0.304556458,
          "target_begin": 1,
          "target_end": 4096,
          "total_nll": 2534.5569354845115
        },
        {
          "first_target": 2048,
          "index": 1,
          "input_begin": 2048,
          "input_end": 6144,
          "mean_nll": 0.44736560533081443,
          "perplexity": 1.5641860695243295,
          "scored_tokens": 2048,
          "seconds": 0.275261676,
          "target_begin": 4096,
          "target_end": 6144,
          "total_nll": 916.204759717508
        },
        {
          "first_target": 2048,
          "index": 2,
          "input_begin": 4096,
          "input_end": 8192,
          "mean_nll": 0.5421895521216032,
          "perplexity": 1.71976826504172,
          "scored_tokens": 2048,
          "seconds": 0.275230985,
          "target_begin": 6144,
          "target_end": 8192,
          "total_nll": 1110.4042027450434
        },
        {
          "first_target": 2048,
          "index": 3,
          "input_begin": 6144,
          "input_end": 10240,
          "mean_nll": 0.42837658698002307,
          "perplexity": 1.5347639442716736,
          "scored_tokens": 2048,
          "seconds": 0.275375945,
          "target_begin": 8192,
          "target_end": 10240,
          "total_nll": 877.3152501350872
        },
        {
          "first_target": 2048,
          "index": 4,
          "input_begin": 8192,
          "input_end": 12288,
          "mean_nll": 0.2535548048021738,
          "perplexity": 1.2885979989085439,
          "scored_tokens": 2048,
          "seconds": 0.275502403,
          "target_begin": 10240,
          "target_end": 12288,
          "total_nll": 519.2802402348519
        },
        {
          "first_target": 2048,
          "index": 5,
          "input_begin": 10240,
          "input_end": 14336,
          "mean_nll": 0.5712639930544812,
          "perplexity": 1.7705035418616328,
          "scored_tokens": 2048,
          "seconds": 0.275122177,
          "target_begin": 12288,
          "target_end": 14336,
          "total_nll": 1169.9486577755774
        },
        {
          "first_target": 2048,
          "index": 6,
          "input_begin": 12288,
          "input_end": 16384,
          "mean_nll": 0.39113749919137086,
          "perplexity": 1.4786618142397463,
          "scored_tokens": 2048,
          "seconds": 0.275205416,
          "target_begin": 14336,
          "target_end": 16384,
          "total_nll": 801.0495983439275
        },
        {
          "first_target": 2048,
          "index": 7,
          "input_begin": 14336,
          "input_end": 18432,
          "mean_nll": 0.4388285013761313,
          "perplexity": 1.5508892892003339,
          "scored_tokens": 2048,
          "seconds": 0.275096427,
          "target_begin": 16384,
          "target_end": 18432,
          "total_nll": 898.7207708183169
        },
        {
          "first_target": 2048,
          "index": 8,
          "input_begin": 16384,
          "input_end": 20480,
          "mean_nll": 0.3432489553427129,
          "perplexity": 1.4095196256914087,
          "scored_tokens": 2048,
          "seconds": 0.275302055,
          "target_begin": 18432,
          "target_end": 20480,
          "total_nll": 702.973860541876
        },
        {
          "first_target": 2048,
          "index": 9,
          "input_begin": 18432,
          "input_end": 22528,
          "mean_nll": 0.37983249660347373,
          "perplexity": 1.4620396723116038,
          "scored_tokens": 2048,
          "seconds": 0.275428314,
          "target_begin": 20480,
          "target_end": 22528,
          "total_nll": 777.8969530439142
        },
        {
          "first_target": 2048,
          "index": 10,
          "input_begin": 20480,
          "input_end": 24576,
          "mean_nll": 0.6971702073557249,
          "perplexity": 2.0080622600618923,
          "scored_tokens": 2048,
          "seconds": 0.275572352,
          "target_begin": 22528,
          "target_end": 24576,
          "total_nll": 1427.8045846645246
        },
        {
          "first_target": 2048,
          "index": 11,
          "input_begin": 22528,
          "input_end": 26624,
          "mean_nll": 0.38884755104592506,
          "perplexity": 1.4752796293524673,
          "scored_tokens": 2048,
          "seconds": 0.27590411,
          "target_begin": 24576,
          "target_end": 26624,
          "total_nll": 796.3597845420545
        },
        {
          "first_target": 2048,
          "index": 12,
          "input_begin": 24576,
          "input_end": 28672,
          "mean_nll": 0.4704398506001082,
          "perplexity": 1.6006981064203902,
          "scored_tokens": 2048,
          "seconds": 0.275966899,
          "target_begin": 26624,
          "target_end": 28672,
          "total_nll": 963.4608140290215
        },
        {
          "first_target": 2048,
          "index": 13,
          "input_begin": 26624,
          "input_end": 30720,
          "mean_nll": 0.349971109745511,
          "perplexity": 1.4190265519628458,
          "scored_tokens": 2048,
          "seconds": 0.276001188,
          "target_begin": 28672,
          "target_end": 30720,
          "total_nll": 716.7408327588065
        },
        {
          "first_target": 2048,
          "index": 14,
          "input_begin": 28672,
          "input_end": 32768,
          "mean_nll": 0.5578154888625981,
          "perplexity": 1.7468523108636325,
          "scored_tokens": 2048,
          "seconds": 0.276150018,
          "target_begin": 30720,
          "target_end": 32768,
          "total_nll": 1142.406121190601
        },
        {
          "first_target": 2048,
          "index": 15,
          "input_begin": 30720,
          "input_end": 34816,
          "mean_nll": 0.6674579326416497,
          "perplexity": 1.949275826630469,
          "scored_tokens": 2048,
          "seconds": 0.276289666,
          "target_begin": 32768,
          "target_end": 34816,
          "total_nll": 1366.9538460500985
        },
        {
          "first_target": 2048,
          "index": 16,
          "input_begin": 32768,
          "input_end": 36864,
          "mean_nll": 0.8222960038393646,
          "perplexity": 2.275718903081176,
          "scored_tokens": 2048,
          "seconds": 0.276034499,
          "target_begin": 34816,
          "target_end": 36864,
          "total_nll": 1684.0622158630188
        },
        {
          "first_target": 2048,
          "index": 17,
          "input_begin": 34816,
          "input_end": 38912,
          "mean_nll": 0.4641425411430687,
          "perplexity": 1.5906496873060652,
          "scored_tokens": 2048,
          "seconds": 0.275867069,
          "target_begin": 36864,
          "target_end": 38912,
          "total_nll": 950.5639242610047
        },
        {
          "first_target": 2048,
          "index": 18,
          "input_begin": 36864,
          "input_end": 40960,
          "mean_nll": 0.2737099946896812,
          "perplexity": 1.3148334382633273,
          "scored_tokens": 2048,
          "seconds": 0.276097108,
          "target_begin": 38912,
          "target_end": 40960,
          "total_nll": 560.5580691244671
        },
        {
          "first_target": 2048,
          "index": 19,
          "input_begin": 38912,
          "input_end": 43008,
          "mean_nll": 0.3395242325715688,
          "perplexity": 1.4042793212443438,
          "scored_tokens": 2048,
          "seconds": 0.276105978,
          "target_begin": 40960,
          "target_end": 43008,
          "total_nll": 695.3456283065728
        },
        {
          "first_target": 2048,
          "index": 20,
          "input_begin": 40960,
          "input_end": 45056,
          "mean_nll": 0.5416375679742995,
          "perplexity": 1.7188192421692676,
          "scored_tokens": 2048,
          "seconds": 0.276365135,
          "target_begin": 43008,
          "target_end": 45056,
          "total_nll": 1109.2737392113654
        },
        {
          "first_target": 2048,
          "index": 21,
          "input_begin": 43008,
          "input_end": 47104,
          "mean_nll": 0.7198574170864112,
          "perplexity": 2.0541403044531457,
          "scored_tokens": 2048,
          "seconds": 0.276202627,
          "target_begin": 45056,
          "target_end": 47104,
          "total_nll": 1474.26799019297
        },
        {
          "first_target": 2048,
          "index": 22,
          "input_begin": 45056,
          "input_end": 49152,
          "mean_nll": 0.4600727616771833,
          "perplexity": 1.5841892490677651,
          "scored_tokens": 2048,
          "seconds": 0.276414115,
          "target_begin": 47104,
          "target_end": 49152,
          "total_nll": 942.2290159148714
        },
        {
          "first_target": 2048,
          "index": 23,
          "input_begin": 47104,
          "input_end": 51200,
          "mean_nll": 0.3178185632382353,
          "perplexity": 1.374126921469661,
          "scored_tokens": 2048,
          "seconds": 0.276670663,
          "target_begin": 49152,
          "target_end": 51200,
          "total_nll": 650.8924175119059
        },
        {
          "first_target": 2048,
          "index": 24,
          "input_begin": 49152,
          "input_end": 53248,
          "mean_nll": 0.6911147141722079,
          "perplexity": 1.9959391953469123,
          "scored_tokens": 2048,
          "seconds": 0.276758391,
          "target_begin": 51200,
          "target_end": 53248,
          "total_nll": 1415.4029346246818
        },
        {
          "first_target": 2048,
          "index": 25,
          "input_begin": 51200,
          "input_end": 55296,
          "mean_nll": 0.677323427423286,
          "perplexity": 1.968601569388846,
          "scored_tokens": 2048,
          "seconds": 0.276879971,
          "target_begin": 53248,
          "target_end": 55296,
          "total_nll": 1387.1583793628897
        },
        {
          "first_target": 2048,
          "index": 26,
          "input_begin": 53248,
          "input_end": 57344,
          "mean_nll": 0.8358944710800587,
          "perplexity": 2.3068765600891834,
          "scored_tokens": 2048,
          "seconds": 0.276889521,
          "target_begin": 55296,
          "target_end": 57344,
          "total_nll": 1711.9118767719601
        },
        {
          "first_target": 2048,
          "index": 27,
          "input_begin": 55296,
          "input_end": 59392,
          "mean_nll": 0.8922951714649627,
          "perplexity": 2.440725110191483,
          "scored_tokens": 2048,
          "seconds": 0.277002259,
          "target_begin": 57344,
          "target_end": 59392,
          "total_nll": 1827.4205111602437
        },
        {
          "first_target": 2048,
          "index": 28,
          "input_begin": 57344,
          "input_end": 61440,
          "mean_nll": 0.39708765750492336,
          "perplexity": 1.487486313675536,
          "scored_tokens": 2048,
          "seconds": 0.276911041,
          "target_begin": 59392,
          "target_end": 61440,
          "total_nll": 813.235522570083
        },
        {
          "first_target": 2048,
          "index": 29,
          "input_begin": 59392,
          "input_end": 63488,
          "mean_nll": 0.22661589290987227,
          "perplexity": 1.254347971446954,
          "scored_tokens": 2048,
          "seconds": 0.276582963,
          "target_begin": 61440,
          "target_end": 63488,
          "total_nll": 464.1093486794184
        },
        {
          "first_target": 2685,
          "index": 30,
          "input_begin": 60803,
          "input_end": 64899,
          "mean_nll": 0.43223436093737344,
          "perplexity": 1.540696151847381,
          "scored_tokens": 1411,
          "seconds": 0.268345769,
          "target_begin": 63488,
          "target_end": 64899,
          "total_nll": 609.8826832826339
        }
      ]
    }
  ],
  "timing": {
    "load_seconds": 3.011956385,
    "read_and_tokenize_seconds": 0.053088175,
    "score_seconds": 34.533052827,
    "scored_tokens_per_second": 7562.812396238657,
    "total_seconds": 37.870070505
  }
}