evaluation/metrics.json

{
  "schema_version": 1,
  "release": "AlicanKiraz0/Kizagan-TTS-v1.0",
  "measurement_scope": {
    "status": "source_derived_frozen",
    "model": "step1500",
    "model_repo": "AlicanKiraz0/Kizagan-TTS-step_1500",
    "model_revision": "458112a0fc3d850de38951796290e48ef710bc26",
    "source_commit": "f772e498a45fbb5fb8e13fbf9b9c48be9fe33e69",
    "protocol_sha256": "2f45c9f8e6de60d5034849050cb35d6eeeccd69cfb15bc66cb384dc119b7421c",
    "runner_sha256": "4e2ffc2efa2b81db94818b118e0bd407ce6d7aa88c14dba16997fc54d7f8545e",
    "reference": {
      "id": "r2",
      "sha256": "5f36097c430ae461dc55d76ebe4ef897442e5a34f19c48cfdb9e909afd85c18d",
      "duration_s": 6.175,
      "header_duration_s": 6.175,
      "manifest_cut_duration_s": 6.175,
      "sample_rate_hz": 24000,
      "channels": 1
    },
    "source_artifacts": {
      "cases": "analysis/naturalness/longform_replication/cases.jsonl",
      "gpu_records": "analysis/naturalness/longform_replication/gpu/records.jsonl",
      "gpu_run": "analysis/naturalness/longform_replication/gpu/run.json",
      "assembly_offsets": "analysis/naturalness/longform_replication/assembly/offsets.json",
      "asr_scores": "analysis/naturalness/longform_replication/asr/scores.json",
      "asr_settings": "analysis/naturalness/longform_replication/asr/settings.json",
      "human_summary": "analysis/naturalness/longform_replication/human/response_02/summary.json"
    }
  },
  "runtime": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition",
    "dtype": "bfloat16",
    "streaming": true,
    "seed": 42,
    "cfg": 2.0,
    "timesteps": 16,
    "output_sample_rate_hz": 48000,
    "timing_excludes": [
      "file_io",
      "model_load",
      "inter_call_gaps",
      "assembly"
    ],
    "timing_note": "The sentence-reset aggregate is sum(elapsed_s) / sum(duration_s) across calls. It is not end-to-end service latency.",
    "warmup_note": "The library whole-call row was the first model call in the process; remaining rows were warm. First-chunk figures exclude model load and are not a matched warmup-latency comparison."
  },
  "performance": [
    {
      "case_id": "longrep_01",
      "label": "library",
      "word_count": 151,
      "sentence_count": 12,
      "whole_call": {
        "audio_duration_s": 59.04,
        "elapsed_s": 21.778395,
        "rtf": 0.368875,
        "first_chunk_e2e_s": 0.582673,
        "peak_allocated_mib": 5638.327,
        "peak_reserved_mib": 5978.0
      },
      "sentence_reset": {
        "segments": 12,
        "audio_duration_sum_s": 68.16,
        "elapsed_sum_s": 24.521308,
        "aggregate_elapsed_over_audio": 0.359761,
        "first_segment_first_chunk_e2e_s": 0.062772,
        "peak_allocated_mib": 5750.052,
        "peak_reserved_mib": 6114.0
      }
    },
    {
      "case_id": "longrep_02",
      "label": "platform",
      "word_count": 149,
      "sentence_count": 13,
      "whole_call": {
        "audio_duration_s": 59.36,
        "elapsed_s": 21.451940,
        "rtf": 0.361387,
        "first_chunk_e2e_s": 0.067999,
        "peak_allocated_mib": 5749.552,
        "peak_reserved_mib": 6114.0
      },
      "sentence_reset": {
        "segments": 13,
        "audio_duration_sum_s": 66.72,
        "elapsed_sum_s": 24.042400,
        "aggregate_elapsed_over_audio": 0.360348,
        "first_segment_first_chunk_e2e_s": 0.067221,
        "peak_allocated_mib": 5750.052,
        "peak_reserved_mib": 6114.0
      }
    }
  ],
  "asr": {
    "method": "faster-whisper-large-v3-turbo",
    "repo": "mobiuslabsgmbh/faster-whisper-large-v3-turbo",
    "revision": "0a363e9161cbc7ed1431c9597a8ceaf0c4f78fcf",
    "model_provenance_sha256": "57039e87eae24b070400ee959e1c0222982e8188f9113f8415dcdfb7215156c4",
    "language": "tr",
    "normalization": "Turkish I->ı, İ->i; Unicode punctuation/symbols to spaces; whitespace collapsed",
    "word_error_counts": [
      {"case_id": "longrep_01", "variant": "whole_call", "errors": 1, "reference_words": 151, "wer": 0.006623},
      {"case_id": "longrep_01", "variant": "sentence_reset", "errors": 0, "reference_words": 151, "wer": 0.0},
      {"case_id": "longrep_02", "variant": "whole_call", "errors": 2, "reference_words": 149, "wer": 0.013423},
      {"case_id": "longrep_02", "variant": "sentence_reset", "errors": 2, "reference_words": 149, "wer": 0.013423}
    ]
  },
  "human_one_user": {
    "respondents": 1,
    "trials": 2,
    "scale": "1-5",
    "ratings": [
      {"case_id": "longrep_01", "sentence_reset": {"naturalness": 5, "similarity": 5}, "whole_call": {"naturalness": 1, "similarity": 1}},
      {"case_id": "longrep_02", "sentence_reset": {"naturalness": 4, "similarity": 5}, "whole_call": {"naturalness": 1, "similarity": 1}}
    ],
    "preferences": {"longrep_01": "sentence_reset", "longrep_02": "sentence_reset", "sentence_reset_count": 2, "trial_count": 2},
    "artifact_reports": {
      "longrep_01": {"sentence_reset": ["none"], "whole_call": ["other"]},
      "longrep_02": {"sentence_reset": [], "sentence_reset_status": "unknown_unanswered_artifact_field", "whole_call": ["other"]}
    },
    "limits": ["one user", "no pooled MOS", "no population success rate", "no universal or checkpoint superiority claim"]
  }
}