{
  "benchmark": {
    "comparability_note": "The official row predates the current deterministic/original-byte DOVER and capped depth-residual fixes; no official run-level artifact identifies the exact private evaluator or generation configuration.",
    "contemporaneous_legacy": {
      "commit": "1b84911a64d9d73c3267fa74baa640fab336b7e0",
      "source_files": {
        "conditional_generation/benchmark_pipelines/scores/transfer_bench/metrics_canny_blur_depth.py": {
          "bytes": 11376,
          "sha256": "63b9178580a72395be8ffdd841a5717a8d84f43e9f80e8736d6d802adc15bfb3"
        },
        "conditional_generation/compute_diversity_score.py": {
          "bytes": 11408,
          "sha256": "c3c48fefcf8fd3b31692869300d43feeca4b95a67a65f5b6a9039a7e9140b0f1"
        },
        "conditional_generation/compute_metrics.py": {
          "bytes": 30773,
          "sha256": "7cbb66e4c5fea2a4ac4a538614577c6ccc42f92f5f5c825fb86241f5538b0d13"
        },
        "conditional_generation/models/dover.py": {
          "bytes": 8856,
          "sha256": "8e0729e2d37b34f0ee4f1d055a0863c1f6fd49f2bea7b705c7843fb87b4b0ed2"
        },
        "conditional_generation/pyproject.toml": {
          "bytes": 1206,
          "sha256": "646ec4e39a9946eaab8e93f4aa4cf3fab10c9aea85e08257e920f1c255233b28"
        }
      }
    },
    "current": {
      "commit": "2f3b687410029b98397fbc51fa4de36bfd45627d",
      "source_files": {
        "conditional_generation/benchmark_pipelines/scores/transfer_bench/metrics_canny_blur_depth.py": {
          "bytes": 11616,
          "sha256": "5aa1beabc56d7f0df23b4ef7b049d4adb41c4cab803de3cb620460827a283962"
        },
        "conditional_generation/compute_diversity_score.py": {
          "bytes": 11408,
          "sha256": "c3c48fefcf8fd3b31692869300d43feeca4b95a67a65f5b6a9039a7e9140b0f1"
        },
        "conditional_generation/compute_metrics.py": {
          "bytes": 30882,
          "sha256": "5001d3bdc81e579c7792b7f737e6620960e2057e638f2e75d419fb06d3d5556d"
        },
        "conditional_generation/models/dover.py": {
          "bytes": 9354,
          "sha256": "9473cbb64744a7436c377eab11fa779c52f85cd0782b2db8d7d29539307e9cb3"
        },
        "conditional_generation/pyproject.toml": {
          "bytes": 1206,
          "sha256": "646ec4e39a9946eaab8e93f4aa4cf3fab10c9aea85e08257e920f1c255233b28"
        }
      }
    },
    "repository": "https://github.com/SHI-Labs/physical-ai-bench"
  },
  "condition": "Depth",
  "diversity_videos_per_variant": 3600,
  "main_cases": 600,
  "metric_definitions": {
    "blur_ssim": "SSIM between bilaterally blurred source and generated RGB videos; higher is better and it is not a sharpness score",
    "depth_si_rmse": "Median-scale-aligned depth RMSE per frame using Video Depth Anything estimates; lower is better",
    "diversity": "Mean VGG-LPIPS over 30 ordered pairs from six prompts sharing one Depth control; higher means more variation",
    "edge_f1": "F1 between Canny edge maps over all video pixels; higher is better",
    "mask_miou": "Prompt-grounded SAM2 mask IoU after Hungarian matching; higher is better",
    "quality_score": "DOVER technical video quality score; higher is better"
  },
  "protocol": {
    "diversity_evaluator": "current",
    "lora_checkpoint_sha256": "47f94f7fc85ed53b5480b28c2bfa3a05659bcf4aeb1278004c0860224787e78a",
    "main_evaluator": "legacy",
    "output_fps": 24,
    "run_contract_sha256": "0fa06f59d3f8a54041ffd9d0982f4f3608fdd6d3fa024bc9d4e7dde403aaf489",
    "shared_frames": 121,
    "shared_seed": 43
  },
  "resolution_buckets": [
    {
      "count": 200,
      "generation": "640x480",
      "input": "640x480",
      "source": "AgiBot"
    },
    {
      "count": 195,
      "generation": "1280x704",
      "input": "1280x720",
      "source": "OpenDV"
    },
    {
      "count": 3,
      "generation": "1280x672",
      "input": "1920x1016",
      "source": "OpenDV"
    },
    {
      "count": 2,
      "generation": "1280x544",
      "input": "1920x808",
      "source": "OpenDV"
    },
    {
      "count": 200,
      "generation": "1280x704",
      "input": "1920x1080",
      "source": "Ego-Exo4D"
    }
  ],
  "rows": [
    {
      "classification": "official_quoted_not_locally_measured",
      "id": "official_reported",
      "label": "Wan2.2-Fun-5B-Control (official leaderboard)",
      "metrics": {
        "blur_ssim": 0.556,
        "depth_si_rmse": 1.819,
        "diversity": 0.481,
        "edge_f1": 0.106,
        "mask_miou": 0.615,
        "quality_score": 9.317
      },
      "protocol": "\u5b98\u65b9 leaderboard\uff1b\u914d\u7f6e\u672a\u62ab\u9732"
    },
    {
      "classification": "local_measured",
      "id": "local_multistep",
      "label": "Wan2.2-Fun-5B-Control (local multistep)",
      "metrics": {
        "blur_ssim": 0.5572094391287489,
        "depth_si_rmse": 4.836493481093092,
        "diversity": 0.4744977790738146,
        "edge_f1": 0.09937129978006591,
        "mask_miou": 0.6002169333646695,
        "quality_score": 8.961508704821268
      },
      "protocol": "\u672c\u5730 40-step \u00b7 CFG 6 \u00b7 TeaCache"
    },
    {
      "classification": "local_measured",
      "id": "base4_cfgfree",
      "label": "Wan2.2-Fun-5B-Control (local direct 4-step)",
      "metrics": {
        "blur_ssim": 0.6823773058925994,
        "depth_si_rmse": 2.6539159381803015,
        "diversity": 0.24838090770112145,
        "edge_f1": 0.0889015064608313,
        "mask_miou": 0.636834080417951,
        "quality_score": 8.89601070801417
      },
      "protocol": "\u672c\u5730 4-step \u00b7 CFG-free \u00b7 no LoRA"
    },
    {
      "classification": "local_measured",
      "id": "lora1600_4step_cfgfree",
      "label": "Wan2.2-Fun-5B-Control + DMD LoRA step 1600 (4-step)",
      "metrics": {
        "blur_ssim": 0.5439233481351533,
        "depth_si_rmse": 1.8050484347511278,
        "diversity": 0.4581396648420228,
        "edge_f1": 0.09165980415187946,
        "mask_miou": 0.5890168254077435,
        "quality_score": 9.290268224477767
      },
      "protocol": "\u672c\u5730 4-step \u00b7 CFG-free \u00b7 LoRA step 1600"
    }
  ]
}