| { |
| "benchmark": { |
| "comparability_note": "The official row predates the current deterministic/original-byte DOVER and capped depth-residual fixes; no official run-level artifact identifies the exact private evaluator or generation configuration.", |
| "contemporaneous_legacy": { |
| "commit": "1b84911a64d9d73c3267fa74baa640fab336b7e0", |
| "source_files": { |
| "conditional_generation/benchmark_pipelines/scores/transfer_bench/metrics_canny_blur_depth.py": { |
| "bytes": 11376, |
| "sha256": "63b9178580a72395be8ffdd841a5717a8d84f43e9f80e8736d6d802adc15bfb3" |
| }, |
| "conditional_generation/compute_diversity_score.py": { |
| "bytes": 11408, |
| "sha256": "c3c48fefcf8fd3b31692869300d43feeca4b95a67a65f5b6a9039a7e9140b0f1" |
| }, |
| "conditional_generation/compute_metrics.py": { |
| "bytes": 30773, |
| "sha256": "7cbb66e4c5fea2a4ac4a538614577c6ccc42f92f5f5c825fb86241f5538b0d13" |
| }, |
| "conditional_generation/models/dover.py": { |
| "bytes": 8856, |
| "sha256": "8e0729e2d37b34f0ee4f1d055a0863c1f6fd49f2bea7b705c7843fb87b4b0ed2" |
| }, |
| "conditional_generation/pyproject.toml": { |
| "bytes": 1206, |
| "sha256": "646ec4e39a9946eaab8e93f4aa4cf3fab10c9aea85e08257e920f1c255233b28" |
| } |
| } |
| }, |
| "current": { |
| "commit": "2f3b687410029b98397fbc51fa4de36bfd45627d", |
| "source_files": { |
| "conditional_generation/benchmark_pipelines/scores/transfer_bench/metrics_canny_blur_depth.py": { |
| "bytes": 11616, |
| "sha256": "5aa1beabc56d7f0df23b4ef7b049d4adb41c4cab803de3cb620460827a283962" |
| }, |
| "conditional_generation/compute_diversity_score.py": { |
| "bytes": 11408, |
| "sha256": "c3c48fefcf8fd3b31692869300d43feeca4b95a67a65f5b6a9039a7e9140b0f1" |
| }, |
| "conditional_generation/compute_metrics.py": { |
| "bytes": 30882, |
| "sha256": "5001d3bdc81e579c7792b7f737e6620960e2057e638f2e75d419fb06d3d5556d" |
| }, |
| "conditional_generation/models/dover.py": { |
| "bytes": 9354, |
| "sha256": "9473cbb64744a7436c377eab11fa779c52f85cd0782b2db8d7d29539307e9cb3" |
| }, |
| "conditional_generation/pyproject.toml": { |
| "bytes": 1206, |
| "sha256": "646ec4e39a9946eaab8e93f4aa4cf3fab10c9aea85e08257e920f1c255233b28" |
| } |
| } |
| }, |
| "repository": "https://github.com/SHI-Labs/physical-ai-bench" |
| }, |
| "condition": "Depth", |
| "diversity_videos_per_variant": 3600, |
| "main_cases": 600, |
| "metric_definitions": { |
| "blur_ssim": "SSIM between bilaterally blurred source and generated RGB videos; higher is better and it is not a sharpness score", |
| "depth_si_rmse": "Median-scale-aligned depth RMSE per frame using Video Depth Anything estimates; lower is better", |
| "diversity": "Mean VGG-LPIPS over 30 ordered pairs from six prompts sharing one Depth control; higher means more variation", |
| "edge_f1": "F1 between Canny edge maps over all video pixels; higher is better", |
| "mask_miou": "Prompt-grounded SAM2 mask IoU after Hungarian matching; higher is better", |
| "quality_score": "DOVER technical video quality score; higher is better" |
| }, |
| "protocol": { |
| "diversity_evaluator": "current", |
| "lora_checkpoint_sha256": "47f94f7fc85ed53b5480b28c2bfa3a05659bcf4aeb1278004c0860224787e78a", |
| "main_evaluator": "legacy", |
| "output_fps": 24, |
| "run_contract_sha256": "0fa06f59d3f8a54041ffd9d0982f4f3608fdd6d3fa024bc9d4e7dde403aaf489", |
| "shared_frames": 121, |
| "shared_seed": 43 |
| }, |
| "resolution_buckets": [ |
| { |
| "count": 200, |
| "generation": "640x480", |
| "input": "640x480", |
| "source": "AgiBot" |
| }, |
| { |
| "count": 195, |
| "generation": "1280x704", |
| "input": "1280x720", |
| "source": "OpenDV" |
| }, |
| { |
| "count": 3, |
| "generation": "1280x672", |
| "input": "1920x1016", |
| "source": "OpenDV" |
| }, |
| { |
| "count": 2, |
| "generation": "1280x544", |
| "input": "1920x808", |
| "source": "OpenDV" |
| }, |
| { |
| "count": 200, |
| "generation": "1280x704", |
| "input": "1920x1080", |
| "source": "Ego-Exo4D" |
| } |
| ], |
| "rows": [ |
| { |
| "classification": "official_quoted_not_locally_measured", |
| "id": "official_reported", |
| "label": "Wan2.2-Fun-5B-Control (official leaderboard)", |
| "metrics": { |
| "blur_ssim": 0.556, |
| "depth_si_rmse": 1.819, |
| "diversity": 0.481, |
| "edge_f1": 0.106, |
| "mask_miou": 0.615, |
| "quality_score": 9.317 |
| }, |
| "protocol": "\u5b98\u65b9 leaderboard\uff1b\u914d\u7f6e\u672a\u62ab\u9732" |
| }, |
| { |
| "classification": "local_measured", |
| "id": "local_multistep", |
| "label": "Wan2.2-Fun-5B-Control (local multistep)", |
| "metrics": { |
| "blur_ssim": 0.5572094391287489, |
| "depth_si_rmse": 4.836493481093092, |
| "diversity": 0.4744977790738146, |
| "edge_f1": 0.09937129978006591, |
| "mask_miou": 0.6002169333646695, |
| "quality_score": 8.961508704821268 |
| }, |
| "protocol": "\u672c\u5730 40-step \u00b7 CFG 6 \u00b7 TeaCache" |
| }, |
| { |
| "classification": "local_measured", |
| "id": "base4_cfgfree", |
| "label": "Wan2.2-Fun-5B-Control (local direct 4-step)", |
| "metrics": { |
| "blur_ssim": 0.6823773058925994, |
| "depth_si_rmse": 2.6539159381803015, |
| "diversity": 0.24838090770112145, |
| "edge_f1": 0.0889015064608313, |
| "mask_miou": 0.636834080417951, |
| "quality_score": 8.89601070801417 |
| }, |
| "protocol": "\u672c\u5730 4-step \u00b7 CFG-free \u00b7 no LoRA" |
| }, |
| { |
| "classification": "local_measured", |
| "id": "lora1600_4step_cfgfree", |
| "label": "Wan2.2-Fun-5B-Control + DMD LoRA step 1600 (4-step)", |
| "metrics": { |
| "blur_ssim": 0.5439233481351533, |
| "depth_si_rmse": 1.8050484347511278, |
| "diversity": 0.4581396648420228, |
| "edge_f1": 0.09165980415187946, |
| "mask_miou": 0.5890168254077435, |
| "quality_score": 9.290268224477767 |
| }, |
| "protocol": "\u672c\u5730 4-step \u00b7 CFG-free \u00b7 LoRA step 1600" |
| } |
| ] |
| } |
|
|
