deployment/portability_audit.json

{
  "date": "2026-09-10",
  "git_metadata": {
    "source_layout": "Four evaluation workspaces in one Hugging Face Git repository",
    "runtime_requires_git": false,
    "upstream_revisions": "Pinned per-method manifests and adapter metadata preserve the official source revisions",
    "worldmem_reference": "Frozen metric and VAE dependency source is included under references/worldmem"
  },
  "active_path_audit": {
    "LIVE": "Config and required VAE/final checkpoint paths come from checkout location and explicit arguments. Metric initialization is disabled.",
    "GeometryForcing": "Hydra configuration comes from unpacked checkout; released weights supplied explicitly. Alignment coefficient zero prevents the training-only VGGT download.",
    "DecMem": "Adapter changes to its own checkout; native wan_models/Wan2.1-T2V-1.3B paths are populated by the deployment asset downloader. The selected pipeline creates no T5 encoder.",
    "Matrix-Game2": "Adapter resolves native config from its checkout and VAE, CLIP, tokenizer and distilled weights from the checkpoint directory. The unused legacy skyreels_models VAE wrapper is not constructed."
  },
  "unpacked_constructor_checks": {
    "conditions": "Real production constructors, CPU, fresh cache directories, Hugging Face and Transformers offline; checkpoint arguments point to the separately staged official weights",
    "GeometryForcing": "passed strict loading of 496 tensors and 458818051 parameters",
    "LIVE": "passed strict loading of 486 tensors, 772346944 backbone parameters and 66456019 VAE parameters",
    "Minecraft_methods": "Strict checkpoint loading and actual generation passed for DecMem and Matrix-Game 2.0 in the local H200 smoke suite, job 4780."
  },
  "matrix_flash_attention": {
    "actual_binary_requirement": "Official FlashAttention 2.8.3 wheel used locally references GLIBC_2.32",
    "cluster_baseline": "RHEL 8 glibc 2.28 baseline; cluster host was not queried",
    "prepared_resolution": "The CPU setup builds FlashAttention 2.8.3 with FLASH_ATTENTION_FORCE_BUILD=TRUE and FLASH_ATTN_CUDA_ARCHS=90. Binary/cache reuse is disabled, including on setup reruns.",
    "cluster_build_status": "Built and installed successfully in user setup job84925; target-cluster GPU runtime smoke remains pending.",
    "evidence": "readelf on the downloaded official extension found required symbols through GLIBC_2.32; the DecMem extension requires at most GLIBC_2.14",
    "build_flag_source": "https://github.com/Dao-AILab/flash-attention/blob/v2.8.3/setup.py"
  },
  "decmem_flash_attention": "Audited local 2.7.4.post1 extension requires at most GLIBC_2.14",
  "torch_and_pyav_binary_audit": "Checked351 native Torch/PyAV/FFmpeg libraries across the four original local environments; none require GLIBC symbols above2.28. This is a local binary audit, not a remote host probe.",
  "source_builds": "The setup builds DecMem VSA with compute_90a/sm_90a. Native extension binaries, environments and checkpoints are excluded from the Git source repository.",
  "effective_evaluation_updates": {
    "LIVE_nonfinite_policy": "The native sampler and VAE decoder raise FloatingPointError for NaN/Inf outputs; failed predictions are not replaced with zeros or scored.",
    "LIVE_nonfinite_guard_verification": "Actual CPU sampler/decoder calls passed six probes: NaN and Inf each raise, while finite controls return finite tensors. No checkpoint was needed.",
    "Matrix_action_timing": "Convert WorldMem incoming controls to Matrix outgoing controls by shifting the complete padded action sequence left one frame and appending a no-op.",
    "Matrix_image_precision": "Resize and normalize RGB in float32, then cast normalized values to bfloat16.",
    "RE10K_scene_seed": "master_seed * 1000000 + 1000003 * metadata_index + 7, matching the existing dense reverse-loop evaluator; the first monitor scene at master seed zero uses 4865014602.",
    "LIVE_numpy_seed": "Use scene_seed modulo 2^32 for NumPy while retaining the complete seed for Torch and Python."
  },
  "berzelius_access_or_execution": "No direct cluster access or execution by the agent; user jobs 84923, 84924 and successful setup 84925 were reviewed through uploaded logs and reports.",
  "environment_setup": {
    "entrypoint": "deployment/setup_environment_berzelius.sh",
    "partition": "berzelius-hopper-cpu",
    "cpu_cores": 14,
    "gpu_count": 0,
    "hours": 8,
    "environment_root": "/proj/cvl/users/x_fahkh2/envs",
    "prefixes": [
      "decmem-eval",
      "matrix-game2-eval",
      "live-eval",
      "geometry-forcing-eval"
    ],
    "outputs": "artifacts/environment_setup_JOBID",
    "cuda_build": "CPU setup links the toolkit driver stub through LIBRARY_PATH only. GPU jobs load the real driver; no runtime stub path is exported.",
    "validation": "Bash syntax passed; local minimal shared-library link against the CUDA driver stub passed without RPATH. Target-cluster builds remain unexecuted."
  },
  "separate_smoke_jobs": {
    "entrypoints": [
      "deployment/smoke_decmem_1h200.sh",
      "deployment/smoke_matrix_game2_1h200.sh",
      "deployment/smoke_live_1h200.sh",
      "deployment/smoke_geometry_forcing_1h200.sh"
    ],
    "gpu_count_per_job": 1,
    "gpu_type": "H200",
    "cases_per_method": 1,
    "generated_frames": {
      "GeometryForcing": 104,
      "LIVE": 104,
      "Matrix-Game2": 24,
      "DecMem": 20
    },
    "failure_behavior": "Generation, scoring or aggregation failure stops that method job with a nonzero exit code.",
    "metric_coverage": "Each smoke runs common aggregation, including FID, using its exact frame count and one-case population."
  },
  "local_gpu_smoke": {
    "job_id": 4780,
    "all_passed": true,
    "methods": [
      "geometry-forcing",
      "live",
      "decmem",
      "matrix-game2"
    ],
    "coverage": "Actual H200 generation, common scoring and aggregation for each model; frozen reference probe also passed.",
    "evidence": "Original local workspace artifacts/smoke-suite-4780/status.tsv and smoke_results.json",
    "berzelius_implication": "Local validation only; the user must run the target-cluster setup and separate smoke jobs."
  },
  "setup_policy_review": {
    "date": "2026-09-10",
    "change": "Install official pinned Miniforge3 24.7.1-2 under /proj/cvl/users/x_fahkh2/envs/baseline-miniforge3 with bash -b -p; call its bin/conda explicitly and create method environments with --override-channels --channel conda-forge. No Miniforge site module, ambient conda or shell-profile initialization is required.",
    "source": "https://www.nsc.liu.se/support/systems/berzelius-software/berzelius-conda-mamba/",
    "resource_request": "Dedicated berzelius-hopper-cpu partition; one node, one task, 14 CPU cores, zero GPUs, eight-hour maximum; no reservation or scheduling override.",
    "execution": "Submitted batch script performs all environment creation, installation, extension compilation, downloads and CPU data checks on the allocated compute node. It submits no further jobs.",
    "storage": "Environment prefixes, persistent caches/checkpoints, temporary files and Triton cache are inside the user project directory. TMPDIR and TRITON_CACHE_DIR both equal /proj/cvl/users/x_fahkh2/worldmem-baseline-evals/cache/ in all nine deployment jobs. Existing dataset files are only read.",
    "remaining_remote_checks": "CPU setup84925, all four dependency checks, downloads, metric caching and dataset preflight completed. GPU model loading/generation/scoring and current job allocation remain to be checked through the user-submitted smoke jobs.",
    "compiler_parallelism": "MAX_JOBS=2 and NVCC_THREADS=4 are explicit; FlashAttention builds cannot inherit a larger NVCC thread setting from the submission shell.",
    "validation": "Pinned official Miniforge installer completed locally with -b -p; its exact conda creation command produced a Python3.10/pip environment containing only conda-forge packages. Python/SSL and wheel-build imports and pip check passed. Setup passes bash -n. No direct cluster access or execution by the agent.",
    "installation_policy": "https://www.nsc.liu.se/software/installation-policy/"
  },
  "berzelius_setup_job_84923": {
    "status": "failed during DecMem FlashAttention installation; dataset preflight and other method dependency installation were not reached",
    "source": "https://huggingface.co/Amshaker/WorldMem/blob/a3a78a134b47657cfe0979c6d12d2fefe5d3c335/slurm_logs/worldmem-baseline-evals/worldmem-baselines-envs-84923_4294967294.out",
    "created_environment_prefixes": [
      "/proj/cvl/users/x_fahkh2/envs/decmem-eval",
      "/proj/cvl/users/x_fahkh2/envs/matrix-game2-eval",
      "/proj/cvl/users/x_fahkh2/envs/live-eval",
      "/proj/cvl/users/x_fahkh2/envs/geometry-forcing-eval"
    ],
    "completed_packages": "DecMem Torch 2.5.1+cu121 and requirements_decmem.txt installed successfully before FlashAttention failed",
    "failure": "FlashAttention 2.7.4.post1 CachedWheelsCommand downloaded its official wheel, then os.rename failed with Errno 18 between source build scratch and the persistent project pip cache.",
    "fix": "Add --no-cache-dir only to the DecMem FlashAttention pip command, so source and ephemeral wheel staging both use TMPDIR. Installed packages remain in the requested project environment.",
    "conda_evidence": "The submitted job used Conda 4.10.1 and pkgs/main. This is inconsistent with the latest explicit Miniforge/conda-forge setup; the uploaded log does not contain a script revision.",
    "retry": "Pull the corrected repository and resubmit the same CPU setup job. Existing prefixes are reused, and installed matching packages need no reinstall. No environments are deleted.",
    "platform_evidence": "CUDA_HOME in the job points into /software/sse2/el9_x86_hopper/. The log does not report host OS release or glibc version; the prior RHEL 8 compatibility baseline was documentation-based, not remotely verified.",
    "local_regression": {
      "pip_version": "26.2.1",
      "mechanism": "Actual pip option parsing and WheelCache/_get_cache_dir selection, followed by the same os.rename operation on a small stand-in file across two local filesystems",
      "original_errno": 18,
      "fixed_errno": 0,
      "fixed_destination": "Same TMPDIR filesystem as source",
      "no_network_or_installation": true,
      "setup_syntax": "bash -n passed"
    }
  },
  "berzelius_setup_job_84924": {
    "status": "failed immediately at unavailable Miniforge module; environment and model installation were not reached",
    "source": "https://huggingface.co/Amshaker/WorldMem/blob/d9e725b97b852e5da0f57ec06d6773b184c0f8e3/slurm_logs/worldmem-baseline-evals/worldmem-baselines-envs-84924_4294967294.out",
    "failure": "Lmod reports Miniforge3/24.7.1-2-hpc1-bdist unknown on the target system. The version was present in NSC documentation but was not verified on this Hopper host.",
    "fix": "Replace the unavailable module with the official pinned installer at an explicit project environment prefix; preserve and reuse all existing method environments.",
    "native_build_path": "Matrix FlashAttention build receives its own environment bin directory in PATH, so Ninja and Python subprocesses resolve to the intended environment.",
    "local_validation": {
      "official_installer_batch_install": "passed",
      "installed_conda_version": "24.7.1",
      "exact_python310_pip_creation_command": "passed",
      "new_environment_packages": 27,
      "all_packages_from_conda_forge": true,
      "python_ssl_import": "passed",
      "wheel_build_import": "passed in fresh subprocess",
      "pip_check": "passed",
      "setup_bash_syntax": "passed",
      "remaining": "Target-cluster setup and model dependency builds still need the user-submitted job."
    }
  },
  "project_temporary_storage": {
    "source": "Explicit user rule, 2026-09-10",
    "TMPDIR": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/cache/",
    "TRITON_CACHE_DIR": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/cache/",
    "scope": "CPU setup, all four one-H200 smoke jobs and all four eight-H200 full evaluation jobs",
    "flashattention_fix": "Keep --no-cache-dir on DecMem FlashAttention so source and temporary wheel staging share TMPDIR; TMPDIR now resides in project storage."
  },
  "berzelius_setup_job_84925": {
    "status": "setup script completed successfully",
    "source": "https://huggingface.co/Amshaker/WorldMem/blob/4e4dd583dd0e12db21c7775d62dad81beefb034d/slurm_logs/worldmem-baseline-evals/worldmem-baselines-envs-84925_4294967294.out",
    "native_installations": [
      "flash-attn2.7.4.post1",
      "vsa0.0.3",
      "flash-attn2.8.3"
    ],
    "package_checks_passed": 4,
    "downloads": "all six author checkpoint snapshots and common metric weights completed",
    "data_preflight": {
      "minecraft_cases": 300,
      "re10k_cases": 100,
      "errors": []
    },
    "gpu_execution": "not yet tested on Berzelius; separate GPU smoke jobs are next"
  },
  "gpu_launcher_initialization": {
    "change": "Remove the unused global conda info/source lookup from all eight GPU launchers; model and metric interpreter paths remain explicit.",
    "reason": "The project-local setup does not initialize the login shell. No conda command or activation is needed to run the absolute method interpreter paths.",
    "scope": "four smoke and four full evaluation scripts; resources, model arguments, data paths and required project cache paths unchanged",
    "verification": "All eight startup fragments pass without conda on PATH (module loader stubbed); bash syntax passes. Each launcher differs from the previous verified revision only by deleting its unused conda initialization line. Both required cache variables retain their exact approved project path."
  },
  "berzelius_smokes_84952_84955": {
    "source_revision": "f8623bb73f72c0e068f67aa0ed72b72a8a894813",
    "source_folder": "https://huggingface.co/Amshaker/WorldMem/tree/f8623bb73f72c0e068f67aa0ed72b72a8a894813/slurm_logs/worldmem-baseline-evals",
    "LIVE": {
      "job_id": 84954,
      "status": "completed generation, scoring and aggregation",
      "generated_frames": 104
    },
    "Matrix-Game2": {
      "job_id": 84953,
      "status": "completed generation, scoring and aggregation",
      "generated_frames": 24
    },
    "GeometryForcing": {
      "job_id": 84955,
      "status": "uploaded snapshot ends during generation without an error",
      "generated_frames_in_snapshot": 37,
      "requested_frames": 104
    },
    "DecMem": {
      "job_id": 84952,
      "status": "failed before generation",
      "cause": "VSA eagerly imports pytest from block_sparse_attn_triton.py, but VSA package metadata declares only torch and the DecMem recipe omitted pytest."
    }
  },
  "decmem_pytest_repair": {
    "required_version": "pytest==9.1.1",
    "working_local_version": "9.1.1",
    "repair_job": "deployment/repair_decmem_environment_berzelius.sh",
    "resources": "berzelius-hopper-cpu, one task, two CPUs, no GPUs, 15 minutes",
    "scope": "Install pytest with its dependencies into the existing decmem-eval environment; no VSA rebuild and no changes to the other model environments. The normal DecMem requirements now include the same pin.",
    "validation": "Exact pytest installation and pip check passed in an isolated Python3.10 environment. The actual complete VSA Triton module imports with pytest9.1.1 and Torch2.5.1+cu121 with zero GPUs visible. Repair script syntax and exact project cache exports passed. GPU generation still requires a DecMem smoke retry.",
    "import_closure": "All four VSA Python files match the working local package. Other top-level dependencies are torch, triton, vsa_cuda and the standard library; torch/triton are installed and the GPU log reached pytest after importing vsa_cuda."
  }
}