Upload folder using huggingface_hub

audio_vae.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:442119db8eaf976f83a105cc8c5f2c930e0f625062f9776b10adcdb7bc96c416
+size 106509048

config.json

@@ -0,0 +1,39 @@
+{
+  "model_version": "2.3.0",
+  "is_v2": true,
+  "model_type": "AudioVideo",
+  "num_attention_heads": 32,
+  "attention_head_dim": 128,
+  "in_channels": 128,
+  "out_channels": 128,
+  "num_layers": 48,
+  "cross_attention_dim": 4096,
+  "caption_channels": null,
+  "apply_gated_attention": true,
+  "audio_num_attention_heads": 32,
+  "audio_attention_head_dim": 64,
+  "audio_in_channels": 128,
+  "audio_out_channels": 128,
+  "audio_cross_attention_dim": 2048,
+  "positional_embedding_theta": 10000.0,
+  "positional_embedding_max_pos": [
+    20,
+    2048,
+    2048
+  ],
+  "audio_positional_embedding_max_pos": [
+    20
+  ],
+  "timestep_scale_multiplier": 1000,
+  "av_ca_timestep_scale_multiplier": 1000,
+  "norm_eps": 1e-06,
+  "connector_positional_embedding_max_pos": [
+    4096
+  ],
+  "connector_rope_type": "SPLIT",
+  "variants": {
+    "distilled": {
+      "cross_attention_adaln": true
+    }
+  }
+}

connector.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5885c2bd7af4e761b3257cf5b17bc1fdb2d187cea3ca520ea723b6a129b7029f
+size 6344495512

embedded_config.json

@@ -0,0 +1,356 @@
+{
+  "transformer": {
+    "_class_name": "AVTransformer3DModel",
+    "activation_fn": "gelu-approximate",
+    "attention_bias": true,
+    "attention_head_dim": 128,
+    "attention_type": "default",
+    "caption_channels": 3840,
+    "cross_attention_dim": 4096,
+    "double_self_attention": false,
+    "dropout": 0.0,
+    "in_channels": 128,
+    "norm_elementwise_affine": false,
+    "norm_eps": 1e-06,
+    "norm_num_groups": 32,
+    "num_attention_heads": 32,
+    "num_embeds_ada_norm": 1000,
+    "num_layers": 48,
+    "num_vector_embeds": null,
+    "only_cross_attention": false,
+    "cross_attention_norm": true,
+    "out_channels": 128,
+    "upcast_attention": false,
+    "use_linear_projection": false,
+    "qk_norm": "rms_norm",
+    "standardization_norm": "rms_norm",
+    "positional_embedding_type": "rope",
+    "positional_embedding_theta": 10000.0,
+    "positional_embedding_max_pos": [
+      20,
+      2048,
+      2048
+    ],
+    "timestep_scale_multiplier": 1000,
+    "av_ca_timestep_scale_multiplier": 1000.0,
+    "causal_temporal_positioning": true,
+    "audio_num_attention_heads": 32,
+    "audio_attention_head_dim": 64,
+    "use_audio_video_cross_attention": true,
+    "share_ff": false,
+    "audio_out_channels": 128,
+    "audio_cross_attention_dim": 2048,
+    "audio_positional_embedding_max_pos": [
+      20
+    ],
+    "av_cross_ada_norm": true,
+    "use_embeddings_connector": true,
+    "connector_attention_head_dim": 128,
+    "connector_num_attention_heads": 32,
+    "connector_num_layers": 8,
+    "connector_positional_embedding_max_pos": [
+      4096
+    ],
+    "connector_num_learnable_registers": 128,
+    "connector_norm_output": true,
+    "use_middle_indices_grid": true,
+    "apply_gated_attention": true,
+    "connector_apply_gated_attention": true,
+    "caption_projection_first_linear": false,
+    "caption_projection_second_linear": false,
+    "caption_proj_input_norm": false,
+    "connector_learnable_registers_std": 1,
+    "caption_proj_before_connector": true,
+    "audio_connector_attention_head_dim": 64,
+    "audio_connector_num_attention_heads": 32,
+    "cross_attention_adaln": true,
+    "text_encoder_norm_type": "per_token_rms",
+    "rope_type": "split",
+    "frequencies_precision": "float64"
+  },
+  "vae": {
+    "_class_name": "CausalVideoAutoencoder",
+    "dims": 3,
+    "in_channels": 3,
+    "out_channels": 3,
+    "latent_channels": 128,
+    "encoder_blocks": [
+      [
+        "res_x",
+        {
+          "num_layers": 4
+        }
+      ],
+      [
+        "compress_space_res",
+        {
+          "multiplier": 2
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 6
+        }
+      ],
+      [
+        "compress_time_res",
+        {
+          "multiplier": 2
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 4
+        }
+      ],
+      [
+        "compress_all_res",
+        {
+          "multiplier": 2
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 2
+        }
+      ],
+      [
+        "compress_all_res",
+        {
+          "multiplier": 1
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 2
+        }
+      ]
+    ],
+    "decoder_blocks": [
+      [
+        "res_x",
+        {
+          "num_layers": 4
+        }
+      ],
+      [
+        "compress_space",
+        {
+          "multiplier": 2
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 6
+        }
+      ],
+      [
+        "compress_time",
+        {
+          "multiplier": 2
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 4
+        }
+      ],
+      [
+        "compress_all",
+        {
+          "multiplier": 1
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 2
+        }
+      ],
+      [
+        "compress_all",
+        {
+          "multiplier": 2
+        }
+      ],
+      [
+        "res_x",
+        {
+          "num_layers": 2
+        }
+      ]
+    ],
+    "scaling_factor": 1.0,
+    "norm_layer": "pixel_norm",
+    "patch_size": 4,
+    "latent_log_var": "uniform",
+    "use_quant_conv": false,
+    "causal_decoder": false,
+    "timestep_conditioning": false,
+    "normalize_latent_channels": false,
+    "encoder_base_channels": 128,
+    "decoder_base_channels": 128,
+    "spatial_padding_mode": "zeros"
+  },
+  "scheduler": {
+    "_class_name": "RectifiedFlowScheduler",
+    "_diffusers_version": "0.25.1",
+    "num_train_timesteps": 1000,
+    "shifting": null,
+    "base_resolution": null,
+    "sampler": "LinearQuadratic"
+  },
+  "audio_vae": {
+    "model": {
+      "params": {
+        "ddconfig": {
+          "double_z": true,
+          "mel_bins": 64,
+          "z_channels": 8,
+          "resolution": 256,
+          "downsample_time": false,
+          "in_channels": 2,
+          "out_ch": 2,
+          "ch": 128,
+          "ch_mult": [
+            1,
+            2,
+            4
+          ],
+          "num_res_blocks": 2,
+          "attn_resolutions": [],
+          "dropout": 0.0,
+          "mid_block_add_attention": false,
+          "norm_type": "pixel",
+          "causality_axis": "height"
+        },
+        "sampling_rate": 16000
+      }
+    },
+    "preprocessing": {
+      "audio": {
+        "sampling_rate": 16000,
+        "max_wav_value": 32768.0,
+        "duration": 5.12,
+        "stereo": true,
+        "causal_padding": 3
+      },
+      "stft": {
+        "filter_length": 1024,
+        "hop_length": 160,
+        "win_length": 1024,
+        "causal": true
+      },
+      "mel": {
+        "n_mel_channels": 64,
+        "mel_fmin": 0,
+        "mel_fmax": 8000
+      }
+    }
+  },
+  "vocoder": {
+    "vocoder": {
+      "upsample_initial_channel": 1536,
+      "resblock": "AMP1",
+      "upsample_rates": [
+        5,
+        2,
+        2,
+        2,
+        2,
+        2
+      ],
+      "resblock_kernel_sizes": [
+        3,
+        7,
+        11
+      ],
+      "upsample_kernel_sizes": [
+        11,
+        4,
+        4,
+        4,
+        4,
+        4
+      ],
+      "resblock_dilation_sizes": [
+        [
+          1,
+          3,
+          5
+        ],
+        [
+          1,
+          3,
+          5
+        ],
+        [
+          1,
+          3,
+          5
+        ]
+      ],
+      "stereo": true,
+      "use_tanh_at_final": false,
+      "activation": "snakebeta",
+      "use_bias_at_final": false
+    },
+    "bwe": {
+      "upsample_initial_channel": 512,
+      "resblock": "AMP1",
+      "upsample_rates": [
+        6,
+        5,
+        2,
+        2,
+        2
+      ],
+      "resblock_kernel_sizes": [
+        3,
+        7,
+        11
+      ],
+      "upsample_kernel_sizes": [
+        12,
+        11,
+        4,
+        4,
+        4
+      ],
+      "resblock_dilation_sizes": [
+        [
+          1,
+          3,
+          5
+        ],
+        [
+          1,
+          3,
+          5
+        ],
+        [
+          1,
+          3,
+          5
+        ]
+      ],
+      "stereo": true,
+      "use_tanh_at_final": false,
+      "activation": "snakebeta",
+      "use_bias_at_final": false,
+      "apply_final_activation": false,
+      "input_sampling_rate": 16000,
+      "output_sampling_rate": 48000,
+      "hop_length": 80,
+      "n_fft": 512,
+      "win_size": 512,
+      "num_mels": 64
+    }
+  }
+}

ltx-2.3-22b-distilled-lora-384-1.1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5d4953f3386197a4b4f5abdb17616ff256171e8075c111d6e7d2dfa6e823b3a
+size 7605507256

ltx-2.3-22b-distilled-lora-384.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2943ab994f3c9d88052e5a2a34cca14e4a2dfc36b1d8c407931d52d5c25dd72b
+size 7605507256

quantize_config.json

@@ -0,0 +1,6 @@
+{
+  "quantization": {
+    "bits": 4,
+    "group_size": 64
+  }
+}

readme.md

@@ -0,0 +1,48 @@
+---
+base_model:
+  - SulphurAI/Sulphur-2-base
+pipeline_tag: image-text-to-video
+---
+
+# LTX2.3 Model Repository
+
+This repository hosts custom implementations of the LTX2.3 video AI model, refined specifically for high-fidelity generation using the Sulphur 2 architecture. It has been converted to Apples MLX architecture and quantized down to Q4, to maximize memory efficiency. It has been tested on a 32GB M5 Silicon Mac, and that is the lowest recommended RAM for this model.
+
+If you are not on a Mac, this madel variant is not for you.
+
+## Model Overview
+
+This implementation represents a highly optimized workflow built around the LTX2.3 core.
+
+- **Base Model:** LTX2.3
+- **Refinement Applied:** Sulphur 2
+- **Fusion Detail:** The Sulphur 2 refinements have been successfully fused into the **`transformer-distilled.safetensors`** checkpoint, providing a unified generation experience.
+- **Implementation:** MLX Conversion
+- **Quantization:** FP4 (Optimized for performance and memory footprint)
+- **Target Pipeline:** 8/3 Pipeline (Optimized for generation workflow)
+
+## Usage Guide
+
+### Core Workflow (Recommended)
+
+For the best results and fastest generation times, users should rely on the integrated 8/3 pipeline.
+
+- **Primary Generation:** Use the fused `transformer-distilled.safetensors` checkpoint to access the Sulphur 2 quality enhancements baked into the LTX2.3 base.
+- **LoRAs:** No external LoRAs are required when using the fused model for Sulphur 2 quality, but have been included in this repo for convenience.
+
+### Hardware & Compute Notes
+
+- **Primary Platform:** Optimized for macOS compute environments on Apple silicon M-series SOC's.
+- **AI Engine:** Built around the MLX framework integration.
+
+## Prompting Guidelines (LTX Specific)
+
+To achieve optimal generation quality with this model, adhere strictly to the following prompting conventions:
+
+1.  **Structure:** Aim for a single, flowing paragraph.
+2.  **Tense:** Use present tense verbs for all actions and movements.
+3.  **Detail Level:** Match the level of descriptive detail to the intended shot scale (e.g., high detail for close-ups, broader strokes for wide shots).
+4.  **Flow:** Describe the camera movement relative to the subject matter.
+5.  **Length Target:** Aim for 4–8 descriptive sentences to maintain focus and coherence.
+
+**Note:** Model coherence (and body horror) has a swift uptake in clips going past ~17 seconds. Test with shorter clips.

spatial_upscaler_x1_5_v1_0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e186732be14a71b953efed15181cbb94595fa836555a07b3b7560d6cf5deeb15
+size 1090127476

spatial_upscaler_x1_5_v1_0_config.json

@@ -0,0 +1,13 @@
+{
+  "config": {
+    "_class_name": "LatentUpsampler",
+    "in_channels": 128,
+    "mid_channels": 1024,
+    "num_blocks_per_stage": 4,
+    "dims": 3,
+    "spatial_upsample": true,
+    "temporal_upsample": false,
+    "spatial_scale": 1.5,
+    "rational_resampler": true
+  }
+}

spatial_upscaler_x2_v1_1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:755f69dcb8a3f9589045925eb4a2237e9969f9c026a016393bc395e65ebd5788
+size 995745061

spatial_upscaler_x2_v1_1_config.json

@@ -0,0 +1,13 @@
+{
+  "config": {
+    "_class_name": "LatentUpsampler",
+    "in_channels": 128,
+    "mid_channels": 1024,
+    "num_blocks_per_stage": 4,
+    "dims": 3,
+    "spatial_upsample": true,
+    "temporal_upsample": false,
+    "spatial_scale": 2.0,
+    "rational_resampler": false
+  }
+}

split_model.json

@@ -0,0 +1,25 @@
+{
+  "format": "split",
+  "model_version": "2.3.0",
+  "components": [
+    "connector",
+    "vae_decoder",
+    "vae_encoder",
+    "audio_vae",
+    "vocoder",
+    "spatial_upscaler_x2_v1_1",
+    "spatial_upscaler_x1_5_v1_0",
+    "temporal_upscaler_x2_v1_0"
+  ],
+  "transformer_variants": [
+    "distilled"
+  ],
+  "lora": [
+    "ltx-2.3-22b-distilled-lora-384.safetensors",
+    "ltx-2.3-22b-distilled-lora-384-1.1.safetensors"
+  ],
+  "source": "Lightricks/LTX-2.3",
+  "notes": {
+    "vocoder": "Also contains BWE (bandwidth extension) generator weights \u2014 upsample layers [6,5,2,2,2] (240x) and mel_stft parameters."
+  }
+}

temporal_upscaler_x2_v1_0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:982bc28b455ed22de1e39a9fcf5213299de4e15f948ae3bd2cee65550e019628
+size 261945581

temporal_upscaler_x2_v1_0_config.json

@@ -0,0 +1,13 @@
+{
+  "config": {
+    "_class_name": "LatentUpsampler",
+    "in_channels": 128,
+    "mid_channels": 512,
+    "num_blocks_per_stage": 4,
+    "dims": 3,
+    "spatial_upsample": false,
+    "temporal_upsample": true,
+    "spatial_scale": 1.0,
+    "rational_resampler": true
+  }
+}

transformer-distilled.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eb719bb8ee018bac999323fc82084c222d17133a75cdb77060e5097af0ec8b0
+size 10698311363

vae_decoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f404026fe0b59418eaec4a3fcdc474125c798f0b787dc390f6eb4e79934d4160
+size 814349531

vae_encoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1512f5ef8387901db0baecb328690676e2667ddf2b63e28f8e6645188c1a8c06
+size 637885319

vocoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:410cee86c14f76956bf487d153211ef15050fe8bf51f86d47a1d27ca6e3ba287
+size 258313851