{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "id": "8ZuJkxvW1cgV" }, "outputs": [], "source": [ "# @title # **Cell 1: Install & Clone ComfyUI**\n", "\n", "import subprocess, sys, os\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "COMFY = f\"{WORK}/ComfyUI\"\n", "\n", "# if not os.path.exists(COMFY):\n", "# print(\"Cloning ComfyUI...\")\n", "# subprocess.run([\"git\", \"clone\", \"https://github.com/comfyanonymous/ComfyUI.git\", COMFY], check=True)\n", "\n", "!git clone https://github.com/comfyanonymous/ComfyUI.git --branch v0.19.4 --single-branch\n", "\n", "subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", f\"{COMFY}/requirements.txt\"], check=True)\n", "subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"accelerate\", \"safetensors\", \"sentencepiece\", \"einops\", \"google-genai\", \"retinaface-pytorch\"], check=True)\n", "subprocess.run([\"apt-get\", \"install\", \"-y\", \"-qq\", \"ffmpeg\"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n", "\n", "try:\n", " subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"sageattention\"], check=True)\n", "except: pass\n", "\n", "print(f\"Done. ComfyUI at {COMFY}\")" ] }, { "cell_type": "code", "source": [ "# @title # **Cell 2: Install Custom Nodes**\n", "\n", "import subprocess, sys, os\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "CN = f\"{WORK}/ComfyUI/custom_nodes\"\n", "os.makedirs(CN, exist_ok=True)\n", "\n", "REPOS = {\n", " \"ComfyUI-KJNodes\": \"https://github.com/kijai/ComfyUI-KJNodes.git\",\n", " \"ComfyUI-VideoHelperSuite\": \"https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git\",\n", " \"ComfyUI-MelBandRoFormer\": \"https://github.com/kijai/ComfyUI-MelBandRoFormer.git\",\n", " \"ComfyUI-Frame-Interpolation\": \"https://github.com/Fannovel16/ComfyUI-Frame-Interpolation.git\",\n", " \"rgthree-comfy\": \"https://github.com/rgthree/rgthree-comfy.git\",\n", " \"ComfyUI-OmniVoice-TTS\": \"https://github.com/Saganaki22/ComfyUI-OmniVoice-TTS.git\",\n", " \"ComfyUI-WanVideoWrapper\": \"https://github.com/kijai/ComfyUI-WanVideoWrapper.git\",\n", " \"ComfyUI-LTXVideo\": \"https://github.com/Lightricks/ComfyUI-LTXVideo.git\",\n", "}\n", "\n", "for name, url in REPOS.items():\n", " dest = os.path.join(CN, name)\n", " if not os.path.exists(dest):\n", " print(f\"Cloning {name}...\")\n", " subprocess.run([\"git\", \"clone\", url, dest], check=True)\n", " req = os.path.join(dest, \"requirements.txt\")\n", " if os.path.exists(req):\n", " subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", req])\n", " else:\n", " print(f\"{name} OK\")\n", "\n", "# OmniVoice needs install.py (not requirements.txt) to protect PyTorch\n", "_omni_install = os.path.join(CN, \"ComfyUI-OmniVoice-TTS\", \"install.py\")\n", "if os.path.exists(_omni_install):\n", " print(\"Running OmniVoice install.py...\")\n", " subprocess.run([sys.executable, _omni_install], check=False)\n", " # OmniVoice needs latest transformers (HiggsAudioV2TokenizerModel)\n", " subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"transformers\"], check=False)\n", "\n", "print(\"Custom nodes ready.\")" ], "metadata": { "id": "FrYr3hxJ1v4j" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ " #KJNodes.# @title # **Cell 3: Download Models**\n", "\n", "import os, subprocess\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "M = f\"{WORK}/ComfyUI/models\"\n", "\n", "for d in [\"diffusion_models\", \"text_encoders\", \"vae\",\n", " \"latent_upscale_models\", \"loras\", \"audio_separator\"]:\n", " os.makedirs(f\"{M}/{d}\", exist_ok=True)\n", "\n", "# Install aria2 for fast multi-connection downloads\n", "subprocess.run([\"apt-get\", \"-qq\", \"-y\", \"install\", \"aria2\"],\n", " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n", "\n", "# ---- Core Models (required for ALL modes) ----\n", "downloads = [\n", " # UNET - Main diffusion model\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/diffusion_models/ltx-2.3-22b-distilled_transformer_only_fp8_input_scaled_v3.safetensors\",\n", " f\"{M}/diffusion_models\"),\n", " # Text Encoders - Gemma 3 12B + LTX projection\n", " (\"https://huggingface.co/Comfy-Org/ltx-2/resolve/main/split_files/text_encoders/gemma_3_12B_it_fpmixed.safetensors\",\n", " f\"{M}/text_encoders\"),\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/text_encoders/ltx-2.3_text_projection_bf16.safetensors\",\n", " f\"{M}/text_encoders\"),\n", " # VAEs - Video, Audio, Preview\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/vae/LTX23_video_vae_bf16.safetensors\",\n", " f\"{M}/vae\"),\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/vae/LTX23_audio_vae_bf16.safetensors\",\n", " f\"{M}/vae\"),\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/vae/taeltx2_3.safetensors\",\n", " f\"{M}/vae\"),\n", " # Upscaler - 2x spatial (used in pass 2)\n", " (\"https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-spatial-upscaler-x2-1.1.safetensors\",\n", " f\"{M}/latent_upscale_models\"),\n", "\n", " # ---- LoRAs ----\n", " # Distilled LoRA (all modes)\n", " (\"https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-22b-distilled-lora-384.safetensors\",\n", " f\"{M}/loras\"),\n", " # Static Camera LoRA - Frees attention budget for lip sync\n", " (\"https://huggingface.co/Lightricks/LTX-2-19b-LoRA-Camera-Control-Static/resolve/main/ltx-2-19b-lora-camera-control-static.safetensors\",\n", " f\"{M}/loras\"),\n", " # Talking Head LoRA - Lip sync for IA2V_TH mode\n", " (\"https://huggingface.co/elix3r/LTX-2.3-22b-AV-LoRA-talking-head/resolve/main/LTX-2.3-22b-AV-LoRA-talking-head-v1.safetensors\",\n", " f\"{M}/loras\"),\n", " # ID-LoRA - Face identity preservation (Custom Audio modes)\n", " (\"https://huggingface.co/AviadDahan/LTX-2.3-ID-LoRA-CelebVHQ-3K/resolve/main/lora_weights.safetensors\",\n", " f\"{M}/loras\", \"LTX-2.3-ID-LoRA-CelebVHQ-3K.safetensors\"),\n", " # Transition LoRA - Smooth first-to-last frame transitions (FL2V modes)\n", " (\"https://huggingface.co/valiantcat/LTX-2.3-Transition-LORA/resolve/main/ltx2.3-transition.safetensors\",\n", " f\"{M}/loras\"),\n", "\n", " # ---- Audio Processing ----\n", " # MelBandRoFormer - Vocal isolation (Custom Audio & V2A modes)\n", " (\"https://huggingface.co/Kijai/MelBandRoFormer_comfy/resolve/main/MelBandRoformer_fp16.safetensors\",\n", " f\"{M}/diffusion_models\"),\n", "]\n", "\n", "# Filter out already-downloaded files and build aria2 input file\n", "aria2_input = f\"{WORK}/dl_list.txt\"\n", "pending = []\n", "for dl in downloads:\n", " url, dest_dir = dl[0], dl[1]\n", " fname = dl[2] if len(dl) > 2 else url.split(\"/\")[-1]\n", " dest_path = f\"{dest_dir}/{fname}\"\n", " if os.path.exists(dest_path):\n", " print(f\" SKIP: {fname}\")\n", " else:\n", " pending.append((url, dest_dir, fname))\n", "\n", "if pending:\n", " with open(aria2_input, \"w\") as f:\n", " for url, dest_dir, fname in pending:\n", " f.write(f\"{url}\\n dir={dest_dir}\\n out={fname}\\n\")\n", " print(f\" Queued: {fname}\")\n", "\n", " print(f\"\\nDownloading {len(pending)} file(s) with aria2c (16 connections)...\")\n", " subprocess.run([\n", " \"aria2c\",\n", " \"--input-file\", aria2_input,\n", " \"--max-connection-per-server=16\",\n", " \"--split=16\",\n", " \"--min-split-size=1M\",\n", " \"--max-concurrent-downloads=4\",\n", " \"--console-log-level=error\",\n", " \"--summary-interval=5\",\n", " ], check=True)\n", " os.remove(aria2_input)\n", "\n", "print(\"\\nAll models downloaded!\")\n", "\n", "# ---- Download Workflow JSONs ----\n", "print(\"\\nDownloading workflows...\")\n", "_wf_dir = f\"{WORK}\"\n", "_workflows = {\n", " \"V2A.json\": \"https://www.dropbox.com/scl/fi/ocosjm021sym378x29uft/V2A.json?rlkey=zj9g2kck84bgppni7rpy3nmv8&st=6phkbg0f&dl=1\",\n", " \"IA2V_A2V.json\": \"https://www.dropbox.com/scl/fi/i6z9gdnh0x46brtxe01i4/IA2V_A2V.json?rlkey=hkts2zhusfe3wgpfw5mjw4tpd&st=iunro7zs&dl=1\",\n", " \"I2V_T2V.json\": \"https://www.dropbox.com/scl/fi/m2cbef0rvdpw1e1ffmysp/I2V_T2V.json?rlkey=t518o2wy2pprbihdxr0zf7o7x&st=0npdht6r&dl=1\",\n", " \"FL2V.json\": \"https://www.dropbox.com/scl/fi/cj2mpitugqghg7d7e8v1y/FL2V.json?rlkey=eynptf1ho9c9e8dj3l2gan541&st=258c6kpl&dl=1\",\n", " \"Extend.json\": \"https://www.dropbox.com/scl/fi/b6d3h4d7t1qm75fi082n9/Extend.json?rlkey=wthux4i3ga9w2zdor2oy7r0k9&st=alnk3cpv&dl=1\",\n", " \"Extend_AV.json\": \"https://www.dropbox.com/scl/fi/67qulvtnqgomj0l9grj64/Extend_AV.json?rlkey=pumte6are667ete8wzxnx6ozp&st=e1nhc7fy&dl=1\",\n", "}\n", "for _wf_name, _wf_url in _workflows.items():\n", " _wf_dest = f\"{_wf_dir}/{_wf_name}\"\n", " subprocess.run([\"wget\", \"-q\", \"-O\", _wf_dest, _wf_url], check=False)\n", " print(f\" Updated: {_wf_name}\")\n", "print(\"Workflows ready!\")" ], "metadata": { "id": "0DikjYdd1x2V" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "\n", "%cd /content/ComfyUI\n", "\n", "# See what changed (optional)\n", "!git status\n", "!git diff --stat comfy_extras/nodes_lt_audio.py | head -20\n", "\n", "# Discard local changes and switch tag\n", "!git reset --hard HEAD\n", "!git fetch --tags origin\n", "!git checkout v0.22.0\n", "!pip install -q -r requirements.txt\n", "\n", "!git describe --tags --always # should show v0.22.0" ], "metadata": { "id": "eD8T_CAU11CO" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title # **: Download outpaint lora, outpaint WF and custom nodes**\n", "\n", "import os, subprocess\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "M = f\"{WORK}/ComfyUI/models\"\n", "\n", "for d in [\"diffusion_models\", \"text_encoders\", \"vae\",\n", " \"latent_upscale_models\", \"loras\", \"audio_separator\"]:\n", " os.makedirs(f\"{M}/{d}\", exist_ok=True)\n", "\n", "# Install aria2 for fast multi-connection downloads\n", "subprocess.run([\"apt-get\", \"-qq\", \"-y\", \"install\", \"aria2\"],\n", " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n", "\n", "# ---- Core Models (required for ALL modes) ----\n", "downloads = [\n", " # Outpaint LoRA - LTX 2.3 video outpainting\n", " (\"https://huggingface.co/oumoumad/LTX-2.3-22b-IC-LoRA-Outpaint/resolve/main/ltx-2.3-22b-ic-lora-outpaint.safetensors\",\n", " f\"{M}/loras\", \"ltx-2.3-22b-ic-lora-outpaint.safetensors\"),\n", "]\n", "\n", "# Filter out already-downloaded files and build aria2 input file\n", "aria2_input = f\"{WORK}/dl_list.txt\"\n", "pending = []\n", "for dl in downloads:\n", " url, dest_dir = dl[0], dl[1]\n", " fname = dl[2] if len(dl) > 2 else url.split(\"/\")[-1]\n", " dest_path = f\"{dest_dir}/{fname}\"\n", " if os.path.exists(dest_path):\n", " print(f\" SKIP: {fname}\")\n", " else:\n", " pending.append((url, dest_dir, fname))\n", "\n", "if pending:\n", " with open(aria2_input, \"w\") as f:\n", " for url, dest_dir, fname in pending:\n", " f.write(f\"{url}\\n dir={dest_dir}\\n out={fname}\\n\")\n", " print(f\" Queued: {fname}\")\n", "\n", " print(f\"\\nDownloading {len(pending)} file(s) with aria2c (16 connections)...\")\n", " subprocess.run([\n", " \"aria2c\",\n", " \"--input-file\", aria2_input,\n", " \"--max-connection-per-server=16\",\n", " \"--split=16\",\n", " \"--min-split-size=1M\",\n", " \"--max-concurrent-downloads=4\",\n", " \"--console-log-level=error\",\n", " \"--summary-interval=5\",\n", " ], check=True)\n", " os.remove(aria2_input)\n", "\n", "print(\"\\nAll models downloaded!\")\n", "\n", "# ---- Download Workflow JSON ----\n", "print(\"\\nDownloading workflows...\")\n", "_wf_dir = f\"{WORK}\"\n", "_workflows = {\n", " \"outpaint_workflow_v4.json\":\"https://huggingface.co/Yuvrajxms09/ltx-outpaint-workflow/resolve/main/outpaint_workflow_v4.json\",\n", "}\n", "for _wf_name, _wf_url in _workflows.items():\n", " _wf_dest = f\"{_wf_dir}/{_wf_name}\"\n", " subprocess.run([\"wget\", \"-q\", \"-O\", _wf_dest, _wf_url], check=False)\n", " print(f\" Updated: {_wf_name}\")\n", "print(\"Workflows ready!\")\n", "\n", "\n", "\n", "\n", "## clone custom nodes\n", "\n", "import subprocess, sys, os\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "CN = f\"{WORK}/ComfyUI/custom_nodes\"\n", "os.makedirs(CN, exist_ok=True)\n", "\n", "REPOS = {\n", " \"ComfyUI-Easy-Use\": \"https://github.com/yolain/ComfyUI-Easy-Use.git\",\n", " \"ComfyMath\":\"https://github.com/evanspearman/ComfyMath\",\n", " \"comfyui-crystools\":\"https://github.com/crystian/comfyui-crystools\",\n", " \"comfy_mtb\":\"https://github.com/melMass/comfy_mtb\",\n", " \"ComfyUI_essentials\":\"https://github.com/cubiq/ComfyUI_essentials\",\n", " \"cg-use-everywhere\": \"https://github.com/chrisgoringe/cg-use-everywhere.git\",\n", "}\n", "\n", "for name, url in REPOS.items():\n", " dest = os.path.join(CN, name)\n", " if not os.path.exists(dest):\n", " print(f\"Cloning {name}...\")\n", " subprocess.run([\"git\", \"clone\", url, dest], check=True)\n", " req = os.path.join(dest, \"requirements.txt\")\n", " if os.path.exists(req):\n", " subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", req])\n", " else:\n", " print(f\"{name} OK\")\n", "\n", "\n", "print(\"Custom nodes ready.\")" ], "metadata": { "id": "XUfTYnAF14ZE" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title # **Cell 4: Download video edit workflow, lora and Unet weights**\n", "\n", "import os, subprocess\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "M = f\"{WORK}/ComfyUI/models\"\n", "\n", "for d in [\"diffusion_models\", \"text_encoders\", \"vae\",\n", " \"latent_upscale_models\", \"loras\", \"audio_separator\"]:\n", " os.makedirs(f\"{M}/{d}\", exist_ok=True)\n", "\n", "# Install aria2 for fast multi-connection downloads\n", "subprocess.run([\"apt-get\", \"-qq\", \"-y\", \"install\", \"aria2\"],\n", " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n", "\n", "# ---- Core Models (required for ALL modes) ----\n", "downloads = [\n", " # UNET - Main diffusion model\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/diffusion_models/ltx-2.3-22b-distilled-1.1_transformer_only_fp8_scaled.safetensors\",\n", " f\"{M}/diffusion_models\"),\n", "\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/diffusion_models/ltx-2.3-22b-dev_transformer_only_fp8_scaled.safetensors\",\n", " f\"{M}/diffusion_models\"),\n", "\n", " #\n", " (\"https://huggingface.co/Kijai/LTX2.3_comfy/resolve/main/loras/ltx-2.3-22b-distilled-1.1_lora-dynamic_fro09_avg_rank_111_bf16.safetensors\",\n", " f\"{M}/loras\", \"ltx-2.3-22b-distilled-1.1_lora-dynamic_fro09_avg_rank_111_bf16.safetensors\"),\n", "\n", " #https://huggingface.co/Alissonerdx/LTX-LoRAs/resolve/main/ltx23_edit_anything_global_rank128_v1_9000steps_adamw.safetensors\n", " (\"https://huggingface.co/Alissonerdx/LTX-LoRAs/resolve/main/ltx23_edit_anything_global_rank128_v1_9000steps_adamw.safetensors\",\n", " f\"{M}/loras\", \"ltx23_edit_anything_global_rank128_v1_9000steps_adamw.safetensors\"),\n", "\n", " (\"https://huggingface.co/Alissonerdx/LTX-LoRAs/resolve/main/ltx23_inpaint_masked_r2v_rank32_v1_3000steps.safetensors\",\n", " f\"{M}/loras\", \"ltx23_inpaint_masked_r2v_rank32_v1_3000steps.safetensors\"),\n", "\n", " (\"https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Detailer/resolve/main/ltx-2-19b-ic-lora-detailer.safetensors\",\n", " f\"{M}/loras\", \"ltx-2-19b-ic-lora-detailer.safetensors\"),\n", "\n", " (\"https://huggingface.co/Cseti/LTX2.3-22B_ReStyle_IC-LoRA/resolve/main/852654_LTX2.3-22B_ReStyle_IC-LoRA_8000_v0.1.safetensors\",\n", " f\"{M}/loras\", \"852654_LTX2.3-22B_ReStyle_IC-LoRA_8000_v0.1.safetensors\"),\n", "\n", "\n", "\n", "]\n", "\n", "# Filter out already-downloaded files and build aria2 input file\n", "aria2_input = f\"{WORK}/dl_list.txt\"\n", "pending = []\n", "for dl in downloads:\n", " url, dest_dir = dl[0], dl[1]\n", " fname = dl[2] if len(dl) > 2 else url.split(\"/\")[-1]\n", " dest_path = f\"{dest_dir}/{fname}\"\n", " if os.path.exists(dest_path):\n", " print(f\" SKIP: {fname}\")\n", " else:\n", " pending.append((url, dest_dir, fname))\n", "\n", "if pending:\n", " with open(aria2_input, \"w\") as f:\n", " for url, dest_dir, fname in pending:\n", " f.write(f\"{url}\\n dir={dest_dir}\\n out={fname}\\n\")\n", " print(f\" Queued: {fname}\")\n", "\n", " print(f\"\\nDownloading {len(pending)} file(s) with aria2c (16 connections)...\")\n", " subprocess.run([\n", " \"aria2c\",\n", " \"--input-file\", aria2_input,\n", " \"--max-connection-per-server=16\",\n", " \"--split=16\",\n", " \"--min-split-size=1M\",\n", " \"--max-concurrent-downloads=4\",\n", " \"--console-log-level=error\",\n", " \"--summary-interval=5\",\n", " ], check=True)\n", " os.remove(aria2_input)\n", "\n", "print(\"\\nAll models downloaded!\")\n", "\n", "\n", "# ---- Download Workflow JSON ----\n", "print(\"\\nDownloading workflows...\")\n", "_wf_dir = f\"{WORK}\"\n", "_workflows = {\n", " \"ltx2.3_edit_workflow.json\":\"https://huggingface.co/Yuvrajxms09/ltx2.3_edit_workflow/resolve/main/ltx2.3_edit_workflow.json\",\n", " \"edit_workflow_v2.json\":\"https://huggingface.co/Yuvrajxms09/ltx2.3_edit_workflow/resolve/main/edit_workflow_v2.json\",\n", " \"v2_restyle_copy.json\":\"https://huggingface.co/Yuvrajxms09/ltx2.3_edit_workflow/resolve/main/v2_restyle_copy.json\",\n", " \"working_style_wf_api.json\":\"https://huggingface.co/Yuvrajxms09/ltx2.3_edit_workflow/resolve/main/working_style_wf_api.json\",\n", " \"external_audio_wf_v1.json\":\"https://huggingface.co/Yuvrajxms09/ltx2.3_edit_workflow/resolve/main/external_audio_wf_v1.json\",\n", " \"errororo222.json\":\"https://huggingface.co/Yuvrajxms09/ltx2.3_edit_workflow/resolve/main/errororo222.json\",\n", "\n", "}\n", "for _wf_name, _wf_url in _workflows.items():\n", " _wf_dest = f\"{_wf_dir}/{_wf_name}\"\n", " subprocess.run([\"wget\", \"-q\", \"-O\", _wf_dest, _wf_url], check=False)\n", " print(f\" Updated: {_wf_name}\")\n", "print(\"Workflows ready!\")\n", "\n", "\n", "## clone custom nodes\n", "\n", "import subprocess, sys, os\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "CN = f\"{WORK}/ComfyUI/custom_nodes\"\n", "os.makedirs(CN, exist_ok=True)\n", "\n", "REPOS = {\n", " \"ComfyUI_essentials\":\"https://github.com/cubiq/ComfyUI_essentials\",\n", " # new_edit.json: LF_EmptyImage (magenta plate)\n", " \"lf-nodes\": \"https://github.com/lucafoscili/lf-nodes.git\",\n", " # SAM3Segment + RMBG (and their model downloads / deps in that repo’s README)\n", " \"ComfyUI-RMBG\": \"https://github.com/1038lab/ComfyUI-RMBG.git\",\n", " # ReservedRegionFrameComposer (and related face-sequence helpers in same pack)\n", " #\"ComfyUI-BFSNodes\": \"https://github.com/alisson-anjos/ComfyUI-BFSNodes.git\",\n", " # aistudynow_QwenVL\n", " \"ComfyUI_Fill-Nodes\": \"https://github.com/filliptm/ComfyUI_Fill-Nodes.git\",\n", " #\"ComfyUI-QwenVL\": \"https://github.com/aistudynow/ComfyUI-QwenVL.git\",\n", " \"RES4LYF\" : \"https://github.com/ClownsharkBatwing/RES4LYF.git\",\n", "\n", "\n", "}\n", "\n", "for name, url in REPOS.items():\n", " dest = os.path.join(CN, name)\n", " if not os.path.exists(dest):\n", " print(f\"Cloning {name}...\")\n", " subprocess.run([\"git\", \"clone\", url, dest], check=True)\n", " req = os.path.join(dest, \"requirements.txt\")\n", " if os.path.exists(req):\n", " subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", req])\n", " else:\n", " print(f\"{name} OK\")\n", "\n", "\n", "print(\"Custom nodes ready.\")\n" ], "metadata": { "id": "Kh7Po8mf16YJ" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "import os, subprocess\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "M = f\"{WORK}/ComfyUI/models\"\n", "LORA_DIR = f\"{M}/loras\"\n", "os.makedirs(LORA_DIR, exist_ok=True)\n", "\n", "token = \"${HF_TOKEN}\"\n", "url = \"https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-LipDub/resolve/main/ltx-2.3-22b-ic-lora-lipdub-0.9.safetensors\"\n", "out_path = f\"{LORA_DIR}/ltx-2.3-22b-ic-lora-lipdub-0.9.safetensors\"\n", "\n", "if os.path.exists(out_path):\n", " print(f\"SKIP: {out_path}\")\n", "else:\n", " subprocess.run([\n", " \"curl\", \"-L\", \"--fail\", \"--retry\", \"3\", \"--retry-delay\", \"2\",\n", " \"-H\", f\"Authorization: Bearer {token}\",\n", " \"-o\", out_path,\n", " url,\n", " ], check=True)\n", " print(f\"Downloaded: {out_path}\")" ], "metadata": { "id": "Ju72nb6L18jd" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "iwGwsUSN7Xem" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "iCbF1Qa77XcO" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "!echo -= Installing custom nodes =-\n", "\n", "# @markdown -= Installing custom nodes =-\n", "\n", "%cd /content/ComfyUI/custom_nodes\n", "!git clone https://github.com/ltdrdata/ComfyUI-Manager.git\n", "# !git clone https://github.com/LykosAI/ComfyUI-Inference-Core-Nodes.git\n", "!git clone https://github.com/kijai/ComfyUI-WanVideoWrapper.git\n", "!git clone https://github.com/kijai/ComfyUI-KJNodes.git\n", "!git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git\n", "\n", "!git clone https://github.com/devnull03/mh-custom-nodes.git\n", "\n", "# !git clone https://github.com/PozzettiAndrea/ComfyUI-SAM3.git\n", "# !uv pip install --system -r ./ComfyUI-SAM3/requirements.txt\n", "# %cd /content/ComfyUI/custom_nodes/ComfyUI-SAM3\n", "# !git checkout b70a2f1399c8a518c5cce77c494a0ee9f91e0c88\n", "# %cd /content/ComfyUI/custom_nodes\n", "\n", "# temp ------------------\n", "# !git clone https://github.com/yolain/ComfyUI-Easy-Use.git\n", "# !git clone https://github.com/kijai/ComfyUI-segment-anything-2.git\n", "# !git clone https://github.com/kijai/ComfyUI-SCAIL-Pose.git\n", "!git clone https://github.com/kijai/ComfyUI-WanAnimatePreprocess.git\n", "!git clone https://github.com/judian17/ComfyUI_YOLO_For_Multi_SDPose_Detection.git\n", "# !git clone https://github.com/rgthree/rgthree-comfy.git\n", "# !git clone https://github.com/pythongosssss/ComfyUI-Custom-Scripts.git\n", "# !git clone https://github.com/cubiq/ComfyUI_essentials.git\n", "# !git clone https://github.com/chflame163/ComfyUI_LayerStyle.git\n", "# !git clone https://github.com/LAOGOU-666/Comfyui_LG_Tools.git\n", "# !git clone https://github.com/ltdrdata/ComfyUI-Impact-Pack.git\n", "# !git clone https://github.com/Suzie1/ComfyUI_Comfyroll_CustomNodes.git\n", "# !uv pip install --system -r ./ComfyUI_LayerStyle/requirements.txt\n", "# !uv pip install --system -r ./Comfyui_LG_Tools/requirements.txt\n", "# !uv pip install --system -r ./ComfyUI-Impact-Pack/requirements.txt\n", "# temp ------------------\n", "\n", "# for debugging\n", "# !git clone https://github.com/atmaranto/ComfyUI-SaveAsScript.git\n", "\n", "%cd /content\n", "!git clone https://github.com/devnull03/mh-video-combiner.git\n", "%cd mh-video-combiner\n", "!uv sync\n", "%cd /content" ], "metadata": { "id": "e3ZMR5KY7XZo" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@markdown -= Install dependencies =-\n", "import os\n", "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n", "\n", "!pip install uv\n", "!uv pip install --system -r /content/ComfyUI/requirements.txt --extra-index-url https://download.pytorch.org/whl/cu121 --extra-index-url https://download.pytorch.org/whl/cu118 --extra-index-url https://download.pytorch.org/whl/cu117\n", "!uv pip install --system PyYAML tomllib # xformers!=0.0.18\n", "!uv pip install --system nest_asyncio opencv-python-headless ipywidgets ipycanvas pillow # numpy~=1.26.4 scipy==1.13.1 accelerate==0.33.0\n", "!uv pip install --system onnxruntime-gpu gguf simpleeval boto3 surrealist onnx redis\n", "!uv pip install --system sageattention==1.0.6\n", "# !uv pip install --system color-matcher\n", "# !uv pip install --system -e /content/ComfyUI/custom_nodes/ComfyUI-Inference-Core-Nodes[cuda]\n", "!uv pip install --system segment-geospatial[samgeo3]\n", "!uv pip install --system ultralytics" ], "metadata": { "id": "Y8HN7D2h7XZH" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "import os\n", "import shutil\n", "\n", "# @markdown -= Downloading models =-\n", "from google.colab import userdata\n", "# os.environ[\"HF_TOKEN\"] =\n", "# os.environ[\"HUGGINGFACE_HUB_TOKEN\"] =\n", "%env HF_HUB_ENABLE_HF_TRANSFER=1\n", "%env HF_TOKEN=${HF_TOKEN}\n", "%env HUGGINGFACE_HUB_TOKEN=${HF_TOKEN} #{userdata.get(\"HF_TOKEN\")}\n", "!uv pip install --system -q huggingface_hub[hf_transfer]\n", "\n", "from huggingface_hub import hf_hub_download\n", "\n", "dirs = {\n", " \"diffusion\": f\"{WORKSPACE}/models/diffusion_models\",\n", " \"vae\": f\"{WORKSPACE}/models/vae\",\n", " \"text_encoder\": f\"{WORKSPACE}/models/text_encoders\",\n", " \"loras\": f\"{WORKSPACE}/models/loras\",\n", " \"controlnet\": f\"{WORKSPACE}/models/controlnet\",\n", " \"clip_vision\": f\"{WORKSPACE}/models/clip_vision\",\n", " \"detection\": f\"{WORKSPACE}/models/detection\",\n", " \"checkpoints\": f\"{WORKSPACE}/models/checkpoints\",\n", " \"yolo\": f\"{WORKSPACE}/models/yolo\",\n", "}\n", "\n", "for d in dirs.values():\n", " os.makedirs(d, exist_ok=True)\n", "\n", "def download_hf(repo_id, filename, target_dir):\n", " print(f\"Downloading {os.path.basename(filename)}...\")\n", " try:\n", " cached_path = hf_hub_download(repo_id=repo_id, filename=filename, force_download=False)\n", " final_path = os.path.join(target_dir, os.path.basename(filename))\n", " if os.path.exists(final_path):\n", " os.remove(final_path)\n", " os.symlink(cached_path, final_path)\n", " print(f\"Linked: {final_path}\")\n", " except Exception as e:\n", " print(f\"Error: {e}\")\n", "\n", "\n", "# download_hf(\"Kijai/WanVideo_comfy\", \"Wan21_Uni3C_controlnet_fp16.safetensors\", dirs[\"controlnet\"])\n", "download_hf(\"Comfy-Org/Wan_2.1_ComfyUI_repackaged\", \"split_files/clip_vision/clip_vision_h.safetensors\", dirs[\"clip_vision\"])\n", "download_hf(\"Wan-AI/Wan2.2-Animate-14B\", \"process_checkpoint/det/yolov10m.onnx\", dirs[\"detection\"])\n", "download_hf(\"JunkyByte/easy_ViTPose\", \"onnx/wholebody/vitpose-l-wholebody.onnx\", dirs[\"detection\"])\n", "\n", "download_hf(\"teemosliang/SDPose-Wholebody\", \"yolo11x.pt\", dirs[\"yolo\"])\n", "# download_hf(\"Ultralytics/YOLO11\", \"yolo11x-pose.pt\", dirs[\"yolo\"])\n", "download_hf(\"Comfy-Org/SDPose\", \"checkpoints/sdpose_wholebody_fp16.safetensors\", dirs[\"checkpoints\"])\n" ], "metadata": { "id": "fKXLv5VG8AsE" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title # **Cell 4: Start ComfyUI Server**\n", "\n", "import subprocess, time, os, json, urllib.request\n", "\n", "WORK = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "COMFY = f\"{WORK}/ComfyUI\"\n", "HOST, PORT = \"127.0.0.1\", \"8188\"\n", "URL = f\"http://{HOST}:{PORT}\"\n", "LOG = f\"{WORK}/comfy_server.log\"\n", "\n", "# Patch LTXVEmptyLatentAudio to allow >1000 frames (for long Foley videos)\n", "_nodes_lt_audio = os.path.join(COMFY, \"comfy_extras\", \"nodes_lt_audio.py\")\n", "if os.path.exists(_nodes_lt_audio):\n", " with open(_nodes_lt_audio, \"r\") as _f:\n", " _code = _f.read()\n", " if \"max=1000,\" in _code:\n", " _code = _code.replace(\"max=1000,\", \"max=4000,\")\n", " with open(_nodes_lt_audio, \"w\") as _f:\n", " _f.write(_code)\n", " print(\"Patched LTXVEmptyLatentAudio: max frames 1000 -> 4000\")\n", "\n", "# Kill existing\n", "subprocess.run([\"pkill\", \"-f\", f\"python3.*main.py.*--port {PORT}\"], capture_output=True)\n", "time.sleep(1)\n", "\n", "# Launch\n", "with open(LOG, \"w\") as log_f:\n", " proc = subprocess.Popen(\n", " [\"python3\", \"-u\", \"main.py\", \"--listen\", HOST, \"--port\", PORT,\n", " \"--disable-auto-launch\", \"--dont-print-server\"],\n", " cwd=COMFY, stdout=log_f, stderr=subprocess.STDOUT,\n", " env={**os.environ, \"PYTHONNOUSERSITE\": \"1\",\n", " \"PYTORCH_CUDA_ALLOC_CONF\": \"expandable_segments:True\"})\n", "\n", "print(f\"Server starting (pid={proc.pid})...\")\n", "\n", "for i in range(240):\n", " try:\n", " urllib.request.urlopen(f\"{URL}/object_info\", timeout=2)\n", " print(f\"Server ready! ({i*0.5:.0f}s)\")\n", " break\n", " except:\n", " time.sleep(0.5)\n", "else:\n", " with open(LOG) as f:\n", " print(\"\".join(f.readlines()[-50:]))\n", " raise RuntimeError(\"Server failed to start\")\n", "\n", "data = json.load(urllib.request.urlopen(f\"{URL}/object_info\"))\n", "needed = [\"UNETLoader\", \"DualCLIPLoader\", \"VAELoaderKJ\",\n", " \"CLIPTextEncode\", \"LTXVConditioning\", \"LTX2_NAG\",\n", " \"CFGGuider\", \"SamplerCustomAdvanced\", \"VHS_VideoCombine\"]\n", "missing = [n for n in needed if n not in data]\n", "print(f\"Nodes: {len(data)} | {'All OK' if not missing else 'MISSING: ' + str(missing)}\")\n" ], "metadata": { "id": "rS_hvQoP1-c1" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "fwmIHc6D2aN6" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "xuR1HI8b2aLT" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "s_2j5wSn2aI8" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title Main character selection / SamGeo mask (Face box mode)\n", "# @markdown Draw a positive box around the desired face on frame 0, then run this cell.\n", "# @markdown Sets global `select_output` (parsed JSON after `<----output---->`).\n", "\n", "import json as _json\n", "import os\n", "import sys\n", "\n", "def run_select_obj_cell(cfg, prep: dict, *, points_store_json=None, positive_boxes_json=None) -> dict:\n", " scaled_pt = str(prep.get(\"scaled_frames_pt\") or \"\").strip()\n", " if not scaled_pt or not os.path.isfile(scaled_pt):\n", " raise FileNotFoundError(f\"scaled_frames_pt missing or not found: {scaled_pt!r}\")\n", "\n", " script = _gen_steps_script(\"select_obj.py\")\n", " cmd = [\n", " sys.executable,\n", " \"-u\",\n", " script,\n", " \"--scaled_frames_pt\",\n", " scaled_pt,\n", " \"--samgeo_prompt_mode\",\n", " \"box\",\n", " \"--samgeo_text_prompt\",\n", " \"person\",\n", " \"--samgeo_object_id\",\n", " str(getattr(cfg, \"samgeo_object_id\", 0)).strip(),\n", " \"--samgeo_prompt_frame_idx\",\n", " \"0\",\n", " \"--force_rate35\",\n", " str(int(cfg.force_rate)),\n", " ]\n", " if positive_boxes_json:\n", " cmd.extend([\"--samgeo_positive_boxes\", str(positive_boxes_json)])\n", " if points_store_json:\n", " cmd.extend([\"--points_store45\", str(points_store_json)])\n", " if getattr(cfg, \"save_mask_output\", False):\n", " cmd.append(\"--save_mask_output\")\n", "\n", " output = run_gen_steps_command(\n", " cmd,\n", " label=\"select_obj.py\",\n", " interactive=True,\n", " )\n", " return parse_gen_steps_output(output)\n", "\n", "if \"config\" not in globals():\n", " raise RuntimeError(\"Run the setup cell first (config is missing).\")\n", "if \"prep_output\" not in globals() or not prep_output:\n", " raise RuntimeError(\"Run the setup cell first (prep_output is missing).\")\n", "if not isinstance(prep_output, dict) or not prep_output.get(\"ok\"):\n", " raise RuntimeError(\n", " \"prep_output invalid or prep_media ok=False; fix setup errors and re-run the setup cell.\"\n", " )\n", "if \"_gen_steps_script\" not in globals() or \"run_gen_steps_command\" not in globals():\n", " raise RuntimeError(\"Run the setup cell first (gen-steps helpers are missing).\")\n", "\n", "points_json = None\n", "positive_boxes_json = None\n", "\n", "if \"bbox_widget\" not in dir() or bbox_widget is None:\n", " raise RuntimeError(\n", " \"bbox_widget is missing. Enable the bbox selection widget and draw a positive face box.\"\n", " )\n", "\n", "samgeo_data = bbox_widget.get_separated_samgeo_format()\n", "if samgeo_data.get(\"has_positive\"):\n", " positive_boxes_json = _json.dumps(samgeo_data[\"positive_boxes\"])\n", "else:\n", " raise RuntimeError(\n", " \"No positive boxes were drawn. Draw a tight box around the target face, then run again.\"\n", " )\n", "\n", "print(\"\\n\" + \"=\" * 50)\n", "print(\"Main character selection (face box mode)\")\n", "print(\"=\" * 50 + \"\\n\")\n", "\n", "select_output = run_select_obj_cell(\n", " config,\n", " prep_output,\n", " points_store_json=points_json,\n", " positive_boxes_json=positive_boxes_json,\n", ")\n", "\n", "if not isinstance(select_output, dict) or not select_output.get(\"ok\"):\n", " raise RuntimeError(\n", " f\"select_obj failed or ok=False: {select_output.get('error') if isinstance(select_output, dict) else select_output!r}\"\n", " )\n", "\n", "print(\n", " f\"\\nselect_output ok={select_output.get('ok')} \"\n", " f\"bundle_path={select_output.get('bundle_path')!r} \"\n", " f\"artifact_dir={select_output.get('artifact_dir')!r}\"\n", ")" ], "metadata": { "id": "wZJGpjwN2aId" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title SamGeo interactive face selector widgets\n", "\n", "import io\n", "import json\n", "import os\n", "\n", "import cv2\n", "import ipywidgets as widgets\n", "from ipycanvas import Canvas\n", "from IPython.display import display\n", "from PIL import Image as PILImage\n", "\n", "\n", "def _parse_prompt_frame_index(config) -> int:\n", " value = getattr(config, \"samgeo_prompt_frame_index\", 0)\n", " parser = getattr(\n", " config.__class__,\n", " \"parse_samgeo_prompt_frame_indices\",\n", " None,\n", " )\n", " if callable(parser):\n", " try:\n", " parsed = parser(value)\n", " if parsed:\n", " return int(parsed[0])\n", " except Exception:\n", " pass\n", " if isinstance(value, (list, tuple)):\n", " return int(value[0]) if value else 0\n", " try:\n", " return int(value)\n", " except Exception:\n", " return 0\n", "\n", "\n", "class InteractiveBBoxCollector:\n", " def __init__(self, config):\n", " self.config = config\n", " self.file_path = str(getattr(config, \"ref_video_path\", \"\") or \"\").strip()\n", " self.force_rate = float(getattr(config, \"force_rate\", 0) or 0)\n", " self.skip_first_frames = int(getattr(config, \"skip_first_frames\", 0) or 0)\n", " self.prompt_frame_idx = _parse_prompt_frame_index(config)\n", " self.draw_mode = \"positive\"\n", " self.boxes = []\n", " self._drag_start = None\n", "\n", " self.original_image = self._load_frame()\n", " self.orig_w, self.orig_h = self.original_image.size\n", "\n", " target_w = int(getattr(config, \"width\", 0) or self.orig_w)\n", " target_h = int(getattr(config, \"height\", 0) or 0)\n", " if target_h <= 0:\n", " self.display_w = target_w\n", " self.display_h = int(round(self.orig_h * (target_w / max(1, self.orig_w))))\n", " else:\n", " self.display_w = target_w\n", " self.display_h = target_h\n", "\n", " self.scale_x = self.orig_w / max(1, self.display_w)\n", " self.scale_y = self.orig_h / max(1, self.display_h)\n", "\n", " self._build_ui()\n", " self._wire_events()\n", "\n", " def _load_frame(self):\n", " if not self.file_path or not os.path.exists(self.file_path):\n", " return PILImage.new(\"RGB\", (512, 512), color=\"black\")\n", "\n", " ext = os.path.splitext(self.file_path)[1].lower()\n", " if ext in {\".mp4\", \".avi\", \".mov\", \".mkv\", \".webm\"}:\n", " cap = cv2.VideoCapture(self.file_path)\n", " if cap.isOpened():\n", " logical_index = self.skip_first_frames + self.prompt_frame_idx\n", " if self.force_rate > 0:\n", " source_fps = cap.get(cv2.CAP_PROP_FPS) or self.force_rate\n", " real_index = int((logical_index / max(1.0, self.force_rate)) * source_fps)\n", " else:\n", " real_index = logical_index\n", " cap.set(cv2.CAP_PROP_POS_FRAMES, real_index)\n", " ok, frame = cap.read()\n", " cap.release()\n", " if ok:\n", " return PILImage.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))\n", "\n", " try:\n", " return PILImage.open(self.file_path).convert(\"RGB\")\n", " except Exception:\n", " return PILImage.new(\"RGB\", (512, 512), color=\"black\")\n", "\n", " def _build_ui(self):\n", " img_bytes = io.BytesIO()\n", " self.original_image.save(img_bytes, format=\"PNG\")\n", "\n", " self.bg = widgets.Image(\n", " value=img_bytes.getvalue(),\n", " format=\"png\",\n", " width=self.display_w,\n", " height=self.display_h,\n", " layout=widgets.Layout(grid_area=\"1 / 1 / 2 / 2\"),\n", " )\n", " self.canvas = Canvas(\n", " width=self.display_w,\n", " height=self.display_h,\n", " layout=widgets.Layout(grid_area=\"1 / 1 / 2 / 2\"),\n", " )\n", " self.stack = widgets.GridBox(\n", " children=[self.bg, self.canvas],\n", " layout=widgets.Layout(\n", " width=f\"{self.display_w}px\",\n", " height=f\"{self.display_h}px\",\n", " grid_template_columns=\"1fr\",\n", " grid_template_rows=\"1fr\",\n", " ),\n", " )\n", "\n", " self.btn_pos = widgets.Button(description=\"Positive\", button_style=\"info\")\n", " self.btn_neg = widgets.Button(description=\"Negative\", button_style=\"danger\")\n", " self.btn_undo = widgets.Button(description=\"Undo\")\n", " self.btn_clear = widgets.Button(description=\"Clear\")\n", " self.btn_go = widgets.Button(description=\"Go\", icon=\"refresh\")\n", " self.btn_print = widgets.Button(description=\"Print\", icon=\"print\")\n", " self.inp_frame = widgets.IntText(value=self.prompt_frame_idx, description=\"Frame\")\n", " self.lbl_info = widgets.Label(\n", " value=(\n", " f\"Mode: {self.draw_mode.upper()} | Boxes: 0 | \"\n", " f\"Size: {self.orig_w}x{self.orig_h} -> {self.display_w}x{self.display_h}\"\n", " )\n", " )\n", " self.layout = widgets.VBox([\n", " widgets.HBox([self.btn_pos, self.btn_neg, self.btn_undo, self.btn_clear]),\n", " widgets.HBox([self.inp_frame, self.btn_go, self.btn_print, self.lbl_info]),\n", " self.stack,\n", " ])\n", "\n", " def _wire_events(self):\n", " self.canvas.on_mouse_down(self._on_mouse_down)\n", " self.canvas.on_mouse_up(self._on_mouse_up)\n", " self.btn_pos.on_click(lambda _: self._set_mode(\"positive\"))\n", " self.btn_neg.on_click(lambda _: self._set_mode(\"negative\"))\n", " self.btn_undo.on_click(lambda _: self._undo())\n", " self.btn_clear.on_click(lambda _: self._clear())\n", " self.btn_go.on_click(lambda _: self._reload_frame())\n", " self.btn_print.on_click(lambda _: self._print_boxes())\n", "\n", " def _set_mode(self, mode: str):\n", " self.draw_mode = mode\n", " self._refresh_info()\n", "\n", " def _refresh_info(self):\n", " self.lbl_info.value = (\n", " f\"Mode: {self.draw_mode.upper()} | Boxes: {len(self.boxes)} | \"\n", " f\"Size: {self.orig_w}x{self.orig_h} -> {self.display_w}x{self.display_h}\"\n", " )\n", "\n", " def _reload_frame(self):\n", " self.prompt_frame_idx = int(self.inp_frame.value)\n", " self.original_image = self._load_frame()\n", " self.orig_w, self.orig_h = self.original_image.size\n", " self._refresh_bg()\n", " self._redraw_boxes()\n", "\n", " def _refresh_bg(self):\n", " img_bytes = io.BytesIO()\n", " self.original_image.save(img_bytes, format=\"PNG\")\n", " self.bg.value = img_bytes.getvalue()\n", "\n", " def _to_norm_box(self, x1, y1, x2, y2):\n", " x1, x2 = sorted([x1, x2])\n", " y1, y2 = sorted([y1, y2])\n", " return [\n", " max(0.0, min(1.0, x1 / self.display_w)),\n", " max(0.0, min(1.0, y1 / self.display_h)),\n", " max(0.0, min(1.0, x2 / self.display_w)),\n", " max(0.0, min(1.0, y2 / self.display_h)),\n", " ]\n", "\n", " def _from_norm_box(self, box):\n", " x1, y1, x2, y2 = box\n", " return [\n", " x1 * self.display_w,\n", " y1 * self.display_h,\n", " x2 * self.display_w,\n", " y2 * self.display_h,\n", " ]\n", "\n", " def _draw_box(self, box, label):\n", " x1, y1, x2, y2 = self._from_norm_box(box)\n", " self.canvas.stroke_style = \"#00aaff\" if label == \"positive\" else \"#ff3344\"\n", " self.canvas.line_width = 3\n", " self.canvas.stroke_rect(x1, y1, x2 - x1, y2 - y1)\n", "\n", " def _redraw_boxes(self):\n", " self.canvas.clear()\n", " for item in self.boxes:\n", " self._draw_box(item[\"box\"], item[\"label\"])\n", " self._refresh_info()\n", "\n", " def _on_mouse_down(self, x, y):\n", " self._drag_start = (float(x), float(y))\n", "\n", " def _on_mouse_up(self, x, y):\n", " if self._drag_start is None:\n", " return\n", " x1, y1 = self._drag_start\n", " x2, y2 = float(x), float(y)\n", " self.boxes.append(\n", " {\n", " \"box\": self._to_norm_box(x1, y1, x2, y2),\n", " \"label\": self.draw_mode,\n", " }\n", " )\n", " self._drag_start = None\n", " self._redraw_boxes()\n", "\n", " def _undo(self):\n", " if self.boxes:\n", " self.boxes.pop()\n", " self._redraw_boxes()\n", "\n", " def _clear(self):\n", " self.boxes = []\n", " self._redraw_boxes()\n", "\n", " def _print_boxes(self):\n", " print(json.dumps(self.get_separated_samgeo_format(), indent=2))\n", "\n", " def get_separated_samgeo_format(self):\n", " positive_boxes = [b[\"box\"] for b in self.boxes if b[\"label\"] == \"positive\"]\n", " negative_boxes = [b[\"box\"] for b in self.boxes if b[\"label\"] == \"negative\"]\n", " return {\n", " \"positive_boxes\": positive_boxes,\n", " \"negative_boxes\": negative_boxes,\n", " \"has_positive\": bool(positive_boxes),\n", " \"has_negative\": bool(negative_boxes),\n", " }\n", "\n", "\n", "if \"config\" not in globals():\n", " raise RuntimeError(\"Run the setup cell first so `config` exists.\")\n", "\n", "bbox_widget = InteractiveBBoxCollector(config)\n", "display(bbox_widget.layout)\n" ], "metadata": { "id": "Q7DCz5C5ArWE" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title Main character selection / SamGeo mask (Face box mode)\n", "# @markdown Draw a positive box around the desired face on frame 0, then run this cell.\n", "# @markdown Sets global `select_output` (parsed JSON after `<----output---->`).\n", "\n", "import json as _json\n", "import os\n", "import sys\n", "\n", "def run_select_obj_cell(cfg, prep: dict, *, points_store_json=None, positive_boxes_json=None) -> dict:\n", " scaled_pt = str(prep.get(\"scaled_frames_pt\") or \"\").strip()\n", " if not scaled_pt or not os.path.isfile(scaled_pt):\n", " raise FileNotFoundError(f\"scaled_frames_pt missing or not found: {scaled_pt!r}\")\n", "\n", " script = _gen_steps_script(\"select_obj.py\")\n", " cmd = [\n", " sys.executable,\n", " \"-u\",\n", " script,\n", " \"--scaled_frames_pt\",\n", " scaled_pt,\n", " \"--samgeo_prompt_mode\",\n", " \"box\",\n", " \"--samgeo_text_prompt\",\n", " \"person\",\n", " \"--samgeo_object_id\",\n", " str(getattr(cfg, \"samgeo_object_id\", 0)).strip(),\n", " \"--samgeo_prompt_frame_idx\",\n", " \"0\",\n", " \"--force_rate35\",\n", " str(int(cfg.force_rate)),\n", " ]\n", " if positive_boxes_json:\n", " cmd.extend([\"--samgeo_positive_boxes\", str(positive_boxes_json)])\n", " if points_store_json:\n", " cmd.extend([\"--points_store45\", str(points_store_json)])\n", " if getattr(cfg, \"save_mask_output\", False):\n", " cmd.append(\"--save_mask_output\")\n", "\n", " output = run_gen_steps_command(\n", " cmd,\n", " label=\"select_obj.py\",\n", " interactive=True,\n", " )\n", " return parse_gen_steps_output(output)\n", "\n", "if \"config\" not in globals():\n", " raise RuntimeError(\"Run the setup cell first (config is missing).\")\n", "if \"prep_output\" not in globals() or not prep_output:\n", " raise RuntimeError(\"Run the setup cell first (prep_output is missing).\")\n", "if not isinstance(prep_output, dict) or not prep_output.get(\"ok\"):\n", " raise RuntimeError(\n", " \"prep_output invalid or prep_media ok=False; fix setup errors and re-run the setup cell.\"\n", " )\n", "if \"_gen_steps_script\" not in globals() or \"run_gen_steps_command\" not in globals():\n", " raise RuntimeError(\"Run the setup cell first (gen-steps helpers are missing).\")\n", "\n", "points_json = None\n", "positive_boxes_json = None\n", "\n", "if \"bbox_widget\" not in dir() or bbox_widget is None:\n", " raise RuntimeError(\n", " \"bbox_widget is missing. Enable the bbox selection widget and draw a positive face box.\"\n", " )\n", "\n", "samgeo_data = bbox_widget.get_separated_samgeo_format()\n", "if samgeo_data.get(\"has_positive\"):\n", " positive_boxes_json = _json.dumps(samgeo_data[\"positive_boxes\"])\n", "else:\n", " raise RuntimeError(\n", " \"No positive boxes were drawn. Draw a tight box around the target face, then run again.\"\n", " )\n", "\n", "print(\"\\n\" + \"=\" * 50)\n", "print(\"Main character selection (face box mode)\")\n", "print(\"=\" * 50 + \"\\n\")\n", "\n", "select_output = run_select_obj_cell(\n", " config,\n", " prep_output,\n", " points_store_json=points_json,\n", " positive_boxes_json=positive_boxes_json,\n", ")\n", "\n", "if not isinstance(select_output, dict) or not select_output.get(\"ok\"):\n", " raise RuntimeError(\n", " f\"select_obj failed or ok=False: {select_output.get('error') if isinstance(select_output, dict) else select_output!r}\"\n", " )\n", "\n", "print(\n", " f\"\\nselect_output ok={select_output.get('ok')} \"\n", " f\"bundle_path={select_output.get('bundle_path')!r} \"\n", " f\"artifact_dir={select_output.get('artifact_dir')!r}\"\n", ")" ], "metadata": { "id": "MHd1Tch92dhD" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title Export selected person video for DWPose\n", "\n", "import cv2\n", "import json\n", "import os\n", "import re\n", "from pathlib import Path\n", "\n", "import numpy as np\n", "\n", "try:\n", " import torch\n", "except Exception:\n", " torch = None\n", "\n", "try:\n", " from IPython.display import Video, display\n", "except Exception:\n", " Video = None\n", " display = None\n", "\n", "input_video = str(getattr(config, \"ref_video_path\", \"/content/ComfyUI/input/boy_speaking_11s.mp4\"))\n", "if not os.path.isabs(input_video):\n", " input_video = str(Path(\"/content/ComfyUI/input\") / input_video)\n", "output_video = \"/content/ComfyUI/input/selected_person.mp4\"\n", "frame_rate = int(getattr(config, \"force_rate\", 24) or 24)\n", "\n", "if \"select_output\" not in globals() or not select_output:\n", " raise RuntimeError(\"Run the face selection cell first; `select_output` is missing.\")\n", "if not isinstance(select_output, dict) or not select_output.get(\"ok\"):\n", " raise RuntimeError(\"select_output invalid or ok=False; re-run the face selection cell.\")\n", "\n", "bundle_path = Path(str(select_output.get(\"bundle_path\") or \"\").strip())\n", "if not bundle_path.is_file():\n", " raise RuntimeError(f\"select_output['bundle_path'] missing or not found: {bundle_path!r}\")\n", "\n", "artifact_dir = Path(str(select_output.get(\"artifact_dir\") or bundle_path.parent).strip())\n", "if not artifact_dir.exists():\n", " artifact_dir = bundle_path.parent\n", "\n", "# The selected object id is the most reliable hint when the bundle contains\n", "# per-frame masks.\n", "selected_obj_id = select_output.get(\"samgeo_object_id\")\n", "try:\n", " selected_obj_id = int(selected_obj_id)\n", "except Exception:\n", " selected_obj_id = None\n", "\n", "\n", "def _natural_key(path: Path):\n", " parts = re.split(r\"(\\d+)\", path.name)\n", " return [int(p) if p.isdigit() else p.lower() for p in parts]\n", "\n", "\n", "def _read_mask_image(path: Path):\n", " mask = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)\n", " if mask is None:\n", " return None\n", " if mask.ndim == 3:\n", " mask = mask[..., 0]\n", " return (mask > 0).astype(np.uint8)\n", "\n", "\n", "def _coerce_frame_outputs(source):\n", " formatted = {}\n", " if not isinstance(source, dict):\n", " return formatted\n", "\n", " for frame_idx, outputs in source.items():\n", " try:\n", " frame_idx = int(frame_idx)\n", " except Exception:\n", " pass\n", "\n", " formatted[frame_idx] = {}\n", " if not isinstance(outputs, dict):\n", " continue\n", "\n", " if \"out_obj_ids\" in outputs:\n", " obj_ids = outputs[\"out_obj_ids\"]\n", " masks = outputs.get(\n", " \"out_binary_masks\",\n", " outputs.get(\"out_mask_logits\", outputs.get(\"masks\", [])),\n", " )\n", " if hasattr(obj_ids, \"tolist\"):\n", " obj_ids = obj_ids.tolist()\n", " for i, obj_id in enumerate(obj_ids):\n", " if i >= len(masks):\n", " continue\n", " mask = masks[i]\n", " if hasattr(mask, \"cpu\"):\n", " mask = (mask > 0.0).cpu().numpy()\n", " elif hasattr(mask, \"numpy\"):\n", " mask = (mask > 0.0).numpy()\n", " else:\n", " mask = np.asarray(mask) > 0.0\n", " try:\n", " obj_id = int(obj_id)\n", " except Exception:\n", " pass\n", " formatted[frame_idx][obj_id] = np.asarray(mask).squeeze().astype(np.uint8)\n", " continue\n", "\n", " if \"object_ids\" in outputs:\n", " obj_ids = outputs[\"object_ids\"]\n", " masks = outputs.get(\"masks\", [])\n", " if hasattr(obj_ids, \"tolist\"):\n", " obj_ids = obj_ids.tolist()\n", " for i, obj_id in enumerate(obj_ids):\n", " if i >= len(masks):\n", " continue\n", " mask = masks[i]\n", " if hasattr(mask, \"cpu\"):\n", " mask = (mask > 0.0).cpu().numpy()\n", " elif hasattr(mask, \"numpy\"):\n", " mask = (mask > 0.0).numpy()\n", " else:\n", " mask = np.asarray(mask) > 0.0\n", " try:\n", " obj_id = int(obj_id)\n", " except Exception:\n", " pass\n", " formatted[frame_idx][obj_id] = np.asarray(mask).squeeze().astype(np.uint8)\n", " continue\n", "\n", " # Already in {obj_id: mask} format.\n", " for obj_id, mask in outputs.items():\n", " if hasattr(mask, \"cpu\"):\n", " mask = (mask > 0.0).cpu().numpy()\n", " elif hasattr(mask, \"numpy\"):\n", " mask = (mask > 0.0).numpy()\n", " else:\n", " mask = np.asarray(mask) > 0.0\n", " try:\n", " obj_id = int(obj_id)\n", " except Exception:\n", " pass\n", " formatted[frame_idx][obj_id] = np.asarray(mask).squeeze().astype(np.uint8)\n", "\n", " return formatted\n", "\n", "\n", "def _load_bundle(path: Path):\n", " if torch is None:\n", " return None\n", " try:\n", " return torch.load(str(path), map_location=\"cpu\", weights_only=False)\n", " except TypeError:\n", " try:\n", " return torch.load(str(path), map_location=\"cpu\")\n", " except Exception as exc:\n", " print(f\"[SamGeo export] torch.load failed for {path.name}: {exc}\")\n", " return None\n", " except Exception as exc:\n", " print(f\"[SamGeo export] torch.load failed for {path.name}: {exc}\")\n", " return None\n", "\n", "\n", "bundle = _load_bundle(bundle_path)\n", "formatted = {}\n", "\n", "if isinstance(bundle, dict):\n", " for key in (\"formatted_outputs\", \"outputs_per_frame\", \"frame_outputs\"):\n", " candidate = bundle.get(key)\n", " if isinstance(candidate, dict) and candidate:\n", " formatted = _coerce_frame_outputs(candidate)\n", " if formatted:\n", " break\n", "\n", " if not formatted:\n", " # Some bundles may store the raw frame mapping at the top level.\n", " formatted = _coerce_frame_outputs(bundle)\n", "\n", "if not formatted:\n", " # Fallback: use saved binary masks under the artifact directory if present.\n", " image_files = [\n", " p for p in artifact_dir.rglob(\"*\")\n", " if p.suffix.lower() in {\".png\", \".jpg\", \".jpeg\", \".bmp\", \".tif\", \".tiff\"}\n", " ]\n", " image_files = sorted(image_files, key=_natural_key)\n", " if not image_files:\n", " raise RuntimeError(\n", " f\"Could not find a usable mask sequence in {bundle_path} or {artifact_dir}\"\n", " )\n", "\n", " fallback_obj_id = selected_obj_id if selected_obj_id is not None else 0\n", " for frame_idx, mask_path in enumerate(image_files):\n", " mask = _read_mask_image(mask_path)\n", " if mask is None:\n", " continue\n", " formatted[frame_idx] = {fallback_obj_id: mask}\n", "\n", "if not formatted:\n", " raise RuntimeError(\"No masks could be decoded from the selection artifact.\")\n", "\n", "# Pick the selected object id.\n", "if selected_obj_id is None or not any(selected_obj_id in frame for frame in formatted.values()):\n", " first_non_empty = next((frame for frame in sorted(formatted) if formatted[frame]), None)\n", " if first_non_empty is None:\n", " raise RuntimeError(\"No non-empty mask frames found in the selection artifact.\")\n", " selected_obj_id = max(\n", " formatted[first_non_empty].keys(),\n", " key=lambda oid: float(np.asarray(formatted[first_non_empty][oid]).sum()),\n", " )\n", "\n", "cap = cv2.VideoCapture(input_video)\n", "if not cap.isOpened():\n", " raise RuntimeError(f\"Could not open input video: {input_video}\")\n", "\n", "w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n", "h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n", "fourcc = cv2.VideoWriter_fourcc(*\"mp4v\")\n", "writer = cv2.VideoWriter(output_video, fourcc, frame_rate, (w, h))\n", "\n", "frame_idx = 0\n", "written = 0\n", "while True:\n", " ret, frame_bgr = cap.read()\n", " if not ret:\n", " break\n", "\n", " frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)\n", " frame_masks = formatted.get(frame_idx, {})\n", " mask = frame_masks.get(selected_obj_id)\n", "\n", " if mask is None:\n", " selected = np.zeros_like(frame_rgb)\n", " else:\n", " mask = np.asarray(mask).squeeze()\n", " if mask.shape != (h, w):\n", " mask = cv2.resize(mask.astype(np.uint8), (w, h), interpolation=cv2.INTER_NEAREST)\n", " mask = (mask > 0).astype(np.uint8)\n", " selected = frame_rgb * mask[..., None]\n", "\n", " writer.write(cv2.cvtColor(selected, cv2.COLOR_RGB2BGR))\n", " frame_idx += 1\n", " written += 1\n", "\n", "cap.release()\n", "writer.release()\n", "\n", "print(f\"Saved selected-person video: {output_video}\")\n", "print(f\"Frames written: {written}\")\n", "print(f\"Target object id: {selected_obj_id}\")\n", "print(f\"Selection artifact dir: {artifact_dir}\")\n", "\n", "if display is not None and Video is not None:\n", " display(Video(str(output_video), embed=True, width=640))\n" ], "metadata": { "id": "8jPAs2qC2dgl" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "ZO92UwYx6KfA" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "EqtvEXcC6Kee" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title **LipDub dw_pose_sam3_api inference**\n", "\n", "import json\n", "import os\n", "import shutil\n", "import time\n", "import uuid\n", "from pathlib import Path\n", "\n", "import requests\n", "\n", "WORK_DIR = \"/workspace\" if os.path.exists(\"/workspace\") else \"/content\"\n", "COMFY_DIR = f\"{WORK_DIR}/ComfyUI\"\n", "COMFY_HOST = \"http://127.0.0.1:8188\"\n", "INPUT_DIR = Path(COMFY_DIR) / \"input\"\n", "OUTPUT_DIR = Path(COMFY_DIR) / \"output\"\n", "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", "\n", "workflow_json = \"dw_pose_sam3_api copy.json\" # @param {type:\"string\"}\n", "input_video = str(getattr(config, \"ref_video_path\", \"boy_speaking_11s.mp4\")) # @param {type:\"string\"}\n", "if not os.path.isabs(input_video):\n", " input_video = str(Path(\"/content/ComfyUI/input\") / input_video)\n", "selected_person_video = \"selected_person.mp4\" # @param {type:\"string\"}\n", "input_audio = \"tts_11s.wav\" # @param {type:\"string\"}\n", "clip_length_sec = 11.0 # @param {type:\"number\"}\n", "force_rate = 24 # @param {type:\"integer\"}\n", "target_width = 1280 # @param {type:\"integer\"}\n", "target_height = 720 # @param {type:\"integer\"}\n", "use_inpainting = True # @param {type:\"boolean\"}\n", "use_chin_mask = False # @param {type:\"boolean\"}\n", "use_sam3geo = False # @param {type:\"boolean\"}\n", "positive_prompt = \"A person speaking in sync with the provided audio.\" # @param {type:\"string\"}\n", "negative_prompt = \"subtitles, text, silent, mouth shut, extra faces, distorted mouth\" # @param {type:\"string\"}\n", "output_prefix = \"dw_pose_sam3_api_lipdub\" # @param {type:\"string\"}\n", "\n", "MAX_WAIT_SEC = 3600\n", "POLL_INTERVAL_SEC = 3\n", "POST_COMPLETE_WAIT_SEC = 60\n", "\n", "\n", "def log(msg: str):\n", " print(f\"[LIPDUB] {msg}\")\n", "\n", "\n", "def resolve_workflow_json(path_value: str) -> Path:\n", " name = (path_value or \"\").strip()\n", " candidates = [\n", " Path(name),\n", " Path(WORK_DIR) / name,\n", " Path(COMFY_DIR) / name,\n", " Path(COMFY_DIR) / \"user\" / \"default\" / \"workflows\" / name,\n", " ]\n", " for candidate in candidates:\n", " if candidate.is_file():\n", " return candidate\n", " raise FileNotFoundError(f\"Workflow JSON not found: {name}\")\n", "\n", "\n", "def resolve_input_file(value: str) -> Path:\n", " name = (value or \"\").strip()\n", " if not name:\n", " raise ValueError(\"input path is empty\")\n", "\n", " candidates = [\n", " Path(name),\n", " INPUT_DIR / name,\n", " Path(WORK_DIR) / name,\n", " Path(COMFY_DIR) / name,\n", " ]\n", " for candidate in candidates:\n", " if candidate.is_file():\n", " return candidate\n", " raise FileNotFoundError(f\"File not found: {name}\")\n", "\n", "\n", "def ensure_comfy_input_file(value: str) -> str:\n", " src = resolve_input_file(value)\n", " dst = INPUT_DIR / src.name\n", " if src.resolve() != dst.resolve():\n", " shutil.copy2(src, dst)\n", " return src.name\n", "\n", "\n", "def set_node_input(prompt_graph: dict, node_id: str, input_key: str, value):\n", " node = prompt_graph.get(str(node_id))\n", " if not isinstance(node, dict):\n", " return False\n", " inputs = node.get(\"inputs\")\n", " if not isinstance(inputs, dict):\n", " return False\n", " inputs[input_key] = value\n", " return True\n", "\n", "\n", "def post_prompt(prompt_graph: dict) -> str:\n", " payload = {\"prompt\": prompt_graph, \"client_id\": str(uuid.uuid4())}\n", " resp = requests.post(f\"{COMFY_HOST}/prompt\", json=payload, timeout=60)\n", " if resp.status_code != 200:\n", " raise RuntimeError(f\"/prompt failed {resp.status_code}: {resp.text[:2000]}\")\n", " data = resp.json()\n", " prompt_id = data.get(\"prompt_id\")\n", " if not prompt_id:\n", " raise RuntimeError(f\"Missing prompt_id in response: {data}\")\n", " return prompt_id\n", "\n", "\n", "def wait_for_completion(prompt_id: str):\n", " start = time.time()\n", " last_entry = None\n", " while time.time() - start < MAX_WAIT_SEC:\n", " try:\n", " history = requests.get(f\"{COMFY_HOST}/history/{prompt_id}\", timeout=10).json()\n", " except Exception:\n", " history = {}\n", "\n", " entry = history.get(prompt_id, {})\n", " last_entry = entry\n", " status = entry.get(\"status\", {}) or {}\n", " if status.get(\"status_str\") == \"error\":\n", " raise RuntimeError(f\"Workflow failed: {json.dumps(status, indent=2)[:6000]}\")\n", " if status.get(\"completed\"):\n", " return entry\n", "\n", " print(\".\", end=\"\", flush=True)\n", " time.sleep(POLL_INTERVAL_SEC)\n", "\n", " raise TimeoutError(\n", " \"Timed out waiting for ComfyUI job.\\n\"\n", " + (json.dumps(last_entry, indent=2)[:30000] if last_entry else \"\")\n", " )\n", "\n", "\n", "def history_mp4_entries(history_entry: dict):\n", " out = []\n", " for bucket in (history_entry.get(\"outputs\", {}) or {}).values():\n", " if not isinstance(bucket, dict):\n", " continue\n", " for arr in bucket.values():\n", " if not isinstance(arr, list):\n", " continue\n", " for obj in arr:\n", " if isinstance(obj, dict) and str(obj.get(\"filename\", \"\")).lower().endswith(\".mp4\"):\n", " out.append({\n", " \"filename\": obj[\"filename\"],\n", " \"subfolder\": obj.get(\"subfolder\", \"\"),\n", " \"type\": obj.get(\"type\", \"output\"),\n", " })\n", " return out\n", "\n", "\n", "def download_mp4_with_fallback(history_entry: dict, output_dir: Path, run_started_at: float) -> Path:\n", " mp4s = history_mp4_entries(history_entry)\n", " if mp4s:\n", " first = mp4s[-1]\n", " r = requests.get(\n", " f\"{COMFY_HOST}/view\",\n", " params={\n", " \"filename\": first[\"filename\"],\n", " \"subfolder\": first.get(\"subfolder\", \"\"),\n", " \"type\": first.get(\"type\", \"output\"),\n", " },\n", " timeout=180,\n", " )\n", " r.raise_for_status()\n", " dest = output_dir / first[\"filename\"]\n", " dest.write_bytes(r.content)\n", " return dest\n", "\n", " candidates = []\n", " for p in output_dir.glob(\"*.mp4\"):\n", " try:\n", " if p.stat().st_mtime >= run_started_at - 1:\n", " candidates.append(p)\n", " except FileNotFoundError:\n", " pass\n", "\n", " if candidates:\n", " return max(candidates, key=lambda p: p.stat().st_mtime)\n", "\n", " raise RuntimeError(\"No MP4 outputs found in history or output directory.\")\n", "\n", "\n", "def display_video(path: Path):\n", " try:\n", " from IPython.display import Video, display\n", " display(Video(str(path), embed=True, width=640))\n", " except Exception:\n", " pass\n", "\n", "\n", "workflow_path = resolve_workflow_json(workflow_json)\n", "video_path = resolve_input_file(input_video)\n", "audio_path = resolve_input_file(input_audio)\n", "selected_video_path = None\n", "if use_sam3geo:\n", " selected_video_path = video_path\n", "else:\n", " selected_video_path = resolve_input_file(selected_person_video)\n", "\n", "video_name = ensure_comfy_input_file(str(video_path))\n", "audio_name = ensure_comfy_input_file(str(audio_path))\n", "selected_video_name = ensure_comfy_input_file(str(selected_video_path))\n", "frame_load_cap = max(1, int(clip_length_sec * force_rate) + 1)\n", "\n", "with open(workflow_path, \"r\", encoding=\"utf-8\") as f:\n", " workflow_prompt = json.load(f)\n", "\n", "patch_results = {\n", " \"video_path\": set_node_input(workflow_prompt, \"5893\", \"video\", video_name),\n", " \"frame_load_cap\": set_node_input(workflow_prompt, \"5893\", \"frame_load_cap\", frame_load_cap),\n", " \"custom_width\": set_node_input(workflow_prompt, \"5893\", \"custom_width\", target_width),\n", " \"custom_height\": set_node_input(workflow_prompt, \"5893\", \"custom_height\", target_height),\n", " \"force_rate\": set_node_input(workflow_prompt, \"5893\", \"force_rate\", force_rate),\n", " \"audio_path\": set_node_input(workflow_prompt, \"5883\", \"audio\", audio_name),\n", " \"audio_start\": set_node_input(workflow_prompt, \"5882\", \"value\", 0),\n", " \"audio_length\": set_node_input(workflow_prompt, \"5885\", \"value\", clip_length_sec),\n", " \"positive_prompt\": set_node_input(workflow_prompt, \"5894:5834\", \"text\", positive_prompt),\n", " \"negative_prompt\": set_node_input(workflow_prompt, \"5894:5835\", \"text\", negative_prompt),\n", " \"output_prefix\": set_node_input(workflow_prompt, \"5921\", \"filename_prefix\", output_prefix),\n", " \"use_inpainting\": set_node_input(workflow_prompt, \"6015\", \"value\", use_inpainting),\n", " \"use_chin_mask\": set_node_input(workflow_prompt, \"6051\", \"value\", use_chin_mask),\n", " \"use_sam3geo\": set_node_input(workflow_prompt, \"6068\", \"value\", use_sam3geo),\n", " \"sam3geo_video\": set_node_input(workflow_prompt, \"6070\", \"video\", selected_video_name),\n", "}\n", "\n", "if not all(patch_results.values()):\n", " raise RuntimeError(f\"One or more required patches failed: {patch_results}\")\n", "\n", "log(f\"workflow: {workflow_path.name}\")\n", "log(f\"input video: {video_name}\")\n", "log(f\"input audio: {audio_name}\")\n", "log(f\"clip length sec: {clip_length_sec}\")\n", "log(f\"frame load cap: {frame_load_cap}\")\n", "log(f\"use inpainting: {use_inpainting}\")\n", "log(f\"use chin mask: {use_chin_mask}\")\n", "log(f\"output prefix: {output_prefix}\")\n", "\n", "prompt_id = post_prompt(workflow_prompt)\n", "print(f\"Queued: {prompt_id}\")\n", "\n", "run_started_at = time.time()\n", "entry = wait_for_completion(prompt_id)\n", "\n", "output_dir = Path(COMFY_DIR) / \"output\"\n", "output_dir.mkdir(parents=True, exist_ok=True)\n", "raw_output_path = None\n", "deadline = time.time() + POST_COMPLETE_WAIT_SEC\n", "while time.time() < deadline:\n", " try:\n", " history = requests.get(f\"{COMFY_HOST}/history/{prompt_id}\", timeout=10).json()\n", " except Exception:\n", " history = {}\n", " entry = history.get(prompt_id, entry)\n", " try:\n", " raw_output_path = download_mp4_with_fallback(entry, output_dir, run_started_at)\n", " break\n", " except RuntimeError:\n", " time.sleep(2)\n", "\n", "if raw_output_path is None:\n", " raise RuntimeError(\"No MP4 outputs found in history or output directory after completion wait window.\")\n", "\n", "final_output_path = output_dir / f\"{video_path.stem}_lipdub_{uuid.uuid4().hex[:6]}.mp4\"\n", "if raw_output_path.resolve() != final_output_path.resolve():\n", " raw_output_path.replace(final_output_path)\n", "else:\n", " final_output_path = raw_output_path\n", "\n", "print(\"\\n\" + \"=\" * 56)\n", "print(\"COMPLETE\")\n", "print(f\"Output: {final_output_path}\")\n", "print(f\"Size: {final_output_path.stat().st_size / (1024 * 1024):.1f} MB\")\n", "print(\"=\" * 56)\n", "\n", "display_video(final_output_path)\n" ], "metadata": { "id": "4jLFyw3x6LIc" }, "execution_count": null, "outputs": [] } ] }