Spaces:
Sleeping
Sleeping
Commit ·
bc6ab10
1
Parent(s): 23c230d
fix: auto-generate PO tokens headlessly for datacenter IP bypass
Browse filesYouTube blocks ALL datacenter IPs regardless of client type.
ANDROID_VR works on residential IPs but fails on HF Spaces.
New approach with cascading fallback:
1. Try ANDROID_VR, ANDROID_MUSIC, WEB_KIDS, WEB_CREATOR (no PO token)
2. If all fail, generate PO token headlessly via Node.js
(npx youtube-po-token-generator — DOM simulation, no browser)
3. Use WEB client with the generated PO token
The PO token makes YouTube think the request comes from a real
browser, bypassing datacenter IP blocking.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
- pipeline_runner.py +69 -11
pipeline_runner.py
CHANGED
|
@@ -153,37 +153,95 @@ def _is_youtube_url(url: str) -> bool:
|
|
| 153 |
return bool(_YT_RE.search(url))
|
| 154 |
|
| 155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
def _download_youtube(url: str, output_path: str) -> str:
|
| 157 |
-
"""Download a YouTube video using pytubefix
|
| 158 |
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
|
|
|
| 162 |
"""
|
| 163 |
from pytubefix import YouTube
|
| 164 |
|
| 165 |
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
|
| 166 |
print(f"[pytubefix] Downloading {url} ...")
|
| 167 |
|
| 168 |
-
#
|
| 169 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
print(f"[pytubefix] Title: {yt.title} ({yt.length}s)")
|
| 171 |
|
| 172 |
-
# Get best progressive stream (video+audio in one file)
|
| 173 |
stream = (
|
| 174 |
yt.streams
|
| 175 |
.filter(progressive=True, file_extension="mp4")
|
| 176 |
.order_by("resolution")
|
| 177 |
.desc()
|
| 178 |
.first()
|
| 179 |
-
)
|
| 180 |
-
|
| 181 |
-
stream = yt.streams.filter(progressive=True).first()
|
| 182 |
if not stream:
|
| 183 |
raise RuntimeError("No downloadable stream found for this video.")
|
| 184 |
|
| 185 |
print(f"[pytubefix] Stream: {stream.resolution} {stream.mime_type}")
|
| 186 |
-
|
| 187 |
out_dir = os.path.dirname(output_path) or "."
|
| 188 |
out_name = os.path.basename(output_path)
|
| 189 |
stream.download(output_path=out_dir, filename=out_name)
|
|
|
|
| 153 |
return bool(_YT_RE.search(url))
|
| 154 |
|
| 155 |
|
| 156 |
+
def _generate_po_token() -> tuple[str, str]:
|
| 157 |
+
"""Generate YouTube PO token headlessly via Node.js.
|
| 158 |
+
|
| 159 |
+
Uses youtube-po-token-generator (npm) which runs a JS DOM simulation
|
| 160 |
+
to create valid {visitorData, poToken} pairs without a browser.
|
| 161 |
+
"""
|
| 162 |
+
import json
|
| 163 |
+
import subprocess
|
| 164 |
+
|
| 165 |
+
print("[po-token] Generating PO token via Node.js...")
|
| 166 |
+
try:
|
| 167 |
+
result = subprocess.run(
|
| 168 |
+
["npx", "--yes", "youtube-po-token-generator"],
|
| 169 |
+
capture_output=True, text=True, timeout=60,
|
| 170 |
+
)
|
| 171 |
+
if result.returncode != 0:
|
| 172 |
+
raise RuntimeError(f"npx failed: {result.stderr.strip()}")
|
| 173 |
+
data = json.loads(result.stdout.strip())
|
| 174 |
+
visitor_data = data["visitorData"]
|
| 175 |
+
po_token = data["poToken"]
|
| 176 |
+
print(f"[po-token] Generated: visitorData={visitor_data[:20]}... poToken={po_token[:20]}...")
|
| 177 |
+
return visitor_data, po_token
|
| 178 |
+
except json.JSONDecodeError:
|
| 179 |
+
raise RuntimeError(f"Invalid JSON from po-token-generator: {result.stdout[:200]}")
|
| 180 |
+
except subprocess.TimeoutExpired:
|
| 181 |
+
raise RuntimeError("PO token generation timed out (60s)")
|
| 182 |
+
|
| 183 |
+
|
| 184 |
def _download_youtube(url: str, output_path: str) -> str:
|
| 185 |
+
"""Download a YouTube video using pytubefix with auto-generated PO token.
|
| 186 |
|
| 187 |
+
YouTube blocks datacenter IPs. We bypass this by:
|
| 188 |
+
1. DoH monkey-patch resolves youtube.com via Cloudflare (bypass DNS block)
|
| 189 |
+
2. youtube-po-token-generator creates valid PO tokens headlessly (bypass bot detection)
|
| 190 |
+
3. pytubefix WEB_CREATOR client downloads with the PO token
|
| 191 |
"""
|
| 192 |
from pytubefix import YouTube
|
| 193 |
|
| 194 |
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
|
| 195 |
print(f"[pytubefix] Downloading {url} ...")
|
| 196 |
|
| 197 |
+
# Try clients that DON'T need PO tokens first (fastest)
|
| 198 |
+
for client_name in ("ANDROID_VR", "ANDROID_MUSIC", "WEB_KIDS", "WEB_CREATOR"):
|
| 199 |
+
try:
|
| 200 |
+
print(f"[pytubefix] Trying {client_name} client...")
|
| 201 |
+
yt = YouTube(url, client=client_name)
|
| 202 |
+
title = yt.title
|
| 203 |
+
print(f"[pytubefix] Title: {title} ({yt.length}s)")
|
| 204 |
+
stream = (
|
| 205 |
+
yt.streams
|
| 206 |
+
.filter(progressive=True, file_extension="mp4")
|
| 207 |
+
.order_by("resolution")
|
| 208 |
+
.desc()
|
| 209 |
+
.first()
|
| 210 |
+
) or yt.streams.filter(progressive=True).first()
|
| 211 |
+
if stream:
|
| 212 |
+
print(f"[pytubefix] Stream: {stream.resolution} {stream.mime_type}")
|
| 213 |
+
out_dir = os.path.dirname(output_path) or "."
|
| 214 |
+
out_name = os.path.basename(output_path)
|
| 215 |
+
stream.download(output_path=out_dir, filename=out_name)
|
| 216 |
+
file_size = os.path.getsize(output_path)
|
| 217 |
+
print(f"[pytubefix] Done: {output_path} ({file_size / 1024 / 1024:.1f} MB)")
|
| 218 |
+
return output_path
|
| 219 |
+
except Exception as exc:
|
| 220 |
+
print(f"[pytubefix] {client_name} failed: {exc}")
|
| 221 |
+
continue
|
| 222 |
+
|
| 223 |
+
# All simple clients failed — generate PO token and use WEB client
|
| 224 |
+
print("[pytubefix] All simple clients blocked. Generating PO token...")
|
| 225 |
+
visitor_data, po_token = _generate_po_token()
|
| 226 |
+
|
| 227 |
+
def _po_verifier():
|
| 228 |
+
return visitor_data, po_token
|
| 229 |
+
|
| 230 |
+
yt = YouTube(url, client="WEB", po_token_verifier=_po_verifier)
|
| 231 |
print(f"[pytubefix] Title: {yt.title} ({yt.length}s)")
|
| 232 |
|
|
|
|
| 233 |
stream = (
|
| 234 |
yt.streams
|
| 235 |
.filter(progressive=True, file_extension="mp4")
|
| 236 |
.order_by("resolution")
|
| 237 |
.desc()
|
| 238 |
.first()
|
| 239 |
+
) or yt.streams.filter(progressive=True).first()
|
| 240 |
+
|
|
|
|
| 241 |
if not stream:
|
| 242 |
raise RuntimeError("No downloadable stream found for this video.")
|
| 243 |
|
| 244 |
print(f"[pytubefix] Stream: {stream.resolution} {stream.mime_type}")
|
|
|
|
| 245 |
out_dir = os.path.dirname(output_path) or "."
|
| 246 |
out_name = os.path.basename(output_path)
|
| 247 |
stream.download(output_path=out_dir, filename=out_name)
|