HeshamHaroon Claude Opus 4.6 (1M context) commited on
Commit
bc6ab10
·
1 Parent(s): 23c230d

fix: auto-generate PO tokens headlessly for datacenter IP bypass

Browse files

YouTube blocks ALL datacenter IPs regardless of client type.
ANDROID_VR works on residential IPs but fails on HF Spaces.

New approach with cascading fallback:
1. Try ANDROID_VR, ANDROID_MUSIC, WEB_KIDS, WEB_CREATOR (no PO token)
2. If all fail, generate PO token headlessly via Node.js
(npx youtube-po-token-generator — DOM simulation, no browser)
3. Use WEB client with the generated PO token

The PO token makes YouTube think the request comes from a real
browser, bypassing datacenter IP blocking.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Files changed (1) hide show
  1. pipeline_runner.py +69 -11
pipeline_runner.py CHANGED
@@ -153,37 +153,95 @@ def _is_youtube_url(url: str) -> bool:
153
  return bool(_YT_RE.search(url))
154
 
155
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
156
  def _download_youtube(url: str, output_path: str) -> str:
157
- """Download a YouTube video using pytubefix ANDROID_VR client.
158
 
159
- The ANDROID_VR client bypasses YouTube's bot detection without needing
160
- PO tokens or cookies. The DoH monkey-patch (applied at import time)
161
- handles DNS resolution on HF Spaces where youtube.com is blocked.
 
162
  """
163
  from pytubefix import YouTube
164
 
165
  os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
166
  print(f"[pytubefix] Downloading {url} ...")
167
 
168
- # ANDROID_VR client: no PO token needed, no bot detection
169
- yt = YouTube(url, client="ANDROID_VR")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
170
  print(f"[pytubefix] Title: {yt.title} ({yt.length}s)")
171
 
172
- # Get best progressive stream (video+audio in one file)
173
  stream = (
174
  yt.streams
175
  .filter(progressive=True, file_extension="mp4")
176
  .order_by("resolution")
177
  .desc()
178
  .first()
179
- )
180
- if not stream:
181
- stream = yt.streams.filter(progressive=True).first()
182
  if not stream:
183
  raise RuntimeError("No downloadable stream found for this video.")
184
 
185
  print(f"[pytubefix] Stream: {stream.resolution} {stream.mime_type}")
186
-
187
  out_dir = os.path.dirname(output_path) or "."
188
  out_name = os.path.basename(output_path)
189
  stream.download(output_path=out_dir, filename=out_name)
 
153
  return bool(_YT_RE.search(url))
154
 
155
 
156
+ def _generate_po_token() -> tuple[str, str]:
157
+ """Generate YouTube PO token headlessly via Node.js.
158
+
159
+ Uses youtube-po-token-generator (npm) which runs a JS DOM simulation
160
+ to create valid {visitorData, poToken} pairs without a browser.
161
+ """
162
+ import json
163
+ import subprocess
164
+
165
+ print("[po-token] Generating PO token via Node.js...")
166
+ try:
167
+ result = subprocess.run(
168
+ ["npx", "--yes", "youtube-po-token-generator"],
169
+ capture_output=True, text=True, timeout=60,
170
+ )
171
+ if result.returncode != 0:
172
+ raise RuntimeError(f"npx failed: {result.stderr.strip()}")
173
+ data = json.loads(result.stdout.strip())
174
+ visitor_data = data["visitorData"]
175
+ po_token = data["poToken"]
176
+ print(f"[po-token] Generated: visitorData={visitor_data[:20]}... poToken={po_token[:20]}...")
177
+ return visitor_data, po_token
178
+ except json.JSONDecodeError:
179
+ raise RuntimeError(f"Invalid JSON from po-token-generator: {result.stdout[:200]}")
180
+ except subprocess.TimeoutExpired:
181
+ raise RuntimeError("PO token generation timed out (60s)")
182
+
183
+
184
  def _download_youtube(url: str, output_path: str) -> str:
185
+ """Download a YouTube video using pytubefix with auto-generated PO token.
186
 
187
+ YouTube blocks datacenter IPs. We bypass this by:
188
+ 1. DoH monkey-patch resolves youtube.com via Cloudflare (bypass DNS block)
189
+ 2. youtube-po-token-generator creates valid PO tokens headlessly (bypass bot detection)
190
+ 3. pytubefix WEB_CREATOR client downloads with the PO token
191
  """
192
  from pytubefix import YouTube
193
 
194
  os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
195
  print(f"[pytubefix] Downloading {url} ...")
196
 
197
+ # Try clients that DON'T need PO tokens first (fastest)
198
+ for client_name in ("ANDROID_VR", "ANDROID_MUSIC", "WEB_KIDS", "WEB_CREATOR"):
199
+ try:
200
+ print(f"[pytubefix] Trying {client_name} client...")
201
+ yt = YouTube(url, client=client_name)
202
+ title = yt.title
203
+ print(f"[pytubefix] Title: {title} ({yt.length}s)")
204
+ stream = (
205
+ yt.streams
206
+ .filter(progressive=True, file_extension="mp4")
207
+ .order_by("resolution")
208
+ .desc()
209
+ .first()
210
+ ) or yt.streams.filter(progressive=True).first()
211
+ if stream:
212
+ print(f"[pytubefix] Stream: {stream.resolution} {stream.mime_type}")
213
+ out_dir = os.path.dirname(output_path) or "."
214
+ out_name = os.path.basename(output_path)
215
+ stream.download(output_path=out_dir, filename=out_name)
216
+ file_size = os.path.getsize(output_path)
217
+ print(f"[pytubefix] Done: {output_path} ({file_size / 1024 / 1024:.1f} MB)")
218
+ return output_path
219
+ except Exception as exc:
220
+ print(f"[pytubefix] {client_name} failed: {exc}")
221
+ continue
222
+
223
+ # All simple clients failed — generate PO token and use WEB client
224
+ print("[pytubefix] All simple clients blocked. Generating PO token...")
225
+ visitor_data, po_token = _generate_po_token()
226
+
227
+ def _po_verifier():
228
+ return visitor_data, po_token
229
+
230
+ yt = YouTube(url, client="WEB", po_token_verifier=_po_verifier)
231
  print(f"[pytubefix] Title: {yt.title} ({yt.length}s)")
232
 
 
233
  stream = (
234
  yt.streams
235
  .filter(progressive=True, file_extension="mp4")
236
  .order_by("resolution")
237
  .desc()
238
  .first()
239
+ ) or yt.streams.filter(progressive=True).first()
240
+
 
241
  if not stream:
242
  raise RuntimeError("No downloadable stream found for this video.")
243
 
244
  print(f"[pytubefix] Stream: {stream.resolution} {stream.mime_type}")
 
245
  out_dir = os.path.dirname(output_path) or "."
246
  out_name = os.path.basename(output_path)
247
  stream.download(output_path=out_dir, filename=out_name)