Spaces:
Running
Running
| import re | |
| import time | |
| from bs4 import BeautifulSoup | |
| from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout | |
| from .base import BaseScraper, Job | |
| class IndeedScraper(BaseScraper): | |
| BASE_URL = "https://in.indeed.com/jobs" | |
| JOB_URL = "https://in.indeed.com/viewjob?jk={jk}" | |
| def __init__(self): | |
| super().__init__("Indeed") | |
| def search(self, role: str, location: str, max_results: int = 25) -> list[Job]: | |
| jobs = [] | |
| loc_map = { | |
| "India": "India", | |
| "Bangalore": "Bengaluru, Karnataka", | |
| "Mumbai": "Mumbai, Maharashtra", | |
| "Delhi NCR": "New Delhi, Delhi", | |
| "Hyderabad": "Hyderabad, Telangana", | |
| "Pune": "Pune, Maharashtra", | |
| "Remote": "Remote", | |
| } | |
| loc = loc_map.get(location, location) | |
| try: | |
| with sync_playwright() as pw: | |
| browser = pw.chromium.launch( | |
| headless=True, | |
| args=["--disable-blink-features=AutomationControlled", "--no-sandbox"], | |
| ) | |
| ctx = browser.new_context( | |
| user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", | |
| viewport={"width": 1366, "height": 768}, | |
| locale="en-IN", | |
| ) | |
| page = ctx.new_page() | |
| page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") | |
| url = f"{self.BASE_URL}?q={role}&l={loc}&fromage=7&sort=date" | |
| try: | |
| page.goto(url, wait_until="networkidle", timeout=30000) | |
| except PWTimeout: | |
| page.goto(url, wait_until="domcontentloaded", timeout=20000) | |
| page.wait_for_timeout(3000) | |
| html = page.content() | |
| soup = BeautifulSoup(html, "lxml") | |
| jobs = self._parse_jobs(soup, max_results) | |
| browser.close() | |
| except Exception: | |
| pass | |
| return jobs[:max_results] | |
| def _parse_jobs(self, soup, max_results: int) -> list[Job]: | |
| jobs = [] | |
| cards = soup.find_all("div", class_="job_seen_beacon") | |
| for card in cards[:max_results]: | |
| try: | |
| link = card.find("a", attrs={"data-jk": True}) | |
| if not link: | |
| continue | |
| jk = link.get("data-jk", "") | |
| aria = link.get("aria-label", "") | |
| title = re.sub(r"^full details of\s*", "", aria, flags=re.I).strip() | |
| if not title: | |
| title = link.get_text(strip=True) | |
| comp_el = ( | |
| card.find("span", attrs={"data-testid": "company-name"}) | |
| or card.find("span", class_=re.compile(r"companyName")) | |
| ) | |
| loc_el = ( | |
| card.find("div", attrs={"data-testid": "text-location"}) | |
| or card.find("div", class_=re.compile(r"companyLocation")) | |
| ) | |
| sal_el = card.find("div", class_=re.compile(r"estimated-salary|salary-snippet")) | |
| jobs.append(Job( | |
| title=title, | |
| company=comp_el.get_text(strip=True) if comp_el else "Unknown", | |
| location=loc_el.get_text(strip=True) if loc_el else "India", | |
| url=self.JOB_URL.format(jk=jk), | |
| platform="Indeed", | |
| salary=sal_el.get_text(strip=True) if sal_el else "Not specified", | |
| job_id=jk, | |
| )) | |
| except Exception: | |
| continue | |
| return jobs | |
| def get_details_bulk(self, jobs: list[Job], progress_cb=None) -> list[Job]: | |
| """Fetch descriptions for ALL jobs using ONE browser session (fast).""" | |
| need = [j for j in jobs if j.job_id and not j.description] | |
| if not need: | |
| return jobs | |
| try: | |
| with sync_playwright() as pw: | |
| browser = pw.chromium.launch( | |
| headless=True, | |
| args=["--disable-blink-features=AutomationControlled", "--no-sandbox"], | |
| ) | |
| ctx = browser.new_context( | |
| user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0.0.0 Safari/537.36", | |
| ) | |
| page = ctx.new_page() | |
| page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") | |
| for i, job in enumerate(need): | |
| try: | |
| page.goto(self.JOB_URL.format(jk=job.job_id), | |
| wait_until="domcontentloaded", timeout=15000) | |
| page.wait_for_timeout(1500) | |
| soup = BeautifulSoup(page.content(), "lxml") | |
| desc_el = ( | |
| soup.find("div", id="jobDescriptionText") | |
| or soup.find("div", class_=re.compile(r"jobDescription|job-description")) | |
| ) | |
| if desc_el: | |
| job.description = desc_el.get_text(separator="\n", strip=True)[:3000] | |
| except Exception: | |
| pass | |
| if progress_cb: | |
| try: | |
| progress_cb(i + 1, len(need)) | |
| except Exception: | |
| pass | |
| browser.close() | |
| except Exception: | |
| pass | |
| return jobs | |
| def get_job_details(self, job: Job) -> Job: | |
| if not job.job_id or job.description: | |
| return job | |
| try: | |
| with sync_playwright() as pw: | |
| browser = pw.chromium.launch( | |
| headless=True, | |
| args=["--disable-blink-features=AutomationControlled", "--no-sandbox"], | |
| ) | |
| ctx = browser.new_context( | |
| user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0.0.0 Safari/537.36", | |
| ) | |
| page = ctx.new_page() | |
| page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") | |
| page.goto(self.JOB_URL.format(jk=job.job_id), wait_until="networkidle", timeout=20000) | |
| html = page.content() | |
| soup = BeautifulSoup(html, "lxml") | |
| desc_el = ( | |
| soup.find("div", id="jobDescriptionText") | |
| or soup.find("div", class_=re.compile(r"jobDescription|job-description")) | |
| ) | |
| if desc_el: | |
| job.description = desc_el.get_text(separator="\n", strip=True)[:3000] | |
| browser.close() | |
| except Exception: | |
| pass | |
| self._sleep(1, 2) | |
| return job | |