Spaces:
Running
Running
File size: 5,299 Bytes
7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 b617fcc 7ff6662 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """
Job deduplication store — prevents the same job appearing twice across runs.
Uses SQLite so history persists across sessions.
Dedup strategy (two layers):
1. URL-primary: exact URL match within last N days (original behavior, unchanged)
2. Content fingerprint: SHA-256 of normalized (title+company) for cross-platform dedup.
Catches cases where the same job is posted on LinkedIn AND Greenhouse (different URLs,
same title+company). Added for ever-jobs integration (R2: dedup at 160+ platform scale).
"""
import sqlite3
import hashlib
from pathlib import Path
from datetime import datetime, timedelta
DB_PATH = Path("data/job_history.db")
def _conn() -> sqlite3.Connection:
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(DB_PATH))
conn.execute("""
CREATE TABLE IF NOT EXISTS seen_jobs (
url TEXT PRIMARY KEY,
title TEXT,
company TEXT,
platform TEXT,
first_seen TEXT,
last_seen TEXT,
run_count INTEGER DEFAULT 1,
content_fp TEXT
)
""")
conn.commit()
try:
conn.execute("ALTER TABLE seen_jobs ADD COLUMN content_fp TEXT")
conn.commit()
except Exception:
pass
return conn
def _canonical_fingerprint(title: str, company: str) -> str:
key = f"{title.lower().strip()}|{company.lower().strip()}"
return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
def is_duplicate(url: str, days: int = 30) -> bool:
if not url:
return False
cutoff = (datetime.now() - timedelta(days=days)).isoformat()
with _conn() as conn:
row = conn.execute(
"SELECT last_seen FROM seen_jobs WHERE url = ? AND last_seen > ?",
(url, cutoff)
).fetchone()
return row is not None
def is_duplicate_by_content(title: str, company: str, days: int = 30) -> bool:
if not title or not company:
return False
fp = _canonical_fingerprint(title, company)
cutoff = (datetime.now() - timedelta(days=days)).isoformat()
with _conn() as conn:
row = conn.execute(
"SELECT last_seen FROM seen_jobs WHERE content_fp = ? AND last_seen > ?",
(fp, cutoff)
).fetchone()
return row is not None
def mark_seen(url: str, title: str = "", company: str = "", platform: str = ""):
if not url:
return
now = datetime.now().isoformat()
fp = _canonical_fingerprint(title, company) if (title and company) else None
with _conn() as conn:
existing = conn.execute(
"SELECT run_count FROM seen_jobs WHERE url = ?", (url,)
).fetchone()
if existing:
conn.execute(
"UPDATE seen_jobs SET last_seen=?, run_count=run_count+1, content_fp=? WHERE url=?",
(now, fp, url)
)
else:
conn.execute(
"INSERT INTO seen_jobs (url, title, company, platform, first_seen, last_seen, content_fp) "
"VALUES (?, ?, ?, ?, ?, ?, ?)",
(url, title, company, platform, now, now, fp)
)
conn.commit()
def bulk_mark_seen(jobs: list):
if not jobs:
return
now = datetime.now().isoformat()
with _conn() as conn:
for job in jobs:
if isinstance(job, dict):
url = job.get("url", "")
title = job.get("title", "")
company = job.get("company", "")
platform = job.get("platform", "")
else:
url = getattr(job, "url", "")
title = getattr(job, "title", "")
company = getattr(job, "company", "")
platform = getattr(job, "platform", "")
if not url:
continue
fp = _canonical_fingerprint(title, company) if (title and company) else None
existing = conn.execute(
"SELECT run_count FROM seen_jobs WHERE url = ?", (url,)
).fetchone()
if existing:
conn.execute(
"UPDATE seen_jobs SET last_seen=?, run_count=run_count+1, content_fp=? WHERE url=?",
(now, fp, url)
)
else:
conn.execute(
"INSERT INTO seen_jobs (url, title, company, platform, first_seen, last_seen, content_fp) "
"VALUES (?, ?, ?, ?, ?, ?, ?)",
(url, title, company, platform, now, now, fp)
)
conn.commit()
def get_stats() -> dict:
with _conn() as conn:
total = conn.execute("SELECT COUNT(*) FROM seen_jobs").fetchone()[0]
recent = conn.execute(
"SELECT COUNT(*) FROM seen_jobs WHERE last_seen > ?",
((datetime.now() - timedelta(days=7)).isoformat(),)
).fetchone()[0]
return {"total_seen": total, "seen_last_7_days": recent, "db_path": str(DB_PATH)}
def clear_old_entries(days: int = 60):
cutoff = (datetime.now() - timedelta(days=days)).isoformat()
with _conn() as conn:
deleted = conn.execute(
"DELETE FROM seen_jobs WHERE last_seen < ?", (cutoff,)
).fetchone()
conn.commit()
|