Upload folder using huggingface_hub

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/WhatsNew-Scraper.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/misc.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_21" default="true" project-jdk-name="jbr-21" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+  <component name="StoredSessionsInfoStore">
+    <option name="sessions">
+      <list>
+        <SessionInfoState>
+          <option name="id" value="20260805-000758-9a7987ea-2159-4416-abc2-15f38fb7cc6d" />
+          <option name="lastUpdateTimestamp" value="1785877678080" />
+          <option name="responseMode" value="Agent" />
+          <option name="title" value="New Agent Session" />
+          <option name="user" value="abdo1234567es@gmail.com" />
+        </SessionInfoState>
+      </list>
+    </option>
+  </component>
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/WhatsNew-Scraper.iml" filepath="$PROJECT_DIR$/.idea/WhatsNew-Scraper.iml" />
+    </modules>
+  </component>
+</project>

.idea/studiobot.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="StudioBotProjectSettings">
+    <option name="shareContext" value="OptedIn" />
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="NONE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="66ee3a4c-7587-42f2-b266-bea278ef11ef" name="Changes" comment="" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="ClangdSettings">
+    <option name="formatViaClangd" value="false" />
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="ProjectColorInfo"><![CDATA[{
+  "associatedIndex": 2,
+  "fromUser": false
+}]]></component>
+  <component name="ProjectId" id="3HdoQC2fewtmSXjlJPtirHTWhuF" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "ModuleVcsDetector.initialDetectionPerformed": "true",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "RunOnceActivity.cidr.known.project.marker": "true",
+    "RunOnceActivity.git.unshallow": "true",
+    "RunOnceActivity.readMode.enableVisualFormatting": "true",
+    "cf.first.check.clang-format": "false",
+    "cidr.known.project.marker": "true",
+    "dart.analysis.tool.window.visible": "false",
+    "git-widget-placeholder": "main",
+    "kotlin-language-version-configured": "true",
+    "last_opened_file_path": "C:/Users/Abdo/AndroidStudioProjects/WhatsNew-Scraper",
+    "settings.editor.selected.configurable": "preferences.lookFeel",
+    "show.migrate.to.gradle.popup": "false"
+  }
+}]]></component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="66ee3a4c-7587-42f2-b266-bea278ef11ef" name="Changes" comment="" />
+      <created>1786207287629</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1786207287629</updated>
+    </task>
+    <servers />
+  </component>
+</project>

app.py

@@ -0,0 +1,269 @@
+"""
+WhatsNew News Intelligence Engine — Hugging Face Gradio Web Space.
+
+Paste a news article URL (Arabic or English) or paste article text directly
+to perform real-time NLP enrichment: Category, Sentiment, NER, Keywords,
+Geocoding, and Metadata.
+"""
+
+import os
+import re
+import json
+import logging
+import requests
+from bs4 import BeautifulSoup
+import gradio as gr
+
+from domain.entities.article import Article
+from services.local_nlp_classifier import LocalNLPClassifier
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize local NLP classifier engine
+nlp_engine = LocalNLPClassifier()
+
+
+def scrape_article_from_url(url: str):
+    """Scrape article title, summary, and image from a news URL."""
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    }
+    try:
+        response = requests.get(url.strip(), headers=headers, timeout=8)
+        if response.status_code != 200:
+            return None, None, None, f"Failed to fetch URL (Status {response.status_code})"
+
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        # 1. Title Extraction
+        title = None
+        og_title = soup.find('meta', property='og:title')
+        if og_title and og_title.get('content'):
+            title = og_title['content'].strip()
+        elif soup.title and soup.title.string:
+            title = soup.title.string.strip()
+        elif soup.find('h1'):
+            title = soup.find('h1').get_text().strip()
+
+        # 2. Image Extraction
+        image_url = None
+        og_image = soup.find('meta', property='og:image') or soup.find('meta', attrs={'name': 'twitter:image'})
+        if og_image and og_image.get('content'):
+            image_url = og_image['content'].strip()
+
+        # 3. Summary / Body Extraction
+        summary = None
+        og_desc = soup.find('meta', property='og:description') or soup.find('meta', attrs={'name': 'description'})
+        if og_desc and og_desc.get('content'):
+            summary = og_desc['content'].strip()
+        else:
+            paragraphs = [p.get_text().strip() for p in soup.find_all('p') if len(p.get_text().strip()) > 30]
+            summary = ' '.join(paragraphs[:3]) if paragraphs else ""
+
+        return title, summary, image_url, None
+    except Exception as e:
+        return None, None, None, f"Error scraping URL: {str(e)}"
+
+
+def process_nlp_analysis(url_input: str, title_input: str, summary_input: str):
+    """Main Gradio processing handler."""
+    url = (url_input or "").strip()
+    title = (title_input or "").strip()
+    summary = (summary_input or "").strip()
+    image_url = None
+    status_msg = ""
+
+    # If URL provided, attempt web scraping first
+    if url:
+        scraped_title, scraped_summary, scraped_img, err = scrape_article_from_url(url)
+        if err:
+            status_msg = f"⚠️ {err}. Using manual text inputs."
+        else:
+            title = scraped_title or title
+            summary = scraped_summary or summary
+            image_url = scraped_img
+            status_msg = "✅ Successfully scraped article from live URL!"
+
+    if not title:
+        return (
+            "⚠️ Please enter an article URL or provide a Title & Summary.",
+            "", "", "", "", "", "", "", "", "", {}
+        )
+
+    # Detect language
+    is_arabic = any('\u0600' <= char <= '\u06FF' for char in f"{title} {summary}")
+    language = "ar" if is_arabic else "en"
+
+    # Construct Article entity
+    article = Article(
+        title=title,
+        summary=summary or title,
+        url=url or "https://whatsnew.ai/demo",
+        source="Gradio Interactive UI",
+        publishedAt=None,
+        language=language,
+        imageUrl=image_url
+    )
+
+    # Perform NLP enrichment
+    enriched = nlp_engine.enrich_article(article)
+
+    # Formatted Visual Displays
+    category_display = f"🏷️ **Category**: {enriched.category or 'General'}\n📌 **Subcategory**: {enriched.subcategory or 'N/A'}"
+    
+    # Sentiment Badge
+    sent = (enriched.sentiment or "Neutral").title()
+    sent_icon = "🟢" if sent == "Positive" else "🔴" if sent == "Negative" else "⚪"
+    sentiment_display = f"{sent_icon} **Sentiment**: {sent}"
+
+    # Named Entities
+    people_str = ", ".join(enriched.people) if enriched.people else "None detected"
+    orgs_str = ", ".join(enriched.organizations) if enriched.organizations else "None detected"
+    locs_str = ", ".join(enriched.locations) if enriched.locations else "None detected"
+    ner_display = f"👤 **People**: {people_str}\n🏢 **Organizations**: {orgs_str}\n📍 **Locations**: {locs_str}"
+
+    # Geocoding
+    geo_display = f"🌍 **Region**: {enriched.region or 'Global'}\n🏳️ **Country**: {enriched.country or 'Global'}"
+
+    # Tags & Keywords
+    tags_display = " ".join([f"#{t.replace(' ', '_')}" for t in (enriched.tags or [])])
+    keywords_display = ", ".join(enriched.keywords or [])
+
+    # Stats
+    stats_display = f"⭐ **Importance Score**: {enriched.importance or 5}/10\n⏱️ **Reading Time**: {enriched.readingTime or 1} min\n🌐 **Language**: {'Arabic 🇦🇪/🇪🇬' if language == 'ar' else 'English 🇬🇧/🇺🇸'}"
+
+    # Full JSON Metadata dictionary
+    json_output = {
+        "title": enriched.title,
+        "summary": enriched.summary,
+        "url": enriched.url,
+        "imageUrl": enriched.imageUrl,
+        "language": enriched.language,
+        "category": enriched.category,
+        "subcategory": enriched.subcategory,
+        "sentiment": enriched.sentiment,
+        "region": enriched.region,
+        "country": enriched.country,
+        "people": enriched.people,
+        "organizations": enriched.organizations,
+        "locations": enriched.locations,
+        "tags": enriched.tags,
+        "keywords": enriched.keywords,
+        "importance": enriched.importance,
+        "readingTime": enriched.readingTime
+    }
+
+    return (
+        status_msg or "✅ Analysis Complete",
+        title,
+        summary,
+        category_display,
+        sentiment_display,
+        ner_display,
+        geo_display,
+        tags_display,
+        keywords_display,
+        stats_display,
+        json_output,
+        image_url or "https://images.unsplash.com/photo-1504711434969-e33886168f5c?q=80&w=1000&auto=format&fit=crop"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Custom CSS Aesthetics
+# ---------------------------------------------------------------------------
+custom_css = """
+.main-title { text-align: center; font-size: 2.2em; font-weight: bold; margin-bottom: 5px; color: #1E293B; }
+.subtitle { text-align: center; font-size: 1.1em; color: #64748B; margin-bottom: 25px; }
+.output-card { border-radius: 12px; padding: 15px; background: #F8FAFC; border: 1px solid #E2E8F0; }
+"""
+
+# ---------------------------------------------------------------------------
+# Gradio UI Layout
+# ---------------------------------------------------------------------------
+with gr.Blocks(title="WhatsNew NLP Engine", css=custom_css, theme=gr.themes.Soft()) as demo:
+    gr.Markdown("<div class='main-title'>📰 WhatsNew Intelligence Engine</div>")
+    gr.Markdown("<div class='subtitle'>Real-Time Multilingual NLP Article Categorization, Sentiment Analysis, NER & Metadata Extraction</div>")
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("### 📥 Article Input")
+            url_input = gr.Textbox(
+                label="Option A: Paste News Article URL (Arabic or English)",
+                placeholder="https://www.aljazeera.net/... or https://techcrunch.com/...",
+                lines=1
+            )
+            gr.Markdown("**OR Enter Manually:**")
+            title_input = gr.Textbox(
+                label="Article Title",
+                placeholder="e.g. الأهلي يفوز بلقب دوري أبطال أفريقيا",
+                lines=2
+            )
+            summary_input = gr.Textbox(
+                label="Article Summary / Text",
+                placeholder="Paste news paragraph or text summary here...",
+                lines=4
+            )
+            analyze_btn = gr.Button("🚀 Analyze & Enrich Article", variant="primary", size="lg")
+            
+            gr.Markdown("---")
+            gr.Markdown("### 💡 Quick Examples")
+            gr.Examples(
+                examples=[
+                    ["", "الأهلي يتوج بكأس السوبر المصري بعد فوزه على الزمالك في ركلات الترجيح", "نجح الفريق الأول لكرة القدم بنادي الأهلي في تحقيق حسم بطولة كأس السوبر المصري بعد مباراة حماسية انتهت بفوزه على غريمه التقليدي الزمالك."],
+                    ["", "Apple unveils new M4 MacBook Pro with advanced AI capabilities", "Apple today announced its next-generation MacBook Pro powered by the M4 chip lineup, featuring enhanced neural processing for on-device artificial intelligence."],
+                    ["", "ارتفاع أسعار النفط العالمية بسبب التوترات الجيوسياسية في الشرق الأوسط", "شهدت أسواق النفط العالمية ارتفاعاً ملحوظاً في أسعار الخام وسط متابعة حثيثة من المستثمرين لتطورات الأوضاع الاقتصادية."]
+                ],
+                inputs=[url_input, title_input, summary_input]
+            )
+
+        with gr.Column(scale=1):
+            status_box = gr.Markdown("Waiting for input...")
+            
+            with gr.Tabs():
+                with gr.TabItem("📊 NLP Insights"):
+                    with gr.Row():
+                        category_box = gr.Markdown(elem_classes=["output-card"])
+                        sentiment_box = gr.Markdown(elem_classes=["output-card"])
+                    
+                    ner_box = gr.Markdown(elem_classes=["output-card"])
+                    geo_box = gr.Markdown(elem_classes=["output-card"])
+                    stats_box = gr.Markdown(elem_classes=["output-card"])
+
+                    gr.Markdown("#### 🏷️ Extracted Hashtags & Tags")
+                    tags_box = gr.Markdown()
+                    
+                    gr.Markdown("#### 🔑 Key Phrases")
+                    keywords_box = gr.Markdown()
+
+                with gr.TabItem("🖼️ Media & Preview"):
+                    image_preview = gr.Image(label="Article Cover Image", type="url")
+                    extracted_title = gr.Textbox(label="Processed Title", interactive=False)
+                    extracted_summary = gr.Textbox(label="Processed Summary", interactive=False, lines=4)
+
+                with gr.TabItem("📄 Raw JSON Metadata"):
+                    json_box = gr.JSON(label="Full Firestore-Ready Document Schema")
+
+    # Connect click event
+    analyze_btn.click(
+        fn=process_nlp_analysis,
+        inputs=[url_input, title_input, summary_input],
+        outputs=[
+            status_box,
+            extracted_title,
+            extracted_summary,
+            category_box,
+            sentiment_box,
+            ner_box,
+            geo_box,
+            tags_box,
+            keywords_box,
+            stats_box,
+            json_box,
+            image_preview
+        ]
+    )
+
+if __name__ == "__main__":
+    demo.launch()

data/repositories/article_repository_impl.py

@@ -0,0 +1,58 @@
+import firebase_admin
+from firebase_admin import credentials, firestore
+from typing import List
+import logging
+from domain.entities.article import Article
+from domain.repositories.article_repository import ArticleRepository
+
+logger = logging.getLogger(__name__)
+
+class FirestoreArticleRepository(ArticleRepository):
+    def __init__(self, service_account_path: str = "service-account.json"):
+        try:
+            if not firebase_admin._apps:
+                cred = credentials.Certificate(service_account_path)
+                firebase_admin.initialize_app(cred)
+            self.db = firestore.client()
+        except Exception as e:
+            logger.error(f"Failed to initialize Firebase Admin: {e}")
+            self.db = None
+    
+    def save_articles(self, articles: List[Article]) -> List[Article]:
+        if not self.db:
+            logger.error("Firestore DB not initialized. Cannot save articles.")
+            return []
+
+        articles_ref = self.db.collection('articles')
+        new_articles = []
+        for article in articles:
+            try:
+                # Use the generated deterministic ID as the document ID
+                doc_id = article.id
+                doc_ref = articles_ref.document(doc_id)
+                
+                # Check if document exists
+                doc_snap = doc_ref.get()
+                if not doc_snap.exists:
+                    doc_ref.set(article.model_dump(mode='json'))
+                    new_articles.append(article)
+                else:
+                    # Backfill/Update image if existing document was saved without an image
+                    existing_data = doc_snap.to_dict() or {}
+                    if not existing_data.get('imageUrl') and article.imageUrl:
+                        logger.info(f"Backfilling missing image for existing article: {article.title[:40]}")
+                        doc_ref.update({
+                            "imageUrl": article.imageUrl,
+                            "hasImage": True
+                        })
+            except Exception as e:
+                logger.error(f"Failed to save article {article.title}: {e}")
+
+        logger.info(f"Saved {len(new_articles)} new articles to Firestore.")
+        return new_articles
+
+    def exists(self, article_id: str) -> bool:
+        if not self.db:
+            return False
+        doc_ref = self.db.collection('articles').document(article_id)
+        return doc_ref.get().exists

domain/entities/article.py

@@ -0,0 +1,74 @@
+from pydantic import BaseModel, HttpUrl, Field, field_validator
+from typing import Optional, List
+import datetime
+import uuid
+import hashlib
+
+class Article(BaseModel):
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    title: str
+    summary: str
+    url: str
+    imageUrl: Optional[str] = None
+    source: str
+    publishedAt: datetime.datetime
+    language: str = "ar"
+
+    # Enriched Fields
+    category: Optional[str] = None
+    subcategory: Optional[str] = None
+    region: Optional[str] = None
+    country: Optional[str] = None
+    city: Optional[str] = None
+    
+    tags: List[str] = Field(default_factory=list)
+    keywords: List[str] = Field(default_factory=list)
+    people: List[str] = Field(default_factory=list)
+    organizations: List[str] = Field(default_factory=list)
+    locations: List[str] = Field(default_factory=list)
+    
+    sentiment: Optional[str] = None
+    importance: Optional[int] = None
+    readingTime: Optional[int] = None
+    hasImage: bool = False
+
+    @classmethod
+    def normalize_url(cls, url: str) -> str:
+        """Strips http/https differences, www prefix, and tracking parameters to ensure 100% identical URL hashing."""
+        from urllib.parse import urlparse, parse_qs, urlunparse, urlencode
+        # Force https and strip www.
+        url_clean_scheme = url.replace('http://', 'https://')
+        parsed = urlparse(url_clean_scheme)
+        netloc = parsed.netloc.lower()
+        if netloc.startswith('www.'):
+            netloc = netloc[4:]
+        
+        # Strip tracking query params
+        query_params = parse_qs(parsed.query, keep_blank_values=True)
+        filtered_params = {k: v for k, v in query_params.items() if not k.startswith('utm_') and k not in ['ref', 'fbclid', 'rss', 'src', 'amp']}
+        clean_query = urlencode(filtered_params, doseq=True)
+        # Reconstruct URL without trailing slash and tracking params
+        clean_url = urlunparse((parsed.scheme, netloc, parsed.path.rstrip('/'), parsed.params, clean_query, ''))
+        return clean_url
+
+    @classmethod
+    def generate_id_from_url(cls, url: str) -> str:
+        """Generates a deterministic ID from the normalized URL to prevent duplicates."""
+        clean_url = cls.normalize_url(url)
+        return hashlib.md5(clean_url.encode('utf-8')).hexdigest()
+
+    @classmethod
+    def generate_title_hash(cls, title: str) -> str:
+        """Generates a normalized hash of the title to detect cross-source headline duplicates."""
+        import re
+        clean_title = re.sub(r'[^\w\s]', '', title.lower()).strip()
+        return hashlib.md5(clean_title.encode('utf-8')).hexdigest()
+
+    @field_validator('readingTime', 'importance', mode='before')
+    def convert_float_to_int(cls, v):
+        if v is not None:
+            try:
+                return max(1, round(float(v)))
+            except (ValueError, TypeError):
+                return v
+        return v

domain/interfaces.py

@@ -0,0 +1,10 @@
+from typing import List, Protocol
+from domain.entities.article import Article
+
+class ScraperInterface(Protocol):
+    source_name: str
+    language: str
+    
+    def scrape(self) -> List[Article]:
+        """Scrapes the target website and returns a list of Articles."""
+        ...

domain/repositories/article_repository.py

@@ -0,0 +1,14 @@
+from abc import ABC, abstractmethod
+from typing import List
+from domain.entities.article import Article
+
+class ArticleRepository(ABC):
+    @abstractmethod
+    def save_articles(self, articles: List[Article]) -> List[Article]:
+        """Saves a list of articles and returns the list of newly saved articles."""
+        pass
+        
+    @abstractmethod
+    def exists(self, article_id: str) -> bool:
+        """Checks if an article already exists by its ID."""
+        pass

domain/usecases/scrape_and_enrich.py

@@ -0,0 +1,69 @@
+import logging
+from typing import List
+from domain.entities.article import Article
+from domain.repositories.article_repository import ArticleRepository
+from domain.interfaces import ScraperInterface
+from services.ai_classifier import HuggingFaceClassifier
+from services.duplicate_detector import DuplicateDetector
+
+from services.lexicons import CATEGORY_IMAGES_MAP
+
+logger = logging.getLogger(__name__)
+
+class ScrapeAndEnrichUseCase:
+    def __init__(self, 
+                 repository: ArticleRepository, 
+                 scrapers: List[ScraperInterface],
+                 classifier: HuggingFaceClassifier,
+                 duplicate_detector: DuplicateDetector):
+        self.repository = repository
+        self.scrapers = scrapers
+        self.classifier = classifier
+        self.duplicate_detector = duplicate_detector
+
+    def execute(self) -> List[Article]:
+        all_articles = []
+        
+        # 1. Scrape raw articles from all sources
+        for scraper in self.scrapers:
+            logger.info(f"Scraping raw feed from {scraper.source_name} ({scraper.language})...")
+            try:
+                articles = scraper.scrape()
+                logger.info(f"Found {len(articles)} raw articles from {scraper.source_name}")
+                all_articles.extend(articles)
+            except Exception as e:
+                logger.error(f"Failed to scrape {scraper.source_name}: {e}")
+                
+        if not all_articles:
+            logger.info("No articles found across all scrapers.")
+            return []
+
+        # 2. Filter duplicates
+        new_articles = self.duplicate_detector.filter_new_articles(all_articles)
+        if not new_articles:
+            logger.info("No new articles to enrich.")
+            return []
+
+        # 3. Enrich articles via AI pipeline & apply Layer 3 Image Fallback
+        enriched_articles = []
+        logger.info(f"Starting AI enrichment for {len(new_articles)} new articles...")
+        
+        for index, article in enumerate(new_articles):
+            logger.info(f"Enriching article {index + 1}/{len(new_articles)}: {article.title[:50]}...")
+            enriched = self.classifier.enrich_article(article)
+            
+            # Layer 3: Ensure imageUrl is NEVER null
+            if not enriched.imageUrl:
+                fallback_img = CATEGORY_IMAGES_MAP.get(enriched.category, CATEGORY_IMAGES_MAP["General"])
+                enriched.imageUrl = fallback_img
+                enriched.hasImage = True
+            else:
+                enriched.hasImage = True
+                
+            enriched_articles.append(enriched)
+            
+        # 4. Save to Repository
+        logger.info(f"Saving {len(enriched_articles)} enriched articles to database...")
+        new_saved_articles = self.repository.save_articles(enriched_articles)
+        
+        return new_saved_articles

main.py

@@ -0,0 +1,133 @@
+import logging
+from scrapers.rss_scraper import RssScraper
+from data.repositories.article_repository_impl import FirestoreArticleRepository
+from services.ai_classifier import HuggingFaceClassifier
+from services.duplicate_detector import DuplicateDetector
+from domain.usecases.scrape_and_enrich import ScrapeAndEnrichUseCase
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Configure our RSS sources across diverse categories (General, Tech, Gaming, Food, Sports, Business, Science, Entertainment)
+SOURCES = [
+    # --- World & Middle East News (English) ---
+    RssScraper("BBC Middle East", "en", "http://feeds.bbci.co.uk/news/world/middle_east/rss.xml", category="Middle East"),
+    RssScraper("BBC World", "en", "http://feeds.bbci.co.uk/news/world/rss.xml", category="World"),
+    RssScraper("Middle East Eye", "en", "https://www.middleeasteye.net/rss", category="Middle East"),
+    RssScraper("Middle East Monitor", "en", "https://www.middleeastmonitor.com/feed/", category="Middle East"),
+    RssScraper("The Guardian Middle East", "en", "https://www.theguardian.com/world/middleeast/rss", category="Middle East"),
+    RssScraper("NY Times Middle East", "en", "https://rss.nytimes.com/services/xml/rss/nyt/MiddleEast.xml", category="Middle East"),
+    RssScraper("Al Jazeera English", "en", "https://www.aljazeera.com/xml/rss/all.xml", category="World"),
+
+    # --- Egyptian & Arab World News (Arabic) ---
+    RssScraper("Youm7", "ar", "http://www.youm7.com/rss/SectionRss?SectionID=65", category="Egypt"),
+    RssScraper("Al-Ahram", "ar", "https://gate.ahram.org.eg/NewsRss.aspx", category="Egypt"),
+    RssScraper("Al Masry Al Youm", "ar", "https://www.almasryalyoum.com/rss/rss", category="Egypt"),
+    RssScraper("Sada El Balad", "ar", "https://www.elbalad.news/rss.aspx", category="Egypt"),
+    RssScraper("Al Shorouk", "ar", "https://www.shorouknews.com/rss/news.aspx", category="Egypt"),
+    RssScraper("Al-Dustour", "ar", "https://www.dostor.org/rss.aspx", category="Egypt"),
+    RssScraper("Sky News Arabia", "ar", "https://www.skynewsarabia.com/rss", category="Middle East"),
+    RssScraper("RT Arabic", "ar", "https://arabic.rt.com/rss/", category="Middle East"),
+    RssScraper("Asharq Al-Awsat", "ar", "https://aawsat.com/feed", category="Middle East"),
+    RssScraper("Alhurra", "ar", "https://www.alhurra.com/rss", category="Middle East"),
+    RssScraper("BBC Arabic", "ar", "http://feeds.bbci.co.uk/arabic/rss.xml", category="Middle East"),
+    RssScraper("CNN Arabic", "ar", "https://arabic.cnn.com/api/v1/rss/middle_east/rss.xml", category="Middle East"),
+    RssScraper("France 24 Arabic", "ar", "https://www.france24.com/ar/rss", category="Middle East"),
+    RssScraper("Hespress", "ar", "https://www.hespress.com/feed", category="North Africa"),
+    RssScraper("Echorouk", "ar", "https://www.echoroukonline.com/feed", category="North Africa"),
+
+    # --- Technology & AI ---
+    RssScraper("TechCrunch", "en", "https://techcrunch.com/feed/", category="Technology"),
+    RssScraper("Wired", "en", "https://www.wired.com/feed/rss", category="Technology"),
+    RssScraper("The Verge", "en", "https://www.theverge.com/rss/index.xml", category="Technology"),
+    RssScraper("Ars Technica", "en", "https://feeds.arstechnica.com/arstechnica/index", category="Technology"),
+    RssScraper("Engadget", "en", "https://www.engadget.com/rss.xml", category="Technology"),
+    RssScraper("Aitnews", "ar", "https://aitnews.com/feed/", category="Technology"),
+    RssScraper("Unlimit Tech", "ar", "https://unlimit-tech.com/feed/", category="Technology"),
+
+    # --- Gaming & Esports ---
+    RssScraper("IGN", "en", "https://feeds.feedburner.com/ign/all", category="Gaming"),
+    RssScraper("Eurogamer", "en", "https://www.eurogamer.net/?format=rss", category="Gaming"),
+    RssScraper("GameSpot", "en", "https://www.gamespot.com/feeds/news/", category="Gaming"),
+    RssScraper("Polygon", "en", "https://www.polygon.com/rss/index.xml", category="Gaming"),
+    RssScraper("Saudi Gamer", "ar", "https://saudigamer.com/feed/", category="Gaming"),
+    RssScraper("TrueGaming", "ar", "https://www.true-gaming.net/home/feed/", category="Gaming"),
+
+    # --- Food, Cooking & Lifestyle ---
+    RssScraper("Serious Eats", "en", "https://www.seriouseats.com/feed", category="Food"),
+    RssScraper("BBC Good Food", "en", "https://www.bbcgoodfood.com/feed/rss2", category="Food"),
+    RssScraper("Food52", "en", "https://food52.com/blog.rss", category="Food"),
+    RssScraper("SuperMama", "ar", "https://www.supermama.me/rss.xml", category="Lifestyle"),
+
+    # --- Business, Economy & Finance ---
+    RssScraper("CNBC World", "en", "https://www.cnbc.com/id/100727362/device/rss/rss.html", category="Business"),
+    RssScraper("Financial Times", "en", "https://www.ft.com/?format=rss", category="Business"),
+    RssScraper("RT Arabic Business", "ar", "https://arabic.rt.com/rss/business/", category="Business"),
+    RssScraper("Al Mal News", "ar", "https://almalnews.com/feed/", category="Business"),
+
+    # --- Sports ---
+    RssScraper("BBC Sport", "en", "http://feeds.bbci.co.uk/sport/rss.xml", category="Sports"),
+    RssScraper("RT Arabic Sports", "ar", "https://arabic.rt.com/rss/sport/", category="Sports"),
+    RssScraper("FilGoal", "ar", "https://www.filgoal.com/rss/news", category="Sports"),
+
+    # --- Science & Space ---
+    RssScraper("ScienceDaily", "en", "https://www.sciencedaily.com/rss/all.xml", category="Science"),
+    RssScraper("NASA News", "en", "https://www.nasa.gov/news-release/feed/", category="Science"),
+    RssScraper("RT Arabic Science", "ar", "https://arabic.rt.com/rss/space/", category="Science"),
+
+    # --- Entertainment & Culture ---
+    RssScraper("Variety", "en", "https://variety.com/feed/", category="Entertainment"),
+    RssScraper("Hollywood Reporter", "en", "https://www.hollywoodreporter.com/feed/", category="Entertainment")
+]
+
+def job():
+    logger.info("Starting scraping job with AI Enrichment...")
+    
+    # 1. Initialize dependencies
+    try:
+        repo = FirestoreArticleRepository(service_account_path="service-account.json")
+    except Exception as e:
+        logger.error(f"Failed to initialize Firestore: {e}")
+        return
+        
+    if not repo.db:
+        logger.error("Skipping job because Firestore is not initialized. Please ensure service-account.json exists.")
+        return
+
+    classifier = HuggingFaceClassifier()
+    duplicate_detector = DuplicateDetector(repository=repo)
+    
+    # 2. Setup Usecase
+    usecase = ScrapeAndEnrichUseCase(
+        repository=repo,
+        scrapers=SOURCES,
+        classifier=classifier,
+        duplicate_detector=duplicate_detector
+    )
+    
+    # 3. Execute Pipeline
+    new_articles = usecase.execute()
+    
+    # 4. Send Notifications
+    if new_articles:
+        try:
+            from firebase_admin import messaging
+            top_article = new_articles[0]
+            message = messaging.Message(
+                notification=messaging.Notification(
+                    title=f"New story from {top_article.source}",
+                    body=top_article.title
+                ),
+                topic='daily_news'
+            )
+            response = messaging.send(message)
+            logger.info(f"Successfully sent FCM notification: {response}")
+        except Exception as e:
+            logger.error(f"Failed to send FCM notification: {e}")
+    else:
+        logger.info("No new articles processed.")
+
+if __name__ == "__main__":
+    logger.info("News Intelligence Engine started.")
+    job()
+    logger.info("Scraping job finished.")

nlp_training/__init__.py

@@ -0,0 +1 @@
+# NLP Training Scripts for WhatsNew News Intelligence Engine

nlp_training/prepare_datasets.py

@@ -0,0 +1,398 @@
+"""
+Dataset Preparation Utility for WhatsNew NLP Training.
+
+Downloads and prepares Arabic + English datasets for fine-tuning:
+  - SANAD + HuffPost  → Category Classification
+  - ASAD + SST-2      → Sentiment Analysis
+  - WikiANN           → Named Entity Recognition
+
+Usage:
+    python nlp_training/prepare_datasets.py --task all
+    python nlp_training/prepare_datasets.py --task category
+    python nlp_training/prepare_datasets.py --task sentiment
+    python nlp_training/prepare_datasets.py --task ner
+"""
+
+import os
+import argparse
+import json
+import logging
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Our unified 13 categories (must match CATEGORIES_MAP keys)
+UNIFIED_CATEGORIES = [
+    "Sports", "Technology", "Gaming", "Business", "Health", "Science",
+    "Food", "Entertainment", "Politics", "Education", "Crime", "Weather", "Travel"
+]
+
+# -----------------------------------------------------------------------
+# Label mapping: Map source dataset labels → our unified category labels
+# -----------------------------------------------------------------------
+
+# SANAD Arabic dataset label → our category
+SANAD_LABEL_MAP = {
+    "culture": "Entertainment",
+    "finance": "Business",
+    "medical": "Health",
+    "politics": "Politics",
+    "religion": "Education",
+    "sports": "Sports",
+    "tech": "Technology",
+}
+
+# HuffPost English dataset label → our category
+HUFFPOST_LABEL_MAP = {
+    "POLITICS": "Politics",
+    "WELLNESS": "Health",
+    "ENTERTAINMENT": "Entertainment",
+    "TRAVEL": "Travel",
+    "STYLE & BEAUTY": "Entertainment",
+    "PARENTING": "Education",
+    "HEALTHY LIVING": "Health",
+    "QUEER VOICES": "Entertainment",
+    "FOOD & DRINK": "Food",
+    "BUSINESS": "Business",
+    "COMEDY": "Entertainment",
+    "SPORTS": "Sports",
+    "BLACK VOICES": "Politics",
+    "HOME & LIVING": "Food",
+    "PARENTS": "Education",
+    "THE WORLDPOST": "Politics",
+    "WEDDINGS": "Entertainment",
+    "WOMEN": "Politics",
+    "IMPACT": "Politics",
+    "DIVORCE": "Entertainment",
+    "CRIME": "Crime",
+    "MEDIA": "Entertainment",
+    "WEIRD NEWS": "Entertainment",
+    "GREEN": "Science",
+    "WORLDPOST": "Politics",
+    "RELIGION": "Education",
+    "SCIENCE": "Science",
+    "TECH": "Technology",
+    "TASTE": "Food",
+    "MONEY": "Business",
+    "ARTS": "Entertainment",
+    "FIFTY": "Entertainment",
+    "GOOD NEWS": "Entertainment",
+    "ARTS & CULTURE": "Entertainment",
+    "ENVIRONMENT": "Science",
+    "COLLEGE": "Education",
+    "LATINO VOICES": "Politics",
+    "CULTURE & ARTS": "Entertainment",
+    "EDUCATION": "Education",
+    "STYLE": "Entertainment",
+}
+
+SENTIMENT_LABELS = ["Positive", "Negative", "Neutral"]
+
+
+def prepare_category_datasets(output_dir: str):
+    """
+    Download and prepare news category classification datasets (Arabic + English).
+    Outputs: category_train.jsonl, category_test.jsonl with {text, label, language} pairs.
+    """
+    from datasets import load_dataset
+
+    os.makedirs(output_dir, exist_ok=True)
+    raw_dir = "nlp_training/raw_data"
+    os.makedirs(raw_dir, exist_ok=True)
+    all_samples = []
+
+    # --- Local Kaggle Raw Data Check ---
+    for filename in os.listdir(raw_dir):
+        filepath = os.path.join(raw_dir, filename)
+        if filename.endswith(".csv"):
+            try:
+                import pandas as pd
+                df = pd.read_csv(filepath)
+                text_col = next((c for c in df.columns if c.lower() in ["text", "article", "content", "headline"]), None)
+                label_col = next((c for c in df.columns if c.lower() in ["label", "category", "class", "topic"]), None)
+                if text_col and label_col:
+                    loaded = 0
+                    for _, row in df.iterrows():
+                        text = str(row[text_col])
+                        raw_label = str(row[label_col]).lower()
+                        mapped = SANAD_LABEL_MAP.get(raw_label) or HUFFPOST_LABEL_MAP.get(raw_label.upper())
+                        if mapped and text:
+                            lang = "ar" if any('\u0600' <= char <= '\u06FF' for char in text[:100]) else "en"
+                            all_samples.append({"text": text[:512], "label": mapped, "language": lang})
+                            loaded += 1
+                    logger.info(f"Loaded {loaded} samples from local CSV '{filename}'")
+            except Exception as e:
+                logger.warning(f"Failed parsing local file {filename}: {e}")
+
+    # --- English: AG News ---
+    logger.info("Loading AG News English dataset...")
+    ag_news_map = {0: "Politics", 1: "Sports", 2: "Business", 3: "Technology"}
+    try:
+        ag_ds = load_dataset("ag_news", split="train")
+        en_count = 0
+        for sample in ag_ds:
+            text = sample.get("text", "")
+            label_id = sample.get("label")
+            mapped = ag_news_map.get(label_id)
+            if mapped and text:
+                all_samples.append({"text": text[:512], "label": mapped, "language": "en"})
+                en_count += 1
+        logger.info(f"Loaded {en_count} English samples from AG News")
+    except Exception as e:
+        logger.warning(f"Could not load AG News: {e}")
+
+    # --- English: 20 Newsgroups ---
+    try:
+        hp_ds = load_dataset("SetFit/20_newsgroups", split="train")
+        newsgroups_map = {
+            "rec.sport.baseball": "Sports", "rec.sport.hockey": "Sports",
+            "comp.graphics": "Technology", "comp.sys.mac.hardware": "Technology",
+            "talk.politics.mideast": "Politics", "talk.politics.misc": "Politics",
+            "sci.space": "Science", "sci.med": "Health"
+        }
+        for sample in hp_ds:
+            text = sample.get("text", "")
+            label_text = sample.get("label_text", "")
+            mapped = newsgroups_map.get(label_text)
+            if mapped and text:
+                all_samples.append({"text": text[:512], "label": mapped, "language": "en"})
+    except Exception:
+        pass
+
+    # --- Arabic: Arabic News Datasets ---
+    logger.info("Loading Arabic news classification dataset...")
+    arabic_loaded = False
+    arabic_ds_candidates = ["arbml/Arabic_News_Tweets", "he3als/sanad", "ahedag/arabic_news"]
+    
+    for candidate in arabic_ds_candidates:
+        try:
+            ar_ds = load_dataset(candidate, split="train")
+            ar_count = 0
+            for sample in ar_ds:
+                text = sample.get("text") or sample.get("article") or sample.get("tweet") or ""
+                raw_label = str(sample.get("label") or sample.get("category") or "").lower()
+                mapped = SANAD_LABEL_MAP.get(raw_label)
+                if not mapped:
+                    if any(k in raw_label for k in ["رياض", "sport"]): mapped = "Sports"
+                    elif any(k in raw_label for k in ["تقن", "tech"]): mapped = "Technology"
+                    elif any(k in raw_label for k in ["سياس", "politic"]): mapped = "Politics"
+                    elif any(k in raw_label for k in ["اقتصاد", "مال", "econ"]): mapped = "Business"
+                    elif any(k in raw_label for k in ["صح", "health"]): mapped = "Health"
+                    elif any(k in raw_label for k in ["فن", "ثقاف", "entertain"]): mapped = "Entertainment"
+
+                if mapped and text:
+                    all_samples.append({"text": text[:512], "label": mapped, "language": "ar"})
+                    ar_count += 1
+            if ar_count > 0:
+                logger.info(f"Loaded {ar_count} Arabic news samples from '{candidate}'")
+                arabic_loaded = True
+                break
+        except Exception:
+            continue
+
+    if not arabic_loaded and len([s for s in all_samples if s["language"] == "ar"]) < 10:
+        logger.info("Using built-in Arabic news samples generator for fallback testing...")
+        sample_ar_news = [
+            ("الأهلي يفوز على الزمالك في قمة الدوري الممتاز", "Sports"),
+            ("إطلاق جديد لهاتف آيفون يدعم تقنيات الذكاء الاصطناعي", "Technology"),
+            ("ارتفاع أسعار النفط والأسهم في البورصة العالمية", "Business"),
+            ("علماء يكتشفون علاجاً جديداً يرفع المناعة ضد الفيروسات", "Health"),
+            ("مفاوضات سياسية جديدة في قمة مجلس الأمن الدولي", "Politics"),
+            ("سوني تعلن عن ألعاب جديدة لجهاز بلايستيشن", "Gaming"),
+            ("وكالة ناسا تطلق تلسكوب فضاء جديد لاستكشاف المريخ", "Science"),
+            ("عرض أول لفيلم سينمائي جديد يتصدر شباك التذاكر", "Entertainment"),
+        ]
+        for text, cat in sample_ar_news * 50:
+            all_samples.append({"text": text, "label": cat, "language": "ar"})
+
+    # --- Shuffle and split 90/10 ---
+    import random
+    random.shuffle(all_samples)
+    split_idx = int(len(all_samples) * 0.9)
+    train_data = all_samples[:split_idx]
+    test_data = all_samples[split_idx:]
+
+    _save_jsonl(train_data, os.path.join(output_dir, "category_train.jsonl"))
+    _save_jsonl(test_data, os.path.join(output_dir, "category_test.jsonl"))
+    logger.info(f"Category dataset saved: {len(train_data)} train, {len(test_data)} test")
+
+
+def prepare_sentiment_datasets(output_dir: str):
+    """
+    Download and prepare Sentiment Analysis datasets (Arabic + English).
+    Outputs: sentiment_train.jsonl, sentiment_test.jsonl with {text, label, language} pairs.
+    """
+    from datasets import load_dataset
+
+    os.makedirs(output_dir, exist_ok=True)
+    all_samples = []
+
+    # --- English: SST-2 / TweetEval ---
+    logger.info("Loading English sentiment dataset...")
+    en_loaded = False
+    for candidate in [("stanfordnlp/sst2", None), ("nyu-mll/glue", "sst2"), ("mteb/tweet_eval_sentiment", None)]:
+        try:
+            ds_name, sub = candidate
+            sst2 = load_dataset(ds_name, sub, split="train") if sub else load_dataset(ds_name, split="train")
+            en_count = 0
+            for sample in sst2:
+                text = sample.get("sentence") or sample.get("text") or ""
+                label = sample.get("label", 0)
+                mapped = "Positive" if label == 1 else "Negative" if label == 0 else "Neutral"
+                if text:
+                    all_samples.append({"text": text[:512], "label": mapped, "language": "en"})
+                    en_count += 1
+            if en_count > 0:
+                logger.info(f"Loaded {en_count} English sentiment samples from '{ds_name}'")
+                en_loaded = True
+                break
+        except Exception:
+            continue
+
+    # --- Arabic: TweetEval / Sentiment ---
+    logger.info("Loading Arabic sentiment dataset...")
+    ar_loaded = False
+    for candidate in ["mteb/tweet_eval_sentiment", "MohamedAtef/LABR"]:
+        try:
+            asad = load_dataset(candidate, split="train")
+            ar_count = 0
+            for sample in asad:
+                text = sample.get("text") or sample.get("text_raw") or ""
+                label = sample.get("label", 1)
+                label_map = {0: "Negative", 1: "Neutral", 2: "Positive"}
+                mapped = label_map.get(label, "Neutral")
+                if text:
+                    all_samples.append({"text": text[:512], "label": mapped, "language": "ar"})
+                    ar_count += 1
+            if ar_count > 0:
+                logger.info(f"Loaded {ar_count} Arabic sentiment samples")
+                ar_loaded = True
+                break
+        except Exception:
+            continue
+
+    if not ar_loaded and len([s for s in all_samples if s["language"] == "ar"]) < 10:
+        sample_ar_sentiment = [
+            ("فوز تاريخي وانتعاش اقتصادي ممتاز للمنطقة", "Positive"),
+            ("تراجع كبير وخسائر ضخمة وتدهور الأوضاع", "Negative"),
+            ("عقد اجتماع عادي لمناقشة جدول الأعمال", "Neutral"),
+        ]
+        for text, sent in sample_ar_sentiment * 100:
+            all_samples.append({"text": text, "label": sent, "language": "ar"})
+
+    # --- Shuffle and split ---
+    import random
+    random.shuffle(all_samples)
+    split_idx = int(len(all_samples) * 0.9)
+
+    _save_jsonl(all_samples[:split_idx], os.path.join(output_dir, "sentiment_train.jsonl"))
+    _save_jsonl(all_samples[split_idx:], os.path.join(output_dir, "sentiment_test.jsonl"))
+    logger.info(f"Sentiment dataset saved: {split_idx} train, {len(all_samples) - split_idx} test")
+
+
+def prepare_ner_datasets(output_dir: str):
+    """
+    Download and prepare WikiANN / CoNLL-2003 for NER.
+    Outputs: ner_train.jsonl, ner_test.jsonl with {tokens, ner_tags, language} pairs.
+    """
+    from datasets import load_dataset
+
+    os.makedirs(output_dir, exist_ok=True)
+    all_train = []
+    all_test = []
+
+    # --- WikiANN Arabic ---
+    logger.info("Loading WikiANN Arabic NER dataset...")
+    try:
+        wikiann_ar = load_dataset("unimelb-nlp/wikiann", "ar")
+        ar_train = [{"tokens": s["tokens"], "ner_tags": s["ner_tags"], "language": "ar"} for s in wikiann_ar["train"]]
+        ar_test = [{"tokens": s["tokens"], "ner_tags": s["ner_tags"], "language": "ar"} for s in wikiann_ar["test"]]
+        all_train.extend(ar_train)
+        all_test.extend(ar_test)
+        logger.info(f"Loaded WikiANN Arabic: {len(ar_train)} train, {len(ar_test)} test")
+    except Exception as e:
+        logger.warning(f"Could not load WikiANN Arabic: {e}")
+
+    # --- WikiANN English ---
+    logger.info("Loading WikiANN English NER dataset...")
+    try:
+        wikiann_en = load_dataset("unimelb-nlp/wikiann", "en")
+        en_train = [{"tokens": s["tokens"], "ner_tags": s["ner_tags"], "language": "en"} for s in wikiann_en["train"]]
+        en_test = [{"tokens": s["tokens"], "ner_tags": s["ner_tags"], "language": "en"} for s in wikiann_en["test"]]
+        all_train.extend(en_train)
+        all_test.extend(en_test)
+        logger.info(f"Loaded WikiANN English: {len(en_train)} train, {len(en_test)} test")
+    except Exception as e:
+        logger.warning(f"Could not load WikiANN English: {e}")
+
+    # --- CoNLL-2003 English ---
+    logger.info("Loading CoNLL-2003 English NER dataset...")
+    try:
+        conll = load_dataset("eriktks/conll2003")
+        conll_train = [{"tokens": s["tokens"], "ner_tags": s["ner_tags"], "language": "en"} for s in conll["train"]]
+        conll_test = [{"tokens": s["tokens"], "ner_tags": s["ner_tags"], "language": "en"} for s in conll["test"]]
+        all_train.extend(conll_train)
+        all_test.extend(conll_test)
+        logger.info(f"Added CoNLL-2003: {len(conll_train)} train, {len(conll_test)} test")
+    except Exception as e:
+        logger.warning(f"Could not load CoNLL-2003: {e}")
+
+    # Fallback if datasets were unavailable
+    if not all_train:
+        sample_ner = [
+            {"tokens": ["الرئيس", "السيسي", "في", "القاهرة"], "ner_tags": [0, 1, 0, 5], "language": "ar"},
+            {"tokens": ["Biden", "visited", "London"], "ner_tags": [1, 0, 5], "language": "en"}
+        ]
+        all_train = sample_ner * 50
+        all_test = sample_ner * 10
+
+    _save_jsonl(all_train, os.path.join(output_dir, "ner_train.jsonl"))
+    _save_jsonl(all_test, os.path.join(output_dir, "ner_test.jsonl"))
+    logger.info(f"NER dataset saved: {len(all_train)} train, {len(all_test)} test")
+
+
+def _save_jsonl(data: list, filepath: str):
+    """Save a list of dicts as JSONL, merging with any existing file to preserve collected data."""
+    existing_samples = []
+    if os.path.exists(filepath):
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                for line in f:
+                    if line.strip():
+                        existing_samples.append(json.loads(line.strip()))
+        except Exception:
+            pass
+
+    # Merge and deduplicate by text/tokens content
+    seen_keys = set()
+    combined_data = []
+
+    for item in existing_samples + data:
+        key = str(item.get("text") or item.get("tokens") or "")
+        if key and key not in seen_keys:
+            seen_keys.add(key)
+            combined_data.append(item)
+
+    with open(filepath, 'w', encoding='utf-8') as f:
+        for item in combined_data:
+            f.write(json.dumps(item, ensure_ascii=False) + '\n')
+    
+    new_added = len(combined_data) - len(existing_samples)
+    logger.info(f"Saved {len(combined_data)} total samples ({len(existing_samples)} existing + {max(0, new_added)} new) to {filepath}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Prepare NLP training datasets for WhatsNew")
+    parser.add_argument("--task", choices=["category", "sentiment", "ner", "all"], default="all",
+                        help="Which dataset to prepare")
+    parser.add_argument("--output-dir", default="nlp_training/data", help="Output directory for processed datasets")
+    args = parser.parse_args()
+
+    if args.task in ("category", "all"):
+        prepare_category_datasets(args.output_dir)
+    if args.task in ("sentiment", "all"):
+        prepare_sentiment_datasets(args.output_dir)
+    if args.task in ("ner", "all"):
+        prepare_ner_datasets(args.output_dir)
+
+    logger.info("Dataset preparation complete!")

nlp_training/train_category_classifier.py

@@ -0,0 +1,190 @@
+"""
+Fine-tune a multilingual category classifier for WhatsNew News Intelligence Engine.
+
+Model: XLM-RoBERTa-base (handles Arabic + English in a single model)
+Dataset: SANAD (Arabic) + HuffPost (English) → 13 unified news categories
+
+Usage:
+    # 1. First prepare the data:
+    python nlp_training/prepare_datasets.py --task category
+
+    # 2. Then fine-tune:
+    python nlp_training/train_category_classifier.py
+
+    # 3. The fine-tuned model will be saved to ./models/category_classifier/
+    #    and can be uploaded to HuggingFace Hub with:
+    python nlp_training/train_category_classifier.py --push-to-hub AbdoCrow/whatsnew-category
+
+Requirements:
+    pip install transformers datasets torch scikit-learn accelerate
+"""
+
+import os
+import json
+import logging
+import argparse
+import numpy as np
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Our unified 13 categories
+LABEL_LIST = [
+    "Sports", "Technology", "Gaming", "Business", "Health", "Science",
+    "Food", "Entertainment", "Politics", "Education", "Crime", "Weather", "Travel"
+]
+LABEL2ID = {label: i for i, label in enumerate(LABEL_LIST)}
+ID2LABEL = {i: label for i, label in enumerate(LABEL_LIST)}
+
+
+def load_jsonl(filepath):
+    """Load a JSONL dataset file."""
+    samples = []
+    with open(filepath, 'r', encoding='utf-8') as f:
+        for line in f:
+            samples.append(json.loads(line.strip()))
+    return samples
+
+
+def main(args):
+    from transformers import (
+        AutoTokenizer,
+        AutoModelForSequenceClassification,
+        TrainingArguments,
+        Trainer,
+        EarlyStoppingCallback
+    )
+    from datasets import Dataset
+    from sklearn.metrics import accuracy_score, f1_score
+
+    # ---------------------------------------------------------------
+    # 1. Load prepared datasets
+    # ---------------------------------------------------------------
+    data_dir = args.data_dir
+    train_path = os.path.join(data_dir, "category_train.jsonl")
+    test_path = os.path.join(data_dir, "category_test.jsonl")
+
+    if not os.path.exists(train_path):
+        logger.error(f"Training data not found at {train_path}. Run prepare_datasets.py first.")
+        return
+
+    train_data = load_jsonl(train_path)
+    test_data = load_jsonl(test_path)
+
+    # Filter to only include our known labels
+    train_data = [s for s in train_data if s["label"] in LABEL2ID]
+    test_data = [s for s in test_data if s["label"] in LABEL2ID]
+
+    logger.info(f"Train: {len(train_data)} samples, Test: {len(test_data)} samples")
+
+    # Convert to HuggingFace Dataset
+    train_dataset = Dataset.from_list([
+        {"text": s["text"], "label": LABEL2ID[s["label"]]} for s in train_data
+    ])
+    test_dataset = Dataset.from_list([
+        {"text": s["text"], "label": LABEL2ID[s["label"]]} for s in test_data
+    ])
+
+    # ---------------------------------------------------------------
+    # 2. Tokenize
+    # ---------------------------------------------------------------
+    model_name = args.model_name
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    def tokenize_fn(examples):
+        return tokenizer(examples["text"], truncation=True, padding="max_length", max_length=256)
+
+    train_dataset = train_dataset.map(tokenize_fn, batched=True, remove_columns=["text"])
+    test_dataset = test_dataset.map(tokenize_fn, batched=True, remove_columns=["text"])
+
+    train_dataset.set_format("torch")
+    test_dataset.set_format("torch")
+
+    # ---------------------------------------------------------------
+    # 3. Load Model
+    # ---------------------------------------------------------------
+    model = AutoModelForSequenceClassification.from_pretrained(
+        model_name,
+        num_labels=len(LABEL_LIST),
+        id2label=ID2LABEL,
+        label2id=LABEL2ID
+    )
+    logger.info(f"Loaded model: {model_name} with {len(LABEL_LIST)} labels")
+
+    # ---------------------------------------------------------------
+    # 4. Training Configuration
+    # ---------------------------------------------------------------
+    output_dir = args.output_dir
+
+    training_args = TrainingArguments(
+        output_dir=output_dir,
+        num_train_epochs=args.epochs,
+        per_device_train_batch_size=args.batch_size,
+        per_device_eval_batch_size=args.batch_size * 2,
+        learning_rate=args.learning_rate,
+        weight_decay=0.01,
+        eval_strategy="epoch",
+        save_strategy="epoch",
+        load_best_model_at_end=True,
+        metric_for_best_model="f1_macro",
+        greater_is_better=True,
+        save_total_limit=2,
+        logging_steps=100,
+        warmup_ratio=0.1,
+        fp16=args.fp16,
+        report_to="none",  # Disable wandb/tensorboard
+    )
+
+    def compute_metrics(eval_pred):
+        logits, labels = eval_pred
+        predictions = np.argmax(logits, axis=-1)
+        acc = accuracy_score(labels, predictions)
+        f1 = f1_score(labels, predictions, average="macro")
+        return {"accuracy": acc, "f1_macro": f1}
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=test_dataset,
+        compute_metrics=compute_metrics,
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=2)]
+    )
+
+    # ---------------------------------------------------------------
+    # 5. Train
+    # ---------------------------------------------------------------
+    logger.info("Starting training...")
+    trainer.train()
+
+    # ---------------------------------------------------------------
+    # 6. Evaluate
+    # ---------------------------------------------------------------
+    results = trainer.evaluate()
+    logger.info(f"Evaluation results: {results}")
+
+    # ---------------------------------------------------------------
+    # 7. Save
+    # ---------------------------------------------------------------
+    trainer.save_model(output_dir)
+    tokenizer.save_pretrained(output_dir)
+    logger.info(f"Model saved to {output_dir}")
+
+    # Optionally push to HuggingFace Hub
+    if args.push_to_hub:
+        trainer.push_to_hub(args.push_to_hub)
+        logger.info(f"Model pushed to HuggingFace Hub: {args.push_to_hub}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Fine-tune a news category classifier")
+    parser.add_argument("--model-name", default="xlm-roberta-base", help="Base model name")
+    parser.add_argument("--data-dir", default="nlp_training/data", help="Directory with prepared datasets")
+    parser.add_argument("--output-dir", default="models/category_classifier", help="Output directory for trained model")
+    parser.add_argument("--epochs", type=int, default=5, help="Number of training epochs")
+    parser.add_argument("--batch-size", type=int, default=16, help="Training batch size")
+    parser.add_argument("--learning-rate", type=float, default=2e-5, help="Learning rate")
+    parser.add_argument("--fp16", action="store_true", help="Use mixed precision training (requires CUDA)")
+    parser.add_argument("--push-to-hub", type=str, default=None, help="HuggingFace Hub repo to push to")
+    args = parser.parse_args()
+    main(args)

nlp_training/train_ner_model.py

@@ -0,0 +1,271 @@
+"""
+Fine-tune a multilingual NER model for WhatsNew News Intelligence Engine.
+
+Model: XLM-RoBERTa-base
+Dataset: WikiANN (Arabic + English) + CoNLL-2003 (English)
+Entities: PER (people), ORG (organizations), LOC (locations)
+
+Usage:
+    # 1. First prepare the data:
+    python nlp_training/prepare_datasets.py --task ner
+
+    # 2. Then fine-tune:
+    python nlp_training/train_ner_model.py
+
+    # 3. The fine-tuned model will be saved to ./models/ner_model/
+
+Requirements:
+    pip install transformers datasets torch scikit-learn seqeval accelerate
+"""
+
+import os
+import json
+import logging
+import argparse
+import numpy as np
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# WikiANN NER tag mapping (IOB2 format)
+# WikiANN uses: O=0, B-PER=1, I-PER=2, B-ORG=3, I-ORG=4, B-LOC=5, I-LOC=6
+LABEL_LIST = ["O", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC"]
+LABEL2ID = {label: i for i, label in enumerate(LABEL_LIST)}
+ID2LABEL = {i: label for i, label in enumerate(LABEL_LIST)}
+
+
+def load_jsonl(filepath):
+    samples = []
+    with open(filepath, 'r', encoding='utf-8') as f:
+        for line in f:
+            samples.append(json.loads(line.strip()))
+    return samples
+
+
+def main(args):
+    from transformers import (
+        AutoTokenizer,
+        AutoModelForTokenClassification,
+        TrainingArguments,
+        Trainer,
+        DataCollatorForTokenClassification,
+        EarlyStoppingCallback
+    )
+    from datasets import Dataset
+
+    # ---------------------------------------------------------------
+    # 1. Load prepared datasets
+    # ---------------------------------------------------------------
+    data_dir = args.data_dir
+    train_path = os.path.join(data_dir, "ner_train.jsonl")
+    test_path = os.path.join(data_dir, "ner_test.jsonl")
+
+    if not os.path.exists(train_path):
+        logger.error(f"Training data not found at {train_path}. Run prepare_datasets.py first.")
+        return
+
+    train_data = load_jsonl(train_path)
+    test_data = load_jsonl(test_path)
+
+    logger.info(f"Train: {len(train_data)} samples, Test: {len(test_data)} samples")
+
+    train_dataset = Dataset.from_list(train_data)
+    test_dataset = Dataset.from_list(test_data)
+
+    # ---------------------------------------------------------------
+    # 2. Tokenize with label alignment
+    # ---------------------------------------------------------------
+    model_name = args.model_name
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    def tokenize_and_align_labels(examples):
+        """
+        Tokenize text and realign NER labels with sub-word tokens.
+        Sub-word continuations receive label -100 (ignored by loss function).
+        """
+        tokenized_inputs = tokenizer(
+            examples["tokens"],
+            truncation=True,
+            padding="max_length",
+            max_length=128,
+            is_split_into_words=True
+        )
+
+        all_labels = []
+        for i, labels in enumerate(examples["ner_tags"]):
+            word_ids = tokenized_inputs.word_ids(batch_index=i)
+            label_ids = []
+            previous_word_idx = None
+            for word_idx in word_ids:
+                if word_idx is None:
+                    label_ids.append(-100)  # Special tokens
+                elif word_idx != previous_word_idx:
+                    # Map WikiANN integer tags to our label space
+                    tag = labels[word_idx] if word_idx < len(labels) else 0
+                    label_ids.append(tag)
+                else:
+                    # Sub-word continuation: use -100 or keep I- tag
+                    tag = labels[word_idx] if word_idx < len(labels) else 0
+                    # For sub-words of an entity, keep the I- variant
+                    if tag in (1, 3, 5):  # B-PER, B-ORG, B-LOC → convert to I-
+                        tag = tag + 1
+                    label_ids.append(tag)
+                previous_word_idx = word_idx
+            all_labels.append(label_ids)
+
+        tokenized_inputs["labels"] = all_labels
+        return tokenized_inputs
+
+    # Remove the "language" column if present
+    columns_to_remove = ["tokens", "ner_tags"]
+    if "language" in train_dataset.column_names:
+        columns_to_remove.append("language")
+
+    train_dataset = train_dataset.map(
+        tokenize_and_align_labels, batched=True, remove_columns=columns_to_remove
+    )
+    test_dataset = test_dataset.map(
+        tokenize_and_align_labels, batched=True, remove_columns=columns_to_remove
+    )
+
+    train_dataset.set_format("torch")
+    test_dataset.set_format("torch")
+
+    # ---------------------------------------------------------------
+    # 3. Load Model
+    # ---------------------------------------------------------------
+    model = AutoModelForTokenClassification.from_pretrained(
+        model_name,
+        num_labels=len(LABEL_LIST),
+        id2label=ID2LABEL,
+        label2id=LABEL2ID
+    )
+    logger.info(f"Loaded model: {model_name}")
+
+    # ---------------------------------------------------------------
+    # 4. Metrics (using seqeval for entity-level F1)
+    # ---------------------------------------------------------------
+    try:
+        from seqeval.metrics import f1_score as seqeval_f1, classification_report as seqeval_report
+        HAS_SEQEVAL = True
+    except ImportError:
+        logger.warning("seqeval not installed. Using token-level accuracy only.")
+        HAS_SEQEVAL = False
+
+    def compute_metrics(eval_pred):
+        logits, labels = eval_pred
+        predictions = np.argmax(logits, axis=-1)
+
+        if HAS_SEQEVAL:
+            # Convert to label strings, skipping -100
+            true_labels_seq = []
+            pred_labels_seq = []
+            for pred_seq, label_seq in zip(predictions, labels):
+                true_sent = []
+                pred_sent = []
+                for p, l in zip(pred_seq, label_seq):
+                    if l != -100:
+                        true_sent.append(ID2LABEL.get(int(l), "O"))
+                        pred_sent.append(ID2LABEL.get(int(p), "O"))
+                true_labels_seq.append(true_sent)
+                pred_labels_seq.append(pred_sent)
+
+            f1 = seqeval_f1(true_labels_seq, pred_labels_seq, average="macro")
+            return {"f1_macro": f1}
+        else:
+            # Fallback: token-level accuracy
+            mask = labels != -100
+            correct = (predictions[mask] == labels[mask]).sum()
+            total = mask.sum()
+            return {"accuracy": float(correct / total)}
+
+    # ---------------------------------------------------------------
+    # 5. Training Configuration
+    # ---------------------------------------------------------------
+    output_dir = args.output_dir
+    data_collator = DataCollatorForTokenClassification(tokenizer)
+
+    training_args = TrainingArguments(
+        output_dir=output_dir,
+        num_train_epochs=args.epochs,
+        per_device_train_batch_size=args.batch_size,
+        per_device_eval_batch_size=args.batch_size * 2,
+        learning_rate=args.learning_rate,
+        weight_decay=0.01,
+        eval_strategy="epoch",
+        save_strategy="epoch",
+        load_best_model_at_end=True,
+        metric_for_best_model="f1_macro" if HAS_SEQEVAL else "accuracy",
+        greater_is_better=True,
+        save_total_limit=2,
+        logging_steps=100,
+        warmup_ratio=0.1,
+        fp16=args.fp16,
+        report_to="none",
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=test_dataset,
+        data_collator=data_collator,
+        compute_metrics=compute_metrics,
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=2)]
+    )
+
+    # ---------------------------------------------------------------
+    # 6. Train
+    # ---------------------------------------------------------------
+    logger.info("Starting NER training...")
+    trainer.train()
+
+    # ---------------------------------------------------------------
+    # 7. Evaluate
+    # ---------------------------------------------------------------
+    results = trainer.evaluate()
+    logger.info(f"Evaluation results: {results}")
+
+    if HAS_SEQEVAL:
+        predictions = trainer.predict(test_dataset)
+        pred_labels = np.argmax(predictions.predictions, axis=-1)
+        true_labels = predictions.label_ids
+
+        true_labels_seq = []
+        pred_labels_seq = []
+        for pred_seq, label_seq in zip(pred_labels, true_labels):
+            true_sent = []
+            pred_sent = []
+            for p, l in zip(pred_seq, label_seq):
+                if l != -100:
+                    true_sent.append(ID2LABEL.get(int(l), "O"))
+                    pred_sent.append(ID2LABEL.get(int(p), "O"))
+            true_labels_seq.append(true_sent)
+            pred_labels_seq.append(pred_sent)
+
+        report = seqeval_report(true_labels_seq, pred_labels_seq)
+        logger.info(f"\nEntity-level Classification Report:\n{report}")
+
+    # ---------------------------------------------------------------
+    # 8. Save
+    # ---------------------------------------------------------------
+    trainer.save_model(output_dir)
+    tokenizer.save_pretrained(output_dir)
+    logger.info(f"Model saved to {output_dir}")
+
+    if args.push_to_hub:
+        trainer.push_to_hub(args.push_to_hub)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Fine-tune a NER model")
+    parser.add_argument("--model-name", default="xlm-roberta-base", help="Base model name")
+    parser.add_argument("--data-dir", default="nlp_training/data", help="Directory with prepared datasets")
+    parser.add_argument("--output-dir", default="models/ner_model", help="Output directory")
+    parser.add_argument("--epochs", type=int, default=5, help="Number of training epochs")
+    parser.add_argument("--batch-size", type=int, default=16, help="Training batch size")
+    parser.add_argument("--learning-rate", type=float, default=3e-5, help="Learning rate")
+    parser.add_argument("--fp16", action="store_true", help="Use mixed precision training")
+    parser.add_argument("--push-to-hub", type=str, default=None, help="HuggingFace Hub repo")
+    args = parser.parse_args()
+    main(args)

nlp_training/train_sentiment_analyzer.py

@@ -0,0 +1,195 @@
+"""
+Fine-tune a multilingual sentiment analyzer for WhatsNew News Intelligence Engine.
+
+Model: XLM-RoBERTa-base (handles Arabic + English in a single model)
+Dataset: ASAD (Arabic tweets) + SST-2 (English sentences) → 3 sentiment labels
+
+Usage:
+    # 1. First prepare the data:
+    python nlp_training/prepare_datasets.py --task sentiment
+
+    # 2. Then fine-tune:
+    python nlp_training/train_sentiment_analyzer.py
+
+    # 3. The fine-tuned model will be saved to ./models/sentiment_analyzer/
+
+Requirements:
+    pip install transformers datasets torch scikit-learn accelerate
+"""
+
+import os
+import json
+import logging
+import argparse
+import numpy as np
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Sentiment labels
+LABEL_LIST = ["Positive", "Negative", "Neutral"]
+LABEL2ID = {label: i for i, label in enumerate(LABEL_LIST)}
+ID2LABEL = {i: label for i, label in enumerate(LABEL_LIST)}
+
+
+def load_jsonl(filepath):
+    samples = []
+    with open(filepath, 'r', encoding='utf-8') as f:
+        for line in f:
+            samples.append(json.loads(line.strip()))
+    return samples
+
+
+def main(args):
+    from transformers import (
+        AutoTokenizer,
+        AutoModelForSequenceClassification,
+        TrainingArguments,
+        Trainer,
+        EarlyStoppingCallback
+    )
+    from datasets import Dataset
+    from sklearn.metrics import accuracy_score, f1_score, classification_report
+
+    # ---------------------------------------------------------------
+    # 1. Load prepared datasets
+    # ---------------------------------------------------------------
+    data_dir = args.data_dir
+    train_path = os.path.join(data_dir, "sentiment_train.jsonl")
+    test_path = os.path.join(data_dir, "sentiment_test.jsonl")
+
+    if not os.path.exists(train_path):
+        logger.error(f"Training data not found at {train_path}. Run prepare_datasets.py first.")
+        return
+
+    train_data = load_jsonl(train_path)
+    test_data = load_jsonl(test_path)
+
+    # Filter to only include our known labels
+    train_data = [s for s in train_data if s["label"] in LABEL2ID]
+    test_data = [s for s in test_data if s["label"] in LABEL2ID]
+
+    logger.info(f"Train: {len(train_data)} samples, Test: {len(test_data)} samples")
+
+    # Label distribution
+    from collections import Counter
+    train_dist = Counter(s["label"] for s in train_data)
+    logger.info(f"Train label distribution: {dict(train_dist)}")
+
+    # Convert to HuggingFace Dataset
+    train_dataset = Dataset.from_list([
+        {"text": s["text"], "label": LABEL2ID[s["label"]]} for s in train_data
+    ])
+    test_dataset = Dataset.from_list([
+        {"text": s["text"], "label": LABEL2ID[s["label"]]} for s in test_data
+    ])
+
+    # ---------------------------------------------------------------
+    # 2. Tokenize
+    # ---------------------------------------------------------------
+    model_name = args.model_name
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    def tokenize_fn(examples):
+        return tokenizer(examples["text"], truncation=True, padding="max_length", max_length=128)
+
+    train_dataset = train_dataset.map(tokenize_fn, batched=True, remove_columns=["text"])
+    test_dataset = test_dataset.map(tokenize_fn, batched=True, remove_columns=["text"])
+
+    train_dataset.set_format("torch")
+    test_dataset.set_format("torch")
+
+    # ---------------------------------------------------------------
+    # 3. Load Model
+    # ---------------------------------------------------------------
+    model = AutoModelForSequenceClassification.from_pretrained(
+        model_name,
+        num_labels=len(LABEL_LIST),
+        id2label=ID2LABEL,
+        label2id=LABEL2ID
+    )
+    logger.info(f"Loaded model: {model_name}")
+
+    # ---------------------------------------------------------------
+    # 4. Training Configuration
+    # ---------------------------------------------------------------
+    output_dir = args.output_dir
+
+    training_args = TrainingArguments(
+        output_dir=output_dir,
+        num_train_epochs=args.epochs,
+        per_device_train_batch_size=args.batch_size,
+        per_device_eval_batch_size=args.batch_size * 2,
+        learning_rate=args.learning_rate,
+        weight_decay=0.01,
+        eval_strategy="epoch",
+        save_strategy="epoch",
+        load_best_model_at_end=True,
+        metric_for_best_model="f1_macro",
+        greater_is_better=True,
+        save_total_limit=2,
+        logging_steps=100,
+        warmup_ratio=0.1,
+        fp16=args.fp16,
+        report_to="none",
+    )
+
+    def compute_metrics(eval_pred):
+        logits, labels = eval_pred
+        predictions = np.argmax(logits, axis=-1)
+        acc = accuracy_score(labels, predictions)
+        f1 = f1_score(labels, predictions, average="macro")
+        return {"accuracy": acc, "f1_macro": f1}
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=test_dataset,
+        compute_metrics=compute_metrics,
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=2)]
+    )
+
+    # ---------------------------------------------------------------
+    # 5. Train
+    # ---------------------------------------------------------------
+    logger.info("Starting sentiment training...")
+    trainer.train()
+
+    # ---------------------------------------------------------------
+    # 6. Evaluate with detailed report
+    # ---------------------------------------------------------------
+    results = trainer.evaluate()
+    logger.info(f"Evaluation results: {results}")
+
+    # Detailed per-class report
+    predictions = trainer.predict(test_dataset)
+    pred_labels = np.argmax(predictions.predictions, axis=-1)
+    true_labels = predictions.label_ids
+    report = classification_report(true_labels, pred_labels, target_names=LABEL_LIST)
+    logger.info(f"\nClassification Report:\n{report}")
+
+    # ---------------------------------------------------------------
+    # 7. Save
+    # ---------------------------------------------------------------
+    trainer.save_model(output_dir)
+    tokenizer.save_pretrained(output_dir)
+    logger.info(f"Model saved to {output_dir}")
+
+    if args.push_to_hub:
+        trainer.push_to_hub(args.push_to_hub)
+        logger.info(f"Model pushed to HuggingFace Hub: {args.push_to_hub}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Fine-tune a sentiment analyzer")
+    parser.add_argument("--model-name", default="xlm-roberta-base", help="Base model name")
+    parser.add_argument("--data-dir", default="nlp_training/data", help="Directory with prepared datasets")
+    parser.add_argument("--output-dir", default="models/sentiment_analyzer", help="Output directory")
+    parser.add_argument("--epochs", type=int, default=4, help="Number of training epochs")
+    parser.add_argument("--batch-size", type=int, default=32, help="Training batch size")
+    parser.add_argument("--learning-rate", type=float, default=2e-5, help="Learning rate")
+    parser.add_argument("--fp16", action="store_true", help="Use mixed precision training")
+    parser.add_argument("--push-to-hub", type=str, default=None, help="HuggingFace Hub repo")
+    args = parser.parse_args()
+    main(args)

requirements.txt

@@ -0,0 +1,54 @@
+annotated-types==0.8.0
+anyio==4.14.2
+beautifulsoup4==4.15.0
+CacheControl==0.14.4
+certifi==2026.7.22
+cffi==2.1.1
+charset-normalizer==3.4.9
+cryptography==50.0.0
+feedparser==6.0.14
+feedparser-sgmllib==2.1.0
+firebase_admin==7.5.0
+google-api-core==2.33.0
+google-auth==2.56.2
+google-cloud-core==2.6.0
+google-cloud-firestore==2.28.0
+google-cloud-storage==3.13.0
+google-crc32c==1.8.0
+google-resumable-media==2.10.0
+googleapis-common-protos==1.75.0
+grpcio==1.83.0
+grpcio-status==1.83.0
+h11==0.16.0
+h2==4.4.1
+hpack==4.2.0
+httpcore==1.0.9
+httpx==0.28.1
+hyperframe==6.1.0
+idna==3.18
+msgpack==1.2.1
+proto-plus==1.28.2
+protobuf==7.35.1
+pyasn1==0.6.4
+pyasn1_modules==0.4.2
+pycparser==3.0
+pydantic==2.13.4
+pydantic_core==2.46.4
+PyJWT==2.13.0
+requests==2.34.2
+schedule==1.2.2
+soupsieve==2.9.1
+typing-inspection==0.4.2
+typing_extensions==4.16.0
+urllib3==2.7.0
+
+# NLP & Machine Learning Dependencies
+transformers
+datasets
+scikit-learn
+accelerate
+seqeval
+keybert
+sentence-transformers
+numpy
+gradio

requirements_space.txt

@@ -0,0 +1,9 @@
+gradio>=4.0.0
+transformers
+torch
+datasets
+keybert
+sentence-transformers
+beautifulsoup4
+requests
+pydantic

scrapers/base_scraper.py

@@ -0,0 +1,21 @@
+import requests
+from bs4 import BeautifulSoup
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+class BaseScraper:
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
+    }
+
+    def fetch_html(self, url: str) -> Optional[BeautifulSoup]:
+        """Fetches the HTML content of the given URL and returns a BeautifulSoup object."""
+        try:
+            response = requests.get(url, headers=self.headers, timeout=10)
+            response.raise_for_status()
+            return BeautifulSoup(response.content, 'html.parser')
+        except Exception as e:
+            logger.error(f"Failed to fetch {url}: {e}")
+            return None

scrapers/rss_scraper.py

@@ -0,0 +1,128 @@
+import datetime
+import logging
+import feedparser
+from typing import List, Optional
+from bs4 import BeautifulSoup
+from domain.entities.article import Article
+from domain.interfaces import ScraperInterface
+import requests
+import time
+
+# Set a custom user agent for feedparser to bypass basic bot protections
+feedparser.USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
+
+logger = logging.getLogger(__name__)
+
+class RssScraper(ScraperInterface):
+    def __init__(self, source_name: str, language: str, feed_url: str, category: str = "World"):
+        self.source_name = source_name
+        self.language = language
+        self.feed_url = feed_url
+        self.category = category
+
+    def scrape(self) -> List[Article]:
+        articles = []
+        try:
+            feed = feedparser.parse(self.feed_url)
+            
+            # Check for network or parsing errors that feedparser hides
+            if getattr(feed, 'bozo', 0) == 1 and hasattr(feed, 'bozo_exception'):
+                logger.warning(f"Feed error for {self.feed_url}: {feed.bozo_exception}")
+                if not feed.entries:
+                    return articles
+                    
+            for entry in feed.entries[:10]:  # Limit to top 10 per feed
+                title = entry.get("title", "")
+                link = entry.get("link", "")
+                
+                # Get summary, remove HTML tags
+                summary_raw = entry.get("description", "")
+                summary = BeautifulSoup(summary_raw, "html.parser").get_text()[:200]
+                
+                if not title or not link:
+                    continue
+                
+                # Try to extract image (from RSS or OpenGraph meta tags)
+                image_url = self._extract_image(entry, link=link)
+                
+                # Parse date or default to now
+                try:
+                    # feedparser parses date into struct_time
+                    published_at = datetime.datetime.fromtimestamp(time.mktime(entry.published_parsed)) if hasattr(entry, 'published_parsed') and entry.published_parsed else datetime.datetime.now()
+                except Exception:
+                    published_at = datetime.datetime.now()
+
+                article = Article(
+                    id=Article.generate_id_from_url(link),
+                    title=title,
+                    source=self.source_name,
+                    url=link,
+                    imageUrl=image_url,
+                    hasImage=True if image_url else False,
+                    summary=summary + "..." if len(summary) >= 200 else summary,
+                    publishedAt=published_at,
+                    language=self.language,
+                    category=self.category
+                )
+                articles.append(article)
+        except Exception as e:
+            logger.error(f"Failed to scrape RSS {self.feed_url}: {e}")
+            
+        return articles
+
+    def _extract_image(self, entry, link: str = None) -> Optional[str]:
+        # 1. Check media_content
+        if hasattr(entry, 'media_content') and entry.media_content:
+            for media in entry.media_content:
+                if media.get('url'):
+                    return media.get('url')
+
+        # 2. Check media_thumbnail
+        if hasattr(entry, 'media_thumbnail') and entry.media_thumbnail:
+            return entry.media_thumbnail[0].get('url')
+
+        # 3. Check enclosures
+        if hasattr(entry, 'enclosures') and entry.enclosures:
+            for enc in entry.enclosures:
+                if 'image' in enc.get('type', '') and enc.get('href'):
+                    return enc.get('href')
+
+        # 4. Check links for image type
+        if hasattr(entry, 'links'):
+            for l in entry.links:
+                if 'image' in l.get('type', ''):
+                    return l.get('href')
+
+        # 5. Check description/content HTML for <img> tags
+        for field in ['description', 'content']:
+            if hasattr(entry, field):
+                content_list = getattr(entry, field)
+                raw_html = content_list[0].get('value', '') if isinstance(content_list, list) else content_list
+                soup = BeautifulSoup(str(raw_html), "html.parser")
+                img = soup.find('img')
+                if img and img.get('src'):
+                    return img.get('src')
+
+        # 6. Layer 2: OpenGraph Web Scraping Fallback (Fast 2s timeout)
+        if link and link.startswith('http'):
+            try:
+                headers = {"User-Agent": feedparser.USER_AGENT}
+                resp = requests.get(link, headers=headers, timeout=2.5)
+                if resp.status_code == 200:
+                    soup = BeautifulSoup(resp.text, "html.parser")
+                    # Check og:image, twitter:image, image_src
+                    og_img = soup.find('meta', property='og:image') or soup.find('meta', attrs={'name': 'og:image'})
+                    if og_img and og_img.get('content'):
+                        return og_img.get('content')
+                        
+                    tw_img = soup.find('meta', attrs={'name': 'twitter:image'}) or soup.find('meta', property='twitter:image')
+                    if tw_img and tw_img.get('content'):
+                        return tw_img.get('content')
+
+                    rel_img = soup.find('link', rel='image_src')
+                    if rel_img and rel_img.get('href'):
+                        return rel_img.get('href')
+            except Exception:
+                pass
+
+        return None

services/ai_classifier.py

@@ -0,0 +1,185 @@
+import requests
+import json
+import os
+import re
+import logging
+from domain.entities.article import Article
+
+from services.lexicons import (
+    POSITIVE_WORDS,
+    NEGATIVE_WORDS,
+    CATEGORIES_MAP,
+    LOCATIONS_MAP,
+    MIDDLE_EAST_COUNTRIES,
+    STOP_WORDS
+)
+from services.local_nlp_classifier import LocalNLPClassifier
+
+logger = logging.getLogger(__name__)
+
+class HuggingFaceClassifier:
+    def __init__(self, api_key: str = None):
+        self.api_key = api_key or os.environ.get("HF_API_KEY")
+        # List of models to try in sequence if one hits rate limits or credit depletion
+        self.fallback_models = [
+            "Qwen/Qwen2.5-7B-Instruct",
+            "Qwen/Qwen2.5-Coder-7B-Instruct",
+            "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
+            "meta-llama/Llama-3.1-8B-Instruct"
+        ]
+        self.api_url = "https://router.huggingface.co/v1/chat/completions"
+        # Tier 2: Local NLP transformer models (loaded lazily on first use)
+        self._local_nlp = LocalNLPClassifier()
+
+    def enrich_article(self, article: Article) -> Article:
+        if not self.api_key:
+            logger.warning("No HF_API_KEY provided. Using local heuristic enrichment.")
+            return self._local_heuristic_fallback(article)
+
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        
+        prompt = """You are an article metadata enrichment service. Analyze the provided article title and summary and return ONLY valid JSON. Do not include any explanations or conversational text. Infer a single high-level category (Politics, Business, Technology, AI, Science, Health, Sports, Entertainment, Gaming, Culture, Education, Economy, Crime, Military, Weather, Environment, Travel, Lifestyle, Religion, Opinion, Local News, World), an optional subcategory, the primary region, country, and city if identifiable, 5–15 normalized tags, important keywords, named entities grouped into people, organizations, and locations, a sentiment (Positive, Negative, Neutral, Mixed), an importance score from 1 to 10 based on global significance, and an estimated readingTime in minutes. Do not invent facts that are not reasonably inferable from the title and summary. If information cannot be determined confidently, return null or an empty array. Return ONLY JSON matching this exact schema:
+{
+  "category": "String",
+  "subcategory": "String",
+  "region": "String",
+  "country": "String",
+  "city": "String",
+  "tags": [],
+  "keywords": [],
+  "people": [],
+  "organizations": [],
+  "locations": [],
+  "sentiment": "String",
+  "importance": 0,
+  "readingTime": 0
+}"""
+
+        # Try models in order until one succeeds
+        for model in self.fallback_models:
+            payload = {
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": prompt},
+                    {"role": "user", "content": f"Title: {article.title}\nSummary: {article.summary}"}
+                ],
+                "max_tokens": 1000,
+                "temperature": 0.1
+            }
+
+            try:
+                response = requests.post(self.api_url, headers=headers, json=payload, timeout=20)
+                if response.status_code == 200:
+                    result = response.json()
+                    content = result['choices'][0]['message']['content'].strip()
+                    
+                    if content.startswith("```json"):
+                        content = content[7:-3].strip()
+                    elif content.startswith("```"):
+                        content = content[3:-3].strip()
+                        
+                    data = json.loads(content)
+                    
+                    article.category = data.get('category') or article.category or "General"
+                    article.subcategory = data.get('subcategory')
+                    article.region = data.get('region')
+                    article.country = data.get('country')
+                    article.city = data.get('city')
+                    article.tags = data.get('tags') or []
+                    article.keywords = data.get('keywords') or []
+                    article.people = data.get('people') or []
+                    article.organizations = data.get('organizations') or []
+                    article.locations = data.get('locations') or []
+                    article.sentiment = data.get('sentiment') or "Neutral"
+                    article.importance = data.get('importance') or 5
+                    article.readingTime = data.get('readingTime') or self._estimate_reading_time(article)
+                    
+                    return article
+                else:
+                    logger.warning(f"HF Model {model} failed with status {response.status_code}. Trying next fallback...")
+            except Exception as e:
+                logger.warning(f"Exception trying model {model}: {e}. Trying next fallback...")
+
+        # If all AI models failed, try Tier 2: local NLP transformer models
+        logger.warning(f"All HF API models failed for article '{article.title[:50]}'. Trying Tier 2 local NLP models...")
+        if self._local_nlp.is_available():
+            try:
+                enriched = self._local_nlp.enrich_article(article)
+                logger.info("Tier 2 local NLP enrichment succeeded.")
+                return enriched
+            except Exception as e:
+                logger.warning(f"Tier 2 local NLP failed: {e}")
+
+        # Tier 3: If all else failed, use lexicon-based heuristic fallback so fields are never null
+        logger.error(f"All enrichment tiers failed for article '{article.title[:50]}'. Falling back to Tier 3 heuristics.")
+        return self._local_heuristic_fallback(article)
+
+    def _estimate_reading_time(self, article: Article) -> int:
+        text = f"{article.title} {article.summary or ''}"
+        # Strip simple HTML if present
+        text = re.sub(r'<[^>]+>', '', text)
+        words = len(text.split())
+        return max(1, round(words / 180))
+
+    def _local_heuristic_fallback(self, article: Article) -> Article:
+        """High-precision rule-based NLP fallback using modular lexicons."""
+        full_text = f"{article.title} {article.summary or ''}".lower()
+
+        # 1. Category & Subcategory Detection
+        if not article.category or article.category in ["General", "Middle East", "Egypt", "World", "North Africa"]:
+            for cat_name, (keywords, subcat) in CATEGORIES_MAP.items():
+                if any(kw in full_text for kw in keywords):
+                    article.category = cat_name
+                    article.subcategory = subcat
+                    break
+            if not article.category:
+                article.category = "General"
+
+        # 2. Entity & Location Extraction
+        detected_locations = []
+        for loc_kw, loc_name in LOCATIONS_MAP.items():
+            if loc_kw in full_text:
+                if loc_name not in detected_locations:
+                    detected_locations.append(loc_name)
+
+        article.locations = detected_locations if detected_locations else ["Global"]
+        if detected_locations:
+            article.country = detected_locations[0]
+            article.region = "Middle East" if any(l in MIDDLE_EAST_COUNTRIES for l in detected_locations) else "Global"
+
+        # 3. Clean Keyword & Tag Extraction
+        clean_words = [
+            re.sub(r'[^\w\s]', '', w) for w in article.title.split()
+            if len(w) > 3 and w.lower() not in STOP_WORDS
+        ]
+        
+        article.keywords = list(dict.fromkeys(clean_words[:6]))
+        article.tags = list(dict.fromkeys(clean_words[:10]))
+
+        # 4. Lexicon Sentiment Analysis
+        pos_count = sum(1 for w in POSITIVE_WORDS if w in full_text)
+        neg_count = sum(1 for w in NEGATIVE_WORDS if w in full_text)
+
+        if pos_count > neg_count:
+            article.sentiment = "Positive"
+        elif neg_count > pos_count:
+            article.sentiment = "Negative"
+        else:
+            article.sentiment = "Neutral"
+
+        # 5. Dynamic Importance Scoring (1 - 10)
+        high_impact_terms = ["عاجل", "قمة", "حرب", "وفاة", "رئيس", "كوارث", "breaking", "war", "president", "disaster", "crisis", "urgent"]
+        importance_score = 5
+        if any(term in full_text for term in high_impact_terms):
+            importance_score += 3
+        if len(detected_locations) > 1:
+            importance_score += 1
+        article.importance = min(10, importance_score)
+
+        # 6. Reading Time Estimation
+        article.readingTime = self._estimate_reading_time(article)
+
+        return article

services/duplicate_detector.py

@@ -0,0 +1,45 @@
+import logging
+from typing import List
+from domain.entities.article import Article
+from domain.repositories.article_repository import ArticleRepository
+
+logger = logging.getLogger(__name__)
+
+class DuplicateDetector:
+    def __init__(self, repository: ArticleRepository):
+        self.repository = repository
+
+    def filter_new_articles(self, articles: List[Article]) -> List[Article]:
+        """Robust multi-level duplicate detection (URL Hash, In-Batch Seen, Title Hash)."""
+        new_articles = []
+        seen_ids = set()
+        seen_title_hashes = set()
+        skipped_count = 0
+
+        for article in articles:
+            # 1. Normalize ID and Title Hash
+            article_id = article.id
+            title_hash = Article.generate_title_hash(article.title)
+
+            # 2. In-batch Duplicate Check (prevents duplicates inside the same scrape run)
+            if article_id in seen_ids or title_hash in seen_title_hashes:
+                logger.debug(f"In-batch duplicate detected: {article.title[:40]}")
+                skipped_count += 1
+                continue
+
+            # 3. Database Check (checks if article ID already exists in Firestore)
+            if self.repository.exists(article_id):
+                logger.debug(f"Article exists in Firestore: {article.title[:40]}")
+                skipped_count += 1
+                # Mark as seen so we don't check Firestore again if it reappears in feed
+                seen_ids.add(article_id)
+                seen_title_hashes.add(title_hash)
+                continue
+
+            # 4. Mark as unique and new
+            seen_ids.add(article_id)
+            seen_title_hashes.add(title_hash)
+            new_articles.append(article)
+
+        logger.info(f"Duplicate Detection complete: {len(new_articles)} new, {skipped_count} duplicates skipped.")
+        return new_articles

services/lexicons/__init__.py

@@ -0,0 +1,15 @@
+from .sentiment_lexicon import POSITIVE_WORDS, NEGATIVE_WORDS
+from .categories_lexicon import CATEGORIES_MAP
+from .locations_lexicon import LOCATIONS_MAP, MIDDLE_EAST_COUNTRIES
+from .stopwords_lexicon import STOP_WORDS
+from .images_lexicon import CATEGORY_IMAGES_MAP
+
+__all__ = [
+    "POSITIVE_WORDS",
+    "NEGATIVE_WORDS",
+    "CATEGORIES_MAP",
+    "LOCATIONS_MAP",
+    "MIDDLE_EAST_COUNTRIES",
+    "STOP_WORDS",
+    "CATEGORY_IMAGES_MAP"
+]

services/lexicons/categories_lexicon.py

@@ -0,0 +1,135 @@
+# Extensive Category and Subcategory Keyword Mappings for Arabic & English News NLP
+
+CATEGORIES_MAP = {
+    "Sports": (
+        [
+            "كرة", "مباراة", "دوري", "بطولة", "الأهلي", "الزمالك", "صلاح", "رياضة", "منتخب", "هداف",
+            "ملعب", "مدرب", "صفقة", "انتقالات", "كأس", "كرة قدم", "سلة", "كرة يد", "تنس", "أولمبياد",
+            "ريال مدريد", "برشلونة", "مانشستر", "ليفربول", "دوري أبطال", "كأس العالم", "رونالدو", "ميسي",
+            "football", "soccer", "match", "league", "cup", "nba", "tennis", "stadium", "coach",
+            "tournament", "player", "fifa", "uefa", "champion", "olympics", "nfl", "formula1", "f1",
+            "real madrid", "barcelona", "liverpool", "champions league", "world cup", "messi", "ronaldo"
+        ],
+        "Football"
+    ),
+    "Technology": (
+        [
+            "ذكاء اصطناعي", "تقنية", "هاتف", "تطبيق", "أبل", "سامسونج", "جوجل", "برمجيات", "روبوت",
+            "سحابة", "أمن سيبراني", "معالج", "آيفون", "أندرويد", "تحديث", "شاشة", "حاسوب", "ابتكار",
+            "شبكات", "5g", "رقائق", "سيليكون", "أنظمة", "خوارزمية", "شات جي بي تي", "إنترنت",
+            "ai", "artificial intelligence", "tech", "technology", "apple", "google", "software",
+            "cyber", "cybersecurity", "iphone", "android", "cloud", "robotics", "processor", "gadget",
+            "app", "application", "startup", "microsoft", "semiconductor", "chip", "chatgpt", "openai"
+        ],
+        "Tech News"
+    ),
+    "Gaming": (
+        [
+            "لعبة", "العاب", "ألعاب", "بلايستيشن", "إكس بوكس", "قيمينق", "جيمرز", "نينتندو", "ستيم",
+            "إي سبورتس", "مطور ألعاب", "تسريبات لعبة", "تحديث لعبة", "سوني", "مهمة", "بطولة العاب",
+            "ببجي", "فورتنايت", "كول اوف ديوتي", "فيفا 24", "العاب فيديو",
+            "game", "gaming", "playstation", "xbox", "nintendo", "esports", "steam", "gamer",
+            "ps5", "unreal engine", "graphics", "gameplay", "gta", "fortnite", "call of duty", "rpg", "roblox"
+        ],
+        "Gaming"
+    ),
+    "Business": (
+        [
+            "اقتصاد", "أسهم", "بورصة", "دولار", "تضخم", "شركات", "أرباح", "استثمار", "بنك", "مركزي",
+            "تجارة", "أسواق", "نفط", "غاز", "عملات", "عقارات", "استثمارات", "ميزانية", "صندوق النقد",
+            "فائدة", "بنك دولي", "استحواذ", "اندماج", "تداول", "ذهب", "سندات", "مشاريع",
+            "economy", "market", "markets", "stock", "stocks", "dollar", "inflation", "bank", "banking",
+            "finance", "financial", "investment", "investor", "revenue", "trade", "crypto", "bitcoin",
+            "real estate", "fed", "central bank", "gdp", "gold", "acquisition", "bonds"
+        ],
+        "Finance"
+    ),
+    "Health": (
+        [
+            "صحة", "مرض", "علاج", "طبيب", "أطباء", "مستشفى", "مستشفيات", "فيروس", "لقاح", "دواء",
+            "أدوية", "سرطان", "مناعة", "قلب", "حمية", "تغذية", "وباء", "منظمة الصحة", "عيادة",
+            "جراحة", "صحة نفسية", "أعراض", "تشخيص", "وقاية", "سمنة",
+            "health", "virus", "medical", "doctor", "hospital", "medicine", "vaccine", "cancer",
+            "disease", "epidemic", "pandemic", "wellness", "diet", "nutrition", "pharma", "who",
+            "surgery", "mental health", "symptoms", "diagnosis", "prevention"
+        ],
+        "Medical"
+    ),
+    "Science": (
+        [
+            "فضاء", "كوكب", "علماء", "مريخ", "ناسا", "بيئة", "مناخ", "أبحاث", "مجرة", "تلسكوب",
+            "احتباس حراري", "فيزياء", "كيمياء", "طبيعة", "احافير", "ذرة", "طاقة تجددة", "اكتشاف",
+            "علمي", "تجربة", "مختبر", "أحياء", "جينات", "وراثة",
+            "science", "space", "nasa", "planet", "climate", "research", "galaxy", "telescope",
+            "astronomy", "physics", "biology", "fossil", "atom", "renewable energy", "nature",
+            "discovery", "scientific", "laboratory", "genetics", "dna"
+        ],
+        "Space & Research"
+    ),
+    "Food": (
+        [
+            "وجبة", "طعام", "وصفة", "وصفات", "مطعم", "مطاعم", "طهي", "أكل", "مطبخ", "شيف", "عشاء",
+            "غداء", "إفطار", "مكونات", "طبق", "حلويات", "مخبوزات", "طبخ", "حلويات شرقية", "مشروبات",
+            "food", "recipe", "recipes", "cooking", "kitchen", "dish", "dishes", "restaurant",
+            "chef", "meal", "cuisine", "flavor", "bakery", "dessert", "dining", "flavor", "baking"
+        ],
+        "Culinary"
+    ),
+    "Entertainment": (
+        [
+            "فيلم", "أفلام", "مسلسل", "مسلسلات", "سينما", "موسيقى", "فنان", "فنانة", "مهرجان", "أغنية",
+            "هوليوود", "دراما", "كوميديا", "شباك التذاكر", "أوسكار", "عرض أول", "حفلة", "أغاني",
+            "تريند", "مشاهير", "نجوم", "بوستر", "تريلر", "برومو",
+            "movie", "movies", "film", "cinema", "music", "actor", "actress", "hollywood", "series",
+            "tv show", "box office", "oscars", "grammy", "concert", "celebrity", "drama", "trailer", "trend"
+        ],
+        "Movies & TV"
+    ),
+    "Politics": (
+        [
+            "رئيس", "وزير", "وزراء", "حكومة", "برلمان", "انتخابات", "سفارة", "سفير", "دبلوماسية",
+            "مفاوضات", "معاهدة", "مجلس الأمن", "الأمم المتحدة", "حزب", "سياسة", "خارجية", "دفاع",
+            "عسكري", "جيش", "قوات", "سياسي", "قمة", "مؤتمر صحفي", "سياسيون",
+            "president", "minister", "government", "parliament", "election", "elections", "diplomacy",
+            "diplomatic", "ambassador", "treaty", "un security council", "senate", "congress", "policy",
+            "military", "army", "forces", "summit", "geopolitics"
+        ],
+        "Government"
+    ),
+    "Education": (
+        [
+            "تعليم", "مدرسة", "مدارس", "جامعة", "جامعات", "طلاب", "طالب", "امتحانات", "امتحان",
+            "نتيجة", "تنسيق", "وزارة التربية والتعليم", "بكالوريا", "ثانوية عامة", "دراسة", "منحة",
+            "education", "school", "schools", "university", "student", "students", "exam", "exams",
+            "degree", "scholarship", "academy", "curriculum", "campus"
+        ],
+        "Education"
+    ),
+    "Crime": (
+        [
+            "جريمة", "جرائم", "شرطة", "قبض", "اعتقال", "مخدرات", "تحقيق", "نيابة", "محكمة", "قضاء",
+            "احتيال", "سرقة", "سطو", "قتل", "عصابة", "تهريب", "قاضي",
+            "crime", "police", "arrest", "arrested", "drugs", "investigation", "court", "judge",
+            "fraud", "theft", "robbery", "murder", "gang", "smuggling", "trial"
+        ],
+        "Crime & Justice"
+    ),
+    "Weather": (
+        [
+            "طقس", "مناخ", "حرارة", "أمطار", "مطر", "عاصفة", "عواصف", "ثلوج", "رياح", "أرصاد",
+            "موجة حارة", "إعصار", "سيول", "درجات الحرارة",
+            "weather", "forecast", "temperature", "rain", "storm", "snow", "wind", "hurricane",
+            "heatwave", "blizzard", "meteorology"
+        ],
+        "Weather & Climate"
+    ),
+    "Travel": (
+        [
+            "سفر", "سياحة", "فندق", "فنادق", "طيران", "رحلة", "رحلات", "وجهة", "تأشيرة", "فيزا",
+            "مطار", "معالم", "شاطئ", "منتجع",
+            "travel", "tourism", "hotel", "hotels", "flight", "flights", "destination", "visa",
+            "airport", "beach", "resort", "vacation"
+        ],
+        "Travel & Leisure"
+    )
+}

services/lexicons/images_lexicon.py

@@ -0,0 +1,21 @@
+# High-Resolution Category Fallback Cover Images (Royalty-Free Unsplash Assets)
+
+CATEGORY_IMAGES_MAP = {
+    "Technology": "https://images.unsplash.com/photo-1518770660439-4636190af475?auto=format&fit=crop&w=1000&q=80",
+    "Gaming": "https://images.unsplash.com/photo-1538481199705-c710c4e965fc?auto=format&fit=crop&w=1000&q=80",
+    "Sports": "https://images.unsplash.com/photo-1461896836934-ffe607ba8211?auto=format&fit=crop&w=1000&q=80",
+    "Business": "https://images.unsplash.com/photo-1611974789855-9c2a0a7236a3?auto=format&fit=crop&w=1000&q=80",
+    "Health": "https://images.unsplash.com/photo-1505751172876-fa1923c5c528?auto=format&fit=crop&w=1000&q=80",
+    "Science": "https://images.unsplash.com/photo-1451187580459-43490279c0fa?auto=format&fit=crop&w=1000&q=80",
+    "Food": "https://images.unsplash.com/photo-1504674900247-0877df9cc836?auto=format&fit=crop&w=1000&q=80",
+    "Entertainment": "https://images.unsplash.com/photo-1489599849927-2ee91cede3ba?auto=format&fit=crop&w=1000&q=80",
+    "Politics": "https://images.unsplash.com/photo-1541872703-74c5e44368f9?auto=format&fit=crop&w=1000&q=80",
+    "Education": "https://images.unsplash.com/photo-1523240795612-9a054b0db644?auto=format&fit=crop&w=1000&q=80",
+    "Crime": "https://images.unsplash.com/photo-1589829545856-d10d557cf95f?auto=format&fit=crop&w=1000&q=80",
+    "Weather": "https://images.unsplash.com/photo-1516912481808-3406841bd33c?auto=format&fit=crop&w=1000&q=80",
+    "Travel": "https://images.unsplash.com/photo-1488646953014-85cb44e25828?auto=format&fit=crop&w=1000&q=80",
+    "Middle East": "https://images.unsplash.com/photo-1512453979798-5ea266f8880c?auto=format&fit=crop&w=1000&q=80",
+    "Egypt": "https://images.unsplash.com/photo-1572252821128-d53b16d39d5e?auto=format&fit=crop&w=1000&q=80",
+    "World": "https://images.unsplash.com/photo-1526778548025-fa2f459cd5c1?auto=format&fit=crop&w=1000&q=80",
+    "General": "https://images.unsplash.com/photo-1504711434969-e33886168f5c?auto=format&fit=crop&w=1000&q=80"
+}

services/lexicons/locations_lexicon.py

@@ -0,0 +1,59 @@
+# Comprehensive Global and Middle East Geographic Gazetteer Mapping
+
+LOCATIONS_MAP = {
+    # Egypt (Governorates & Cities)
+    "مصر": "Egypt", "القاهرة": "Cairo", "الإسكندرية": "Alexandria", "الجيزة": "Giza",
+    "سيناء": "Sinai", "شرم الشيخ": "Sharm El Sheikh", "العلمين": "El Alamein", "أسوان": "Aswan",
+    "الأقصر": "Luxor", "مطروح": "Matrouh", "بورسعيد": "Port Said", "السويس": "Suez",
+    "طنطا": "Tanta", "المنصورة": "Mansoura", "الزقازيق": "Zagazig", "الإسماعيلية": "Ismailia",
+    "الفيوم": "Faiyum", "بني سويف": "Beni Suef", "المنيا": "Minya", "أسيوط": "Asyut",
+    "سوهاج": "Sohag", "قنا": "Qena", "الغردقة": "Hurghada", "دمياط": "Damietta",
+    "egypt": "Egypt", "cairo": "Cairo", "alexandria": "Alexandria", "giza": "Giza",
+
+    # Palestine & Levant
+    "غزة": "Gaza", "فلسطين": "Palestine", "القدس": "Jerusalem", "الضفة": "West Bank",
+    "رام الله": "Ramallah", "رفح": "Rafah", "خان يونس": "Khan Younis", "جنين": "Jenin",
+    "نابلس": "Nablus", "بيروت": "Beirut", "لبنان": "Lebanon", "دمشق": "Damascus", "حلب": "Aleppo",
+    "سوريا": "Syria", "عمان": "Amman", "الأردن": "Jordan", "إربد": "Irbid", "الزرعاء": "Zarqaa",
+    "gaza": "Gaza", "palestine": "Palestine", "jerusalem": "Jerusalem", "lebanon": "Lebanon",
+    "syria": "Syria", "jordan": "Jordan", "beirut": "Beirut", "damascus": "Damascus",
+
+    # Gulf & Arabian Peninsula
+    "السعودية": "Saudi Arabia", "الرياض": "Riyadh", "جدة": "Jeddah", "مكة": "Mekkah", "المدينة": "Madinah",
+    "الدمام": "Dammam", "الخبر": "Khobar", "الإمارات": "UAE", "دبي": "Dubai", "أبوظبي": "Abu Dhabi",
+    "الشارقة": "Sharjah", "قطر": "Qatar", "الدوحة": "Doha", "الكويت": "Kuwait", "البحرين": "Bahrain",
+    "المنامة": "Manama", "عمان": "Oman", "مسقط": "Muscat", "اليمن": "Yemen", "صنعاء": "Sanaa", "عدن": "Aden",
+    "saudi arabia": "Saudi Arabia", "riyadh": "Riyadh", "dubai": "Dubai", "uae": "UAE",
+    "qatar": "Qatar", "doha": "Doha", "kuwait": "Kuwait", "abu dhabi": "Abu Dhabi",
+
+    # North Africa
+    "المغرب": "Morocco", "الرباط": "Rabat", "الدار البيضاء": "Casablanca", "مراكش": "Marrakech",
+    "فاس": "Fes", "طنجة": "Tangier", "الجزائر": "Algeria", "وهران": "Oran", "قسنطينة": "Constantine",
+    "تونس": "Tunisia", "صفاقس": "Sfax", "ليبيا": "Libya", "طرابلس": "Tripoli", "بنغازي": "Benghazi",
+    "السودان": "Sudan", "الخرطوم": "Khartoum", "أم درمان": "Omdurman",
+    "morocco": "Morocco", "algeria": "Algeria", "tunisia": "Tunisia", "libya": "Libya", "sudan": "Sudan",
+
+    # Iraq
+    "العراق": "Iraq", "بغداد": "Baghdad", "أربيل": "Erbil", "البصرة": "Basra", "الموصل": "Mosul",
+    "iraq": "Iraq", "baghdad": "Baghdad",
+
+    # Global Powers & Major Capitals
+    "واشنطن": "Washington", "أمريكا": "USA", "الولايات المتحدة": "USA", "نيويورك": "New York",
+    "شيكاغو": "Chicago", "كاليفورنيا": "California", "تكساس": "Texas", "فلوريدا": "Florida",
+    "باريس": "Paris", "فرنسا": "France", "مارسيليا": "Marseille", "لندن": "London", "بريطانيا": "UK",
+    "إنجلترا": "UK", "مانشستر": "Manchester", "برلين": "Berlin", "ألمانيا": "Germany", "ميونيخ": "Munich",
+    "بكين": "Beijing", "الصين": "China", "شنغهاي": "Shanghai", "موسكو": "Moscow", "روسيا": "Russia",
+    "طوكيو": "Tokyo", "اليابان": "Japan", "روما": "Rome", "إيطاليا": "Italy", "ميلانو": "Milan",
+    "مدريد": "Madrid", "إسبانيا": "Spain", "برشلونة": "Barcelona", "أنقرة": "Ankara", "تركيا": "Turkey",
+    "إسطنبول": "Istanbul", "كييف": "Kyiv", "أوكرانيا": "Ukraine", "سول": "Seoul", "كوريا": "Korea",
+    "نيودلهي": "New Delhi", "الهند": "India", "برازيليا": "Brasilia", "البرازيل": "Brazil",
+    "سدني": "Sydney", "أستراليا": "Australia", "تورونتو": "Toronto", "كندا": "Canada",
+    "usa": "USA", "us": "USA", "uk": "UK", "china": "China", "russia": "Russia", "france": "France",
+    "germany": "Germany", "japan": "Japan", "turkey": "Turkey", "ukraine": "Ukraine", "canada": "Canada"
+}
+
+MIDDLE_EAST_COUNTRIES = {
+    "Egypt", "Gaza", "Palestine", "Saudi Arabia", "UAE", "Qatar", "Kuwait", "Lebanon",
+    "Syria", "Jordan", "Iraq", "Yemen", "Oman", "Bahrain", "Sudan", "Libya", "Algeria",
+    "Morocco", "Tunisia"
+}

services/lexicons/sentiment_lexicon.py

@@ -0,0 +1,46 @@
+# Extensive sentiment lexicons for Arabic and English NLP classification
+
+POSITIVE_WORDS = {
+    # Arabic Positive Cues
+    "فوز", "نجاح", "انتصار", "تألق", "تفوق", "نمو", "ازدهار", "أرباح", "ربح", "استقرار",
+    "اتفاق", "إنجاز", "افتتاح", "مكاسب", "تبرع", "تبرعات", "مساعدة", "حليف", "صعود", "انتعاش",
+    "ابتكار", "تطوير", "حسم", "ريادة", "امتياز", "سلام", "تعافي", "تسهيل", "دعم", "شراكة",
+    "تأهل", "ذهبية", "فضية", "برونزية", "ترقية", "جائزة", "تكريم", "مبادرة", "هدنة", "إيجابي",
+    "تقدم", "اعتماد", "موافقة", "تسهيلات", "شفاء", "علاج", "بشرى", "نجاحات", "تفاؤل", "فرحة",
+    "مكافأة", "تحسين", "توسيع", "إنقاذ", "صفقة", "عقد", "استثمار", "تسهيل", "إشادة", "ثراء",
+    "خير", "بركة", "بهجة", "سعادة", "انتصارات", "مكتسبات", "تكاتف", "تضامن", "مرونة", "تأهيل",
+    "إعفاء", "تسوية", "تسامح", "إنعاش", "بشائر", "طفرة", "تنمية", "إعادة إعمار", "جاهزية",
+
+    # English Positive Cues
+    "win", "winning", "winner", "victory", "triumph", "success", "successful", "growth", "grow",
+    "profit", "profitable", "boom", "booming", "recovery", "recover", "peace", "agreement",
+    "breakthrough", "milestone", "boost", "boosting", "gain", "gains", "innovation", "innovative",
+    "positive", "flourish", "thrive", "thriving", "achievement", "award", "awarded", "hero",
+    "heroic", "stability", "stable", "champion", "champions", "championship", "progress",
+    "partnership", "soar", "soaring", "upgrade", "advance", "advancement", "relief", "cure",
+    "prosper", "prosperity", "benefit", "beneficial", "glory", "glorious", "praise", "praised",
+    "accord", "reconciliation", "revival", "revive", "opportunity", "record", "surge", "surging",
+    "optimism", "optimistic", "solidarity", "resilience", "fund", "funded", "grant", "granted"
+}
+
+NEGATIVE_WORDS = {
+    # Arabic Negative Cues
+    "مقتل", "قتيل", "وفاة", "وفيات", "أزمة", "أزمات", "تراجع", "انخفاض", "خسارة", "خسائر",
+    "انهيار", "جريمة", "جرائم", "حادث", "حوادث", "كارثة", "كوارث", "تحذير", "عقوبات", "إفلاس",
+    "تهديد", "خيانة", "سرقة", "قلق", "زلزال", "فيضان", "حريق", "حرائق", "قتلى", "جرحى",
+    "مصابين", "إصابة", "حرب", "حروب", "قصف", "هجوم", "انفجار", "انفجارات", "توتر", "مأساة",
+    "ضحايا", "نزاع", "اعتقال", "اشتباكات", "شهداء", "عنف", "دمار", "تدمير", "مغادرة", "احتجاجات",
+    "نزوح", "لاجئين", "فقر", "مجاعة", "غلاء", "ركود", "إغلاق", "إلغاء", "إضراب", "اتهام",
+    "ادانة", "إدانة", "خراب", "سقوط", "نزيف", "تدهور", "مخاطر", "خطر", "فشل", "اضطراب",
+    "فضيحة", "فساد", "اختلاس", "احتيال", "مداهمة", "اعتقالات", "حصار", "احتلال", "قمع",
+
+    # English Negative Cues
+    "death", "dead", "died", "kill", "killed", "killing", "murder", "attack", "attacks", "war",
+    "warfare", "crisis", "crash", "crashed", "decline", "declining", "loss", "losses", "collapse",
+    "crime", "criminal", "warning", "warn", "threat", "threatening", "disaster", "disastrous",
+    "accident", "explosion", "explosions", "sanction", "sanctions", "inflation", "tragedy",
+    "tragic", "casualty", "casualties", "scandal", "fatal", "fatality", "strike", "strikes",
+    "conflict", "violence", "violent", "injury", "injured", "ruin", "ruined", "protest", "riot",
+    "poverty", "famine", "recession", "cancel", "cancelled", "fraud", "corruption", "arrest",
+    "arrested", "siege", "occupation", "devastation", "devastating", "slump", "downfall", "fear"
+}

services/lexicons/stopwords_lexicon.py

@@ -0,0 +1,20 @@
+# Comprehensive Arabic & English Stopwords for NLP Tokenization & Tagging
+
+STOP_WORDS = {
+    # Arabic Stopwords
+    "في", "من", "على", "عن", "أن", "إن", "إلى", "مع", "هذا", "هذه", "تم", "بعد", "قبل",
+    "كان", "كانت", "يكون", "تكون", "أو", "أم", "ثم", "حتى", "التي", "الذي", "الذين",
+    "اللذين", "اللتين", "بين", "حول", "ضد", "نحو", "كل", "جميع", "بعض", "غير", "ذات",
+    "ما", "ماذا", "منذ", "خلال", "عند", "أمام", "خلف", "تحت", "فوق", "كما", "لكن",
+    "غير", "أيضا", "فقط", "جدا", "قد", "لقد", "حيث", "إذا", "سوف", "سيكون", "اليوم",
+
+    # English Stopwords
+    "the", "a", "an", "in", "on", "at", "to", "for", "of", "with", "and", "is", "are",
+    "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did",
+    "but", "or", "because", "as", "until", "while", "about", "against", "between",
+    "into", "through", "during", "before", "after", "above", "below", "from", "up",
+    "down", "in", "out", "over", "under", "again", "further", "then", "once", "here",
+    "there", "when", "where", "why", "how", "all", "any", "both", "each", "few", "more",
+    "most", "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so",
+    "than", "too", "very", "s", "t", "can", "will", "just", "don", "should", "now"
+}

services/local_nlp_classifier.py

@@ -0,0 +1,399 @@
+"""
+Tier 2 Local NLP Classifier — Pre-trained Transformer Models.
+
+This module provides near-LLM-quality enrichment using locally-downloaded
+HuggingFace transformer models. It sits BETWEEN the HuggingFace API (Tier 1)
+and the lexicon-based heuristics (Tier 3) in the fallback chain.
+
+Models used:
+  - Category:  facebook/bart-large-mnli (zero-shot classification, multilingual)
+  - Sentiment: CAMeL-Lab/bert-base-arabic-camelbert-msa-sentiment (Arabic)
+               distilbert-base-uncased-finetuned-sst-2-english (English)
+  - NER:       CAMeL-Lab/bert-base-arabic-camelbert-msa-ner (Arabic)
+               dslim/bert-base-NER (English)
+  - Keywords:  KeyBERT with paraphrase-multilingual-MiniLM-L12-v2
+"""
+
+import re
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Lazy-loaded singletons — models are heavy, we only load them once and only when needed.
+_category_pipeline = None
+_sentiment_ar_pipeline = None
+_sentiment_en_pipeline = None
+_ner_ar_pipeline = None
+_ner_en_pipeline = None
+_keyword_model = None
+_models_available = None  # None = not checked yet, True/False after first attempt
+
+
+# ---------------------------------------------------------------------------
+# Our 13 categories (must match CATEGORIES_MAP keys in categories_lexicon.py)
+# ---------------------------------------------------------------------------
+CATEGORY_LABELS = [
+    "Sports", "Technology", "Gaming", "Business", "Health", "Science",
+    "Food", "Entertainment", "Politics", "Education", "Crime", "Weather", "Travel"
+]
+
+# ---------------------------------------------------------------------------
+# Region inference from NER-detected locations
+# ---------------------------------------------------------------------------
+from services.lexicons import MIDDLE_EAST_COUNTRIES, LOCATIONS_MAP
+
+
+def _check_dependencies() -> bool:
+    """Check if transformers & torch are installed."""
+    global _models_available
+    if _models_available is not None:
+        return _models_available
+    try:
+        import transformers  # noqa: F401
+        import torch  # noqa: F401
+        _models_available = True
+        logger.info("NLP model dependencies (transformers, torch) are available.")
+    except ImportError as e:
+        _models_available = False
+        logger.warning(f"NLP model dependencies not installed ({e}). Tier 2 fallback disabled.")
+    return _models_available
+
+
+# ========================= MODEL LOADERS =========================
+
+def _get_category_pipeline():
+    """Zero-shot classification using BART-MNLI. Handles Arabic & English."""
+    global _category_pipeline
+    if _category_pipeline is not None:
+        return _category_pipeline
+    try:
+        from transformers import pipeline
+        _category_pipeline = pipeline(
+            "zero-shot-classification",
+            model="facebook/bart-large-mnli",
+            device=-1  # CPU only (GitHub Actions has no GPU)
+        )
+        logger.info("Loaded zero-shot category classifier (facebook/bart-large-mnli)")
+    except Exception as e:
+        logger.warning(f"Failed to load category classifier: {e}")
+        _category_pipeline = None
+    return _category_pipeline
+
+
+def _get_sentiment_pipeline(language: str):
+    """Language-specific sentiment pipeline."""
+    global _sentiment_ar_pipeline, _sentiment_en_pipeline
+
+    if language == "ar":
+        if _sentiment_ar_pipeline is not None:
+            return _sentiment_ar_pipeline
+        try:
+            from transformers import pipeline
+            _sentiment_ar_pipeline = pipeline(
+                "text-classification",
+                model="CAMeL-Lab/bert-base-arabic-camelbert-msa-sentiment",
+                device=-1
+            )
+            logger.info("Loaded Arabic sentiment model (CAMeL-Lab/camelbert-msa-sentiment)")
+        except Exception as e:
+            logger.warning(f"Failed to load Arabic sentiment model: {e}")
+            _sentiment_ar_pipeline = None
+        return _sentiment_ar_pipeline
+    else:
+        if _sentiment_en_pipeline is not None:
+            return _sentiment_en_pipeline
+        try:
+            from transformers import pipeline
+            _sentiment_en_pipeline = pipeline(
+                "text-classification",
+                model="distilbert-base-uncased-finetuned-sst-2-english",
+                device=-1
+            )
+            logger.info("Loaded English sentiment model (distilbert-sst-2)")
+        except Exception as e:
+            logger.warning(f"Failed to load English sentiment model: {e}")
+            _sentiment_en_pipeline = None
+        return _sentiment_en_pipeline
+
+
+def _get_ner_pipeline(language: str):
+    """Language-specific NER pipeline."""
+    global _ner_ar_pipeline, _ner_en_pipeline
+
+    if language == "ar":
+        if _ner_ar_pipeline is not None:
+            return _ner_ar_pipeline
+        try:
+            from transformers import pipeline
+            _ner_ar_pipeline = pipeline(
+                "ner",
+                model="CAMeL-Lab/bert-base-arabic-camelbert-msa-ner",
+                aggregation_strategy="simple",
+                device=-1
+            )
+            logger.info("Loaded Arabic NER model (CAMeL-Lab/camelbert-msa-ner)")
+        except Exception as e:
+            logger.warning(f"Failed to load Arabic NER model: {e}")
+            _ner_ar_pipeline = None
+        return _ner_ar_pipeline
+    else:
+        if _ner_en_pipeline is not None:
+            return _ner_en_pipeline
+        try:
+            from transformers import pipeline
+            _ner_en_pipeline = pipeline(
+                "ner",
+                model="dslim/bert-base-NER",
+                aggregation_strategy="simple",
+                device=-1
+            )
+            logger.info("Loaded English NER model (dslim/bert-base-NER)")
+        except Exception as e:
+            logger.warning(f"Failed to load English NER model: {e}")
+            _ner_en_pipeline = None
+        return _ner_en_pipeline
+
+
+def _get_keyword_model():
+    """KeyBERT with multilingual sentence-transformer embeddings."""
+    global _keyword_model
+    if _keyword_model is not None:
+        return _keyword_model
+    try:
+        from keybert import KeyBERT
+        _keyword_model = KeyBERT(model="paraphrase-multilingual-MiniLM-L12-v2")
+        logger.info("Loaded KeyBERT keyword extractor (multilingual-MiniLM)")
+    except Exception as e:
+        logger.warning(f"Failed to load KeyBERT: {e}")
+        _keyword_model = None
+    return _keyword_model
+
+
+# ========================= ENRICHMENT =========================
+
+class LocalNLPClassifier:
+    """
+    Tier 2 NLP enrichment engine using pre-trained transformer models.
+
+    Usage:
+        classifier = LocalNLPClassifier()
+        if classifier.is_available():
+            enriched_article = classifier.enrich_article(article)
+    """
+
+    def is_available(self) -> bool:
+        """Check if the required ML libraries are installed."""
+        return _check_dependencies()
+
+    def enrich_article(self, article) -> "Article":
+        """
+        Enrich a single article using local transformer models.
+        Each sub-task is wrapped in try/except so a failure in one
+        (e.g. sentiment) does not block the others (e.g. NER).
+        """
+        text = f"{article.title} {article.summary or ''}"
+        lang = getattr(article, 'language', 'en')
+
+        # 1. Category (zero-shot classification)
+        try:
+            self._classify_category(article, text)
+        except Exception as e:
+            logger.warning(f"Tier 2 category classification failed: {e}")
+
+        # 2. Sentiment
+        try:
+            self._classify_sentiment(article, text, lang)
+        except Exception as e:
+            logger.warning(f"Tier 2 sentiment analysis failed: {e}")
+
+        # 3. Named Entity Recognition -> people, organizations, locations, country, region
+        try:
+            self._extract_entities(article, text, lang)
+        except Exception as e:
+            logger.warning(f"Tier 2 NER failed: {e}")
+
+        # 4. Keywords & Tags
+        try:
+            self._extract_keywords(article, text)
+        except Exception as e:
+            logger.warning(f"Tier 2 keyword extraction failed: {e}")
+
+        # 5. Importance scoring (derived from NER entity density + category)
+        try:
+            self._score_importance(article, text)
+        except Exception as e:
+            logger.warning(f"Tier 2 importance scoring failed: {e}")
+
+        # 6. Reading time estimation
+        if not article.readingTime:
+            article.readingTime = self._estimate_reading_time(text)
+
+        return article
+
+    # ---------------------------------------------------------------
+    # Sub-task implementations
+    # ---------------------------------------------------------------
+
+    def _classify_category(self, article, text: str):
+        """Zero-shot classification into our 13 news categories."""
+        if article.category and article.category not in ("General", "Middle East", "Egypt", "World", "North Africa"):
+            return  # Already classified by the RSS source metadata
+
+        pipe = _get_category_pipeline()
+        if pipe is None:
+            return
+
+        # Truncate to 512 tokens for BART
+        result = pipe(text[:1024], candidate_labels=CATEGORY_LABELS, multi_label=False)
+        if result and result['scores'][0] > 0.25:
+            article.category = result['labels'][0]
+            # Subcategory: use the second-highest label if confident
+            if len(result['labels']) > 1 and result['scores'][1] > 0.20:
+                article.subcategory = result['labels'][1]
+        else:
+            article.category = article.category or "General"
+
+    def _classify_sentiment(self, article, text: str, lang: str):
+        """Sentiment analysis using language-specific pre-trained models."""
+        pipe = _get_sentiment_pipeline(lang)
+        if pipe is None:
+            return
+
+        result = pipe(text[:512])
+        if not result:
+            return
+
+        label = result[0]['label'].upper()
+        score = result[0]['score']
+
+        if lang == "ar":
+            # CAMeLBERT sentiment outputs: positive, negative, neutral
+            label_map = {"POSITIVE": "Positive", "NEGATIVE": "Negative", "NEUTRAL": "Neutral"}
+            article.sentiment = label_map.get(label, "Neutral")
+        else:
+            # DistilBERT SST-2 outputs: POSITIVE, NEGATIVE
+            if label == "POSITIVE":
+                article.sentiment = "Positive"
+            elif label == "NEGATIVE":
+                article.sentiment = "Negative"
+            else:
+                article.sentiment = "Neutral"
+
+            # Mixed sentiment: if confidence is low (close to 50/50)
+            if 0.45 < score < 0.60:
+                article.sentiment = "Mixed"
+
+    def _extract_entities(self, article, text: str, lang: str):
+        """NER extraction: people, organizations, locations."""
+        pipe = _get_ner_pipeline(lang)
+        if pipe is None:
+            return
+
+        entities = pipe(text[:512])
+        if not entities:
+            return
+
+        people = []
+        organizations = []
+        locations = []
+
+        for ent in entities:
+            entity_group = ent.get('entity_group', '')
+            word = ent.get('word', '').strip()
+            score = ent.get('score', 0)
+
+            # Filter low-confidence entities
+            if score < 0.5 or len(word) < 2:
+                continue
+
+            # Clean up subword tokens (##prefix)
+            word = word.replace('##', '')
+
+            if entity_group in ('PER', 'B-PER', 'I-PER'):
+                if word not in people:
+                    people.append(word)
+            elif entity_group in ('ORG', 'B-ORG', 'I-ORG'):
+                if word not in organizations:
+                    organizations.append(word)
+            elif entity_group in ('LOC', 'B-LOC', 'I-LOC', 'GPE', 'B-GPE', 'I-GPE'):
+                if word not in locations:
+                    locations.append(word)
+
+        article.people = people[:10]       # Cap at 10
+        article.organizations = organizations[:10]
+        article.locations = locations[:10] if locations else article.locations
+
+        # Derive country and region from detected locations
+        if locations:
+            # Try to map NER location names to our gazetteer
+            for loc in locations:
+                loc_lower = loc.lower()
+                if loc_lower in LOCATIONS_MAP:
+                    article.country = LOCATIONS_MAP[loc_lower]
+                    article.region = "Middle East" if article.country in MIDDLE_EAST_COUNTRIES else "Global"
+                    break
+            if not article.country:
+                article.country = locations[0]
+                article.region = "Global"
+
+    def _extract_keywords(self, article, text: str):
+        """KeyBERT keyword extraction with multilingual embeddings."""
+        kw_model = _get_keyword_model()
+        if kw_model is None:
+            return
+
+        try:
+            keywords = kw_model.extract_keywords(
+                text,
+                keyphrase_ngram_range=(1, 2),
+                stop_words=None,  # KeyBERT handles multilingual stop words
+                top_n=12,
+                use_maxsum=True,
+                nr_candidates=20
+            )
+        except Exception:
+            return
+
+        if keywords:
+            article.keywords = [kw[0] for kw in keywords[:6]]
+            article.tags = [kw[0] for kw in keywords[:10]]
+
+    def _score_importance(self, article, text: str):
+        """
+        Importance scoring (1-10) based on:
+        - Number of named entities (more entities = more important)
+        - High-impact trigger words
+        - Category weight
+        """
+        score = 5  # Base importance
+
+        # Named entity density bonus
+        entity_count = len(article.people or []) + len(article.organizations or []) + len(article.locations or [])
+        if entity_count >= 5:
+            score += 2
+        elif entity_count >= 2:
+            score += 1
+
+        # High-impact trigger words
+        high_impact_terms = [
+            "عاجل", "قمة", "حرب", "وفاة", "رئيس", "كوارث", "اغتيال", "انقلاب", "زلزال",
+            "breaking", "war", "president", "disaster", "crisis", "urgent", "killed",
+            "earthquake", "assassination", "coup", "summit", "death"
+        ]
+        text_lower = text.lower()
+        if any(term in text_lower for term in high_impact_terms):
+            score += 3
+
+        # Category bonus
+        high_importance_categories = {"Politics", "Crime", "Health", "Science"}
+        if article.category in high_importance_categories:
+            score += 1
+
+        article.importance = min(10, max(1, score))
+
+    def _estimate_reading_time(self, text: str) -> int:
+        """Estimate reading time in minutes from word count."""
+        clean_text = re.sub(r'<[^>]+>', '', text)
+        words = len(clean_text.split())
+        return max(1, round(words / 180))