import logging from scrapers.rss_scraper import RssScraper from data.repositories.article_repository_impl import FirestoreArticleRepository from services.ai_classifier import HuggingFaceClassifier from services.duplicate_detector import DuplicateDetector from domain.usecases.scrape_and_enrich import ScrapeAndEnrichUseCase logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) # Configure our RSS sources across diverse categories (General, Tech, Gaming, Food, Sports, Business, Science, Entertainment) SOURCES = [ # --- World & Middle East News (English) --- RssScraper("BBC Middle East", "en", "http://feeds.bbci.co.uk/news/world/middle_east/rss.xml", category="Middle East"), RssScraper("BBC World", "en", "http://feeds.bbci.co.uk/news/world/rss.xml", category="World"), RssScraper("Middle East Eye", "en", "https://www.middleeasteye.net/rss", category="Middle East"), RssScraper("Middle East Monitor", "en", "https://www.middleeastmonitor.com/feed/", category="Middle East"), RssScraper("The Guardian Middle East", "en", "https://www.theguardian.com/world/middleeast/rss", category="Middle East"), RssScraper("NY Times Middle East", "en", "https://rss.nytimes.com/services/xml/rss/nyt/MiddleEast.xml", category="Middle East"), RssScraper("Al Jazeera English", "en", "https://www.aljazeera.com/xml/rss/all.xml", category="World"), # --- Egyptian & Arab World News (Arabic) --- RssScraper("Youm7", "ar", "http://www.youm7.com/rss/SectionRss?SectionID=65", category="Egypt"), RssScraper("Al-Ahram", "ar", "https://gate.ahram.org.eg/NewsRss.aspx", category="Egypt"), RssScraper("Al Masry Al Youm", "ar", "https://www.almasryalyoum.com/rss/rss", category="Egypt"), RssScraper("Sada El Balad", "ar", "https://www.elbalad.news/rss.aspx", category="Egypt"), RssScraper("Al Shorouk", "ar", "https://www.shorouknews.com/rss/news.aspx", category="Egypt"), RssScraper("Al-Dustour", "ar", "https://www.dostor.org/rss.aspx", category="Egypt"), RssScraper("Sky News Arabia", "ar", "https://www.skynewsarabia.com/rss", category="Middle East"), RssScraper("RT Arabic", "ar", "https://arabic.rt.com/rss/", category="Middle East"), RssScraper("Asharq Al-Awsat", "ar", "https://aawsat.com/feed", category="Middle East"), RssScraper("Alhurra", "ar", "https://www.alhurra.com/rss", category="Middle East"), RssScraper("BBC Arabic", "ar", "http://feeds.bbci.co.uk/arabic/rss.xml", category="Middle East"), RssScraper("CNN Arabic", "ar", "https://arabic.cnn.com/api/v1/rss/middle_east/rss.xml", category="Middle East"), RssScraper("France 24 Arabic", "ar", "https://www.france24.com/ar/rss", category="Middle East"), RssScraper("Hespress", "ar", "https://www.hespress.com/feed", category="North Africa"), RssScraper("Echorouk", "ar", "https://www.echoroukonline.com/feed", category="North Africa"), # --- Technology & AI --- RssScraper("TechCrunch", "en", "https://techcrunch.com/feed/", category="Technology"), RssScraper("Wired", "en", "https://www.wired.com/feed/rss", category="Technology"), RssScraper("The Verge", "en", "https://www.theverge.com/rss/index.xml", category="Technology"), RssScraper("Ars Technica", "en", "https://feeds.arstechnica.com/arstechnica/index", category="Technology"), RssScraper("Engadget", "en", "https://www.engadget.com/rss.xml", category="Technology"), RssScraper("Aitnews", "ar", "https://aitnews.com/feed/", category="Technology"), RssScraper("Unlimit Tech", "ar", "https://unlimit-tech.com/feed/", category="Technology"), # --- Gaming & Esports --- RssScraper("IGN", "en", "https://feeds.feedburner.com/ign/all", category="Gaming"), RssScraper("Eurogamer", "en", "https://www.eurogamer.net/?format=rss", category="Gaming"), RssScraper("GameSpot", "en", "https://www.gamespot.com/feeds/news/", category="Gaming"), RssScraper("Polygon", "en", "https://www.polygon.com/rss/index.xml", category="Gaming"), RssScraper("Saudi Gamer", "ar", "https://saudigamer.com/feed/", category="Gaming"), RssScraper("TrueGaming", "ar", "https://www.true-gaming.net/home/feed/", category="Gaming"), # --- Food, Cooking & Lifestyle --- RssScraper("Serious Eats", "en", "https://www.seriouseats.com/feed", category="Food"), RssScraper("BBC Good Food", "en", "https://www.bbcgoodfood.com/feed/rss2", category="Food"), RssScraper("Food52", "en", "https://food52.com/blog.rss", category="Food"), RssScraper("SuperMama", "ar", "https://www.supermama.me/rss.xml", category="Lifestyle"), # --- Business, Economy & Finance --- RssScraper("CNBC World", "en", "https://www.cnbc.com/id/100727362/device/rss/rss.html", category="Business"), RssScraper("Financial Times", "en", "https://www.ft.com/?format=rss", category="Business"), RssScraper("RT Arabic Business", "ar", "https://arabic.rt.com/rss/business/", category="Business"), RssScraper("Al Mal News", "ar", "https://almalnews.com/feed/", category="Business"), # --- Sports --- RssScraper("BBC Sport", "en", "http://feeds.bbci.co.uk/sport/rss.xml", category="Sports"), RssScraper("RT Arabic Sports", "ar", "https://arabic.rt.com/rss/sport/", category="Sports"), RssScraper("FilGoal", "ar", "https://www.filgoal.com/rss/news", category="Sports"), # --- Science & Space --- RssScraper("ScienceDaily", "en", "https://www.sciencedaily.com/rss/all.xml", category="Science"), RssScraper("NASA News", "en", "https://www.nasa.gov/news-release/feed/", category="Science"), RssScraper("RT Arabic Science", "ar", "https://arabic.rt.com/rss/space/", category="Science"), # --- Entertainment & Culture --- RssScraper("Variety", "en", "https://variety.com/feed/", category="Entertainment"), RssScraper("Hollywood Reporter", "en", "https://www.hollywoodreporter.com/feed/", category="Entertainment") ] def job(): logger.info("Starting scraping job with AI Enrichment...") # 1. Initialize dependencies try: repo = FirestoreArticleRepository(service_account_path="service-account.json") except Exception as e: logger.error(f"Failed to initialize Firestore: {e}") return if not repo.db: logger.error("Skipping job because Firestore is not initialized. Please ensure service-account.json exists.") return classifier = HuggingFaceClassifier() duplicate_detector = DuplicateDetector(repository=repo) # 2. Setup Usecase usecase = ScrapeAndEnrichUseCase( repository=repo, scrapers=SOURCES, classifier=classifier, duplicate_detector=duplicate_detector ) # 3. Execute Pipeline new_articles = usecase.execute() # 4. Send Notifications if new_articles: try: from firebase_admin import messaging top_article = new_articles[0] message = messaging.Message( notification=messaging.Notification( title=f"New story from {top_article.source}", body=top_article.title ), topic='daily_news' ) response = messaging.send(message) logger.info(f"Successfully sent FCM notification: {response}") except Exception as e: logger.error(f"Failed to send FCM notification: {e}") else: logger.info("No new articles processed.") if __name__ == "__main__": logger.info("News Intelligence Engine started.") job() logger.info("Scraping job finished.")