Advanced Newspaper Scraping with Python: Complete Guide 2025

# Advanced newspaper scraping with Newspaper3k
from newspaper import Article
import requests
from datetime import datetime
import json

class NewsArticleScraper:
    def __init__(self, proxies=None):
        self.proxies = proxies or {}
        self.session = requests.Session()
        if proxies:
            self.session.proxies.update(proxies)
    
    def scrape_article(self, url, config=None):
        """
        Scrape a single news article with comprehensive data extraction
        """
        try:
            # Initialize article with custom configuration
            article = Article(url, config=config)
            
            # Set session for proxy support
            if self.proxies:
                article.set_config(proxies=self.proxies)
            
            # Download and parse article
            article.download()
            article.parse()
            
            # Optional: Run NLP processing
            article.nlp()
            
            return {
                'url': url,
                'title': article.title,
                'authors': article.authors,
                'publish_date': article.publish_date.isoformat() if article.publish_date else None,
                'text': article.text,
                'summary': article.summary,
                'keywords': article.keywords,
                'top_image': article.top_image,
                'images': list(article.images),
                'videos': list(article.movies),
                'meta_description': article.meta_description,
                'meta_keywords': article.meta_keywords,
                'canonical_link': article.canonical_link,
                'scraped_at': datetime.now().isoformat()
            }
            
        except Exception as e:
            return {
                'url': url,
                'error': str(e),
                'scraped_at': datetime.now().isoformat()
            }
    
    def scrape_multiple_articles(self, urls, max_workers=5):
        """
        Scrape multiple articles concurrently
        """
        from concurrent.futures import ThreadPoolExecutor
        
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            results = list(executor.map(self.scrape_article, urls))
        
        return results

# Example usage with mobile proxies
if __name__ == "__main__":
    # Configure mobile proxy (get from Coronium dashboard)
    mobile_proxies = {
        'http': 'http://username:password@proxy.coronium.io:8080',
        'https': 'http://username:password@proxy.coronium.io:8080'
    }
    
    # Initialize scraper
    scraper = NewsArticleScraper(proxies=mobile_proxies)
    
    # Scrape article
    article_data = scraper.scrape_article('https://example-news-site.com/article')
    
    # Print results
    print(json.dumps(article_data, indent=2, ensure_ascii=False))