Discover RSS Feeds From Any Website in Python

import requests
import re
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup

def discover_rss_feeds(url):
    """Discover all RSS/Atom feeds linked from a given website."""
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (compatible; RSSDiscovery/1.0)'}
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
    except requests.RequestException as e:
        print(f"Error fetching {url}: {e}")
        return []

    soup = BeautifulSoup(response.text, 'html.parser')
    feeds = set()

    # Find <link> tags with RSS/Atom types
    for link in soup.find_all('link', type=re.compile(r'application/(rss|atom)\+xml', re.I)):
        href = link.get('href')
        if href:
            feeds.add(urljoin(url, href))

    # Find <a> tags linking to .rss or .xml or containing/feed in URL
    for a in soup.find_all('a', href=True):
        href = a['href']
        if re.search(r'\.(rss|xml)$', href, re.I) or '/feed' in href.lower():
            full_url = urljoin(url, href)
            if urlparse(full_url).netloc == urlparse(url).netloc:
                feeds.add(full_url)

    return sorted(feeds)

if __name__ == "__main__":
    # Example usage
    website = "https://news.ycombinator.com"
    feeds = discover_rss_feeds(website)
    if feeds:
        print(f"Found {len(feeds)} feed(s) on {website}:")
        for feed in feeds:
            print(f"  {feed}")
    else:
        print(f"No feeds discovered on {website}")