Build a Complete Website Sitemap Generator Without External Services

import json
from urllib.parse import urlparse, urljoin
from collections import deque
import urllib.request
import urllib.error
import re
from html.parser import HTMLParser

class SitemapParser(HTMLParser):
    def __init__(self, base_url):
        super().__init__()
        self.base_url = base_url
        self.links = set()
        
    def handle_starttag(self, tag, attrs):
        if tag == 'a':
            attrs_dict = dict(attrs)
            href = attrs_dict.get('href')
            if href:
                full_url = urljoin(self.base_url, href)
                parsed = urlparse(full_url)
                if parsed.netloc and parsed.scheme in ('http', 'https'):
                    self.links.add(full_url)

def crawl_website(start_url: str, max_pages: int = 50) -> dict:
    visited = set()
    queue = deque([start_url])
    sitemap = {}
    
    while queue and len(visited) < max_pages:
        current_url = queue.popleft()
        if current_url in visited:
            continue
            
        try:
            with urllib.request.urlopen(current_url, timeout=5) as response:
                if response.status == 200:
                    content = response.read().decode('utf-8', errors='ignore')
                    parser = SitemapParser(current_url)
                    parser.feed(content)
                    visited.add(current_url)
                    sitemap[current_url] = list(parser.links)
                    
                    for link in parser.links:
                        if link not in visited and link not in queue and len(visited) < max_pages:
                            queue.append(link)
        except (urllib.error.URLError, ValueError, TimeoutError):
            continue
    
    return {"sitemap": sitemap, "total_pages": len(visited), "base_url": start_url}

if __name__ == "__main__":
    result = crawl_website("https://example.com", max_pages=10)
    print(json.dumps(result, indent=2))