Build a Complete Web Scraper with Requests and BeautifulSoup in Python

import requests
from bs4 import BeautifulSoup
import csv
import time
from typing import List, Dict, Optional

class WebScraper:
    def __init__(self, base_url: str, output_file: str = "scraped_data.csv"):
        self.base_url = base_url
        self.output_file = output_file
        self.session = requests.Session()
        
    def fetch_page(self, url: str, retries: int = 3) -> Optional[BeautifulSoup]:
        for attempt in range(retries):
            try:
                response = self.session.get(url, timeout=10)
                response.raise_for_status()
                return BeautifulSoup(response.text, 'html.parser')
            except (requests.RequestException, Exception) as e:
                if attempt == retries - 1:
                    print(f"Error fetching {url}: {e}")
                    return None
                time.sleep(1)
                
    def parse_page(self, soup: BeautifulSoup) -> List[Dict[str, str]]:
        items = []
        for product in soup.select('.product-item'):
            item = {
                'name': product.select_one('.product-name').get_text(strip=True) if product.select_one('.product-name') else '',
                'price': product.select_one('.product-price').get_text(strip=True) if product.select_one('.product-price') else '',
                'rating': product.select_one('.rating').get_text(strip=True) if product.select_one('.rating') else ''
            }
            items.append(item)
        return items
    
    def get_next_page(self, soup: BeautifulSoup) -> Optional[str]:
        next_link = soup.select_one('a.next-page')
        if next_link and next_link.get('href'):
            return self.base_url + next_link['href']
        return None
    
    def scrape_all_pages(self) -> List[Dict[str, str]]:
        all_items = []
        current_url = self.base_url
        page_num = 1
        
        while current_url:
            print(f"Scraping page {page_num}: {current_url}")
            soup = self.fetch_page(current_url)
            if not soup:
                break
                
            items = self.parse_page(soup)
            all_items.extend(items)
            print(f"Found {len(items)} items on page {page_num}")
            
            current_url = self.get_next_page(soup)
            page_num += 1
            time.sleep(0.5)
            
        return all_items
    
    def export_to_csv(self, data: List[Dict[str, str]]):
        if not data:
            print("No data to export")
            return
            
        with open(self.output_file, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=['name', 'price', 'rating'])
            writer.writeheader()
            writer.writerows(data)
        print(f"Exported {len(data)} items to {self.output_file}")

if __name__ == "__main__":
    scraper = WebScraper("http://books.toscrape.com/catalogue/page-1.html")
    all_data = scraper.scrape_all_pages()
    scraper.export_to_csv(all_data)