How to Stream Large CSV Files in Python

import csv
from pathlib import Path

def process_csv_in_chunks(file_path, chunk_size=1000):
    """Yield rows from a large CSV file in chunks without loading all into memory."""
    with open(file_path, 'r', newline='') as f:
        reader = csv.DictReader(f)
        chunk = []
        for row in reader:
            chunk.append(row)
            if len(chunk) == chunk_size:
                yield chunk
                chunk = []
        if chunk:
            yield chunk

if __name__ == "__main__":
    # Create a sample large CSV file
    sample_file = Path("large_data.csv")
    with open(sample_file, 'w', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=["id", "value"])
        writer.writeheader()
        for i in range(2500):
            writer.writerow({"id": i, "value": f"item_{i}"})

    # Process it in chunks
    total_rows = 0
    for i, chunk in enumerate(process_csv_in_chunks(sample_file)):
        print(f"Chunk {i}: {len(chunk)} rows (first ID: {chunk[0]['id']})")
        total_rows += len(chunk)

    print(f"Total rows processed: {total_rows}")
    sample_file.unlink()  # Clean up test file