Split CSV Files into Smaller Chunks in Python

import csv
import os

def split_csv(input_file, chunk_size=1000, output_prefix="chunk"):
    """Split a large CSV file into smaller chunks."""
    with open(input_file, 'r', newline='') as infile:
        reader = csv.reader(infile)
        header = next(reader)
        
        file_count = 1
        row_count = 0
        outfile = None
        writer = None
        
        for row in reader:
            if row_count % chunk_size == 0:
                if outfile:
                    outfile.close()
                output_file = f"{output_prefix}_{file_count}.csv"
                outfile = open(output_file, 'w', newline='')
                writer = csv.writer(outfile)
                writer.writerow(header)
                file_count += 1
            writer.writerow(row)
            row_count += 1
        
        if outfile:
            outfile.close()
    
    print(f"Split '{input_file}' into {file_count-1} chunks.")

if __name__ == "__main__":
    # Create a sample large CSV for demonstration
    sample_file = "large_data.csv"
    with open(sample_file, 'w', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(["id", "name", "value"])
        for i in range(2500):
            writer.writerow([i, f"item_{i}", i * 1.5])
    
    # Split into chunks of 1000 rows each
    split_csv(sample_file, chunk_size=1000, output_prefix="split_chunk")
    
    # Cleanup sample files
    os.remove(sample_file)
    for i in range(1, 4):
        os.remove(f"split_chunk_{i}.csv")