How to Stream a Large JSONL File Line by Line in Python

import json

def process_large_file(filepath, chunk_size=8192):
    """
    Stream a large JSON-lines file line by line, processing each record
    without loading the entire file into memory.
    """
    total_count = 0
    total_sum = 0
    
    with open(filepath, 'r') as f:
        while True:
            chunk = f.readlines(chunk_size)
            if not chunk:
                break
            for line in chunk:
                line = line.strip()
                if not line:
                    continue
                record = json.loads(line)
                total_count += 1
                total_sum += record.get('value', 0)
    
    return total_count, total_sum

if __name__ == "__main__":
    # Simulate a large file with a generator to demonstrate streaming
    import tempfile
    import os
    
    # Create a sample large file (1000 records)
    with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
        for i in range(1000):
            f.write(json.dumps({"id": i, "value": i * 2}) + "\n")
        filepath = f.name
    
    # Process it in streaming fashion
    count, total = process_large_file(filepath)
    print(f"Processed {count} records, total sum: {total}")
    
    # Clean up
    os.unlink(filepath)