How to Compress Pipeline Output Gzip Per Partition in Python

import gzip
import io
import random
from pathlib import Path


def compress_partition(partition_data: list[str], output_path: Path) -> int:
    """Compress a partition of data to a gzip file, returns bytes written."""
    with gzip.open(output_path, 'wt', encoding='utf-8') as f:
        f.writelines(partition_data)
    return output_path.stat().st_size


def generate_partition(size: int = 10_000) -> list[str]:
    """Generate sample partition data (lines with random numbers)."""
    return [f"Record {i}: value={random.randint(1, 1000)}\n"
            for i in range(size)]


if __name__ == "__main__":
    # Create a temp directory for output
    out_dir = Path("./compressed_partitions")
    out_dir.mkdir(exist_ok=True)

    # Simulate processing 3 partitions of a pipeline
    for partition_num in range(3):
        data = generate_partition(size=10_000)
        output_file = out_dir / f"partition_{partition_num}.csv.gz"
        compressed_size = compress_partition(data, output_file)

        # Verify decompression works
        with gzip.open(output_file, 'rt', encoding='utf-8') as f:
            first_line = f.readline().strip()
            lines_count = sum(1 for _ in f) + 1

        print(f"Partition {partition_num}: {compressed_size} bytes, "
              f"{lines_count} lines, first line: '{first_line}'")