How to Mock a Parquet partitionBy Sink in Python

import pyarrow as pa
import pyarrow.parquet as pq
from pathlib import Path
import tempfile
import shutil


def mock_partition_by_sink(data, output_dir, partition_cols):
    table = pa.Table.from_pandas(data)
    schema = table.schema
    unique_combos = table.select(partition_cols).to_pylist()
    seen = set()
    for combo in unique_combos:
        key = tuple(combo[col] for col in partition_cols)
        if key in seen:
            continue
        seen.add(key)
        mask = pa.compute.equal(table[partition_cols[0]], combo[partition_cols[0]])
        for col in partition_cols[1:]:
            mask = pa.compute.and_(mask, pa.compute.equal(table[col], combo[col]))
        subset = table.filter(mask)
        partition_path = Path(output_dir) / "/".join(
            f"{col}={combo[col]}" for col in partition_cols
        )
        partition_path.mkdir(parents=True, exist_ok=True)
        pq.write_table(subset, partition_path / "data.parquet", schema=schema)


if __name__ == "__main__":
    import pandas as pd

    df = pd.DataFrame({
        "year": [2020, 2020, 2021, 2021],
        "month": [1, 2, 1, 2],
        "value": [10, 20, 30, 40],
    })

    tmp_dir = tempfile.mkdtemp()
    try:
        mock_partition_by_sink(df, tmp_dir, ["year", "month"])
        result = sorted(
            str(p.relative_to(tmp_dir))
            for p in Path(tmp_dir).rglob("*.parquet")
        )
        print(result)
    finally:
        shutil.rmtree(tmp_dir)