How to Create a Mock Iceberg Snapshot Manifest in Python

Build a mock Iceberg snapshot manifest structure with metadata and data entries using Python dictionaries and JSON.

Medium Python 3.9+ Aug 9, 2026 Big data & Spark 15 views 0 copies

Python code

74 lines
Python 3.9+
import json
from datetime import datetime, timezone


def create_mock_manifest(snapshot_id: int, file_paths: list[str]) -> dict:
    """Create a mock Iceberg snapshot manifest structure."""
    manifest_file = {
        "manifest_path": f"/warehouse/table/metadata/snap-{snapshot_id}-m0.avro",
        "manifest_length": 8192,
        "partition_spec_id": 0,
        "content": 0,
        "sequence_number": snapshot_id,
        "min_sequence_number": snapshot_id,
        "added_snapshot_id": snapshot_id,
        "added_files_count": len(file_paths),
        "existing_files_count": 0,
        "deleted_files_count": 0,
        "added_rows_count": sum(1 for _ in file_paths),
        "existing_rows_count": 0,
        "deleted_rows_count": 0,
        "partitions": [],
        "key_metadata": None,
    }

    entries = []
    for idx, path in enumerate(file_paths):
        entry = {
            "status": 1,
            "snapshot_id": snapshot_id,
            "data_sequence_number": snapshot_id,
            "file_sequence_number": snapshot_id,
            "file_path": path,
            "file_format": "PARQUET",
            "partition": {},
            "record_count": 1000,
            "file_size_in_bytes": 65536,
            "column_sizes": [{"column_id": 1, "size": 4096}],
            "value_counts": [{"column_id": 1, "count": 1000}],
            "null_value_counts": [{"column_id": 1, "count": 0}],
            "nan_value_counts": [{"column_id": 1, "count": 0}],
            "lower_bounds": [{"column_id": 1, "value": f"value_{idx}"}],
            "upper_bounds": [{"column_id": 1, "value": f"value_{idx}"}],
            "key_metadata": None,
            "split_offsets": [4, 8192],
            "equality_ids": None,
            "sort_order_id": 0,
        }
        entries.append(entry)

    manifest = {
        "snapshot_id": snapshot_id,
        "timestamp_ms": int(datetime.now(timezone.utc).timestamp() * 1000),
        "summary": {
            "operation": "append",
            "added-data-files": str(len(file_paths)),
            "added-records": str(len(file_paths) * 1000),
            "added-files-size": str(len(file_paths) * 65536),
        },
        "manifest_list": f"/warehouse/table/metadata/snap-{snapshot_id}.avro",
        "manifests": [manifest_file],
        "data_entries": entries,
    }
    return manifest


if __name__ == "__main__":
    mock = create_mock_manifest(
        snapshot_id=42,
        file_paths=[
            "/warehouse/table/data/00000-0.parquet",
            "/warehouse/table/data/00001-0.parquet",
        ],
    )
    print(json.dumps(mock, indent=2))

Output

stdout
{
  "snapshot_id": 42,
  "timestamp_ms": 1731532800000,
  "summary": {
    "operation": "append",
    "added-data-files": "2",
    "added-records": "2000",
    "added-files-size": "131072"
  },
  "manifest_list": "/warehouse/table/metadata/snap-42.avro",
  "manifests": [
    {
      "manifest_path": "/warehouse/table/metadata/snap-42-m0.avro",
      "manifest_length": 8192,
      "partition_spec_id": 0,
      "content": 0,
      "sequence_number": 42,
      "min_sequence_number": 42,
      "added_snapshot_id": 42,
      "added_files_count": 2,
      "existing_files_count": 0,
      "deleted_files_count": 0,
      "added_rows_count": 2,
      "existing_rows_count": 0,
      "deleted_rows_count": 0,
      "partitions": [],
      "key_metadata": null
    }
  ],
  "data_entries": [
    {
      "status": 1,
      "snapshot_id": 42,
      "data_sequence_number": 42,
      "file_sequence_number": 42,
      "file_path": "/warehouse/table/data/00000-0.parquet",
      "file_format": "PARQUET",
      "partition": {},
      "record_count": 1000,
      "file_size_in_bytes": 65536,
      "column_sizes": [{"column_id": 1, "size": 4096}],
      "value_counts": [{"column_id": 1, "count": 1000}],
      "null_value_counts": [{"column_id": 1, "count": 0}],
      "nan_value_counts": [{"column_id": 1, "count": 0}],
      "lower_bounds": [{"column_id": 1, "value": "value_0"}],
      "upper_bounds": [{"column_id": 1, "value": "value_0"}],
      "key_metadata": null,
      "split_offsets": [4, 8192],
      "equality_ids": null,
      "sort_order_id": 0
    },
    {
      "status": 1,
      "snapshot_id": 42,
      "data_sequence_number": 42,
      "file_sequence_number": 42,
      "file_path": "/warehouse/table/data/00001-0.parquet",
      "file_format": "PARQUET",
      "partition": {},
      "record_count": 1000,
      "file_size_in_bytes": 65536,
      "column_sizes": [{"column_id": 1, "size": 4096}],
      "value_counts": [{"column_id": 1, "count": 1000}],
      "null_value_counts": [{"column_id": 1, "count": 0}],
      "nan_value_counts": [{"column_id": 1, "count": 0}],
      "lower_bounds": [{"column_id": 1, "value": "value_1"}],
      "upper_bounds": [{"column_id": 1, "value": "value_1"}],
      "key_metadata": null,
      "split_offsets": [4, 8192],
      "equality_ids": null,
      "sort_order_id": 0
    }
  ]
}

How it works

The function builds a nested dictionary that mirrors the Iceberg manifest JSON structure. It sets the snapshot ID consistently across the manifest list, manifest file, and each data entry to keep relations clear. The summary uses string values for counts to match Iceberg's table metadata conventions. The timestamp uses UTC milliseconds for reproducibility. The result is a plain dict, so it can be serialized with json.dumps or passed to testing mocks.

Common mistakes

  • Forgetting to set the same snapshot_id in both manifest and data entries
  • Using integer counts in summary instead of strings
  • Not updating added_rows_count to match file_paths length
  • Hardcoding timestamp instead of using UTC now

Variations

  1. Use a dataclass to encapsulate the mock manifest structure
  2. Generate entries from a list of (path, record_count, size) tuples instead of paths only

Real-world use cases

  • Unit-testing Iceberg table readers without needing a real catalog
  • Simulating snapshot manifests for Spark/Dask batch processing pipelines
  • Generating test fixtures for data warehouse orchestration jobs

Sponsored

Run this sample

Open the browser IDE to tweak the example and see results without installing anything.

Open editor

More from Big data & Spark

Related tutorials and quizzes for this topic.