Big data &amp; Spark

from datetime import datetime, timedelta
import time

class StreamingWatermark:
    """Mock watermark tracker for structured streaming."""

    def __init__(self, watermark_delay_seconds):
        self.watermark_delay = timedelta(seconds=watermark_delay_seconds)
        self.max_event_time = None

    def observe_even…

spark streaming micro-batch

How to Mock Spark Streaming Micro-Batches in Python

Simulate Spark's micro-batch streaming with a simple deque-based class that collects events over time and processes them in timed batches.

import time
from collections import deque
from datetime import datetime


class MicroBatchStream:
    def __init__(self, batch_interval_sec=2):
        self.batch_interval = batch_interval_sec
        self.source = deque()
        self.processed = []

    def add_events(self, events):
        self.source.extend(events…

apache-spark logical-plan catalyst

How to Mock a Catalyst Logical Plan in Python

Build a small Python class that mimics Spark Catalyst's logical plan tree for teaching or testing query optimizations.

from typing import Any, Dict, List, Optional


class CatalystLogicalPlan:
    """A minimal mock of Catalyst's logical plan for teaching purposes."""
    
    def __init__(self, node_type: str, **kwargs: Any) -> None:
        self.node_type = node_type
        self.attributes: Dict[str, Any] = kwargs
        self.child…

How to Mock a Compute-Collect Action Trigger in Python

Mock a compute-collect action trigger using Python's unittest.mock to simulate Spark-style job execution and assert trigger behavior.

testing mock spark

Here's a Python code sample for the problem title "Action trigger compute collect mock":

parquet pyarrow partition

How to Mock a Parquet partitionBy Sink in Python

Manually write a DataFrame to partitioned Parquet files, mimicking Spark's partitionBy sink behavior without Spark.

import pyarrow as pa
import pyarrow.parquet as pq
from pathlib import Path
import tempfile
import shutil


def mock_partition_by_sink(data, output_dir, partition_cols):
    table = pa.Table.from_pandas(data)
    schema = table.schema
    unique_combos = table.select(partition_cols).to_pylist()
    seen = set()
    for…

How to Mock a UDAF Aggregate Function in Python

This code provides a minimal mock of a User-Defined Aggregate Function (UDAF), simulating the initialize-update-merge-finalize lifecycle with a defaultdict counter.

udaf aggregate mock

from collections import defaultdict

class MockUDAF:
    """A minimal mock of a User-Defined Aggregate Function.

    Simulates aggregate lifecycle: initialize, update per row,
    and finalize the result.
    """

    def __init__(self):
        self._buffer = defaultdict(int)

    def initialize(self):
        """Re…

pyspark structured-streaming foreachbatch

How to use foreachBatch with a mock sink in PySpark

Demonstrates using Spark Structured Streaming's foreachBatch sink to capture and verify streaming batches by writing them into a custom mock sink object.

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, lit

class MockSink:
    def __init__(self):
        self.batches = []
    
    def write_batch(self, batch_df, batch_id):
        # Collect batch data as list of dicts for verification
        records = batch_df.collect()
        self.batches…

Mock RDD in Python: Simulate Spark RDD Lazy Transformations

Simulate Apache Spark RDD behavior in Python with lazy maps, filters, partitions, and a collect action.

spark rdd big-data

import random

def mock_rdd(data, num_slices=2):
    """
    A simple simulation of Spark RDD behavior with lazy evaluation,
    transformations, and an action.
    """
    class SimpleRDD:
        def __init__(self, data, num_slices=2):
            self.data = data
            self.num_slices = num_slices
           …

skew join salting distributed

Skew Join Salting Key in Python (Demo)

Demonstrates skew join salting by expanding a smaller side with salt keys and matching rows on the larger side via random salt assignment.

import random


def skew_join_salting_key(left_df, right_df, salt_range=4):
    """
    Demonstrates skew join salting: expand the smaller side with salt keys,
    then attach a salt key to each row on the larger side.
    Returns a list of (left, right, salt) tuples.
    """
    skewed_left = []
    for row in left_d…