Big data & Spark

import unittest
from unittest.mock import patch

# Module-level global counter accumulator
counter = 0

def increment(by=1):
    """Increment the global counter in place (accumulator pattern)."""
    global counter
    counter += by
    return counter

def reset():
    """Reset the counter to zero."""
    global count…

from typing import Any, Dict, List, Optional


class CatalystLogicalPlan:
    """A minimal mock of Catalyst's logical plan for teaching purposes."""
    
    def __init__(self, node_type: str, **kwargs: Any) -> None:
        self.node_type = node_type
        self.attributes: Dict[str, Any] = kwargs
        self.child…

Here's a Python code sample for the problem title "Action trigger compute collect mock":

import time
from collections import deque


class RateLimitedSource:
    def __init__(self, max_rate, window=1.0):
        self.max_rate = max_rate
        self.window = window
        self._timestamps = deque()

    def emit(self, item):
        now = time.monotonic()
        while self._timestamps and self._timestam…

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, lit

class MockSink:
    def __init__(self):
        self.batches = []
    
    def write_batch(self, batch_df, batch_id):
        # Collect batch data as list of dicts for verification
        records = batch_df.collect()
        self.batches…