Big data & Spark

import json 

def explode_array_column(data, column):
    """Mock explode: split array column into multiple rows."""
    exploded = []
    for row in data:
        values = row.get(column, [])
        for value in values:
            new_row = dict(row)
            new_row[column] = value
            exploded.append(n…

from unittest.mock import Mock, patch


def get_hive_tables(spark):
    """Mock Hive support by returning a fixed list of tables."""
    return spark.sql("SHOW TABLES").collect()


class HiveTable:
    """Simple class that mimics a Hive table row."""
    def __init__(self, database, tableName):
        self.database =…

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, lit

class MockSink:
    def __init__(self):
        self.batches = []
    
    def write_batch(self, batch_df, batch_id):
        # Collect batch data as list of dicts for verification
        records = batch_df.collect()
        self.batches…

import random


def skew_join_salting_key(left_df, right_df, salt_range=4):
    """
    Demonstrates skew join salting: expand the smaller side with salt keys,
    then attach a salt key to each row on the larger side.
    Returns a list of (left, right, salt) tuples.
    """
    skewed_left = []
    for row in left_d…

class ZOrderLayout:
    """
    Minimal mock for z-order layout optimization using a stacking score.
    Elements overlap; higher z_index is drawn on top.
    """
    def __init__(self):
        self.elements = []

    def add_element(self, name, area, z_index):
        self.elements.append({"name": name, "area": area…