Data pipelines & processing

import pandas as pd
from pathlib import Path

def find_duplicate_records_across_sheets(file_path: str, key_columns: list, sheet_names: list) -> dict:
    """
    Detect duplicate records across multiple Excel sheets based on specified key columns.
    
    Args:
        file_path: Path to the Excel file
        key_co…

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, sum, count

def check_null_rate(df, threshold=0.2, columns=None):
    """
    Check null rate for specified columns (or all) against a threshold.
    Returns columns that exceed the threshold.
    """
    cols = columns or df.columns
    total…

from collections import defaultdict, deque


def topological_order(dependencies):
    graph = defaultdict(list)
    in_degree = defaultdict(int)
    tasks = set(dependencies.keys())

    for task, depends_on in dependencies.items():
        for d in depends_on:
            graph[d].append(task)
            in_degree[t…