Data pipelines & processing
ETL-style flows, batch transforms, validation, and moving data between formats.
Build a Python Utility That Detects Duplicate Records Across Multiple Excel Sheets
A Python utility that uses pandas to find overlapping records across different Excel sheets based on specified key columns.
import pandas as pd
from pathlib import Path
def find_duplicate_records_across_sheets(file_path: str, key_columns: list, sheet_names: list) -> dict:
"""
Detect duplicate records across multiple Excel sheets based on specified key columns.
Args:
file_path: Path to the Excel file
key_co…Check Null Rate Threshold in PySpark DataFrame
This PySpark code checks the null rate of specified DataFrame columns against a threshold and returns violations.
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, sum, count
def check_null_rate(df, threshold=0.2, columns=None):
"""
Check null rate for specified columns (or all) against a threshold.
Returns columns that exceed the threshold.
"""
cols = columns or df.columns
total…Group Python Events into Sessions with a Gap Timeout
Groups timestamped events into sessions, starting a new session when the time gap exceeds a specified timeout.
from itertools import groupby
from datetime import datetime, timedelta
def session_window_group(events, gap_seconds=300):
"""Group events into sessions where gap > gap_seconds starts a new session."""
if not events:
return []
events = sorted(events, key=lambda x: x[0])
sessions = []
c…How to Filter Data in Python
Filter a list of dictionaries by exact key-value matches or numerical ranges using concise list comprehensions.
from typing import List, Dict, Any
def filter_data(
data: List[Dict[str, Any]], key: str, value: Any
) -> List[Dict[str, Any]]:
"""Return records where data[key] equals value."""
return [record for record in data if record.get(key) == value]
def filter_by_range(
data: List[Dict[str, Any]], key: str…How to Group Data by Key in Python
Group a list of dictionaries by a specified key using a defaultdict and compute per-group averages.
from collections import defaultdict
def group_by_key(data, key):
grouped = defaultdict(list)
for item in data:
grouped[item[key]].append(item)
return dict(grouped)
if __name__ == "__main__":
records = [
{"name": "Alice", "dept": "Engineering", "score": 85},
{"name": "Bob", "de…How to Group Rows by Key into Nested Arrays in Python
This code groups rows in a list of dictionaries by a specified key and returns a dictionary with each key mapped to a list of values from another key.
from collections import defaultdict
def implode_rows(rows, key, value_key):
grouped = defaultdict(list)
for row in rows:
grouped[row[key]].append(row[value_key])
return dict(grouped)
if __name__ == "__main__":
data = [
{"category": "fruit", "item": "apple"},
{"category": "fr…How to Implement a Sliding Window Average in Python
Compute the average of the most recent N values in a stream using a bounded deque, efficiently updating the total as new values arrive.
from collections import deque
class SlidingWindowAverage:
def __init__(self, window_size):
self.window_size = window_size
self.window = deque(maxlen=window_size)
self.total = 0
def add(self, value):
if len(self.window) == self.window_size:
self.total -= self.windo…How to Sort a List of Dictionaries by Key in Python
A reusable helper function that sorts a list of dictionaries by a specified key, with optional descending order support.
from typing import List
def sort_records(records: List[dict], key: str, descending: bool = False) -> List[dict]:
"""Sort a list of dictionaries by a specified key."""
return sorted(records, key=lambda record: record[key], reverse=descending)
def demonstrate_sorting() -> None:
users = [
{"name": …How to Topologically Sort a DAG in Python
Compute a valid execution order for tasks with dependencies using Kahn's algorithm in Python.
from collections import defaultdict, deque
def topological_order(dependencies):
graph = defaultdict(list)
in_degree = defaultdict(int)
tasks = set(dependencies.keys())
for task, depends_on in dependencies.items():
for d in depends_on:
graph[d].append(task)
in_degree[t…How to Track Checkpoint Offset After Batch Commit in Python
A batch processor that tracks the last successfully committed offset after processing records in batches, advancing the checkpoint only when each batch commits successfully.
import json
from typing import Any
class BatchProcessor:
"""Tracks checkpoint offset after committing batches."""
def __init__(self, batch_size: int = 3):
self.batch_size = batch_size
self.offset = 0 # last successfully committed offset (exclusive)
self.total_committed = 0
def …Validate dict schema at pipeline boundary in Python
This code validates a dictionary against a TypedDict schema at a pipeline boundary, enforcing required fields and types with custom error messages.
from typing import Any, TypedDict
class Person(TypedDict):
name: str
age: int
email: str
def validate_person(data: dict[str, Any]) -> Person:
errors: list[str] = []
if not isinstance(data.get("name"), str) or not data["name"].strip():
errors.append("name must be a non-empty string")
…Browse by section
Each section groups closely related Python snippets.
Data pipelines & processing — Python code examples
What you will find here
This page collects data pipelines & processing snippets — short, copy-ready Python you can paste into our free online IDE and run without installing anything. Each sample includes a plain-English explanation and the full source code.
Samples vs tutorials and challenges
Samples are quick reference — one concept per page. For step-by-step teaching, use our Python tutorials. To test yourself, try quizzes or coding challenges. Clean up style with the Python formatter.