Streaming & messaging
Kafka-style pub/sub, event consumers, async pipelines, and message-driven workflows.
At Most Once Fire-and-Forget Mock in Python
A Python mock that enforces send() is called at most once and records the arguments for verification.
class FireForgetMock:
def __init__(self):
self._calls = 0
self._last_args = None
self._last_kwargs = None
def send(self, *args, **kwargs):
if self._calls > 0:
raise RuntimeError("send() called more than once")
self._calls += 1
self._last_args = args
…Dead Letter Queue Failed Messages List Mock in Python
Implements a simple in-memory dead letter queue to collect, list, and retry failed messages, with JSON serialization for inspection in streaming pipelines.
import json
from collections import deque
class Message:
def __init__(self, message_id, payload, attempts=0):
self.message_id = message_id
self.payload = payload
self.attempts = attempts
def __repr__(self):
return f"Message(id={self.message_id}, attempts={self.attempts})"
c…Dedupe processed message IDs in Python
Filters an inbox of messages by removing items whose IDs have already been processed, using a set for fast lookups.
from pathlib import Path
import json
def dedupe_processed_ids(inbox_file: Path, processed_file: Path) -> list:
processed = set(json.loads(processed_file.read_text()))
inbox = json.loads(inbox_file.read_text())
deduped = [item for item in inbox if item["id"] not in processed]
return deduped
if __nam…Event Envelope with Schema Version Field in Python
Build a typed event envelope dataclass with an explicit schema version field for mock streaming scenarios.
from dataclasses import dataclass, field
from datetime import datetime
import uuid
@dataclass
class Event:
event_id: str = field(default_factory=lambda: str(uuid.uuid4()))
event_type: str = "user.created"
version: str = "1.0.0"
created_at: str = field(default_factory=lambda: datetime.utcnow().isoform…Exactly Once Idempotent Consumer Store in Python
A mock key-value store that guarantees exactly-once processing by rejecting duplicate message keys in a message or event stream.
from collections import defaultdict
class ExactlyOnceStore:
def __init__(self):
self.processed = defaultdict(set)
self.data = {}
def consume(self, key, value):
if key in self.data:
return False
self.data[key] = value
return True
def get_processed_count…How to Aggregate Periodic Snapshot Data in Python
Generates mock snapshot data and groups values into periods to compute average aggregates with Python's standard library.
import random
from collections import defaultdict
def snapshot_aggregate(n=10, period=3):
data = defaultdict(list)
for i in range(n):
key = f"item_{i % period}"
data[key].append(random.randint(1, 100))
return dict(data)
def aggregate_periodic(snapshots, period=3):
result = {}
for …How to Build a Mock Change Data Capture Event Stream in Python
Generate a deterministic list of mock CDC events with event IDs, stream positions, payloads, and timestamps for testing streaming pipelines.
from itertools import count
from random import choice, randint, seed
from datetime import datetime, timedelta
seed(42) # Make output deterministic
event_types = ["INSERT", "UPDATE", "DELETE"]
table_names = ["users", "orders", "products", "payments"]
counter = count(1)
def mock_cdc_event(stream_index: int) -> dict:
…How to Encode and Decode Avro Data in Python (Roundtrip)
Serialize a Python dict to Avro binary bytes and decode it back using the fastavro-compatible avro library.
import io
import json
from avro.schema import parse
from avro.io import DatumWriter, DatumReader, BinaryEncoder, BinaryDecoder
def avro_roundtrip(schema_json, data):
schema = parse(json.dumps(schema_json))
bytes_writer = io.BytesIO()
encoder = BinaryEncoder(bytes_writer)
writer = DatumWriter(schema)
…How to Implement a Priority Queue for Messages in Python
Build a message priority queue with heapq and dataclasses that pops messages by priority, using sequence numbers to keep insertion order.
import heapq
from dataclasses import dataclass, field
from typing import Any
@dataclass(order=True)
class Message:
priority: int
sequence: int = field(compare=False)
content: str = field(compare=False)
class PriorityQueue:
def __init__(self):
self._heap = []
def push(self, priority: int,…How to Implement a Tumbling Window Counter in Python
Count events that fall within a fixed-size sliding time window using a deque and pruning logic.
from collections import deque
import time
class TumblingWindowCounter:
def __init__(self, window_size_seconds):
self.window_size = window_size_seconds
self.window = deque()
def add_event(self, timestamp):
self.window.append(timestamp)
def count(self, current_time):
while…How to Mock Kafka Topic Partitions with a Python dict of lists
Mocks a Kafka topic and its partitions using a defaultdict of lists to simulate message production, consumption, and per-partition counts.
from collections import defaultdict
class KafkaTopicPartitionMock:
"""A simple mock for Kafka topic-partition assignment using dict of lists."""
def __init__(self, topic):
self.topic = topic
self.partitions = defaultdict(list) # partition_id -> list of messages
def produce(self, message…How to Mock MQTT Topic Subscriptions with QoS in Python
Build a lightweight MQTT client mock that tracks topic subscriptions with QoS levels and simulates wildcard message delivery.
import time
from collections import defaultdict
class MockMQTTClient:
def __init__(self):
self.subscriptions = defaultdict(list)
self.messages = []
def subscribe(self, topic, qos=0):
self.subscriptions[topic].append(qos)
print(f"Subscribed to '{topic}' with QoS {qos}")
…How to Mock Offset Commit Auto vs Manual in Python
Demonstrates a Kafka-style offset commit function with auto/manual modes and tests it using unittest.mock.patch.
from unittest.mock import Mock, patch
def commit_offsets(topic_partition_offsets, auto_commit=False):
"""Manually commit offsets or simulate auto-commit."""
if auto_commit:
print(f"Auto-committing offsets: {topic_partition_offsets}")
return {"status": "auto_committed"}
print(f"Manuall…How to Mock a Kafka Producer Batch Send in Python
Simulate a Kafka producer in Python that sends batched JSON events with mock partitions and latency for testing streaming pipelines without a real broker.
import json
import random
import time
from datetime import datetime
class MockKafkaProducer:
def __init__(self, topic):
self.topic = topic
self.sent_messages = []
def send(self, value, key=None):
message = {
"topic": self.topic,
"key": key,
"value"…How to Mock a Kafka Rebalance Listener in Python
Simulate Kafka consumer rebalance callbacks (on_partitions_revoked and on_partitions_assigned) with a mock consumer to test listener logic.
import time
from collections import defaultdict
class MockKafkaConsumer:
def __init__(self):
self.assignments = defaultdict(list)
self.rebalances = 0
def assign(self, partitions):
self.rebalances += 1
self.assignments.clear()
for partition in partitions:
s…How to Partition and Order Kafka-Style Messages by Key in Python
Group messages with the same key into ordered buckets using hashing and a defaultdict, mimicking Kafka partition ordering.
from dataclasses import dataclass
from collections import defaultdict
@dataclass
class Message:
key: str
content: str
def partition_and_order(messages, num_partitions=3):
partitions = defaultdict(list)
for msg in messages:
partition_id = hash(msg.key) % num_partitions
partitions[parti…How to Read Redis Streams with XREADGROUP in Python
Read new messages from a Redis stream using a consumer group with XREADGROUP, handling JSON payloads and group creation.
import redis
import json
def read_group_messages(stream_key, group_name, consumer_name, count=10):
r = redis.Redis(host="localhost", port=6379, decode_responses=True)
try:
r.xgroup_create(stream_key, group_name, id="0", mkstream=True)
except redis.exceptions.ResponseError:
pass
messag…How to Serialize and Deserialize JSON Event Payloads in Python
Define an EventPayload class with custom to_json and from_json methods to convert event objects to JSON strings and back, using datetime parsing.
import json
from datetime import datetime
class EventPayload:
def __init__(self, event_id, event_type, timestamp, data):
self.event_id = event_id
self.event_type = event_type
self.timestamp = timestamp
self.data = data
def to_json(self):
return json.dumps({
…How to Simulate RabbitMQ Exchange Routing in Python
Simulate RabbitMQ exchange routing using a nested dict, matching routing keys against patterns like error.* and info.# to return bound queues.
from collections import defaultdict
def route_message(exchanges, exchange_name, routing_key):
"""
Simulate RabbitMQ exchange routing using a nested dict structure.
Returns list of queue names that match the routing key.
"""
queues = exchanges.get(exchange_name, {})
matched = []
for pa…How to Track Session Windows with Gap Timeout in Python
A Python class that groups events into sessions, closing a session when the gap between events exceeds a timeout threshold.
import time
class SessionWindow:
"""Track sessions with a gap timeout (mock)."""
def __init__(self, timeout_seconds=5):
self.timeout = timeout_seconds
self.session_start = None
self.last_event_time = None
self.event_count = 0
self.events = []
def add_event…How to deduplicate messages by ID in Python
Track seen message IDs in a set to skip duplicate messages and store unique content in a dict, with exact output showing which messages were added or skipped.
import time
class MessageStore:
def __init__(self):
self.seen_ids = set()
self.messages = {}
def add(self, message_id, content, timestamp=None):
timestamp = timestamp or time.time()
if message_id in self.seen_ids:
return False
self.seen_ids.add(message_…How to mock RabbitMQ queue binding with routing keys in Python
A mock demonstration of binding a queue to an exchange with multiple routing keys in RabbitMQ using Python and pika, without a real broker connection.
import pika
import sys
def bind_queue_with_routing(channel, queue_name, exchange_name, routing_keys):
"""
Mock RabbitMQ queue binding with routing keys.
Prints the binding configuration instead of connecting to a real broker.
"""
for routing_key in routing_keys:
binding = {
"q…Implement a FIFO Message Queue in Python with deque
This code implements a FIFO (first-in-first-out) message queue class using Python's collections.deque, providing enqueue, dequeue, peek, and size operations.
from collections import deque
class MessageQueue:
def __init__(self):
self.queue = deque()
def enqueue(self, message):
self.queue.append(message)
print(f"Enqueued: {message}")
def dequeue(self):
if self.is_empty():
print("Queue is empty, cannot dequeue.")
…Implement a retry queue with visibility timeout in Python
This code simulates a message queue with a visibility timeout, allowing messages to be retried if not deleted before the timeout expires.
import time
from collections import deque
class SimpleQueue:
def __init__(self, visibility_timeout=2):
self.queue = deque()
self.in_flight = {}
self.visibility_timeout = visibility_timeout
def send(self, message):
self.queue.append(message)
def receive(self):
if …Browse by section
Each section groups closely related Python snippets.
Streaming & messaging — Python code examples
What you will find here
This page collects streaming & messaging snippets — short, copy-ready Python you can paste into our free online IDE and run without installing anything. Each sample includes a plain-English explanation and the full source code.
Samples vs tutorials and challenges
Samples are quick reference — one concept per page. For step-by-step teaching, use our Python tutorials. To test yourself, try quizzes or coding challenges. Clean up style with the Python formatter.