Streaming & messaging
Kafka-style pub/sub, event consumers, async pipelines, and message-driven workflows.
At Most Once Fire-and-Forget Mock in Python
A Python mock that enforces send() is called at most once and records the arguments for verification.
class FireForgetMock:
def __init__(self):
self._calls = 0
self._last_args = None
self._last_kwargs = None
def send(self, *args, **kwargs):
if self._calls > 0:
raise RuntimeError("send() called more than once")
self._calls += 1
self._last_args = args
…Batch Consume Process Commit Pattern in Python
A mock batch processor that accumulates items in a queue, processes full batches, commits successful or failed results, and flushes remaining items.
import random
import threading
import time
from collections import deque
class MockBatchProcessor:
def __init__(self, process_func, commit_func, batch_size=5):
self.queue = deque()
self.batch_size = batch_size
self.process_func = process_func
self.commit_func = commit_func
de…Build a Streaming Messaging Helper in Python
Create a simple message stream class that stores recent messages, sends user messages, and retrieves history or latest messages with timestamps.
from collections import deque
from dataclasses import dataclass
from datetime import datetime
import time
@dataclass
class Message:
user: str
text: str
timestamp: str = ""
def __post_init__(self):
if not self.timestamp:
self.timestamp = datetime.now().strftime("%H:%M:%S")
class…Dead Letter Queue Failed Messages List Mock in Python
Implements a simple in-memory dead letter queue to collect, list, and retry failed messages, with JSON serialization for inspection in streaming pipelines.
import json
from collections import deque
class Message:
def __init__(self, message_id, payload, attempts=0):
self.message_id = message_id
self.payload = payload
self.attempts = attempts
def __repr__(self):
return f"Message(id={self.message_id}, attempts={self.attempts})"
c…Dedupe processed message IDs in Python
Filters an inbox of messages by removing items whose IDs have already been processed, using a set for fast lookups.
from pathlib import Path
import json
def dedupe_processed_ids(inbox_file: Path, processed_file: Path) -> list:
processed = set(json.loads(processed_file.read_text()))
inbox = json.loads(inbox_file.read_text())
deduped = [item for item in inbox if item["id"] not in processed]
return deduped
if __nam…Event Envelope with Schema Version Field in Python
Build a typed event envelope dataclass with an explicit schema version field for mock streaming scenarios.
from dataclasses import dataclass, field
from datetime import datetime
import uuid
@dataclass
class Event:
event_id: str = field(default_factory=lambda: str(uuid.uuid4()))
event_type: str = "user.created"
version: str = "1.0.0"
created_at: str = field(default_factory=lambda: datetime.utcnow().isoform…Event sourcing append store replay in Python
A simple in-memory event store that appends events per aggregate and replays them on demand.
import json
from collections import defaultdict
class EventStore:
def __init__(self):
self._events = defaultdict(list)
def append(self, aggregate_id, event_type, data):
event = {"type": event_type, "data": data}
self._events[aggregate_id].append(event)
def replay(self, aggregate…Exactly Once Idempotent Consumer Store in Python
A mock key-value store that guarantees exactly-once processing by rejecting duplicate message keys in a message or event stream.
from collections import defaultdict
class ExactlyOnceStore:
def __init__(self):
self.processed = defaultdict(set)
self.data = {}
def consume(self, key, value):
if key in self.data:
return False
self.data[key] = value
return True
def get_processed_count…How to Aggregate Periodic Snapshot Data in Python
Generates mock snapshot data and groups values into periods to compute average aggregates with Python's standard library.
import random
from collections import defaultdict
def snapshot_aggregate(n=10, period=3):
data = defaultdict(list)
for i in range(n):
key = f"item_{i % period}"
data[key].append(random.randint(1, 100))
return dict(data)
def aggregate_periodic(snapshots, period=3):
result = {}
for …How to Build a Flow Control Credit Window in Python
A Python class that reserves, confirms, releases, and settles credit to limit message flow and prevent overload in streaming pipelines.
class CreditWindow:
def __init__(self, max_credit=1000):
self.max_credit = max_credit
self.used_credit = 0
self.pending_credit = 0
def try_reserve(self, amount):
available = self.max_credit - self.used_credit - self.pending_credit
if available >= amount:
…How to Build a Materialized View Updater Consumer Mock in Python
A mock consumer that queues change events and triggers refresh callbacks to simulate materialized view updates.
import time
from collections import deque
from dataclasses import dataclass, field
from typing import Callable, Deque, Optional
@dataclass
class MaterializedViewUpdater:
"""Mock updater that consumes change events and refreshes a view."""
refresh: Optional[Callable[[str], None]] = None
queue: Deque[tuple…How to Build a Message Stream Queue in Python
A beginner-friendly MessageStream class built on deque that sends messages one at a time, tracks unread counts, and records sent items.
from collections import deque
import time
class MessageStream:
def __init__(self, messages):
self._queue = deque(messages)
self._sent = []
def send_next(self):
if not self._queue:
return None
message = self._queue.popleft()
self._sent.append(message)
…How to Build a Mock Change Data Capture Event Stream in Python
Generate a deterministic list of mock CDC events with event IDs, stream positions, payloads, and timestamps for testing streaming pipelines.
from itertools import count
from random import choice, randint, seed
from datetime import datetime, timedelta
seed(42) # Make output deterministic
event_types = ["INSERT", "UPDATE", "DELETE"]
table_names = ["users", "orders", "products", "payments"]
counter = count(1)
def mock_cdc_event(stream_index: int) -> dict:
…How to Encode and Decode Avro Data in Python (Roundtrip)
Serialize a Python dict to Avro binary bytes and decode it back using the fastavro-compatible avro library.
import io
import json
from avro.schema import parse
from avro.io import DatumWriter, DatumReader, BinaryEncoder, BinaryDecoder
def avro_roundtrip(schema_json, data):
schema = parse(json.dumps(schema_json))
bytes_writer = io.BytesIO()
encoder = BinaryEncoder(bytes_writer)
writer = DatumWriter(schema)
…How to Implement At-Least-Once Delivery with Acknowledgment in Python
This code demonstrates a mock message broker with at-least-once delivery, including retry logic and acknowledgment after successful processing.
import time
import uuid
from collections import deque
class MockMessageBroker:
def __init__(self):
self.queue = deque()
self.acked = set()
def publish(self, payload: str) -> str:
msg_id = str(uuid.uuid4())
self.queue.append((msg_id, payload))
return msg_id
def po…How to Implement Backpressure Pause Producer with a Bounded Queue in Python
Places a Producer thread that sends items into a bounded queue with backpressure: on Full, it pauses to let the consumer catch up.
import threading
import time
import queue
import random
class Producer:
def __init__(self, q):
self.q = q
self.running = True
def produce(self):
while self.running:
item = random.randint(1, 100)
try:
self.q.put(item, timeout=0.5)
…How to Implement Publish-Subscribe Fanout with Multiple Subscribers in Python
Create a simple publish-subscribe system in Python that broadcasts messages to multiple subscriber callbacks for a given topic.
import time
class PubSub:
def __init__(self):
self.subscribers = {}
def subscribe(self, topic, callback):
if topic not in self.subscribers:
self.subscribers[topic] = []
self.subscribers[topic].append(callback)
def publish(self, topic, message):
if topic in sel…How to Implement a Priority Queue for Messages in Python
Build a message priority queue with heapq and dataclasses that pops messages by priority, using sequence numbers to keep insertion order.
import heapq
from dataclasses import dataclass, field
from typing import Any
@dataclass(order=True)
class Message:
priority: int
sequence: int = field(compare=False)
content: str = field(compare=False)
class PriorityQueue:
def __init__(self):
self._heap = []
def push(self, priority: int,…How to Implement a Tumbling Window Counter in Python
Count events that fall within a fixed-size sliding time window using a deque and pruning logic.
from collections import deque
import time
class TumblingWindowCounter:
def __init__(self, window_size_seconds):
self.window_size = window_size_seconds
self.window = deque()
def add_event(self, timestamp):
self.window.append(timestamp)
def count(self, current_time):
while…How to Implement an Outbox Table Poll Publisher in Python
This code simulates an outbox pattern with a class that polls for pending records and publishes them as JSON messages, removing only those that are due.
import time
import json
from dataclasses import dataclass, asdict
from datetime import datetime, timedelta
@dataclass
class OutboxRecord:
id: int
topic: str
payload: dict
created_at: datetime
class OutboxPollPublisher:
def __init__(self, poll_interval_seconds=1):
self.poll_interval = poll…How to Mock Offset Commit Auto vs Manual in Python
Demonstrates a Kafka-style offset commit function with auto/manual modes and tests it using unittest.mock.patch.
from unittest.mock import Mock, patch
def commit_offsets(topic_partition_offsets, auto_commit=False):
"""Manually commit offsets or simulate auto-commit."""
if auto_commit:
print(f"Auto-committing offsets: {topic_partition_offsets}")
return {"status": "auto_committed"}
print(f"Manuall…How to Mock RabbitMQ Ack Nack Requeue in Python
A mock RabbitMQ channel and consumer that simulates ack, nack, and requeue handling for testing message processing logic without a broker.
import json
from collections import deque
class MockChannel:
def __init__(self):
self.acked = []
self.nacked = []
self.requeued = []
def basic_ack(self, delivery_tag):
self.acked.append(delivery_tag)
def basic_nack(self, delivery_tag, requeue=False):
self.nacked.…How to Mock a Kafka Producer Batch Send in Python
Simulate a Kafka producer in Python that sends batched JSON events with mock partitions and latency for testing streaming pipelines without a real broker.
import json
import random
import time
from datetime import datetime
class MockKafkaProducer:
def __init__(self, topic):
self.topic = topic
self.sent_messages = []
def send(self, value, key=None):
message = {
"topic": self.topic,
"key": key,
"value"…How to Mock a Kafka Rebalance Listener in Python
Simulate Kafka consumer rebalance callbacks (on_partitions_revoked and on_partitions_assigned) with a mock consumer to test listener logic.
import time
from collections import defaultdict
class MockKafkaConsumer:
def __init__(self):
self.assignments = defaultdict(list)
self.rebalances = 0
def assign(self, partitions):
self.rebalances += 1
self.assignments.clear()
for partition in partitions:
s…Browse by section
Each section groups closely related Python snippets.
Streaming & messaging — Python code examples
What you will find here
This page collects streaming & messaging snippets — short, copy-ready Python you can paste into our free online IDE and run without installing anything. Each sample includes a plain-English explanation and the full source code.
Samples vs tutorials and challenges
Samples are quick reference — one concept per page. For step-by-step teaching, use our Python tutorials. To test yourself, try quizzes or coding challenges. Clean up style with the Python formatter.