Build a Simple ETL Pipeline in Python

import json
from pathlib import Path


def read_input(file_path: Path) -> list[dict]:
    """Read JSON lines file into list of dicts."""
    with file_path.open("r", encoding="utf-8") as f:
        return [json.loads(line) for line in f if line.strip()]


def transform(records: list[dict]) -> list[dict]:
    """Transform raw records: filter invalid, normalize fields."""
    cleaned = []
    for rec in records:
        if rec.get("value") is None:
            continue
        cleaned.append({
            "id": rec["id"],
            "value": float(rec["value"]),
            "status": rec.get("status", "unknown"),
        })
    return cleaned


def write_output(records: list[dict], output_path: Path) -> None:
    """Write transformed records to JSON file."""
    with output_path.open("w", encoding="utf-8") as f:
        json.dump(records, f, indent=2)


if __name__ == "__main__":
    input_file = Path("input.jsonl")
    output_file = Path("output.json")

    # Sample input data for demonstration
    sample_records = [
        {"id": 1, "value": "10.5", "status": "ok"},
        {"id": 2, "value": "20.0"},
        {"id": 3, "value": None},
        {"id": 4, "value": "30.25", "status": "flagged"},
    ]
    with input_file.open("w", encoding="utf-8") as f:
        for rec in sample_records:
            f.write(json.dumps(rec) + "\n")

    # ETL pipeline
    raw_data = read_input(input_file)
    transformed = transform(raw_data)
    write_output(transformed, output_file)

    # Verify and print result
    with output_file.open("r", encoding="utf-8") as f:
        result = json.load(f)
    print(f"Transformed {len(transformed)} records:")
    print(json.dumps(result, indent=2))