Generate a Deterministic Hash for Deduplication in Python

import hashlib
import json
from pathlib import Path

def natural_key_hash(data, salt=""):
    """
    Generate a deterministic fingerprint from raw data (dict/list/str).
    Uses JSON canonical-ish serialization with sorted keys and SHA-256.
    """
    canonical = json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
    return hashlib.sha256((salt + canonical).encode("utf-8")).hexdigest()

def file_fingerprint(filepath, chunk_size=65536):
    """Compute a deduplication fingerprint for a file's content."""
    hasher = hashlib.sha256()
    with Path(filepath).open("rb") as f:
        for chunk in iter(lambda: f.read(chunk_size), b""):
            hasher.update(chunk)
    return hasher.hexdigest()

if __name__ == "__main__":
    # Deduplicate identical records with different field order
    records = [
        {"name": "Alice", "age": 30, "city": "NYC"},
        {"city": "NYC", "age": 30, "name": "Alice"},
        {"name": "Bob", "age": 25, "city": "LA"},
    ]
    seen = set()
    unique = []
    for rec in records:
        fp = natural_key_hash(rec)
        if fp not in seen:
            seen.add(fp)
            unique.append(rec)
    print("Unique records:", unique)

    # Example file fingerprint (temporary file)
    p = Path("/tmp/dedup_test.txt")
    p.write_text("hello world")
    print("File fingerprint:", file_fingerprint(p))
    p.unlink()