How to Chunk a Long Document for RAG Retrieval in Python

import re
from pathlib import Path

def chunk_document(text, chunk_size=500, overlap=100):
    """Split text into overlapping chunks suitable for RAG retrieval."""
    # Normalize whitespace
    text = re.sub(r'\s+', ' ', text).strip()
    
    chunks = []
    start = 0
    while start < len(text):
        end = min(start + chunk_size, len(text))
        
        # If not at the end, try to break at a sentence boundary
        if end < len(text):
            # Look back for the last period, question mark, or exclamation
            boundary = max(text.rfind('.', start, end), 
                         text.rfind('?', start, end), 
                         text.rfind('!', start, end))
            if boundary > start + chunk_size // 2:
                end = boundary + 1
        
        chunk = text[start:end].strip()
        if chunk:
            chunks.append(chunk)
        
        # Move start with overlap
        start = max(end - overlap, start + 1)
    
    return chunks

if __name__ == "__main__":
    document = (
        "Python is a powerful programming language. It is widely used in data science. "
        "Machine learning models can be built quickly. RAG retrieval enhances AI answers. "
        "Chunking documents is essential for efficient search. Proper overlap preserves context. "
        "This ensures no important information is lost between chunks."
    )
    
    chunks = chunk_document(document, chunk_size=100, overlap=30)
    for i, chunk in enumerate(chunks):
        print(f"Chunk {i+1}: {chunk}")