aigenrec/luminabackend

1from typing import List2from langchain_text_splitters import RecursiveCharacterTextSplitter3 4class TextChunker:5    def __init__(self, chunk_size: int = 800, overlap: int = 100):6        self.splitter = RecursiveCharacterTextSplitter(7            chunk_size=chunk_size,8            chunk_overlap=overlap,9            length_function=len,10            is_separator_regex=False,11            separators=["\n\n", "\n", " ", ""]12        )13 14    def chunk_text(self, text: str, chunk_size: int = None, overlap: int = None) -> List[str]:15        """16        Split text using LangChain's RecursiveCharacterTextSplitter.17        If chunk_size/overlap provided here, they override init values (re-initializing splitter).18        """19        if not text:20            return []21            22        # If overrides are provided and different, create a temporary splitter23        current_splitter = self.splitter24        if (chunk_size is not None and chunk_size != self.splitter._chunk_size) or \25           (overlap is not None and overlap != self.splitter._chunk_overlap):26            current_splitter = RecursiveCharacterTextSplitter(27                chunk_size=chunk_size or self.splitter._chunk_size,28                chunk_overlap=overlap or self.splitter._chunk_overlap,29                separators=["\n\n", "\n", " ", ""]30            )31            32        return current_splitter.split_text(text)33    34    def chunk_by_tokens(self, text: str, max_tokens: int = 512, overlap_tokens: int = 50) -> List[str]:35        """36        Approximate token-based chunking using char count (1 token ~ 4 chars).37        """38        return self.chunk_text(text, chunk_size=max_tokens*4, overlap=overlap_tokens*4)