jdwh08s/Autodoc-Lifter
0
1#####################################################2### DOCUMENT PROCESSOR [PDF READER]3#####################################################4# Jonathan Wang5 6# ABOUT: 7# This project creates an app to chat with PDFs.8 9# This is the PDF READER.10# It converts a PDF into LlamaIndex nodes11# using UnstructuredIO.12#####################################################13# TODO Board:14# I don't think the current code is elegent... :(15 16# TODO: Replace chunk_by_header with a custom solution replicating bySimilarity17# https://docs.unstructured.io/api-reference/api-services/chunking#by-similarity-chunking-strategy18# Some hybrid thing...19 20 21# Come up with a awy to handle summarizing images and tables using MultiModalLLM after the processing into nodes.22 # TODO: Put this into PDFReaderUtilities? Along with the other functions for stuff like email?23 24# Investigate PDFPlumber as a backup/alternative for Unstructured. 25 # `https://github.com/jsvine/pdfplumber`26 # nevermind, this is essentially pdfminer.six but nicer27 28# Chunk hierarchy from https://www.reddit.com/r/LocalLLaMA/comments/1dpb9ow/how_we_chunk_turning_pdfs_into_hierarchical/29# Investigate document parsing algorithms from https://github.com/BobLd/DocumentLayoutAnalysis?tab=readme-ov-file30# Investigate document parsing algorithms from https://github.com/Filimoa/open-parse?tab=readme-ov-file31 32# Competition:33 # https://github.com/infiniflow/ragflow34 # https://github.com/deepdoctection/deepdoctection35 36#####################################################37## IMPORTS38import os39import re40import regex41from copy import deepcopy42 43from abc import ABC, abstractmethod44from typing import Any, List, Tuple, IO, Optional, Type, Generic, TypeVar45from llama_index.core.bridge.pydantic import Field46 47import numpy as np48 49from io import BytesIO50from base64 import b64encode, b64decode51from PIL import Image as PILImage52 53# from pdf_reader_utils import clean_pdf_chunk, dedupe_title_chunks, combine_listitem_chunks54 55# Unstructured Document Parsing56from unstructured.partition.pdf import partition_pdf57# from unstructured.cleaners.core import clean_extra_whitespace, group_broken_paragraphs #, clean_ordered_bullets, clean_bullets, clean_dashes58# from unstructured.chunking.title import chunk_by_title59# Unstructured Element Types60from unstructured.documents import elements, email_elements61from unstructured.partition.utils.constants import PartitionStrategy62 63# Llamaindex Nodes64from llama_index.core.settings import Settings65from llama_index.core.schema import Document, BaseNode, TextNode, ImageNode, NodeRelationship, RelatedNodeInfo66from llama_index.core.readers.base import BaseReader67from llama_index.core.base.embeddings.base import BaseEmbedding68from llama_index.core.node_parser import NodeParser69 70# Parallelism for cleaning chunks71from joblib import Parallel, delayed72 73## Lazy Imports74# import nltk75#####################################################76 77# Additional padding around the PDF extracted images78PDF_IMAGE_HORIZONTAL_PADDING = 2079PDF_IMAGE_VERTICAL_PADDING = 2080os.environ['EXTRACT_IMAGE_BLOCK_CROP_HORIZONTAL_PAD'] = str(PDF_IMAGE_HORIZONTAL_PADDING)81os.environ['EXTRACT_IMAGE_BLOCK_CROP_VERTICAL_PAD'] = str(PDF_IMAGE_VERTICAL_PADDING)82 83# class TextReader(BaseReader):84# def __init__(self, text: str) -> None:85# """Init params."""86# self.text = text87 88 89# class ImageReader(BaseReader):90# def __init__(self, image: Any) -> None:91# """Init params."""92# self.image = image93 94GenericNode = TypeVar("GenericNode", bound=BaseNode) # https://mypy.readthedocs.io/en/stable/generics.html95 96class UnstructuredPDFReader():97 # Yes, we could inherit from LlamaIndex BaseReader even though I don't think it's a good idea.98 # Have you seen the Llamaindex Base Reader? It's silly. """OOP"""99 # https://docs.llamaindex.ai/en/stable/api_reference/readers/100 101 # here I'm basically cargo culting off the (not-very-good) pre-built Llamaindex one.102 # https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/readers/llama-index-readers-file/llama_index/readers/file/unstructured/base.py103 104 # yes I do want to bind these to the class. 105 # you better not be changing the embedding model or node parser on me across different PDFReaders. that's absurd.106 # embed_model: BaseEmbedding107 # _node_parser: NodeParser# = Field(108 # description="Node parser to run on each Unstructured Title Chunk",109 # default=Settings.node_parser,110 # )111 _max_characters: int# = Field(112 # description="The maximum number of characters in a node",113 # default=8192,114 # )115 _new_after_n_chars: int #= Field(116 # description="The number of characters after which a new node is created",117 # default=1024,118 # )119 _overlap_n_chars: int #= Field(120 # description="The number of characters to overlap between nodes",121 # default=128,122 # )123 _overlap: int #= Field(124 # description="The number of characters to overlap between nodes",125 # default=128,126 # )127 _overlap_all: bool #= Field(128 # description="Whether to overlap all nodes",129 # default=False,130 # )131 _multipage_sections: bool #= Field(132 # description="Whether to include multipage sections",133 # default=False,134 # )135 136 ## TODO: Fix this big ball of primiatives and turn it into a class.137 def __init__(138 self,139 # node_parser: Optional[NodeParser], # Suggest using a SemanticNodeParser.140 max_characters: int = 2048, 141 new_after_n_chars: int = 512, 142 overlap_n_chars: int = 128, 143 overlap: int = 128, 144 overlap_all: bool = False, 145 multipage_sections: bool = True, 146 **kwargs: Any147 ) -> None:148 # node_parser = node_parser or Settings.node_parser149 """Init params."""150 super().__init__(**kwargs)151 152 self._max_characters = max_characters153 self._new_after_n_chars = new_after_n_chars154 self._overlap_n_chars = overlap_n_chars155 self._overlap = overlap156 self._overlap_all = overlap_all157 self._multipage_sections = multipage_sections158 # self._node_parser = node_parser or Settings.node_parser # set node parser to run on each Unstructured Title Chunk159 160 # Prerequisites for Unstructured.io to work161 # import nltk162 # nltk.data.path = ['./nltk_data']163 # try: 164 # if not nltk.data.find("tokenizers/punkt"):165 # # nltk.download("punkt")166 # print("Can't find punkt.")167 # except Exception as e:168 # # nltk.download("punkt")169 # print(e)170 # try: 171 # if not nltk.data.find("taggers/averaged_perceptron_tagger"):172 # # nltk.download("averaged_perceptron_tagger")173 # print("Can't find averaged_perceptron_tagger.")174 # except Exception as e:175 # # nltk.download("averaged_perceptron_tagger")176 # print(e)177 178 179 # """DATA LOADING FUNCTIONS"""180 def _node_rel_prev_next(self, prev_node: GenericNode, next_node: GenericNode) -> Tuple[GenericNode, GenericNode]:181 """Update pre-next node relationships between two nodes."""182 prev_node.relationships[NodeRelationship.NEXT] = RelatedNodeInfo(183 node_id=next_node.node_id,184 metadata={"filename": next_node.metadata['filename']}185 )186 next_node.relationships[NodeRelationship.PREVIOUS] = RelatedNodeInfo(187 node_id=prev_node.node_id,188 metadata={"filename": prev_node.metadata['filename']}189 )190 return (prev_node, next_node)191 192 def _node_rel_parent_child(self, parent_node: GenericNode, child_node: GenericNode) -> Tuple[GenericNode, GenericNode]:193 """Update parent-child node relationships between two nodes."""194 parent_node.relationships[NodeRelationship.CHILD] = RelatedNodeInfo(195 node_id=child_node.node_id,196 metadata={"filename": child_node.metadata['filename']}197 )198 child_node.relationships[NodeRelationship.PARENT] = RelatedNodeInfo(199 node_id=parent_node.node_id,200 metadata={"filename": parent_node.metadata['filename']}201 )202 return (parent_node, child_node)203 204 def _handle_metadata(205 self, 206 pdf_chunk: elements.Element, 207 node: GenericNode, 208 kept_metadata: List[str] = [209 'filename', 'file_directory', 'coordinates', 210 'page_number', 'page_name', 'section',211 'sent_from', 'sent_to', 'subject',212 'parent_id', 'category_depth', 213 'text_as_html', 'languages', 214 'emphasized_text_contents', 'link_texts', 'link_urls',215 'is_continuation', 'detection_class_prob',216 ]) -> GenericNode:217 """Add common unstructured element metadata to LlamaIndex node."""218 pdf_chunk_metadata = pdf_chunk.metadata.to_dict() if pdf_chunk.metadata else {}219 current_kept_metadata = deepcopy(kept_metadata)220 221 # Handle some interesting keys222 node.metadata['type'] = pdf_chunk.category223 if (('filename' in current_kept_metadata) and ('filename' in pdf_chunk_metadata) and ('file_directory' in pdf_chunk_metadata)):224 filename = os.path.join(str(pdf_chunk_metadata['file_directory']), str(pdf_chunk_metadata['filename']))225 node.metadata['filename'] = filename226 current_kept_metadata.remove('file_directory') if ('file_directory' in current_kept_metadata) else None227 if (('text_as_html' in current_kept_metadata) and ('text_as_html' in pdf_chunk_metadata)):228 node.metadata['orignal_table_text'] = getattr(node, 'text', '')229 node.text = pdf_chunk_metadata['text_as_html']230 current_kept_metadata.remove('text_as_html')231 if (('coordinates' in current_kept_metadata) and (pdf_chunk_metadata.get('coordinates') is not None)):232 node.metadata['coordinates'] = pdf_chunk_metadata['coordinates']233 current_kept_metadata.remove('coordinates')234 if (('page_number' in current_kept_metadata) and ('page_number' in pdf_chunk_metadata)):235 node.metadata['page_number'] = [pdf_chunk_metadata['page_number']] # save as list to allow for multiple pages236 current_kept_metadata.remove('page_number')237 if (('page_name' in current_kept_metadata) and ('page_name' in pdf_chunk_metadata)):238 node.metadata['page_name'] = [pdf_chunk_metadata['page_name']] # save as list to allow for multiple sheets239 current_kept_metadata.remove('page_name')240 241 # Handle the remaining keys242 for key in set(current_kept_metadata).intersection(set(pdf_chunk_metadata.keys())):243 node.metadata[key] = pdf_chunk_metadata[key]244 245 return node246 247 def _handle_text_chunk(self, pdf_text_chunk: elements.Element) -> TextNode:248 """Given a text chunk from Unstructured, convert it to a TextNode for LlamaIndex.249 250 Args:251 pdf_text_chunk (elements.Element): Input text chunk from Unstructured.252 253 Returns:254 TextNode: LlamaIndex TextNode which saves the text as HTML for structure.255 """256 new_node = TextNode(257 text=pdf_text_chunk.text, 258 id_=pdf_text_chunk.id,259 excluded_llm_metadata_keys=['type', 'parent_id', 'depth', 'filename', 'coordinates', 'link_texts', 'link_urls', 'link_start_indexes', 'orig_nodes', 'orignal_table_text', 'languages', 'detection_class_prob', 'keyword_metadata'],260 excluded_embed_metadata_keys=['type', 'parent_id', 'depth', 'filename', 'coordinates', 'page number', 'original_text', 'window', 'link_texts', 'link_urls', 'link_start_indexes', 'orig_nodes', 'orignal_table_text', 'languages', 'detection_class_prob']261 )262 new_node = self._handle_metadata(pdf_text_chunk, new_node)263 return (new_node)264 265 266 def _handle_table_chunk(self, pdf_table_chunk: elements.Table | elements.TableChunk) -> TextNode:267 """Given a table chunk from Unstructured, convert it to a TextNode for LlamaIndex.268 269 Args:270 pdf_table_chunk (elements.Table | elements.TableChunk): Input table chunk from Unstructured271 272 Returns:273 TextNode: LlamaIndex TextNode which saves the table as HTML for structure.274 275 NOTE: You will need to get the summary of the table for better performance.276 """277 new_node = TextNode(278 text=pdf_table_chunk.metadata.text_as_html if pdf_table_chunk.metadata.text_as_html else pdf_table_chunk.text,279 id_=pdf_table_chunk.id,280 excluded_llm_metadata_keys=['type', 'parent_id', 'depth', 'filename', 'coordinates', 'link_texts', 'link_urls', 'link_start_indexes', 'orig_nodes', 'orignal_table_text', 'languages', 'detection_class_prob', 'keyword_metadata'],281 excluded_embed_metadata_keys=['type', 'parent_id', 'depth', 'filename', 'coordinates', 'page number', 'original_text', 'window', 'link_texts', 'link_urls', 'link_start_indexes', 'orig_nodes', 'orignal_table_text', 'languages', 'detection_class_prob']282 )283 new_node = self._handle_metadata(pdf_table_chunk, new_node)284 return (new_node)285 286 287 def _handle_image_chunk(self, pdf_image_chunk: elements.Element) -> ImageNode:288 """Given an image chunk from UnstructuredIO, read it in and convert it into a Llamaindex ImageNode.289 290 Args:291 pdf_image_chunk (elements.Element): The input image element from UnstructuredIO. We'll allow all types, just in case you want to process some weird chunks.292 293 Returns:294 ImageNode: The image saved as a Llamaindex ImageNode.295 """296 pdf_image_chunk_data_available = pdf_image_chunk.metadata.to_dict()297 298 # Check for either saved image_path or image_base64/image_mime_type299 if (('image_path' not in pdf_image_chunk_data_available) and ('image_base64' not in pdf_image_chunk_data_available)):300 raise Exception('Image chunk does not have either image_path or image_base64/image_mime_type. Are you sure this is an image?')301 302 # Make the image node.303 new_node = ImageNode(304 text=pdf_image_chunk.text,305 id_=pdf_image_chunk.id,306 excluded_llm_metadata_keys=['type', 'parent_id', 'depth', 'filename', 'coordinates', 'link_texts', 'link_urls', 'link_start_indexes', 'orig_nodes', 'languages', 'detection_class_prob', 'keyword_metadata'],307 excluded_embed_metadata_keys=['type', 'parent_id', 'depth', 'filename', 'coordinates', 'page number', 'original_text', 'window', 'link_texts', 'link_urls', 'link_start_indexes', 'orig_nodes', 'languages', 'detection_class_prob']308 )309 new_node = self._handle_metadata(pdf_image_chunk, new_node)310 311 # Add image data to image node312 image = None313 if ('image_path' in pdf_image_chunk_data_available):314 # Save image path to image node315 new_node.image_path = pdf_image_chunk_data_available['image_path']316 317 # Load image from path, convert to base64318 image_pil = PILImage.open(pdf_image_chunk_data_available['image_path'])319 image_buffer = BytesIO()320 image_pil.save(image_buffer, format='JPEG')321 image = b64encode(image_buffer.getvalue()).decode('utf-8')322 323 new_node.image = image324 new_node.image_mimetype = 'image/jpeg'325 del image_buffer, image_pil326 elif ('image_base64' in pdf_image_chunk_data_available):327 # Save image base64 to image node328 new_node.image = pdf_image_chunk_data_available['image_base64']329 new_node.image_mimetype = pdf_image_chunk_data_available['image_mime_type']330 331 return (new_node)332 333 334 def _handle_composite_chunk(self, pdf_composite_chunk: elements.CompositeElement) -> BaseNode:335 """Given a composite chunk from Unstructured, convert it into a node and handle it dependencies as well."""336 # Start by getting a list of all the nodes which were combined into the composite chunk.337 # child_chunks = pdf_composite_chunk.metadata.to_dict()['orig_elements']338 child_chunks = pdf_composite_chunk.metadata.orig_elements or []339 child_nodes = []340 for chunk in child_chunks:341 child_nodes.append(self._handle_chunk(chunk)) # process all the child chunks.342 343 # Then build the Composite Chunk into a Node.344 composite_node = self._handle_text_chunk(pdf_text_chunk=pdf_composite_chunk)345 composite_node = self._handle_metadata(pdf_composite_chunk, composite_node)346 347 # Set relationships between chunks.348 for index in range(1, len(child_nodes)):349 child_nodes[index-1], child_nodes[index] = self._node_rel_prev_next(child_nodes[index-1], child_nodes[index])350 for index, node in enumerate(child_nodes):351 composite_node, child_nodes[index] = self._node_rel_parent_child(composite_node, child_nodes[index])352 353 composite_node.metadata['orig_nodes'] = child_nodes354 composite_node.excluded_llm_metadata_keys = ['filename', 'coordinates', 'chunk_number', 'window', 'orig_nodes', 'languages', 'detection_class_prob', 'keyword_metadata']355 composite_node.excluded_embed_metadata_keys = ['filename', 'coordinates', 'chunk_number', 'page number', 'original_text', 'window', 'summary', 'orig_nodes', 'languages', 'detection_class_prob']356 return(composite_node)357 358 359 def _handle_chunk(self, chunk: elements.Element) -> BaseNode:360 """Convert Unstructured element chunks to Llamaindex Node. Determine which chunk handling to use based on the element type."""361 # Composite (multiple nodes combined together by chunking)362 if (isinstance(chunk, elements.CompositeElement)):363 return (self._handle_composite_chunk(pdf_composite_chunk=chunk))364 # Tables365 elif ((chunk.category == 'Table') and isinstance(chunk, (elements.Table, elements.TableChunk))):366 return(self._handle_table_chunk(pdf_table_chunk=chunk))367 # Images368 elif (any(True for chunk_info in ['image', 'image_base64', 'image_path'] if chunk_info in chunk.metadata.to_dict())):369 return(self._handle_image_chunk(pdf_image_chunk=chunk))370 # Text371 else:372 return(self._handle_text_chunk(pdf_text_chunk=chunk))373 374 375 def pdf_to_chunks(376 self, 377 file_path: Optional[str],378 file: Optional[IO[bytes]],379 ) -> List[elements.Element]:380 """381 Given the file path to a PDF, read it in with UnstructuredIO and return its elements.382 """383 print("NEWPDF: Partitioning into Chunks...")384 # 1. attempt using AUTO to have it decide.385 # NOTE: this takes care of pdfminer, and also choses between using detectron2 vs tesseract only.386 # However, it sometimes gets confused by PDFs where text elements are added on later, e.g., CIDs for linking, or REDACTED387 pdf_chunks = partition_pdf(388 filename=file_path,389 file=file,390 unique_element_ids=True, # UUIDs that are unique for each element391 strategy=PartitionStrategy.HI_RES, # auto: it decides, hi_res: detectron2, but issues with multi-column, ocr_only: pytesseract, fast: pdfminer392 hi_res_model_name='yolox',393 include_page_breaks=False,394 metadata_filename=file_path,395 infer_table_structure=True,396 extract_images_in_pdf=True,397 extract_image_block_types=['Image', 'Table', 'Formula'], # element types to save as images398 extract_image_block_to_payload=False, # needs to be false; we'll convert into base64 later.399 extract_forms=False, # not currently available400 extract_image_block_output_dir=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/pdfimgs/')401 )402 403 # # 2. Check if it got good output.404 # pdf_read_in_okay = self.check_pdf_read_in(pdf_file_path=pdf_file_path, pdf_file=pdf_file, pdf_chunks=pdf_chunks)405 # if (pdf_read_in_okay):406 # return pdf_chunks407 408 # # 3. Okay, PDF didn't read in well, so we'll use the back-up strategy409 # # According to Unstructured's Github: https://github.com/Unstructured-IO/unstructured/blob/main/unstructured/partition/pdf.py410 # # that is "OCR_ONLY" as opposed to "HI_RES".411 # pdf_chunks = partition_pdf(412 # filename=pdf_file_path,413 # file=pdf_file,414 # strategy="ocr_only" # auto: it decides, hi_res: detectron2, but issues with multi-column, ocr_only: pytesseract, fast: pdfminer415 # )416 return pdf_chunks417 418 419 def chunks_to_nodes(self, pdf_chunks: List[elements.Element]) -> List[BaseNode]:420 """421 Given a PDF from Unstructured broken by header,422 convert them into nodes using the node_parser.423 E.g., to have all sentences with similar meaning as a node, use the SemanticNodeParser424 """425 # 0. Setup.426 unstructured_chunk_nodes = []427 428 # Hash of node ID and index429 node_id_to_index = {}430 431 # 1. Convert each page's text to Nodes.432 for index, chunk in enumerate(pdf_chunks):433 # Create new node based on node type434 new_node = self._handle_chunk(chunk)435 436 # Update hash of node ID and index437 node_id_to_index[new_node.id_] = index438 439 # Add relationship to prior node440 if (len(unstructured_chunk_nodes) > 0):441 unstructured_chunk_nodes[-1], new_node = self._node_rel_prev_next(prev_node=unstructured_chunk_nodes[-1], next_node=new_node)442 443 # Add parent-child relationships for Title Chunks444 if (chunk.metadata.parent_id is not None):445 # Find the index of the parent node based on parent_id446 parent_index = node_id_to_index[chunk.metadata.parent_id]447 if (parent_index is not None):448 unstructured_chunk_nodes[parent_index], new_node = self._node_rel_parent_child(parent_node=unstructured_chunk_nodes[parent_index], child_node=new_node)449 450 # Append to list451 unstructured_chunk_nodes.append(new_node)452 453 del node_id_to_index454 455 ## TODO: Move this chunk into a separate ReaderPostProcessor thing into PDFReaderUtils. Bundle in the sumamrization for tables and images into this.456 # 2. Node Parse each page to split when new information is different457 # NOTE: This was built for the Semantic Parser, but I guess we'll technically allow any parser here.458 # unstructured_parsed_nodes = self._node_parser.get_nodes_from_documents(unstructured_chunk_nodes)459 460 # 3. Node Attributes461 # for index, node in enumerate(unstructured_parsed_nodes):462 # # Keywords and Summary463 # # node_keywords = ', '.join(pdfrutils.get_keywords(node.text, top_k=5))464 # # node_summary = get_t5_summary(node.text, summary_length=64) # get_t5_summary465 # node.metadata['keywords'] = node_keywords466 # # node.metadata['summary'] = node_summary + (("\n" + node.metadata['summary']) if node.metadata['summary'] is not None else "")467 468 # # Get additional information about the node.469 # # Email: check for address.470 # info_types = []471 # if (pdfrutils.has_date(node.text)):472 # info_types.append("date")473 # if (pdfrutils.has_email(node.text)):474 # info_types.append("contact email")475 # if (pdfrutils.has_mail_addr(node.text)):476 # info_types.append("mailing postal address")477 # if (pdfrutils.has_phone(node.text)):478 # info_types.append("contact phone")479 480 # node.metadata['information types'] = ", ".join(info_types)481 # node.excluded_llm_metadata_keys = ['filename', 'coordinates', 'chunk_number', 'window', 'orig_nodes']482 # node.excluded_embed_metadata_keys = ['filename', 'coordinates', 'chunk_number', 'page number', 'original_text', 'window', 'keywords', 'summary', 'orig_nodes']483 484 # if (index > 0):485 # unstructured_parsed_nodes[index-1], node = self._node_rel_prev_next(unstructured_parsed_nodes[index-1], node)486 return(unstructured_chunk_nodes)487 488 # """Main user-interaction function"""489 def load_data(490 self, 491 file_path: Optional[str] = None,492 file: Optional[IO[bytes]] = None493 ) -> List: #[GenericNode]:494 """Given a path to a PDF file, load it with Unstructured and convert it into a list of Llamaindex Base Nodes.495 Input:496 - pdf_file_path (str): the path to the PDF file.497 Output:498 - List[GenericNode]: a list of LlamaIndex nodes. Creates one node for each parsed node, for each Unstructured Title Chunk.499 """500 # 1. PDF to Chunks501 print("NEWPDF: Reading Input File...")502 pdf_chunks = self.pdf_to_chunks(file_path=file_path, file=file)503 # return (pdf_chunks)504 505 # Chunk processing506 # pdf_chunks = clean_pdf_chunk, dedupe_title_chunks, combine_listitem_chunks, remove_header_footer_pagenum507 508 # 2. Chunks to titles509 # TODO: I hate this, make our own chunker.510 # pdf_titlechunks = chunk_by_title(511 # pdf_chunks,512 # max_characters=self._max_characters, 513 # new_after_n_chars=self._new_after_n_chars,514 # overlap=self._overlap, 515 # overlap_all=self._overlap_all,516 # multipage_sections=self._multipage_sections,517 # include_orig_elements=True,518 # combine_text_under_n_chars=self._new_after_n_chars519 # )520 # 3. Cleaning521 # pdf_titlechunks = Parallel(n_jobs=max(int(os.cpu_count())-1, 1))( # type: ignore522 # delayed(self.clean_pdf_chunk)(chunk) for chunk in pdf_chunks # pdf_titlechunks523 # )524 # pdf_titlechunks = list(pdf_titlechunks)525 # 4. Headlines to llamaindex nodes526 print("NEWPDF: Converting chunks to nodes...")527 parsed_chunks = self.chunks_to_nodes(pdf_chunks)528 return (parsed_chunks)