Fix chunker (#2014)

backend/danswer/indexing/chunker.py

@@ -41,7 +41,10 @@ ChunkFunc = Callable[[Document], list[DocAwareChunk]]
 
 
 def extract_blurb(text: str, blurb_splitter: "SentenceSplitter") -> str:
-    return blurb_splitter.split_text(text)[0]
+    texts = blurb_splitter.split_text(text)
+    if not texts:
+        return ""
+    return texts[0]
 
 
 def chunk_large_section(

backend/danswer/indexing/indexing_pipeline.py

@@ -181,7 +181,7 @@ def index_doc_batch(
     )
 
     logger.debug("Starting chunking")
-    # The first chunk additionally contains the Title of the Document
+    # The embedder is needed here to get the correct tokenizer
     chunks: list[DocAwareChunk] = [
         chunk
         for document in updatable_docs