highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-09-20 13:05:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add more metadata options for File connector (#797)

Browse Source
This commit is contained in:
Yuhong Sun
2023-11-30 13:24:22 -08:00
committed by GitHub
parent 9cff294a71
commit 17c2f06338
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/danswer/connectors/file/connector.py
View File

@@ -12,6 +12,7 @@ from danswer.connectors.cross_connector_utils.file_utils import detect_encoding
from danswer.connectors.cross_connector_utils.file_utils import load_files_from_zip from danswer.connectors.cross_connector_utils.file_utils import load_files_from_zip
from danswer.connectors.cross_connector_utils.file_utils import read_file from danswer.connectors.cross_connector_utils.file_utils import read_file
from danswer.connectors.cross_connector_utils.file_utils import read_pdf_file from danswer.connectors.cross_connector_utils.file_utils import read_pdf_file
from danswer.connectors.cross_connector_utils.time_utils import time_str_to_utc
from danswer.connectors.file.utils import check_file_ext_is_valid from danswer.connectors.file.utils import check_file_ext_is_valid
from danswer.connectors.file.utils import get_file_ext from danswer.connectors.file.utils import get_file_ext
from danswer.connectors.interfaces import GenerateDocumentsOutput from danswer.connectors.interfaces import GenerateDocumentsOutput
@@ -63,15 +64,20 @@ def _process_file(
else: else:
file_content_raw, metadata = read_file(file) file_content_raw, metadata = read_file(file)
dt_str = metadata.get("doc_updated_at")
final_time_updated = time_str_to_utc(dt_str) if dt_str else time_updated
return [ return [
Document( Document(
id=file_name, id=file_name,
sections=[ sections=[
Section(link=metadata.get("link", None), text=file_content_raw.strip()) Section(link=metadata.get("link"), text=file_content_raw.strip())
], ],
source=DocumentSource.FILE, source=DocumentSource.FILE,
semantic_identifier=file_name, semantic_identifier=file_name,
doc_updated_at=time_updated, doc_updated_at=final_time_updated,
primary_owners=metadata.get("primary_owners"),
secondary_owners=metadata.get("secondary_owners"),
metadata={}, metadata={},
) )
] ]