From c2f3302aa051a188dc6fcc93d985165442b02d75 Mon Sep 17 00:00:00 2001 From: Weves Date: Thu, 13 Feb 2025 18:00:51 -0800 Subject: [PATCH] Fix mypy --- backend/onyx/connectors/gitbook/connector.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/backend/onyx/connectors/gitbook/connector.py b/backend/onyx/connectors/gitbook/connector.py index 7f9d82a57..68ee812cc 100644 --- a/backend/onyx/connectors/gitbook/connector.py +++ b/backend/onyx/connectors/gitbook/connector.py @@ -1,4 +1,3 @@ -import time from datetime import datetime from datetime import timezone from typing import Any @@ -46,18 +45,18 @@ def _extract_text_from_document(document: dict[str, Any]) -> str: """Extract text content from GitBook document structure by parsing the document nodes into markdown format.""" - def parse_leaf(leaf): + def parse_leaf(leaf: dict[str, Any]) -> str: text = leaf.get("text", "") leaf.get("marks", []) return text - def parse_text_node(node): + def parse_text_node(node: dict[str, Any]) -> str: text = "" for leaf in node.get("leaves", []): text += parse_leaf(leaf) return text - def parse_block_node(node): + def parse_block_node(node: dict[str, Any]) -> str: block_type = node.get("type", "") result = "" @@ -232,7 +231,7 @@ class GitbookConnector(LoadConnector, PollConnector): content = self.client.get(f"/spaces/{self.space_id}/content") pages = content.get("pages", []) - current_batch = [] + current_batch: list[Document] = [] for page in pages: updated_at = datetime.fromisoformat(page["updatedAt"]) @@ -250,7 +249,6 @@ class GitbookConnector(LoadConnector, PollConnector): if len(current_batch) >= self.batch_size: yield current_batch current_batch = [] - time.sleep(0.1) # Rate limiting if current_batch: yield current_batch