Restructure APIs (#803)

backend/danswer/access/access.py

@@ -4,7 +4,7 @@ from danswer.access.models import DocumentAccess
 from danswer.configs.constants import PUBLIC_DOC_PAT
 from danswer.db.document import get_acccess_info_for_documents
 from danswer.db.models import User
-from danswer.server.models import ConnectorCredentialPairIdentifier
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 
 

backend/danswer/background/celery/celery_utils.py

@@ -2,7 +2,7 @@ from sqlalchemy.orm import Session
 
 from danswer.background.task_utils import name_cc_cleanup_task
 from danswer.db.tasks import get_latest_task
-from danswer.server.models import DeletionAttemptSnapshot
+from danswer.server.documents.models import DeletionAttemptSnapshot
 
 
 def get_deletion_status(

backend/danswer/background/connector_deletion.py

@@ -33,7 +33,7 @@ from danswer.db.index_attempt import delete_index_attempts
 from danswer.db.models import ConnectorCredentialPair
 from danswer.document_index.interfaces import DocumentIndex
 from danswer.document_index.interfaces import UpdateRequest
-from danswer.server.models import ConnectorCredentialPairIdentifier
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()

backend/danswer/chat/chat_llm.py

@@ -43,7 +43,7 @@ from danswer.search.models import SearchQuery
 from danswer.search.models import SearchType
 from danswer.search.search_runner import chunks_to_search_docs
 from danswer.search.search_runner import full_chunk_search
-from danswer.server.models import RetrievalDocs
+from danswer.server.chat.models import RetrievalDocs
 from danswer.utils.logger import setup_logger
 from danswer.utils.text_processing import extract_embedded_json
 from danswer.utils.text_processing import has_unescaped_quote

backend/danswer/connectors/google_drive/connector_auth.py

@@ -25,9 +25,9 @@ from danswer.connectors.google_drive.constants import SCOPES
 from danswer.db.credentials import update_credential_json
 from danswer.db.models import User
 from danswer.dynamic_configs import get_dynamic_config_store
-from danswer.server.models import CredentialBase
-from danswer.server.models import GoogleAppCredentials
-from danswer.server.models import GoogleServiceAccountKey
+from danswer.server.documents.models import CredentialBase
+from danswer.server.documents.models import GoogleAppCredentials
+from danswer.server.documents.models import GoogleServiceAccountKey
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()

backend/danswer/danswerbot/slack/blocks.py

@@ -18,7 +18,7 @@ from danswer.danswerbot.slack.utils import build_feedback_block_id
 from danswer.danswerbot.slack.utils import remove_slack_text_interactions
 from danswer.danswerbot.slack.utils import translate_vespa_highlight_to_slack
 from danswer.direct_qa.interfaces import DanswerQuote
-from danswer.server.models import SearchDoc
+from danswer.server.chat.models import SearchDoc
 from danswer.utils.text_processing import replace_whitespaces_w_space
 
 

backend/danswer/danswerbot/slack/handlers/handle_message.py

@@ -27,8 +27,8 @@ from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.models import SlackBotConfig
 from danswer.direct_qa.answer_question import answer_qa_query
 from danswer.search.models import BaseFilters
-from danswer.server.models import NewMessageRequest
-from danswer.server.models import QAResponse
+from danswer.server.chat.models import NewMessageRequest
+from danswer.server.chat.models import QAResponse
 from danswer.utils.logger import setup_logger
 
 logger_base = setup_logger()

backend/danswer/danswerbot/slack/tokens.py

@@ -2,7 +2,7 @@ import os
 from typing import cast
 
 from danswer.dynamic_configs import get_dynamic_config_store
-from danswer.server.models import SlackBotTokens
+from danswer.server.manage.models import SlackBotTokens
 
 
 _SLACK_BOT_TOKENS_CONFIG_KEY = "slack_bot_tokens_config_key"

backend/danswer/db/connector.py

@@ -11,8 +11,8 @@ from danswer.configs.constants import DocumentSource
 from danswer.connectors.models import InputType
 from danswer.db.models import Connector
 from danswer.db.models import IndexAttempt
-from danswer.server.models import ConnectorBase
-from danswer.server.models import ObjectCreationIdResponse
+from danswer.server.documents.models import ConnectorBase
+from danswer.server.documents.models import ObjectCreationIdResponse
 from danswer.server.models import StatusResponse
 from danswer.utils.logger import setup_logger
 

backend/danswer/db/credentials.py

@@ -11,7 +11,7 @@ from danswer.connectors.google_drive.constants import (
 )
 from danswer.db.models import Credential
 from danswer.db.models import User
-from danswer.server.models import CredentialBase
+from danswer.server.documents.models import CredentialBase
 from danswer.utils.logger import setup_logger
 
 

backend/danswer/db/document.py

@@ -19,7 +19,7 @@ from danswer.db.models import Document as DbDocument
 from danswer.db.models import DocumentByConnectorCredentialPair
 from danswer.db.utils import model_to_dict
 from danswer.document_index.interfaces import DocumentMetadata
-from danswer.server.models import ConnectorCredentialPairIdentifier
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()

backend/danswer/db/document_set.py

@@ -14,8 +14,8 @@ from danswer.db.models import Document
 from danswer.db.models import DocumentByConnectorCredentialPair
 from danswer.db.models import DocumentSet as DocumentSetDBModel
 from danswer.db.models import DocumentSet__ConnectorCredentialPair
-from danswer.server.models import DocumentSetCreationRequest
-from danswer.server.models import DocumentSetUpdateRequest
+from danswer.server.features.document_set.models import DocumentSetCreationRequest
+from danswer.server.features.document_set.models import DocumentSetUpdateRequest
 
 
 def _delete_document_set_cc_pairs__no_commit(

backend/danswer/db/index_attempt.py

@@ -12,7 +12,7 @@ from sqlalchemy.orm import Session
 
 from danswer.db.models import IndexAttempt
 from danswer.db.models import IndexingStatus
-from danswer.server.models import ConnectorCredentialPairIdentifier
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.utils.logger import setup_logger
 from danswer.utils.telemetry import optional_telemetry
 from danswer.utils.telemetry import RecordType

backend/danswer/direct_qa/answer_question.py

@@ -33,10 +33,10 @@ from danswer.search.search_runner import chunks_to_search_docs
 from danswer.search.search_runner import full_chunk_search
 from danswer.search.search_runner import full_chunk_search_generator
 from danswer.secondary_llm_flows.answer_validation import get_answer_validity
-from danswer.server.models import LLMRelevanceFilterResponse
-from danswer.server.models import NewMessageRequest
-from danswer.server.models import QADocsResponse
-from danswer.server.models import QAResponse
+from danswer.server.chat.models import LLMRelevanceFilterResponse
+from danswer.server.chat.models import NewMessageRequest
+from danswer.server.chat.models import QADocsResponse
+from danswer.server.chat.models import QAResponse
 from danswer.server.utils import get_json_line
 from danswer.utils.logger import setup_logger
 from danswer.utils.timing import log_function_time

backend/danswer/main.py

@@ -44,19 +44,19 @@ from danswer.direct_qa.factory import get_default_qa_model
 from danswer.document_index.factory import get_default_document_index
 from danswer.llm.factory import get_default_llm
 from danswer.search.search_nlp_models import warm_up_models
-from danswer.server.cc_pair.api import router as cc_pair_router
-from danswer.server.chat.api import router as chat_router
-from danswer.server.connector import router as connector_router
-from danswer.server.credential import router as credential_router
-from danswer.server.danswer_api import get_danswer_api_key
-from danswer.server.danswer_api import router as danswer_api_router
-from danswer.server.document_set import router as document_set_router
-from danswer.server.manage import router as admin_router
-from danswer.server.persona.api import router as persona_router
-from danswer.server.search_backend import router as backend_router
-from danswer.server.slack_bot_management import router as slack_bot_management_router
-from danswer.server.state import router as state_router
-from danswer.server.users import router as user_router
+from danswer.server.chat.chat_backend import router as chat_router
+from danswer.server.chat.search_backend import router as backend_router
+from danswer.server.danswer_api.ingestion import get_danswer_api_key
+from danswer.server.danswer_api.ingestion import router as danswer_api_router
+from danswer.server.documents.cc_pair import router as cc_pair_router
+from danswer.server.documents.connector import router as connector_router
+from danswer.server.documents.credential import router as credential_router
+from danswer.server.features.document_set.api import router as document_set_router
+from danswer.server.features.persona.api import router as persona_router
+from danswer.server.manage.administrative import router as admin_router
+from danswer.server.manage.get_state import router as state_router
+from danswer.server.manage.slack_bot import router as slack_bot_management_router
+from danswer.server.manage.users import router as user_router
 from danswer.utils.logger import setup_logger
 from danswer.utils.telemetry import optional_telemetry
 from danswer.utils.telemetry import RecordType

backend/danswer/search/danswer_helper.py

@@ -5,7 +5,7 @@ from danswer.search.models import SearchType
 from danswer.search.search_nlp_models import get_default_tokenizer
 from danswer.search.search_nlp_models import IntentModel
 from danswer.search.search_runner import remove_stop_words_and_punctuation
-from danswer.server.models import HelperResponse
+from danswer.server.chat.models import HelperResponse
 from danswer.utils.logger import setup_logger
 from danswer.utils.timing import log_function_time
 

backend/danswer/search/request_preprocessing.py

@@ -13,7 +13,7 @@ from danswer.search.models import SearchQuery
 from danswer.search.models import SearchType
 from danswer.secondary_llm_flows.source_filter import extract_source_filter
 from danswer.secondary_llm_flows.time_filter import extract_time_filter
-from danswer.server.models import NewMessageRequest
+from danswer.server.chat.models import NewMessageRequest
 from danswer.utils.threadpool_concurrency import FunctionCall
 from danswer.utils.threadpool_concurrency import run_functions_in_parallel
 

backend/danswer/search/search_runner.py

@@ -32,7 +32,7 @@ from danswer.search.search_nlp_models import CrossEncoderEnsembleModel
 from danswer.search.search_nlp_models import EmbeddingModel
 from danswer.secondary_llm_flows.chunk_usefulness import llm_batch_eval_chunks
 from danswer.secondary_llm_flows.query_expansion import rephrase_query
-from danswer.server.models import SearchDoc
+from danswer.server.chat.models import SearchDoc
 from danswer.utils.logger import setup_logger
 from danswer.utils.threadpool_concurrency import FunctionCall
 from danswer.utils.threadpool_concurrency import run_functions_in_parallel

backend/danswer/secondary_llm_flows/query_validation.py

@@ -8,7 +8,7 @@ from danswer.llm.utils import dict_based_prompt_to_langchain_prompt
 from danswer.prompts.constants import ANSWERABLE_PAT
 from danswer.prompts.constants import THOUGHT_PAT
 from danswer.prompts.secondary_llm_flows import ANSWERABLE_PROMPT
-from danswer.server.models import QueryValidationResponse
+from danswer.server.chat.models import QueryValidationResponse
 from danswer.server.utils import get_json_line
 from danswer.utils.logger import setup_logger
 

backend/danswer/server/cc_pair/models.py

@@ -1,43 +0,0 @@
-from pydantic import BaseModel
-
-from danswer.db.models import ConnectorCredentialPair
-from danswer.db.models import IndexAttempt
-from danswer.server.models import ConnectorSnapshot
-from danswer.server.models import CredentialSnapshot
-from danswer.server.models import DeletionAttemptSnapshot
-from danswer.server.models import IndexAttemptSnapshot
-
-
-class CCPairFullInfo(BaseModel):
-    id: int
-    name: str
-    num_docs_indexed: int
-    connector: ConnectorSnapshot
-    credential: CredentialSnapshot
-    index_attempts: list[IndexAttemptSnapshot]
-    latest_deletion_attempt: DeletionAttemptSnapshot | None
-
-    @classmethod
-    def from_models(
-        cls,
-        cc_pair_model: ConnectorCredentialPair,
-        index_attempt_models: list[IndexAttempt],
-        latest_deletion_attempt: DeletionAttemptSnapshot | None,
-        num_docs_indexed: int,  # not ideal, but this must be computed separately
-    ) -> "CCPairFullInfo":
-        return cls(
-            id=cc_pair_model.id,
-            name=cc_pair_model.name,
-            num_docs_indexed=num_docs_indexed,
-            connector=ConnectorSnapshot.from_connector_db_model(
-                cc_pair_model.connector
-            ),
-            credential=CredentialSnapshot.from_credential_db_model(
-                cc_pair_model.credential
-            ),
-            index_attempts=[
-                IndexAttemptSnapshot.from_index_attempt_db_model(index_attempt_model)
-                for index_attempt_model in index_attempt_models
-            ],
-            latest_deletion_attempt=latest_deletion_attempt,
-        )

backend/danswer/server/chat/chat_backend.py

@@ -26,19 +26,19 @@ from danswer.db.models import User
 from danswer.direct_qa.interfaces import DanswerAnswerPiece
 from danswer.llm.utils import get_default_llm_token_encode
 from danswer.secondary_llm_flows.chat_helpers import get_new_chat_name
+from danswer.server.chat.models import ChatFeedbackRequest
+from danswer.server.chat.models import ChatMessageDetail
+from danswer.server.chat.models import ChatMessageIdentifier
+from danswer.server.chat.models import ChatRenameRequest
+from danswer.server.chat.models import ChatSession
 from danswer.server.chat.models import ChatSessionCreationRequest
-from danswer.server.models import ChatFeedbackRequest
-from danswer.server.models import ChatMessageDetail
-from danswer.server.models import ChatMessageIdentifier
-from danswer.server.models import ChatRenameRequest
-from danswer.server.models import ChatSession
-from danswer.server.models import ChatSessionDetailResponse
-from danswer.server.models import ChatSessionsResponse
-from danswer.server.models import CreateChatMessageRequest
-from danswer.server.models import CreateChatSessionID
-from danswer.server.models import RegenerateMessageRequest
-from danswer.server.models import RenameChatSessionResponse
-from danswer.server.models import RetrievalDocs
+from danswer.server.chat.models import ChatSessionDetailResponse
+from danswer.server.chat.models import ChatSessionsResponse
+from danswer.server.chat.models import CreateChatMessageRequest
+from danswer.server.chat.models import CreateChatSessionID
+from danswer.server.chat.models import RegenerateMessageRequest
+from danswer.server.chat.models import RenameChatSessionResponse
+from danswer.server.chat.models import RetrievalDocs
 from danswer.server.utils import get_json_line
 from danswer.utils.logger import setup_logger
 from danswer.utils.timing import log_generator_function_time

backend/danswer/server/chat/models.py

@@ -1,5 +1,200 @@
+from datetime import datetime
+from typing import Any
+
 from pydantic import BaseModel
 
+from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
+from danswer.configs.constants import DocumentSource
+from danswer.configs.constants import MessageType
+from danswer.configs.constants import QAFeedbackType
+from danswer.configs.constants import SearchFeedbackType
+from danswer.direct_qa.interfaces import DanswerAnswer
+from danswer.direct_qa.interfaces import DanswerQuote
+from danswer.search.models import BaseFilters
+from danswer.search.models import QueryFlow
+from danswer.search.models import SearchType
+
 
 class ChatSessionCreationRequest(BaseModel):
     persona_id: int | None = None
+
+
+class HelperResponse(BaseModel):
+    values: dict[str, str]
+    details: list[str] | None = None
+
+
+class SearchDoc(BaseModel):
+    document_id: str
+    semantic_identifier: str
+    link: str | None
+    blurb: str
+    source_type: str
+    boost: int
+    # whether the document is hidden when doing a standard search
+    # since a standard search will never find a hidden doc, this can only ever
+    # be `True` when doing an admin search
+    hidden: bool
+    score: float | None
+    # Matched sections in the doc. Uses Vespa syntax e.g. <hi>TEXT</hi>
+    # to specify that a set of words should be highlighted. For example:
+    # ["<hi>the</hi> <hi>answer</hi> is 42", "the answer is <hi>42</hi>""]
+    match_highlights: list[str]
+    # when the doc was last updated
+    updated_at: datetime | None
+
+    def dict(self, *args: list, **kwargs: dict[str, Any]) -> dict[str, Any]:  # type: ignore
+        initial_dict = super().dict(*args, **kwargs)  # type: ignore
+        initial_dict["updated_at"] = (
+            self.updated_at.isoformat() if self.updated_at else None
+        )
+        return initial_dict
+
+
+class RetrievalDocs(BaseModel):
+    top_documents: list[SearchDoc]
+
+
+# First chunk of info for streaming QA
+class QADocsResponse(RetrievalDocs):
+    predicted_flow: QueryFlow
+    predicted_search: SearchType
+    time_cutoff: datetime | None
+    favor_recent: bool
+
+    def dict(self, *args: list, **kwargs: dict[str, Any]) -> dict[str, Any]:  # type: ignore
+        initial_dict = super().dict(*args, **kwargs)  # type: ignore
+        initial_dict["time_cutoff"] = (
+            self.time_cutoff.isoformat() if self.time_cutoff else None
+        )
+        return initial_dict
+
+
+# Second chunk of info for streaming QA
+class LLMRelevanceFilterResponse(BaseModel):
+    relevant_chunk_indices: list[int]
+
+
+# TODO: rename/consolidate once the chat / QA flows are merged
+class NewMessageRequest(BaseModel):
+    chat_session_id: int
+    query: str
+    filters: BaseFilters
+    collection: str = DOCUMENT_INDEX_NAME
+    search_type: SearchType = SearchType.HYBRID
+    enable_auto_detect_filters: bool = True
+    favor_recent: bool | None = None
+    # Is this a real-time/streaming call or a question where Danswer can take more time?
+    real_time: bool = True
+    # Pagination purposes, offset is in batches, not by document count
+    offset: int | None = None
+
+
+class CreateChatSessionID(BaseModel):
+    chat_session_id: int
+
+
+class ChatFeedbackRequest(BaseModel):
+    chat_session_id: int
+    message_number: int
+    edit_number: int
+    is_positive: bool | None = None
+    feedback_text: str | None = None
+
+
+class CreateChatMessageRequest(BaseModel):
+    chat_session_id: int
+    message_number: int
+    parent_edit_number: int | None
+    message: str
+    persona_id: int | None
+
+
+class ChatMessageIdentifier(BaseModel):
+    chat_session_id: int
+    message_number: int
+    edit_number: int
+
+
+class RegenerateMessageRequest(ChatMessageIdentifier):
+    persona_id: int | None
+
+
+class ChatRenameRequest(BaseModel):
+    chat_session_id: int
+    name: str | None
+    first_message: str | None
+
+
+class RenameChatSessionResponse(BaseModel):
+    new_name: str  # This is only really useful if the name is generated
+
+
+class ChatSession(BaseModel):
+    id: int
+    name: str
+    time_created: str
+
+
+class ChatSessionsResponse(BaseModel):
+    sessions: list[ChatSession]
+
+
+class ChatMessageDetail(BaseModel):
+    message_number: int
+    edit_number: int
+    parent_edit_number: int | None
+    latest: bool
+    message: str
+    context_docs: RetrievalDocs | None
+    message_type: MessageType
+    time_sent: datetime
+
+
+class ChatSessionDetailResponse(BaseModel):
+    chat_session_id: int
+    description: str
+    messages: list[ChatMessageDetail]
+
+
+class QueryValidationResponse(BaseModel):
+    reasoning: str
+    answerable: bool
+
+
+class QAFeedbackRequest(BaseModel):
+    query_id: int
+    feedback: QAFeedbackType
+
+
+class SearchFeedbackRequest(BaseModel):
+    query_id: int
+    document_id: str
+    document_rank: int
+    click: bool
+    search_feedback: SearchFeedbackType
+
+
+class AdminSearchRequest(BaseModel):
+    query: str
+    filters: BaseFilters
+
+
+class AdminSearchResponse(BaseModel):
+    documents: list[SearchDoc]
+
+
+class SearchResponse(RetrievalDocs):
+    query_event_id: int
+    source_type: list[DocumentSource] | None
+    time_cutoff: datetime | None
+    favor_recent: bool
+
+
+class QAResponse(SearchResponse, DanswerAnswer):
+    quotes: list[DanswerQuote] | None
+    predicted_flow: QueryFlow
+    predicted_search: SearchType
+    eval_res_valid: bool | None = None
+    llm_chunks_indices: list[int] | None = None
+    error_msg: str | None = None

backend/danswer/server/chat/search_backend.py

@@ -23,16 +23,16 @@ from danswer.search.search_runner import chunks_to_search_docs
 from danswer.search.search_runner import full_chunk_search
 from danswer.secondary_llm_flows.query_validation import get_query_answerability
 from danswer.secondary_llm_flows.query_validation import stream_query_answerability
-from danswer.server.models import AdminSearchRequest
-from danswer.server.models import AdminSearchResponse
-from danswer.server.models import HelperResponse
-from danswer.server.models import NewMessageRequest
-from danswer.server.models import QAFeedbackRequest
-from danswer.server.models import QAResponse
-from danswer.server.models import QueryValidationResponse
-from danswer.server.models import SearchDoc
-from danswer.server.models import SearchFeedbackRequest
-from danswer.server.models import SearchResponse
+from danswer.server.chat.models import AdminSearchRequest
+from danswer.server.chat.models import AdminSearchResponse
+from danswer.server.chat.models import HelperResponse
+from danswer.server.chat.models import NewMessageRequest
+from danswer.server.chat.models import QAFeedbackRequest
+from danswer.server.chat.models import QAResponse
+from danswer.server.chat.models import QueryValidationResponse
+from danswer.server.chat.models import SearchDoc
+from danswer.server.chat.models import SearchFeedbackRequest
+from danswer.server.chat.models import SearchResponse
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()

backend/danswer/server/danswer_api/ingestion.py

@@ -18,9 +18,9 @@ from danswer.db.engine import get_session
 from danswer.dynamic_configs import get_dynamic_config_store
 from danswer.dynamic_configs.interface import ConfigNotFoundError
 from danswer.indexing.indexing_pipeline import build_indexing_pipeline
+from danswer.server.danswer_api.models import IngestionDocument
+from danswer.server.danswer_api.models import IngestionResult
 from danswer.server.models import ApiKey
-from danswer.server.models import IngestionDocument
-from danswer.server.models import IngestionResult
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()

backend/danswer/server/danswer_api/models.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel
+
+from danswer.connectors.models import DocumentBase
+
+
+class IngestionDocument(BaseModel):
+    document: DocumentBase
+    connector_id: int | None = None  # Takes precedence over the name
+    connector_name: str | None = None
+    credential_id: int | None = None
+    create_connector: bool = False  # Currently not allowed
+    public_doc: bool = True  # To attach to the cc_pair, currently unused
+
+
+class IngestionResult(BaseModel):
+    document_id: str
+    already_existed: bool

backend/danswer/server/documents/cc_pair.py

@@ -1,18 +1,23 @@
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from danswer.auth.users import current_admin_user
+from danswer.auth.users import current_user
 from danswer.background.celery.celery_utils import get_deletion_status
+from danswer.db.connector_credential_pair import add_credential_to_connector
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
+from danswer.db.connector_credential_pair import remove_credential_from_connector
 from danswer.db.document import get_document_cnts_for_cc_pairs
 from danswer.db.engine import get_session
 from danswer.db.index_attempt import get_index_attempts_for_cc_pair
 from danswer.db.models import User
-from danswer.server.cc_pair.models import CCPairFullInfo
-from danswer.server.models import ConnectorCredentialPairIdentifier
-
+from danswer.server.documents.models import CCPairFullInfo
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
+from danswer.server.documents.models import ConnectorCredentialPairMetadata
+from danswer.server.models import StatusResponse
 
 router = APIRouter(prefix="/manage")
 
@@ -65,3 +70,35 @@ def get_cc_pair_full_info(
         latest_deletion_attempt=latest_deletion_attempt,
         num_docs_indexed=documents_indexed,
     )
+
+
+@router.put("/connector/{connector_id}/credential/{credential_id}")
+def associate_credential_to_connector(
+    connector_id: int,
+    credential_id: int,
+    metadata: ConnectorCredentialPairMetadata,
+    user: User = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> StatusResponse[int]:
+    try:
+        return add_credential_to_connector(
+            connector_id=connector_id,
+            credential_id=credential_id,
+            cc_pair_name=metadata.name,
+            user=user,
+            db_session=db_session,
+        )
+    except IntegrityError:
+        raise HTTPException(status_code=400, detail="Name must be unique")
+
+
+@router.delete("/connector/{connector_id}/credential/{credential_id}")
+def dissociate_credential_from_connector(
+    connector_id: int,
+    credential_id: int,
+    user: User = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> StatusResponse[int]:
+    return remove_credential_from_connector(
+        connector_id, credential_id, user, db_session
+    )

backend/danswer/server/documents/connector.py

@@ -46,21 +46,21 @@ from danswer.db.index_attempt import create_index_attempt
 from danswer.db.index_attempt import get_latest_index_attempts
 from danswer.db.models import User
 from danswer.dynamic_configs.interface import ConfigNotFoundError
-from danswer.server.models import AuthStatus
-from danswer.server.models import AuthUrl
-from danswer.server.models import ConnectorBase
-from danswer.server.models import ConnectorCredentialPairIdentifier
-from danswer.server.models import ConnectorIndexingStatus
-from danswer.server.models import ConnectorSnapshot
-from danswer.server.models import CredentialSnapshot
-from danswer.server.models import FileUploadResponse
-from danswer.server.models import GDriveCallback
-from danswer.server.models import GoogleAppCredentials
-from danswer.server.models import GoogleServiceAccountCredentialRequest
-from danswer.server.models import GoogleServiceAccountKey
-from danswer.server.models import IndexAttemptSnapshot
-from danswer.server.models import ObjectCreationIdResponse
-from danswer.server.models import RunConnectorRequest
+from danswer.server.documents.models import AuthStatus
+from danswer.server.documents.models import AuthUrl
+from danswer.server.documents.models import ConnectorBase
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
+from danswer.server.documents.models import ConnectorIndexingStatus
+from danswer.server.documents.models import ConnectorSnapshot
+from danswer.server.documents.models import CredentialSnapshot
+from danswer.server.documents.models import FileUploadResponse
+from danswer.server.documents.models import GDriveCallback
+from danswer.server.documents.models import GoogleAppCredentials
+from danswer.server.documents.models import GoogleServiceAccountCredentialRequest
+from danswer.server.documents.models import GoogleServiceAccountKey
+from danswer.server.documents.models import IndexAttemptSnapshot
+from danswer.server.documents.models import ObjectCreationIdResponse
+from danswer.server.documents.models import RunConnectorRequest
 from danswer.server.models import StatusResponse
 
 _GOOGLE_DRIVE_CREDENTIAL_ID_COOKIE_NAME = "google_drive_credential_id"

backend/danswer/server/documents/credential.py

@@ -13,9 +13,9 @@ from danswer.db.credentials import fetch_credentials
 from danswer.db.credentials import update_credential
 from danswer.db.engine import get_session
 from danswer.db.models import User
-from danswer.server.models import CredentialBase
-from danswer.server.models import CredentialSnapshot
-from danswer.server.models import ObjectCreationIdResponse
+from danswer.server.documents.models import CredentialBase
+from danswer.server.documents.models import CredentialSnapshot
+from danswer.server.documents.models import ObjectCreationIdResponse
 from danswer.server.models import StatusResponse
 
 

backend/danswer/server/documents/models.py

@@ -0,0 +1,238 @@
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from pydantic import BaseModel
+
+from danswer.configs.app_configs import MASK_CREDENTIAL_PREFIX
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.models import InputType
+from danswer.db.models import Connector
+from danswer.db.models import ConnectorCredentialPair
+from danswer.db.models import Credential
+from danswer.db.models import IndexAttempt
+from danswer.db.models import IndexingStatus
+from danswer.db.models import TaskStatus
+from danswer.server.utils import mask_credential_dict
+
+
+class IndexAttemptSnapshot(BaseModel):
+    id: int
+    status: IndexingStatus | None
+    new_docs_indexed: int  # only includes completely new docs
+    total_docs_indexed: int  # includes docs that are updated
+    error_msg: str | None
+    time_started: str | None
+    time_updated: str
+
+    @classmethod
+    def from_index_attempt_db_model(
+        cls, index_attempt: IndexAttempt
+    ) -> "IndexAttemptSnapshot":
+        return IndexAttemptSnapshot(
+            id=index_attempt.id,
+            status=index_attempt.status,
+            new_docs_indexed=index_attempt.new_docs_indexed or 0,
+            total_docs_indexed=index_attempt.total_docs_indexed or 0,
+            error_msg=index_attempt.error_msg,
+            time_started=index_attempt.time_started.isoformat()
+            if index_attempt.time_started
+            else None,
+            time_updated=index_attempt.time_updated.isoformat(),
+        )
+
+
+class DeletionAttemptSnapshot(BaseModel):
+    connector_id: int
+    credential_id: int
+    status: TaskStatus
+
+
+class ConnectorBase(BaseModel):
+    name: str
+    source: DocumentSource
+    input_type: InputType
+    connector_specific_config: dict[str, Any]
+    refresh_freq: int | None  # In seconds, None for one time index with no refresh
+    disabled: bool
+
+
+class ConnectorSnapshot(ConnectorBase):
+    id: int
+    credential_ids: list[int]
+    time_created: datetime
+    time_updated: datetime
+
+    @classmethod
+    def from_connector_db_model(cls, connector: Connector) -> "ConnectorSnapshot":
+        return ConnectorSnapshot(
+            id=connector.id,
+            name=connector.name,
+            source=connector.source,
+            input_type=connector.input_type,
+            connector_specific_config=connector.connector_specific_config,
+            refresh_freq=connector.refresh_freq,
+            credential_ids=[
+                association.credential.id for association in connector.credentials
+            ],
+            time_created=connector.time_created,
+            time_updated=connector.time_updated,
+            disabled=connector.disabled,
+        )
+
+
+class CredentialBase(BaseModel):
+    credential_json: dict[str, Any]
+    # if `true`, then all Admins will have access to the credential
+    admin_public: bool
+
+
+class CredentialSnapshot(CredentialBase):
+    id: int
+    user_id: UUID | None
+    time_created: datetime
+    time_updated: datetime
+
+    @classmethod
+    def from_credential_db_model(cls, credential: Credential) -> "CredentialSnapshot":
+        return CredentialSnapshot(
+            id=credential.id,
+            credential_json=mask_credential_dict(credential.credential_json)
+            if MASK_CREDENTIAL_PREFIX
+            else credential.credential_json,
+            user_id=credential.user_id,
+            admin_public=credential.admin_public,
+            time_created=credential.time_created,
+            time_updated=credential.time_updated,
+        )
+
+
+class CCPairFullInfo(BaseModel):
+    id: int
+    name: str
+    num_docs_indexed: int
+    connector: ConnectorSnapshot
+    credential: CredentialSnapshot
+    index_attempts: list[IndexAttemptSnapshot]
+    latest_deletion_attempt: DeletionAttemptSnapshot | None
+
+    @classmethod
+    def from_models(
+        cls,
+        cc_pair_model: ConnectorCredentialPair,
+        index_attempt_models: list[IndexAttempt],
+        latest_deletion_attempt: DeletionAttemptSnapshot | None,
+        num_docs_indexed: int,  # not ideal, but this must be computed separately
+    ) -> "CCPairFullInfo":
+        return cls(
+            id=cc_pair_model.id,
+            name=cc_pair_model.name,
+            num_docs_indexed=num_docs_indexed,
+            connector=ConnectorSnapshot.from_connector_db_model(
+                cc_pair_model.connector
+            ),
+            credential=CredentialSnapshot.from_credential_db_model(
+                cc_pair_model.credential
+            ),
+            index_attempts=[
+                IndexAttemptSnapshot.from_index_attempt_db_model(index_attempt_model)
+                for index_attempt_model in index_attempt_models
+            ],
+            latest_deletion_attempt=latest_deletion_attempt,
+        )
+
+
+class ConnectorIndexingStatus(BaseModel):
+    """Represents the latest indexing status of a connector"""
+
+    cc_pair_id: int
+    name: str | None
+    connector: ConnectorSnapshot
+    credential: CredentialSnapshot
+    owner: str
+    public_doc: bool
+    last_status: IndexingStatus | None
+    last_success: datetime | None
+    docs_indexed: int
+    error_msg: str | None
+    latest_index_attempt: IndexAttemptSnapshot | None
+    deletion_attempt: DeletionAttemptSnapshot | None
+    is_deletable: bool
+
+
+class ConnectorCredentialPairIdentifier(BaseModel):
+    connector_id: int
+    credential_id: int
+
+
+class ConnectorCredentialPairMetadata(BaseModel):
+    name: str | None
+
+
+class ConnectorCredentialPairDescriptor(BaseModel):
+    id: int
+    name: str | None
+    connector: ConnectorSnapshot
+    credential: CredentialSnapshot
+
+
+class RunConnectorRequest(BaseModel):
+    connector_id: int
+    credential_ids: list[int] | None
+
+
+"""Connectors Models"""
+
+
+class GoogleAppWebCredentials(BaseModel):
+    client_id: str
+    project_id: str
+    auth_uri: str
+    token_uri: str
+    auth_provider_x509_cert_url: str
+    client_secret: str
+    redirect_uris: list[str]
+    javascript_origins: list[str]
+
+
+class GoogleAppCredentials(BaseModel):
+    web: GoogleAppWebCredentials
+
+
+class GoogleServiceAccountKey(BaseModel):
+    type: str
+    project_id: str
+    private_key_id: str
+    private_key: str
+    client_email: str
+    client_id: str
+    auth_uri: str
+    token_uri: str
+    auth_provider_x509_cert_url: str
+    client_x509_cert_url: str
+    universe_domain: str
+
+
+class GoogleServiceAccountCredentialRequest(BaseModel):
+    google_drive_delegated_user: str | None  # email of user to impersonate
+
+
+class FileUploadResponse(BaseModel):
+    file_paths: list[str]
+
+
+class ObjectCreationIdResponse(BaseModel):
+    id: int | str
+
+
+class AuthStatus(BaseModel):
+    authenticated: bool
+
+
+class AuthUrl(BaseModel):
+    auth_url: str
+
+
+class GDriveCallback(BaseModel):
+    state: str
+    code: str

backend/danswer/server/features/document_set/api.py

@@ -12,14 +12,14 @@ from danswer.db.document_set import mark_document_set_as_to_be_deleted
 from danswer.db.document_set import update_document_set
 from danswer.db.engine import get_session
 from danswer.db.models import User
-from danswer.server.models import CheckDocSetPublicRequest
-from danswer.server.models import CheckDocSetPublicResponse
-from danswer.server.models import ConnectorCredentialPairDescriptor
-from danswer.server.models import ConnectorSnapshot
-from danswer.server.models import CredentialSnapshot
-from danswer.server.models import DocumentSet
-from danswer.server.models import DocumentSetCreationRequest
-from danswer.server.models import DocumentSetUpdateRequest
+from danswer.server.documents.models import ConnectorCredentialPairDescriptor
+from danswer.server.documents.models import ConnectorSnapshot
+from danswer.server.documents.models import CredentialSnapshot
+from danswer.server.features.document_set.models import CheckDocSetPublicRequest
+from danswer.server.features.document_set.models import CheckDocSetPublicResponse
+from danswer.server.features.document_set.models import DocumentSet
+from danswer.server.features.document_set.models import DocumentSetCreationRequest
+from danswer.server.features.document_set.models import DocumentSetUpdateRequest
 
 
 router = APIRouter(prefix="/manage")

backend/danswer/server/features/document_set/models.py

@@ -0,0 +1,63 @@
+from pydantic import BaseModel
+
+from danswer.db.models import DocumentSet as DocumentSetDBModel
+from danswer.server.documents.models import ConnectorCredentialPairDescriptor
+from danswer.server.documents.models import ConnectorSnapshot
+from danswer.server.documents.models import CredentialSnapshot
+
+
+class DocumentSetCreationRequest(BaseModel):
+    name: str
+    description: str
+    cc_pair_ids: list[int]
+
+
+class DocumentSetUpdateRequest(BaseModel):
+    id: int
+    description: str
+    cc_pair_ids: list[int]
+
+
+class CheckDocSetPublicRequest(BaseModel):
+    document_set_ids: list[int]
+
+
+class CheckDocSetPublicResponse(BaseModel):
+    is_public: bool
+
+
+class DocumentSet(BaseModel):
+    id: int
+    name: str
+    description: str
+    cc_pair_descriptors: list[ConnectorCredentialPairDescriptor]
+    is_up_to_date: bool
+    contains_non_public: bool
+
+    @classmethod
+    def from_model(cls, document_set_model: DocumentSetDBModel) -> "DocumentSet":
+        return cls(
+            id=document_set_model.id,
+            name=document_set_model.name,
+            description=document_set_model.description,
+            contains_non_public=any(
+                [
+                    not cc_pair.is_public
+                    for cc_pair in document_set_model.connector_credential_pairs
+                ]
+            ),
+            cc_pair_descriptors=[
+                ConnectorCredentialPairDescriptor(
+                    id=cc_pair.id,
+                    name=cc_pair.name,
+                    connector=ConnectorSnapshot.from_connector_db_model(
+                        cc_pair.connector
+                    ),
+                    credential=CredentialSnapshot.from_credential_db_model(
+                        cc_pair.credential
+                    ),
+                )
+                for cc_pair in document_set_model.connector_credential_pairs
+            ],
+            is_up_to_date=document_set_model.is_up_to_date,
+        )

backend/danswer/server/features/persona/api.py

@@ -13,9 +13,9 @@ from danswer.db.document_set import get_document_sets_by_ids
 from danswer.db.engine import get_session
 from danswer.db.models import User
 from danswer.direct_qa.qa_block import PersonaBasedQAHandler
-from danswer.server.persona.models import CreatePersonaRequest
-from danswer.server.persona.models import PersonaSnapshot
-from danswer.server.persona.models import PromptTemplateResponse
+from danswer.server.features.persona.models import CreatePersonaRequest
+from danswer.server.features.persona.models import PersonaSnapshot
+from danswer.server.features.persona.models import PromptTemplateResponse
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()

backend/danswer/server/features/persona/models.py

@@ -1,7 +1,7 @@
 from pydantic import BaseModel
 
 from danswer.db.models import Persona
-from danswer.server.models import DocumentSet
+from danswer.server.features.document_set.models import DocumentSet
 
 
 class CreatePersonaRequest(BaseModel):

backend/danswer/server/manage/administrative.py

@@ -6,17 +6,13 @@ from typing import cast
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from danswer.auth.users import current_admin_user
-from danswer.auth.users import current_user
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
 from danswer.configs.app_configs import GENERATIVE_MODEL_ACCESS_CHECK_FREQ
 from danswer.configs.constants import GEN_AI_API_KEY_STORAGE_KEY
-from danswer.db.connector_credential_pair import add_credential_to_connector
 from danswer.db.connector_credential_pair import get_connector_credential_pair
-from danswer.db.connector_credential_pair import remove_credential_from_connector
 from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed
 from danswer.db.engine import get_session
 from danswer.db.feedback import fetch_docs_ranked_by_boost
@@ -30,14 +26,11 @@ from danswer.dynamic_configs.interface import ConfigNotFoundError
 from danswer.llm.factory import get_default_llm
 from danswer.llm.utils import get_gen_ai_api_key
 from danswer.llm.utils import test_llm
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
+from danswer.server.manage.models import BoostDoc
+from danswer.server.manage.models import BoostUpdateRequest
+from danswer.server.manage.models import HiddenUpdateRequest
 from danswer.server.models import ApiKey
-from danswer.server.models import BoostDoc
-from danswer.server.models import BoostUpdateRequest
-from danswer.server.models import ConnectorCredentialPairIdentifier
-from danswer.server.models import ConnectorCredentialPairMetadata
-from danswer.server.models import HiddenUpdateRequest
-from danswer.server.models import StatusResponse
-from danswer.server.models import UserRoleResponse
 from danswer.utils.logger import setup_logger
 
 router = APIRouter(prefix="/manage")
@@ -225,45 +218,3 @@ def create_deletion_attempt_for_connector_id(
     cleanup_connector_credential_pair_task.apply_async(
         kwargs=dict(connector_id=connector_id, credential_id=credential_id),
     )
-
-
-"""Endpoints for basic users"""
-
-
-@router.get("/get-user-role", response_model=UserRoleResponse)
-async def get_user_role(user: User = Depends(current_user)) -> UserRoleResponse:
-    if user is None:
-        raise ValueError("Invalid or missing user.")
-    return UserRoleResponse(role=user.role)
-
-
-@router.put("/connector/{connector_id}/credential/{credential_id}")
-def associate_credential_to_connector(
-    connector_id: int,
-    credential_id: int,
-    metadata: ConnectorCredentialPairMetadata,
-    user: User = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> StatusResponse[int]:
-    try:
-        return add_credential_to_connector(
-            connector_id=connector_id,
-            credential_id=credential_id,
-            cc_pair_name=metadata.name,
-            user=user,
-            db_session=db_session,
-        )
-    except IntegrityError:
-        raise HTTPException(status_code=400, detail="Name must be unique")
-
-
-@router.delete("/connector/{connector_id}/credential/{credential_id}")
-def dissociate_credential_from_connector(
-    connector_id: int,
-    credential_id: int,
-    user: User = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> StatusResponse[int]:
-    return remove_credential_from_connector(
-        connector_id, credential_id, user, db_session
-    )

backend/danswer/server/manage/get_state.py

@@ -2,9 +2,9 @@ from fastapi import APIRouter
 
 from danswer import __version__
 from danswer.configs.app_configs import AUTH_TYPE
-from danswer.server.models import AuthTypeResponse
+from danswer.server.manage.models import AuthTypeResponse
+from danswer.server.manage.models import VersionResponse
 from danswer.server.models import StatusResponse
-from danswer.server.models import VersionResponse
 
 router = APIRouter()
 

backend/danswer/server/manage/models.py

@@ -0,0 +1,88 @@
+from pydantic import BaseModel
+from pydantic import validator
+
+from danswer.auth.schemas import UserRole
+from danswer.configs.constants import AuthType
+from danswer.danswerbot.slack.config import VALID_SLACK_FILTERS
+from danswer.db.models import AllowedAnswerFilters
+from danswer.db.models import ChannelConfig
+from danswer.server.features.document_set.models import DocumentSet
+
+
+class VersionResponse(BaseModel):
+    backend_version: str
+
+
+class AuthTypeResponse(BaseModel):
+    auth_type: AuthType
+
+
+class UserInfo(BaseModel):
+    id: str
+    email: str
+    is_active: bool
+    is_superuser: bool
+    is_verified: bool
+    role: UserRole
+
+
+class UserByEmail(BaseModel):
+    user_email: str
+
+
+class UserRoleResponse(BaseModel):
+    role: str
+
+
+class BoostDoc(BaseModel):
+    document_id: str
+    semantic_id: str
+    link: str
+    boost: int
+    hidden: bool
+
+
+class BoostUpdateRequest(BaseModel):
+    document_id: str
+    boost: int
+
+
+class HiddenUpdateRequest(BaseModel):
+    document_id: str
+    hidden: bool
+
+
+class SlackBotTokens(BaseModel):
+    bot_token: str
+    app_token: str
+
+
+class SlackBotConfigCreationRequest(BaseModel):
+    # currently, a persona is created for each slack bot config
+    # in the future, `document_sets` will probably be replaced
+    # by an optional `PersonaSnapshot` object. Keeping it like this
+    # for now for simplicity / speed of development
+    document_sets: list[int]
+    channel_names: list[str]
+    respond_tag_only: bool = False
+    # If no team members, assume respond in the channel to everyone
+    respond_team_member_list: list[str] = []
+    answer_filters: list[AllowedAnswerFilters] = []
+
+    @validator("answer_filters", pre=True)
+    def validate_filters(cls, value: list[str]) -> list[str]:
+        if any(test not in VALID_SLACK_FILTERS for test in value):
+            raise ValueError(
+                f"Slack Answer filters must be one of {VALID_SLACK_FILTERS}"
+            )
+        return value
+
+
+class SlackBotConfig(BaseModel):
+    id: int
+    # currently, a persona is created for each slack bot config
+    # in the future, `document_sets` will probably be replaced
+    # by an optional `PersonaSnapshot` object. Keeping it like this
+    # for now for simplicity / speed of development
+    document_sets: list[DocumentSet]
+    channel_config: ChannelConfig

backend/danswer/server/manage/slack_bot.py

@@ -15,10 +15,10 @@ from danswer.db.slack_bot_config import insert_slack_bot_config
 from danswer.db.slack_bot_config import remove_slack_bot_config
 from danswer.db.slack_bot_config import update_slack_bot_config
 from danswer.dynamic_configs.interface import ConfigNotFoundError
-from danswer.server.models import DocumentSet
-from danswer.server.models import SlackBotConfig
-from danswer.server.models import SlackBotConfigCreationRequest
-from danswer.server.models import SlackBotTokens
+from danswer.server.features.document_set.models import DocumentSet
+from danswer.server.manage.models import SlackBotConfig
+from danswer.server.manage.models import SlackBotConfigCreationRequest
+from danswer.server.manage.models import SlackBotTokens
 
 
 router = APIRouter(prefix="/manage")

backend/danswer/server/manage/users.py

@@ -14,8 +14,9 @@ from danswer.db.engine import get_session
 from danswer.db.engine import get_sqlalchemy_async_engine
 from danswer.db.models import User
 from danswer.db.users import list_users
-from danswer.server.models import UserByEmail
-from danswer.server.models import UserInfo
+from danswer.server.manage.models import UserByEmail
+from danswer.server.manage.models import UserInfo
+from danswer.server.manage.models import UserRoleResponse
 
 router = APIRouter(prefix="/manage")
 
@@ -46,6 +47,13 @@ def list_all_users(
     return [UserRead.from_orm(user) for user in users]
 
 
+@router.get("/get-user-role", response_model=UserRoleResponse)
+async def get_user_role(user: User = Depends(current_user)) -> UserRoleResponse:
+    if user is None:
+        raise ValueError("Invalid or missing user.")
+    return UserRoleResponse(role=user.role)
+
+
 @router.get("/me")
 def verify_user_logged_in(user: User | None = Depends(current_user)) -> UserInfo:
     if user is None:

backend/danswer/server/models.py

@@ -1,40 +1,10 @@
-from datetime import datetime
-from typing import Any
 from typing import Generic
 from typing import Optional
 from typing import TypeVar
-from uuid import UUID
 
 from pydantic import BaseModel
-from pydantic import validator
 from pydantic.generics import GenericModel
 
-from danswer.auth.schemas import UserRole
-from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
-from danswer.configs.app_configs import MASK_CREDENTIAL_PREFIX
-from danswer.configs.constants import AuthType
-from danswer.configs.constants import DocumentSource
-from danswer.configs.constants import MessageType
-from danswer.configs.constants import QAFeedbackType
-from danswer.configs.constants import SearchFeedbackType
-from danswer.connectors.models import DocumentBase
-from danswer.connectors.models import InputType
-from danswer.danswerbot.slack.config import VALID_SLACK_FILTERS
-from danswer.db.models import AllowedAnswerFilters
-from danswer.db.models import ChannelConfig
-from danswer.db.models import Connector
-from danswer.db.models import Credential
-from danswer.db.models import DocumentSet as DocumentSetDBModel
-from danswer.db.models import IndexAttempt
-from danswer.db.models import IndexingStatus
-from danswer.db.models import TaskStatus
-from danswer.direct_qa.interfaces import DanswerAnswer
-from danswer.direct_qa.interfaces import DanswerQuote
-from danswer.search.models import BaseFilters
-from danswer.search.models import QueryFlow
-from danswer.search.models import SearchType
-from danswer.server.utils import mask_credential_dict
-
 
 DataT = TypeVar("DataT")
 
@@ -45,529 +15,5 @@ class StatusResponse(GenericModel, Generic[DataT]):
     data: Optional[DataT] = None
 
 
-class AuthTypeResponse(BaseModel):
-    auth_type: AuthType
-
-
-class VersionResponse(BaseModel):
-    backend_version: str
-
-
-class DataRequest(BaseModel):
-    data: str
-
-
-class HelperResponse(BaseModel):
-    values: dict[str, str]
-    details: list[str] | None = None
-
-
-class UserInfo(BaseModel):
-    id: str
-    email: str
-    is_active: bool
-    is_superuser: bool
-    is_verified: bool
-    role: UserRole
-
-
-class GoogleAppWebCredentials(BaseModel):
-    client_id: str
-    project_id: str
-    auth_uri: str
-    token_uri: str
-    auth_provider_x509_cert_url: str
-    client_secret: str
-    redirect_uris: list[str]
-    javascript_origins: list[str]
-
-
-class GoogleAppCredentials(BaseModel):
-    web: GoogleAppWebCredentials
-
-
-class GoogleServiceAccountKey(BaseModel):
-    type: str
-    project_id: str
-    private_key_id: str
-    private_key: str
-    client_email: str
-    client_id: str
-    auth_uri: str
-    token_uri: str
-    auth_provider_x509_cert_url: str
-    client_x509_cert_url: str
-    universe_domain: str
-
-
-class GoogleServiceAccountCredentialRequest(BaseModel):
-    google_drive_delegated_user: str | None  # email of user to impersonate
-
-
-class FileUploadResponse(BaseModel):
-    file_paths: list[str]
-
-
-class ObjectCreationIdResponse(BaseModel):
-    id: int | str
-
-
-class AuthStatus(BaseModel):
-    authenticated: bool
-
-
-class AuthUrl(BaseModel):
-    auth_url: str
-
-
-class GDriveCallback(BaseModel):
-    state: str
-    code: str
-
-
-class UserRoleResponse(BaseModel):
-    role: str
-
-
-class BoostDoc(BaseModel):
-    document_id: str
-    semantic_id: str
-    link: str
-    boost: int
-    hidden: bool
-
-
-class BoostUpdateRequest(BaseModel):
-    document_id: str
-    boost: int
-
-
-class HiddenUpdateRequest(BaseModel):
-    document_id: str
-    hidden: bool
-
-
-class SearchDoc(BaseModel):
-    document_id: str
-    semantic_identifier: str
-    link: str | None
-    blurb: str
-    source_type: str
-    boost: int
-    # whether the document is hidden when doing a standard search
-    # since a standard search will never find a hidden doc, this can only ever
-    # be `True` when doing an admin search
-    hidden: bool
-    score: float | None
-    # Matched sections in the doc. Uses Vespa syntax e.g. <hi>TEXT</hi>
-    # to specify that a set of words should be highlighted. For example:
-    # ["<hi>the</hi> <hi>answer</hi> is 42", "the answer is <hi>42</hi>""]
-    match_highlights: list[str]
-    # when the doc was last updated
-    updated_at: datetime | None
-
-    def dict(self, *args: list, **kwargs: dict[str, Any]) -> dict[str, Any]:  # type: ignore
-        initial_dict = super().dict(*args, **kwargs)  # type: ignore
-        initial_dict["updated_at"] = (
-            self.updated_at.isoformat() if self.updated_at else None
-        )
-        return initial_dict
-
-
-# TODO: rename/consolidate once the chat / QA flows are merged
-class NewMessageRequest(BaseModel):
-    chat_session_id: int
-    query: str
-    filters: BaseFilters
-    collection: str = DOCUMENT_INDEX_NAME
-    search_type: SearchType = SearchType.HYBRID
-    enable_auto_detect_filters: bool = True
-    favor_recent: bool | None = None
-    # Is this a real-time/streaming call or a question where Danswer can take more time?
-    real_time: bool = True
-    # Pagination purposes, offset is in batches, not by document count
-    offset: int | None = None
-
-
-class QAFeedbackRequest(BaseModel):
-    query_id: int
-    feedback: QAFeedbackType
-
-
-class SearchFeedbackRequest(BaseModel):
-    query_id: int
-    document_id: str
-    document_rank: int
-    click: bool
-    search_feedback: SearchFeedbackType
-
-
-class RetrievalDocs(BaseModel):
-    top_documents: list[SearchDoc]
-
-
-# First chunk of info for streaming QA
-class QADocsResponse(RetrievalDocs):
-    predicted_flow: QueryFlow
-    predicted_search: SearchType
-    time_cutoff: datetime | None
-    favor_recent: bool
-
-    def dict(self, *args: list, **kwargs: dict[str, Any]) -> dict[str, Any]:  # type: ignore
-        initial_dict = super().dict(*args, **kwargs)  # type: ignore
-        initial_dict["time_cutoff"] = (
-            self.time_cutoff.isoformat() if self.time_cutoff else None
-        )
-        return initial_dict
-
-
-# second chunk of info for streaming QA
-class LLMRelevanceFilterResponse(BaseModel):
-    relevant_chunk_indices: list[int]
-
-
-class CreateChatSessionID(BaseModel):
-    chat_session_id: int
-
-
-class ChatFeedbackRequest(BaseModel):
-    chat_session_id: int
-    message_number: int
-    edit_number: int
-    is_positive: bool | None = None
-    feedback_text: str | None = None
-
-
-class CreateChatMessageRequest(BaseModel):
-    chat_session_id: int
-    message_number: int
-    parent_edit_number: int | None
-    message: str
-    persona_id: int | None
-
-
-class ChatMessageIdentifier(BaseModel):
-    chat_session_id: int
-    message_number: int
-    edit_number: int
-
-
-class RegenerateMessageRequest(ChatMessageIdentifier):
-    persona_id: int | None
-
-
-class ChatRenameRequest(BaseModel):
-    chat_session_id: int
-    name: str | None
-    first_message: str | None
-
-
-class RenameChatSessionResponse(BaseModel):
-    new_name: str  # This is only really useful if the name is generated
-
-
-class ChatSession(BaseModel):
-    id: int
-    name: str
-    time_created: str
-
-
-class ChatSessionsResponse(BaseModel):
-    sessions: list[ChatSession]
-
-
-class ChatMessageDetail(BaseModel):
-    message_number: int
-    edit_number: int
-    parent_edit_number: int | None
-    latest: bool
-    message: str
-    context_docs: RetrievalDocs | None
-    message_type: MessageType
-    time_sent: datetime
-
-
-class ChatSessionDetailResponse(BaseModel):
-    chat_session_id: int
-    description: str
-    messages: list[ChatMessageDetail]
-
-
-class QueryValidationResponse(BaseModel):
-    reasoning: str
-    answerable: bool
-
-
-class AdminSearchRequest(BaseModel):
-    query: str
-    filters: BaseFilters
-
-
-class AdminSearchResponse(BaseModel):
-    documents: list[SearchDoc]
-
-
-class SearchResponse(RetrievalDocs):
-    query_event_id: int
-    source_type: list[DocumentSource] | None
-    time_cutoff: datetime | None
-    favor_recent: bool
-
-
-class QAResponse(SearchResponse, DanswerAnswer):
-    quotes: list[DanswerQuote] | None
-    predicted_flow: QueryFlow
-    predicted_search: SearchType
-    eval_res_valid: bool | None = None
-    llm_chunks_indices: list[int] | None = None
-    error_msg: str | None = None
-
-
-class UserByEmail(BaseModel):
-    user_email: str
-
-
-class IndexAttemptRequest(BaseModel):
-    input_type: InputType = InputType.POLL
-    connector_specific_config: dict[str, Any]
-
-
-class IndexAttemptSnapshot(BaseModel):
-    id: int
-    status: IndexingStatus | None
-    new_docs_indexed: int  # only includes completely new docs
-    total_docs_indexed: int  # includes docs that are updated
-    error_msg: str | None
-    time_started: str | None
-    time_updated: str
-
-    @classmethod
-    def from_index_attempt_db_model(
-        cls, index_attempt: IndexAttempt
-    ) -> "IndexAttemptSnapshot":
-        return IndexAttemptSnapshot(
-            id=index_attempt.id,
-            status=index_attempt.status,
-            new_docs_indexed=index_attempt.new_docs_indexed or 0,
-            total_docs_indexed=index_attempt.total_docs_indexed or 0,
-            error_msg=index_attempt.error_msg,
-            time_started=index_attempt.time_started.isoformat()
-            if index_attempt.time_started
-            else None,
-            time_updated=index_attempt.time_updated.isoformat(),
-        )
-
-
-class DeletionAttemptSnapshot(BaseModel):
-    connector_id: int
-    credential_id: int
-    status: TaskStatus
-
-
-class ConnectorBase(BaseModel):
-    name: str
-    source: DocumentSource
-    input_type: InputType
-    connector_specific_config: dict[str, Any]
-    refresh_freq: int | None  # In seconds, None for one time index with no refresh
-    disabled: bool
-
-
-class ConnectorSnapshot(ConnectorBase):
-    id: int
-    credential_ids: list[int]
-    time_created: datetime
-    time_updated: datetime
-
-    @classmethod
-    def from_connector_db_model(cls, connector: Connector) -> "ConnectorSnapshot":
-        return ConnectorSnapshot(
-            id=connector.id,
-            name=connector.name,
-            source=connector.source,
-            input_type=connector.input_type,
-            connector_specific_config=connector.connector_specific_config,
-            refresh_freq=connector.refresh_freq,
-            credential_ids=[
-                association.credential.id for association in connector.credentials
-            ],
-            time_created=connector.time_created,
-            time_updated=connector.time_updated,
-            disabled=connector.disabled,
-        )
-
-
-class RunConnectorRequest(BaseModel):
-    connector_id: int
-    credential_ids: list[int] | None
-
-
-class CredentialBase(BaseModel):
-    credential_json: dict[str, Any]
-    # if `true`, then all Admins will have access to the credential
-    admin_public: bool
-
-
-class CredentialSnapshot(CredentialBase):
-    id: int
-    user_id: UUID | None
-    time_created: datetime
-    time_updated: datetime
-
-    @classmethod
-    def from_credential_db_model(cls, credential: Credential) -> "CredentialSnapshot":
-        return CredentialSnapshot(
-            id=credential.id,
-            credential_json=mask_credential_dict(credential.credential_json)
-            if MASK_CREDENTIAL_PREFIX
-            else credential.credential_json,
-            user_id=credential.user_id,
-            admin_public=credential.admin_public,
-            time_created=credential.time_created,
-            time_updated=credential.time_updated,
-        )
-
-
-class ConnectorIndexingStatus(BaseModel):
-    """Represents the latest indexing status of a connector"""
-
-    cc_pair_id: int
-    name: str | None
-    connector: ConnectorSnapshot
-    credential: CredentialSnapshot
-    owner: str
-    public_doc: bool
-    last_status: IndexingStatus | None
-    last_success: datetime | None
-    docs_indexed: int
-    error_msg: str | None
-    latest_index_attempt: IndexAttemptSnapshot | None
-    deletion_attempt: DeletionAttemptSnapshot | None
-    is_deletable: bool
-
-
-class ConnectorCredentialPairIdentifier(BaseModel):
-    connector_id: int
-    credential_id: int
-
-
-class ConnectorCredentialPairMetadata(BaseModel):
-    name: str | None
-
-
-class ConnectorCredentialPairDescriptor(BaseModel):
-    id: int
-    name: str | None
-    connector: ConnectorSnapshot
-    credential: CredentialSnapshot
-
-
 class ApiKey(BaseModel):
     api_key: str
-
-
-class DocumentSetCreationRequest(BaseModel):
-    name: str
-    description: str
-    cc_pair_ids: list[int]
-
-
-class DocumentSetUpdateRequest(BaseModel):
-    id: int
-    description: str
-    cc_pair_ids: list[int]
-
-
-class CheckDocSetPublicRequest(BaseModel):
-    document_set_ids: list[int]
-
-
-class CheckDocSetPublicResponse(BaseModel):
-    is_public: bool
-
-
-class DocumentSet(BaseModel):
-    id: int
-    name: str
-    description: str
-    cc_pair_descriptors: list[ConnectorCredentialPairDescriptor]
-    is_up_to_date: bool
-    contains_non_public: bool
-
-    @classmethod
-    def from_model(cls, document_set_model: DocumentSetDBModel) -> "DocumentSet":
-        return cls(
-            id=document_set_model.id,
-            name=document_set_model.name,
-            description=document_set_model.description,
-            contains_non_public=any(
-                [
-                    not cc_pair.is_public
-                    for cc_pair in document_set_model.connector_credential_pairs
-                ]
-            ),
-            cc_pair_descriptors=[
-                ConnectorCredentialPairDescriptor(
-                    id=cc_pair.id,
-                    name=cc_pair.name,
-                    connector=ConnectorSnapshot.from_connector_db_model(
-                        cc_pair.connector
-                    ),
-                    credential=CredentialSnapshot.from_credential_db_model(
-                        cc_pair.credential
-                    ),
-                )
-                for cc_pair in document_set_model.connector_credential_pairs
-            ],
-            is_up_to_date=document_set_model.is_up_to_date,
-        )
-
-
-class IngestionDocument(BaseModel):
-    document: DocumentBase
-    connector_id: int | None = None  # Takes precedence over the name
-    connector_name: str | None = None
-    credential_id: int | None = None
-    create_connector: bool = False  # Currently not allowed
-    public_doc: bool = True  # To attach to the cc_pair, currently unused
-
-
-class IngestionResult(BaseModel):
-    document_id: str
-    already_existed: bool
-
-
-class SlackBotTokens(BaseModel):
-    bot_token: str
-    app_token: str
-
-
-class SlackBotConfigCreationRequest(BaseModel):
-    # currently, a persona is created for each slack bot config
-    # in the future, `document_sets` will probably be replaced
-    # by an optional `PersonaSnapshot` object. Keeping it like this
-    # for now for simplicity / speed of development
-    document_sets: list[int]
-    channel_names: list[str]
-    respond_tag_only: bool = False
-    # If no team members, assume respond in the channel to everyone
-    respond_team_member_list: list[str] = []
-    answer_filters: list[AllowedAnswerFilters] = []
-
-    @validator("answer_filters", pre=True)
-    def validate_filters(cls, value: list[str]) -> list[str]:
-        if any(test not in VALID_SLACK_FILTERS for test in value):
-            raise ValueError(
-                f"Slack Answer filters must be one of {VALID_SLACK_FILTERS}"
-            )
-        return value
-
-
-class SlackBotConfig(BaseModel):
-    id: int
-    # currently, a persona is created for each slack bot config
-    # in the future, `document_sets` will probably be replaced
-    # by an optional `PersonaSnapshot` object. Keeping it like this
-    # for now for simplicity / speed of development
-    document_sets: list[DocumentSet]
-    channel_config: ChannelConfig

backend/tests/regression/answer_quality/eval_direct_qa.py

@@ -15,7 +15,7 @@ from danswer.direct_qa.models import LLMMetricsContainer
 from danswer.search.models import IndexFilters
 from danswer.search.models import RerankMetricsContainer
 from danswer.search.models import RetrievalMetricsContainer
-from danswer.server.models import NewMessageRequest
+from danswer.server.chat.models import NewMessageRequest
 from danswer.utils.callbacks import MetricsHander