diff --git a/backend/alembic/versions/dbaa756c2ccf_embedding_models.py b/backend/alembic/versions/dbaa756c2ccf_embedding_models.py index c7b6fd58db1b..7b104e1217f4 100644 --- a/backend/alembic/versions/dbaa756c2ccf_embedding_models.py +++ b/backend/alembic/versions/dbaa756c2ccf_embedding_models.py @@ -7,13 +7,7 @@ Create Date: 2024-01-25 17:12:31.813160 """ from alembic import op import sqlalchemy as sa -from sqlalchemy import table, column, String, Integer, Boolean -from danswer.configs.model_configs import DOCUMENT_ENCODER_MODEL -from danswer.configs.model_configs import DOC_EMBEDDING_DIM -from danswer.configs.model_configs import NORMALIZE_EMBEDDINGS -from danswer.configs.model_configs import ASYM_QUERY_PREFIX -from danswer.configs.model_configs import ASYM_PASSAGE_PREFIX from danswer.db.models import IndexModelStatus # revision identifiers, used by Alembic. @@ -40,33 +34,6 @@ def upgrade() -> None: ), sa.PrimaryKeyConstraint("id"), ) - EmbeddingModel = table( - "embedding_model", - column("id", Integer), - column("model_name", String), - column("model_dim", Integer), - column("normalize", Boolean), - column("query_prefix", String), - column("passage_prefix", String), - column("index_name", String), - column( - "status", sa.Enum(IndexModelStatus, name="indexmodelstatus", native=False) - ), - ) - op.bulk_insert( - EmbeddingModel, - [ - { - "model_name": DOCUMENT_ENCODER_MODEL, - "model_dim": DOC_EMBEDDING_DIM, - "normalize": NORMALIZE_EMBEDDINGS, - "query_prefix": ASYM_QUERY_PREFIX, - "passage_prefix": ASYM_PASSAGE_PREFIX, - "index_name": "danswer_chunk", - "status": IndexModelStatus.PRESENT, - } - ], - ) op.add_column( "index_attempt", sa.Column("embedding_model_id", sa.Integer(), nullable=True), diff --git a/backend/danswer/configs/model_configs.py b/backend/danswer/configs/model_configs.py index 62563b111c0a..c2dc2c044bed 100644 --- a/backend/danswer/configs/model_configs.py +++ b/backend/danswer/configs/model_configs.py @@ -10,33 +10,38 @@ CHUNK_SIZE = 512 # Inference/Indexing speed # https://huggingface.co/DOCUMENT_ENCODER_MODEL # The useable models configured as below must be SentenceTransformer compatible +# NOTE: DO NOT CHANGE SET THESE UNLESS YOU KNOW WHAT YOU ARE DOING +# IDEALLY, YOU SHOULD CHANGE EMBEDDING MODELS VIA THE UI +DEFAULT_DOCUMENT_ENCODER_MODEL = "intfloat/e5-base-v2" DOCUMENT_ENCODER_MODEL = ( - # This is not a good model anymore, but this default needs to be kept for not breaking existing - # deployments, will eventually be retired/swapped for a different default model - os.environ.get("DOCUMENT_ENCODER_MODEL") - or "thenlper/gte-small" + os.environ.get("DOCUMENT_ENCODER_MODEL") or DEFAULT_DOCUMENT_ENCODER_MODEL ) # If the below is changed, Vespa deployment must also be changed -DOC_EMBEDDING_DIM = int(os.environ.get("DOC_EMBEDDING_DIM") or 384) +DOC_EMBEDDING_DIM = int(os.environ.get("DOC_EMBEDDING_DIM") or 768) # Model should be chosen with 512 context size, ideally don't change this DOC_EMBEDDING_CONTEXT_SIZE = 512 NORMALIZE_EMBEDDINGS = ( - os.environ.get("NORMALIZE_EMBEDDINGS") or "False" + os.environ.get("NORMALIZE_EMBEDDINGS") or "true" ).lower() == "true" + +# Old default model settings, which are needed for an automatic easy upgrade +OLD_DEFAULT_DOCUMENT_ENCODER_MODEL = "thenlper/gte-small" +OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM = 384 +OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS = False + # These are only used if reranking is turned off, to normalize the direct retrieval scores for display # Currently unused SIM_SCORE_RANGE_LOW = float(os.environ.get("SIM_SCORE_RANGE_LOW") or 0.0) SIM_SCORE_RANGE_HIGH = float(os.environ.get("SIM_SCORE_RANGE_HIGH") or 1.0) # Certain models like e5, BGE, etc use a prefix for asymmetric retrievals (query generally shorter than docs) -ASYM_QUERY_PREFIX = os.environ.get("ASYM_QUERY_PREFIX", "") -ASYM_PASSAGE_PREFIX = os.environ.get("ASYM_PASSAGE_PREFIX", "") +ASYM_QUERY_PREFIX = os.environ.get("ASYM_QUERY_PREFIX", "query: ") +ASYM_PASSAGE_PREFIX = os.environ.get("ASYM_PASSAGE_PREFIX", "passage: ") # Purely an optimization, memory limitation consideration BATCH_SIZE_ENCODE_CHUNKS = 8 # This controls the minimum number of pytorch "threads" to allocate to the embedding # model. If torch finds more threads on its own, this value is not used. MIN_THREADS_ML_MODELS = int(os.environ.get("MIN_THREADS_ML_MODELS") or 1) - # Cross Encoder Settings ENABLE_RERANKING_ASYNC_FLOW = ( os.environ.get("ENABLE_RERANKING_ASYNC_FLOW", "").lower() == "true" diff --git a/backend/danswer/db/embedding_model.py b/backend/danswer/db/embedding_model.py index 790763310695..e6b77df5c39d 100644 --- a/backend/danswer/db/embedding_model.py +++ b/backend/danswer/db/embedding_model.py @@ -1,6 +1,16 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from danswer.configs.model_configs import ASYM_PASSAGE_PREFIX +from danswer.configs.model_configs import ASYM_QUERY_PREFIX +from danswer.configs.model_configs import DEFAULT_DOCUMENT_ENCODER_MODEL +from danswer.configs.model_configs import DOC_EMBEDDING_DIM +from danswer.configs.model_configs import DOCUMENT_ENCODER_MODEL +from danswer.configs.model_configs import NORMALIZE_EMBEDDINGS +from danswer.configs.model_configs import OLD_DEFAULT_DOCUMENT_ENCODER_MODEL +from danswer.configs.model_configs import OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM +from danswer.configs.model_configs import OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS +from danswer.db.connector_credential_pair import get_connector_credential_pairs from danswer.db.models import EmbeddingModel from danswer.db.models import IndexModelStatus from danswer.indexing.models import EmbeddingModelDetail @@ -65,3 +75,55 @@ def update_embedding_model_status( ) -> None: embedding_model.status = new_status db_session.commit() + + +def insert_initial_embedding_models(db_session: Session) -> None: + """Should be called on startup to ensure that the initial + embedding model is present in the DB.""" + existing_embedding_models = db_session.scalars(select(EmbeddingModel)).all() + if existing_embedding_models: + logger.error( + "Called `insert_initial_embedding_models` but models already exist in the DB. Skipping." + ) + return + + existing_cc_pairs = get_connector_credential_pairs(db_session) + + # if the user is overriding the `DOCUMENT_ENCODER_MODEL`, then + # allow them to continue to use that model and do nothing fancy + # in the background OR if the user has no connectors, then we can + # also just use the new model immediately + can_skip_upgrade = ( + DOCUMENT_ENCODER_MODEL != DEFAULT_DOCUMENT_ENCODER_MODEL + or not existing_cc_pairs + ) + + # if we need to automatically upgrade the user, then create + # an entry which will automatically be replaced by the + # below desired model + if not can_skip_upgrade: + embedding_model_to_upgrade = EmbeddingModel( + model_name=OLD_DEFAULT_DOCUMENT_ENCODER_MODEL, + model_dim=OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM, + normalize=OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS, + query_prefix="", + passage_prefix="", + status=IndexModelStatus.PRESENT, + index_name="danswer_chunk", + ) + db_session.add(embedding_model_to_upgrade) + + desired_embedding_model = EmbeddingModel( + model_name=DOCUMENT_ENCODER_MODEL, + model_dim=DOC_EMBEDDING_DIM, + normalize=NORMALIZE_EMBEDDINGS, + query_prefix=ASYM_QUERY_PREFIX, + passage_prefix=ASYM_PASSAGE_PREFIX, + status=IndexModelStatus.PRESENT + if can_skip_upgrade + else IndexModelStatus.FUTURE, + index_name=f"danswer_chunk_{clean_model_name(DOCUMENT_ENCODER_MODEL)}", + ) + db_session.add(desired_embedding_model) + + db_session.commit() diff --git a/backend/danswer/llm/utils.py b/backend/danswer/llm/utils.py index 379ccfbae7f5..1f29dd80ef8a 100644 --- a/backend/danswer/llm/utils.py +++ b/backend/danswer/llm/utils.py @@ -192,16 +192,18 @@ def get_gen_ai_api_key() -> str | None: return GEN_AI_API_KEY -def test_llm(llm: LLM) -> bool: +def test_llm(llm: LLM) -> str | None: # try for up to 2 timeouts (e.g. 10 seconds in total) + error_msg = None for _ in range(2): try: llm.invoke("Do not respond") - return True + return None except Exception as e: - logger.warning(f"GenAI API key failed for the following reason: {e}") + error_msg = str(e) + logger.warning(f"Failed to call LLM with the following error: {error_msg}") - return False + return error_msg def get_llm_max_tokens( diff --git a/backend/danswer/main.py b/backend/danswer/main.py index 5f8bdd85b569..cd5addac6178 100644 --- a/backend/danswer/main.py +++ b/backend/danswer/main.py @@ -44,6 +44,7 @@ from danswer.db.connector_credential_pair import associate_default_cc_pair from danswer.db.credentials import create_initial_public_credential from danswer.db.embedding_model import get_current_db_embedding_model from danswer.db.embedding_model import get_secondary_db_embedding_model +from danswer.db.embedding_model import insert_initial_embedding_models from danswer.db.engine import get_sqlalchemy_engine from danswer.db.index_attempt import cancel_indexing_attempts_past_model from danswer.document_index.factory import get_default_document_index @@ -247,9 +248,16 @@ def get_application() -> FastAPI: ) with Session(engine) as db_session: - db_embedding_model = get_current_db_embedding_model(db_session) + try: + db_embedding_model = get_current_db_embedding_model(db_session) + except RuntimeError: + logger.info("No embedding model's found in DB, creating initial model.") + insert_initial_embedding_models(db_session) + db_embedding_model = get_current_db_embedding_model(db_session) + secondary_db_embedding_model = get_secondary_db_embedding_model(db_session) + # cleanup "NOT_STARTED" indexing attempts for embedding models that are no longer used cancel_indexing_attempts_past_model(db_session) logger.info(f'Using Embedding model: "{db_embedding_model.model_name}"') diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py index a5fa93d40214..8c3e50936e95 100644 --- a/backend/danswer/server/documents/connector.py +++ b/backend/danswer/server/documents/connector.py @@ -6,6 +6,7 @@ from fastapi import HTTPException from fastapi import Request from fastapi import Response from fastapi import UploadFile +from pydantic import BaseModel from sqlalchemy.orm import Session from danswer.auth.users import current_admin_user @@ -689,3 +690,43 @@ def get_connector_by_id( time_updated=connector.time_updated, disabled=connector.disabled, ) + + +class BasicCCPairInfo(BaseModel): + docs_indexed: int + has_successful_run: bool + source: DocumentSource + + +@router.get("/indexing-status") +def get_basic_connector_indexing_status( + _: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> list[BasicCCPairInfo]: + cc_pairs = get_connector_credential_pairs(db_session) + cc_pair_identifiers = [ + ConnectorCredentialPairIdentifier( + connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id + ) + for cc_pair in cc_pairs + ] + document_count_info = get_document_cnts_for_cc_pairs( + db_session=db_session, + cc_pair_identifiers=cc_pair_identifiers, + ) + cc_pair_to_document_cnt = { + (connector_id, credential_id): cnt + for connector_id, credential_id, cnt in document_count_info + } + return [ + BasicCCPairInfo( + docs_indexed=cc_pair_to_document_cnt.get( + (cc_pair.connector_id, cc_pair.credential_id) + ) + or 0, + has_successful_run=cc_pair.last_successful_index_time is not None, + source=cc_pair.connector.source, + ) + for cc_pair in cc_pairs + if cc_pair.connector.source != DocumentSource.INGESTION_API + ] diff --git a/backend/danswer/server/manage/administrative.py b/backend/danswer/server/manage/administrative.py index 2d12ca148e04..e33c428bdeb9 100644 --- a/backend/danswer/server/manage/administrative.py +++ b/backend/danswer/server/manage/administrative.py @@ -9,7 +9,6 @@ from fastapi import HTTPException from sqlalchemy.orm import Session from danswer.auth.users import current_admin_user -from danswer.configs.app_configs import GENERATIVE_MODEL_ACCESS_CHECK_FREQ from danswer.configs.constants import GEN_AI_API_KEY_STORAGE_KEY from danswer.db.connector_credential_pair import get_connector_credential_pair from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed @@ -107,7 +106,7 @@ def document_hidden_update( raise HTTPException(status_code=400, detail=str(e)) -@router.head("/admin/genai-api-key/validate") +@router.get("/admin/genai-api-key/validate") def validate_existing_genai_api_key( _: User = Depends(current_admin_user), ) -> None: @@ -119,7 +118,8 @@ def validate_existing_genai_api_key( last_check = datetime.fromtimestamp( cast(float, kv_store.load(check_key_time)), tz=timezone.utc ) - check_freq_sec = timedelta(seconds=GENERATIVE_MODEL_ACCESS_CHECK_FREQ) + # GENERATIVE_MODEL_ACCESS_CHECK_FREQ + check_freq_sec = timedelta(seconds=1) if curr_time - last_check < check_freq_sec: return except ConfigNotFoundError: @@ -133,12 +133,12 @@ def validate_existing_genai_api_key( except GenAIDisabledException: return - is_valid = test_llm(llm) + error_msg = test_llm(llm) - if not is_valid: + if error_msg: if genai_api_key is None: raise HTTPException(status_code=404, detail="Key not found") - raise HTTPException(status_code=400, detail="Invalid API key provided") + raise HTTPException(status_code=400, detail=error_msg) # Mark check as successful get_dynamic_config_store().store(check_key_time, curr_time.timestamp()) @@ -172,10 +172,10 @@ def store_genai_api_key( raise HTTPException(400, "No API key provided") llm = get_default_llm(api_key=request.api_key, timeout=10) - is_valid = test_llm(llm) + error_msg = test_llm(llm) - if not is_valid: - raise HTTPException(400, "Invalid API key provided") + if error_msg: + raise HTTPException(400, detail=error_msg) get_dynamic_config_store().store(GEN_AI_API_KEY_STORAGE_KEY, request.api_key) except GenAIDisabledException: diff --git a/web/src/app/chat/Chat.tsx b/web/src/app/chat/Chat.tsx index 331eb40f64a8..6ca2ee4ed02d 100644 --- a/web/src/app/chat/Chat.tsx +++ b/web/src/app/chat/Chat.tsx @@ -481,9 +481,7 @@ export const Chat = ({ } }; - const retrievalDisabled = selectedPersona - ? !personaIncludesRetrieval(selectedPersona) - : false; + const retrievalDisabled = !personaIncludesRetrieval(livePersona); return (
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx index 97ed47fad986..980a98bc360b 100644 --- a/web/src/app/chat/ChatPage.tsx +++ b/web/src/app/chat/ChatPage.tsx @@ -8,7 +8,6 @@ import { DocumentSet, Tag, User, ValidSources } from "@/lib/types"; import { Persona } from "../admin/personas/interfaces"; import { Header } from "@/components/Header"; import { HealthCheckBanner } from "@/components/health/healthcheck"; -import { ApiKeyModal } from "@/components/openai/ApiKeyModal"; import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh"; export function ChatLayout({ @@ -44,7 +43,6 @@ export function ChatLayout({
-
diff --git a/web/src/app/chat/page.tsx b/web/src/app/chat/page.tsx index 5099c7226d40..24c7a442db2a 100644 --- a/web/src/app/chat/page.tsx +++ b/web/src/app/chat/page.tsx @@ -5,12 +5,21 @@ import { } from "@/lib/userSS"; import { redirect } from "next/navigation"; import { fetchSS } from "@/lib/utilsSS"; -import { Connector, DocumentSet, Tag, User, ValidSources } from "@/lib/types"; +import { + CCPairBasicInfo, + DocumentSet, + Tag, + User, + ValidSources, +} from "@/lib/types"; import { ChatSession } from "./interfaces"; import { unstable_noStore as noStore } from "next/cache"; import { Persona } from "../admin/personas/interfaces"; import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh"; -import { WelcomeModal } from "@/components/WelcomeModal"; +import { + WelcomeModal, + hasCompletedWelcomeFlowSS, +} from "@/components/initialSetup/welcome/WelcomeModalWrapper"; import { ApiKeyModal } from "@/components/openai/ApiKeyModal"; import { cookies } from "next/headers"; import { DOCUMENT_SIDEBAR_WIDTH_COOKIE_NAME } from "@/components/resizable/contants"; @@ -21,6 +30,8 @@ import { checkModelNameIsValid, } from "../admin/models/embedding/embeddingModels"; import { SwitchModelModal } from "@/components/SwitchModelModal"; +import { NoSourcesModal } from "@/components/initialSetup/search/NoSourcesModal"; +import { NoCompleteSourcesModal } from "@/components/initialSetup/search/NoCompleteSourceModal"; export default async function Page({ searchParams, @@ -32,7 +43,7 @@ export default async function Page({ const tasks = [ getAuthTypeMetadataSS(), getCurrentUserSS(), - fetchSS("/manage/connector"), + fetchSS("/manage/indexing-status"), fetchSS("/manage/document-set"), fetchSS("/persona?include_default=true"), fetchSS("/chat/get-user-chat-sessions"), @@ -57,7 +68,7 @@ export default async function Page({ } const authTypeMetadata = results[0] as AuthTypeMetadata | null; const user = results[1] as User | null; - const connectorsResponse = results[2] as Response | null; + const ccPairsResponse = results[2] as Response | null; const documentSetsResponse = results[3] as Response | null; const personasResponse = results[4] as Response | null; const chatSessionsResponse = results[5] as Response | null; @@ -73,16 +84,16 @@ export default async function Page({ return redirect("/auth/waiting-on-verification"); } - let connectors: Connector[] = []; - if (connectorsResponse?.ok) { - connectors = await connectorsResponse.json(); + let ccPairs: CCPairBasicInfo[] = []; + if (ccPairsResponse?.ok) { + ccPairs = await ccPairsResponse.json(); } else { - console.log(`Failed to fetch connectors - ${connectorsResponse?.status}`); + console.log(`Failed to fetch connectors - ${ccPairsResponse?.status}`); } const availableSources: ValidSources[] = []; - connectors.forEach((connector) => { - if (!availableSources.includes(connector.source)) { - availableSources.push(connector.source); + ccPairs.forEach((ccPair) => { + if (!availableSources.includes(ccPair.source)) { + availableSources.push(ccPair.source); } }); @@ -145,19 +156,31 @@ export default async function Page({ ? parseInt(documentSidebarCookieInitialWidth.value) : undefined; + const shouldShowWelcomeModal = !hasCompletedWelcomeFlowSS(); + const hasAnyConnectors = ccPairs.length > 0; + const shouldDisplaySourcesIncompleteModal = + hasAnyConnectors && + !shouldShowWelcomeModal && + !ccPairs.some( + (ccPair) => ccPair.has_successful_run && ccPair.docs_indexed > 0 + ); + + // if no connectors are setup, only show personas that are pure + // passthrough and don't do any retrieval + if (!hasAnyConnectors) { + personas = personas.filter((persona) => persona.num_chunks === 0); + } + return ( <> - - {connectors.length === 0 ? ( - - ) : ( - embeddingModelVersionInfo && - !checkModelNameIsValid(currentEmbeddingModelName) && - !nextEmbeddingModelName && ( - - ) + {shouldShowWelcomeModal && } + {!shouldShowWelcomeModal && !shouldDisplaySourcesIncompleteModal && ( + + )} + {shouldDisplaySourcesIncompleteModal && ( + )} [] = []; - if (connectorsResponse?.ok) { - connectors = await connectorsResponse.json(); + let ccPairs: CCPairBasicInfo[] = []; + if (ccPairsResponse?.ok) { + ccPairs = await ccPairsResponse.json(); } else { - console.log(`Failed to fetch connectors - ${connectorsResponse?.status}`); + console.log(`Failed to fetch connectors - ${ccPairsResponse?.status}`); } let documentSets: DocumentSet[] = []; @@ -126,29 +127,37 @@ export default async function Home() { ? (storedSearchType as SearchType) : SearchType.SEMANTIC; // default to semantic + const shouldShowWelcomeModal = !hasCompletedWelcomeFlowSS(); + const shouldDisplayNoSourcesModal = + ccPairs.length === 0 && !shouldShowWelcomeModal; + const shouldDisplaySourcesIncompleteModal = + !ccPairs.some( + (ccPair) => ccPair.has_successful_run && ccPair.docs_indexed > 0 + ) && + !shouldDisplayNoSourcesModal && + !shouldShowWelcomeModal; + return ( <>
- - - - {connectors.length === 0 ? ( - - ) : ( - embeddingModelVersionInfo && - !checkModelNameIsValid(currentEmbeddingModelName) && - !nextEmbeddingModelName && ( - - ) + {shouldShowWelcomeModal && } + {!shouldShowWelcomeModal && + !shouldDisplayNoSourcesModal && + !shouldDisplaySourcesIncompleteModal && } + {shouldDisplayNoSourcesModal && } + {shouldDisplaySourcesIncompleteModal && ( + )} + +
void; +}) { const router = useRouter(); return ( @@ -20,7 +24,13 @@ export function BackButton() { cursor-pointer rounded-lg text-sm`} - onClick={() => router.back()} + onClick={() => { + if (behaviorOverride) { + behaviorOverride(); + } else { + router.back(); + } + }} > Back diff --git a/web/src/components/Modal.tsx b/web/src/components/Modal.tsx index aa08b0e4bd7d..645c418a92cb 100644 --- a/web/src/components/Modal.tsx +++ b/web/src/components/Modal.tsx @@ -7,6 +7,8 @@ interface ModalProps { onOutsideClick?: () => void; className?: string; width?: string; + titleSize?: string; + hideDividerForTitle?: boolean; } export function Modal({ @@ -15,6 +17,8 @@ export function Modal({ onOutsideClick, className, width, + titleSize, + hideDividerForTitle, }: ModalProps) { return (
@@ -36,7 +40,11 @@ export function Modal({ {title && ( <>
-

{title}

+

+ {title} +

{onOutsideClick && (
)}
- + {!hideDividerForTitle && } )} {children} diff --git a/web/src/components/WelcomeModal.tsx b/web/src/components/WelcomeModal.tsx deleted file mode 100644 index 8b0fd2b0b146..000000000000 --- a/web/src/components/WelcomeModal.tsx +++ /dev/null @@ -1,71 +0,0 @@ -"use client"; - -import { Button, Text } from "@tremor/react"; -import { Modal } from "./Modal"; -import Link from "next/link"; -import { FiCheckCircle } from "react-icons/fi"; -import { checkModelNameIsValid } from "@/app/admin/models/embedding/embeddingModels"; - -export function WelcomeModal({ - embeddingModelName, -}: { - embeddingModelName: undefined | null | string; -}) { - const validModelSelected = checkModelNameIsValid(embeddingModelName); - - return ( - -
-

- Welcome to Danswer 🎉 -

-
-

- Danswer is the AI-powered search engine for your organization's - internal knowledge. Whenever you need to find any piece of internal - information, Danswer is there to help! -

-
-
- {validModelSelected && ( - - )} - Step 1: Choose Your Embedding Model -
- {!validModelSelected && ( - <> - To get started, the first step is to choose your{" "} - embedding model. This machine learning model helps power - Danswer's search. Different models have different strengths, - but don't worry we'll guide you through the process of - choosing the right one for your organization. - - )} -
- - - -
- - Step 2: Add Your First Connector - - Next, we need to to configure some connectors. Connectors are the - way that Danswer gets data from your organization's various data - sources. Once setup, we'll automatically sync data from your apps - and docs into Danswer, so you can search all through all of them in one - place. -
- - - -
-
-
- ); -} diff --git a/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx b/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx new file mode 100644 index 000000000000..2a43a9e73775 --- /dev/null +++ b/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx @@ -0,0 +1,71 @@ +"use client"; + +import { Modal } from "../../Modal"; +import Link from "next/link"; +import { useEffect, useState } from "react"; +import { CCPairBasicInfo } from "@/lib/types"; +import { useRouter } from "next/navigation"; + +export function NoCompleteSourcesModal({ + ccPairs, +}: { + ccPairs: CCPairBasicInfo[]; +}) { + const router = useRouter(); + const [isHidden, setIsHidden] = useState(false); + + useEffect(() => { + const interval = setInterval(() => { + router.refresh(); + }, 5000); + + return () => clearInterval(interval); + }, []); + + if (isHidden) { + return null; + } + + const totalDocs = ccPairs.reduce( + (acc, ccPair) => acc + ccPair.docs_indexed, + 0 + ); + + return ( + setIsHidden(true)} + > +
+
+
+ You've connected some sources, but none of them have finished + syncing. Depending on the size of the knowledge base(s) you've + connected to Danswer, it can take anywhere between 30 seconds to a + few days for the initial sync to complete. So far we've synced{" "} + {totalDocs} documents. +
+
+ To view the status of your syncing connectors, head over to the{" "} + + Existing Connectors page + + . +
+
+

{ + setIsHidden(true); + }} + > + Or, click here to continue and ask questions on the partially + synced knowledge set. +

+
+
+
+
+ ); +} diff --git a/web/src/components/initialSetup/search/NoSourcesModal.tsx b/web/src/components/initialSetup/search/NoSourcesModal.tsx new file mode 100644 index 000000000000..7510c9393a10 --- /dev/null +++ b/web/src/components/initialSetup/search/NoSourcesModal.tsx @@ -0,0 +1,51 @@ +"use client"; + +import { Button, Divider } from "@tremor/react"; +import { Modal } from "../../Modal"; +import Link from "next/link"; +import { FiMessageSquare, FiShare2 } from "react-icons/fi"; +import { useState } from "react"; + +export function NoSourcesModal() { + const [isHidden, setIsHidden] = useState(false); + + if (isHidden) { + return null; + } + + return ( + setIsHidden(true)} + > +
+
+

+ Before using Search you'll need to connect at least one source. + Without any connected knowledge sources, there isn't anything + to search over. +

+ + + + +
+

+ Or, if you're looking for a pure ChatGPT-like experience + without any organization specific knowledge, then you can head + over to the Chat page and start chatting with Danswer right away! +

+ + + +
+
+
+
+ ); +} diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx new file mode 100644 index 000000000000..21b7b9f7d39f --- /dev/null +++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx @@ -0,0 +1,287 @@ +"use client"; + +import { Button, Divider, Text } from "@tremor/react"; +import { Modal } from "../../Modal"; +import Link from "next/link"; +import Cookies from "js-cookie"; +import { useRouter } from "next/navigation"; +import { COMPLETED_WELCOME_FLOW_COOKIE } from "./constants"; +import { FiCheckCircle, FiMessageSquare, FiShare2 } from "react-icons/fi"; +import { useEffect, useState } from "react"; +import { BackButton } from "@/components/BackButton"; +import { ApiKeyForm } from "@/components/openai/ApiKeyForm"; +import { checkApiKey } from "@/components/openai/ApiKeyModal"; + +function setWelcomeFlowComplete() { + Cookies.set(COMPLETED_WELCOME_FLOW_COOKIE, "true", { expires: 365 }); +} + +export function _CompletedWelcomeFlowDummyComponent() { + setWelcomeFlowComplete(); + return null; +} + +function UsageTypeSection({ + title, + description, + callToAction, + icon, + onClick, +}: { + title: string; + description: string | JSX.Element; + callToAction: string; + icon?: React.ElementType; + onClick: () => void; +}) { + return ( +
+ {title} +
{description}
+
{ + e.preventDefault(); + onClick(); + }} + > +
+ {callToAction} +
+
+
+ ); +} + +export function _WelcomeModal() { + const router = useRouter(); + const [selectedFlow, setSelectedFlow] = useState( + null + ); + const [isHidden, setIsHidden] = useState(false); + const [apiKeyVerified, setApiKeyVerified] = useState(false); + + useEffect(() => { + checkApiKey().then((error) => { + if (!error) { + setApiKeyVerified(true); + } + }); + }, []); + + if (isHidden) { + return null; + } + + let title; + let body; + switch (selectedFlow) { + case "search": + title = undefined; + body = ( + <> + setSelectedFlow(null)} /> +
+ + {apiKeyVerified && ( + + )} + Step 1: Provide OpenAI API Key + +
+ {apiKeyVerified ? ( +
+ API Key setup complete! +

+ If you want to change the key later, you'll be able to + easily to do so in the Admin Panel. +
+ ) : ( + { + if (response.ok) { + setApiKeyVerified(true); + } + }} + /> + )} +
+ + Step 2: Connect Data Sources + +
+

+ Connectors are the way that Danswer gets data from your + organization's various data sources. Once setup, we'll + automatically sync data from your apps and docs into Danswer, so + you can search through all of them in one place. +

+ +
+ { + e.preventDefault(); + setWelcomeFlowComplete(); + router.push("/admin/add-connector"); + }} + className="w-fit mx-auto" + > + + +
+
+
+ + ); + break; + case "chat": + title = undefined; + body = ( + <> + setSelectedFlow(null)} /> + +
+
+ To start using Danswer as a secure ChatGPT, we just need to + configure our LLM! +
+
+ Danswer supports connections with a wide range of LLMs, including + self-hosted open-source LLMs. For more details, check out the{" "} + + documentation + + . +
+
+ If you haven't done anything special with the Gen AI configs, + then we default to use OpenAI. +
+ + + {apiKeyVerified && ( + + )} + Step 1: Provide LLM API Key + +
+ {apiKeyVerified ? ( +
+ LLM setup complete! +

+ If you want to change the key later or choose a different LLM, + you'll be able to easily to do so in the Admin Panel / by + changing some environment variables. +
+ ) : ( + { + if (response.ok) { + setApiKeyVerified(true); + } + }} + /> + )} +
+ + + Step 2: Start Chatting! + + +
+ Click the button below to start chatting with the LLM setup above! + Don't worry, if you do decide later on you want to connect + your organization's knowledge, you can always do that in the{" "} + { + e.preventDefault(); + setWelcomeFlowComplete(); + router.push("/admin/add-connector"); + }} + > + Admin Panel + + . +
+ +
+ { + e.preventDefault(); + setWelcomeFlowComplete(); + router.push("/chat"); + setIsHidden(true); + }} + className="w-fit mx-auto" + > + + +
+
+ + ); + break; + default: + title = "🎉 Welcome to Danswer"; + body = ( + <> +
+

How are you planning on using Danswer?

+
+ + + If you're looking to search through, chat with, or ask + direct questions of your organization's knowledge, then + this is the option for you! +
+ } + callToAction="Get Started" + onClick={() => setSelectedFlow("search")} + /> + + + If you're looking for a pure ChatGPT-like experience, then + this is the option for you! + + } + icon={FiMessageSquare} + callToAction="Get Started" + onClick={() => { + setSelectedFlow("chat"); + }} + /> + + {/* TODO: add a Slack option here */} + {/* + */} + + ); + } + + return ( + +
{body}
+
+ ); +} diff --git a/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx b/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx new file mode 100644 index 000000000000..a3d3aa9bf93b --- /dev/null +++ b/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx @@ -0,0 +1,23 @@ +import { cookies } from "next/headers"; +import { + _CompletedWelcomeFlowDummyComponent, + _WelcomeModal, +} from "./WelcomeModal"; +import { COMPLETED_WELCOME_FLOW_COOKIE } from "./constants"; + +export function hasCompletedWelcomeFlowSS() { + const cookieStore = cookies(); + return ( + cookieStore.get(COMPLETED_WELCOME_FLOW_COOKIE)?.value?.toLowerCase() === + "true" + ); +} + +export function WelcomeModal() { + const hasCompletedWelcomeFlow = hasCompletedWelcomeFlowSS(); + if (hasCompletedWelcomeFlow) { + return <_CompletedWelcomeFlowDummyComponent />; + } + + return <_WelcomeModal />; +} diff --git a/web/src/components/initialSetup/welcome/constants.ts b/web/src/components/initialSetup/welcome/constants.ts new file mode 100644 index 000000000000..cc077d37ba1e --- /dev/null +++ b/web/src/components/initialSetup/welcome/constants.ts @@ -0,0 +1 @@ +export const COMPLETED_WELCOME_FLOW_COOKIE = "completed_welcome_flow"; diff --git a/web/src/components/openai/ApiKeyForm.tsx b/web/src/components/openai/ApiKeyForm.tsx index 77da6ad72445..7861bd8d10b0 100644 --- a/web/src/components/openai/ApiKeyForm.tsx +++ b/web/src/components/openai/ApiKeyForm.tsx @@ -51,25 +51,24 @@ export const ApiKeyForm = ({ handleResponse }: Props) => { } setTimeout(() => { setPopup(null); - }, 4000); + }, 10000); } }} > {({ isSubmitting }) => isSubmitting ? ( - +
+ +
) : (
- +
diff --git a/web/src/components/openai/ApiKeyModal.tsx b/web/src/components/openai/ApiKeyModal.tsx index 8ed58bd45405..a0bc5dc56e81 100644 --- a/web/src/components/openai/ApiKeyModal.tsx +++ b/web/src/components/openai/ApiKeyModal.tsx @@ -3,46 +3,62 @@ import { useState, useEffect } from "react"; import { ApiKeyForm } from "./ApiKeyForm"; import { Modal } from "../Modal"; -import { Text } from "@tremor/react"; +import { Divider, Text } from "@tremor/react"; + +export async function checkApiKey() { + const response = await fetch("/api/manage/admin/genai-api-key/validate"); + if (!response.ok && (response.status === 404 || response.status === 400)) { + const jsonResponse = await response.json(); + return jsonResponse.detail; + } + return null; +} export const ApiKeyModal = () => { - const [isOpen, setIsOpen] = useState(false); + const [errorMsg, setErrorMsg] = useState(null); useEffect(() => { - fetch("/api/manage/admin/genai-api-key/validate", { - method: "HEAD", - }).then((res) => { - // show popup if either the API key is not set or the API key is invalid - if (!res.ok && (res.status === 404 || res.status === 400)) { - setIsOpen(true); + checkApiKey().then((error) => { + console.log(error); + if (error) { + setErrorMsg(error); } }); }, []); - if (!isOpen) { + if (!errorMsg) { return null; } return ( - setIsOpen(false)}> + setErrorMsg(null)} + >
- - Can't find a valid registered OpenAI API key. Please provide - one to be able to ask questions! Or if you'd rather just look - around for now,{" "} +
+ Please provide a valid OpenAI API key below in order to start using + Danswer Search or Danswer Chat. +
+
+ Or if you'd rather look around first,{" "} setIsOpen(false)} + onClick={() => setErrorMsg(null)} className="text-link cursor-pointer" > skip this step . - +
+ + + { if (response.ok) { - setIsOpen(false); + setErrorMsg(null); } }} /> diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx index 383d6a936f02..ea07066ade89 100644 --- a/web/src/components/search/SearchSection.tsx +++ b/web/src/components/search/SearchSection.tsx @@ -4,7 +4,7 @@ import { useRef, useState } from "react"; import { SearchBar } from "./SearchBar"; import { SearchResultsDisplay } from "./SearchResultsDisplay"; import { SourceSelector } from "./filtering/Filters"; -import { Connector, DocumentSet, Tag } from "@/lib/types"; +import { CCPairBasicInfo, Connector, DocumentSet, Tag } from "@/lib/types"; import { DanswerDocument, Quote, @@ -36,7 +36,7 @@ const VALID_QUESTION_RESPONSE_DEFAULT: ValidQuestionResponse = { }; interface SearchSectionProps { - connectors: Connector[]; + ccPairs: CCPairBasicInfo[]; documentSets: DocumentSet[]; personas: Persona[]; tags: Tag[]; @@ -44,7 +44,7 @@ interface SearchSectionProps { } export const SearchSection = ({ - connectors, + ccPairs, documentSets, personas, tags, @@ -72,7 +72,7 @@ export const SearchSection = ({ // Filters const filterManager = useFilters(); - const availableSources = connectors.map((connector) => connector.source); + const availableSources = ccPairs.map((ccPair) => ccPair.source); const [finalAvailableSources, finalAvailableDocumentSets] = computeAvailableFilters({ selectedPersona: personas.find( @@ -214,7 +214,7 @@ export const SearchSection = ({ return (
- {(connectors.length > 0 || documentSets.length > 0) && ( + {(ccPairs.length > 0 || documentSets.length > 0) && ( { credential_json: T;