Improve initial flow

backend/alembic/versions/dbaa756c2ccf_embedding_models.py

@@ -7,13 +7,7 @@ Create Date: 2024-01-25 17:12:31.813160
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy import table, column, String, Integer, Boolean
 
-from danswer.configs.model_configs import DOCUMENT_ENCODER_MODEL
-from danswer.configs.model_configs import DOC_EMBEDDING_DIM
-from danswer.configs.model_configs import NORMALIZE_EMBEDDINGS
-from danswer.configs.model_configs import ASYM_QUERY_PREFIX
-from danswer.configs.model_configs import ASYM_PASSAGE_PREFIX
 from danswer.db.models import IndexModelStatus
 
 # revision identifiers, used by Alembic.
@@ -40,33 +34,6 @@ def upgrade() -> None:
         ),
         sa.PrimaryKeyConstraint("id"),
     )
-    EmbeddingModel = table(
-        "embedding_model",
-        column("id", Integer),
-        column("model_name", String),
-        column("model_dim", Integer),
-        column("normalize", Boolean),
-        column("query_prefix", String),
-        column("passage_prefix", String),
-        column("index_name", String),
-        column(
-            "status", sa.Enum(IndexModelStatus, name="indexmodelstatus", native=False)
-        ),
-    )
-    op.bulk_insert(
-        EmbeddingModel,
-        [
-            {
-                "model_name": DOCUMENT_ENCODER_MODEL,
-                "model_dim": DOC_EMBEDDING_DIM,
-                "normalize": NORMALIZE_EMBEDDINGS,
-                "query_prefix": ASYM_QUERY_PREFIX,
-                "passage_prefix": ASYM_PASSAGE_PREFIX,
-                "index_name": "danswer_chunk",
-                "status": IndexModelStatus.PRESENT,
-            }
-        ],
-    )
     op.add_column(
         "index_attempt",
         sa.Column("embedding_model_id", sa.Integer(), nullable=True),

backend/danswer/configs/model_configs.py

@@ -10,33 +10,38 @@ CHUNK_SIZE = 512
 # Inference/Indexing speed
 # https://huggingface.co/DOCUMENT_ENCODER_MODEL
 # The useable models configured as below must be SentenceTransformer compatible
+# NOTE: DO NOT CHANGE SET THESE UNLESS YOU KNOW WHAT YOU ARE DOING
+# IDEALLY, YOU SHOULD CHANGE EMBEDDING MODELS VIA THE UI
+DEFAULT_DOCUMENT_ENCODER_MODEL = "intfloat/e5-base-v2"
 DOCUMENT_ENCODER_MODEL = (
-    # This is not a good model anymore, but this default needs to be kept for not breaking existing
+    os.environ.get("DOCUMENT_ENCODER_MODEL") or DEFAULT_DOCUMENT_ENCODER_MODEL
-    # deployments, will eventually be retired/swapped for a different default model
-    os.environ.get("DOCUMENT_ENCODER_MODEL")
-    or "thenlper/gte-small"
 )
 # If the below is changed, Vespa deployment must also be changed
-DOC_EMBEDDING_DIM = int(os.environ.get("DOC_EMBEDDING_DIM") or 384)
+DOC_EMBEDDING_DIM = int(os.environ.get("DOC_EMBEDDING_DIM") or 768)
 # Model should be chosen with 512 context size, ideally don't change this
 DOC_EMBEDDING_CONTEXT_SIZE = 512
 NORMALIZE_EMBEDDINGS = (
-    os.environ.get("NORMALIZE_EMBEDDINGS") or "False"
+    os.environ.get("NORMALIZE_EMBEDDINGS") or "true"
 ).lower() == "true"
+
+# Old default model settings, which are needed for an automatic easy upgrade
+OLD_DEFAULT_DOCUMENT_ENCODER_MODEL = "thenlper/gte-small"
+OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM = 384
+OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS = False
+
 # These are only used if reranking is turned off, to normalize the direct retrieval scores for display
 # Currently unused
 SIM_SCORE_RANGE_LOW = float(os.environ.get("SIM_SCORE_RANGE_LOW") or 0.0)
 SIM_SCORE_RANGE_HIGH = float(os.environ.get("SIM_SCORE_RANGE_HIGH") or 1.0)
 # Certain models like e5, BGE, etc use a prefix for asymmetric retrievals (query generally shorter than docs)
-ASYM_QUERY_PREFIX = os.environ.get("ASYM_QUERY_PREFIX", "")
+ASYM_QUERY_PREFIX = os.environ.get("ASYM_QUERY_PREFIX", "query: ")
-ASYM_PASSAGE_PREFIX = os.environ.get("ASYM_PASSAGE_PREFIX", "")
+ASYM_PASSAGE_PREFIX = os.environ.get("ASYM_PASSAGE_PREFIX", "passage: ")
 # Purely an optimization, memory limitation consideration
 BATCH_SIZE_ENCODE_CHUNKS = 8
 # This controls the minimum number of pytorch "threads" to allocate to the embedding
 # model. If torch finds more threads on its own, this value is not used.
 MIN_THREADS_ML_MODELS = int(os.environ.get("MIN_THREADS_ML_MODELS") or 1)
 
-
 # Cross Encoder Settings
 ENABLE_RERANKING_ASYNC_FLOW = (
     os.environ.get("ENABLE_RERANKING_ASYNC_FLOW", "").lower() == "true"

backend/danswer/db/embedding_model.py

@@ -1,6 +1,16 @@
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
+from danswer.configs.model_configs import ASYM_PASSAGE_PREFIX
+from danswer.configs.model_configs import ASYM_QUERY_PREFIX
+from danswer.configs.model_configs import DEFAULT_DOCUMENT_ENCODER_MODEL
+from danswer.configs.model_configs import DOC_EMBEDDING_DIM
+from danswer.configs.model_configs import DOCUMENT_ENCODER_MODEL
+from danswer.configs.model_configs import NORMALIZE_EMBEDDINGS
+from danswer.configs.model_configs import OLD_DEFAULT_DOCUMENT_ENCODER_MODEL
+from danswer.configs.model_configs import OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM
+from danswer.configs.model_configs import OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS
+from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.models import EmbeddingModel
 from danswer.db.models import IndexModelStatus
 from danswer.indexing.models import EmbeddingModelDetail
@@ -65,3 +75,55 @@ def update_embedding_model_status(
 ) -> None:
     embedding_model.status = new_status
     db_session.commit()
+
+
+def insert_initial_embedding_models(db_session: Session) -> None:
+    """Should be called on startup to ensure that the initial
+    embedding model is present in the DB."""
+    existing_embedding_models = db_session.scalars(select(EmbeddingModel)).all()
+    if existing_embedding_models:
+        logger.error(
+            "Called `insert_initial_embedding_models` but models already exist in the DB. Skipping."
+        )
+        return
+
+    existing_cc_pairs = get_connector_credential_pairs(db_session)
+
+    # if the user is overriding the `DOCUMENT_ENCODER_MODEL`, then
+    # allow them to continue to use that model and do nothing fancy
+    # in the background OR if the user has no connectors, then we can
+    # also just use the new model immediately
+    can_skip_upgrade = (
+        DOCUMENT_ENCODER_MODEL != DEFAULT_DOCUMENT_ENCODER_MODEL
+        or not existing_cc_pairs
+    )
+
+    # if we need to automatically upgrade the user, then create
+    # an entry which will automatically be replaced by the
+    # below desired model
+    if not can_skip_upgrade:
+        embedding_model_to_upgrade = EmbeddingModel(
+            model_name=OLD_DEFAULT_DOCUMENT_ENCODER_MODEL,
+            model_dim=OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM,
+            normalize=OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS,
+            query_prefix="",
+            passage_prefix="",
+            status=IndexModelStatus.PRESENT,
+            index_name="danswer_chunk",
+        )
+        db_session.add(embedding_model_to_upgrade)
+
+    desired_embedding_model = EmbeddingModel(
+        model_name=DOCUMENT_ENCODER_MODEL,
+        model_dim=DOC_EMBEDDING_DIM,
+        normalize=NORMALIZE_EMBEDDINGS,
+        query_prefix=ASYM_QUERY_PREFIX,
+        passage_prefix=ASYM_PASSAGE_PREFIX,
+        status=IndexModelStatus.PRESENT
+        if can_skip_upgrade
+        else IndexModelStatus.FUTURE,
+        index_name=f"danswer_chunk_{clean_model_name(DOCUMENT_ENCODER_MODEL)}",
+    )
+    db_session.add(desired_embedding_model)
+
+    db_session.commit()

backend/danswer/llm/utils.py

@@ -192,16 +192,18 @@ def get_gen_ai_api_key() -> str | None:
     return GEN_AI_API_KEY
 
 
-def test_llm(llm: LLM) -> bool:
+def test_llm(llm: LLM) -> str | None:
     # try for up to 2 timeouts (e.g. 10 seconds in total)
+    error_msg = None
     for _ in range(2):
         try:
             llm.invoke("Do not respond")
-            return True
+            return None
         except Exception as e:
-            logger.warning(f"GenAI API key failed for the following reason: {e}")
+            error_msg = str(e)
+            logger.warning(f"Failed to call LLM with the following error: {error_msg}")
 
-    return False
+    return error_msg
 
 
 def get_llm_max_tokens(

backend/danswer/main.py

@@ -44,6 +44,7 @@ from danswer.db.connector_credential_pair import associate_default_cc_pair
 from danswer.db.credentials import create_initial_public_credential
 from danswer.db.embedding_model import get_current_db_embedding_model
 from danswer.db.embedding_model import get_secondary_db_embedding_model
+from danswer.db.embedding_model import insert_initial_embedding_models
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.index_attempt import cancel_indexing_attempts_past_model
 from danswer.document_index.factory import get_default_document_index
@@ -247,9 +248,16 @@ def get_application() -> FastAPI:
             )
 
         with Session(engine) as db_session:
-            db_embedding_model = get_current_db_embedding_model(db_session)
+            try:
+                db_embedding_model = get_current_db_embedding_model(db_session)
+            except RuntimeError:
+                logger.info("No embedding model's found in DB, creating initial model.")
+                insert_initial_embedding_models(db_session)
+                db_embedding_model = get_current_db_embedding_model(db_session)
+
             secondary_db_embedding_model = get_secondary_db_embedding_model(db_session)
 
+            # cleanup "NOT_STARTED" indexing attempts for embedding models that are no longer used
             cancel_indexing_attempts_past_model(db_session)
 
             logger.info(f'Using Embedding model: "{db_embedding_model.model_name}"')

backend/danswer/server/documents/connector.py

@@ -6,6 +6,7 @@ from fastapi import HTTPException
 from fastapi import Request
 from fastapi import Response
 from fastapi import UploadFile
+from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
 from danswer.auth.users import current_admin_user
@@ -689,3 +690,43 @@ def get_connector_by_id(
         time_updated=connector.time_updated,
         disabled=connector.disabled,
     )
+
+
+class BasicCCPairInfo(BaseModel):
+    docs_indexed: int
+    has_successful_run: bool
+    source: DocumentSource
+
+
+@router.get("/indexing-status")
+def get_basic_connector_indexing_status(
+    _: User = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> list[BasicCCPairInfo]:
+    cc_pairs = get_connector_credential_pairs(db_session)
+    cc_pair_identifiers = [
+        ConnectorCredentialPairIdentifier(
+            connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
+        )
+        for cc_pair in cc_pairs
+    ]
+    document_count_info = get_document_cnts_for_cc_pairs(
+        db_session=db_session,
+        cc_pair_identifiers=cc_pair_identifiers,
+    )
+    cc_pair_to_document_cnt = {
+        (connector_id, credential_id): cnt
+        for connector_id, credential_id, cnt in document_count_info
+    }
+    return [
+        BasicCCPairInfo(
+            docs_indexed=cc_pair_to_document_cnt.get(
+                (cc_pair.connector_id, cc_pair.credential_id)
+            )
+            or 0,
+            has_successful_run=cc_pair.last_successful_index_time is not None,
+            source=cc_pair.connector.source,
+        )
+        for cc_pair in cc_pairs
+        if cc_pair.connector.source != DocumentSource.INGESTION_API
+    ]

backend/danswer/server/manage/administrative.py

@@ -9,7 +9,6 @@ from fastapi import HTTPException
 from sqlalchemy.orm import Session
 
 from danswer.auth.users import current_admin_user
-from danswer.configs.app_configs import GENERATIVE_MODEL_ACCESS_CHECK_FREQ
 from danswer.configs.constants import GEN_AI_API_KEY_STORAGE_KEY
 from danswer.db.connector_credential_pair import get_connector_credential_pair
 from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed
@@ -107,7 +106,7 @@ def document_hidden_update(
         raise HTTPException(status_code=400, detail=str(e))
 
 
-@router.head("/admin/genai-api-key/validate")
+@router.get("/admin/genai-api-key/validate")
 def validate_existing_genai_api_key(
     _: User = Depends(current_admin_user),
 ) -> None:
@@ -119,7 +118,8 @@ def validate_existing_genai_api_key(
         last_check = datetime.fromtimestamp(
             cast(float, kv_store.load(check_key_time)), tz=timezone.utc
         )
-        check_freq_sec = timedelta(seconds=GENERATIVE_MODEL_ACCESS_CHECK_FREQ)
+        # GENERATIVE_MODEL_ACCESS_CHECK_FREQ
+        check_freq_sec = timedelta(seconds=1)
         if curr_time - last_check < check_freq_sec:
             return
     except ConfigNotFoundError:
@@ -133,12 +133,12 @@ def validate_existing_genai_api_key(
     except GenAIDisabledException:
         return
 
-    is_valid = test_llm(llm)
+    error_msg = test_llm(llm)
 
-    if not is_valid:
+    if error_msg:
         if genai_api_key is None:
             raise HTTPException(status_code=404, detail="Key not found")
-        raise HTTPException(status_code=400, detail="Invalid API key provided")
+        raise HTTPException(status_code=400, detail=error_msg)
 
     # Mark check as successful
     get_dynamic_config_store().store(check_key_time, curr_time.timestamp())
@@ -172,10 +172,10 @@ def store_genai_api_key(
             raise HTTPException(400, "No API key provided")
 
         llm = get_default_llm(api_key=request.api_key, timeout=10)
-        is_valid = test_llm(llm)
+        error_msg = test_llm(llm)
 
-        if not is_valid:
+        if error_msg:
-            raise HTTPException(400, "Invalid API key provided")
+            raise HTTPException(400, detail=error_msg)
 
         get_dynamic_config_store().store(GEN_AI_API_KEY_STORAGE_KEY, request.api_key)
     except GenAIDisabledException:

web/src/app/chat/Chat.tsx

@@ -481,9 +481,7 @@ export const Chat = ({
     }
   };
 
-  const retrievalDisabled = selectedPersona
+  const retrievalDisabled = !personaIncludesRetrieval(livePersona);
-    ? !personaIncludesRetrieval(selectedPersona)
-    : false;
 
   return (
     <div className="flex w-full overflow-x-hidden" ref={masterFlexboxRef}>

web/src/app/chat/ChatPage.tsx

@@ -8,7 +8,6 @@ import { DocumentSet, Tag, User, ValidSources } from "@/lib/types";
 import { Persona } from "../admin/personas/interfaces";
 import { Header } from "@/components/Header";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
-import { ApiKeyModal } from "@/components/openai/ApiKeyModal";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 
 export function ChatLayout({
@@ -44,7 +43,6 @@ export function ChatLayout({
         <Header user={user} />
       </div>
       <HealthCheckBanner />
-      <ApiKeyModal />
       <InstantSSRAutoRefresh />
 
       <div className="flex relative bg-background text-default overflow-x-hidden">

web/src/app/chat/page.tsx

@@ -5,12 +5,21 @@ import {
 } from "@/lib/userSS";
 import { redirect } from "next/navigation";
 import { fetchSS } from "@/lib/utilsSS";
-import { Connector, DocumentSet, Tag, User, ValidSources } from "@/lib/types";
+import {
+  CCPairBasicInfo,
+  DocumentSet,
+  Tag,
+  User,
+  ValidSources,
+} from "@/lib/types";
 import { ChatSession } from "./interfaces";
 import { unstable_noStore as noStore } from "next/cache";
 import { Persona } from "../admin/personas/interfaces";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
-import { WelcomeModal } from "@/components/WelcomeModal";
+import {
+  WelcomeModal,
+  hasCompletedWelcomeFlowSS,
+} from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 import { ApiKeyModal } from "@/components/openai/ApiKeyModal";
 import { cookies } from "next/headers";
 import { DOCUMENT_SIDEBAR_WIDTH_COOKIE_NAME } from "@/components/resizable/contants";
@@ -21,6 +30,8 @@ import {
   checkModelNameIsValid,
 } from "../admin/models/embedding/embeddingModels";
 import { SwitchModelModal } from "@/components/SwitchModelModal";
+import { NoSourcesModal } from "@/components/initialSetup/search/NoSourcesModal";
+import { NoCompleteSourcesModal } from "@/components/initialSetup/search/NoCompleteSourceModal";
 
 export default async function Page({
   searchParams,
@@ -32,7 +43,7 @@ export default async function Page({
   const tasks = [
     getAuthTypeMetadataSS(),
     getCurrentUserSS(),
-    fetchSS("/manage/connector"),
+    fetchSS("/manage/indexing-status"),
     fetchSS("/manage/document-set"),
     fetchSS("/persona?include_default=true"),
     fetchSS("/chat/get-user-chat-sessions"),
@@ -57,7 +68,7 @@ export default async function Page({
   }
   const authTypeMetadata = results[0] as AuthTypeMetadata | null;
   const user = results[1] as User | null;
-  const connectorsResponse = results[2] as Response | null;
+  const ccPairsResponse = results[2] as Response | null;
   const documentSetsResponse = results[3] as Response | null;
   const personasResponse = results[4] as Response | null;
   const chatSessionsResponse = results[5] as Response | null;
@@ -73,16 +84,16 @@ export default async function Page({
     return redirect("/auth/waiting-on-verification");
   }
 
-  let connectors: Connector<any>[] = [];
+  let ccPairs: CCPairBasicInfo[] = [];
-  if (connectorsResponse?.ok) {
+  if (ccPairsResponse?.ok) {
-    connectors = await connectorsResponse.json();
+    ccPairs = await ccPairsResponse.json();
   } else {
-    console.log(`Failed to fetch connectors - ${connectorsResponse?.status}`);
+    console.log(`Failed to fetch connectors - ${ccPairsResponse?.status}`);
   }
   const availableSources: ValidSources[] = [];
-  connectors.forEach((connector) => {
+  ccPairs.forEach((ccPair) => {
-    if (!availableSources.includes(connector.source)) {
+    if (!availableSources.includes(ccPair.source)) {
-      availableSources.push(connector.source);
+      availableSources.push(ccPair.source);
     }
   });
 
@@ -145,19 +156,31 @@ export default async function Page({
     ? parseInt(documentSidebarCookieInitialWidth.value)
     : undefined;
 
+  const shouldShowWelcomeModal = !hasCompletedWelcomeFlowSS();
+  const hasAnyConnectors = ccPairs.length > 0;
+  const shouldDisplaySourcesIncompleteModal =
+    hasAnyConnectors &&
+    !shouldShowWelcomeModal &&
+    !ccPairs.some(
+      (ccPair) => ccPair.has_successful_run && ccPair.docs_indexed > 0
+    );
+
+  // if no connectors are setup, only show personas that are pure
+  // passthrough and don't do any retrieval
+  if (!hasAnyConnectors) {
+    personas = personas.filter((persona) => persona.num_chunks === 0);
+  }
+
   return (
     <>
       <InstantSSRAutoRefresh />
-      <ApiKeyModal />
 
-      {connectors.length === 0 ? (
+      {shouldShowWelcomeModal && <WelcomeModal />}
-        <WelcomeModal embeddingModelName={currentEmbeddingModelName} />
+      {!shouldShowWelcomeModal && !shouldDisplaySourcesIncompleteModal && (
-      ) : (
+        <ApiKeyModal />
-        embeddingModelVersionInfo &&
+      )}
-        !checkModelNameIsValid(currentEmbeddingModelName) &&
+      {shouldDisplaySourcesIncompleteModal && (
-        !nextEmbeddingModelName && (
+        <NoCompleteSourcesModal ccPairs={ccPairs} />
-          <SwitchModelModal embeddingModelName={currentEmbeddingModelName} />
-        )
       )}
 
       <ChatLayout

web/src/app/search/page.tsx

@@ -9,19 +9,20 @@ import { redirect } from "next/navigation";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { ApiKeyModal } from "@/components/openai/ApiKeyModal";
 import { fetchSS } from "@/lib/utilsSS";
-import { Connector, DocumentSet, Tag, User, ValidSources } from "@/lib/types";
+import { CCPairBasicInfo, DocumentSet, Tag, User } from "@/lib/types";
 import { cookies } from "next/headers";
 import { SearchType } from "@/lib/search/interfaces";
 import { Persona } from "../admin/personas/interfaces";
-import { WelcomeModal } from "@/components/WelcomeModal";
+import {
+  WelcomeModal,
+  hasCompletedWelcomeFlowSS,
+} from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 import { unstable_noStore as noStore } from "next/cache";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 import { personaComparator } from "../admin/personas/lib";
-import {
+import { FullEmbeddingModelResponse } from "../admin/models/embedding/embeddingModels";
-  FullEmbeddingModelResponse,
+import { NoSourcesModal } from "@/components/initialSetup/search/NoSourcesModal";
-  checkModelNameIsValid,
+import { NoCompleteSourcesModal } from "@/components/initialSetup/search/NoCompleteSourceModal";
-} from "../admin/models/embedding/embeddingModels";
-import { SwitchModelModal } from "@/components/SwitchModelModal";
 
 export default async function Home() {
   // Disable caching so we always get the up to date connector / document set / persona info
@@ -32,7 +33,7 @@ export default async function Home() {
   const tasks = [
     getAuthTypeMetadataSS(),
     getCurrentUserSS(),
-    fetchSS("/manage/connector"),
+    fetchSS("/manage/indexing-status"),
     fetchSS("/manage/document-set"),
     fetchSS("/persona"),
     fetchSS("/query/valid-tags"),
@@ -56,7 +57,7 @@ export default async function Home() {
   }
   const authTypeMetadata = results[0] as AuthTypeMetadata | null;
   const user = results[1] as User | null;
-  const connectorsResponse = results[2] as Response | null;
+  const ccPairsResponse = results[2] as Response | null;
   const documentSetsResponse = results[3] as Response | null;
   const personaResponse = results[4] as Response | null;
   const tagsResponse = results[5] as Response | null;
@@ -71,11 +72,11 @@ export default async function Home() {
     return redirect("/auth/waiting-on-verification");
   }
 
-  let connectors: Connector<any>[] = [];
+  let ccPairs: CCPairBasicInfo[] = [];
-  if (connectorsResponse?.ok) {
+  if (ccPairsResponse?.ok) {
-    connectors = await connectorsResponse.json();
+    ccPairs = await ccPairsResponse.json();
   } else {
-    console.log(`Failed to fetch connectors - ${connectorsResponse?.status}`);
+    console.log(`Failed to fetch connectors - ${ccPairsResponse?.status}`);
   }
 
   let documentSets: DocumentSet[] = [];
@@ -126,29 +127,37 @@ export default async function Home() {
       ? (storedSearchType as SearchType)
       : SearchType.SEMANTIC; // default to semantic
 
+  const shouldShowWelcomeModal = !hasCompletedWelcomeFlowSS();
+  const shouldDisplayNoSourcesModal =
+    ccPairs.length === 0 && !shouldShowWelcomeModal;
+  const shouldDisplaySourcesIncompleteModal =
+    !ccPairs.some(
+      (ccPair) => ccPair.has_successful_run && ccPair.docs_indexed > 0
+    ) &&
+    !shouldDisplayNoSourcesModal &&
+    !shouldShowWelcomeModal;
+
   return (
     <>
       <Header user={user} />
       <div className="m-3">
         <HealthCheckBanner />
       </div>
-      <ApiKeyModal />
+      {shouldShowWelcomeModal && <WelcomeModal />}
-      <InstantSSRAutoRefresh />
+      {!shouldShowWelcomeModal &&
-
+        !shouldDisplayNoSourcesModal &&
-      {connectors.length === 0 ? (
+        !shouldDisplaySourcesIncompleteModal && <ApiKeyModal />}
-        <WelcomeModal embeddingModelName={currentEmbeddingModelName} />
+      {shouldDisplayNoSourcesModal && <NoSourcesModal />}
-      ) : (
+      {shouldDisplaySourcesIncompleteModal && (
-        embeddingModelVersionInfo &&
+        <NoCompleteSourcesModal ccPairs={ccPairs} />
-        !checkModelNameIsValid(currentEmbeddingModelName) &&
-        !nextEmbeddingModelName && (
-          <SwitchModelModal embeddingModelName={currentEmbeddingModelName} />
-        )
       )}
 
+      <InstantSSRAutoRefresh />
+
       <div className="px-24 pt-10 flex flex-col items-center min-h-screen">
         <div className="w-full">
           <SearchSection
-            connectors={connectors}
+            ccPairs={ccPairs}
             documentSets={documentSets}
             personas={personas}
             tags={tags}

web/src/components/BackButton.tsx

@@ -4,7 +4,11 @@ import { useRouter } from "next/navigation";
 
 import { FiChevronLeft } from "react-icons/fi";
 
-export function BackButton() {
+export function BackButton({
+  behaviorOverride,
+}: {
+  behaviorOverride?: () => void;
+}) {
   const router = useRouter();
 
   return (
@@ -20,7 +24,13 @@ export function BackButton() {
         cursor-pointer 
         rounded-lg 
         text-sm`}
-      onClick={() => router.back()}
+      onClick={() => {
+        if (behaviorOverride) {
+          behaviorOverride();
+        } else {
+          router.back();
+        }
+      }}
     >
       <FiChevronLeft className="mr-1 my-auto" />
       Back

web/src/components/Modal.tsx

@@ -7,6 +7,8 @@ interface ModalProps {
   onOutsideClick?: () => void;
   className?: string;
   width?: string;
+  titleSize?: string;
+  hideDividerForTitle?: boolean;
 }
 
 export function Modal({
@@ -15,6 +17,8 @@ export function Modal({
   onOutsideClick,
   className,
   width,
+  titleSize,
+  hideDividerForTitle,
 }: ModalProps) {
   return (
     <div>
@@ -36,7 +40,11 @@ export function Modal({
           {title && (
             <>
               <div className="flex mb-4">
-                <h2 className="my-auto text-2xl font-bold">{title}</h2>
+                <h2
+                  className={"my-auto font-bold " + (titleSize || "text-2xl")}
+                >
+                  {title}
+                </h2>
                 {onOutsideClick && (
                   <div
                     onClick={onOutsideClick}
@@ -46,7 +54,7 @@ export function Modal({
                   </div>
                 )}
               </div>
-              <Divider />
+              {!hideDividerForTitle && <Divider />}
             </>
           )}
           {children}

web/src/components/WelcomeModal.tsx

@@ -1,71 +0,0 @@
-"use client";
-
-import { Button, Text } from "@tremor/react";
-import { Modal } from "./Modal";
-import Link from "next/link";
-import { FiCheckCircle } from "react-icons/fi";
-import { checkModelNameIsValid } from "@/app/admin/models/embedding/embeddingModels";
-
-export function WelcomeModal({
-  embeddingModelName,
-}: {
-  embeddingModelName: undefined | null | string;
-}) {
-  const validModelSelected = checkModelNameIsValid(embeddingModelName);
-
-  return (
-    <Modal className="max-w-4xl">
-      <div className="text-base">
-        <h2 className="text-xl font-bold mb-4 pb-2 border-b border-border flex">
-          Welcome to Danswer 🎉
-        </h2>
-        <div>
-          <p>
-            Danswer is the AI-powered search engine for your organization&apos;s
-            internal knowledge. Whenever you need to find any piece of internal
-            information, Danswer is there to help!
-          </p>
-        </div>
-        <div className="flex mt-8 mb-2">
-          {validModelSelected && (
-            <FiCheckCircle className="my-auto mr-2 text-success" />
-          )}
-          <Text className="font-bold">Step 1: Choose Your Embedding Model</Text>
-        </div>
-        {!validModelSelected && (
-          <>
-            To get started, the first step is to choose your{" "}
-            <i>embedding model</i>. This machine learning model helps power
-            Danswer&apos;s search. Different models have different strengths,
-            but don&apos;t worry we&apos;ll guide you through the process of
-            choosing the right one for your organization.
-          </>
-        )}
-        <div className="flex mt-3">
-          <Link href="/admin/models/embedding">
-            <Button size="xs">
-              {validModelSelected
-                ? "Change your Embedding Model"
-                : "Choose your Embedding Model"}
-            </Button>
-          </Link>
-        </div>
-        <Text className="font-bold mt-8 mb-2">
-          Step 2: Add Your First Connector
-        </Text>
-        Next, we need to to configure some <i>connectors</i>. Connectors are the
-        way that Danswer gets data from your organization&apos;s various data
-        sources. Once setup, we&apos;ll automatically sync data from your apps
-        and docs into Danswer, so you can search all through all of them in one
-        place.
-        <div className="flex mt-3">
-          <Link href="/admin/add-connector">
-            <Button size="xs" disabled={!validModelSelected}>
-              Setup your first connector!
-            </Button>
-          </Link>
-        </div>
-      </div>
-    </Modal>
-  );
-}

web/src/components/initialSetup/search/NoCompleteSourceModal.tsx

@@ -0,0 +1,71 @@
+"use client";
+
+import { Modal } from "../../Modal";
+import Link from "next/link";
+import { useEffect, useState } from "react";
+import { CCPairBasicInfo } from "@/lib/types";
+import { useRouter } from "next/navigation";
+
+export function NoCompleteSourcesModal({
+  ccPairs,
+}: {
+  ccPairs: CCPairBasicInfo[];
+}) {
+  const router = useRouter();
+  const [isHidden, setIsHidden] = useState(false);
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      router.refresh();
+    }, 5000);
+
+    return () => clearInterval(interval);
+  }, []);
+
+  if (isHidden) {
+    return null;
+  }
+
+  const totalDocs = ccPairs.reduce(
+    (acc, ccPair) => acc + ccPair.docs_indexed,
+    0
+  );
+
+  return (
+    <Modal
+      className="max-w-4xl"
+      title="⏳ None of your connectors have finished a full sync yet"
+      onOutsideClick={() => setIsHidden(true)}
+    >
+      <div className="text-base">
+        <div>
+          <div>
+            You&apos;ve connected some sources, but none of them have finished
+            syncing. Depending on the size of the knowledge base(s) you&apos;ve
+            connected to Danswer, it can take anywhere between 30 seconds to a
+            few days for the initial sync to complete. So far we&apos;ve synced{" "}
+            <b>{totalDocs}</b> documents.
+            <br />
+            <br />
+            To view the status of your syncing connectors, head over to the{" "}
+            <Link className="text-link" href="admin/indexing/status">
+              Existing Connectors page
+            </Link>
+            .
+            <br />
+            <br />
+            <p
+              className="text-link cursor-pointer inline"
+              onClick={() => {
+                setIsHidden(true);
+              }}
+            >
+              Or, click here to continue and ask questions on the partially
+              synced knowledge set.
+            </p>
+          </div>
+        </div>
+      </div>
+    </Modal>
+  );
+}

web/src/components/initialSetup/search/NoSourcesModal.tsx

@@ -0,0 +1,51 @@
+"use client";
+
+import { Button, Divider } from "@tremor/react";
+import { Modal } from "../../Modal";
+import Link from "next/link";
+import { FiMessageSquare, FiShare2 } from "react-icons/fi";
+import { useState } from "react";
+
+export function NoSourcesModal() {
+  const [isHidden, setIsHidden] = useState(false);
+
+  if (isHidden) {
+    return null;
+  }
+
+  return (
+    <Modal
+      className="max-w-4xl"
+      title="🧐 No sources connected"
+      onOutsideClick={() => setIsHidden(true)}
+    >
+      <div className="text-base">
+        <div>
+          <p>
+            Before using Search you&apos;ll need to connect at least one source.
+            Without any connected knowledge sources, there isn&apos;t anything
+            to search over.
+          </p>
+          <Link href="/admin/add-connector">
+            <Button className="mt-3" size="xs" icon={FiShare2}>
+              Connect a Source!
+            </Button>
+          </Link>
+          <Divider />
+          <div>
+            <p>
+              Or, if you&apos;re looking for a pure ChatGPT-like experience
+              without any organization specific knowledge, then you can head
+              over to the Chat page and start chatting with Danswer right away!
+            </p>
+            <Link href="/chat">
+              <Button className="mt-3" size="xs" icon={FiMessageSquare}>
+                Start Chatting!
+              </Button>
+            </Link>
+          </div>
+        </div>
+      </div>
+    </Modal>
+  );
+}

web/src/components/initialSetup/welcome/WelcomeModal.tsx

@@ -0,0 +1,287 @@
+"use client";
+
+import { Button, Divider, Text } from "@tremor/react";
+import { Modal } from "../../Modal";
+import Link from "next/link";
+import Cookies from "js-cookie";
+import { useRouter } from "next/navigation";
+import { COMPLETED_WELCOME_FLOW_COOKIE } from "./constants";
+import { FiCheckCircle, FiMessageSquare, FiShare2 } from "react-icons/fi";
+import { useEffect, useState } from "react";
+import { BackButton } from "@/components/BackButton";
+import { ApiKeyForm } from "@/components/openai/ApiKeyForm";
+import { checkApiKey } from "@/components/openai/ApiKeyModal";
+
+function setWelcomeFlowComplete() {
+  Cookies.set(COMPLETED_WELCOME_FLOW_COOKIE, "true", { expires: 365 });
+}
+
+export function _CompletedWelcomeFlowDummyComponent() {
+  setWelcomeFlowComplete();
+  return null;
+}
+
+function UsageTypeSection({
+  title,
+  description,
+  callToAction,
+  icon,
+  onClick,
+}: {
+  title: string;
+  description: string | JSX.Element;
+  callToAction: string;
+  icon?: React.ElementType;
+  onClick: () => void;
+}) {
+  return (
+    <div>
+      <Text className="font-bold">{title}</Text>
+      <div className="text-base mt-1 mb-3">{description}</div>
+      <div
+        onClick={(e) => {
+          e.preventDefault();
+          onClick();
+        }}
+      >
+        <div className="text-link font-medium cursor-pointer select-none">
+          {callToAction}
+        </div>
+      </div>
+    </div>
+  );
+}
+
+export function _WelcomeModal() {
+  const router = useRouter();
+  const [selectedFlow, setSelectedFlow] = useState<null | "search" | "chat">(
+    null
+  );
+  const [isHidden, setIsHidden] = useState(false);
+  const [apiKeyVerified, setApiKeyVerified] = useState<boolean>(false);
+
+  useEffect(() => {
+    checkApiKey().then((error) => {
+      if (!error) {
+        setApiKeyVerified(true);
+      }
+    });
+  }, []);
+
+  if (isHidden) {
+    return null;
+  }
+
+  let title;
+  let body;
+  switch (selectedFlow) {
+    case "search":
+      title = undefined;
+      body = (
+        <>
+          <BackButton behaviorOverride={() => setSelectedFlow(null)} />
+          <div className="mt-3">
+            <Text className="font-bold mt-6 mb-2 flex">
+              {apiKeyVerified && (
+                <FiCheckCircle className="my-auto mr-2 text-success" />
+              )}
+              Step 1: Provide OpenAI API Key
+            </Text>
+            <div>
+              {apiKeyVerified ? (
+                <div>
+                  API Key setup complete!
+                  <br /> <br />
+                  If you want to change the key later, you&apos;ll be able to
+                  easily to do so in the Admin Panel.
+                </div>
+              ) : (
+                <ApiKeyForm
+                  handleResponse={async (response) => {
+                    if (response.ok) {
+                      setApiKeyVerified(true);
+                    }
+                  }}
+                />
+              )}
+            </div>
+            <Text className="font-bold mt-6 mb-2">
+              Step 2: Connect Data Sources
+            </Text>
+            <div>
+              <p>
+                Connectors are the way that Danswer gets data from your
+                organization&apos;s various data sources. Once setup, we&apos;ll
+                automatically sync data from your apps and docs into Danswer, so
+                you can search through all of them in one place.
+              </p>
+
+              <div className="flex mt-3">
+                <Link
+                  href="/admin/add-connector"
+                  onClick={(e) => {
+                    e.preventDefault();
+                    setWelcomeFlowComplete();
+                    router.push("/admin/add-connector");
+                  }}
+                  className="w-fit mx-auto"
+                >
+                  <Button size="xs" icon={FiShare2} disabled={!apiKeyVerified}>
+                    Setup your first connector!
+                  </Button>
+                </Link>
+              </div>
+            </div>
+          </div>
+        </>
+      );
+      break;
+    case "chat":
+      title = undefined;
+      body = (
+        <>
+          <BackButton behaviorOverride={() => setSelectedFlow(null)} />
+
+          <div className="mt-3">
+            <div>
+              To start using Danswer as a secure ChatGPT, we just need to
+              configure our LLM!
+              <br />
+              <br />
+              Danswer supports connections with a wide range of LLMs, including
+              self-hosted open-source LLMs. For more details, check out the{" "}
+              <a
+                className="text-link"
+                href="https://docs.danswer.dev/gen_ai_configs/overview"
+              >
+                documentation
+              </a>
+              .
+              <br />
+              <br />
+              If you haven&apos;t done anything special with the Gen AI configs,
+              then we default to use OpenAI.
+            </div>
+
+            <Text className="font-bold mt-6 mb-2 flex">
+              {apiKeyVerified && (
+                <FiCheckCircle className="my-auto mr-2 text-success" />
+              )}
+              Step 1: Provide LLM API Key
+            </Text>
+            <div>
+              {apiKeyVerified ? (
+                <div>
+                  LLM setup complete!
+                  <br /> <br />
+                  If you want to change the key later or choose a different LLM,
+                  you&apos;ll be able to easily to do so in the Admin Panel / by
+                  changing some environment variables.
+                </div>
+              ) : (
+                <ApiKeyForm
+                  handleResponse={async (response) => {
+                    if (response.ok) {
+                      setApiKeyVerified(true);
+                    }
+                  }}
+                />
+              )}
+            </div>
+
+            <Text className="font-bold mt-6 mb-2 flex">
+              Step 2: Start Chatting!
+            </Text>
+
+            <div>
+              Click the button below to start chatting with the LLM setup above!
+              Don&apos;t worry, if you do decide later on you want to connect
+              your organization&apos;s knowledge, you can always do that in the{" "}
+              <Link
+                className="text-link"
+                href="/admin/add-connector"
+                onClick={(e) => {
+                  e.preventDefault();
+                  setWelcomeFlowComplete();
+                  router.push("/admin/add-connector");
+                }}
+              >
+                Admin Panel
+              </Link>
+              .
+            </div>
+
+            <div className="flex mt-3">
+              <Link
+                href="/chat"
+                onClick={(e) => {
+                  e.preventDefault();
+                  setWelcomeFlowComplete();
+                  router.push("/chat");
+                  setIsHidden(true);
+                }}
+                className="w-fit mx-auto"
+              >
+                <Button size="xs" icon={FiShare2} disabled={!apiKeyVerified}>
+                  Start chatting!
+                </Button>
+              </Link>
+            </div>
+          </div>
+        </>
+      );
+      break;
+    default:
+      title = "🎉 Welcome to Danswer";
+      body = (
+        <>
+          <div>
+            <p>How are you planning on using Danswer?</p>
+          </div>
+          <Divider />
+          <UsageTypeSection
+            title="Search / Chat with Knowledge"
+            description={
+              <div>
+                If you&apos;re looking to search through, chat with, or ask
+                direct questions of your organization&apos;s knowledge, then
+                this is the option for you!
+              </div>
+            }
+            callToAction="Get Started"
+            onClick={() => setSelectedFlow("search")}
+          />
+          <Divider />
+          <UsageTypeSection
+            title="Secure ChatGPT"
+            description={
+              <>
+                If you&apos;re looking for a pure ChatGPT-like experience, then
+                this is the option for you!
+              </>
+            }
+            icon={FiMessageSquare}
+            callToAction="Get Started"
+            onClick={() => {
+              setSelectedFlow("chat");
+            }}
+          />
+
+          {/* TODO: add a Slack option here */}
+          {/* <Divider />
+          <UsageTypeSection
+            title="AI-powered Slack Assistant"
+            description="If you're looking to setup a bot to auto-answer questions in Slack"
+            callToAction="Connect your company knowledge!"
+            link="/admin/add-connector"
+          /> */}
+        </>
+      );
+  }
+
+  return (
+    <Modal title={title} className="max-w-4xl">
+      <div className="text-base">{body}</div>
+    </Modal>
+  );
+}

web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx

@@ -0,0 +1,23 @@
+import { cookies } from "next/headers";
+import {
+  _CompletedWelcomeFlowDummyComponent,
+  _WelcomeModal,
+} from "./WelcomeModal";
+import { COMPLETED_WELCOME_FLOW_COOKIE } from "./constants";
+
+export function hasCompletedWelcomeFlowSS() {
+  const cookieStore = cookies();
+  return (
+    cookieStore.get(COMPLETED_WELCOME_FLOW_COOKIE)?.value?.toLowerCase() ===
+    "true"
+  );
+}
+
+export function WelcomeModal() {
+  const hasCompletedWelcomeFlow = hasCompletedWelcomeFlowSS();
+  if (hasCompletedWelcomeFlow) {
+    return <_CompletedWelcomeFlowDummyComponent />;
+  }
+
+  return <_WelcomeModal />;
+}

web/src/components/initialSetup/welcome/constants.ts

@@ -0,0 +1 @@
+export const COMPLETED_WELCOME_FLOW_COOKIE = "completed_welcome_flow";

web/src/components/openai/ApiKeyForm.tsx

@@ -51,25 +51,24 @@ export const ApiKeyForm = ({ handleResponse }: Props) => {
             }
             setTimeout(() => {
               setPopup(null);
-            }, 4000);
+            }, 10000);
           }
         }}
       >
         {({ isSubmitting }) =>
           isSubmitting ? (
-            <LoadingAnimation text="Validating API key" />
+            <div className="text-base">
+              <LoadingAnimation text="Validating API key" />
+            </div>
           ) : (
             <Form>
-              <TextFormField
+              <TextFormField name="apiKey" type="password" label="API Key:" />
-                name="apiKey"
-                type="password"
-                label="OpenAI API Key:"
-              />
               <div className="flex">
                 <Button
+                  size="xs"
                   type="submit"
                   disabled={isSubmitting}
-                  className="w-64 mx-auto"
+                  className="w-48 mx-auto"
                 >
                   Submit
                 </Button>

web/src/components/openai/ApiKeyModal.tsx

@@ -3,46 +3,62 @@
 import { useState, useEffect } from "react";
 import { ApiKeyForm } from "./ApiKeyForm";
 import { Modal } from "../Modal";
-import { Text } from "@tremor/react";
+import { Divider, Text } from "@tremor/react";
+
+export async function checkApiKey() {
+  const response = await fetch("/api/manage/admin/genai-api-key/validate");
+  if (!response.ok && (response.status === 404 || response.status === 400)) {
+    const jsonResponse = await response.json();
+    return jsonResponse.detail;
+  }
+  return null;
+}
 
 export const ApiKeyModal = () => {
-  const [isOpen, setIsOpen] = useState(false);
+  const [errorMsg, setErrorMsg] = useState<string | null>(null);
 
   useEffect(() => {
-    fetch("/api/manage/admin/genai-api-key/validate", {
+    checkApiKey().then((error) => {
-      method: "HEAD",
+      console.log(error);
-    }).then((res) => {
+      if (error) {
-      // show popup if either the API key is not set or the API key is invalid
+        setErrorMsg(error);
-      if (!res.ok && (res.status === 404 || res.status === 400)) {
-        setIsOpen(true);
       }
     });
   }, []);
 
-  if (!isOpen) {
+  if (!errorMsg) {
     return null;
   }
 
   return (
-    <Modal className="max-w-4xl" onOutsideClick={() => setIsOpen(false)}>
+    <Modal
+      title="LLM Key Setup"
+      className="max-w-4xl"
+      onOutsideClick={() => setErrorMsg(null)}
+    >
       <div>
         <div>
-          <Text className="mb-2.5">
+          <div className="mb-2.5 text-base">
-            Can&apos;t find a valid registered OpenAI API key. Please provide
+            Please provide a valid OpenAI API key below in order to start using
-            one to be able to ask questions! Or if you&apos;d rather just look
+            Danswer Search or Danswer Chat.
-            around for now,{" "}
+            <br />
+            <br />
+            Or if you&apos;d rather look around first,{" "}
             <strong
-              onClick={() => setIsOpen(false)}
+              onClick={() => setErrorMsg(null)}
               className="text-link cursor-pointer"
             >
               skip this step
             </strong>
             .
-          </Text>
+          </div>
+
+          <Divider />
+
           <ApiKeyForm
             handleResponse={(response) => {
               if (response.ok) {
-                setIsOpen(false);
+                setErrorMsg(null);
               }
             }}
           />

web/src/components/search/SearchSection.tsx

@@ -4,7 +4,7 @@ import { useRef, useState } from "react";
 import { SearchBar } from "./SearchBar";
 import { SearchResultsDisplay } from "./SearchResultsDisplay";
 import { SourceSelector } from "./filtering/Filters";
-import { Connector, DocumentSet, Tag } from "@/lib/types";
+import { CCPairBasicInfo, Connector, DocumentSet, Tag } from "@/lib/types";
 import {
   DanswerDocument,
   Quote,
@@ -36,7 +36,7 @@ const VALID_QUESTION_RESPONSE_DEFAULT: ValidQuestionResponse = {
 };
 
 interface SearchSectionProps {
-  connectors: Connector<any>[];
+  ccPairs: CCPairBasicInfo[];
   documentSets: DocumentSet[];
   personas: Persona[];
   tags: Tag[];
@@ -44,7 +44,7 @@ interface SearchSectionProps {
 }
 
 export const SearchSection = ({
-  connectors,
+  ccPairs,
   documentSets,
   personas,
   tags,
@@ -72,7 +72,7 @@ export const SearchSection = ({
 
   // Filters
   const filterManager = useFilters();
-  const availableSources = connectors.map((connector) => connector.source);
+  const availableSources = ccPairs.map((ccPair) => ccPair.source);
   const [finalAvailableSources, finalAvailableDocumentSets] =
     computeAvailableFilters({
       selectedPersona: personas.find(
@@ -214,7 +214,7 @@ export const SearchSection = ({
   return (
     <div className="relative max-w-[2000px] xl:max-w-[1430px] mx-auto">
       <div className="absolute left-0 hidden 2xl:block w-52 3xl:w-64">
-        {(connectors.length > 0 || documentSets.length > 0) && (
+        {(ccPairs.length > 0 || documentSets.length > 0) && (
           <SourceSelector
             {...filterManager}
             availableDocumentSets={finalAvailableDocumentSets}

web/src/lib/types.ts

@@ -191,6 +191,12 @@ export interface ConnectorIndexingStatus<
   is_deletable: boolean;
 }
 
+export interface CCPairBasicInfo {
+  docs_indexed: number;
+  has_successful_run: boolean;
+  source: ValidSources;
+}
+
 // CREDENTIALS
 export interface CredentialBase<T> {
   credential_json: T;