Add Filters to UI (#86)

* Adding filters

* Fix get_connector_indexing_status endpoint bug

backend/danswer/server/manage.py

@@ -213,7 +213,10 @@ def get_connector_indexing_status(
     for index_attempt in index_attempts:
         # don't consider index attempts where the connector has been deleted
         # or the credential has been deleted
-        if index_attempt.connector_id and index_attempt.credential_id:
+        if (
+            index_attempt.connector_id is not None
+            and index_attempt.credential_id is not None
+        ):
             connector_credential_pair_to_index_attempts[
                 (index_attempt.connector_id, index_attempt.credential_id)
             ].append(index_attempt)

backend/danswer/server/models.py

@@ -7,6 +7,7 @@ from typing import TypeVar
 
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.models import InputType
+from danswer.datastores.interfaces import IndexFilter
 from danswer.db.models import Connector
 from danswer.db.models import IndexingStatus
 from pydantic import BaseModel
@@ -77,7 +78,7 @@ class QuestionRequest(BaseModel):
     query: str
     collection: str
     use_keyword: bool | None
-    filters: str | None  # string of list[IndexFilter]
+    filters: list[IndexFilter] | None
 
 
 class SearchResponse(BaseModel):

backend/danswer/server/search_backend.py

@@ -27,13 +27,13 @@ logger = setup_logger()
 router = APIRouter()
 
 
-@router.get("/semantic-search")
+@router.post("/semantic-search")
 def semantic_search(
-    question: QuestionRequest = Depends(), user: User = Depends(current_user)
+    question: QuestionRequest, user: User = Depends(current_user)
 ) -> SearchResponse:
     query = question.query
     collection = question.collection
-    filters = json.loads(question.filters) if question.filters is not None else None
+    filters = question.filters
     logger.info(f"Received semantic search query: {query}")
 
     user_id = None if user is None else int(user.id)
@@ -49,13 +49,13 @@ def semantic_search(
     return SearchResponse(top_ranked_docs=top_docs, semi_ranked_docs=other_top_docs)
 
 
-@router.get("/keyword-search", response_model=SearchResponse)
+@router.post("/keyword-search")
 def keyword_search(
-    question: QuestionRequest = Depends(), user: User = Depends(current_user)
+    question: QuestionRequest, user: User = Depends(current_user)
 ) -> SearchResponse:
     query = question.query
     collection = question.collection
-    filters = json.loads(question.filters) if question.filters is not None else None
+    filters = question.filters
     logger.info(f"Received keyword search query: {query}")
 
     user_id = None if user is None else int(user.id)
@@ -69,15 +69,15 @@ def keyword_search(
     return SearchResponse(top_ranked_docs=top_docs, semi_ranked_docs=None)
 
 
-@router.get("/direct-qa", response_model=QAResponse)
+@router.post("/direct-qa")
 def direct_qa(
-    question: QuestionRequest = Depends(), user: User = Depends(current_user)
+    question: QuestionRequest, user: User = Depends(current_user)
 ) -> QAResponse:
     start_time = time.time()
 
     query = question.query
     collection = question.collection
-    filters = json.loads(question.filters) if question.filters is not None else None
+    filters = question.filters
     use_keyword = question.use_keyword
     logger.info(f"Received QA query: {query}")
 
@@ -115,9 +115,9 @@ def direct_qa(
     )
 
 
-@router.get("/stream-direct-qa")
+@router.post("/stream-direct-qa")
 def stream_direct_qa(
-    question: QuestionRequest = Depends(), user: User = Depends(current_user)
+    question: QuestionRequest, user: User = Depends(current_user)
 ) -> StreamingResponse:
     top_documents_key = "top_documents"
     unranked_top_docs_key = "unranked_top_documents"
@@ -125,7 +125,7 @@ def stream_direct_qa(
     def stream_qa_portions() -> Generator[str, None, None]:
         query = question.query
         collection = question.collection
-        filters = json.loads(question.filters) if question.filters is not None else None
+        filters = question.filters
         use_keyword = question.use_keyword
         logger.info(f"Received QA query: {query}")
 

web/src/app/page.tsx

@@ -23,7 +23,7 @@ export default async function Home() {
       </div>
       <ApiKeyModal />
       <div className="px-24 pt-10 flex flex-col items-center min-h-screen bg-gray-900 text-gray-100">
-        <div className="max-w-[800px] w-full">
+        <div className="w-full">
           <SearchSection />
         </div>
       </div>

web/src/components/search/Filters.tsx

@@ -0,0 +1,59 @@
+import React from "react";
+import { Source } from "./interfaces";
+import { getSourceIcon } from "../source";
+import { Funnel } from "@phosphor-icons/react";
+
+interface SourceSelectorProps {
+  selectedSources: Source[];
+  setSelectedSources: React.Dispatch<React.SetStateAction<Source[]>>;
+}
+
+const sources: Source[] = [
+  { displayName: "Google Drive", internalName: "google_drive" },
+  { displayName: "Slack", internalName: "slack" },
+  { displayName: "Confluence", internalName: "confluence" },
+  { displayName: "Github PRs", internalName: "github" },
+  { displayName: "Web", internalName: "web" },
+];
+
+export function SourceSelector({
+  selectedSources,
+  setSelectedSources,
+}: SourceSelectorProps) {
+  const handleSelect = (source: Source) => {
+    setSelectedSources((prev: Source[]) => {
+      if (prev.includes(source)) {
+        return prev.filter((s) => s.internalName !== source.internalName);
+      } else {
+        return [...prev, source];
+      }
+    });
+  };
+
+  return (
+    <div className="bg-gray-900 p-6">
+      <div className="flex mb-3 mx-2">
+        <h2 className="font-bold my-auto">Filters</h2>
+        <Funnel className="my-auto ml-2" size="20" />
+      </div>
+      {sources.map((source) => (
+        <div
+          key={source.internalName}
+          className={
+            "flex cursor-pointer w-full items-center text-white " +
+            "py-1.5 my-1.5 rounded-lg px-2 " +
+            (selectedSources.includes(source)
+              ? "bg-gray-700"
+              : "hover:bg-gray-800")
+          }
+          onClick={() => handleSelect(source)}
+        >
+          {getSourceIcon(source.internalName, "16")}
+          <span className="ml-2 text-sm text-gray-200">
+            {source.displayName}
+          </span>
+        </div>
+      ))}
+    </div>
+  );
+}

web/src/components/search/SearchResultsDisplay.tsx

@@ -45,11 +45,7 @@ export const SearchResultsDisplay: React.FC<SearchResultsDisplayProps> = ({
   }
 
   if (answer === null && documents === null && quotes === null) {
-    return (
+    return <div className="text-gray-300">No matching documents found.</div>;
-      <div className="text-red-500">
-        Something went wrong, please try again.
-      </div>
-    );
   }
 
   const dedupedQuotes: Quote[] = [];

web/src/components/search/SearchSection.tsx

@@ -4,6 +4,8 @@ import { useState } from "react";
 import { SearchBar } from "./SearchBar";
 import { SearchResultsDisplay } from "./SearchResultsDisplay";
 import { Quote, Document, SearchResponse } from "./types";
+import { SourceSelector } from "./Filters";
+import { Source } from "./interfaces";
 
 const initialSearchResponse: SearchResponse = {
   answer: null,
@@ -55,24 +57,44 @@ const processRawChunkString = (
   return [parsedChunkSections, currPartialChunk];
 };
 
-const searchRequestStreamed = async (
+interface SearchRequestStreamedArgs {
-  query: string,
+  query: string;
-  updateCurrentAnswer: (val: string) => void,
+  sources: Source[];
-  updateQuotes: (quotes: Record<string, Quote>) => void,
+  updateCurrentAnswer: (val: string) => void;
-  updateDocs: (docs: Document[]) => void
+  updateQuotes: (quotes: Record<string, Quote>) => void;
-) => {
+  updateDocs: (docs: Document[]) => void;
-  const url = new URL("/api/stream-direct-qa", window.location.origin);
+}
-  const params = new URLSearchParams({
-    query,
-    collection: "danswer_index",
-  }).toString();
-  url.search = params;
 
+const searchRequestStreamed = async ({
+  query,
+  sources,
+  updateCurrentAnswer,
+  updateQuotes,
+  updateDocs,
+}: SearchRequestStreamedArgs) => {
   let answer = "";
   let quotes: Record<string, Quote> | null = null;
   let relevantDocuments: Document[] | null = null;
   try {
-    const response = await fetch(url);
+    const response = await fetch("/api/stream-direct-qa", {
+      method: "POST",
+      body: JSON.stringify({
+        query,
+        collection: "danswer_index",
+        ...(sources.length > 0
+          ? {
+              filters: [
+                {
+                  source_type: sources.map((source) => source.internalName),
+                },
+              ],
+            }
+          : {}),
+      }),
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
     const reader = response.body?.getReader();
     const decoder = new TextDecoder("utf-8");
 
@@ -139,13 +161,24 @@ const searchRequestStreamed = async (
 };
 
 export const SearchSection: React.FC<{}> = () => {
+  // Search
   const [searchResponse, setSearchResponse] = useState<SearchResponse | null>(
     null
   );
   const [isFetching, setIsFetching] = useState(false);
 
+  // Filters
+  const [sources, setSources] = useState<Source[]>([]);
+
   return (
-    <>
+    <div className="relative max-w-[1500px] mx-auto">
+      <div className="absolute left-0 ml-24 hidden 2xl:block">
+        <SourceSelector
+          selectedSources={sources}
+          setSelectedSources={setSources}
+        />
+      </div>
+      <div className="w-[800px] mx-auto">
         <SearchBar
           onSearch={(query) => {
             setIsFetching(true);
@@ -154,24 +187,25 @@ export const SearchSection: React.FC<{}> = () => {
               quotes: null,
               documents: null,
             });
-          searchRequestStreamed(
+            searchRequestStreamed({
               query,
-            (answer) =>
+              sources,
+              updateCurrentAnswer: (answer) =>
                 setSearchResponse((prevState) => ({
                   ...(prevState || initialSearchResponse),
                   answer,
                 })),
-            (quotes) =>
+              updateQuotes: (quotes) =>
                 setSearchResponse((prevState) => ({
                   ...(prevState || initialSearchResponse),
                   quotes,
                 })),
-            (documents) =>
+              updateDocs: (documents) =>
                 setSearchResponse((prevState) => ({
                   ...(prevState || initialSearchResponse),
                   documents,
-              }))
+                })),
-          ).then(() => {
+            }).then(() => {
               setIsFetching(false);
             });
           }}
@@ -182,6 +216,7 @@ export const SearchSection: React.FC<{}> = () => {
             isFetching={isFetching}
           />
         </div>
-    </>
+      </div>
+    </div>
   );
 };

web/src/components/search/interfaces.tsx

@@ -0,0 +1,6 @@
+import { ValidSources } from "@/lib/types";
+
+export interface Source {
+  displayName: string;
+  internalName: ValidSources;
+}