Add Filters to UI (#86)

* Adding filters

* Fix get_connector_indexing_status endpoint bug
This commit is contained in:
Chris Weaver 2023-06-05 00:41:48 -07:00 committed by GitHub
parent c4e8afe4d2
commit 711e66184e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 169 additions and 69 deletions

View File

@ -213,7 +213,10 @@ def get_connector_indexing_status(
for index_attempt in index_attempts:
# don't consider index attempts where the connector has been deleted
# or the credential has been deleted
if index_attempt.connector_id and index_attempt.credential_id:
if (
index_attempt.connector_id is not None
and index_attempt.credential_id is not None
):
connector_credential_pair_to_index_attempts[
(index_attempt.connector_id, index_attempt.credential_id)
].append(index_attempt)

View File

@ -7,6 +7,7 @@ from typing import TypeVar
from danswer.configs.constants import DocumentSource
from danswer.connectors.models import InputType
from danswer.datastores.interfaces import IndexFilter
from danswer.db.models import Connector
from danswer.db.models import IndexingStatus
from pydantic import BaseModel
@ -77,7 +78,7 @@ class QuestionRequest(BaseModel):
query: str
collection: str
use_keyword: bool | None
filters: str | None # string of list[IndexFilter]
filters: list[IndexFilter] | None
class SearchResponse(BaseModel):

View File

@ -27,13 +27,13 @@ logger = setup_logger()
router = APIRouter()
@router.get("/semantic-search")
@router.post("/semantic-search")
def semantic_search(
question: QuestionRequest = Depends(), user: User = Depends(current_user)
question: QuestionRequest, user: User = Depends(current_user)
) -> SearchResponse:
query = question.query
collection = question.collection
filters = json.loads(question.filters) if question.filters is not None else None
filters = question.filters
logger.info(f"Received semantic search query: {query}")
user_id = None if user is None else int(user.id)
@ -49,13 +49,13 @@ def semantic_search(
return SearchResponse(top_ranked_docs=top_docs, semi_ranked_docs=other_top_docs)
@router.get("/keyword-search", response_model=SearchResponse)
@router.post("/keyword-search")
def keyword_search(
question: QuestionRequest = Depends(), user: User = Depends(current_user)
question: QuestionRequest, user: User = Depends(current_user)
) -> SearchResponse:
query = question.query
collection = question.collection
filters = json.loads(question.filters) if question.filters is not None else None
filters = question.filters
logger.info(f"Received keyword search query: {query}")
user_id = None if user is None else int(user.id)
@ -69,15 +69,15 @@ def keyword_search(
return SearchResponse(top_ranked_docs=top_docs, semi_ranked_docs=None)
@router.get("/direct-qa", response_model=QAResponse)
@router.post("/direct-qa")
def direct_qa(
question: QuestionRequest = Depends(), user: User = Depends(current_user)
question: QuestionRequest, user: User = Depends(current_user)
) -> QAResponse:
start_time = time.time()
query = question.query
collection = question.collection
filters = json.loads(question.filters) if question.filters is not None else None
filters = question.filters
use_keyword = question.use_keyword
logger.info(f"Received QA query: {query}")
@ -115,9 +115,9 @@ def direct_qa(
)
@router.get("/stream-direct-qa")
@router.post("/stream-direct-qa")
def stream_direct_qa(
question: QuestionRequest = Depends(), user: User = Depends(current_user)
question: QuestionRequest, user: User = Depends(current_user)
) -> StreamingResponse:
top_documents_key = "top_documents"
unranked_top_docs_key = "unranked_top_documents"
@ -125,7 +125,7 @@ def stream_direct_qa(
def stream_qa_portions() -> Generator[str, None, None]:
query = question.query
collection = question.collection
filters = json.loads(question.filters) if question.filters is not None else None
filters = question.filters
use_keyword = question.use_keyword
logger.info(f"Received QA query: {query}")

View File

@ -23,7 +23,7 @@ export default async function Home() {
</div>
<ApiKeyModal />
<div className="px-24 pt-10 flex flex-col items-center min-h-screen bg-gray-900 text-gray-100">
<div className="max-w-[800px] w-full">
<div className="w-full">
<SearchSection />
</div>
</div>

View File

@ -0,0 +1,59 @@
import React from "react";
import { Source } from "./interfaces";
import { getSourceIcon } from "../source";
import { Funnel } from "@phosphor-icons/react";
interface SourceSelectorProps {
selectedSources: Source[];
setSelectedSources: React.Dispatch<React.SetStateAction<Source[]>>;
}
const sources: Source[] = [
{ displayName: "Google Drive", internalName: "google_drive" },
{ displayName: "Slack", internalName: "slack" },
{ displayName: "Confluence", internalName: "confluence" },
{ displayName: "Github PRs", internalName: "github" },
{ displayName: "Web", internalName: "web" },
];
export function SourceSelector({
selectedSources,
setSelectedSources,
}: SourceSelectorProps) {
const handleSelect = (source: Source) => {
setSelectedSources((prev: Source[]) => {
if (prev.includes(source)) {
return prev.filter((s) => s.internalName !== source.internalName);
} else {
return [...prev, source];
}
});
};
return (
<div className="bg-gray-900 p-6">
<div className="flex mb-3 mx-2">
<h2 className="font-bold my-auto">Filters</h2>
<Funnel className="my-auto ml-2" size="20" />
</div>
{sources.map((source) => (
<div
key={source.internalName}
className={
"flex cursor-pointer w-full items-center text-white " +
"py-1.5 my-1.5 rounded-lg px-2 " +
(selectedSources.includes(source)
? "bg-gray-700"
: "hover:bg-gray-800")
}
onClick={() => handleSelect(source)}
>
{getSourceIcon(source.internalName, "16")}
<span className="ml-2 text-sm text-gray-200">
{source.displayName}
</span>
</div>
))}
</div>
);
}

View File

@ -45,11 +45,7 @@ export const SearchResultsDisplay: React.FC<SearchResultsDisplayProps> = ({
}
if (answer === null && documents === null && quotes === null) {
return (
<div className="text-red-500">
Something went wrong, please try again.
</div>
);
return <div className="text-gray-300">No matching documents found.</div>;
}
const dedupedQuotes: Quote[] = [];

View File

@ -4,6 +4,8 @@ import { useState } from "react";
import { SearchBar } from "./SearchBar";
import { SearchResultsDisplay } from "./SearchResultsDisplay";
import { Quote, Document, SearchResponse } from "./types";
import { SourceSelector } from "./Filters";
import { Source } from "./interfaces";
const initialSearchResponse: SearchResponse = {
answer: null,
@ -55,24 +57,44 @@ const processRawChunkString = (
return [parsedChunkSections, currPartialChunk];
};
const searchRequestStreamed = async (
query: string,
updateCurrentAnswer: (val: string) => void,
updateQuotes: (quotes: Record<string, Quote>) => void,
updateDocs: (docs: Document[]) => void
) => {
const url = new URL("/api/stream-direct-qa", window.location.origin);
const params = new URLSearchParams({
query,
collection: "danswer_index",
}).toString();
url.search = params;
interface SearchRequestStreamedArgs {
query: string;
sources: Source[];
updateCurrentAnswer: (val: string) => void;
updateQuotes: (quotes: Record<string, Quote>) => void;
updateDocs: (docs: Document[]) => void;
}
const searchRequestStreamed = async ({
query,
sources,
updateCurrentAnswer,
updateQuotes,
updateDocs,
}: SearchRequestStreamedArgs) => {
let answer = "";
let quotes: Record<string, Quote> | null = null;
let relevantDocuments: Document[] | null = null;
try {
const response = await fetch(url);
const response = await fetch("/api/stream-direct-qa", {
method: "POST",
body: JSON.stringify({
query,
collection: "danswer_index",
...(sources.length > 0
? {
filters: [
{
source_type: sources.map((source) => source.internalName),
},
],
}
: {}),
}),
headers: {
"Content-Type": "application/json",
},
});
const reader = response.body?.getReader();
const decoder = new TextDecoder("utf-8");
@ -139,49 +161,62 @@ const searchRequestStreamed = async (
};
export const SearchSection: React.FC<{}> = () => {
// Search
const [searchResponse, setSearchResponse] = useState<SearchResponse | null>(
null
);
const [isFetching, setIsFetching] = useState(false);
// Filters
const [sources, setSources] = useState<Source[]>([]);
return (
<>
<SearchBar
onSearch={(query) => {
setIsFetching(true);
setSearchResponse({
answer: null,
quotes: null,
documents: null,
});
searchRequestStreamed(
query,
(answer) =>
setSearchResponse((prevState) => ({
...(prevState || initialSearchResponse),
answer,
})),
(quotes) =>
setSearchResponse((prevState) => ({
...(prevState || initialSearchResponse),
quotes,
})),
(documents) =>
setSearchResponse((prevState) => ({
...(prevState || initialSearchResponse),
documents,
}))
).then(() => {
setIsFetching(false);
});
}}
/>
<div className="mt-2">
<SearchResultsDisplay
searchResponse={searchResponse}
isFetching={isFetching}
<div className="relative max-w-[1500px] mx-auto">
<div className="absolute left-0 ml-24 hidden 2xl:block">
<SourceSelector
selectedSources={sources}
setSelectedSources={setSources}
/>
</div>
</>
<div className="w-[800px] mx-auto">
<SearchBar
onSearch={(query) => {
setIsFetching(true);
setSearchResponse({
answer: null,
quotes: null,
documents: null,
});
searchRequestStreamed({
query,
sources,
updateCurrentAnswer: (answer) =>
setSearchResponse((prevState) => ({
...(prevState || initialSearchResponse),
answer,
})),
updateQuotes: (quotes) =>
setSearchResponse((prevState) => ({
...(prevState || initialSearchResponse),
quotes,
})),
updateDocs: (documents) =>
setSearchResponse((prevState) => ({
...(prevState || initialSearchResponse),
documents,
})),
}).then(() => {
setIsFetching(false);
});
}}
/>
<div className="mt-2">
<SearchResultsDisplay
searchResponse={searchResponse}
isFetching={isFetching}
/>
</div>
</div>
</div>
);
};

View File

@ -0,0 +1,6 @@
import { ValidSources } from "@/lib/types";
export interface Source {
displayName: string;
internalName: ValidSources;
}