Add document set-based filters in UI (#497)

web/src/app/admin/bot/SlackBotConfigCreationForm.tsx

@@ -12,7 +12,7 @@ import { channel } from "diagnostics_channel";
 interface SetCreationPopupProps {
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
   existingSlackBotConfig?: SlackBotConfig;
 }
 

web/src/app/admin/bot/page.tsx

@@ -29,7 +29,7 @@ const EditRow = ({
 }: {
   existingSlackBotConfig: SlackBotConfig;
   setPopup: (popupSpec: PopupSpec | null) => void;
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
   refreshSlackBotConfigs: () => void;
 }) => {
   const [isEditPopupOpen, setEditPopupOpen] = useState(false);
@@ -58,7 +58,7 @@ const EditRow = ({
 
 interface DocumentFeedbackTableProps {
   slackBotConfigs: SlackBotConfig[];
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
   refresh: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
 }

web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx

@@ -10,7 +10,7 @@ interface SetCreationPopupProps {
   ccPairs: ConnectorIndexingStatus<any, any>[];
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
-  existingDocumentSet?: DocumentSet<any, any>;
+  existingDocumentSet?: DocumentSet;
 }
 
 export const DocumentSetCreationForm = ({

web/src/app/admin/documents/sets/hooks.tsx

@@ -4,10 +4,7 @@ import useSWR, { mutate } from "swr";
 
 export const useDocumentSets = () => {
   const url = "/api/manage/document-set";
-  const swrResponse = useSWR<DocumentSet<any, any>[]>(
+  const swrResponse = useSWR<DocumentSet[]>(url, errorHandlingFetcher);
-    url,
-    errorHandlingFetcher
-  );
 
   return {
     ...swrResponse,

web/src/app/admin/documents/sets/page.tsx

@@ -27,7 +27,7 @@ const EditRow = ({
   setPopup,
   refreshDocumentSets,
 }: {
-  documentSet: DocumentSet<any, any>;
+  documentSet: DocumentSet;
   ccPairs: ConnectorIndexingStatus<any, any>[];
   setPopup: (popupSpec: PopupSpec | null) => void;
   refreshDocumentSets: () => void;
@@ -81,7 +81,7 @@ const EditRow = ({
 };
 
 interface DocumentFeedbackTableProps {
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
   ccPairs: ConnectorIndexingStatus<any, any>[];
   refresh: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;

web/src/app/page.tsx

@@ -6,7 +6,7 @@ import { DISABLE_AUTH } from "@/lib/constants";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { ApiKeyModal } from "@/components/openai/ApiKeyModal";
 import { buildUrl } from "@/lib/utilsSS";
-import { Connector, User } from "@/lib/types";
+import { Connector, DocumentSet, User } from "@/lib/types";
 import { cookies } from "next/headers";
 import { SearchType } from "@/lib/search/interfaces";
 
@@ -19,6 +19,12 @@ export default async function Home() {
         cookie: processCookies(cookies()),
       },
     }),
+    fetch(buildUrl("/manage/document-set"), {
+      next: { revalidate: 0 },
+      headers: {
+        cookie: processCookies(cookies()),
+      },
+    }),
   ];
 
   // catch cases where the backend is completely unreachable here
@@ -32,6 +38,7 @@ export default async function Home() {
   }
   const user = results[0] as User | null;
   const connectorsResponse = results[1] as Response | null;
+  const documentSetsResponse = results[2] as Response | null;
 
   if (!DISABLE_AUTH && !user) {
     return redirect("/auth/login");
@@ -44,6 +51,15 @@ export default async function Home() {
     console.log(`Failed to fetch connectors - ${connectorsResponse?.status}`);
   }
 
+  let documentSets: DocumentSet[] = [];
+  if (documentSetsResponse?.ok) {
+    documentSets = await documentSetsResponse.json();
+  } else {
+    console.log(
+      `Failed to fetch document sets - ${documentSetsResponse?.status}`
+    );
+  }
+
   // needs to be done in a non-client side component due to nextjs
   const storedSearchType = cookies().get("searchType")?.value as
     | string
@@ -65,6 +81,7 @@ export default async function Home() {
         <div className="w-full">
           <SearchSection
             connectors={connectors}
+            documentSets={documentSets}
             defaultSearchType={searchTypeDefault}
           />
         </div>

web/src/components/HoverPopup.tsx

@@ -0,0 +1,35 @@
+import { useState } from "react";
+
+interface HoverPopupProps {
+  mainContent: string | JSX.Element;
+  popupContent: string | JSX.Element;
+  classNameModifications?: string;
+}
+
+export const HoverPopup = ({
+  mainContent,
+  popupContent,
+  classNameModifications,
+}: HoverPopupProps) => {
+  const [hovered, setHovered] = useState(false);
+
+  return (
+    <div
+      className="relative flex"
+      onMouseEnter={() => setHovered(true)}
+      onMouseLeave={() => setHovered(false)}
+    >
+      {hovered && (
+        <div
+          className={
+            `absolute top-0 left-0 mt-8 bg-gray-700 px-3 py-2 rounded shadow-lg z-30 ` +
+              classNameModifications || ""
+          }
+        >
+          {popupContent}
+        </div>
+      )}
+      {mainContent}
+    </div>
+  );
+};

web/src/components/icons/icons.tsx

@@ -4,7 +4,6 @@ import {
   Notebook,
   Key,
   Trash,
-  Info,
   XSquare,
   LinkBreak,
   Link,
@@ -33,6 +32,7 @@ import {
   FiCopy,
   FiBookmark,
   FiCpu,
+  FiInfo,
 } from "react-icons/fi";
 import { SiBookstack } from "react-icons/si";
 import Image from "next/image";
@@ -47,7 +47,7 @@ interface IconProps {
   className?: string;
 }
 
-const defaultTailwindCSS = "my-auto flex flex-shrink-0 text-blue-400";
+export const defaultTailwindCSS = "my-auto flex flex-shrink-0 text-blue-400";
 
 export const PlugIcon = ({
   size = 16,
@@ -123,7 +123,7 @@ export const InfoIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => {
-  return <Info size={size} className={className} />;
+  return <FiInfo size={size} className={className} />;
 };
 
 export const QuestionIcon = ({

web/src/components/search/Filters.tsx

@@ -1,8 +1,16 @@
 import React from "react";
 import { getSourceIcon } from "../source";
 import { Funnel } from "@phosphor-icons/react";
-import { ValidSources } from "@/lib/types";
+import { DocumentSet, ValidSources } from "@/lib/types";
 import { Source } from "@/lib/search/interfaces";
+import {
+  BookmarkIcon,
+  InfoIcon,
+  NotebookIcon,
+  defaultTailwindCSS,
+} from "../icons/icons";
+import { HoverPopup } from "../HoverPopup";
+import { FiFilter } from "react-icons/fi";
 
 const sources: Source[] = [
   { displayName: "Google Drive", internalName: "google_drive" },
@@ -24,12 +32,18 @@ const sources: Source[] = [
 interface SourceSelectorProps {
   selectedSources: Source[];
   setSelectedSources: React.Dispatch<React.SetStateAction<Source[]>>;
+  selectedDocumentSets: string[];
+  setSelectedDocumentSets: React.Dispatch<React.SetStateAction<string[]>>;
+  availableDocumentSets: DocumentSet[];
   existingSources: ValidSources[];
 }
 
 export function SourceSelector({
   selectedSources,
   setSelectedSources,
+  selectedDocumentSets,
+  setSelectedDocumentSets,
+  availableDocumentSets,
   existingSources,
 }: SourceSelectorProps) {
   const handleSelect = (source: Source) => {
@@ -42,13 +56,27 @@ export function SourceSelector({
     });
   };
 
+  const handleDocumentSetSelect = (documentSetName: string) => {
+    setSelectedDocumentSets((prev: string[]) => {
+      if (prev.includes(documentSetName)) {
+        return prev.filter((s) => s !== documentSetName);
+      } else {
+        return [...prev, documentSetName];
+      }
+    });
+  };
+
   return (
-    <div className="bg-gray-900">
+    <div>
-      <div className="flex mb-2 pb-1 pl-2 border-b border-gray-800 mx-2">
+      <div className="flex mb-2 pb-1 border-b border-gray-800">
         <h2 className="font-bold my-auto">Filters</h2>
-        <Funnel className="my-auto ml-2" size="20" />
+        <FiFilter className="my-auto ml-2" size="18" />
       </div>
-      <div className="px-2">
+
+      {existingSources.length > 0 && (
+        <>
+          <div className="font-medium text-sm flex">Sources</div>
+          <div className="px-1">
             {sources
               .filter((source) => existingSources.includes(source.internalName))
               .map((source) => (
@@ -70,6 +98,55 @@ export function SourceSelector({
                 </div>
               ))}
           </div>
+        </>
+      )}
+
+      {availableDocumentSets.length > 0 && (
+        <>
+          <div className="mt-4">
+            <div className="font-medium text-sm flex">Knowledge Sets</div>
+          </div>
+          <div className="px-1">
+            {availableDocumentSets.map((documentSet) => (
+              <div key={documentSet.name} className="my-1.5 flex">
+                <div
+                  key={documentSet.name}
+                  className={
+                    "flex cursor-pointer w-full items-center text-white " +
+                    "py-1.5 rounded-lg px-2 " +
+                    (selectedDocumentSets.includes(documentSet.name)
+                      ? "bg-gray-700"
+                      : "hover:bg-gray-800")
+                  }
+                  onClick={() => handleDocumentSetSelect(documentSet.name)}
+                >
+                  <HoverPopup
+                    mainContent={
+                      <div className="flex my-auto mr-2">
+                        <InfoIcon className={defaultTailwindCSS} />
+                      </div>
+                    }
+                    popupContent={
+                      <div className="text-sm w-64">
+                        <div className="flex font-medium text-gray-200">
+                          Description
+                        </div>
+                        <div className="mt-1 text-gray-300">
+                          {documentSet.description}
+                        </div>
+                      </div>
+                    }
+                    classNameModifications="-ml-2"
+                  />
+                  <span className="text-sm text-gray-200">
+                    {documentSet.name}
+                  </span>
+                </div>
+              </div>
+            ))}
+          </div>
+        </>
+      )}
     </div>
   );
 }

web/src/components/search/SearchSection.tsx

@@ -4,7 +4,7 @@ import { useRef, useState } from "react";
 import { SearchBar } from "./SearchBar";
 import { SearchResultsDisplay } from "./SearchResultsDisplay";
 import { SourceSelector } from "./Filters";
-import { Connector } from "@/lib/types";
+import { Connector, DocumentSet } from "@/lib/types";
 import { SearchTypeSelector } from "./SearchTypeSelector";
 import {
   DanswerDocument,
@@ -38,13 +38,16 @@ const VALID_QUESTION_RESPONSE_DEFAULT: ValidQuestionResponse = {
 
 interface SearchSectionProps {
   connectors: Connector<any>[];
+  documentSets: DocumentSet[];
   defaultSearchType: SearchType;
 }
 
 export const SearchSection: React.FC<SearchSectionProps> = ({
   connectors,
+  documentSets,
   defaultSearchType,
 }) => {
+  console.log(documentSets);
   // Search Bar
   const [query, setQuery] = useState<string>("");
 
@@ -59,6 +62,9 @@ export const SearchSection: React.FC<SearchSectionProps> = ({
 
   // Filters
   const [sources, setSources] = useState<Source[]>([]);
+  const [selectedDocumentSets, setSelectedDocumentSets] = useState<string[]>(
+    []
+  );
 
   // Search Type
   const [selectedSearchType, setSelectedSearchType] =
@@ -135,6 +141,7 @@ export const SearchSection: React.FC<SearchSectionProps> = ({
     const searchFnArgs = {
       query,
       sources,
+      documentSets: selectedDocumentSets,
       updateCurrentAnswer: cancellable({
         cancellationToken: lastSearchCancellationToken.current,
         fn: updateCurrentAnswer,
@@ -183,10 +190,13 @@ export const SearchSection: React.FC<SearchSectionProps> = ({
   return (
     <div className="relative max-w-[2000px] xl:max-w-[1400px] mx-auto">
       <div className="absolute left-0 hidden 2xl:block w-64">
-        {connectors.length > 0 && (
+        {(connectors.length > 0 || documentSets.length > 0) && (
           <SourceSelector
             selectedSources={sources}
             setSelectedSources={setSources}
+            selectedDocumentSets={selectedDocumentSets}
+            setSelectedDocumentSets={setSelectedDocumentSets}
+            availableDocumentSets={documentSets}
             existingSources={connectors.map((connector) => connector.source)}
           />
         )}

web/src/lib/search/interfaces.ts

@@ -59,6 +59,7 @@ export interface SearchDefaultOverrides {
 export interface SearchRequestArgs {
   query: string;
   sources: Source[];
+  documentSets: string[];
   updateCurrentAnswer: (val: string) => void;
   updateQuotes: (quotes: Quote[]) => void;
   updateDocs: (documents: DanswerDocument[]) => void;

web/src/lib/search/qa.ts

@@ -5,10 +5,12 @@ import {
   SearchRequestArgs,
   SearchType,
 } from "./interfaces";
+import { buildFilters } from "./utils";
 
 export const searchRequest = async ({
   query,
   sources,
+  documentSets,
   updateCurrentAnswer,
   updateQuotes,
   updateDocs,
@@ -27,19 +29,16 @@ export const searchRequest = async ({
   let quotes: Quote[] | null = null;
   let relevantDocuments: DanswerDocument[] | null = null;
   try {
+    const filters = buildFilters(sources, documentSets);
     const response = await fetch("/api/direct-qa", {
       method: "POST",
       body: JSON.stringify({
         query,
         collection: "danswer_index",
         use_keyword: useKeyword,
-        ...(sources.length > 0
+        ...(filters.length > 0
           ? {
-              filters: [
+              filters,
-                {
-                  source_type: sources.map((source) => source.internalName),
-                },
-              ],
             }
           : {}),
         offset: offset,

web/src/lib/search/streamingQa.ts

@@ -4,6 +4,7 @@ import {
   SearchRequestArgs,
   SearchType,
 } from "./interfaces";
+import { buildFilters } from "./utils";
 
 const processSingleChunk = (
   chunk: string,
@@ -54,6 +55,7 @@ const processRawChunkString = (
 export const searchRequestStreamed = async ({
   query,
   sources,
+  documentSets,
   updateCurrentAnswer,
   updateQuotes,
   updateDocs,
@@ -73,19 +75,16 @@ export const searchRequestStreamed = async ({
   let quotes: Quote[] | null = null;
   let relevantDocuments: DanswerDocument[] | null = null;
   try {
+    const filters = buildFilters(sources, documentSets);
     const response = await fetch("/api/stream-direct-qa", {
       method: "POST",
       body: JSON.stringify({
         query,
         collection: "danswer_index",
         use_keyword: useKeyword,
-        ...(sources.length > 0
+        ...(filters.length > 0
           ? {
-              filters: [
+              filters,
-                {
-                  source_type: sources.map((source) => source.internalName),
-                },
-              ],
             }
           : {}),
         offset: offset,

web/src/lib/search/utils.ts

@@ -0,0 +1,16 @@
+import { Source } from "./interfaces";
+
+export const buildFilters = (sources: Source[], documentSets: string[]) => {
+  const filters = [];
+  if (sources.length > 0) {
+    filters.push({
+      source_type: sources.map((source) => source.internalName),
+    });
+  }
+  if (documentSets.length > 0) {
+    filters.push({
+      document_sets: documentSets,
+    });
+  }
+  return filters;
+};

web/src/lib/types.ts

@@ -222,11 +222,11 @@ export interface CCPairDescriptor<ConnectorType, CredentialType> {
   credential: Credential<CredentialType>;
 }
 
-export interface DocumentSet<ConnectorType, CredentialType> {
+export interface DocumentSet {
   id: number;
   name: string;
   description: string;
-  cc_pair_descriptors: CCPairDescriptor<ConnectorType, CredentialType>[];
+  cc_pair_descriptors: CCPairDescriptor<any, any>[];
   is_up_to_date: boolean;
 }
 
@@ -239,7 +239,7 @@ export interface ChannelConfig {
 
 export interface SlackBotConfig {
   id: number;
-  document_sets: DocumentSet<any, any>[];
+  document_sets: DocumentSet[];
   channel_config: ChannelConfig;
 }