Add document set-based filters in UI (#497)

2025-09-25 19:37:29 +02:00 · 2023-10-01 10:46:04 -07:00
parent aa9071e441
commit 2d06008f6f
15 changed files with 208 additions and 57 deletions
--- a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
+++ b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
@@ -12,7 +12,7 @@ import { channel } from "diagnostics_channel";
 interface SetCreationPopupProps {
  onClose: () => void;
  setPopup: (popupSpec: PopupSpec | null) => void;
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
  existingSlackBotConfig?: SlackBotConfig;
 }

--- a/web/src/app/admin/bot/page.tsx
+++ b/web/src/app/admin/bot/page.tsx
@@ -29,7 +29,7 @@ const EditRow = ({
 }: {
  existingSlackBotConfig: SlackBotConfig;
  setPopup: (popupSpec: PopupSpec | null) => void;
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
  refreshSlackBotConfigs: () => void;
 }) => {
  const [isEditPopupOpen, setEditPopupOpen] = useState(false);
@@ -58,7 +58,7 @@ const EditRow = ({

 interface DocumentFeedbackTableProps {
  slackBotConfigs: SlackBotConfig[];
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
  refresh: () => void;
  setPopup: (popupSpec: PopupSpec | null) => void;
 }
--- a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
+++ b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
@@ -10,7 +10,7 @@ interface SetCreationPopupProps {
  ccPairs: ConnectorIndexingStatus<any, any>[];
  onClose: () => void;
  setPopup: (popupSpec: PopupSpec | null) => void;
-  existingDocumentSet?: DocumentSet<any, any>;
+  existingDocumentSet?: DocumentSet;
 }

 export const DocumentSetCreationForm = ({
--- a/web/src/app/admin/documents/sets/hooks.tsx
+++ b/web/src/app/admin/documents/sets/hooks.tsx
@@ -4,10 +4,7 @@ import useSWR, { mutate } from "swr";

 export const useDocumentSets = () => {
  const url = "/api/manage/document-set";
-  const swrResponse = useSWR<DocumentSet<any, any>[]>(
-    url,
-    errorHandlingFetcher
-  );
+  const swrResponse = useSWR<DocumentSet[]>(url, errorHandlingFetcher);

  return {
    ...swrResponse,
--- a/web/src/app/admin/documents/sets/page.tsx
+++ b/web/src/app/admin/documents/sets/page.tsx
@@ -27,7 +27,7 @@ const EditRow = ({
  setPopup,
  refreshDocumentSets,
 }: {
-  documentSet: DocumentSet<any, any>;
+  documentSet: DocumentSet;
  ccPairs: ConnectorIndexingStatus<any, any>[];
  setPopup: (popupSpec: PopupSpec | null) => void;
  refreshDocumentSets: () => void;
@@ -81,7 +81,7 @@ const EditRow = ({
 };

 interface DocumentFeedbackTableProps {
-  documentSets: DocumentSet<any, any>[];
+  documentSets: DocumentSet[];
  ccPairs: ConnectorIndexingStatus<any, any>[];
  refresh: () => void;
  setPopup: (popupSpec: PopupSpec | null) => void;
--- a/web/src/app/page.tsx
+++ b/web/src/app/page.tsx
@@ -6,7 +6,7 @@ import { DISABLE_AUTH } from "@/lib/constants";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { ApiKeyModal } from "@/components/openai/ApiKeyModal";
 import { buildUrl } from "@/lib/utilsSS";
-import { Connector, User } from "@/lib/types";
+import { Connector, DocumentSet, User } from "@/lib/types";
 import { cookies } from "next/headers";
 import { SearchType } from "@/lib/search/interfaces";

@@ -19,6 +19,12 @@ export default async function Home() {
        cookie: processCookies(cookies()),
      },
    }),
+    fetch(buildUrl("/manage/document-set"), {
+      next: { revalidate: 0 },
+      headers: {
+        cookie: processCookies(cookies()),
+      },
+    }),
  ];

  // catch cases where the backend is completely unreachable here
@@ -32,6 +38,7 @@ export default async function Home() {
  }
  const user = results[0] as User | null;
  const connectorsResponse = results[1] as Response | null;
+  const documentSetsResponse = results[2] as Response | null;

  if (!DISABLE_AUTH && !user) {
    return redirect("/auth/login");
@@ -44,6 +51,15 @@ export default async function Home() {
    console.log(`Failed to fetch connectors - ${connectorsResponse?.status}`);
  }

+  let documentSets: DocumentSet[] = [];
+  if (documentSetsResponse?.ok) {
+    documentSets = await documentSetsResponse.json();
+  } else {
+    console.log(
+      `Failed to fetch document sets - ${documentSetsResponse?.status}`
+    );
+  }
+
  // needs to be done in a non-client side component due to nextjs
  const storedSearchType = cookies().get("searchType")?.value as
    | string
@@ -65,6 +81,7 @@ export default async function Home() {
        <div className="w-full">
          <SearchSection
            connectors={connectors}
+            documentSets={documentSets}
            defaultSearchType={searchTypeDefault}
          />
        </div>
--- a/web/src/components/HoverPopup.tsx
+++ b/web/src/components/HoverPopup.tsx
@@ -0,0 +1,35 @@
+import { useState } from "react";
+
+interface HoverPopupProps {
+  mainContent: string | JSX.Element;
+  popupContent: string | JSX.Element;
+  classNameModifications?: string;
+}
+
+export const HoverPopup = ({
+  mainContent,
+  popupContent,
+  classNameModifications,
+}: HoverPopupProps) => {
+  const [hovered, setHovered] = useState(false);
+
+  return (
+    <div
+      className="relative flex"
+      onMouseEnter={() => setHovered(true)}
+      onMouseLeave={() => setHovered(false)}
+    >
+      {hovered && (
+        <div
+          className={
+            `absolute top-0 left-0 mt-8 bg-gray-700 px-3 py-2 rounded shadow-lg z-30 ` +
+              classNameModifications || ""
+          }
+        >
+          {popupContent}
+        </div>
+      )}
+      {mainContent}
+    </div>
+  );
+};
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -4,7 +4,6 @@ import {
  Notebook,
  Key,
  Trash,
-  Info,
  XSquare,
  LinkBreak,
  Link,
@@ -33,6 +32,7 @@ import {
  FiCopy,
  FiBookmark,
  FiCpu,
+  FiInfo,
 } from "react-icons/fi";
 import { SiBookstack } from "react-icons/si";
 import Image from "next/image";
@@ -47,7 +47,7 @@ interface IconProps {
  className?: string;
 }

-const defaultTailwindCSS = "my-auto flex flex-shrink-0 text-blue-400";
+export const defaultTailwindCSS = "my-auto flex flex-shrink-0 text-blue-400";

 export const PlugIcon = ({
  size = 16,
@@ -123,7 +123,7 @@ export const InfoIcon = ({
  size = 16,
  className = defaultTailwindCSS,
 }: IconProps) => {
-  return <Info size={size} className={className} />;
+  return <FiInfo size={size} className={className} />;
 };

 export const QuestionIcon = ({
--- a/web/src/components/search/Filters.tsx
+++ b/web/src/components/search/Filters.tsx
@@ -1,8 +1,16 @@
 import React from "react";
 import { getSourceIcon } from "../source";
 import { Funnel } from "@phosphor-icons/react";
-import { ValidSources } from "@/lib/types";
+import { DocumentSet, ValidSources } from "@/lib/types";
 import { Source } from "@/lib/search/interfaces";
+import {
+  BookmarkIcon,
+  InfoIcon,
+  NotebookIcon,
+  defaultTailwindCSS,
+} from "../icons/icons";
+import { HoverPopup } from "../HoverPopup";
+import { FiFilter } from "react-icons/fi";

 const sources: Source[] = [
  { displayName: "Google Drive", internalName: "google_drive" },
@@ -24,12 +32,18 @@ const sources: Source[] = [
 interface SourceSelectorProps {
  selectedSources: Source[];
  setSelectedSources: React.Dispatch<React.SetStateAction<Source[]>>;
+  selectedDocumentSets: string[];
+  setSelectedDocumentSets: React.Dispatch<React.SetStateAction<string[]>>;
+  availableDocumentSets: DocumentSet[];
  existingSources: ValidSources[];
 }

 export function SourceSelector({
  selectedSources,
  setSelectedSources,
+  selectedDocumentSets,
+  setSelectedDocumentSets,
+  availableDocumentSets,
  existingSources,
 }: SourceSelectorProps) {
  const handleSelect = (source: Source) => {
@@ -42,34 +56,97 @@ export function SourceSelector({
    });
  };

+  const handleDocumentSetSelect = (documentSetName: string) => {
+    setSelectedDocumentSets((prev: string[]) => {
+      if (prev.includes(documentSetName)) {
+        return prev.filter((s) => s !== documentSetName);
+      } else {
+        return [...prev, documentSetName];
+      }
+    });
+  };
+
  return (
-    <div className="bg-gray-900">
-      <div className="flex mb-2 pb-1 pl-2 border-b border-gray-800 mx-2">
+    <div>
+      <div className="flex mb-2 pb-1 border-b border-gray-800">
        <h2 className="font-bold my-auto">Filters</h2>
-        <Funnel className="my-auto ml-2" size="20" />
-      </div>
-      <div className="px-2">
-        {sources
-          .filter((source) => existingSources.includes(source.internalName))
-          .map((source) => (
-            <div
-              key={source.internalName}
-              className={
-                "flex cursor-pointer w-full items-center text-white " +
-                "py-1.5 my-1.5 rounded-lg px-2 " +
-                (selectedSources.includes(source)
-                  ? "bg-gray-700"
-                  : "hover:bg-gray-800")
-              }
-              onClick={() => handleSelect(source)}
-            >
-              {getSourceIcon(source.internalName, 16)}
-              <span className="ml-2 text-sm text-gray-200">
-                {source.displayName}
-              </span>
-            </div>
-          ))}
+        <FiFilter className="my-auto ml-2" size="18" />
      </div>
+
+      {existingSources.length > 0 && (
+        <>
+          <div className="font-medium text-sm flex">Sources</div>
+          <div className="px-1">
+            {sources
+              .filter((source) => existingSources.includes(source.internalName))
+              .map((source) => (
+                <div
+                  key={source.internalName}
+                  className={
+                    "flex cursor-pointer w-full items-center text-white " +
+                    "py-1.5 my-1.5 rounded-lg px-2 " +
+                    (selectedSources.includes(source)
+                      ? "bg-gray-700"
+                      : "hover:bg-gray-800")
+                  }
+                  onClick={() => handleSelect(source)}
+                >
+                  {getSourceIcon(source.internalName, 16)}
+                  <span className="ml-2 text-sm text-gray-200">
+                    {source.displayName}
+                  </span>
+                </div>
+              ))}
+          </div>
+        </>
+      )}
+
+      {availableDocumentSets.length > 0 && (
+        <>
+          <div className="mt-4">
+            <div className="font-medium text-sm flex">Knowledge Sets</div>
+          </div>
+          <div className="px-1">
+            {availableDocumentSets.map((documentSet) => (
+              <div key={documentSet.name} className="my-1.5 flex">
+                <div
+                  key={documentSet.name}
+                  className={
+                    "flex cursor-pointer w-full items-center text-white " +
+                    "py-1.5 rounded-lg px-2 " +
+                    (selectedDocumentSets.includes(documentSet.name)
+                      ? "bg-gray-700"
+                      : "hover:bg-gray-800")
+                  }
+                  onClick={() => handleDocumentSetSelect(documentSet.name)}
+                >
+                  <HoverPopup
+                    mainContent={
+                      <div className="flex my-auto mr-2">
+                        <InfoIcon className={defaultTailwindCSS} />
+                      </div>
+                    }
+                    popupContent={
+                      <div className="text-sm w-64">
+                        <div className="flex font-medium text-gray-200">
+                          Description
+                        </div>
+                        <div className="mt-1 text-gray-300">
+                          {documentSet.description}
+                        </div>
+                      </div>
+                    }
+                    classNameModifications="-ml-2"
+                  />
+                  <span className="text-sm text-gray-200">
+                    {documentSet.name}
+                  </span>
+                </div>
+              </div>
+            ))}
+          </div>
+        </>
+      )}
    </div>
  );
 }
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -4,7 +4,7 @@ import { useRef, useState } from "react";
 import { SearchBar } from "./SearchBar";
 import { SearchResultsDisplay } from "./SearchResultsDisplay";
 import { SourceSelector } from "./Filters";
-import { Connector } from "@/lib/types";
+import { Connector, DocumentSet } from "@/lib/types";
 import { SearchTypeSelector } from "./SearchTypeSelector";
 import {
  DanswerDocument,
@@ -38,13 +38,16 @@ const VALID_QUESTION_RESPONSE_DEFAULT: ValidQuestionResponse = {

 interface SearchSectionProps {
  connectors: Connector<any>[];
+  documentSets: DocumentSet[];
  defaultSearchType: SearchType;
 }

 export const SearchSection: React.FC<SearchSectionProps> = ({
  connectors,
+  documentSets,
  defaultSearchType,
 }) => {
+  console.log(documentSets);
  // Search Bar
  const [query, setQuery] = useState<string>("");

@@ -59,6 +62,9 @@ export const SearchSection: React.FC<SearchSectionProps> = ({

  // Filters
  const [sources, setSources] = useState<Source[]>([]);
+  const [selectedDocumentSets, setSelectedDocumentSets] = useState<string[]>(
+    []
+  );

  // Search Type
  const [selectedSearchType, setSelectedSearchType] =
@@ -135,6 +141,7 @@ export const SearchSection: React.FC<SearchSectionProps> = ({
    const searchFnArgs = {
      query,
      sources,
+      documentSets: selectedDocumentSets,
      updateCurrentAnswer: cancellable({
        cancellationToken: lastSearchCancellationToken.current,
        fn: updateCurrentAnswer,
@@ -183,10 +190,13 @@ export const SearchSection: React.FC<SearchSectionProps> = ({
  return (
    <div className="relative max-w-[2000px] xl:max-w-[1400px] mx-auto">
      <div className="absolute left-0 hidden 2xl:block w-64">
-        {connectors.length > 0 && (
+        {(connectors.length > 0 || documentSets.length > 0) && (
          <SourceSelector
            selectedSources={sources}
            setSelectedSources={setSources}
+            selectedDocumentSets={selectedDocumentSets}
+            setSelectedDocumentSets={setSelectedDocumentSets}
+            availableDocumentSets={documentSets}
            existingSources={connectors.map((connector) => connector.source)}
          />
        )}
--- a/web/src/lib/search/interfaces.ts
+++ b/web/src/lib/search/interfaces.ts
@@ -59,6 +59,7 @@ export interface SearchDefaultOverrides {
 export interface SearchRequestArgs {
  query: string;
  sources: Source[];
+  documentSets: string[];
  updateCurrentAnswer: (val: string) => void;
  updateQuotes: (quotes: Quote[]) => void;
  updateDocs: (documents: DanswerDocument[]) => void;
--- a/web/src/lib/search/qa.ts
+++ b/web/src/lib/search/qa.ts
@@ -5,10 +5,12 @@ import {
  SearchRequestArgs,
  SearchType,
 } from "./interfaces";
+import { buildFilters } from "./utils";

 export const searchRequest = async ({
  query,
  sources,
+  documentSets,
  updateCurrentAnswer,
  updateQuotes,
  updateDocs,
@@ -27,19 +29,16 @@ export const searchRequest = async ({
  let quotes: Quote[] | null = null;
  let relevantDocuments: DanswerDocument[] | null = null;
  try {
+    const filters = buildFilters(sources, documentSets);
    const response = await fetch("/api/direct-qa", {
      method: "POST",
      body: JSON.stringify({
        query,
        collection: "danswer_index",
        use_keyword: useKeyword,
-        ...(sources.length > 0
+        ...(filters.length > 0
          ? {
-              filters: [
-                {
-                  source_type: sources.map((source) => source.internalName),
-                },
-              ],
+              filters,
            }
          : {}),
        offset: offset,
--- a/web/src/lib/search/streamingQa.ts
+++ b/web/src/lib/search/streamingQa.ts
@@ -4,6 +4,7 @@ import {
  SearchRequestArgs,
  SearchType,
 } from "./interfaces";
+import { buildFilters } from "./utils";

 const processSingleChunk = (
  chunk: string,
@@ -54,6 +55,7 @@ const processRawChunkString = (
 export const searchRequestStreamed = async ({
  query,
  sources,
+  documentSets,
  updateCurrentAnswer,
  updateQuotes,
  updateDocs,
@@ -73,19 +75,16 @@ export const searchRequestStreamed = async ({
  let quotes: Quote[] | null = null;
  let relevantDocuments: DanswerDocument[] | null = null;
  try {
+    const filters = buildFilters(sources, documentSets);
    const response = await fetch("/api/stream-direct-qa", {
      method: "POST",
      body: JSON.stringify({
        query,
        collection: "danswer_index",
        use_keyword: useKeyword,
-        ...(sources.length > 0
+        ...(filters.length > 0
          ? {
-              filters: [
-                {
-                  source_type: sources.map((source) => source.internalName),
-                },
-              ],
+              filters,
            }
          : {}),
        offset: offset,
--- a/web/src/lib/search/utils.ts
+++ b/web/src/lib/search/utils.ts
@@ -0,0 +1,16 @@
+import { Source } from "./interfaces";
+
+export const buildFilters = (sources: Source[], documentSets: string[]) => {
+  const filters = [];
+  if (sources.length > 0) {
+    filters.push({
+      source_type: sources.map((source) => source.internalName),
+    });
+  }
+  if (documentSets.length > 0) {
+    filters.push({
+      document_sets: documentSets,
+    });
+  }
+  return filters;
+};
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -222,11 +222,11 @@ export interface CCPairDescriptor<ConnectorType, CredentialType> {
  credential: Credential<CredentialType>;
 }

-export interface DocumentSet<ConnectorType, CredentialType> {
+export interface DocumentSet {
  id: number;
  name: string;
  description: string;
-  cc_pair_descriptors: CCPairDescriptor<ConnectorType, CredentialType>[];
+  cc_pair_descriptors: CCPairDescriptor<any, any>[];
  is_up_to_date: boolean;
 }

@@ -239,7 +239,7 @@ export interface ChannelConfig {

 export interface SlackBotConfig {
  id: number;
-  document_sets: DocumentSet<any, any>[];
+  document_sets: DocumentSet[];
  channel_config: ChannelConfig;
 }