Add hiding of documents to feedback page (#585)

backend/danswer/configs/constants.py

@@ -24,6 +24,7 @@ PUBLIC_DOC_PAT = "PUBLIC"
 PUBLIC_DOCUMENT_SET = "__PUBLIC"
 QUOTE = "quote"
 BOOST = "boost"
+HIDDEN = "hidden"
 SCORE = "score"
 ID_SEPARATOR = ":;:"
 DEFAULT_BOOST = 0

backend/danswer/datastores/interfaces.py

@@ -36,6 +36,7 @@ class UpdateRequest:
     access: DocumentAccess | None = None
     document_sets: set[str] | None = None
     boost: float | None = None
+    hidden: bool | None = None
 
 
 class Verifiable(abc.ABC):

backend/danswer/datastores/vespa/app_config/schemas/danswer_chunk.sd

@@ -47,6 +47,9 @@ schema danswer_chunk {
         field boost type float {
             indexing: summary | attribute
         }
+        field hidden type bool {
+            indexing: summary | attribute
+        }
         field metadata type string {
             indexing: summary | attribute
         }

backend/danswer/datastores/vespa/store.py

@@ -30,6 +30,7 @@ from danswer.configs.constants import DEFAULT_BOOST
 from danswer.configs.constants import DOCUMENT_ID
 from danswer.configs.constants import DOCUMENT_SETS
 from danswer.configs.constants import EMBEDDINGS
+from danswer.configs.constants import HIDDEN
 from danswer.configs.constants import MATCH_HIGHLIGHTS
 from danswer.configs.constants import METADATA
 from danswer.configs.constants import SCORE
@@ -271,8 +272,10 @@ def _build_vespa_filters(filters: list[IndexFilter] | None) -> str:
     # via the `filters` arg. These are set either in the Web UI or in the Slack
     # listener
 
+    # ignore hidden docs
+    filter_str = f"!({HIDDEN}=true) and "
+
     # Handle provided query filters
-    filter_str = ""
     if filters:
         for filter_dict in filters:
             valid_filters = {
@@ -424,16 +427,26 @@ class VespaIndex(DocumentIndex):
         batch_size: int = _BATCH_SIZE,
     ) -> None:
         """Runs a batch of updates in parallel via the ThreadPoolExecutor."""
+
+        def _update_chunk(update: _VespaUpdateRequest) -> Response:
+            update_body = json.dumps(update.update_request)
+            logger.debug(
+                f"Updating with request to {update.url} with body {update_body}"
+            )
+            return requests.put(
+                update.url,
+                headers={"Content-Type": "application/json"},
+                data=update_body,
+            )
+
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=_NUM_THREADS
         ) as executor:
             for update_batch in batch_generator(updates, batch_size):
                 future_to_document_id = {
                     executor.submit(
-                        requests.put,
-                        update.url,
-                        headers={"Content-Type": "application/json"},
-                        data=json.dumps(update.update_request),
+                        _update_chunk,
+                        update,
                     ): update.document_id
                     for update in update_batch
                 }
@@ -451,14 +464,6 @@ class VespaIndex(DocumentIndex):
 
         processed_updates_requests: list[_VespaUpdateRequest] = []
         for update_request in update_requests:
-            if (
-                update_request.boost is None
-                and update_request.access is None
-                and update_request.document_sets is None
-            ):
-                logger.error("Update request received but nothing to update")
-                continue
-
             update_dict: dict[str, dict] = {"fields": {}}
             if update_request.boost is not None:
                 update_dict["fields"][BOOST] = {"assign": update_request.boost}
@@ -474,6 +479,12 @@ class VespaIndex(DocumentIndex):
                         acl_entry: 1 for acl_entry in update_request.access.to_acl()
                     }
                 }
+            if update_request.hidden is not None:
+                update_dict["fields"][HIDDEN] = {"assign": update_request.hidden}
+
+            if not update_dict["fields"]:
+                logger.error("Update request received but nothing to update")
+                continue
 
             for document_id in update_request.document_ids:
                 for doc_chunk_id in _get_vespa_chunk_ids_by_document_id(document_id):

backend/danswer/db/feedback.py

@@ -46,7 +46,11 @@ def fetch_docs_ranked_by_boost(
     db_session: Session, ascending: bool = False, limit: int = 100
 ) -> list[DbDocument]:
     order_func = asc if ascending else desc
-    stmt = select(DbDocument).order_by(order_func(DbDocument.boost)).limit(limit)
+    stmt = (
+        select(DbDocument)
+        .order_by(order_func(DbDocument.boost), order_func(DbDocument.semantic_id))
+        .limit(limit)
+    )
     result = db_session.execute(stmt)
     doc_list = result.scalars().all()
 
@@ -71,6 +75,24 @@ def update_document_boost(db_session: Session, document_id: str, boost: int) ->
     db_session.commit()
 
 
+def update_document_hidden(db_session: Session, document_id: str, hidden: bool) -> None:
+    stmt = select(DbDocument).where(DbDocument.id == document_id)
+    result = db_session.execute(stmt).scalar_one_or_none()
+    if result is None:
+        raise ValueError(f"No document found with ID: '{document_id}'")
+
+    result.hidden = hidden
+
+    update = UpdateRequest(
+        document_ids=[document_id],
+        hidden=hidden,
+    )
+
+    get_default_document_index().update([update])
+
+    db_session.commit()
+
+
 def create_query_event(
     query: str,
     selected_flow: SearchType | None,

backend/danswer/server/manage.py

@@ -54,6 +54,7 @@ from danswer.db.document import get_document_cnts_for_cc_pairs
 from danswer.db.engine import get_session
 from danswer.db.feedback import fetch_docs_ranked_by_boost
 from danswer.db.feedback import update_document_boost
+from danswer.db.feedback import update_document_hidden
 from danswer.db.index_attempt import create_index_attempt
 from danswer.db.index_attempt import get_latest_index_attempts
 from danswer.db.models import User
@@ -78,6 +79,7 @@ from danswer.server.models import GDriveCallback
 from danswer.server.models import GoogleAppCredentials
 from danswer.server.models import GoogleServiceAccountCredentialRequest
 from danswer.server.models import GoogleServiceAccountKey
+from danswer.server.models import HiddenUpdateRequest
 from danswer.server.models import IndexAttemptSnapshot
 from danswer.server.models import ObjectCreationIdResponse
 from danswer.server.models import RunConnectorRequest
@@ -133,6 +135,22 @@ def document_boost_update(
         raise HTTPException(status_code=400, detail=str(e))
 
 
+@router.post("/admin/doc-hidden")
+def document_hidden_update(
+    hidden_update: HiddenUpdateRequest,
+    _: User | None = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
+) -> None:
+    try:
+        update_document_hidden(
+            db_session=db_session,
+            document_id=hidden_update.document_id,
+            hidden=hidden_update.hidden,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
 @router.get("/admin/connector/google-drive/app-credential")
 def check_google_app_credentials_exist(
     _: User = Depends(current_admin_user),

backend/danswer/server/models.py

@@ -135,6 +135,11 @@ class BoostUpdateRequest(BaseModel):
     boost: int
 
 
+class HiddenUpdateRequest(BaseModel):
+    document_id: str
+    hidden: bool
+
+
 class SearchDoc(BaseModel):
     document_id: str
     semantic_identifier: str

web/src/app/admin/connectors/document360/page.tsx

@@ -96,14 +96,15 @@ const MainSection = () => {
         <>
           <p className="text-sm mb-4">
             To use the Document360 connector, you must first provide the API
-            token and portal ID corresponding to your Document360 setup. See setup guide{" "}
+            token and portal ID corresponding to your Document360 setup. See
+            setup guide{" "}
             <a
               className="text-blue-500"
               href="https://docs.danswer.dev/connectors/document360"
             >
               here
-            </a>
-            {" "}for more detail.
+            </a>{" "}
+            for more detail.
           </p>
           <div className="border-solid border-gray-600 border rounded-md p-6 mt-2">
             <CredentialForm<Document360CredentialJson>

web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx

@@ -0,0 +1,243 @@
+import { BasicTable } from "@/components/admin/connectors/BasicTable";
+import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
+import { useState } from "react";
+import { PageSelector } from "@/components/PageSelector";
+import { DocumentBoostStatus } from "@/lib/types";
+import { updateBoost, updateHiddenStatus } from "./lib";
+import { CheckmarkIcon, EditIcon } from "@/components/icons/icons";
+import { numToDisplay } from "./constants";
+import { FiCheck, FiCheckSquare, FiEye, FiEyeOff, FiX } from "react-icons/fi";
+import { getErrorMsg } from "@/lib/fetchUtils";
+import { HoverPopup } from "@/components/HoverPopup";
+import { CustomCheckbox } from "@/components/CustomCheckbox";
+
+const IsVisibleSection = ({
+  document,
+  onUpdate,
+}: {
+  document: DocumentBoostStatus;
+  onUpdate: (response: Response) => void;
+}) => {
+  return (
+    <HoverPopup
+      mainContent={
+        document.hidden ? (
+          <div
+            onClick={async () => {
+              const response = await updateHiddenStatus(
+                document.document_id,
+                false
+              );
+              onUpdate(response);
+            }}
+            className="flex text-red-700 cursor-pointer hover:bg-gray-700 py-1 px-2 w-fit rounded-full"
+          >
+            <div className="select-none">Hidden</div>
+            <div className="ml-1 my-auto">
+              <CustomCheckbox checked={false} />
+            </div>
+          </div>
+        ) : (
+          <div
+            onClick={async () => {
+              const response = await updateHiddenStatus(
+                document.document_id,
+                true
+              );
+              onUpdate(response);
+            }}
+            className="flex text-gray-400 cursor-pointer hover:bg-gray-700 py-1 px-2 w-fit rounded-full"
+          >
+            <div className="text-gray-400 my-auto select-none">Visible</div>
+            <div className="ml-1 my-auto">
+              <CustomCheckbox checked={true} />
+            </div>
+          </div>
+        )
+      }
+      popupContent={
+        <div className="text-xs text-gray-300">
+          {document.hidden ? (
+            <div className="flex">
+              <FiEye className="my-auto mr-1" /> Unhide
+            </div>
+          ) : (
+            <div className="flex">
+              <FiEyeOff className="my-auto mr-1" />
+              Hide
+            </div>
+          )}
+        </div>
+      }
+      direction="left"
+    />
+  );
+};
+
+const ScoreSection = ({
+  documentId,
+  initialScore,
+  setPopup,
+  refresh,
+}: {
+  documentId: string;
+  initialScore: number;
+  setPopup: (popupSpec: PopupSpec | null) => void;
+  refresh: () => void;
+}) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const [score, setScore] = useState(initialScore.toString());
+
+  const onSubmit = async () => {
+    const numericScore = Number(score);
+    if (isNaN(numericScore)) {
+      setPopup({
+        message: "Score must be a number",
+        type: "error",
+      });
+      return;
+    }
+
+    const errorMsg = await updateBoost(documentId, numericScore);
+    if (errorMsg) {
+      setPopup({
+        message: errorMsg,
+        type: "error",
+      });
+    } else {
+      setPopup({
+        message: "Updated score!",
+        type: "success",
+      });
+      refresh();
+      setIsOpen(false);
+    }
+  };
+
+  if (isOpen) {
+    return (
+      <div className="m-auto flex">
+        <input
+          value={score}
+          onChange={(e) => {
+            setScore(e.target.value);
+          }}
+          onKeyDown={(e) => {
+            if (e.key === "Enter") {
+              onSubmit();
+            }
+            if (e.key === "Escape") {
+              setIsOpen(false);
+              setScore(initialScore.toString());
+            }
+          }}
+          className="border bg-slate-700 text-gray-200 border-gray-300 rounded py-1 px-3 w-16"
+        />
+        <div onClick={onSubmit} className="cursor-pointer my-auto ml-2">
+          <CheckmarkIcon size={20} className="text-green-700" />
+        </div>
+      </div>
+    );
+  }
+
+  return (
+    <div className="h-full flex flex-col">
+      <div className="flex my-auto">
+        <div className="w-6 flex">
+          <div className="ml-auto">{initialScore}</div>
+        </div>
+        <div className="cursor-pointer ml-2" onClick={() => setIsOpen(true)}>
+          <EditIcon size={20} />
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export const DocumentFeedbackTable = ({
+  documents,
+  refresh,
+}: {
+  documents: DocumentBoostStatus[];
+  refresh: () => void;
+}) => {
+  const [page, setPage] = useState(1);
+  const { popup, setPopup } = usePopup();
+
+  return (
+    <div>
+      {popup}
+      <BasicTable
+        columns={[
+          {
+            header: "Document Name",
+            key: "name",
+          },
+          {
+            header: "Is Searchable?",
+            key: "visible",
+          },
+          {
+            header: "Score",
+            key: "score",
+            alignment: "right",
+          },
+        ]}
+        data={documents
+          .slice((page - 1) * numToDisplay, page * numToDisplay)
+          .map((document) => {
+            return {
+              name: (
+                <a
+                  className="text-blue-600"
+                  href={document.link}
+                  target="_blank"
+                  rel="noopener noreferrer"
+                >
+                  {document.semantic_id}
+                </a>
+              ),
+              visible: (
+                <IsVisibleSection
+                  document={document}
+                  onUpdate={async (response) => {
+                    if (response.ok) {
+                      refresh();
+                    } else {
+                      setPopup({
+                        message: `Error updating hidden status - ${getErrorMsg(
+                          response
+                        )}`,
+                        type: "error",
+                      });
+                    }
+                  }}
+                />
+              ),
+              score: (
+                <div className="ml-auto flex w-16">
+                  <div key={document.document_id} className="h-10 ml-auto mr-8">
+                    <ScoreSection
+                      documentId={document.document_id}
+                      initialScore={document.boost}
+                      refresh={refresh}
+                      setPopup={setPopup}
+                    />
+                  </div>
+                </div>
+              ),
+            };
+          })}
+      />
+      <div className="mt-3 flex">
+        <div className="mx-auto">
+          <PageSelector
+            totalPages={Math.ceil(documents.length / numToDisplay)}
+            currentPage={page}
+            onPageChange={(newPage) => setPage(newPage)}
+          />
+        </div>
+      </div>
+    </div>
+  );
+};

web/src/app/admin/documents/feedback/constants.ts

@@ -0,0 +1,2 @@
+export const numPages = 8;
+export const numToDisplay = 10;

web/src/app/admin/documents/feedback/lib.ts

@@ -0,0 +1,34 @@
+export const updateBoost = async (documentId: string, boost: number) => {
+  const response = await fetch("/api/manage/admin/doc-boosts", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      document_id: documentId,
+      boost,
+    }),
+  });
+  if (response.ok) {
+    return null;
+  }
+  const responseJson = await response.json();
+  return responseJson.message || responseJson.detail || "Unknown error";
+};
+
+export const updateHiddenStatus = async (
+  documentId: string,
+  isHidden: boolean
+) => {
+  const response = await fetch("/api/manage/admin/doc-hidden", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      document_id: documentId,
+      hidden: isHidden,
+    }),
+  });
+  return response;
+};

web/src/app/admin/documents/feedback/page.tsx

@@ -1,187 +1,10 @@
 "use client";
 
 import { LoadingAnimation } from "@/components/Loading";
-import { PageSelector } from "@/components/PageSelector";
-import { BasicTable } from "@/components/admin/connectors/BasicTable";
-import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
-import {
-  CheckmarkIcon,
-  EditIcon,
-  ThumbsUpIcon,
-} from "@/components/icons/icons";
+import { ThumbsUpIcon } from "@/components/icons/icons";
 import { useMostReactedToDocuments } from "@/lib/hooks";
-import { DocumentBoostStatus, User } from "@/lib/types";
-import { useState } from "react";
-
-const numPages = 8;
-const numToDisplay = 10;
-
-const updateBoost = async (documentId: string, boost: number) => {
-  const response = await fetch("/api/manage/admin/doc-boosts", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify({
-      document_id: documentId,
-      boost,
-    }),
-  });
-  if (response.ok) {
-    return null;
-  }
-  const responseJson = await response.json();
-  return responseJson.message || responseJson.detail || "Unknown error";
-};
-
-const ScoreSection = ({
-  documentId,
-  initialScore,
-  setPopup,
-  refresh,
-}: {
-  documentId: string;
-  initialScore: number;
-  setPopup: (popupSpec: PopupSpec | null) => void;
-  refresh: () => void;
-}) => {
-  const [isOpen, setIsOpen] = useState(false);
-  const [score, setScore] = useState(initialScore.toString());
-
-  const onSubmit = async () => {
-    const numericScore = Number(score);
-    if (isNaN(numericScore)) {
-      setPopup({
-        message: "Score must be a number",
-        type: "error",
-      });
-      return;
-    }
-
-    const errorMsg = await updateBoost(documentId, numericScore);
-    if (errorMsg) {
-      setPopup({
-        message: errorMsg,
-        type: "error",
-      });
-    } else {
-      setPopup({
-        message: "Updated score!",
-        type: "success",
-      });
-      refresh();
-      setIsOpen(false);
-    }
-  };
-
-  if (isOpen) {
-    return (
-      <div className="m-auto flex">
-        <input
-          value={score}
-          onChange={(e) => {
-            setScore(e.target.value);
-          }}
-          onKeyDown={(e) => {
-            if (e.key === "Enter") {
-              onSubmit();
-            }
-            if (e.key === "Escape") {
-              setIsOpen(false);
-              setScore(initialScore.toString());
-            }
-          }}
-          className="border bg-slate-700 text-gray-200 border-gray-300 rounded py-1 px-3 w-16"
-        />
-        <div onClick={onSubmit} className="cursor-pointer my-auto ml-2">
-          <CheckmarkIcon size={20} className="text-green-700" />
-        </div>
-      </div>
-    );
-  }
-
-  return (
-    <div className="h-full flex flex-col">
-      <div className="flex my-auto">
-        <div className="w-6 flex">
-          <div className="ml-auto">{initialScore}</div>
-        </div>
-        <div className="cursor-pointer ml-2" onClick={() => setIsOpen(true)}>
-          <EditIcon size={20} />
-        </div>
-      </div>
-    </div>
-  );
-};
-
-interface DocumentFeedbackTableProps {
-  documents: DocumentBoostStatus[];
-  refresh: () => void;
-}
-
-const DocumentFeedbackTable = ({
-  documents,
-  refresh,
-}: DocumentFeedbackTableProps) => {
-  const [page, setPage] = useState(1);
-  const { popup, setPopup } = usePopup();
-
-  return (
-    <div>
-      {popup}
-      <BasicTable
-        columns={[
-          {
-            header: "Document Name",
-            key: "name",
-          },
-          {
-            header: "Score",
-            key: "score",
-            alignment: "right",
-          },
-        ]}
-        data={documents
-          .slice((page - 1) * numToDisplay, page * numToDisplay)
-          .map((document) => {
-            return {
-              name: (
-                <a
-                  className="text-blue-600"
-                  href={document.link}
-                  target="_blank"
-                  rel="noopener noreferrer"
-                >
-                  {document.semantic_id}
-                </a>
-              ),
-              score: (
-                <div className="ml-auto flex w-16">
-                  <div key={document.document_id} className="h-10 ml-auto mr-8">
-                    <ScoreSection
-                      documentId={document.document_id}
-                      initialScore={document.boost}
-                      refresh={refresh}
-                      setPopup={setPopup}
-                    />
-                  </div>
-                </div>
-              ),
-            };
-          })}
-      />
-      <div className="mt-3 flex">
-        <div className="mx-auto">
-          <PageSelector
-            totalPages={Math.ceil(documents.length / numToDisplay)}
-            currentPage={page}
-            onPageChange={(newPage) => setPage(newPage)}
-          />
-        </div>
-      </div>
-    </div>
-  );
-};
+import { DocumentFeedbackTable } from "./DocumentFeedbackTable";
+import { numPages, numToDisplay } from "./constants";
 
 const Main = () => {
   const {

web/src/components/CustomCheckbox.tsx

@@ -0,0 +1,34 @@
+export const CustomCheckbox = ({
+  checked,
+  onChange,
+}: {
+  checked: boolean;
+  onChange?: () => void;
+}) => {
+  return (
+    <label className="flex items-center cursor-pointer">
+      <input
+        type="checkbox"
+        className="hidden"
+        checked={checked}
+        onChange={onChange}
+      />
+      <span className="relative">
+        <span
+          className={`block w-3 h-3 border border-gray-600 rounded ${
+            checked ? "bg-green-700" : "bg-gray-800"
+          } transition duration-300`}
+        >
+          {checked && (
+            <svg
+              className="absolute top-0 left-0 w-3 h-3 fill-current text-gray-200"
+              viewBox="0 0 20 20"
+            >
+              <path d="M0 11l2-2 5 5L18 3l2 2L7 18z" />
+            </svg>
+          )}
+        </span>
+      </span>
+    </label>
+  );
+};

web/src/components/HoverPopup.tsx

@@ -4,26 +4,42 @@ interface HoverPopupProps {
   mainContent: string | JSX.Element;
   popupContent: string | JSX.Element;
   classNameModifications?: string;
+  direction?: "left" | "bottom";
 }
 
 export const HoverPopup = ({
   mainContent,
   popupContent,
   classNameModifications,
+  direction = "bottom",
 }: HoverPopupProps) => {
   const [hovered, setHovered] = useState(false);
 
+  let popupDirectionClass;
+  switch (direction) {
+    case "left":
+      popupDirectionClass = "top-0 left-0 transform translate-x-[-110%]";
+      break;
+    case "bottom":
+      popupDirectionClass = "top-0 left-0 mt-8";
+      break;
+  }
+
   return (
     <div
       className="relative flex"
-      onMouseEnter={() => setHovered(true)}
+      onMouseEnter={() => {
+        setHovered(true);
+        console.log("HIII");
+      }}
       onMouseLeave={() => setHovered(false)}
     >
       {hovered && (
         <div
           className={
-            `absolute top-0 left-0 mt-8 bg-gray-700 px-3 py-2 rounded shadow-lg z-30 ` +
-              classNameModifications || ""
+            `absolute bg-gray-700 px-3 py-2 rounded shadow-lg z-30 ` +
+            (classNameModifications || "") +
+            ` ${popupDirectionClass}`
           }
         >
           {popupContent}

web/src/components/search/Filters.tsx

@@ -1,14 +1,8 @@
 import React from "react";
 import { getSourceIcon } from "../source";
-import { Funnel } from "@phosphor-icons/react";
 import { DocumentSet, ValidSources } from "@/lib/types";
 import { Source } from "@/lib/search/interfaces";
-import {
-  BookmarkIcon,
-  InfoIcon,
-  NotebookIcon,
-  defaultTailwindCSS,
-} from "../icons/icons";
+import { InfoIcon, defaultTailwindCSS } from "../icons/icons";
 import { HoverPopup } from "../HoverPopup";
 import { FiFilter } from "react-icons/fi";
 

web/src/lib/fetchUtils.ts

@@ -0,0 +1,7 @@
+export const getErrorMsg = async (response: Response) => {
+  if (response.ok) {
+    return null;
+  }
+  const responseJson = await response.json();
+  return responseJson.message || responseJson.detail || "Unknown error";
+};