Slack bot improve source feedback (#827)

---------

Co-authored-by: Yuhong Sun <yuhongsun96@gmail.com>
Co-authored-by: Matthieu Boret <matthieu.boret@fr.clara.net>

backend/danswer/configs/danswerbot_configs.py

@@ -48,7 +48,3 @@ ENABLE_DANSWERBOT_REFLEXION = (
 )
 # Currently not support chain of thought, probably will add back later
 DANSWER_BOT_DISABLE_COT = True
-# Add the per document feedback blocks that affect the document rankings via boosting
-ENABLE_SLACK_DOC_FEEDBACK = (
-    os.environ.get("ENABLE_SLACK_DOC_FEEDBACK", "").lower() == "true"
-)

backend/danswer/danswerbot/slack/blocks.py

@@ -5,19 +5,20 @@ import timeago  # type: ignore
 from slack_sdk.models.blocks import ActionsBlock
 from slack_sdk.models.blocks import Block
 from slack_sdk.models.blocks import ButtonElement
-from slack_sdk.models.blocks import ConfirmObject
 from slack_sdk.models.blocks import DividerBlock
 from slack_sdk.models.blocks import HeaderBlock
+from slack_sdk.models.blocks import Option
+from slack_sdk.models.blocks import RadioButtonsElement
 from slack_sdk.models.blocks import SectionBlock
 
 from danswer.chat.models import DanswerQuote
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import SearchFeedbackType
 from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_DOCS_TO_DISPLAY
-from danswer.configs.danswerbot_configs import ENABLE_SLACK_DOC_FEEDBACK
 from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
+from danswer.danswerbot.slack.constants import FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
-from danswer.danswerbot.slack.utils import build_feedback_block_id
+from danswer.danswerbot.slack.utils import build_feedback_id
 from danswer.danswerbot.slack.utils import remove_slack_text_interactions
 from danswer.danswerbot.slack.utils import translate_vespa_highlight_to_slack
 from danswer.search.models import SavedSearchDoc
@@ -28,7 +29,7 @@ _MAX_BLURB_LEN = 75
 
 def build_qa_feedback_block(message_id: int) -> Block:
     return ActionsBlock(
-        block_id=build_feedback_block_id(message_id),
+        block_id=build_feedback_id(message_id),
         elements=[
             ButtonElement(
                 action_id=LIKE_BLOCK_ACTION_ID,
@@ -44,33 +45,43 @@ def build_qa_feedback_block(message_id: int) -> Block:
     )
 
 
+def get_document_feedback_blocks() -> Block:
+    return SectionBlock(
+        text=(
+            "If this document is a good source of information and should be shown more often, "
+            "please select 'Boost'. Conversely, if it seems to be a bad source of information, "
+            "select 'Down-boost'. You can also select 'Hide' if the source is deprecated and "
+            "should not be taken into account anymore."
+        ),
+        accessory=RadioButtonsElement(
+            options=[
+                Option(
+                    text=":thumbsup: Boost",
+                    value=SearchFeedbackType.ENDORSE.value,
+                ),
+                Option(
+                    text=":thumbsdown: Down-Boost",
+                    value=SearchFeedbackType.REJECT.value,
+                ),
+                Option(
+                    text=":palms_up_together: Hide",
+                    value=SearchFeedbackType.HIDE.value,
+                ),
+            ]
+        ),
+    )
+
+
 def build_doc_feedback_block(
     message_id: int,
     document_id: str,
     document_rank: int,
-) -> Block:
+) -> ButtonElement:
-    return ActionsBlock(
+    feedback_id = build_feedback_id(message_id, document_id, document_rank)
-        block_id=build_feedback_block_id(message_id, document_id, document_rank),
+    return ButtonElement(
-        elements=[
+        action_id=FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID,
-            ButtonElement(
+        value=feedback_id,
-                action_id=SearchFeedbackType.ENDORSE.value,
+        text="Source feedback",
-                text="⬆",
-                style="primary",
-                confirm=ConfirmObject(
-                    title="Endorse this Document",
-                    text="This is a good source of information and should be shown more often!",
-                ),
-            ),
-            ButtonElement(
-                action_id=SearchFeedbackType.REJECT.value,
-                text="⬇",
-                style="danger",
-                confirm=ConfirmObject(
-                    title="Reject this Document",
-                    text="This is a bad source of information and should be shown less often.",
-                ),
-            ),
-        ],
     )
 
 
@@ -92,7 +103,6 @@ def build_documents_blocks(
     documents: list[SavedSearchDoc],
     message_id: int | None,
     num_docs_to_display: int = DANSWER_BOT_NUM_DOCS_TO_DISPLAY,
-    include_feedback: bool = ENABLE_SLACK_DOC_FEEDBACK,
 ) -> list[Block]:
     seen_docs_identifiers = set()
     section_blocks: list[Block] = [HeaderBlock(text="Reference Documents")]
@@ -125,17 +135,18 @@ def build_documents_blocks(
 
         block_text = header_line + updated_at_line + body_text
 
-        section_blocks.append(SectionBlock(text=block_text))
+        feedback: ButtonElement | dict = {}
-
+        if message_id is not None:
-        if include_feedback and message_id is not None:
+            feedback = build_doc_feedback_block(
-            section_blocks.append(
+                message_id=message_id,
-                build_doc_feedback_block(
+                document_id=d.document_id,
-                    message_id=message_id,
+                document_rank=rank,
-                    document_id=d.document_id,
-                    document_rank=rank,
-                ),
             )
 
+        section_blocks.append(
+            SectionBlock(text=block_text, accessory=feedback),
+        )
+
         section_blocks.append(DividerBlock())
 
         if included_docs >= num_docs_to_display:

backend/danswer/danswerbot/slack/constants.py

@@ -1,3 +1,5 @@
 LIKE_BLOCK_ACTION_ID = "feedback-like"
 DISLIKE_BLOCK_ACTION_ID = "feedback-dislike"
+FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID = "feedback-doc-button"
 SLACK_CHANNEL_ID = "channel_id"
+VIEW_DOC_FEEDBACK_ID = "view-doc-feedback"

backend/danswer/danswerbot/slack/handlers/handle_feedback.py

@@ -1,18 +1,60 @@
 from slack_sdk import WebClient
+from slack_sdk.models.views import View
+from slack_sdk.socket_mode import SocketModeClient
+from slack_sdk.socket_mode.request import SocketModeRequest
 from sqlalchemy.orm import Session
 
 from danswer.configs.constants import SearchFeedbackType
+from danswer.danswerbot.slack.blocks import get_document_feedback_blocks
 from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
-from danswer.danswerbot.slack.utils import decompose_block_id
+from danswer.danswerbot.slack.constants import VIEW_DOC_FEEDBACK_ID
+from danswer.danswerbot.slack.utils import build_feedback_id
+from danswer.danswerbot.slack.utils import decompose_feedback_id
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.feedback import create_chat_message_feedback
 from danswer.db.feedback import create_doc_retrieval_feedback
 from danswer.document_index.factory import get_default_document_index
+from danswer.utils.logger import setup_logger
+
+logger_base = setup_logger()
+
+
+def handle_doc_feedback_button(
+    req: SocketModeRequest,
+    client: SocketModeClient,
+) -> None:
+    if not (actions := req.payload.get("actions")):
+        logger_base.error("Missing actions. Unable to build the source feedback view")
+        return
+
+    # Extracts the feedback_id coming from the 'source feedback' button
+    # and generates a new one for the View, to keep track of the doc info
+    query_event_id, doc_id, doc_rank = decompose_feedback_id(actions[0].get("value"))
+    external_id = build_feedback_id(query_event_id, doc_id, doc_rank)
+
+    channel_id = req.payload["container"]["channel_id"]
+    thread_ts = req.payload["container"]["thread_ts"]
+
+    data = View(
+        type="modal",
+        callback_id=VIEW_DOC_FEEDBACK_ID,
+        external_id=external_id,
+        # We use the private metadata to keep track of the channel id and thread ts
+        private_metadata=f"{channel_id}_{thread_ts}",
+        title="Source Feedback",
+        blocks=[get_document_feedback_blocks()],
+        submit="send",
+        close="cancel",
+    )
+
+    client.web_client.views_open(
+        trigger_id=req.payload["trigger_id"], view=data.to_dict()
+    )
 
 
 def handle_slack_feedback(
-    block_id: str,
+    feedback_id: str,
     feedback_type: str,
     client: WebClient,
     user_id_to_post_confirmation: str,
@@ -21,7 +63,7 @@ def handle_slack_feedback(
 ) -> None:
     engine = get_sqlalchemy_engine()
 
-    message_id, doc_id, doc_rank = decompose_block_id(block_id)
+    message_id, doc_id, doc_rank = decompose_feedback_id(feedback_id)
 
     with Session(engine) as db_session:
         if feedback_type in [LIKE_BLOCK_ACTION_ID, DISLIKE_BLOCK_ACTION_ID]:
@@ -32,13 +74,21 @@ def handle_slack_feedback(
                 user_id=None,  # no "user" for Slack bot for now
                 db_session=db_session,
             )
-        if feedback_type in [
+        elif feedback_type in [
             SearchFeedbackType.ENDORSE.value,
             SearchFeedbackType.REJECT.value,
+            SearchFeedbackType.HIDE.value,
         ]:
             if doc_id is None or doc_rank is None:
                 raise ValueError("Missing information for Document Feedback")
 
+            if feedback_type == SearchFeedbackType.ENDORSE.value:
+                feedback = SearchFeedbackType.ENDORSE
+            elif feedback_type == SearchFeedbackType.REJECT.value:
+                feedback = SearchFeedbackType.REJECT
+            else:
+                feedback = SearchFeedbackType.HIDE
+
             create_doc_retrieval_feedback(
                 message_id=message_id,
                 document_id=doc_id,
@@ -46,10 +96,10 @@ def handle_slack_feedback(
                 document_index=get_default_document_index(),
                 db_session=db_session,
                 clicked=False,  # Not tracking this for Slack
-                feedback=SearchFeedbackType.ENDORSE
+                feedback=feedback,
-                if feedback_type == SearchFeedbackType.ENDORSE.value
-                else SearchFeedbackType.REJECT,
             )
+        else:
+            logger_base.error(f"Feedback type '{feedback_type}' not supported")
 
     # post message to slack confirming that feedback was received
     client.chat_postEphemeral(

backend/danswer/danswerbot/slack/listener.py

@@ -14,17 +14,23 @@ from danswer.configs.danswerbot_configs import DANSWER_BOT_RESPOND_EVERY_CHANNEL
 from danswer.configs.danswerbot_configs import NOTIFY_SLACKBOT_NO_ANSWER
 from danswer.configs.model_configs import ENABLE_RERANKING_ASYNC_FLOW
 from danswer.danswerbot.slack.config import get_slack_bot_config_for_channel
+from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
+from danswer.danswerbot.slack.constants import FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID
+from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import SLACK_CHANNEL_ID
+from danswer.danswerbot.slack.constants import VIEW_DOC_FEEDBACK_ID
+from danswer.danswerbot.slack.handlers.handle_feedback import handle_doc_feedback_button
 from danswer.danswerbot.slack.handlers.handle_feedback import handle_slack_feedback
 from danswer.danswerbot.slack.handlers.handle_message import handle_message
 from danswer.danswerbot.slack.models import SlackMessageInfo
 from danswer.danswerbot.slack.tokens import fetch_tokens
 from danswer.danswerbot.slack.utils import ChannelIdAdapter
-from danswer.danswerbot.slack.utils import decompose_block_id
+from danswer.danswerbot.slack.utils import decompose_feedback_id
 from danswer.danswerbot.slack.utils import get_channel_name_from_id
 from danswer.danswerbot.slack.utils import get_danswer_bot_app_id
 from danswer.danswerbot.slack.utils import read_slack_thread
 from danswer.danswerbot.slack.utils import remove_danswer_bot_tag
+from danswer.danswerbot.slack.utils import get_view_values
 from danswer.danswerbot.slack.utils import respond_in_thread
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.dynamic_configs.interface import ConfigNotFoundError
@@ -33,7 +39,6 @@ from danswer.search.search_nlp_models import warm_up_models
 from danswer.server.manage.models import SlackBotTokens
 from danswer.utils.logger import setup_logger
 
-
 logger = setup_logger()
 
 
@@ -131,28 +136,41 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool
 
 
 def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None:
-    actions = req.payload.get("actions")
+    # Answer feedback
-    if not actions:
+    if actions := req.payload.get("actions"):
-        logger.error("Unable to process block actions - no actions found")
+        action = cast(dict[str, Any], actions[0])
+        feedback_type = cast(str, action.get("action_id"))
+        feedback_id = cast(str, action.get("block_id"))
+        channel_id = cast(str, req.payload["container"]["channel_id"])
+        thread_ts = cast(str, req.payload["container"]["thread_ts"])
+    # Doc feedback
+    elif view := req.payload.get("view"):
+        view_values = get_view_values(view["state"]["values"])
+        private_metadata = view.get("private_metadata").split("_")
+        if not view_values:
+            logger.error("Unable to process feedback. Missing view values")
+            return
+
+        feedback_type = [x for x in view_values.values()][0]
+        feedback_id = cast(str, view.get("external_id"))
+        channel_id = private_metadata[0]
+        thread_ts = private_metadata[1]
+    else:
+        logger.error("Unable to process feedback. Actions or View not found")
         return
 
-    action = cast(dict[str, Any], actions[0])
-    action_id = cast(str, action.get("action_id"))
-    block_id = cast(str, action.get("block_id"))
     user_id = cast(str, req.payload["user"]["id"])
-    channel_id = cast(str, req.payload["container"]["channel_id"])
-    thread_ts = cast(str, req.payload["container"]["thread_ts"])
 
     handle_slack_feedback(
-        block_id=block_id,
+        feedback_id=feedback_id,
-        feedback_type=action_id,
+        feedback_type=feedback_type,
         client=client.web_client,
         user_id_to_post_confirmation=user_id,
         channel_id_to_post_confirmation=channel_id,
         thread_ts_to_post_confirmation=thread_ts,
     )
 
-    query_event_id, _, _ = decompose_block_id(block_id)
+    query_event_id, _, _ = decompose_feedback_id(feedback_id)
     logger.info(f"Successfully handled QA feedback for event: {query_event_id}")
 
 
@@ -273,15 +291,35 @@ def acknowledge_message(req: SocketModeRequest, client: SocketModeClient) -> Non
     client.send_socket_mode_response(response)
 
 
+def action_routing(req: SocketModeRequest, client: SocketModeClient) -> None:
+    if actions := req.payload.get("actions"):
+        action = cast(dict[str, Any], actions[0])
+
+        if action["action_id"] in [DISLIKE_BLOCK_ACTION_ID, LIKE_BLOCK_ACTION_ID]:
+            # AI Answer feedback
+            return process_feedback(req, client)
+        elif action["action_id"] == FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID:
+            # Activation of the "source feedback" button
+            return handle_doc_feedback_button(req, client)
+
+
+def view_routing(req: SocketModeRequest, client: SocketModeClient) -> None:
+    if view := req.payload.get("view"):
+        if view["callback_id"] == VIEW_DOC_FEEDBACK_ID:
+            return process_feedback(req, client)
+
+
 def process_slack_event(client: SocketModeClient, req: SocketModeRequest) -> None:
     # Always respond right away, if Slack doesn't receive these frequently enough
     # it will assume the Bot is DEAD!!! :(
     acknowledge_message(req, client)
 
     try:
-        if req.type == "interactive" and req.payload.get("type") == "block_actions":
+        if req.type == "interactive":
-            return process_feedback(req, client)
+            if req.payload.get("type") == "block_actions":
-
+                return action_routing(req, client)
+            elif req.payload.get("type") == "view_submission":
+                return view_routing(req, client)
         elif req.type == "events_api" or req.type == "slash_commands":
             return process_message(req, client)
     except Exception:

backend/danswer/danswerbot/slack/utils.py

@@ -112,7 +112,7 @@ def respond_in_thread(
                 raise RuntimeError(f"Failed to post message: {response}")
 
 
-def build_feedback_block_id(
+def build_feedback_id(
     message_id: int,
     document_id: str | None = None,
     document_rank: int | None = None,
@@ -125,19 +125,21 @@ def build_feedback_block_id(
             raise ValueError(
                 "Separator pattern should not already exist in document id"
             )
-        block_id = ID_SEPARATOR.join([str(message_id), document_id, str(document_rank)])
+        feedback_id = ID_SEPARATOR.join(
+            [str(message_id), document_id, str(document_rank)]
+        )
     else:
-        block_id = str(message_id)
+        feedback_id = str(message_id)
 
-    return unique_prefix + ID_SEPARATOR + block_id
+    return unique_prefix + ID_SEPARATOR + feedback_id
 
 
-def decompose_block_id(block_id: str) -> tuple[int, str | None, int | None]:
+def decompose_feedback_id(feedback_id: str) -> tuple[int, str | None, int | None]:
     """Decompose into query_id, document_id, document_rank, see above function"""
     try:
-        components = block_id.split(ID_SEPARATOR)
+        components = feedback_id.split(ID_SEPARATOR)
         if len(components) != 2 and len(components) != 4:
-            raise ValueError("Block ID does not contain right number of elements")
+            raise ValueError("Feedback ID does not contain right number of elements")
 
         if len(components) == 2:
             return int(components[-1]), None, None
@@ -146,7 +148,36 @@ def decompose_block_id(block_id: str) -> tuple[int, str | None, int | None]:
 
     except Exception as e:
         logger.error(e)
-        raise ValueError("Received invalid Feedback Block Identifier")
+        raise ValueError("Received invalid Feedback Identifier")
+
+
+def get_view_values(state_values: dict[str, Any]) -> dict[str, str]:
+    """Extract view values
+
+    Args:
+        state_values (dict): The Slack view-submission values
+
+    Returns:
+        dict: keys/values of the view state content
+    """
+    view_values = {}
+    for _, view_data in state_values.items():
+        for k, v in view_data.items():
+            if (
+                "selected_option" in v
+                and isinstance(v["selected_option"], dict)
+                and "value" in v["selected_option"]
+            ):
+                view_values[k] = v["selected_option"]["value"]
+            elif "selected_options" in v and isinstance(v["selected_options"], list):
+                view_values[k] = [
+                    x["value"] for x in v["selected_options"] if "value" in x
+                ]
+            elif "selected_date" in v:
+                view_values[k] = v["selected_date"]
+            elif "value" in v:
+                view_values[k] = v["value"]
+    return view_values
 
 
 def translate_vespa_highlight_to_slack(match_strs: list[str], used_chars: int) -> str:

backend/danswer/db/feedback.py

@@ -114,10 +114,15 @@ def create_doc_retrieval_feedback(
         else:
             raise ValueError("Unhandled document feedback type")
 
-    if feedback in [SearchFeedbackType.ENDORSE, SearchFeedbackType.REJECT]:
+    if feedback in [
+        SearchFeedbackType.ENDORSE,
+        SearchFeedbackType.REJECT,
+        SearchFeedbackType.HIDE,
+    ]:
         update = UpdateRequest(
             document_ids=[document_id],
             boost=db_doc.boost,
+            hidden=db_doc.hidden
         )
         # Updates are generally batched for efficiency, this case only 1 doc/value is updated
         document_index.update([update])