Slack bot improve source feedback (#827)

--------- Co-authored-by: Yuhong Sun <yuhongsun96@gmail.com> Co-authored-by: Matthieu Boret <matthieu.boret@fr.clara.net>
2025-07-24 20:06:32 +02:00 · 2023-12-22 05:33:20 +01:00
parent dc0b3672ac
commit 25a73b9921
7 changed files with 206 additions and 73 deletions
--- a/backend/danswer/configs/danswerbot_configs.py
+++ b/backend/danswer/configs/danswerbot_configs.py
@@ -48,7 +48,3 @@ ENABLE_DANSWERBOT_REFLEXION = (
 )
 # Currently not support chain of thought, probably will add back later
 DANSWER_BOT_DISABLE_COT = True
-# Add the per document feedback blocks that affect the document rankings via boosting
-ENABLE_SLACK_DOC_FEEDBACK = (
-    os.environ.get("ENABLE_SLACK_DOC_FEEDBACK", "").lower() == "true"
-)
--- a/backend/danswer/danswerbot/slack/blocks.py
+++ b/backend/danswer/danswerbot/slack/blocks.py
@@ -5,19 +5,20 @@ import timeago  # type: ignore
 from slack_sdk.models.blocks import ActionsBlock
 from slack_sdk.models.blocks import Block
 from slack_sdk.models.blocks import ButtonElement
-from slack_sdk.models.blocks import ConfirmObject
 from slack_sdk.models.blocks import DividerBlock
 from slack_sdk.models.blocks import HeaderBlock
+from slack_sdk.models.blocks import Option
+from slack_sdk.models.blocks import RadioButtonsElement
 from slack_sdk.models.blocks import SectionBlock

 from danswer.chat.models import DanswerQuote
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import SearchFeedbackType
 from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_DOCS_TO_DISPLAY
-from danswer.configs.danswerbot_configs import ENABLE_SLACK_DOC_FEEDBACK
 from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
+from danswer.danswerbot.slack.constants import FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
-from danswer.danswerbot.slack.utils import build_feedback_block_id
+from danswer.danswerbot.slack.utils import build_feedback_id
 from danswer.danswerbot.slack.utils import remove_slack_text_interactions
 from danswer.danswerbot.slack.utils import translate_vespa_highlight_to_slack
 from danswer.search.models import SavedSearchDoc
@@ -28,7 +29,7 @@ _MAX_BLURB_LEN = 75

 def build_qa_feedback_block(message_id: int) -> Block:
    return ActionsBlock(
-        block_id=build_feedback_block_id(message_id),
+        block_id=build_feedback_id(message_id),
        elements=[
            ButtonElement(
                action_id=LIKE_BLOCK_ACTION_ID,
@@ -44,33 +45,43 @@ def build_qa_feedback_block(message_id: int) -> Block:
    )


+def get_document_feedback_blocks() -> Block:
+    return SectionBlock(
+        text=(
+            "If this document is a good source of information and should be shown more often, "
+            "please select 'Boost'. Conversely, if it seems to be a bad source of information, "
+            "select 'Down-boost'. You can also select 'Hide' if the source is deprecated and "
+            "should not be taken into account anymore."
+        ),
+        accessory=RadioButtonsElement(
+            options=[
+                Option(
+                    text=":thumbsup: Boost",
+                    value=SearchFeedbackType.ENDORSE.value,
+                ),
+                Option(
+                    text=":thumbsdown: Down-Boost",
+                    value=SearchFeedbackType.REJECT.value,
+                ),
+                Option(
+                    text=":palms_up_together: Hide",
+                    value=SearchFeedbackType.HIDE.value,
+                ),
+            ]
+        ),
+    )
+
+
 def build_doc_feedback_block(
    message_id: int,
    document_id: str,
    document_rank: int,
-) -> Block:
-    return ActionsBlock(
-        block_id=build_feedback_block_id(message_id, document_id, document_rank),
-        elements=[
-            ButtonElement(
-                action_id=SearchFeedbackType.ENDORSE.value,
-                text="⬆",
-                style="primary",
-                confirm=ConfirmObject(
-                    title="Endorse this Document",
-                    text="This is a good source of information and should be shown more often!",
-                ),
-            ),
-            ButtonElement(
-                action_id=SearchFeedbackType.REJECT.value,
-                text="⬇",
-                style="danger",
-                confirm=ConfirmObject(
-                    title="Reject this Document",
-                    text="This is a bad source of information and should be shown less often.",
-                ),
-            ),
-        ],
+) -> ButtonElement:
+    feedback_id = build_feedback_id(message_id, document_id, document_rank)
+    return ButtonElement(
+        action_id=FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID,
+        value=feedback_id,
+        text="Source feedback",
    )


@@ -92,7 +103,6 @@ def build_documents_blocks(
    documents: list[SavedSearchDoc],
    message_id: int | None,
    num_docs_to_display: int = DANSWER_BOT_NUM_DOCS_TO_DISPLAY,
-    include_feedback: bool = ENABLE_SLACK_DOC_FEEDBACK,
 ) -> list[Block]:
    seen_docs_identifiers = set()
    section_blocks: list[Block] = [HeaderBlock(text="Reference Documents")]
@@ -125,17 +135,18 @@ def build_documents_blocks(

        block_text = header_line + updated_at_line + body_text

-        section_blocks.append(SectionBlock(text=block_text))
-
-        if include_feedback and message_id is not None:
-            section_blocks.append(
-                build_doc_feedback_block(
-                    message_id=message_id,
-                    document_id=d.document_id,
-                    document_rank=rank,
-                ),
+        feedback: ButtonElement | dict = {}
+        if message_id is not None:
+            feedback = build_doc_feedback_block(
+                message_id=message_id,
+                document_id=d.document_id,
+                document_rank=rank,
            )

+        section_blocks.append(
+            SectionBlock(text=block_text, accessory=feedback),
+        )
+
        section_blocks.append(DividerBlock())

        if included_docs >= num_docs_to_display:
--- a/backend/danswer/danswerbot/slack/constants.py
+++ b/backend/danswer/danswerbot/slack/constants.py
@@ -1,3 +1,5 @@
 LIKE_BLOCK_ACTION_ID = "feedback-like"
 DISLIKE_BLOCK_ACTION_ID = "feedback-dislike"
+FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID = "feedback-doc-button"
 SLACK_CHANNEL_ID = "channel_id"
+VIEW_DOC_FEEDBACK_ID = "view-doc-feedback"
--- a/backend/danswer/danswerbot/slack/handlers/handle_feedback.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_feedback.py
@@ -1,18 +1,60 @@
 from slack_sdk import WebClient
+from slack_sdk.models.views import View
+from slack_sdk.socket_mode import SocketModeClient
+from slack_sdk.socket_mode.request import SocketModeRequest
 from sqlalchemy.orm import Session

 from danswer.configs.constants import SearchFeedbackType
+from danswer.danswerbot.slack.blocks import get_document_feedback_blocks
 from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
-from danswer.danswerbot.slack.utils import decompose_block_id
+from danswer.danswerbot.slack.constants import VIEW_DOC_FEEDBACK_ID
+from danswer.danswerbot.slack.utils import build_feedback_id
+from danswer.danswerbot.slack.utils import decompose_feedback_id
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.feedback import create_chat_message_feedback
 from danswer.db.feedback import create_doc_retrieval_feedback
 from danswer.document_index.factory import get_default_document_index
+from danswer.utils.logger import setup_logger
+
+logger_base = setup_logger()
+
+
+def handle_doc_feedback_button(
+    req: SocketModeRequest,
+    client: SocketModeClient,
+) -> None:
+    if not (actions := req.payload.get("actions")):
+        logger_base.error("Missing actions. Unable to build the source feedback view")
+        return
+
+    # Extracts the feedback_id coming from the 'source feedback' button
+    # and generates a new one for the View, to keep track of the doc info
+    query_event_id, doc_id, doc_rank = decompose_feedback_id(actions[0].get("value"))
+    external_id = build_feedback_id(query_event_id, doc_id, doc_rank)
+
+    channel_id = req.payload["container"]["channel_id"]
+    thread_ts = req.payload["container"]["thread_ts"]
+
+    data = View(
+        type="modal",
+        callback_id=VIEW_DOC_FEEDBACK_ID,
+        external_id=external_id,
+        # We use the private metadata to keep track of the channel id and thread ts
+        private_metadata=f"{channel_id}_{thread_ts}",
+        title="Source Feedback",
+        blocks=[get_document_feedback_blocks()],
+        submit="send",
+        close="cancel",
+    )
+
+    client.web_client.views_open(
+        trigger_id=req.payload["trigger_id"], view=data.to_dict()
+    )


 def handle_slack_feedback(
-    block_id: str,
+    feedback_id: str,
    feedback_type: str,
    client: WebClient,
    user_id_to_post_confirmation: str,
@@ -21,7 +63,7 @@ def handle_slack_feedback(
 ) -> None:
    engine = get_sqlalchemy_engine()

-    message_id, doc_id, doc_rank = decompose_block_id(block_id)
+    message_id, doc_id, doc_rank = decompose_feedback_id(feedback_id)

    with Session(engine) as db_session:
        if feedback_type in [LIKE_BLOCK_ACTION_ID, DISLIKE_BLOCK_ACTION_ID]:
@@ -32,13 +74,21 @@ def handle_slack_feedback(
                user_id=None,  # no "user" for Slack bot for now
                db_session=db_session,
            )
-        if feedback_type in [
+        elif feedback_type in [
            SearchFeedbackType.ENDORSE.value,
            SearchFeedbackType.REJECT.value,
+            SearchFeedbackType.HIDE.value,
        ]:
            if doc_id is None or doc_rank is None:
                raise ValueError("Missing information for Document Feedback")

+            if feedback_type == SearchFeedbackType.ENDORSE.value:
+                feedback = SearchFeedbackType.ENDORSE
+            elif feedback_type == SearchFeedbackType.REJECT.value:
+                feedback = SearchFeedbackType.REJECT
+            else:
+                feedback = SearchFeedbackType.HIDE
+
            create_doc_retrieval_feedback(
                message_id=message_id,
                document_id=doc_id,
@@ -46,10 +96,10 @@ def handle_slack_feedback(
                document_index=get_default_document_index(),
                db_session=db_session,
                clicked=False,  # Not tracking this for Slack
-                feedback=SearchFeedbackType.ENDORSE
-                if feedback_type == SearchFeedbackType.ENDORSE.value
-                else SearchFeedbackType.REJECT,
+                feedback=feedback,
            )
+        else:
+            logger_base.error(f"Feedback type '{feedback_type}' not supported")

    # post message to slack confirming that feedback was received
    client.chat_postEphemeral(
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -14,17 +14,23 @@ from danswer.configs.danswerbot_configs import DANSWER_BOT_RESPOND_EVERY_CHANNEL
 from danswer.configs.danswerbot_configs import NOTIFY_SLACKBOT_NO_ANSWER
 from danswer.configs.model_configs import ENABLE_RERANKING_ASYNC_FLOW
 from danswer.danswerbot.slack.config import get_slack_bot_config_for_channel
+from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
+from danswer.danswerbot.slack.constants import FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID
+from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import SLACK_CHANNEL_ID
+from danswer.danswerbot.slack.constants import VIEW_DOC_FEEDBACK_ID
+from danswer.danswerbot.slack.handlers.handle_feedback import handle_doc_feedback_button
 from danswer.danswerbot.slack.handlers.handle_feedback import handle_slack_feedback
 from danswer.danswerbot.slack.handlers.handle_message import handle_message
 from danswer.danswerbot.slack.models import SlackMessageInfo
 from danswer.danswerbot.slack.tokens import fetch_tokens
 from danswer.danswerbot.slack.utils import ChannelIdAdapter
-from danswer.danswerbot.slack.utils import decompose_block_id
+from danswer.danswerbot.slack.utils import decompose_feedback_id
 from danswer.danswerbot.slack.utils import get_channel_name_from_id
 from danswer.danswerbot.slack.utils import get_danswer_bot_app_id
 from danswer.danswerbot.slack.utils import read_slack_thread
 from danswer.danswerbot.slack.utils import remove_danswer_bot_tag
+from danswer.danswerbot.slack.utils import get_view_values
 from danswer.danswerbot.slack.utils import respond_in_thread
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.dynamic_configs.interface import ConfigNotFoundError
@@ -33,7 +39,6 @@ from danswer.search.search_nlp_models import warm_up_models
 from danswer.server.manage.models import SlackBotTokens
 from danswer.utils.logger import setup_logger

-
 logger = setup_logger()


@@ -131,28 +136,41 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool


 def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None:
-    actions = req.payload.get("actions")
-    if not actions:
-        logger.error("Unable to process block actions - no actions found")
+    # Answer feedback
+    if actions := req.payload.get("actions"):
+        action = cast(dict[str, Any], actions[0])
+        feedback_type = cast(str, action.get("action_id"))
+        feedback_id = cast(str, action.get("block_id"))
+        channel_id = cast(str, req.payload["container"]["channel_id"])
+        thread_ts = cast(str, req.payload["container"]["thread_ts"])
+    # Doc feedback
+    elif view := req.payload.get("view"):
+        view_values = get_view_values(view["state"]["values"])
+        private_metadata = view.get("private_metadata").split("_")
+        if not view_values:
+            logger.error("Unable to process feedback. Missing view values")
+            return
+
+        feedback_type = [x for x in view_values.values()][0]
+        feedback_id = cast(str, view.get("external_id"))
+        channel_id = private_metadata[0]
+        thread_ts = private_metadata[1]
+    else:
+        logger.error("Unable to process feedback. Actions or View not found")
        return

-    action = cast(dict[str, Any], actions[0])
-    action_id = cast(str, action.get("action_id"))
-    block_id = cast(str, action.get("block_id"))
    user_id = cast(str, req.payload["user"]["id"])
-    channel_id = cast(str, req.payload["container"]["channel_id"])
-    thread_ts = cast(str, req.payload["container"]["thread_ts"])

    handle_slack_feedback(
-        block_id=block_id,
-        feedback_type=action_id,
+        feedback_id=feedback_id,
+        feedback_type=feedback_type,
        client=client.web_client,
        user_id_to_post_confirmation=user_id,
        channel_id_to_post_confirmation=channel_id,
        thread_ts_to_post_confirmation=thread_ts,
    )

-    query_event_id, _, _ = decompose_block_id(block_id)
+    query_event_id, _, _ = decompose_feedback_id(feedback_id)
    logger.info(f"Successfully handled QA feedback for event: {query_event_id}")


@@ -273,15 +291,35 @@ def acknowledge_message(req: SocketModeRequest, client: SocketModeClient) -> Non
    client.send_socket_mode_response(response)


+def action_routing(req: SocketModeRequest, client: SocketModeClient) -> None:
+    if actions := req.payload.get("actions"):
+        action = cast(dict[str, Any], actions[0])
+
+        if action["action_id"] in [DISLIKE_BLOCK_ACTION_ID, LIKE_BLOCK_ACTION_ID]:
+            # AI Answer feedback
+            return process_feedback(req, client)
+        elif action["action_id"] == FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID:
+            # Activation of the "source feedback" button
+            return handle_doc_feedback_button(req, client)
+
+
+def view_routing(req: SocketModeRequest, client: SocketModeClient) -> None:
+    if view := req.payload.get("view"):
+        if view["callback_id"] == VIEW_DOC_FEEDBACK_ID:
+            return process_feedback(req, client)
+
+
 def process_slack_event(client: SocketModeClient, req: SocketModeRequest) -> None:
    # Always respond right away, if Slack doesn't receive these frequently enough
    # it will assume the Bot is DEAD!!! :(
    acknowledge_message(req, client)

    try:
-        if req.type == "interactive" and req.payload.get("type") == "block_actions":
-            return process_feedback(req, client)
-
+        if req.type == "interactive":
+            if req.payload.get("type") == "block_actions":
+                return action_routing(req, client)
+            elif req.payload.get("type") == "view_submission":
+                return view_routing(req, client)
        elif req.type == "events_api" or req.type == "slash_commands":
            return process_message(req, client)
    except Exception:
--- a/backend/danswer/danswerbot/slack/utils.py
+++ b/backend/danswer/danswerbot/slack/utils.py
@@ -112,7 +112,7 @@ def respond_in_thread(
                raise RuntimeError(f"Failed to post message: {response}")


-def build_feedback_block_id(
+def build_feedback_id(
    message_id: int,
    document_id: str | None = None,
    document_rank: int | None = None,
@@ -125,19 +125,21 @@ def build_feedback_block_id(
            raise ValueError(
                "Separator pattern should not already exist in document id"
            )
-        block_id = ID_SEPARATOR.join([str(message_id), document_id, str(document_rank)])
+        feedback_id = ID_SEPARATOR.join(
+            [str(message_id), document_id, str(document_rank)]
+        )
    else:
-        block_id = str(message_id)
+        feedback_id = str(message_id)

-    return unique_prefix + ID_SEPARATOR + block_id
+    return unique_prefix + ID_SEPARATOR + feedback_id


-def decompose_block_id(block_id: str) -> tuple[int, str | None, int | None]:
+def decompose_feedback_id(feedback_id: str) -> tuple[int, str | None, int | None]:
    """Decompose into query_id, document_id, document_rank, see above function"""
    try:
-        components = block_id.split(ID_SEPARATOR)
+        components = feedback_id.split(ID_SEPARATOR)
        if len(components) != 2 and len(components) != 4:
-            raise ValueError("Block ID does not contain right number of elements")
+            raise ValueError("Feedback ID does not contain right number of elements")

        if len(components) == 2:
            return int(components[-1]), None, None
@@ -146,7 +148,36 @@ def decompose_block_id(block_id: str) -> tuple[int, str | None, int | None]:

    except Exception as e:
        logger.error(e)
-        raise ValueError("Received invalid Feedback Block Identifier")
+        raise ValueError("Received invalid Feedback Identifier")
+
+
+def get_view_values(state_values: dict[str, Any]) -> dict[str, str]:
+    """Extract view values
+
+    Args:
+        state_values (dict): The Slack view-submission values
+
+    Returns:
+        dict: keys/values of the view state content
+    """
+    view_values = {}
+    for _, view_data in state_values.items():
+        for k, v in view_data.items():
+            if (
+                "selected_option" in v
+                and isinstance(v["selected_option"], dict)
+                and "value" in v["selected_option"]
+            ):
+                view_values[k] = v["selected_option"]["value"]
+            elif "selected_options" in v and isinstance(v["selected_options"], list):
+                view_values[k] = [
+                    x["value"] for x in v["selected_options"] if "value" in x
+                ]
+            elif "selected_date" in v:
+                view_values[k] = v["selected_date"]
+            elif "value" in v:
+                view_values[k] = v["value"]
+    return view_values


 def translate_vespa_highlight_to_slack(match_strs: list[str], used_chars: int) -> str:
--- a/backend/danswer/db/feedback.py
+++ b/backend/danswer/db/feedback.py
@@ -114,10 +114,15 @@ def create_doc_retrieval_feedback(
        else:
            raise ValueError("Unhandled document feedback type")

-    if feedback in [SearchFeedbackType.ENDORSE, SearchFeedbackType.REJECT]:
+    if feedback in [
+        SearchFeedbackType.ENDORSE,
+        SearchFeedbackType.REJECT,
+        SearchFeedbackType.HIDE,
+    ]:
        update = UpdateRequest(
            document_ids=[document_id],
            boost=db_doc.boost,
+            hidden=db_doc.hidden
        )
        # Updates are generally batched for efficiency, this case only 1 doc/value is updated
        document_index.update([update])