Add Slack feedback reminder (#1262)

---------

Co-authored-by: Matthieu Boret <matthieu.boret@fr.clara.net>

backend/danswer/configs/danswerbot_configs.py

@@ -67,3 +67,9 @@ DANSWER_BOT_USE_QUOTES = os.environ.get("DANSWER_BOT_USE_QUOTES", "").lower() ==
 DANSWER_BOT_MAX_QPM = int(os.environ.get("DANSWER_BOT_MAX_QPM") or 0) or None
 # Maximum time to wait when a question is queued
 DANSWER_BOT_MAX_WAIT_TIME = int(os.environ.get("DANSWER_BOT_MAX_WAIT_TIME") or 180)
+
+# Time (in minutes) after which a Slack message is sent to the user to remind him to give feedback.
+# Set to 0 to disable it (default)
+DANSWER_BOT_FEEDBACK_REMINDER = int(
+    os.environ.get("DANSWER_BOT_FEEDBACK_REMINDER") or 0
+)

backend/danswer/danswerbot/slack/blocks.py

@@ -38,6 +38,16 @@ from danswer.utils.text_processing import replace_whitespaces_w_space
 _MAX_BLURB_LEN = 45
 
 
+def get_feedback_reminder_blocks(thread_link: str) -> Block:
+    return SectionBlock(
+        text=(
+            f"Eh! You forget to give feedback on <{thread_link}|this answer>. "
+            "It's essential to help us to improve the quality of the answers. "
+            "Please rate it by clicking the `Helpful` or `Not helpful` button. Thanks!"
+        )
+    )
+
+
 def _process_citations_for_slack(text: str) -> str:
     """
     Converts instances of [[x]](LINK) in the input text to Slack's link format <LINK|[x]>.
@@ -88,7 +98,9 @@ def clean_markdown_link_text(text: str) -> str:
     return text.replace("\n", " ").strip()
 
 
-def build_qa_feedback_block(message_id: int) -> Block:
+def build_qa_feedback_block(
+    message_id: int, feedback_reminder_id: str | None = None
+) -> Block:
     return ActionsBlock(
         block_id=build_feedback_id(message_id),
         elements=[
@@ -96,10 +108,12 @@ def build_qa_feedback_block(message_id: int) -> Block:
                 action_id=LIKE_BLOCK_ACTION_ID,
                 text="👍 Helpful",
                 style="primary",
+                value=feedback_reminder_id,
             ),
             ButtonElement(
                 action_id=DISLIKE_BLOCK_ACTION_ID,
                 text="👎 Not helpful",
+                value=feedback_reminder_id,
             ),
         ],
     )
@@ -345,6 +359,7 @@ def build_qa_response_blocks(
     skip_quotes: bool = False,
     process_message_for_citations: bool = False,
     skip_ai_feedback: bool = False,
+    feedback_reminder_id: str | None = None,
 ) -> list[Block]:
     if DISABLE_GENERATIVE_AI:
         return []
@@ -397,7 +412,11 @@ def build_qa_response_blocks(
     response_blocks.extend(answer_blocks)
 
     if message_id is not None and not skip_ai_feedback:
-        response_blocks.append(build_qa_feedback_block(message_id=message_id))
+        response_blocks.append(
+            build_qa_feedback_block(
+                message_id=message_id, feedback_reminder_id=feedback_reminder_id
+            )
+        )
 
     if not skip_quotes:
         response_blocks.extend(quotes_blocks)

backend/danswer/danswerbot/slack/handlers/handle_buttons.py

@@ -18,6 +18,9 @@ from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import FeedbackVisibility
 from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import VIEW_DOC_FEEDBACK_ID
+from danswer.danswerbot.slack.handlers.handle_message import (
+    remove_scheduled_feedback_reminder,
+)
 from danswer.danswerbot.slack.utils import build_feedback_id
 from danswer.danswerbot.slack.utils import decompose_action_id
 from danswer.danswerbot.slack.utils import fetch_groupids_from_names
@@ -72,6 +75,7 @@ def handle_doc_feedback_button(
 def handle_slack_feedback(
     feedback_id: str,
     feedback_type: str,
+    feedback_msg_reminder: str,
     client: WebClient,
     user_id_to_post_confirmation: str,
     channel_id_to_post_confirmation: str,
@@ -90,6 +94,11 @@ def handle_slack_feedback(
                 user_id=None,  # no "user" for Slack bot for now
                 db_session=db_session,
             )
+            remove_scheduled_feedback_reminder(
+                client=client,
+                channel=user_id_to_post_confirmation,
+                msg_id=feedback_msg_reminder,
+            )
         elif feedback_type in [
             SearchFeedbackType.ENDORSE.value,
             SearchFeedbackType.REJECT.value,

backend/danswer/danswerbot/slack/handlers/handle_message.py

@@ -1,3 +1,4 @@
+import datetime
 import functools
 import logging
 from collections.abc import Callable
@@ -16,6 +17,7 @@ from danswer.configs.danswerbot_configs import DANSWER_BOT_ANSWER_GENERATION_TIM
 from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_COT
 from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER
 from danswer.configs.danswerbot_configs import DANSWER_BOT_DISPLAY_ERROR_MSGS
+from danswer.configs.danswerbot_configs import DANSWER_BOT_FEEDBACK_REMINDER
 from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_RETRIES
 from danswer.configs.danswerbot_configs import DANSWER_BOT_TARGET_CHUNK_PERCENTAGE
 from danswer.configs.danswerbot_configs import DANSWER_BOT_USE_QUOTES
@@ -27,6 +29,7 @@ from danswer.danswerbot.slack.blocks import build_documents_blocks
 from danswer.danswerbot.slack.blocks import build_follow_up_block
 from danswer.danswerbot.slack.blocks import build_qa_response_blocks
 from danswer.danswerbot.slack.blocks import build_sources_blocks
+from danswer.danswerbot.slack.blocks import get_feedback_reminder_blocks
 from danswer.danswerbot.slack.blocks import get_restate_blocks
 from danswer.danswerbot.slack.constants import SLACK_CHANNEL_ID
 from danswer.danswerbot.slack.models import SlackMessageInfo
@@ -102,10 +105,74 @@ def send_msg_ack_to_user(details: SlackMessageInfo, client: WebClient) -> None:
     )
 
 
+def schedule_feedback_reminder(
+    details: SlackMessageInfo, client: WebClient
+) -> str | None:
+    logger = cast(
+        logging.Logger,
+        ChannelIdAdapter(
+            logger_base, extra={SLACK_CHANNEL_ID: details.channel_to_respond}
+        ),
+    )
+    if not DANSWER_BOT_FEEDBACK_REMINDER:
+        logger.info("Scheduled feedback reminder disabled...")
+        return None
+
+    try:
+        permalink = client.chat_getPermalink(
+            channel=details.channel_to_respond,
+            message_ts=details.msg_to_respond,  # type:ignore
+        )
+    except SlackApiError as e:
+        logger.error(f"Unable to generate the feedback reminder permalink: {e}")
+        return None
+
+    now = datetime.datetime.now()
+    future = now + datetime.timedelta(minutes=DANSWER_BOT_FEEDBACK_REMINDER)
+
+    try:
+        response = client.chat_scheduleMessage(
+            channel=details.sender,  # type:ignore
+            post_at=int(future.timestamp()),
+            blocks=[
+                get_feedback_reminder_blocks(
+                    thread_link=permalink.data["permalink"]  # type:ignore
+                )
+            ],
+            text="",
+        )
+        logger.info("Scheduled feedback reminder configured")
+        return response.data["scheduled_message_id"]  # type:ignore
+    except SlackApiError as e:
+        logger.error(f"Unable to generate the feedback reminder message: {e}")
+        return None
+
+
+def remove_scheduled_feedback_reminder(
+    client: WebClient, channel: str | None, msg_id: str
+) -> None:
+    logger = cast(
+        logging.Logger,
+        ChannelIdAdapter(logger_base, extra={SLACK_CHANNEL_ID: channel}),
+    )
+
+    try:
+        client.chat_deleteScheduledMessage(
+            channel=channel, scheduled_message_id=msg_id  # type:ignore
+        )
+        logger.info("Scheduled feedback reminder deleted")
+    except SlackApiError as e:
+        if e.response["error"] == "invalid_scheduled_message_id":
+            logger.info(
+                "Unable to delete the scheduled message. It must have already been posted"
+            )
+
+
 def handle_message(
     message_info: SlackMessageInfo,
     channel_config: SlackBotConfig | None,
     client: WebClient,
+    feedback_reminder_id: str | None,
     num_retries: int = DANSWER_BOT_NUM_RETRIES,
     answer_generation_timeout: int = DANSWER_BOT_ANSWER_GENERATION_TIMEOUT,
     should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS,
@@ -425,6 +492,7 @@ def handle_message(
         # if citations are enabled, also don't use quotes
         skip_quotes=persona is not None or use_citations,
         process_message_for_citations=use_citations,
+        feedback_reminder_id=feedback_reminder_id,
     )
 
     # Get the chunks fed to the LLM only, then fill with other docs

backend/danswer/danswerbot/slack/listener.py

@@ -28,6 +28,10 @@ from danswer.danswerbot.slack.handlers.handle_buttons import (
 )
 from danswer.danswerbot.slack.handlers.handle_buttons import handle_slack_feedback
 from danswer.danswerbot.slack.handlers.handle_message import handle_message
+from danswer.danswerbot.slack.handlers.handle_message import (
+    remove_scheduled_feedback_reminder,
+)
+from danswer.danswerbot.slack.handlers.handle_message import schedule_feedback_reminder
 from danswer.danswerbot.slack.models import SlackMessageInfo
 from danswer.danswerbot.slack.tokens import fetch_tokens
 from danswer.danswerbot.slack.utils import ChannelIdAdapter
@@ -160,6 +164,7 @@ def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None:
     if actions := req.payload.get("actions"):
         action = cast(dict[str, Any], actions[0])
         feedback_type = cast(str, action.get("action_id"))
+        feedback_msg_reminder = cast(str, action.get("value"))
         feedback_id = cast(str, action.get("block_id"))
         channel_id = cast(str, req.payload["container"]["channel_id"])
         thread_ts = cast(str, req.payload["container"]["thread_ts"])
@@ -172,6 +177,7 @@ def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None:
     handle_slack_feedback(
         feedback_id=feedback_id,
         feedback_type=feedback_type,
+        feedback_msg_reminder=feedback_msg_reminder,
         client=client.web_client,
         user_id_to_post_confirmation=user_id,
         channel_id_to_post_confirmation=channel_id,
@@ -286,15 +292,27 @@ def process_message(
         ):
             return
 
+        feedback_reminder_id = schedule_feedback_reminder(
+            details=details, client=client.web_client
+        )
+
         failed = handle_message(
             message_info=details,
             channel_config=slack_bot_config,
             client=client.web_client,
+            feedback_reminder_id=feedback_reminder_id,
         )
 
-        # Skipping answering due to pre-filtering is not considered a failure
-        if failed and notify_no_answer:
-            apologize_for_fail(details, client)
+        if failed:
+            if feedback_reminder_id:
+                remove_scheduled_feedback_reminder(
+                    client=client.web_client,
+                    channel=details.sender,
+                    msg_id=feedback_reminder_id,
+                )
+            # Skipping answering due to pre-filtering is not considered a failure
+            if notify_no_answer:
+                apologize_for_fail(details, client)
 
 
 def acknowledge_message(req: SocketModeRequest, client: SocketModeClient) -> None: