From a467999984c9bc4a4b68ba86d1938e143eb5d667 Mon Sep 17 00:00:00 2001 From: mattboret Date: Sat, 11 May 2024 19:27:09 +0200 Subject: [PATCH] Add Slack feedback reminder (#1262) --------- Co-authored-by: Matthieu Boret --- backend/danswer/configs/danswerbot_configs.py | 6 ++ backend/danswer/danswerbot/slack/blocks.py | 23 ++++++- .../slack/handlers/handle_buttons.py | 9 +++ .../slack/handlers/handle_message.py | 68 +++++++++++++++++++ backend/danswer/danswerbot/slack/listener.py | 24 ++++++- 5 files changed, 125 insertions(+), 5 deletions(-) diff --git a/backend/danswer/configs/danswerbot_configs.py b/backend/danswer/configs/danswerbot_configs.py index 192a0594d..1dc01bca1 100644 --- a/backend/danswer/configs/danswerbot_configs.py +++ b/backend/danswer/configs/danswerbot_configs.py @@ -67,3 +67,9 @@ DANSWER_BOT_USE_QUOTES = os.environ.get("DANSWER_BOT_USE_QUOTES", "").lower() == DANSWER_BOT_MAX_QPM = int(os.environ.get("DANSWER_BOT_MAX_QPM") or 0) or None # Maximum time to wait when a question is queued DANSWER_BOT_MAX_WAIT_TIME = int(os.environ.get("DANSWER_BOT_MAX_WAIT_TIME") or 180) + +# Time (in minutes) after which a Slack message is sent to the user to remind him to give feedback. +# Set to 0 to disable it (default) +DANSWER_BOT_FEEDBACK_REMINDER = int( + os.environ.get("DANSWER_BOT_FEEDBACK_REMINDER") or 0 +) diff --git a/backend/danswer/danswerbot/slack/blocks.py b/backend/danswer/danswerbot/slack/blocks.py index 2cd81a369..dd50cd490 100644 --- a/backend/danswer/danswerbot/slack/blocks.py +++ b/backend/danswer/danswerbot/slack/blocks.py @@ -38,6 +38,16 @@ from danswer.utils.text_processing import replace_whitespaces_w_space _MAX_BLURB_LEN = 45 +def get_feedback_reminder_blocks(thread_link: str) -> Block: + return SectionBlock( + text=( + f"Eh! You forget to give feedback on <{thread_link}|this answer>. " + "It's essential to help us to improve the quality of the answers. " + "Please rate it by clicking the `Helpful` or `Not helpful` button. Thanks!" + ) + ) + + def _process_citations_for_slack(text: str) -> str: """ Converts instances of [[x]](LINK) in the input text to Slack's link format . @@ -88,7 +98,9 @@ def clean_markdown_link_text(text: str) -> str: return text.replace("\n", " ").strip() -def build_qa_feedback_block(message_id: int) -> Block: +def build_qa_feedback_block( + message_id: int, feedback_reminder_id: str | None = None +) -> Block: return ActionsBlock( block_id=build_feedback_id(message_id), elements=[ @@ -96,10 +108,12 @@ def build_qa_feedback_block(message_id: int) -> Block: action_id=LIKE_BLOCK_ACTION_ID, text="👍 Helpful", style="primary", + value=feedback_reminder_id, ), ButtonElement( action_id=DISLIKE_BLOCK_ACTION_ID, text="👎 Not helpful", + value=feedback_reminder_id, ), ], ) @@ -345,6 +359,7 @@ def build_qa_response_blocks( skip_quotes: bool = False, process_message_for_citations: bool = False, skip_ai_feedback: bool = False, + feedback_reminder_id: str | None = None, ) -> list[Block]: if DISABLE_GENERATIVE_AI: return [] @@ -397,7 +412,11 @@ def build_qa_response_blocks( response_blocks.extend(answer_blocks) if message_id is not None and not skip_ai_feedback: - response_blocks.append(build_qa_feedback_block(message_id=message_id)) + response_blocks.append( + build_qa_feedback_block( + message_id=message_id, feedback_reminder_id=feedback_reminder_id + ) + ) if not skip_quotes: response_blocks.extend(quotes_blocks) diff --git a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py index bec1959e3..3a0209b07 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py @@ -18,6 +18,9 @@ from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID from danswer.danswerbot.slack.constants import FeedbackVisibility from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID from danswer.danswerbot.slack.constants import VIEW_DOC_FEEDBACK_ID +from danswer.danswerbot.slack.handlers.handle_message import ( + remove_scheduled_feedback_reminder, +) from danswer.danswerbot.slack.utils import build_feedback_id from danswer.danswerbot.slack.utils import decompose_action_id from danswer.danswerbot.slack.utils import fetch_groupids_from_names @@ -72,6 +75,7 @@ def handle_doc_feedback_button( def handle_slack_feedback( feedback_id: str, feedback_type: str, + feedback_msg_reminder: str, client: WebClient, user_id_to_post_confirmation: str, channel_id_to_post_confirmation: str, @@ -90,6 +94,11 @@ def handle_slack_feedback( user_id=None, # no "user" for Slack bot for now db_session=db_session, ) + remove_scheduled_feedback_reminder( + client=client, + channel=user_id_to_post_confirmation, + msg_id=feedback_msg_reminder, + ) elif feedback_type in [ SearchFeedbackType.ENDORSE.value, SearchFeedbackType.REJECT.value, diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py index fb627fb69..14efe52a5 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_message.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_message.py @@ -1,3 +1,4 @@ +import datetime import functools import logging from collections.abc import Callable @@ -16,6 +17,7 @@ from danswer.configs.danswerbot_configs import DANSWER_BOT_ANSWER_GENERATION_TIM from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_COT from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER from danswer.configs.danswerbot_configs import DANSWER_BOT_DISPLAY_ERROR_MSGS +from danswer.configs.danswerbot_configs import DANSWER_BOT_FEEDBACK_REMINDER from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_RETRIES from danswer.configs.danswerbot_configs import DANSWER_BOT_TARGET_CHUNK_PERCENTAGE from danswer.configs.danswerbot_configs import DANSWER_BOT_USE_QUOTES @@ -27,6 +29,7 @@ from danswer.danswerbot.slack.blocks import build_documents_blocks from danswer.danswerbot.slack.blocks import build_follow_up_block from danswer.danswerbot.slack.blocks import build_qa_response_blocks from danswer.danswerbot.slack.blocks import build_sources_blocks +from danswer.danswerbot.slack.blocks import get_feedback_reminder_blocks from danswer.danswerbot.slack.blocks import get_restate_blocks from danswer.danswerbot.slack.constants import SLACK_CHANNEL_ID from danswer.danswerbot.slack.models import SlackMessageInfo @@ -102,10 +105,74 @@ def send_msg_ack_to_user(details: SlackMessageInfo, client: WebClient) -> None: ) +def schedule_feedback_reminder( + details: SlackMessageInfo, client: WebClient +) -> str | None: + logger = cast( + logging.Logger, + ChannelIdAdapter( + logger_base, extra={SLACK_CHANNEL_ID: details.channel_to_respond} + ), + ) + if not DANSWER_BOT_FEEDBACK_REMINDER: + logger.info("Scheduled feedback reminder disabled...") + return None + + try: + permalink = client.chat_getPermalink( + channel=details.channel_to_respond, + message_ts=details.msg_to_respond, # type:ignore + ) + except SlackApiError as e: + logger.error(f"Unable to generate the feedback reminder permalink: {e}") + return None + + now = datetime.datetime.now() + future = now + datetime.timedelta(minutes=DANSWER_BOT_FEEDBACK_REMINDER) + + try: + response = client.chat_scheduleMessage( + channel=details.sender, # type:ignore + post_at=int(future.timestamp()), + blocks=[ + get_feedback_reminder_blocks( + thread_link=permalink.data["permalink"] # type:ignore + ) + ], + text="", + ) + logger.info("Scheduled feedback reminder configured") + return response.data["scheduled_message_id"] # type:ignore + except SlackApiError as e: + logger.error(f"Unable to generate the feedback reminder message: {e}") + return None + + +def remove_scheduled_feedback_reminder( + client: WebClient, channel: str | None, msg_id: str +) -> None: + logger = cast( + logging.Logger, + ChannelIdAdapter(logger_base, extra={SLACK_CHANNEL_ID: channel}), + ) + + try: + client.chat_deleteScheduledMessage( + channel=channel, scheduled_message_id=msg_id # type:ignore + ) + logger.info("Scheduled feedback reminder deleted") + except SlackApiError as e: + if e.response["error"] == "invalid_scheduled_message_id": + logger.info( + "Unable to delete the scheduled message. It must have already been posted" + ) + + def handle_message( message_info: SlackMessageInfo, channel_config: SlackBotConfig | None, client: WebClient, + feedback_reminder_id: str | None, num_retries: int = DANSWER_BOT_NUM_RETRIES, answer_generation_timeout: int = DANSWER_BOT_ANSWER_GENERATION_TIMEOUT, should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS, @@ -425,6 +492,7 @@ def handle_message( # if citations are enabled, also don't use quotes skip_quotes=persona is not None or use_citations, process_message_for_citations=use_citations, + feedback_reminder_id=feedback_reminder_id, ) # Get the chunks fed to the LLM only, then fill with other docs diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py index 829c5bbf6..819b37dd6 100644 --- a/backend/danswer/danswerbot/slack/listener.py +++ b/backend/danswer/danswerbot/slack/listener.py @@ -28,6 +28,10 @@ from danswer.danswerbot.slack.handlers.handle_buttons import ( ) from danswer.danswerbot.slack.handlers.handle_buttons import handle_slack_feedback from danswer.danswerbot.slack.handlers.handle_message import handle_message +from danswer.danswerbot.slack.handlers.handle_message import ( + remove_scheduled_feedback_reminder, +) +from danswer.danswerbot.slack.handlers.handle_message import schedule_feedback_reminder from danswer.danswerbot.slack.models import SlackMessageInfo from danswer.danswerbot.slack.tokens import fetch_tokens from danswer.danswerbot.slack.utils import ChannelIdAdapter @@ -160,6 +164,7 @@ def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None: if actions := req.payload.get("actions"): action = cast(dict[str, Any], actions[0]) feedback_type = cast(str, action.get("action_id")) + feedback_msg_reminder = cast(str, action.get("value")) feedback_id = cast(str, action.get("block_id")) channel_id = cast(str, req.payload["container"]["channel_id"]) thread_ts = cast(str, req.payload["container"]["thread_ts"]) @@ -172,6 +177,7 @@ def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None: handle_slack_feedback( feedback_id=feedback_id, feedback_type=feedback_type, + feedback_msg_reminder=feedback_msg_reminder, client=client.web_client, user_id_to_post_confirmation=user_id, channel_id_to_post_confirmation=channel_id, @@ -286,15 +292,27 @@ def process_message( ): return + feedback_reminder_id = schedule_feedback_reminder( + details=details, client=client.web_client + ) + failed = handle_message( message_info=details, channel_config=slack_bot_config, client=client.web_client, + feedback_reminder_id=feedback_reminder_id, ) - # Skipping answering due to pre-filtering is not considered a failure - if failed and notify_no_answer: - apologize_for_fail(details, client) + if failed: + if feedback_reminder_id: + remove_scheduled_feedback_reminder( + client=client.web_client, + channel=details.sender, + msg_id=feedback_reminder_id, + ) + # Skipping answering due to pre-filtering is not considered a failure + if notify_no_answer: + apologize_for_fail(details, client) def acknowledge_message(req: SocketModeRequest, client: SocketModeClient) -> None: