slackbot doesnt respond without citations/quotes (#1798)

* slackbot doesnt respond without citations/quotes

fixed logical issues

fixed dict logic

* added slackbot shim for the llm source/time feature

* mypy fixes

* slackbot doesnt respond without citations/quotes

fixed logical issues

fixed dict logic

* Update handle_regular_answer.py

* added bypass_filter check

* final fixes

backend/alembic/versions/7aea705850d5_added_slack_auto_filter.py

@@ -0,0 +1,35 @@
+"""added slack_auto_filter
+
+Revision ID: 7aea705850d5
+Revises: 4505fd7302e1
+Create Date: 2024-07-10 11:01:23.581015
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "7aea705850d5"
+down_revision = "4505fd7302e1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "slack_bot_config",
+        sa.Column("enable_auto_filters", sa.Boolean(), nullable=True),
+    )
+    op.execute(
+        "UPDATE slack_bot_config SET enable_auto_filters = FALSE WHERE enable_auto_filters IS NULL"
+    )
+    op.alter_column(
+        "slack_bot_config",
+        "enable_auto_filters",
+        existing_type=sa.Boolean(),
+        nullable=False,
+        server_default=sa.false(),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("slack_bot_config", "enable_auto_filters")

backend/danswer/configs/chat_configs.py

@@ -28,9 +28,6 @@ BASE_RECENCY_DECAY = 0.5
 FAVOR_RECENT_DECAY_MULTIPLIER = 2.0
 # Currently this next one is not configurable via env
 DISABLE_LLM_QUERY_ANSWERABILITY = QA_PROMPT_OVERRIDE == "weak"
-DISABLE_LLM_FILTER_EXTRACTION = (
-    os.environ.get("DISABLE_LLM_FILTER_EXTRACTION", "").lower() == "true"
-)
 # Whether the LLM should evaluate all of the document chunks passed in for usefulness
 # in relation to the user query
 DISABLE_LLM_CHUNK_FILTER = (

backend/danswer/configs/danswerbot_configs.py

@@ -47,10 +47,6 @@ DANSWER_BOT_DISPLAY_ERROR_MSGS = os.environ.get(
 DANSWER_BOT_RESPOND_EVERY_CHANNEL = (
     os.environ.get("DANSWER_BOT_RESPOND_EVERY_CHANNEL", "").lower() == "true"
 )
-# Auto detect query options like time cutoff or heavily favor recently updated docs
-DISABLE_DANSWER_BOT_FILTER_DETECT = (
-    os.environ.get("DISABLE_DANSWER_BOT_FILTER_DETECT", "").lower() == "true"
-)
 # Add a second LLM call post Answer to verify if the Answer is valid
 # Throws out answers that don't directly or fully answer the user query
 # This is the default for all DanswerBot channels unless the channel is configured individually

backend/danswer/danswerbot/slack/handlers/handle_message.py

@@ -168,8 +168,6 @@ def handle_message(
     if slack_bot_config and slack_bot_config.channel_config:
         channel_conf = slack_bot_config.channel_config
         if not bypass_filters and "answer_filters" in channel_conf:
-            "well_answered_postfilter" in channel_conf["answer_filters"]
-
             if (
                 "questionmark_prefilter" in channel_conf["answer_filters"]
                 and "?" not in messages[-1].message

backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py

@@ -22,7 +22,6 @@ from danswer.configs.danswerbot_configs import DANSWER_BOT_TARGET_CHUNK_PERCENTA
 from danswer.configs.danswerbot_configs import DANSWER_BOT_USE_QUOTES
 from danswer.configs.danswerbot_configs import DANSWER_FOLLOWUP_EMOJI
 from danswer.configs.danswerbot_configs import DANSWER_REACT_EMOJI
-from danswer.configs.danswerbot_configs import DISABLE_DANSWER_BOT_FILTER_DETECT
 from danswer.configs.danswerbot_configs import ENABLE_DANSWERBOT_REFLEXION
 from danswer.danswerbot.slack.blocks import build_documents_blocks
 from danswer.danswerbot.slack.blocks import build_follow_up_block
@@ -91,7 +90,6 @@ def handle_regular_answer(
     thread_context_percent: float = DANSWER_BOT_TARGET_CHUNK_PERCENTAGE,
     should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS,
     disable_docs_only_answer: bool = DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER,
-    disable_auto_detect_filters: bool = DISABLE_DANSWER_BOT_FILTER_DETECT,
     disable_cot: bool = DANSWER_BOT_DISABLE_COT,
     reflexion: bool = ENABLE_DANSWERBOT_REFLEXION,
 ) -> bool:
@@ -207,12 +205,17 @@ def handle_regular_answer(
         )
 
         # Default True because no other ways to apply filters in Slack (no nice UI)
+        # Commenting this out because this is only available to the slackbot for now
+        # later we plan to implement this at the persona level where this will get
+        # commented back in
+        # auto_detect_filters = (
+        #     persona.llm_filter_extraction if persona is not None else True
+        # )
         auto_detect_filters = (
-            persona.llm_filter_extraction if persona is not None else True
+            slack_bot_config.enable_auto_filters
+            if slack_bot_config is not None
+            else False
         )
-        if disable_auto_detect_filters:
-            auto_detect_filters = False
-
         retrieval_details = RetrievalDetails(
             run_search=OptionalSearchSetting.ALWAYS,
             real_time=False,
@@ -352,6 +355,31 @@ def handle_regular_answer(
         )
         return True
 
+    only_respond_with_citations_or_quotes = (
+        channel_conf
+        and "well_answered_postfilter" in channel_conf.get("answer_filters", [])
+    )
+    has_citations_or_quotes = bool(answer.citations or answer.quotes)
+    if (
+        only_respond_with_citations_or_quotes
+        and not has_citations_or_quotes
+        and not message_info.bypass_filters
+    ):
+        logger.error(
+            f"Unable to find citations or quotes to answer: '{answer.rephrase}' - not answering!"
+        )
+        # Optionally, respond in thread with the error message
+        # Used primarily for debugging purposes
+        if should_respond_with_error_msgs:
+            respond_in_thread(
+                client=client,
+                channel=channel,
+                receiver_ids=None,
+                text="Found no citations or quotes when trying to answer.",
+                thread_ts=message_ts_to_respond_to,
+            )
+        return True
+
     # If called with the DanswerBot slash command, the question is lost so we have to reshow it
     restate_question_block = get_restate_blocks(messages[-1].message, is_bot_msg)
 

backend/danswer/db/models.py

@@ -1195,6 +1195,9 @@ class SlackBotConfig(Base):
     response_type: Mapped[SlackBotResponseType] = mapped_column(
         Enum(SlackBotResponseType, native_enum=False), nullable=False
     )
+    enable_auto_filters: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=False
+    )
 
     persona: Mapped[Persona | None] = relationship("Persona")
     standard_answer_categories: Mapped[list[StandardAnswerCategory]] = relationship(

backend/danswer/db/slack_bot_config.py

@@ -53,7 +53,7 @@ def create_slack_bot_persona(
         description="",
         num_chunks=num_chunks,
         llm_relevance_filter=True,
-        llm_filter_extraction=True,
+        llm_filter_extraction=False,
         recency_bias=RecencyBiasSetting.AUTO,
         prompt_ids=[default_prompt.id],
         document_set_ids=document_set_ids,
@@ -74,6 +74,7 @@ def insert_slack_bot_config(
     channel_config: ChannelConfig,
     response_type: SlackBotResponseType,
     standard_answer_category_ids: list[int],
+    enable_auto_filters: bool,
     db_session: Session,
 ) -> SlackBotConfig:
     existing_standard_answer_categories = fetch_standard_answer_categories_by_ids(
@@ -90,6 +91,7 @@ def insert_slack_bot_config(
         channel_config=channel_config,
         response_type=response_type,
         standard_answer_categories=existing_standard_answer_categories,
+        enable_auto_filters=enable_auto_filters,
     )
     db_session.add(slack_bot_config)
     db_session.commit()
@@ -103,6 +105,7 @@ def update_slack_bot_config(
     channel_config: ChannelConfig,
     response_type: SlackBotResponseType,
     standard_answer_category_ids: list[int],
+    enable_auto_filters: bool,
     db_session: Session,
 ) -> SlackBotConfig:
     slack_bot_config = db_session.scalar(
@@ -134,6 +137,7 @@ def update_slack_bot_config(
     slack_bot_config.standard_answer_categories = list(
         existing_standard_answer_categories
     )
+    slack_bot_config.enable_auto_filters = enable_auto_filters
 
     # if the persona has changed, then clean up the old persona
     if persona_id != existing_persona_id and existing_persona_id:

backend/danswer/search/preprocessing/preprocessing.py

@@ -2,7 +2,6 @@ from sqlalchemy.orm import Session
 
 from danswer.configs.chat_configs import BASE_RECENCY_DECAY
 from danswer.configs.chat_configs import DISABLE_LLM_CHUNK_FILTER
-from danswer.configs.chat_configs import DISABLE_LLM_FILTER_EXTRACTION
 from danswer.configs.chat_configs import FAVOR_RECENT_DECAY_MULTIPLIER
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
 from danswer.db.models import User
@@ -36,8 +35,6 @@ def retrieval_preprocessing(
     db_session: Session,
     bypass_acl: bool = False,
     include_query_intent: bool = True,
-    enable_auto_detect_filters: bool = False,
-    disable_llm_filter_extraction: bool = DISABLE_LLM_FILTER_EXTRACTION,
     disable_llm_chunk_filter: bool = DISABLE_LLM_CHUNK_FILTER,
     base_recency_decay: float = BASE_RECENCY_DECAY,
     favor_recent_decay_multiplier: float = FAVOR_RECENT_DECAY_MULTIPLIER,
@@ -63,10 +60,7 @@ def retrieval_preprocessing(
 
     auto_detect_time_filter = True
     auto_detect_source_filter = True
-    if disable_llm_filter_extraction:
+    if not search_request.enable_auto_detect_filters:
-        auto_detect_time_filter = False
-        auto_detect_source_filter = False
-    elif enable_auto_detect_filters is False:
         logger.debug("Retrieval details disables auto detect filters")
         auto_detect_time_filter = False
         auto_detect_source_filter = False

backend/danswer/server/manage/models.py

@@ -155,6 +155,7 @@ class SlackBotConfigCreationRequest(BaseModel):
     channel_names: list[str]
     respond_tag_only: bool = False
     respond_to_bots: bool = False
+    enable_auto_filters: bool = False
     # If no team members, assume respond in the channel to everyone
     respond_team_member_list: list[str] = []
     respond_slack_group_list: list[str] = []
@@ -188,6 +189,7 @@ class SlackBotConfig(BaseModel):
     channel_config: ChannelConfig
     response_type: SlackBotResponseType
     standard_answer_categories: list[StandardAnswerCategory]
+    enable_auto_filters: bool
 
     @classmethod
     def from_model(
@@ -208,6 +210,7 @@ class SlackBotConfig(BaseModel):
                 StandardAnswerCategory.from_model(standard_answer_category_model)
                 for standard_answer_category_model in slack_bot_config_model.standard_answer_categories
             ],
+            enable_auto_filters=slack_bot_config_model.enable_auto_filters,
         )
 
 

backend/danswer/server/manage/slack_bot.py

@@ -115,6 +115,7 @@ def create_slack_bot_config(
         response_type=slack_bot_config_creation_request.response_type,
         standard_answer_category_ids=slack_bot_config_creation_request.standard_answer_categories,
         db_session=db_session,
+        enable_auto_filters=slack_bot_config_creation_request.enable_auto_filters,
     )
     return SlackBotConfig.from_model(slack_bot_config_model)
 
@@ -174,6 +175,7 @@ def patch_slack_bot_config(
         response_type=slack_bot_config_creation_request.response_type,
         standard_answer_category_ids=slack_bot_config_creation_request.standard_answer_categories,
         db_session=db_session,
+        enable_auto_filters=slack_bot_config_creation_request.enable_auto_filters,
     )
     return SlackBotConfig.from_model(slack_bot_config_model)
 

backend/danswer/tools/search/search_tool.py

@@ -225,6 +225,9 @@ class SearchTool(Tool):
                 chunks_above=self.chunks_above,
                 chunks_below=self.chunks_below,
                 full_doc=self.full_doc,
+                enable_auto_detect_filters=self.retrieval_options.enable_auto_detect_filters
+                if self.retrieval_options
+                else None,
             ),
             user=self.user,
             llm=self.llm,

deployment/docker_compose/docker-compose.dev.yml

@@ -44,7 +44,6 @@ services:
       - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
       - QA_TIMEOUT=${QA_TIMEOUT:-}
       - MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
-      - DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
       - DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
       - DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
       - DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}
@@ -130,7 +129,6 @@ services:
       - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
       - QA_TIMEOUT=${QA_TIMEOUT:-}
       - MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
-      - DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
       - DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
       - DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
       - DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}

deployment/docker_compose/docker-compose.gpu-dev.yml

@@ -41,7 +41,6 @@ services:
       - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
       - QA_TIMEOUT=${QA_TIMEOUT:-}
       - MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
-      - DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
       - DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
       - DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
       - DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}
@@ -123,7 +122,6 @@ services:
       - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
       - QA_TIMEOUT=${QA_TIMEOUT:-}
       - MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
-      - DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
       - DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
       - DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
       - DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}

deployment/helm/values.yaml

@@ -402,7 +402,6 @@ configMap:
   GEN_AI_MAX_TOKENS: ""
   QA_TIMEOUT: "60"
   MAX_CHUNKS_FED_TO_CHAT: ""
-  DISABLE_LLM_FILTER_EXTRACTION: ""
   DISABLE_LLM_CHUNK_FILTER: ""
   DISABLE_LLM_CHOOSE_SEARCH: ""
   DISABLE_LLM_QUERY_REPHRASE: ""

deployment/kubernetes/env-configmap.yaml

@@ -24,7 +24,6 @@ data:
   GEN_AI_MAX_TOKENS: ""
   QA_TIMEOUT: "60"
   MAX_CHUNKS_FED_TO_CHAT: ""
-  DISABLE_LLM_FILTER_EXTRACTION: ""
   DISABLE_LLM_CHUNK_FILTER: ""
   DISABLE_LLM_CHOOSE_SEARCH: ""
   DISABLE_LLM_QUERY_REPHRASE: ""

web/src/app/admin/bot/SlackBotConfigCreationForm.tsx

@@ -77,6 +77,8 @@ export const SlackBotCreationForm = ({
               existingSlackBotConfig?.channel_config?.respond_tag_only || false,
             respond_to_bots:
               existingSlackBotConfig?.channel_config?.respond_to_bots || false,
+            enable_auto_filters:
+              existingSlackBotConfig?.enable_auto_filters || false,
             respond_member_group_list: (
               existingSlackBotConfig?.channel_config
                 ?.respond_team_member_list ?? []
@@ -114,6 +116,7 @@ export const SlackBotCreationForm = ({
             questionmark_prefilter_enabled: Yup.boolean().required(),
             respond_tag_only: Yup.boolean().required(),
             respond_to_bots: Yup.boolean().required(),
+            enable_auto_filters: Yup.boolean().required(),
             respond_member_group_list: Yup.array().of(Yup.string()).required(),
             still_need_help_enabled: Yup.boolean().required(),
             follow_up_tags: Yup.array().of(Yup.string()),
@@ -247,6 +250,11 @@ export const SlackBotCreationForm = ({
                   label="Responds to Bot messages"
                   subtext="If not set, DanswerBot will always ignore messages from Bots"
                 />
+                <BooleanFormField
+                  name="enable_auto_filters"
+                  label="Enable LLM Autofiltering"
+                  subtext="If set, the LLM will generate source and time filters based on the user's query"
+                />
                 <TextArrayField
                   name="respond_member_group_list"
                   label="Team Member Emails Or Slack Group Names"

web/src/app/admin/bot/lib.ts

@@ -8,6 +8,7 @@ import { Persona } from "../assistants/interfaces";
 interface SlackBotConfigCreationRequest {
   document_sets: number[];
   persona_id: number | null;
+  enable_auto_filters: boolean;
   channel_names: string[];
   answer_validity_check_enabled: boolean;
   questionmark_prefilter_enabled: boolean;
@@ -41,6 +42,7 @@ const buildRequestBodyFromCreationRequest = (
     channel_names: creationRequest.channel_names,
     respond_tag_only: creationRequest.respond_tag_only,
     respond_to_bots: creationRequest.respond_to_bots,
+    enable_auto_filters: creationRequest.enable_auto_filters,
     respond_team_member_list: creationRequest.respond_team_member_list,
     respond_slack_group_list: creationRequest.respond_slack_group_list,
     answer_filters: buildFiltersFromCreationRequest(creationRequest),

web/src/lib/types.ts

@@ -553,6 +553,7 @@ export interface SlackBotConfig {
   channel_config: ChannelConfig;
   response_type: SlackBotResponseType;
   standard_answer_categories: StandardAnswerCategory[];
+  enable_auto_filters: boolean;
 }
 
 export interface SlackBotTokens {