highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-04-08 11:58:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

slackbot doesnt respond without citations/quotes (#1798)

Browse Source 
* slackbot doesnt respond without citations/quotes

fixed logical issues

fixed dict logic

* added slackbot shim for the llm source/time feature

* mypy fixes

* slackbot doesnt respond without citations/quotes

fixed logical issues

fixed dict logic

* Update handle_regular_answer.py

* added bypass_filter check

* final fixes
This commit is contained in:
hagen-danswer 2024-07-10 17:18:26 -07:00 committed by GitHub
parent 511f619212
commit 47a550221f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 97 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
backend/alembic/versions/7aea705850d5_added_slack_auto_filter.py Normal file
View File

@ -0,0 +1,35 @@
"""added slack_auto_filter
Revision ID: 7aea705850d5
Revises: 4505fd7302e1
Create Date: 2024-07-10 11:01:23.581015
"""
from alembic import op
import sqlalchemy as sa
revision = "7aea705850d5"
down_revision = "4505fd7302e1"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"slack_bot_config",
sa.Column("enable_auto_filters", sa.Boolean(), nullable=True),
)
op.execute(
"UPDATE slack_bot_config SET enable_auto_filters = FALSE WHERE enable_auto_filters IS NULL"
)
op.alter_column(
"slack_bot_config",
"enable_auto_filters",
existing_type=sa.Boolean(),
nullable=False,
server_default=sa.false(),
)
def downgrade() -> None:
op.drop_column("slack_bot_config", "enable_auto_filters")

3
backend/danswer/configs/chat_configs.py
View File

@ -28,9 +28,6 @@ BASE_RECENCY_DECAY = 0.5
FAVOR_RECENT_DECAY_MULTIPLIER = 2.0
# Currently this next one is not configurable via env
DISABLE_LLM_QUERY_ANSWERABILITY = QA_PROMPT_OVERRIDE == "weak"
DISABLE_LLM_FILTER_EXTRACTION = (
os.environ.get("DISABLE_LLM_FILTER_EXTRACTION", "").lower() == "true"
)
# Whether the LLM should evaluate all of the document chunks passed in for usefulness
# in relation to the user query
DISABLE_LLM_CHUNK_FILTER = (

4
backend/danswer/configs/danswerbot_configs.py
View File

@ -47,10 +47,6 @@ DANSWER_BOT_DISPLAY_ERROR_MSGS = os.environ.get(
DANSWER_BOT_RESPOND_EVERY_CHANNEL = (
os.environ.get("DANSWER_BOT_RESPOND_EVERY_CHANNEL", "").lower() == "true"
)
# Auto detect query options like time cutoff or heavily favor recently updated docs
DISABLE_DANSWER_BOT_FILTER_DETECT = (
os.environ.get("DISABLE_DANSWER_BOT_FILTER_DETECT", "").lower() == "true"
)
# Add a second LLM call post Answer to verify if the Answer is valid
# Throws out answers that don't directly or fully answer the user query
# This is the default for all DanswerBot channels unless the channel is configured individually

2
backend/danswer/danswerbot/slack/handlers/handle_message.py
View File

@ -168,8 +168,6 @@ def handle_message(
if slack_bot_config and slack_bot_config.channel_config:
channel_conf = slack_bot_config.channel_config
if not bypass_filters and "answer_filters" in channel_conf:
"well_answered_postfilter" in channel_conf["answer_filters"]
if (
"questionmark_prefilter" in channel_conf["answer_filters"]
and "?" not in messages[-1].message

40
backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
View File

@ -22,7 +22,6 @@ from danswer.configs.danswerbot_configs import DANSWER_BOT_TARGET_CHUNK_PERCENTA
from danswer.configs.danswerbot_configs import DANSWER_BOT_USE_QUOTES
from danswer.configs.danswerbot_configs import DANSWER_FOLLOWUP_EMOJI
from danswer.configs.danswerbot_configs import DANSWER_REACT_EMOJI
from danswer.configs.danswerbot_configs import DISABLE_DANSWER_BOT_FILTER_DETECT
from danswer.configs.danswerbot_configs import ENABLE_DANSWERBOT_REFLEXION
from danswer.danswerbot.slack.blocks import build_documents_blocks
from danswer.danswerbot.slack.blocks import build_follow_up_block
@ -91,7 +90,6 @@ def handle_regular_answer(
thread_context_percent: float = DANSWER_BOT_TARGET_CHUNK_PERCENTAGE,
should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS,
disable_docs_only_answer: bool = DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER,
disable_auto_detect_filters: bool = DISABLE_DANSWER_BOT_FILTER_DETECT,
disable_cot: bool = DANSWER_BOT_DISABLE_COT,
reflexion: bool = ENABLE_DANSWERBOT_REFLEXION,
) -> bool:
@ -207,12 +205,17 @@ def handle_regular_answer(
)
# Default True because no other ways to apply filters in Slack (no nice UI)
# Commenting this out because this is only available to the slackbot for now
# later we plan to implement this at the persona level where this will get
# commented back in
# auto_detect_filters = (
# persona.llm_filter_extraction if persona is not None else True
# )
auto_detect_filters = (
persona.llm_filter_extraction if persona is not None else True
slack_bot_config.enable_auto_filters
if slack_bot_config is not None
else False
)
if disable_auto_detect_filters:
auto_detect_filters = False
retrieval_details = RetrievalDetails(
run_search=OptionalSearchSetting.ALWAYS,
real_time=False,
@ -352,6 +355,31 @@ def handle_regular_answer(
)
return True
only_respond_with_citations_or_quotes = (
channel_conf
and "well_answered_postfilter" in channel_conf.get("answer_filters", [])
)
has_citations_or_quotes = bool(answer.citations or answer.quotes)
if (
only_respond_with_citations_or_quotes
and not has_citations_or_quotes
and not message_info.bypass_filters
):
logger.error(
f"Unable to find citations or quotes to answer: '{answer.rephrase}' - not answering!"
)
# Optionally, respond in thread with the error message
# Used primarily for debugging purposes
if should_respond_with_error_msgs:
respond_in_thread(
client=client,
channel=channel,
receiver_ids=None,
text="Found no citations or quotes when trying to answer.",
thread_ts=message_ts_to_respond_to,
)
return True
# If called with the DanswerBot slash command, the question is lost so we have to reshow it
restate_question_block = get_restate_blocks(messages[-1].message, is_bot_msg)

3
backend/danswer/db/models.py
View File

@ -1195,6 +1195,9 @@ class SlackBotConfig(Base):
response_type: Mapped[SlackBotResponseType] = mapped_column(
Enum(SlackBotResponseType, native_enum=False), nullable=False
)
enable_auto_filters: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False
)
persona: Mapped[Persona | None] = relationship("Persona")
standard_answer_categories: Mapped[list[StandardAnswerCategory]] = relationship(

6
backend/danswer/db/slack_bot_config.py
View File

@ -53,7 +53,7 @@ def create_slack_bot_persona(
description="",
num_chunks=num_chunks,
llm_relevance_filter=True,
llm_filter_extraction=True,
llm_filter_extraction=False,
recency_bias=RecencyBiasSetting.AUTO,
prompt_ids=[default_prompt.id],
document_set_ids=document_set_ids,
@ -74,6 +74,7 @@ def insert_slack_bot_config(
channel_config: ChannelConfig,
response_type: SlackBotResponseType,
standard_answer_category_ids: list[int],
enable_auto_filters: bool,
db_session: Session,
) -> SlackBotConfig:
existing_standard_answer_categories = fetch_standard_answer_categories_by_ids(
@ -90,6 +91,7 @@ def insert_slack_bot_config(
channel_config=channel_config,
response_type=response_type,
standard_answer_categories=existing_standard_answer_categories,
enable_auto_filters=enable_auto_filters,
)
db_session.add(slack_bot_config)
db_session.commit()
@ -103,6 +105,7 @@ def update_slack_bot_config(
channel_config: ChannelConfig,
response_type: SlackBotResponseType,
standard_answer_category_ids: list[int],
enable_auto_filters: bool,
db_session: Session,
) -> SlackBotConfig:
slack_bot_config = db_session.scalar(
@ -134,6 +137,7 @@ def update_slack_bot_config(
slack_bot_config.standard_answer_categories = list(
existing_standard_answer_categories
)
slack_bot_config.enable_auto_filters = enable_auto_filters
# if the persona has changed, then clean up the old persona
if persona_id != existing_persona_id and existing_persona_id:

8
backend/danswer/search/preprocessing/preprocessing.py
View File

@ -2,7 +2,6 @@ from sqlalchemy.orm import Session
from danswer.configs.chat_configs import BASE_RECENCY_DECAY
from danswer.configs.chat_configs import DISABLE_LLM_CHUNK_FILTER
from danswer.configs.chat_configs import DISABLE_LLM_FILTER_EXTRACTION
from danswer.configs.chat_configs import FAVOR_RECENT_DECAY_MULTIPLIER
from danswer.configs.chat_configs import NUM_RETURNED_HITS
from danswer.db.models import User
@ -36,8 +35,6 @@ def retrieval_preprocessing(
db_session: Session,
bypass_acl: bool = False,
include_query_intent: bool = True,
enable_auto_detect_filters: bool = False,
disable_llm_filter_extraction: bool = DISABLE_LLM_FILTER_EXTRACTION,
disable_llm_chunk_filter: bool = DISABLE_LLM_CHUNK_FILTER,
base_recency_decay: float = BASE_RECENCY_DECAY,
favor_recent_decay_multiplier: float = FAVOR_RECENT_DECAY_MULTIPLIER,
@ -63,10 +60,7 @@ def retrieval_preprocessing(
auto_detect_time_filter = True
auto_detect_source_filter = True
if disable_llm_filter_extraction:
auto_detect_time_filter = False
auto_detect_source_filter = False
elif enable_auto_detect_filters is False:
if not search_request.enable_auto_detect_filters:
logger.debug("Retrieval details disables auto detect filters")
auto_detect_time_filter = False
auto_detect_source_filter = False

3
backend/danswer/server/manage/models.py
View File

@ -155,6 +155,7 @@ class SlackBotConfigCreationRequest(BaseModel):
channel_names: list[str]
respond_tag_only: bool = False
respond_to_bots: bool = False
enable_auto_filters: bool = False
# If no team members, assume respond in the channel to everyone
respond_team_member_list: list[str] = []
respond_slack_group_list: list[str] = []
@ -188,6 +189,7 @@ class SlackBotConfig(BaseModel):
channel_config: ChannelConfig
response_type: SlackBotResponseType
standard_answer_categories: list[StandardAnswerCategory]
enable_auto_filters: bool
@classmethod
def from_model(
@ -208,6 +210,7 @@ class SlackBotConfig(BaseModel):
StandardAnswerCategory.from_model(standard_answer_category_model)
for standard_answer_category_model in slack_bot_config_model.standard_answer_categories
],
enable_auto_filters=slack_bot_config_model.enable_auto_filters,
)

2
backend/danswer/server/manage/slack_bot.py
View File

@ -115,6 +115,7 @@ def create_slack_bot_config(
response_type=slack_bot_config_creation_request.response_type,
standard_answer_category_ids=slack_bot_config_creation_request.standard_answer_categories,
db_session=db_session,
enable_auto_filters=slack_bot_config_creation_request.enable_auto_filters,
)
return SlackBotConfig.from_model(slack_bot_config_model)
@ -174,6 +175,7 @@ def patch_slack_bot_config(
response_type=slack_bot_config_creation_request.response_type,
standard_answer_category_ids=slack_bot_config_creation_request.standard_answer_categories,
db_session=db_session,
enable_auto_filters=slack_bot_config_creation_request.enable_auto_filters,
)
return SlackBotConfig.from_model(slack_bot_config_model)

3
backend/danswer/tools/search/search_tool.py
View File

@ -225,6 +225,9 @@ class SearchTool(Tool):
chunks_above=self.chunks_above,
chunks_below=self.chunks_below,
full_doc=self.full_doc,
enable_auto_detect_filters=self.retrieval_options.enable_auto_detect_filters
if self.retrieval_options
else None,
),
user=self.user,
llm=self.llm,

2
deployment/docker_compose/docker-compose.dev.yml
View File

@ -44,7 +44,6 @@ services:
- GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
- QA_TIMEOUT=${QA_TIMEOUT:-}
- MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
- DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
- DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
- DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
- DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}
@ -130,7 +129,6 @@ services:
- GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
- QA_TIMEOUT=${QA_TIMEOUT:-}
- MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
- DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
- DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
- DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
- DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}

2
deployment/docker_compose/docker-compose.gpu-dev.yml
View File

@ -41,7 +41,6 @@ services:
- GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
- QA_TIMEOUT=${QA_TIMEOUT:-}
- MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
- DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
- DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
- DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
- DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}
@ -123,7 +122,6 @@ services:
- GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
- QA_TIMEOUT=${QA_TIMEOUT:-}
- MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
- DISABLE_LLM_FILTER_EXTRACTION=${DISABLE_LLM_FILTER_EXTRACTION:-}
- DISABLE_LLM_CHUNK_FILTER=${DISABLE_LLM_CHUNK_FILTER:-}
- DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
- DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}

1
deployment/helm/values.yaml
View File

@ -402,7 +402,6 @@ configMap:
GEN_AI_MAX_TOKENS: ""
QA_TIMEOUT: "60"
MAX_CHUNKS_FED_TO_CHAT: ""
DISABLE_LLM_FILTER_EXTRACTION: ""
DISABLE_LLM_CHUNK_FILTER: ""
DISABLE_LLM_CHOOSE_SEARCH: ""
DISABLE_LLM_QUERY_REPHRASE: ""

1
deployment/kubernetes/env-configmap.yaml
View File

@ -24,7 +24,6 @@ data:
GEN_AI_MAX_TOKENS: ""
QA_TIMEOUT: "60"
MAX_CHUNKS_FED_TO_CHAT: ""
DISABLE_LLM_FILTER_EXTRACTION: ""
DISABLE_LLM_CHUNK_FILTER: ""
DISABLE_LLM_CHOOSE_SEARCH: ""
DISABLE_LLM_QUERY_REPHRASE: ""

8
web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
View File

@ -77,6 +77,8 @@ export const SlackBotCreationForm = ({
existingSlackBotConfig?.channel_config?.respond_tag_only || false,
respond_to_bots:
existingSlackBotConfig?.channel_config?.respond_to_bots || false,
enable_auto_filters:
existingSlackBotConfig?.enable_auto_filters || false,
respond_member_group_list: (
existingSlackBotConfig?.channel_config
?.respond_team_member_list ?? []
@ -114,6 +116,7 @@ export const SlackBotCreationForm = ({
questionmark_prefilter_enabled: Yup.boolean().required(),
respond_tag_only: Yup.boolean().required(),
respond_to_bots: Yup.boolean().required(),
enable_auto_filters: Yup.boolean().required(),
respond_member_group_list: Yup.array().of(Yup.string()).required(),
still_need_help_enabled: Yup.boolean().required(),
follow_up_tags: Yup.array().of(Yup.string()),
@ -247,6 +250,11 @@ export const SlackBotCreationForm = ({
label="Responds to Bot messages"
subtext="If not set, DanswerBot will always ignore messages from Bots"
/>
<BooleanFormField
name="enable_auto_filters"
label="Enable LLM Autofiltering"
subtext="If set, the LLM will generate source and time filters based on the user's query"
/>
<TextArrayField
name="respond_member_group_list"
label="Team Member Emails Or Slack Group Names"

2
web/src/app/admin/bot/lib.ts
View File

@ -8,6 +8,7 @@ import { Persona } from "../assistants/interfaces";
interface SlackBotConfigCreationRequest {
document_sets: number[];
persona_id: number | null;
enable_auto_filters: boolean;
channel_names: string[];
answer_validity_check_enabled: boolean;
questionmark_prefilter_enabled: boolean;
@ -41,6 +42,7 @@ const buildRequestBodyFromCreationRequest = (
channel_names: creationRequest.channel_names,
respond_tag_only: creationRequest.respond_tag_only,
respond_to_bots: creationRequest.respond_to_bots,
enable_auto_filters: creationRequest.enable_auto_filters,
respond_team_member_list: creationRequest.respond_team_member_list,
respond_slack_group_list: creationRequest.respond_slack_group_list,
answer_filters: buildFiltersFromCreationRequest(creationRequest),

1
web/src/lib/types.ts
View File

@ -553,6 +553,7 @@ export interface SlackBotConfig {
channel_config: ChannelConfig;
response_type: SlackBotResponseType;
standard_answer_categories: StandardAnswerCategory[];
enable_auto_filters: boolean;
}
export interface SlackBotTokens {