highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-21 21:41:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Allow to define custom conditions for the answer prompt answer validation (#1347)

Browse Source 
Co-authored-by: Matthieu Boret <matthieu.boret@fr.clara.net>
This commit is contained in:
mattboret 2024-05-11 19:43:05 +02:00 committed by GitHub
parent b8af38bb95
commit eb1b604b8c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 40 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/danswer/configs/app_configs.py
View File

@ -1,3 +1,4 @@
import json
import os import os
import urllib.parse import urllib.parse
@ -261,3 +262,9 @@ DISABLE_TELEMETRY = os.environ.get("DISABLE_TELEMETRY", "").lower() == "true"
TOKEN_BUDGET_GLOBALLY_ENABLED = ( TOKEN_BUDGET_GLOBALLY_ENABLED = (
os.environ.get("TOKEN_BUDGET_GLOBALLY_ENABLED", "").lower() == "true" os.environ.get("TOKEN_BUDGET_GLOBALLY_ENABLED", "").lower() == "true"
) )
# Defined custom query/answer conditions to validate the query and the LLM answer.
# Format: list of strings
CUSTOM_ANSWER_VALIDITY_CONDITIONS = json.loads(
os.environ.get("CUSTOM_ANSWER_VALIDITY_CONDITIONS", "[]")
)

40
backend/danswer/prompts/answer_validation.py
View File

@ -1,18 +1,46 @@
# The following prompts are used for verifying the LLM answer after it is already produced. # The following prompts are used for verifying the LLM answer after it is already produced.
# Reflexion flow essentially. This feature can be toggled on/off # Reflexion flow essentially. This feature can be toggled on/off
from danswer.configs.app_configs import CUSTOM_ANSWER_VALIDITY_CONDITIONS
from danswer.prompts.constants import ANSWER_PAT from danswer.prompts.constants import ANSWER_PAT
from danswer.prompts.constants import QUESTION_PAT from danswer.prompts.constants import QUESTION_PAT
ANSWER_VALIDITY_CONDITIONS = (
ANSWER_VALIDITY_PROMPT = f""" """
You are an assistant to identify invalid query/answer pairs coming from a large language model.
The query/answer pair is invalid if any of the following are True:
1. Query is asking for information that varies by person or is subjective. If there is not a \ 1. Query is asking for information that varies by person or is subjective. If there is not a \
globally true answer, the language model should not respond, therefore any answer is invalid. globally true answer, the language model should not respond, therefore any answer is invalid.
2. Answer addresses a related but different query. To be helpful, the model may provide \ 2. Answer addresses a related but different query. To be helpful, the model may provide \
related information about a query but it won't match what the user is asking, this is invalid. related information about a query but it won't match what the user is asking, this is invalid.
3. Answer is just some form of "I don\'t know" or "not enough information" without significant \ 3. Answer is just some form of "I don\'t know" or "not enough information" without significant \
additional useful information. Explaining why it does not know or cannot answer is invalid. additional useful information. Explaining why it does not know or cannot answer is invalid.
"""
if not CUSTOM_ANSWER_VALIDITY_CONDITIONS
else "\n".join(
[
f"{indice+1}. {condition}"
for indice, condition in enumerate(CUSTOM_ANSWER_VALIDITY_CONDITIONS)
]
)
)
ANSWER_FORMAT = (
"""
1. True or False
2. True or False
3. True or False
"""
if not CUSTOM_ANSWER_VALIDITY_CONDITIONS
else "\n".join(
[
f"{indice+1}. True or False"
for indice, _ in enumerate(CUSTOM_ANSWER_VALIDITY_CONDITIONS)
]
)
)
ANSWER_VALIDITY_PROMPT = f"""
You are an assistant to identify invalid query/answer pairs coming from a large language model.
The query/answer pair is invalid if any of the following are True:
{ANSWER_VALIDITY_CONDITIONS}
{QUESTION_PAT} {{user_query}} {QUESTION_PAT} {{user_query}}
{ANSWER_PAT} {{llm_answer}} {ANSWER_PAT} {{llm_answer}}
@ -20,9 +48,7 @@ additional useful information. Explaining why it does not know or cannot answer
------------------------ ------------------------
You MUST answer in EXACTLY the following format: You MUST answer in EXACTLY the following format:
``` ```
1. True or False {ANSWER_FORMAT}
2. True or False
3. True or False
Final Answer: Valid or Invalid Final Answer: Valid or Invalid
``` ```