highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-05-18 07:40:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

Enrichment prompts, prompt improvements, dispatch logging & reinsert empty tool response

Browse Source
This commit is contained in:
joachim-danswer 2025-01-29 11:33:56 -08:00 committed by Evan Lohn
parent 4e17fc06ff
commit 7e98936c58
8 changed files with 207 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/onyx/agents/agent_search/deep_search_a/initial__individual_sub_answer__subgraph/nodes/answer_generation.py
View File

@ -82,7 +82,7 @@ def answer_generation(
)
start_stream_token = datetime.now()
dispatch_custom_event(
"sub_answers",
"refined_agent_answer",
AgentAnswerPiece(
answer_piece=content,
level=level,

89
backend/onyx/agents/agent_search/deep_search_a/initial__retrieval_sub_answers__subgraph/nodes/generate_initial_answer.py
View File

@ -24,7 +24,7 @@ from onyx.agents.agent_search.deep_search_a.main__graph.states import (
)
from onyx.agents.agent_search.models import AgentSearchConfig
from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
build_history_prompt,
get_prompt_enrichment_components,
)
from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
trim_prompt_piece,
@ -45,16 +45,10 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
dispatch_main_answer_stop_info,
)
from onyx.agents.agent_search.shared_graph_utils.utils import format_docs
from onyx.agents.agent_search.shared_graph_utils.utils import (
get_persona_agent_prompt_expressions,
)
from onyx.agents.agent_search.shared_graph_utils.utils import get_today_prompt
from onyx.agents.agent_search.shared_graph_utils.utils import parse_question_id
from onyx.agents.agent_search.shared_graph_utils.utils import summarize_history
from onyx.chat.models import AgentAnswerPiece
from onyx.chat.models import ExtendedToolResponse
from onyx.configs.agent_configs import AGENT_MAX_ANSWER_CONTEXT_DOCS
from onyx.configs.agent_configs import AGENT_MAX_STATIC_HISTORY_CHAR_LENGTH
from onyx.configs.agent_configs import AGENT_MIN_ORIG_QUESTION_DOCS
from onyx.context.search.models import InferenceSection
from onyx.tools.tool_implementations.search.search_tool import yield_search_responses
@ -69,13 +63,7 @@ def generate_initial_answer(
agent_a_config = cast(AgentSearchConfig, config["metadata"]["config"])
question = agent_a_config.search_request.query
persona_prompts = get_persona_agent_prompt_expressions(
agent_a_config.search_request.persona
)
history = build_history_prompt(agent_a_config, question)
date_str = get_today_prompt()
prompt_enrichment_components = get_prompt_enrichment_components(agent_a_config)
sub_questions_cited_docs = state.cited_docs
all_original_question_documents = state.all_original_question_documents
@ -98,6 +86,28 @@ def generate_initial_answer(
decomp_questions = []
# Use the query info from the base document retrieval
query_info = get_query_info(state.original_question_retrieval_results)
if agent_a_config.search_tool is None:
raise ValueError("search_tool must be provided for agentic search")
for tool_response in yield_search_responses(
query=question,
reranked_sections=relevant_docs,
final_context_sections=relevant_docs,
search_query_info=query_info,
get_section_relevance=lambda: None, # TODO: add relevance
search_tool=agent_a_config.search_tool,
):
dispatch_custom_event(
"tool_response",
ExtendedToolResponse(
id=tool_response.id,
response=tool_response.response,
level=0,
level_question_nr=0, # 0, 0 is the base question
),
)
if len(relevant_docs) == 0:
dispatch_custom_event(
"initial_agent_answer",
@ -118,28 +128,6 @@ def generate_initial_answer(
)
else:
# Use the query info from the base document retrieval
query_info = get_query_info(state.original_question_retrieval_results)
if agent_a_config.search_tool is None:
raise ValueError("search_tool must be provided for agentic search")
for tool_response in yield_search_responses(
query=question,
reranked_sections=relevant_docs,
final_context_sections=relevant_docs,
search_query_info=query_info,
get_section_relevance=lambda: None, # TODO: add relevance
search_tool=agent_a_config.search_tool,
):
dispatch_custom_event(
"tool_response",
ExtendedToolResponse(
id=tool_response.id,
response=tool_response.response,
level=0,
level_question_nr=0, # 0, 0 is the base question
),
)
decomp_answer_results = state.decomp_answer_results
good_qa_list: list[str] = []
@ -176,21 +164,15 @@ def generate_initial_answer(
model = agent_a_config.fast_llm
# summarize the history iff too long
if len(history) > AGENT_MAX_STATIC_HISTORY_CHAR_LENGTH:
history = summarize_history(
history, question, persona_prompts.base_prompt, model
)
doc_context = format_docs(relevant_docs)
doc_context = trim_prompt_piece(
model.config,
doc_context,
base_prompt
+ sub_question_answer_str
+ persona_prompts.contextualized_prompt
+ history
+ date_str,
+ prompt_enrichment_components.persona_prompts.contextualized_prompt
+ prompt_enrichment_components.history
+ prompt_enrichment_components.date_str,
)
msg = [
@ -201,14 +183,15 @@ def generate_initial_answer(
sub_question_answer_str
),
relevant_docs=format_docs(relevant_docs),
persona_specification=persona_prompts.contextualized_prompt,
history=history,
date_prompt=date_str,
persona_specification=prompt_enrichment_components.persona_prompts.contextualized_prompt,
history=prompt_enrichment_components.history,
date_prompt=prompt_enrichment_components.date_str,
)
)
]
streamed_tokens: list[str | list[str | dict[str, Any]]] = [""]
dispatch_timings: list[float] = []
for message in model.stream(msg):
# TODO: in principle, the answer here COULD contain images, but we don't support that yet
content = message.content
@ -216,6 +199,8 @@ def generate_initial_answer(
raise ValueError(
f"Expected content to be a string, but got {type(content)}"
)
start_stream_token = datetime.now()
dispatch_custom_event(
"initial_agent_answer",
AgentAnswerPiece(
@ -225,8 +210,16 @@ def generate_initial_answer(
answer_type="agent_level_answer",
),
)
end_stream_token = datetime.now()
dispatch_timings.append(
(end_stream_token - start_stream_token).microseconds
)
streamed_tokens.append(content)
logger.info(
f"Average dispatch time for initial answer: {sum(dispatch_timings) / len(dispatch_timings)}"
)
dispatch_main_answer_stop_info(0)
response = merge_content(*streamed_tokens)
answer = cast(str, response)

30
backend/onyx/agents/agent_search/deep_search_a/main__graph/nodes/generate_refined_answer.py
View File

@ -21,7 +21,7 @@ from onyx.agents.agent_search.deep_search_a.main__graph.states import (
)
from onyx.agents.agent_search.models import AgentSearchConfig
from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
build_history_prompt,
get_prompt_enrichment_components,
)
from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
trim_prompt_piece,
@ -43,10 +43,6 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
dispatch_main_answer_stop_info,
)
from onyx.agents.agent_search.shared_graph_utils.utils import format_docs
from onyx.agents.agent_search.shared_graph_utils.utils import (
get_persona_agent_prompt_expressions,
)
from onyx.agents.agent_search.shared_graph_utils.utils import get_today_prompt
from onyx.agents.agent_search.shared_graph_utils.utils import parse_question_id
from onyx.chat.models import AgentAnswerPiece
from onyx.chat.models import ExtendedToolResponse
@ -64,12 +60,12 @@ def generate_refined_answer(
agent_a_config = cast(AgentSearchConfig, config["metadata"]["config"])
question = agent_a_config.search_request.query
persona_contextualized_prompt = get_persona_agent_prompt_expressions(
agent_a_config.search_request.persona
).contextualized_prompt
prompt_enrichment_components = get_prompt_enrichment_components(agent_a_config)
persona_contextualized_prompt = (
prompt_enrichment_components.persona_prompts.contextualized_prompt
)
history = build_history_prompt(agent_a_config, question)
date_str = get_today_prompt()
initial_documents = state.documents
refined_documents = state.refined_documents
sub_questions_cited_docs = state.cited_docs
@ -197,21 +193,21 @@ def generate_refined_answer(
+ sub_question_answer_str
+ initial_answer
+ persona_contextualized_prompt
+ history,
+ prompt_enrichment_components.history,
)
msg = [
HumanMessage(
content=base_prompt.format(
question=question,
history=history,
history=prompt_enrichment_components.history,
answered_sub_questions=remove_document_citations(
sub_question_answer_str
),
relevant_docs=relevant_docs,
initial_answer=remove_document_citations(initial_answer),
persona_specification=persona_contextualized_prompt,
date_prompt=date_str,
date_prompt=prompt_enrichment_components.date_str,
)
)
]
@ -219,6 +215,7 @@ def generate_refined_answer(
# Grader
streamed_tokens: list[str | list[str | dict[str, Any]]] = [""]
dispatch_timings: list[float] = []
for message in model.stream(msg):
# TODO: in principle, the answer here COULD contain images, but we don't support that yet
content = message.content
@ -226,6 +223,8 @@ def generate_refined_answer(
raise ValueError(
f"Expected content to be a string, but got {type(content)}"
)
start_stream_token = datetime.now()
dispatch_custom_event(
"refined_agent_answer",
AgentAnswerPiece(
@ -235,8 +234,13 @@ def generate_refined_answer(
answer_type="agent_level_answer",
),
)
end_stream_token = datetime.now()
dispatch_timings.append((end_stream_token - start_stream_token).microseconds)
streamed_tokens.append(content)
logger.info(
f"Average dispatch time for refined answer: {sum(dispatch_timings) / len(dispatch_timings)}"
)
dispatch_main_answer_stop_info(1)
response = merge_content(*streamed_tokens)
answer = cast(str, response)

4
backend/onyx/agents/agent_search/deep_search_a/main__graph/states.py
View File

@ -104,7 +104,9 @@ class RequireRefinedAnswerUpdate(LoggerUpdate):
class DecompAnswersUpdate(LoggerUpdate):
documents: Annotated[list[InferenceSection], dedup_inference_sections] = []
context_documents: Annotated[list[InferenceSection], dedup_inference_sections] = []
cited_docs: Annotated[list[InferenceSection], dedup_inference_sections] = []
cited_docs: Annotated[
list[InferenceSection], dedup_inference_sections
] = [] # cited docs from sub-answers are used for answer context
decomp_answer_results: Annotated[
list[QuestionAnswerResults], dedup_question_answer_results
] = []

7
backend/onyx/agents/agent_search/models.py
View File

@ -5,6 +5,7 @@ from pydantic import BaseModel
from pydantic import model_validator
from sqlalchemy.orm import Session
from onyx.agents.agent_search.shared_graph_utils.models import PersonaExpressions
from onyx.chat.prompt_builder.answer_prompt_builder import AnswerPromptBuilder
from onyx.context.search.models import SearchRequest
from onyx.file_store.utils import InMemoryChatFile
@ -91,3 +92,9 @@ class AgentDocumentCitations(BaseModel):
document_id: str
document_title: str
link: str
class AgentPromptEnrichmentComponents(BaseModel):
persona_prompts: PersonaExpressions
history: str
date_str: str

145
backend/onyx/agents/agent_search/run_graph.py
View File

@ -198,80 +198,83 @@ def run_basic_graph(
if __name__ == "__main__":
from onyx.llm.factory import get_default_llms
now_start = datetime.now()
logger.debug(f"Start at {now_start}")
for _ in range(3):
now_start = datetime.now()
logger.debug(f"Start at {now_start}")
if GRAPH_VERSION_NAME == "a":
graph = main_graph_builder_a()
else:
graph = main_graph_builder_a()
compiled_graph = graph.compile()
now_end = datetime.now()
logger.debug(f"Graph compiled in {now_end - now_start} seconds")
primary_llm, fast_llm = get_default_llms()
search_request = SearchRequest(
# query="what can you do with gitlab?",
# query="What are the guiding principles behind the development of cockroachDB",
# query="What are the temperatures in Munich, Hawaii, and New York?",
# query="When was Washington born?",
# query="What is Onyx?",
# query="What is the difference between astronomy and astrology?",
query="Do a search to tell me hat is the difference between astronomy and astrology?",
)
# Joachim custom persona
with get_session_context_manager() as db_session:
config, search_tool = get_test_config(
db_session, primary_llm, fast_llm, search_request
)
# search_request.persona = get_persona_by_id(1, None, db_session)
config.use_persistence = True
# config.perform_initial_search_path_decision = False
config.perform_initial_search_decomposition = True
if GRAPH_VERSION_NAME == "a":
input = MainInput_a(
base_question=config.search_request.query, log_messages=[]
)
graph = main_graph_builder_a()
else:
input = MainInput_a(
base_question=config.search_request.query, log_messages=[]
graph = main_graph_builder_a()
compiled_graph = graph.compile()
now_end = datetime.now()
logger.debug(f"Graph compiled in {now_end - now_start} seconds")
primary_llm, fast_llm = get_default_llms()
search_request = SearchRequest(
# query="what can you do with gitlab?",
# query="What are the guiding principles behind the development of cockroachDB",
# query="What are the temperatures in Munich, Hawaii, and New York?",
# query="When was Washington born?",
# query="What is Onyx?",
# query="What is the difference between astronomy and astrology?",
query="Do a search to tell me hat is the difference between astronomy and astrology?",
)
# Joachim custom persona
with get_session_context_manager() as db_session:
config, search_tool = get_test_config(
db_session, primary_llm, fast_llm, search_request
)
# with open("output.txt", "w") as f:
tool_responses: list = []
for output in run_graph(compiled_graph, config, input):
# pass
# search_request.persona = get_persona_by_id(1, None, db_session)
config.use_persistence = True
# config.perform_initial_search_path_decision = False
config.perform_initial_search_decomposition = True
if GRAPH_VERSION_NAME == "a":
input = MainInput_a(
base_question=config.search_request.query, log_messages=[]
)
else:
input = MainInput_a(
base_question=config.search_request.query, log_messages=[]
)
# with open("output.txt", "w") as f:
tool_responses: list = []
for output in run_graph(compiled_graph, config, input):
# pass
if isinstance(output, ToolCallKickoff):
pass
elif isinstance(output, ExtendedToolResponse):
tool_responses.append(output.response)
logger.info(
f" ---- ET {output.level} - {output.level_question_nr} | "
)
elif isinstance(output, SubQueryPiece):
logger.info(
f"Sq {output.level} - {output.level_question_nr} - {output.sub_query} | "
)
elif isinstance(output, SubQuestionPiece):
logger.info(
f"SQ {output.level} - {output.level_question_nr} - {output.sub_question} | "
)
elif (
isinstance(output, AgentAnswerPiece)
and output.answer_type == "agent_sub_answer"
):
logger.info(
f" ---- SA {output.level} - {output.level_question_nr} {output.answer_piece} | "
)
elif (
isinstance(output, AgentAnswerPiece)
and output.answer_type == "agent_level_answer"
):
logger.info(
f" ---------- FA {output.level} - {output.level_question_nr} {output.answer_piece} | "
)
elif isinstance(output, RefinedAnswerImprovement):
logger.info(f" ---------- RE {output.refined_answer_improvement} | ")
if isinstance(output, ToolCallKickoff):
pass
elif isinstance(output, ExtendedToolResponse):
tool_responses.append(output.response)
logger.info(
f" ---- ET {output.level} - {output.level_question_nr} | "
)
elif isinstance(output, SubQueryPiece):
logger.info(
f"Sq {output.level} - {output.level_question_nr} - {output.sub_query} | "
)
elif isinstance(output, SubQuestionPiece):
logger.info(
f"SQ {output.level} - {output.level_question_nr} - {output.sub_question} | "
)
elif (
isinstance(output, AgentAnswerPiece)
and output.answer_type == "agent_sub_answer"
):
logger.info(
f" ---- SA {output.level} - {output.level_question_nr} {output.answer_piece} | "
)
elif (
isinstance(output, AgentAnswerPiece)
and output.answer_type == "agent_level_answer"
):
logger.info(
f" ---------- FA {output.level} - {output.level_question_nr} {output.answer_piece} | "
)
elif isinstance(output, RefinedAnswerImprovement):
logger.info(
f" ---------- RE {output.refined_answer_improvement} | "
)
# for tool_response in tool_responses:
# logger.debug(tool_response)
# for tool_response in tool_responses:
# logger.debug(tool_response)

19
backend/onyx/agents/agent_search/shared_graph_utils/agent_prompt_ops.py
View File

@ -3,6 +3,7 @@ from langchain.schema import HumanMessage
from langchain.schema import SystemMessage
from langchain_core.messages.tool import ToolMessage
from onyx.agents.agent_search.models import AgentPromptEnrichmentComponents
from onyx.agents.agent_search.models import AgentSearchConfig
from onyx.agents.agent_search.shared_graph_utils.prompts import BASE_RAG_PROMPT_v2
from onyx.agents.agent_search.shared_graph_utils.prompts import HISTORY_PROMPT
@ -113,3 +114,21 @@ def build_history_prompt(config: AgentSearchConfig, question: str) -> str:
history = summarize_history(history, question, persona_base, model)
return HISTORY_PROMPT.format(history=history) if history else ""
def get_prompt_enrichment_components(
config: AgentSearchConfig,
) -> AgentPromptEnrichmentComponents:
persona_prompts = get_persona_agent_prompt_expressions(
config.search_request.persona
)
history = build_history_prompt(config, config.search_request.query)
date_str = get_today_prompt()
return AgentPromptEnrichmentComponents(
persona_prompts=persona_prompts,
history=history,
date_str=date_str,
)

85
backend/onyx/agents/agent_search/shared_graph_utils/prompts.py
View File

@ -5,14 +5,14 @@ NO_RECOVERED_DOCS = "No relevant information recovered"
DATE_PROMPT = """Today is {date}.\n\n"""
HISTORY_PROMPT = """\n
For more context, here is the history of the conversation so far that preceeded this question:
For more context, here is the history of the conversation so far that preceded this question:
\n ------- \n
{history}
\n ------- \n\n
"""
REWRITE_PROMPT_MULTI_ORIGINAL = """ \n
Please convert an initial user question into a 2-3 more appropriate short and pointed search queries for retrievel from a
Please convert an initial user question into a 2-3 more appropriate short and pointed search queries for retrivel from a
document store. Particularly, try to think about resolving ambiguities and make the search queries more specific,
enabling the system to search more broadly.
Also, try to make the search queries not redundant, i.e. not too similar! \n\n
@ -41,7 +41,7 @@ BASE_RAG_PROMPT = (
{persona_specification}
{date_prompt}
Use the context provided below - and only the
provided context - to answer the given question. (Note that the answer is in service of anserwing a broader
provided context - to answer the given question. (Note that the answer is in service of ansering a broader
question, given below as 'motivation'.)
Again, only use the provided context and do not use your internal knowledge! If you cannot answer the
@ -162,11 +162,11 @@ Answer:
INITIAL_DECOMPOSITION_PROMPT_BASIC = """ \n
If you think it is helpful, please decompose an initial user question into not more
than 4 appropriate sub-questions that help to answer the original question.
The purpose for this decomposition is to isolate individulal entities
The purpose for this decomposition is to isolate individual entities
(i.e., 'compare sales of company A and company B' -> 'what are sales for company A' + 'what are sales
for company B'), split ambiguous terms (i.e., 'what is our success with company A' -> 'what are our
sales with company A' + 'what is our market share with company A' + 'is company A a reference customer
for us'), etc. Each sub-question should be realistically be answerable by a good RAG system.
for us'), etc. Each sub-question should realistically be answerable by a good RAG system.
Importantly, if you think it is not needed or helpful, please just return an empty list. That is ok too.
@ -181,7 +181,7 @@ Answer:
"""
REWRITE_PROMPT_SINGLE = """ \n
Please convert an initial user question into a more appropriate search query for retrievel from a
Please convert an initial user question into a more appropriate search query for retrieval from a
document store. \n
Here is the initial question:
\n ------- \n
@ -194,7 +194,7 @@ MODIFIED_RAG_PROMPT = (
"""You are an assistant for question-answering tasks. Use the context provided below
- and only this context - to answer the question. It is a matter of life and death that you do NOT
use your internal knowledge, just the provided information!
If you don't have enough infortmation to generate an answer, just say """
If you don't have enough information to generate an answer, just say """
+ f'"{UNKNOWN_ANSWER}"'
+ """.
Use three sentences maximum and keep the answer concise.
@ -212,7 +212,7 @@ ERT_INFORMED_DEEP_DECOMPOSE_PROMPT = """ \n
good enough. Also, some sub-questions had been answered and this information has been used to provide
the initial answer. Some other subquestions may have been suggested based on little knowledge, but they
were not directly answerable. Also, some entities, relationships and terms are givenm to you so that
you have an idea of how the avaiolable data looks like.
you have an idea of how the available data looks like.
Your role is to generate 3-5 new sub-questions that would help to answer the initial question,
considering:
@ -232,7 +232,7 @@ ERT_INFORMED_DEEP_DECOMPOSE_PROMPT = """ \n
- The sub-questions should be specific to the question and provide richer context for the question,
resolve ambiguities, or address shortcoming of the initial answer
- Each sub-question - when answered - should be relevant for the answer to the original question
- The sub-questions should be free from comparisions, ambiguities,judgements, aggregations, or any
- The sub-questions should be free from comparisons, ambiguities,judgements, aggregations, or any
other complications that may require extra context.
- The sub-questions MUST have the full context of the original question so that it can be executed by
a RAG system independently without the original question available
@ -288,7 +288,7 @@ DOC_INFORMED_DEEP_DECOMPOSE_PROMPT = """ \n
An initial user question needs to be answered. An initial answer has been provided but it wasn't quite
good enough. Also, some sub-questions had been answered and this information has been used to provide
the initial answer. Some other subquestions may have been suggested based on little knowledge, but they
were not directly answerable. Also, some entities, relationships and terms are givenm to you so that
were not directly answerable. Also, some entities, relationships and terms are given to you so that
you have an idea of how the avaiolable data looks like.
Your role is to generate 2-4 new sub-questions that would help to answer the initial question,
@ -298,7 +298,7 @@ DOC_INFORMED_DEEP_DECOMPOSE_PROMPT = """ \n
2) The initial answer that was found to be unsatisfactory
3) The sub-questions that were answered
4) The sub-questions that were suggested but not answered
5) A sample of the TYPE of documents that may be in the databse in order to inform
5) A sample of the TYPE of documents that may be in the database in order to inform
you what type of entities, relationships, and terms you may want to consider asking about.
(But do not build the questions strictly on these documents! They are only examples!
Take the, as illustrations.)
@ -347,7 +347,7 @@ DOC_INFORMED_DEEP_DECOMPOSE_PROMPT = """ \n
\n ------- \n
And here some reference documents that show you what type of entities, relationships,
and terms you may want to consider toask about as relevamt to your initial question.
and terms you may want to consider to be as relevant to your initial question.
\n ------- \n
{docs_str}
\n ------- \n
@ -395,7 +395,7 @@ DEEP_DECOMPOSE_PROMPT_WITH_ENTITIES = """ \n
- The sub-questions should be specific to the question and provide richer context for the question,
resolve ambiguities, or address shortcoming of the initial answer
- Each sub-question - when answered - should be relevant for the answer to the original question
- The sub-questions should be free from comparisions, ambiguities,judgements, aggregations, or any
- The sub-questions should be free from comparisons, ambiguities,judgements, aggregations, or any
other complications that may require extra context.
- The sub-questions MUST have the full context of the original question so that it can be executed by
a RAG system independently without the original question available
@ -466,7 +466,7 @@ DECOMPOSE_PROMPT = """ \n
- The sub-questions should be specific to the question and provide richer context for the question,
and or resolve ambiguities
- Each sub-question - when answered - should be relevant for the answer to the original question
- The sub-questions should be free from comparisions, ambiguities,judgements, aggregations, or any
- The sub-questions should be free from comparisons, ambiguities,judgements, aggregations, or any
other complications that may require extra context.
- The sub-questions MUST have the full context of the original question so that it can be executed by
a RAG system independently without the original question available
@ -522,7 +522,7 @@ SUB_QUESTION_EXPLANATION_RANKER_PROMPT = """-------
Please rank the motivations in order of relevance for answering the original question. Also, try to
ensure that the top questions do not duplicate too much, i.e. that they are not too similar.
Ultimately, create a list with the motivation numbers where the number of the most relevant
motivations comes first.
motivation comes first.
Here is the original question:
\n\n {original_question} \n\n
@ -579,16 +579,16 @@ answer the original question. The purpose for this decomposition may be to
'what do we do to improve stability of product X', ...])
4) research an area that could really help to answer the question. (But clarifications or disambiguations are more important.)
Here are some other ruleds:
Here are some other rules:
1) To give you some context, you will see below also some documents that relate to the question. Please only
use this information to learn what the question is approximately asking about, but do not focus on the details
to construct the sub-questions! Also, some of the entities, relationships and terms that are in the dataset may
not be in these few documents, so DO NOT focusd too much on the documents when constructing the sub-questions! Decomposition and
disambiguation are most important!
not be in these few documents, so DO NOT focussed too much on the documents when constructing the sub-questions! Decomposition and
disambiguations are most important!
2) If you think that a decomposition is not needed or helpful, please just return an empty string. That is very much ok too.
Here are the sampple docs to give you some context:
Here are the sample docs to give you some context:
-------
{sample_doc_str}
-------
@ -610,11 +610,11 @@ Answer:"""
INITIAL_DECOMPOSITION_PROMPT = """ \n
Please decompose an initial user question into 2 or 3 appropriate sub-questions that help to
answer the original question. The purpose for this decomposition is to isolate individulal entities
answer the original question. The purpose for this decomposition is to isolate individual entities
(i.e., 'compare sales of company A and company B' -> 'what are sales for company A' + 'what are sales
for company B'), split ambiguous terms (i.e., 'what is our success with company A' -> 'what are our
sales with company A' + 'what is our market share with company A' + 'is company A a reference customer
for us'), etc. Each sub-question should be realistically be answerable by a good RAG system. \n
for us'), etc. Each sub-question should realistically be answerable by a good RAG system. \n
For each sub-question, please also create one search term that can be used to retrieve relevant
documents from a document store.
@ -659,14 +659,14 @@ Answer:"""
AGENT_DECISION_PROMPT = """
You are an large language model assistant helping users address their information needs. You are tasked with deciding
You are a large language model assistant helping users address their information needs. You are tasked with deciding
whether to use a thorough agent search ('research') of a document store to answer a question or request, or whether you want to
address the question or request yourself as an LLM.
Here are some rules:
- If you think that a thorough search through a document store will help answer the question
or address the request, you should choose the 'research' option.
- If the question asks you do do somethng ('please create...', 'write for me...', etc.), you should choose the 'LLM' option.
- If the question asks you do something ('please create...', 'write for me...', etc.), you should choose the 'LLM' option.
- If you think the question is very general and does not refer to a contents of a document store, you should choose
the 'LLM' option.
- Otherwise, you should choose the 'research' option.
@ -683,7 +683,7 @@ Please decide whether to use the agent search or the LLM to answer the question.
Answer:"""
AGENT_DECISION_PROMPT_AFTER_SEARCH = """
You are an large language model assistant helping users address their information needs. You are given an initial question
You are a large language model assistant helping users address their information needs. You are given an initial question
or request and very few sample of documents that a preliminary and fast search from a document store returned.
You are tasked with deciding whether to use a thorough agent search ('research') of the document store to answer a question
or request, or whether you want to address the question or request yourself as an LLM.
@ -693,7 +693,7 @@ Here are some rules:
store to answer or materially help with the request, you should choose the 'research' option.
- If you think that the retrieved document do not help to answer the question or do not help with the request, AND
you know the answer/can handle the request, you should choose the 'LLM' option.
- If the question asks you do do somethng ('please create...', 'write for me...', etc.), you should choose the 'LLM' option.
- If the question asks you do something ('please create...', 'write for me...', etc.), you should choose the 'LLM' option.
- If in doubt, choose the 'research' option.
{history}
@ -747,7 +747,7 @@ Use the information provided below - and only the provided information - to answ
The information provided below consists of:
1) a number of sub-questions and supporting document information that would help answer them.
2) a broader collection of documents that were deemed relevant for the question. These documents contain informattion
2) a broader collection of documents that were deemed relevant for the question. These documents contain information
that was also provided in the sub-questions and often more.
IMPORTANT RULES:
@ -759,7 +759,7 @@ IMPORTANT RULES:
- If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.
- The answers to the subquestions should help you to structure your thoughts in order to answer the question.
It is critical that you provide proper inline citations of documentsin the format [[D1]](), [[D2]](), [[D3]](), etc.!
It is critical that you provide proper inline citations of documents in the format [[D1]](), [[D2]](), [[D3]](), etc.!
It is important that the citation is close to the information it supports. If you have multiple citations,
please cite for example as [[D1]]()[[D3]](), or [[D2]]()[[D4]](), etc. Feel free to cite documents in addition
to the sub-questions! Proper citations are important for the final answer to be verifiable! \n\n\n
@ -857,8 +857,8 @@ INITIAL_RAG_PROMPT_NO_SUB_QUESTIONS = (
"""{answered_sub_questions}
{persona_specification}
{date_prompt}
Use the information provided below
- and only the provided information - to answer the provided question.
Use the information provided below - and only the provided information - to answer the provided question.
The information provided below consists of a number of documents that were deemed relevant for the question.
{history}
@ -898,19 +898,24 @@ REVISED_RAG_PROMPT = (
"""\n
{persona_specification}
{date_prompt}
Use the information provided below - and only the provided information - to answer the provided main question.
Your task is to improve on a given answer to a question, as the initial answer was found to be lacking in some way.
Use the information provided below - and only the provided information - to write your new and improved answer.
The information provided below consists of:
1) an initial answer that was given but found to be lacking in some way.
2) a number of answered sub-questions - these are very important(!) and definitely should help yoiu to answer
2) a number of answered sub-questions - these are very important(!) and definitely should help you to answer
the main question. Note that the sub-questions have a type, 'initial' and 'revised'. The 'initial'
ones were available for the initial answer, and the 'revised' were not. So please use the 'revised' sub-questions in
particular to update/extend/correct the initial answer!
3) a number of documents that were deemed relevant for the question. This the is the context that you use largey for
citations (see below).
ones were available for the initial answer, and the 'revised' were not, they are new. So please use
the 'revised' sub-questions in particular to update/extend/correct the initial answer!
3) a number of documents that were deemed relevant for the question. This the is the context that you use largely for
citations (see below). So consider the answers to the sub-questions as guidelines to construct your new answer, but
make sure you cite the relevant document for a fact!
It is critical that you provide proper inline citations to documents in the format [[D1]](), [[D2]](), [[D3]](), etc!
It is important that the citation is close to the information it supports. If you have multiple
It is important that the citation is close to the information it supports. If you have multiple
citations, please cite for example as [[D1]]()[[D3]](), or [[D2]]()[[D4]](), etc.
Feel free to also cite sub-questions in addition to documents, but make sure that you have documents cited with the sub-question
citation. If you want to cite both a document and a sub-question, please use [[D1]]()[[Q3]](), or [[D2]]()[[D7]]()[[Q4]](), etc.
@ -925,9 +930,9 @@ IMPORTANT RULES:
- If the information is empty or irrelevant, just say """
+ f'"{UNKNOWN_ANSWER}"'
+ """.
- If the information is relevant but not fully conclusive, provide and answer to the extent you can but also
- If the information is relevant but not fully conclusive, provide an answer to the extent you can but also
specify that the information is not conclusive and why.
- Ignore any exisiting citations within the answered sub-questions, like [[D1]]()... and [[Q2]]()!
- Ignore any existing citations within the answered sub-questions, like [[D1]]()... and [[Q2]]()!
The citations you will need to use will need to refer to the documents (and sub-questions) that you are explicitly
presented with below!
@ -942,7 +947,7 @@ Here is the contextual information:
*Initial Answer that was found to be lacking:
{initial_answer}
*Answered Sub-questions (these should really help ypu to research your answer! They also contain questions/answers
*Answered Sub-questions (these should really help you to research your answer! They also contain questions/answers
that were not available when the original answer was constructed):
{answered_sub_questions}
@ -1017,7 +1022,7 @@ Answer:"""
ENTITY_TERM_PROMPT = """ \n
Based on the original question and some context retieved from a dataset, please generate a list of
Based on the original question and some context retrieved from a dataset, please generate a list of
entities (e.g. companies, organizations, industries, products, locations, etc.), terms and concepts
(e.g. sales, revenue, etc.) that are relevant for the question, plus their relations to each other.