From 7c8f8dba17b22fd800c114f741944a17d1657704 Mon Sep 17 00:00:00 2001 From: Yuhong Sun Date: Tue, 9 Jul 2024 18:18:29 -0700 Subject: [PATCH] Break the Danswer LLM logging from LiteLLM Verbose (#1795) --- .vscode/env_template.txt | 2 +- .vscode/launch.template.jsonc | 2 +- backend/danswer/configs/app_configs.py | 6 +++++- backend/danswer/llm/chat_llm.py | 7 ++++--- deployment/docker_compose/docker-compose.dev.yml | 8 ++++++-- deployment/docker_compose/docker-compose.gpu-dev.yml | 8 ++++++-- deployment/helm/values.yaml | 1 + deployment/kubernetes/env-configmap.yaml | 1 + 8 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.vscode/env_template.txt b/.vscode/env_template.txt index 6ebf500c3..015672a22 100644 --- a/.vscode/env_template.txt +++ b/.vscode/env_template.txt @@ -8,7 +8,7 @@ AUTH_TYPE=disabled # Always keep these on for Dev # Logs all model prompts to stdout -LOG_ALL_MODEL_INTERACTIONS=True +LOG_DANSWER_MODEL_INTERACTIONS=True # More verbose logging LOG_LEVEL=debug diff --git a/.vscode/launch.template.jsonc b/.vscode/launch.template.jsonc index 719693c4b..19bfc513b 100644 --- a/.vscode/launch.template.jsonc +++ b/.vscode/launch.template.jsonc @@ -49,7 +49,7 @@ "cwd": "${workspaceFolder}/backend", "envFile": "${workspaceFolder}/.env", "env": { - "LOG_ALL_MODEL_INTERACTIONS": "True", + "LOG_DANSWER_MODEL_INTERACTIONS": "True", "LOG_LEVEL": "DEBUG", "PYTHONUNBUFFERED": "1" }, diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py index c40c661b4..47cfee37f 100644 --- a/backend/danswer/configs/app_configs.py +++ b/backend/danswer/configs/app_configs.py @@ -266,10 +266,14 @@ JOB_TIMEOUT = 60 * 60 * 6 # 6 hours default CURRENT_PROCESS_IS_AN_INDEXING_JOB = ( os.environ.get("CURRENT_PROCESS_IS_AN_INDEXING_JOB", "").lower() == "true" ) -# Logs every model prompt and output, mostly used for development or exploration purposes +# Sets LiteLLM to verbose logging LOG_ALL_MODEL_INTERACTIONS = ( os.environ.get("LOG_ALL_MODEL_INTERACTIONS", "").lower() == "true" ) +# Logs Danswer only model interactions like prompts, responses, messages etc. +LOG_DANSWER_MODEL_INTERACTIONS = ( + os.environ.get("LOG_DANSWER_MODEL_INTERACTIONS", "").lower() == "true" +) # If set to `true` will enable additional logs about Vespa query performance # (time spent on finding the right docs + time spent fetching summaries from disk) LOG_VESPA_TIMING_INFORMATION = ( diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py index 2f60708bb..90ad481d4 100644 --- a/backend/danswer/llm/chat_llm.py +++ b/backend/danswer/llm/chat_llm.py @@ -23,6 +23,7 @@ from langchain_core.messages.tool import ToolCallChunk from langchain_core.messages.tool import ToolMessage from danswer.configs.app_configs import LOG_ALL_MODEL_INTERACTIONS +from danswer.configs.app_configs import LOG_DANSWER_MODEL_INTERACTIONS from danswer.configs.model_configs import DISABLE_LITELLM_STREAMING from danswer.configs.model_configs import GEN_AI_API_ENDPOINT from danswer.configs.model_configs import GEN_AI_API_VERSION @@ -316,7 +317,7 @@ class DefaultMultiLLM(LLM): tools: list[dict] | None = None, tool_choice: ToolChoiceOptions | None = None, ) -> BaseMessage: - if LOG_ALL_MODEL_INTERACTIONS: + if LOG_DANSWER_MODEL_INTERACTIONS: self.log_model_configs() self._log_prompt(prompt) @@ -333,7 +334,7 @@ class DefaultMultiLLM(LLM): tools: list[dict] | None = None, tool_choice: ToolChoiceOptions | None = None, ) -> Iterator[BaseMessage]: - if LOG_ALL_MODEL_INTERACTIONS: + if LOG_DANSWER_MODEL_INTERACTIONS: self.log_model_configs() self._log_prompt(prompt) @@ -361,7 +362,7 @@ class DefaultMultiLLM(LLM): "The AI model failed partway through generation, please try again." ) - if LOG_ALL_MODEL_INTERACTIONS and output: + if LOG_DANSWER_MODEL_INTERACTIONS and output: content = output.content or "" if isinstance(output, AIMessage): if content: diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml index 20c6bbc8b..78654d7f5 100644 --- a/deployment/docker_compose/docker-compose.dev.yml +++ b/deployment/docker_compose/docker-compose.dev.yml @@ -83,7 +83,9 @@ services: # https://docs.danswer.dev/more/telemetry - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-} - LOG_LEVEL=${LOG_LEVEL:-info} # Set to debug to get more fine-grained logs - - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # Log all of the prompts to the LLM + - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # LiteLLM Verbose Logging + # Log all of Danswer prompts and interactions with the LLM + - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-} # If set to `true` will enable additional logs about Vespa query performance # (time spent on finding the right docs + time spent fetching summaries from disk) - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-} @@ -191,7 +193,9 @@ services: # https://docs.danswer.dev/more/telemetry - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-} - LOG_LEVEL=${LOG_LEVEL:-info} # Set to debug to get more fine-grained logs - - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # Log all of the prompts to the LLM + - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # LiteLLM Verbose Logging + # Log all of Danswer prompts and interactions with the LLM + - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-} - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-} # Enterprise Edition stuff diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml index 82cfd6b40..71f0f920b 100644 --- a/deployment/docker_compose/docker-compose.gpu-dev.yml +++ b/deployment/docker_compose/docker-compose.gpu-dev.yml @@ -79,7 +79,9 @@ services: # https://docs.danswer.dev/more/telemetry - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-} - LOG_LEVEL=${LOG_LEVEL:-info} # Set to debug to get more fine-grained logs - - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # Log all of the prompts to the LLM + - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # LiteLLM Verbose Logging + # Log all of Danswer prompts and interactions with the LLM + - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-} # If set to `true` will enable additional logs about Vespa query performance # (time spent on finding the right docs + time spent fetching summaries from disk) - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-} @@ -183,7 +185,9 @@ services: # https://docs.danswer.dev/more/telemetry - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-} - LOG_LEVEL=${LOG_LEVEL:-info} # Set to debug to get more fine-grained logs - - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # Log all of the prompts to the LLM + - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # LiteLLM Verbose Logging + # Log all of Danswer prompts and interactions with the LLM + - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-} - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-} # Enterprise Edition only diff --git a/deployment/helm/values.yaml b/deployment/helm/values.yaml index c16f19719..b844b5be0 100644 --- a/deployment/helm/values.yaml +++ b/deployment/helm/values.yaml @@ -449,6 +449,7 @@ configMap: DISABLE_TELEMETRY: "" LOG_LEVEL: "" LOG_ALL_MODEL_INTERACTIONS: "" + LOG_DANSWER_MODEL_INTERACTIONS: "" LOG_VESPA_TIMING_INFORMATION: "" # Shared or Non-backend Related WEB_DOMAIN: "http://localhost:3000" # for web server and api server diff --git a/deployment/kubernetes/env-configmap.yaml b/deployment/kubernetes/env-configmap.yaml index 42451a24a..da2f7e589 100644 --- a/deployment/kubernetes/env-configmap.yaml +++ b/deployment/kubernetes/env-configmap.yaml @@ -78,6 +78,7 @@ data: DISABLE_TELEMETRY: "" LOG_LEVEL: "" LOG_ALL_MODEL_INTERACTIONS: "" + LOG_DANSWER_MODEL_INTERACTIONS: "" LOG_VESPA_TIMING_INFORMATION: "" # Shared or Non-backend Related INTERNAL_URL: "http://api-server-service:80" # for web server