diff --git a/backend/danswer/configs/model_configs.py b/backend/danswer/configs/model_configs.py
index aa2ffec2e..0c6327ccc 100644
--- a/backend/danswer/configs/model_configs.py
+++ b/backend/danswer/configs/model_configs.py
@@ -75,7 +75,10 @@ INTERNAL_MODEL_VERSION = os.environ.get(
 )
 
 # If the Generative AI model requires an API key for access, otherwise can leave blank
-GEN_AI_API_KEY = os.environ.get("GEN_AI_API_KEY", "")
+GEN_AI_API_KEY = (
+    os.environ.get("GEN_AI_API_KEY", os.environ.get("OPENAI_API_KEY"))
+    or "dummy_llm_key"
+)
 
 # If using GPT4All, HuggingFace Inference API, or OpenAI - specify the model version
 GEN_AI_MODEL_VERSION = os.environ.get(
diff --git a/backend/danswer/direct_qa/qa_utils.py b/backend/danswer/direct_qa/qa_utils.py
index f1dab5ae9..cd754783e 100644
--- a/backend/danswer/direct_qa/qa_utils.py
+++ b/backend/danswer/direct_qa/qa_utils.py
@@ -32,8 +32,9 @@ logger = setup_logger()
 
 
 def get_gen_ai_api_key() -> str:
-    return GEN_AI_API_KEY or cast(
-        str, get_dynamic_config_store().load(GEN_AI_API_KEY_STORAGE_KEY)
+    return (
+        cast(str, get_dynamic_config_store().load(GEN_AI_API_KEY_STORAGE_KEY))
+        or GEN_AI_API_KEY
     )
 
 
diff --git a/backend/danswer/llm/azure.py b/backend/danswer/llm/azure.py
index 973172378..60ee3b409 100644
--- a/backend/danswer/llm/azure.py
+++ b/backend/danswer/llm/azure.py
@@ -1,4 +1,3 @@
-import os
 from typing import Any
 
 from langchain.chat_models.azure_openai import AzureChatOpenAI
@@ -6,6 +5,7 @@ from langchain.chat_models.azure_openai import AzureChatOpenAI
 from danswer.configs.model_configs import API_BASE_OPENAI
 from danswer.configs.model_configs import API_VERSION_OPENAI
 from danswer.configs.model_configs import AZURE_DEPLOYMENT_ID
+from danswer.configs.model_configs import GEN_AI_API_KEY
 from danswer.llm.llm import LangChainChatLLM
 from danswer.llm.utils import should_be_verbose
 
@@ -27,7 +27,7 @@ class AzureGPT(LangChainChatLLM):
         # exception when trying to initialize the LLM which would prevent the API
         # server from starting up
         if not api_key:
-            api_key = os.environ.get("OPENAI_API_KEY") or "dummy_api_key"
+            api_key = GEN_AI_API_KEY
         self._llm = AzureChatOpenAI(
             model=model_version,
             openai_api_type="azure",
diff --git a/backend/danswer/llm/openai.py b/backend/danswer/llm/openai.py
index 06e252a85..638aa134c 100644
--- a/backend/danswer/llm/openai.py
+++ b/backend/danswer/llm/openai.py
@@ -1,9 +1,9 @@
-import os
 from typing import Any
 from typing import cast
 
 from langchain.chat_models.openai import ChatOpenAI
 
+from danswer.configs.model_configs import GEN_AI_API_KEY
 from danswer.configs.model_configs import GEN_AI_TEMPERATURE
 from danswer.llm.llm import LangChainChatLLM
 from danswer.llm.utils import should_be_verbose
@@ -24,7 +24,7 @@ class OpenAIGPT(LangChainChatLLM):
         # exception when trying to initialize the LLM which would prevent the API
         # server from starting up
         if not api_key:
-            api_key = os.environ.get("OPENAI_API_KEY") or "dummy_api_key"
+            api_key = GEN_AI_API_KEY
 
         self._llm = ChatOpenAI(
             model=model_version,