highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-07-21 02:23:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

add SSL parameter support for redis (#2389)

Browse Source 
* add SSL parameter support for redis

* add ssl support to redis pool
This commit is contained in:
rkuo-danswer
2024-09-12 09:18:11 -07:00
committed by GitHub
parent 7af572d0e7
commit 4bc4da29f5
3 changed files with 42 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
backend/danswer/background/celery/celeryconfig.py
View File

@ -3,6 +3,9 @@ from danswer.configs.app_configs import REDIS_DB_NUMBER_CELERY
from danswer.configs.app_configs import REDIS_HOST from danswer.configs.app_configs import REDIS_HOST
from danswer.configs.app_configs import REDIS_PASSWORD from danswer.configs.app_configs import REDIS_PASSWORD
from danswer.configs.app_configs import REDIS_PORT from danswer.configs.app_configs import REDIS_PORT
from danswer.configs.app_configs import REDIS_SSL
from danswer.configs.app_configs import REDIS_SSL_CA_CERTS
from danswer.configs.app_configs import REDIS_SSL_CERT_REQS
from danswer.configs.constants import DanswerCeleryPriority from danswer.configs.constants import DanswerCeleryPriority
CELERY_SEPARATOR = ":" CELERY_SEPARATOR = ":"
@ -11,16 +14,22 @@ CELERY_PASSWORD_PART = ""
if REDIS_PASSWORD: if REDIS_PASSWORD:
CELERY_PASSWORD_PART = f":{REDIS_PASSWORD}@" CELERY_PASSWORD_PART = f":{REDIS_PASSWORD}@"
REDIS_SCHEME = "redis"
# SSL-specific query parameters for Redis URL
SSL_QUERY_PARAMS = ""
if REDIS_SSL:
REDIS_SCHEME == "rediss"
SSL_QUERY_PARAMS = f"?ssl_cert_reqs={REDIS_SSL_CERT_REQS}"
if REDIS_SSL_CA_CERTS:
SSL_QUERY_PARAMS += f"&ssl_ca_certs={REDIS_SSL_CA_CERTS}"
# example celery_broker_url: "redis://:password@localhost:6379/15" # example celery_broker_url: "redis://:password@localhost:6379/15"
broker_url = ( broker_url = f"{REDIS_SCHEME}://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}{SSL_QUERY_PARAMS}"
f"redis://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}"
)
result_backend = ( result_backend = f"{REDIS_SCHEME}://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}{SSL_QUERY_PARAMS}"
f"redis://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}"
)
# NOTE: prefetch 4 is significantly faster than prefetch 1 # NOTE: prefetch 4 is significantly faster than prefetch 1 for small tasks
# however, prefetching is bad when tasks are lengthy as those tasks # however, prefetching is bad when tasks are lengthy as those tasks
# can stall other tasks. # can stall other tasks.
worker_prefetch_multiplier = 4 worker_prefetch_multiplier = 4

4
backend/danswer/configs/app_configs.py
View File

@ -150,6 +150,7 @@ try:
except ValueError: except ValueError:
POSTGRES_POOL_RECYCLE = POSTGRES_POOL_RECYCLE_DEFAULT POSTGRES_POOL_RECYCLE = POSTGRES_POOL_RECYCLE_DEFAULT
REDIS_SSL = os.getenv("REDIS_SSL", "").lower() == "true"
REDIS_HOST = os.environ.get("REDIS_HOST") or "localhost" REDIS_HOST = os.environ.get("REDIS_HOST") or "localhost"
REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379)) REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD") or "" REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD") or ""
@ -160,6 +161,9 @@ REDIS_DB_NUMBER = int(os.environ.get("REDIS_DB_NUMBER", 0))
# Used by celery as broker and backend # Used by celery as broker and backend
REDIS_DB_NUMBER_CELERY = int(os.environ.get("REDIS_DB_NUMBER_CELERY", 15)) REDIS_DB_NUMBER_CELERY = int(os.environ.get("REDIS_DB_NUMBER_CELERY", 15))
REDIS_SSL_CERT_REQS = os.getenv("REDIS_SSL_CERT_REQS", "CERT_NONE")
REDIS_SSL_CA_CERTS = os.getenv("REDIS_SSL_CA_CERTS", "")
##### #####
# Connector Configs # Connector Configs
##### #####

29
backend/danswer/redis/redis_pool.py
View File

@ -9,6 +9,9 @@ from danswer.configs.app_configs import REDIS_DB_NUMBER
from danswer.configs.app_configs import REDIS_HOST from danswer.configs.app_configs import REDIS_HOST
from danswer.configs.app_configs import REDIS_PASSWORD from danswer.configs.app_configs import REDIS_PASSWORD
from danswer.configs.app_configs import REDIS_PORT from danswer.configs.app_configs import REDIS_PORT
from danswer.configs.app_configs import REDIS_SSL
from danswer.configs.app_configs import REDIS_SSL_CA_CERTS
from danswer.configs.app_configs import REDIS_SSL_CERT_REQS
REDIS_POOL_MAX_CONNECTIONS = 10 REDIS_POOL_MAX_CONNECTIONS = 10
@ -27,13 +30,25 @@ class RedisPool:
return cls._instance return cls._instance
def _init_pool(self) -> None: def _init_pool(self) -> None:
self._pool = redis.ConnectionPool( if REDIS_SSL:
host=REDIS_HOST, self._pool = redis.ConnectionPool(
port=REDIS_PORT, host=REDIS_HOST,
db=REDIS_DB_NUMBER, port=REDIS_PORT,
password=REDIS_PASSWORD, db=REDIS_DB_NUMBER,
max_connections=REDIS_POOL_MAX_CONNECTIONS, password=REDIS_PASSWORD,
) max_connections=REDIS_POOL_MAX_CONNECTIONS,
ssl=True,
ssl_ca_certs=REDIS_SSL_CA_CERTS,
ssl_cert_reqs=REDIS_SSL_CERT_REQS,
)
else:
self._pool = redis.ConnectionPool(
host=REDIS_HOST,
port=REDIS_PORT,
db=REDIS_DB_NUMBER,
password=REDIS_PASSWORD,
max_connections=REDIS_POOL_MAX_CONNECTIONS,
)
def get_client(self) -> Redis: def get_client(self) -> Redis:
return redis.Redis(connection_pool=self._pool) return redis.Redis(connection_pool=self._pool)