highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-19 04:20:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix local background jobs (#4241)

Browse Source
This commit is contained in:
pablonyx 2025-03-08 14:47:56 -08:00 committed by GitHub
parent 0d3c72acbf
commit 18df63dfd9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 33 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
backend/onyx/background/celery/memory_monitoring.py
View File

@ -5,40 +5,53 @@ from logging.handlers import RotatingFileHandler
import psutil import psutil
from onyx.utils.logger import is_running_in_container
from onyx.utils.logger import setup_logger from onyx.utils.logger import setup_logger
# Regular application logger # Regular application logger
logger = setup_logger() logger = setup_logger()
# Set up a dedicated memory monitoring logger # Only set up memory monitoring in container environment
MEMORY_LOG_DIR = "/var/log/persisted-logs/memory" if is_running_in_container():
MEMORY_LOG_FILE = os.path.join(MEMORY_LOG_DIR, "memory_usage.log") # Set up a dedicated memory monitoring logger
MEMORY_LOG_MAX_BYTES = 10 * 1024 * 1024 # 10MB MEMORY_LOG_DIR = "/var/log/persisted-logs/memory"
MEMORY_LOG_BACKUP_COUNT = 5 # Keep 5 backup files MEMORY_LOG_FILE = os.path.join(MEMORY_LOG_DIR, "memory_usage.log")
MEMORY_LOG_MAX_BYTES = 10 * 1024 * 1024 # 10MB
MEMORY_LOG_BACKUP_COUNT = 5 # Keep 5 backup files
# Ensure log directory exists # Ensure log directory exists
os.makedirs(MEMORY_LOG_DIR, exist_ok=True) os.makedirs(MEMORY_LOG_DIR, exist_ok=True)
# Create a dedicated logger for memory monitoring # Create a dedicated logger for memory monitoring
memory_logger = logging.getLogger("memory_monitoring") memory_logger = logging.getLogger("memory_monitoring")
memory_logger.setLevel(logging.INFO) memory_logger.setLevel(logging.INFO)
# Create a rotating file handler # Create a rotating file handler
memory_handler = RotatingFileHandler( memory_handler = RotatingFileHandler(
MEMORY_LOG_FILE, maxBytes=MEMORY_LOG_MAX_BYTES, backupCount=MEMORY_LOG_BACKUP_COUNT MEMORY_LOG_FILE,
) maxBytes=MEMORY_LOG_MAX_BYTES,
backupCount=MEMORY_LOG_BACKUP_COUNT,
)
# Create a formatter that includes all relevant information # Create a formatter that includes all relevant information
memory_formatter = logging.Formatter( memory_formatter = logging.Formatter(
"%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S" "%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
) )
memory_handler.setFormatter(memory_formatter) memory_handler.setFormatter(memory_formatter)
memory_logger.addHandler(memory_handler) memory_logger.addHandler(memory_handler)
else:
# Create a null logger when not in container
memory_logger = logging.getLogger("memory_monitoring")
memory_logger.addHandler(logging.NullHandler())
def emit_process_memory( def emit_process_memory(
pid: int, process_name: str, additional_metadata: dict[str, str | int] pid: int, process_name: str, additional_metadata: dict[str, str | int]
) -> None: ) -> None:
# Skip memory monitoring if not in container
if not is_running_in_container():
return
try: try:
process = psutil.Process(pid) process = psutil.Process(pid)
memory_info = process.memory_info() memory_info = process.memory_info()