From a95f4298adf727b8daf9f2ab6ddfaef680a741a0 Mon Sep 17 00:00:00 2001 From: hagen-danswer Date: Tue, 7 Jan 2025 13:53:08 -0800 Subject: [PATCH] Improved logging for confluence calls (#3622) * Improved logging for confluence calls * cleanup * idk * combined logging --- .../confluence/group_sync.py | 1 + .../connectors/confluence/onyx_confluence.py | 29 +++++++++++++++++-- backend/onyx/connectors/confluence/utils.py | 10 ++++--- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/backend/ee/onyx/external_permissions/confluence/group_sync.py b/backend/ee/onyx/external_permissions/confluence/group_sync.py index c43783340..e57e6a976 100644 --- a/backend/ee/onyx/external_permissions/confluence/group_sync.py +++ b/backend/ee/onyx/external_permissions/confluence/group_sync.py @@ -30,6 +30,7 @@ def _build_group_member_email_map( ) if not email: # If we still don't have an email, skip this user + logger.warning(f"user result missing email field: {user_result}") continue for group in confluence_client.paginated_groups_by_user_retrieval(user): diff --git a/backend/onyx/connectors/confluence/onyx_confluence.py b/backend/onyx/connectors/confluence/onyx_confluence.py index 1ae765dca..ea8a7a67e 100644 --- a/backend/onyx/connectors/confluence/onyx_confluence.py +++ b/backend/onyx/connectors/confluence/onyx_confluence.py @@ -186,9 +186,18 @@ class OnyxConfluence(Confluence): url_suffix += f"{connection_char}limit={limit}" while url_suffix: + logger.debug(f"Making confluence call to {url_suffix}") try: - logger.debug(f"Making confluence call to {url_suffix}") - next_response = self.get(url_suffix) + raw_response = self.get( + path=url_suffix, + advanced_mode=True, + ) + except Exception as e: + logger.exception(f"Error in confluence call to {url_suffix}") + raise e + + try: + raw_response.raise_for_status() except Exception as e: logger.warning(f"Error in confluence call to {url_suffix}") @@ -196,8 +205,14 @@ class OnyxConfluence(Confluence): # with the replacement expansion and try again # If that fails, raise the error if _PROBLEMATIC_EXPANSIONS not in url_suffix: - logger.exception(f"Error in confluence call to {url_suffix}") + logger.exception( + f"Error in confluence call to {url_suffix} \n" + f"Raw Response Text: {raw_response.text} \n" + f"Full Response: {raw_response.__dict__} \n" + f"Error: {e} \n" + ) raise e + logger.warning( f"Replacing {_PROBLEMATIC_EXPANSIONS} with {_REPLACEMENT_EXPANSIONS}" " and trying again." @@ -208,6 +223,14 @@ class OnyxConfluence(Confluence): ) continue + try: + next_response = raw_response.json() + except Exception as e: + logger.exception( + f"Failed to parse response as JSON. Response: {raw_response.__dict__}" + ) + raise e + # yield the results individually yield from next_response.get("results", []) diff --git a/backend/onyx/connectors/confluence/utils.py b/backend/onyx/connectors/confluence/utils.py index fcb9474c7..49fe60a94 100644 --- a/backend/onyx/connectors/confluence/utils.py +++ b/backend/onyx/connectors/confluence/utils.py @@ -32,11 +32,13 @@ def get_user_email_from_username__server( response = confluence_client.get_mobile_parameters(user_name) email = response.get("email") except Exception: - # For now, we'll just return a string that indicates failure - # We may want to revert to returning None in the future - # email = None - email = f"FAILED TO GET CONFLUENCE EMAIL FOR {user_name}" logger.warning(f"failed to get confluence email for {user_name}") + # For now, we'll just return None and log a warning. This means + # we will keep retrying to get the email every group sync. + email = None + # We may want to just return a string that indicates failure so we dont + # keep retrying + # email = f"FAILED TO GET CONFLUENCE EMAIL FOR {user_name}" _USER_EMAIL_CACHE[user_name] = email return _USER_EMAIL_CACHE[user_name]