Improved logging for confluence calls (#3622)

* Improved logging for confluence calls

* cleanup

* idk

* combined logging

backend/ee/onyx/external_permissions/confluence/group_sync.py

@@ -30,6 +30,7 @@ def _build_group_member_email_map(
                 )
         if not email:
             # If we still don't have an email, skip this user
+            logger.warning(f"user result missing email field: {user_result}")
             continue
 
         for group in confluence_client.paginated_groups_by_user_retrieval(user):

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -186,9 +186,18 @@ class OnyxConfluence(Confluence):
         url_suffix += f"{connection_char}limit={limit}"
 
         while url_suffix:
+            logger.debug(f"Making confluence call to {url_suffix}")
             try:
-                logger.debug(f"Making confluence call to {url_suffix}")
+                raw_response = self.get(
-                next_response = self.get(url_suffix)
+                    path=url_suffix,
+                    advanced_mode=True,
+                )
+            except Exception as e:
+                logger.exception(f"Error in confluence call to {url_suffix}")
+                raise e
+
+            try:
+                raw_response.raise_for_status()
             except Exception as e:
                 logger.warning(f"Error in confluence call to {url_suffix}")
 
@@ -196,8 +205,14 @@ class OnyxConfluence(Confluence):
                 # with the replacement expansion and try again
                 # If that fails, raise the error
                 if _PROBLEMATIC_EXPANSIONS not in url_suffix:
-                    logger.exception(f"Error in confluence call to {url_suffix}")
+                    logger.exception(
+                        f"Error in confluence call to {url_suffix} \n"
+                        f"Raw Response Text: {raw_response.text} \n"
+                        f"Full Response: {raw_response.__dict__} \n"
+                        f"Error: {e} \n"
+                    )
                     raise e
+
                 logger.warning(
                     f"Replacing {_PROBLEMATIC_EXPANSIONS} with {_REPLACEMENT_EXPANSIONS}"
                     " and trying again."
@@ -208,6 +223,14 @@ class OnyxConfluence(Confluence):
                 )
                 continue
 
+            try:
+                next_response = raw_response.json()
+            except Exception as e:
+                logger.exception(
+                    f"Failed to parse response as JSON. Response: {raw_response.__dict__}"
+                )
+                raise e
+
             # yield the results individually
             yield from next_response.get("results", [])
 

backend/onyx/connectors/confluence/utils.py

@@ -32,11 +32,13 @@ def get_user_email_from_username__server(
             response = confluence_client.get_mobile_parameters(user_name)
             email = response.get("email")
         except Exception:
-            # For now, we'll just return a string that indicates failure
-            # We may want to revert to returning None in the future
-            # email = None
-            email = f"FAILED TO GET CONFLUENCE EMAIL FOR {user_name}"
             logger.warning(f"failed to get confluence email for {user_name}")
+            # For now, we'll just return None and log a warning. This means
+            # we will keep retrying to get the email every group sync.
+            email = None
+            # We may want to just return a string that indicates failure so we dont
+            # keep retrying
+            # email = f"FAILED TO GET CONFLUENCE EMAIL FOR {user_name}"
         _USER_EMAIL_CACHE[user_name] = email
     return _USER_EMAIL_CACHE[user_name]