From 10be91a8cca91b3454f802f215dde06800543c2f Mon Sep 17 00:00:00 2001 From: Yuhong Sun Date: Sun, 5 May 2024 16:02:42 -0700 Subject: [PATCH] Track Slack questions Autoresolved (#86) --- backend/ee/danswer/db/analytics.py | 94 ++++++++++++++++++- backend/ee/danswer/server/analytics/api.py | 40 +++++++- web/src/app/ee/admin/performance/lib.ts | 14 +++ .../performance/usage/DanswerBotChart.tsx | 78 +++++++++++++++ .../app/ee/admin/performance/usage/page.tsx | 2 + .../app/ee/admin/performance/usage/types.ts | 6 ++ 6 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx diff --git a/backend/ee/danswer/db/analytics.py b/backend/ee/danswer/db/analytics.py index a0b623646..e0eff7850 100644 --- a/backend/ee/danswer/db/analytics.py +++ b/backend/ee/danswer/db/analytics.py @@ -6,6 +6,7 @@ from sqlalchemy import case from sqlalchemy import cast from sqlalchemy import Date from sqlalchemy import func +from sqlalchemy import or_ from sqlalchemy import select from sqlalchemy.orm import Session @@ -16,9 +17,9 @@ from danswer.db.models import ChatSession def fetch_query_analytics( - db_session: Session, start: datetime.datetime, end: datetime.datetime, + db_session: Session, ) -> Sequence[tuple[int, int, int, datetime.date]]: stmt = ( select( @@ -51,9 +52,9 @@ def fetch_query_analytics( def fetch_per_user_query_analytics( - db_session: Session, start: datetime.datetime, end: datetime.datetime, + db_session: Session, ) -> Sequence[tuple[int, int, int, datetime.date, UUID]]: stmt = ( select( @@ -80,3 +81,92 @@ def fetch_per_user_query_analytics( ) return db_session.execute(stmt).all() # type: ignore + + +def fetch_danswerbot_analytics( + start: datetime.datetime, + end: datetime.datetime, + db_session: Session, +) -> Sequence[tuple[int, int, datetime.date]]: + """Gets the: + Date of each set of aggregated statistics + Number of DanswerBot Queries (Chat Sessions) + Number of instances of Negative feedback OR Needing additional help + (only counting the last feedback) + """ + # Get every chat session in the time range which is a Danswerbot flow + # along with the first Assistant message which is the response to the user question. + # Generally there should not be more than one AI message per chat session of this type + subquery_first_ai_response = ( + db_session.query( + ChatMessage.chat_session_id.label("chat_session_id"), + func.min(ChatMessage.id).label("chat_message_id"), + ) + .join(ChatSession, ChatSession.id == ChatMessage.chat_session_id) + .where( + ChatSession.time_created >= start, + ChatSession.time_created <= end, + ChatSession.danswerbot_flow.is_(True), + ) + .where( + ChatMessage.message_type == MessageType.ASSISTANT, + ) + .group_by(ChatMessage.chat_session_id) + .subquery() + ) + + # Get the chat message ids and most recent feedback for each of those chat messages, + # not including the messages that have no feedback + subquery_last_feedback = ( + db_session.query( + ChatMessageFeedback.chat_message_id.label("chat_message_id"), + func.max(ChatMessageFeedback.id).label("max_feedback_id"), + ) + .group_by(ChatMessageFeedback.chat_message_id) + .subquery() + ) + + results = ( + db_session.query( + func.count(ChatSession.id).label("total_sessions"), + # Need to explicitly specify this as False to handle the NULL case so the cases without + # feedback aren't counted against Danswerbot + func.sum( + case( + ( + or_( + ChatMessageFeedback.is_positive.is_(False), + ChatMessageFeedback.required_followup, + ), + 1, + ), + else_=0, + ) + ).label("negative_answer"), + cast(ChatSession.time_created, Date).label("session_date"), + ) + .join( + subquery_first_ai_response, + ChatSession.id == subquery_first_ai_response.c.chat_session_id, + ) + # Combine the chat sessions with latest feedback to get the latest feedback for the first AI + # message of the chat session where the chat session is Danswerbot type and within the time + # range specified. Left/outer join used here to ensure that if no feedback, a null is used + # for the feedback id + .outerjoin( + subquery_last_feedback, + subquery_first_ai_response.c.chat_message_id + == subquery_last_feedback.c.chat_message_id, + ) + # Join the actual feedback table to get the feedback info for the sums + # Outer join because the "last feedback" may be null + .outerjoin( + ChatMessageFeedback, + ChatMessageFeedback.id == subquery_last_feedback.c.max_feedback_id, + ) + .group_by(cast(ChatSession.time_created, Date)) + .order_by(cast(ChatSession.time_created, Date)) + .all() + ) + + return results diff --git a/backend/ee/danswer/server/analytics/api.py b/backend/ee/danswer/server/analytics/api.py index 72910446e..19415e506 100644 --- a/backend/ee/danswer/server/analytics/api.py +++ b/backend/ee/danswer/server/analytics/api.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import Session import danswer.db.models as db_models from danswer.auth.users import current_admin_user from danswer.db.engine import get_session +from ee.danswer.db.analytics import fetch_danswerbot_analytics from ee.danswer.db.analytics import fetch_per_user_query_analytics from ee.danswer.db.analytics import fetch_query_analytics @@ -30,12 +31,12 @@ def get_query_analytics( db_session: Session = Depends(get_session), ) -> list[QueryAnalyticsResponse]: daily_query_usage_info = fetch_query_analytics( - db_session=db_session, start=start or ( datetime.datetime.utcnow() - datetime.timedelta(days=30) ), # default is 30d lookback end=end or datetime.datetime.utcnow(), + db_session=db_session, ) return [ QueryAnalyticsResponse( @@ -61,12 +62,12 @@ def get_user_analytics( db_session: Session = Depends(get_session), ) -> list[UserAnalyticsResponse]: daily_query_usage_info_per_user = fetch_per_user_query_analytics( - db_session=db_session, start=start or ( datetime.datetime.utcnow() - datetime.timedelta(days=30) ), # default is 30d lookback end=end or datetime.datetime.utcnow(), + db_session=db_session, ) user_analytics: dict[datetime.date, int] = defaultdict(int) @@ -79,3 +80,38 @@ def get_user_analytics( ) for date, cnt in user_analytics.items() ] + + +class DanswerbotAnalyticsResponse(BaseModel): + total_queries: int + auto_resolved: int + date: datetime.date + + +@router.get("/admin/danswerbot") +def get_danswerbot_analytics( + start: datetime.datetime | None = None, + end: datetime.datetime | None = None, + _: db_models.User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[DanswerbotAnalyticsResponse]: + daily_danswerbot_info = fetch_danswerbot_analytics( + start=start + or ( + datetime.datetime.utcnow() - datetime.timedelta(days=30) + ), # default is 30d lookback + end=end or datetime.datetime.utcnow(), + db_session=db_session, + ) + + resolution_results = [ + DanswerbotAnalyticsResponse( + total_queries=total_queries, + # If it hits negatives, something has gone wrong... + auto_resolved=max(0, total_queries - total_negatives), + date=date, + ) + for total_queries, total_negatives, date in daily_danswerbot_info + ] + + return resolution_results diff --git a/web/src/app/ee/admin/performance/lib.ts b/web/src/app/ee/admin/performance/lib.ts index 38e972ae9..cd0e56600 100644 --- a/web/src/app/ee/admin/performance/lib.ts +++ b/web/src/app/ee/admin/performance/lib.ts @@ -2,6 +2,7 @@ import { errorHandlingFetcher } from "@/lib/fetcher"; import useSWR, { mutate } from "swr"; import { ChatSessionSnapshot, + DanswerBotAnalytics, QueryAnalytics, UserAnalytics, } from "./usage/types"; @@ -50,6 +51,19 @@ export const useUserAnalytics = (timeRange: DateRangePickerValue) => { }; }; +export const useDanswerBotAnalytics = (timeRange: DateRangePickerValue) => { + const url = buildApiPath("/api/analytics/admin/danswerbot", { + start: convertDateToStartOfDay(timeRange.from)?.toISOString(), + end: convertDateToEndOfDay(timeRange.to)?.toISOString(), + }); + const swrResponse = useSWR(url, errorHandlingFetcher); // TODO + + return { + ...swrResponse, + refreshDanswerBotAnalytics: () => mutate(url), + }; +}; + export const useQueryHistory = () => { const [selectedFeedbackType, setSelectedFeedbackType] = useState(null); diff --git a/web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx b/web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx new file mode 100644 index 000000000..9d15abcf5 --- /dev/null +++ b/web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx @@ -0,0 +1,78 @@ +import { ThreeDotsLoader } from "@/components/Loading"; +import { getDatesList, useDanswerBotAnalytics } from "../lib"; +import { + AreaChart, + Card, + Title, + Text, + DateRangePickerValue, +} from "@tremor/react"; + +export function DanswerBotChart({ + timeRange, +}: { + timeRange: DateRangePickerValue; +}) { + const { + data: danswerBotAnalyticsData, + isLoading: isDanswerBotAnalyticsLoading, + error: danswerBotAnalyticsError, + } = useDanswerBotAnalytics(timeRange); + + let chart; + if (isDanswerBotAnalyticsLoading) { + chart = ( +
+ +
+ ); + } else if (!danswerBotAnalyticsData || danswerBotAnalyticsError) { + chart = ( +
+

Failed to fetch feedback data...

+
+ ); + } else { + const initialDate = + timeRange.from || new Date(danswerBotAnalyticsData[0].date); + const dateRange = getDatesList(initialDate); + + const dateToDanswerBotAnalytics = new Map( + danswerBotAnalyticsData.map((danswerBotAnalyticsEntry) => [ + danswerBotAnalyticsEntry.date, + danswerBotAnalyticsEntry, + ]) + ); + + chart = ( + { + const danswerBotAnalyticsForDate = + dateToDanswerBotAnalytics.get(dateStr); + return { + Day: dateStr, + "Total Queries": danswerBotAnalyticsForDate?.total_queries || 0, + "Automatically Resolved": + danswerBotAnalyticsForDate?.auto_resolved || 0, + }; + })} + categories={["Total Queries", "Automatically Resolved"]} + index="Day" + colors={["indigo", "fuchsia"]} + valueFormatter={(number: number) => + `${Intl.NumberFormat("us").format(number).toString()}` + } + yAxisWidth={60} + /> + ); + } + + return ( + + Slack Bot + Total Queries vs Auto Resolved + {chart} + + ); +} diff --git a/web/src/app/ee/admin/performance/usage/page.tsx b/web/src/app/ee/admin/performance/usage/page.tsx index 6e7751c71..b47a9a73c 100644 --- a/web/src/app/ee/admin/performance/usage/page.tsx +++ b/web/src/app/ee/admin/performance/usage/page.tsx @@ -1,6 +1,7 @@ "use client"; import { DateRangeSelector } from "../DateRangeSelector"; +import { DanswerBotChart } from "./DanswerBotChart"; import { FeedbackChart } from "./FeedbackChart"; import { QueryPerformanceChart } from "./QueryPerformanceChart"; import { BarChartIcon } from "@/components/icons/icons"; @@ -23,6 +24,7 @@ export default function AnalyticsPage() { + ); } diff --git a/web/src/app/ee/admin/performance/usage/types.ts b/web/src/app/ee/admin/performance/usage/types.ts index 1c7c8e0e9..72a9b19c5 100644 --- a/web/src/app/ee/admin/performance/usage/types.ts +++ b/web/src/app/ee/admin/performance/usage/types.ts @@ -12,6 +12,12 @@ export interface UserAnalytics { date: string; } +export interface DanswerBotAnalytics { + total_queries: number; + auto_resolved: number; + date: string; +} + export interface AbridgedSearchDoc { document_id: string; semantic_identifier: string;