From 14772dee71fa93cd65830fc291f1dc0f71bd356d Mon Sep 17 00:00:00 2001 From: hagen-danswer Date: Thu, 5 Dec 2024 09:15:56 -0800 Subject: [PATCH] Add persona stats (#3282) * Added a chart to display persona message stats * polish * k * hope this works * cleanup --- backend/ee/danswer/db/analytics.py | 64 +++++ backend/ee/danswer/server/analytics/api.py | 82 ++++++- web/src/app/ee/admin/performance/lib.ts | 66 +++++ .../usage/PersonaMessagesChart.tsx | 231 ++++++++++++++++++ .../app/ee/admin/performance/usage/page.tsx | 2 + 5 files changed, 442 insertions(+), 3 deletions(-) create mode 100644 web/src/app/ee/admin/performance/usage/PersonaMessagesChart.tsx diff --git a/backend/ee/danswer/db/analytics.py b/backend/ee/danswer/db/analytics.py index e0eff7850e4b..8d27af06899e 100644 --- a/backend/ee/danswer/db/analytics.py +++ b/backend/ee/danswer/db/analytics.py @@ -170,3 +170,67 @@ def fetch_danswerbot_analytics( ) return results + + +def fetch_persona_message_analytics( + db_session: Session, + persona_id: int, + start: datetime.datetime, + end: datetime.datetime, +) -> list[tuple[int, datetime.date]]: + """Gets the daily message counts for a specific persona within the given time range.""" + query = ( + select( + func.count(ChatMessage.id), + cast(ChatMessage.time_sent, Date), + ) + .join( + ChatSession, + ChatMessage.chat_session_id == ChatSession.id, + ) + .where( + or_( + ChatMessage.alternate_assistant_id == persona_id, + ChatSession.persona_id == persona_id, + ), + ChatMessage.time_sent >= start, + ChatMessage.time_sent <= end, + ChatMessage.message_type == MessageType.ASSISTANT, + ) + .group_by(cast(ChatMessage.time_sent, Date)) + .order_by(cast(ChatMessage.time_sent, Date)) + ) + + return [tuple(row) for row in db_session.execute(query).all()] + + +def fetch_persona_unique_users( + db_session: Session, + persona_id: int, + start: datetime.datetime, + end: datetime.datetime, +) -> list[tuple[int, datetime.date]]: + """Gets the daily unique user counts for a specific persona within the given time range.""" + query = ( + select( + func.count(func.distinct(ChatSession.user_id)), + cast(ChatMessage.time_sent, Date), + ) + .join( + ChatSession, + ChatMessage.chat_session_id == ChatSession.id, + ) + .where( + or_( + ChatMessage.alternate_assistant_id == persona_id, + ChatSession.persona_id == persona_id, + ), + ChatMessage.time_sent >= start, + ChatMessage.time_sent <= end, + ChatMessage.message_type == MessageType.ASSISTANT, + ) + .group_by(cast(ChatMessage.time_sent, Date)) + .order_by(cast(ChatMessage.time_sent, Date)) + ) + + return [tuple(row) for row in db_session.execute(query).all()] diff --git a/backend/ee/danswer/server/analytics/api.py b/backend/ee/danswer/server/analytics/api.py index f79199323f5c..2963dc2134c7 100644 --- a/backend/ee/danswer/server/analytics/api.py +++ b/backend/ee/danswer/server/analytics/api.py @@ -11,11 +11,16 @@ from danswer.db.engine import get_session from danswer.db.models import User from ee.danswer.db.analytics import fetch_danswerbot_analytics from ee.danswer.db.analytics import fetch_per_user_query_analytics +from ee.danswer.db.analytics import fetch_persona_message_analytics +from ee.danswer.db.analytics import fetch_persona_unique_users from ee.danswer.db.analytics import fetch_query_analytics router = APIRouter(prefix="/analytics") +_DEFAULT_LOOKBACK_DAYS = 30 + + class QueryAnalyticsResponse(BaseModel): total_queries: int total_likes: int @@ -33,7 +38,7 @@ def get_query_analytics( daily_query_usage_info = fetch_query_analytics( start=start or ( - datetime.datetime.utcnow() - datetime.timedelta(days=30) + datetime.datetime.utcnow() - datetime.timedelta(days=_DEFAULT_LOOKBACK_DAYS) ), # default is 30d lookback end=end or datetime.datetime.utcnow(), db_session=db_session, @@ -64,7 +69,7 @@ def get_user_analytics( daily_query_usage_info_per_user = fetch_per_user_query_analytics( start=start or ( - datetime.datetime.utcnow() - datetime.timedelta(days=30) + datetime.datetime.utcnow() - datetime.timedelta(days=_DEFAULT_LOOKBACK_DAYS) ), # default is 30d lookback end=end or datetime.datetime.utcnow(), db_session=db_session, @@ -98,7 +103,7 @@ def get_danswerbot_analytics( daily_danswerbot_info = fetch_danswerbot_analytics( start=start or ( - datetime.datetime.utcnow() - datetime.timedelta(days=30) + datetime.datetime.utcnow() - datetime.timedelta(days=_DEFAULT_LOOKBACK_DAYS) ), # default is 30d lookback end=end or datetime.datetime.utcnow(), db_session=db_session, @@ -115,3 +120,74 @@ def get_danswerbot_analytics( ] return resolution_results + + +class PersonaMessageAnalyticsResponse(BaseModel): + total_messages: int + date: datetime.date + persona_id: int + + +@router.get("/admin/persona/messages") +def get_persona_messages( + persona_id: int, + start: datetime.datetime | None = None, + end: datetime.datetime | None = None, + _: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[PersonaMessageAnalyticsResponse]: + """Fetch daily message counts for a single persona within the given time range.""" + start = start or ( + datetime.datetime.utcnow() - datetime.timedelta(days=_DEFAULT_LOOKBACK_DAYS) + ) + end = end or datetime.datetime.utcnow() + + persona_message_counts = [] + for count, date in fetch_persona_message_analytics( + db_session=db_session, + persona_id=persona_id, + start=start, + end=end, + ): + persona_message_counts.append( + PersonaMessageAnalyticsResponse( + total_messages=count, + date=date, + persona_id=persona_id, + ) + ) + + return persona_message_counts + + +class PersonaUniqueUsersResponse(BaseModel): + unique_users: int + date: datetime.date + persona_id: int + + +@router.get("/admin/persona/unique-users") +def get_persona_unique_users( + persona_id: int, + start: datetime.datetime, + end: datetime.datetime, + _: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[PersonaUniqueUsersResponse]: + """Get unique users per day for a single persona.""" + unique_user_counts = [] + daily_counts = fetch_persona_unique_users( + db_session=db_session, + persona_id=persona_id, + start=start, + end=end, + ) + for count, date in daily_counts: + unique_user_counts.append( + PersonaUniqueUsersResponse( + unique_users=count, + date=date, + persona_id=persona_id, + ) + ) + return unique_user_counts diff --git a/web/src/app/ee/admin/performance/lib.ts b/web/src/app/ee/admin/performance/lib.ts index 0837df1dea05..59042a38766e 100644 --- a/web/src/app/ee/admin/performance/lib.ts +++ b/web/src/app/ee/admin/performance/lib.ts @@ -97,3 +97,69 @@ export function getDatesList(startDate: Date): string[] { return datesList; } + +export interface PersonaMessageAnalytics { + total_messages: number; + date: string; + persona_id: number; +} + +export interface PersonaSnapshot { + id: number; + name: string; + description: string; + is_visible: boolean; + is_public: boolean; +} + +export const usePersonaMessages = ( + personaId: number | undefined, + timeRange: DateRangePickerValue +) => { + const url = buildApiPath(`/api/analytics/admin/persona/messages`, { + persona_id: personaId?.toString(), + start: convertDateToStartOfDay(timeRange.from)?.toISOString(), + end: convertDateToEndOfDay(timeRange.to)?.toISOString(), + }); + + const { data, error, isLoading } = useSWR( + personaId !== undefined ? url : null, + errorHandlingFetcher + ); + + return { + data, + error, + isLoading, + refreshPersonaMessages: () => mutate(url), + }; +}; + +export interface PersonaUniqueUserAnalytics { + unique_users: number; + date: string; + persona_id: number; +} + +export const usePersonaUniqueUsers = ( + personaId: number | undefined, + timeRange: DateRangePickerValue +) => { + const url = buildApiPath(`/api/analytics/admin/persona/unique-users`, { + persona_id: personaId?.toString(), + start: convertDateToStartOfDay(timeRange.from)?.toISOString(), + end: convertDateToEndOfDay(timeRange.to)?.toISOString(), + }); + + const { data, error, isLoading } = useSWR( + personaId !== undefined ? url : null, + errorHandlingFetcher + ); + + return { + data, + error, + isLoading, + refreshPersonaUniqueUsers: () => mutate(url), + }; +}; diff --git a/web/src/app/ee/admin/performance/usage/PersonaMessagesChart.tsx b/web/src/app/ee/admin/performance/usage/PersonaMessagesChart.tsx new file mode 100644 index 000000000000..593ab6ba4dee --- /dev/null +++ b/web/src/app/ee/admin/performance/usage/PersonaMessagesChart.tsx @@ -0,0 +1,231 @@ +import { ThreeDotsLoader } from "@/components/Loading"; +import { X, Search } from "lucide-react"; +import { + getDatesList, + usePersonaMessages, + usePersonaUniqueUsers, +} from "../lib"; +import { useAssistants } from "@/components/context/AssistantsContext"; +import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector"; +import Text from "@/components/ui/text"; +import Title from "@/components/ui/title"; +import CardSection from "@/components/admin/CardSection"; +import { AreaChartDisplay } from "@/components/ui/areaChart"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { useState, useMemo, useEffect } from "react"; + +export function PersonaMessagesChart({ + timeRange, +}: { + timeRange: DateRangePickerValue; +}) { + const [selectedPersonaId, setSelectedPersonaId] = useState< + number | undefined + >(undefined); + const [searchQuery, setSearchQuery] = useState(""); + const [highlightedIndex, setHighlightedIndex] = useState(-1); + const { allAssistants: personaList } = useAssistants(); + + const { + data: personaMessagesData, + isLoading: isPersonaMessagesLoading, + error: personaMessagesError, + } = usePersonaMessages(selectedPersonaId, timeRange); + + const { + data: personaUniqueUsersData, + isLoading: isPersonaUniqueUsersLoading, + error: personaUniqueUsersError, + } = usePersonaUniqueUsers(selectedPersonaId, timeRange); + + const isLoading = isPersonaMessagesLoading || isPersonaUniqueUsersLoading; + const hasError = personaMessagesError || personaUniqueUsersError; + + const filteredPersonaList = useMemo(() => { + if (!personaList) return []; + return personaList.filter((persona) => + persona.name.toLowerCase().includes(searchQuery.toLowerCase()) + ); + }, [personaList, searchQuery]); + + const handleKeyDown = (e: React.KeyboardEvent) => { + e.stopPropagation(); + + switch (e.key) { + case "ArrowDown": + e.preventDefault(); + setHighlightedIndex((prev) => + prev < filteredPersonaList.length - 1 ? prev + 1 : prev + ); + break; + case "ArrowUp": + e.preventDefault(); + setHighlightedIndex((prev) => (prev > 0 ? prev - 1 : prev)); + break; + case "Enter": + if ( + highlightedIndex >= 0 && + highlightedIndex < filteredPersonaList.length + ) { + setSelectedPersonaId(filteredPersonaList[highlightedIndex].id); + setSearchQuery(""); + setHighlightedIndex(-1); + } + break; + case "Escape": + setSearchQuery(""); + setHighlightedIndex(-1); + break; + } + }; + + // Reset highlight when search query changes + useEffect(() => { + setHighlightedIndex(-1); + }, [searchQuery]); + + const chartData = useMemo(() => { + if ( + !personaMessagesData?.length || + !personaUniqueUsersData?.length || + selectedPersonaId === undefined + ) { + return null; + } + + const initialDate = + timeRange.from || + new Date( + Math.min( + ...personaMessagesData.map((entry) => new Date(entry.date).getTime()) + ) + ); + const dateRange = getDatesList(initialDate); + + // Create maps for messages and unique users data + const messagesMap = new Map( + personaMessagesData.map((entry) => [entry.date, entry]) + ); + const uniqueUsersMap = new Map( + personaUniqueUsersData.map((entry) => [entry.date, entry]) + ); + + return dateRange.map((dateStr) => { + const messageData = messagesMap.get(dateStr); + const uniqueUserData = uniqueUsersMap.get(dateStr); + return { + Day: dateStr, + Messages: messageData?.total_messages || 0, + "Unique Users": uniqueUserData?.unique_users || 0, + }; + }); + }, [ + personaMessagesData, + personaUniqueUsersData, + timeRange.from, + selectedPersonaId, + ]); + + let content; + if (isLoading) { + content = ( +
+ +
+ ); + } else if (!personaList || hasError) { + content = ( +
+

Failed to fetch data...

+
+ ); + } else if (selectedPersonaId === undefined) { + content = ( +
+

Select a persona to view analytics

+
+ ); + } else if (!personaMessagesData?.length) { + content = ( +
+

+ No data found for selected persona in the selected time range +

+
+ ); + } else if (chartData) { + content = ( + + ); + } + + const selectedPersona = personaList?.find((p) => p.id === selectedPersonaId); + + return ( + + Persona Analytics +
+ Messages and unique users per day for selected persona +
+ setSearchQuery(e.target.value)} + onClick={(e) => e.stopPropagation()} + onMouseDown={(e) => e.stopPropagation()} + onKeyDown={handleKeyDown} + /> + {searchQuery && ( + { + setSearchQuery(""); + setHighlightedIndex(-1); + }} + /> + )} +
+ {filteredPersonaList.map((persona, index) => ( + setHighlightedIndex(index)} + > + {persona.name} + + ))} + + +
+ + {content} + + ); +} diff --git a/web/src/app/ee/admin/performance/usage/page.tsx b/web/src/app/ee/admin/performance/usage/page.tsx index e1fffc323a2e..967f16a377ef 100644 --- a/web/src/app/ee/admin/performance/usage/page.tsx +++ b/web/src/app/ee/admin/performance/usage/page.tsx @@ -4,6 +4,7 @@ import { DateRangeSelector } from "../DateRangeSelector"; import { DanswerBotChart } from "./DanswerBotChart"; import { FeedbackChart } from "./FeedbackChart"; import { QueryPerformanceChart } from "./QueryPerformanceChart"; +import { PersonaMessagesChart } from "./PersonaMessagesChart"; import { useTimeRange } from "../lib"; import { AdminPageTitle } from "@/components/admin/Title"; import { FiActivity } from "react-icons/fi"; @@ -26,6 +27,7 @@ export default function AnalyticsPage() { +