Track Slack questions Autoresolved (#86)

backend/ee/danswer/db/analytics.py

@@ -6,6 +6,7 @@ from sqlalchemy import case
 from sqlalchemy import cast
 from sqlalchemy import Date
 from sqlalchemy import func
+from sqlalchemy import or_
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -16,9 +17,9 @@ from danswer.db.models import ChatSession
 
 
 def fetch_query_analytics(
-    db_session: Session,
     start: datetime.datetime,
     end: datetime.datetime,
+    db_session: Session,
 ) -> Sequence[tuple[int, int, int, datetime.date]]:
     stmt = (
         select(
@@ -51,9 +52,9 @@ def fetch_query_analytics(
 
 
 def fetch_per_user_query_analytics(
-    db_session: Session,
     start: datetime.datetime,
     end: datetime.datetime,
+    db_session: Session,
 ) -> Sequence[tuple[int, int, int, datetime.date, UUID]]:
     stmt = (
         select(
@@ -80,3 +81,92 @@ def fetch_per_user_query_analytics(
     )
 
     return db_session.execute(stmt).all()  # type: ignore
+
+
+def fetch_danswerbot_analytics(
+    start: datetime.datetime,
+    end: datetime.datetime,
+    db_session: Session,
+) -> Sequence[tuple[int, int, datetime.date]]:
+    """Gets the:
+    Date of each set of aggregated statistics
+    Number of DanswerBot Queries (Chat Sessions)
+    Number of instances of Negative feedback OR Needing additional help
+        (only counting the last feedback)
+    """
+    # Get every chat session in the time range which is a Danswerbot flow
+    # along with the first Assistant message which is the response to the user question.
+    # Generally there should not be more than one AI message per chat session of this type
+    subquery_first_ai_response = (
+        db_session.query(
+            ChatMessage.chat_session_id.label("chat_session_id"),
+            func.min(ChatMessage.id).label("chat_message_id"),
+        )
+        .join(ChatSession, ChatSession.id == ChatMessage.chat_session_id)
+        .where(
+            ChatSession.time_created >= start,
+            ChatSession.time_created <= end,
+            ChatSession.danswerbot_flow.is_(True),
+        )
+        .where(
+            ChatMessage.message_type == MessageType.ASSISTANT,
+        )
+        .group_by(ChatMessage.chat_session_id)
+        .subquery()
+    )
+
+    # Get the chat message ids and most recent feedback for each of those chat messages,
+    # not including the messages that have no feedback
+    subquery_last_feedback = (
+        db_session.query(
+            ChatMessageFeedback.chat_message_id.label("chat_message_id"),
+            func.max(ChatMessageFeedback.id).label("max_feedback_id"),
+        )
+        .group_by(ChatMessageFeedback.chat_message_id)
+        .subquery()
+    )
+
+    results = (
+        db_session.query(
+            func.count(ChatSession.id).label("total_sessions"),
+            # Need to explicitly specify this as False to handle the NULL case so the cases without
+            # feedback aren't counted against Danswerbot
+            func.sum(
+                case(
+                    (
+                        or_(
+                            ChatMessageFeedback.is_positive.is_(False),
+                            ChatMessageFeedback.required_followup,
+                        ),
+                        1,
+                    ),
+                    else_=0,
+                )
+            ).label("negative_answer"),
+            cast(ChatSession.time_created, Date).label("session_date"),
+        )
+        .join(
+            subquery_first_ai_response,
+            ChatSession.id == subquery_first_ai_response.c.chat_session_id,
+        )
+        # Combine the chat sessions with latest feedback to get the latest feedback for the first AI
+        # message of the chat session where the chat session is Danswerbot type and within the time
+        # range specified. Left/outer join used here to ensure that if no feedback, a null is used
+        # for the feedback id
+        .outerjoin(
+            subquery_last_feedback,
+            subquery_first_ai_response.c.chat_message_id
+            == subquery_last_feedback.c.chat_message_id,
+        )
+        # Join the actual feedback table to get the feedback info for the sums
+        # Outer join because the "last feedback" may be null
+        .outerjoin(
+            ChatMessageFeedback,
+            ChatMessageFeedback.id == subquery_last_feedback.c.max_feedback_id,
+        )
+        .group_by(cast(ChatSession.time_created, Date))
+        .order_by(cast(ChatSession.time_created, Date))
+        .all()
+    )
+
+    return results

backend/ee/danswer/server/analytics/api.py

@@ -9,6 +9,7 @@ from sqlalchemy.orm import Session
 import danswer.db.models as db_models
 from danswer.auth.users import current_admin_user
 from danswer.db.engine import get_session
+from ee.danswer.db.analytics import fetch_danswerbot_analytics
 from ee.danswer.db.analytics import fetch_per_user_query_analytics
 from ee.danswer.db.analytics import fetch_query_analytics
 
@@ -30,12 +31,12 @@ def get_query_analytics(
     db_session: Session = Depends(get_session),
 ) -> list[QueryAnalyticsResponse]:
     daily_query_usage_info = fetch_query_analytics(
-        db_session=db_session,
         start=start
         or (
             datetime.datetime.utcnow() - datetime.timedelta(days=30)
         ),  # default is 30d lookback
         end=end or datetime.datetime.utcnow(),
+        db_session=db_session,
     )
     return [
         QueryAnalyticsResponse(
@@ -61,12 +62,12 @@ def get_user_analytics(
     db_session: Session = Depends(get_session),
 ) -> list[UserAnalyticsResponse]:
     daily_query_usage_info_per_user = fetch_per_user_query_analytics(
-        db_session=db_session,
         start=start
         or (
             datetime.datetime.utcnow() - datetime.timedelta(days=30)
         ),  # default is 30d lookback
         end=end or datetime.datetime.utcnow(),
+        db_session=db_session,
     )
 
     user_analytics: dict[datetime.date, int] = defaultdict(int)
@@ -79,3 +80,38 @@ def get_user_analytics(
         )
         for date, cnt in user_analytics.items()
     ]
+
+
+class DanswerbotAnalyticsResponse(BaseModel):
+    total_queries: int
+    auto_resolved: int
+    date: datetime.date
+
+
+@router.get("/admin/danswerbot")
+def get_danswerbot_analytics(
+    start: datetime.datetime | None = None,
+    end: datetime.datetime | None = None,
+    _: db_models.User | None = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
+) -> list[DanswerbotAnalyticsResponse]:
+    daily_danswerbot_info = fetch_danswerbot_analytics(
+        start=start
+        or (
+            datetime.datetime.utcnow() - datetime.timedelta(days=30)
+        ),  # default is 30d lookback
+        end=end or datetime.datetime.utcnow(),
+        db_session=db_session,
+    )
+
+    resolution_results = [
+        DanswerbotAnalyticsResponse(
+            total_queries=total_queries,
+            # If it hits negatives, something has gone wrong...
+            auto_resolved=max(0, total_queries - total_negatives),
+            date=date,
+        )
+        for total_queries, total_negatives, date in daily_danswerbot_info
+    ]
+
+    return resolution_results

web/src/app/ee/admin/performance/lib.ts

@@ -2,6 +2,7 @@ import { errorHandlingFetcher } from "@/lib/fetcher";
 import useSWR, { mutate } from "swr";
 import {
   ChatSessionSnapshot,
+  DanswerBotAnalytics,
   QueryAnalytics,
   UserAnalytics,
 } from "./usage/types";
@@ -50,6 +51,19 @@ export const useUserAnalytics = (timeRange: DateRangePickerValue) => {
   };
 };
 
+export const useDanswerBotAnalytics = (timeRange: DateRangePickerValue) => {
+  const url = buildApiPath("/api/analytics/admin/danswerbot", {
+    start: convertDateToStartOfDay(timeRange.from)?.toISOString(),
+    end: convertDateToEndOfDay(timeRange.to)?.toISOString(),
+  });
+  const swrResponse = useSWR<DanswerBotAnalytics[]>(url, errorHandlingFetcher); // TODO
+
+  return {
+    ...swrResponse,
+    refreshDanswerBotAnalytics: () => mutate(url),
+  };
+};
+
 export const useQueryHistory = () => {
   const [selectedFeedbackType, setSelectedFeedbackType] =
     useState<Feedback | null>(null);

web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx

@@ -0,0 +1,78 @@
+import { ThreeDotsLoader } from "@/components/Loading";
+import { getDatesList, useDanswerBotAnalytics } from "../lib";
+import {
+  AreaChart,
+  Card,
+  Title,
+  Text,
+  DateRangePickerValue,
+} from "@tremor/react";
+
+export function DanswerBotChart({
+  timeRange,
+}: {
+  timeRange: DateRangePickerValue;
+}) {
+  const {
+    data: danswerBotAnalyticsData,
+    isLoading: isDanswerBotAnalyticsLoading,
+    error: danswerBotAnalyticsError,
+  } = useDanswerBotAnalytics(timeRange);
+
+  let chart;
+  if (isDanswerBotAnalyticsLoading) {
+    chart = (
+      <div className="h-80 flex flex-col">
+        <ThreeDotsLoader />
+      </div>
+    );
+  } else if (!danswerBotAnalyticsData || danswerBotAnalyticsError) {
+    chart = (
+      <div className="h-80 text-red-600 text-bold flex flex-col">
+        <p className="m-auto">Failed to fetch feedback data...</p>
+      </div>
+    );
+  } else {
+    const initialDate =
+      timeRange.from || new Date(danswerBotAnalyticsData[0].date);
+    const dateRange = getDatesList(initialDate);
+
+    const dateToDanswerBotAnalytics = new Map(
+      danswerBotAnalyticsData.map((danswerBotAnalyticsEntry) => [
+        danswerBotAnalyticsEntry.date,
+        danswerBotAnalyticsEntry,
+      ])
+    );
+
+    chart = (
+      <AreaChart
+        className="mt-4 h-80"
+        data={dateRange.map((dateStr) => {
+          const danswerBotAnalyticsForDate =
+            dateToDanswerBotAnalytics.get(dateStr);
+          return {
+            Day: dateStr,
+            "Total Queries": danswerBotAnalyticsForDate?.total_queries || 0,
+            "Automatically Resolved":
+              danswerBotAnalyticsForDate?.auto_resolved || 0,
+          };
+        })}
+        categories={["Total Queries", "Automatically Resolved"]}
+        index="Day"
+        colors={["indigo", "fuchsia"]}
+        valueFormatter={(number: number) =>
+          `${Intl.NumberFormat("us").format(number).toString()}`
+        }
+        yAxisWidth={60}
+      />
+    );
+  }
+
+  return (
+    <Card className="mt-8">
+      <Title>Slack Bot</Title>
+      <Text>Total Queries vs Auto Resolved</Text>
+      {chart}
+    </Card>
+  );
+}

web/src/app/ee/admin/performance/usage/page.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { DateRangeSelector } from "../DateRangeSelector";
+import { DanswerBotChart } from "./DanswerBotChart";
 import { FeedbackChart } from "./FeedbackChart";
 import { QueryPerformanceChart } from "./QueryPerformanceChart";
 import { BarChartIcon } from "@/components/icons/icons";
@@ -23,6 +24,7 @@ export default function AnalyticsPage() {
 
       <QueryPerformanceChart timeRange={timeRange} />
       <FeedbackChart timeRange={timeRange} />
+      <DanswerBotChart timeRange={timeRange} />
     </main>
   );
 }

web/src/app/ee/admin/performance/usage/types.ts

@@ -12,6 +12,12 @@ export interface UserAnalytics {
   date: string;
 }
 
+export interface DanswerBotAnalytics {
+  total_queries: number;
+  auto_resolved: number;
+  date: string;
+}
+
 export interface AbridgedSearchDoc {
   document_id: string;
   semantic_identifier: string;