Allow slack channels to be specified (#238)

Adds the capability to specify specific channels to index when using the Slack connector

backend/danswer/connectors/slack/connector.py

@@ -46,7 +46,10 @@ def get_channel_info(client: WebClient, channel_id: str) -> ChannelType:
     ]
 
 
-def get_channels(client: WebClient, exclude_archived: bool = True) -> list[ChannelType]:
+def get_channels(
+    client: WebClient,
+    exclude_archived: bool = True,
+) -> list[ChannelType]:
     """Get all channels in the workspace"""
     channels: list[dict[str, Any]] = []
     for result in _make_slack_api_call(
@@ -133,9 +136,22 @@ def _default_msg_filter(message: MessageType) -> bool:
     return message.get("subtype", "") in _DISALLOWED_MSG_SUBTYPES
 
 
+def _filter_channels(
+    all_channels: list[dict[str, Any]], channels_to_connect: list[str] | None
+) -> list[dict[str, Any]]:
+    if channels_to_connect:
+        return [
+            channel
+            for channel in all_channels
+            if channel["name"] in channels_to_connect
+        ]
+    return all_channels
+
+
 def get_all_docs(
     client: WebClient,
     workspace: str,
+    channels: list[str] | None = None,
     oldest: str | None = None,
     latest: str | None = None,
     msg_filter_func: Callable[[MessageType], bool] = _default_msg_filter,
@@ -143,9 +159,10 @@ def get_all_docs(
     """Get all documents in the workspace, channel by channel"""
     user_id_replacer = UserIdReplacer(client=client)
 
-    channels = get_channels(client)
+    all_channels = get_channels(client)
+    filtered_channels = _filter_channels(all_channels, channels)
 
-    for channel in channels:
+    for channel in filtered_channels:
         channel_docs = 0
         channel_message_batches = get_channel_messages(
             client=client, channel=channel, oldest=oldest, latest=latest
@@ -181,9 +198,14 @@ def get_all_docs(
 
 class SlackLoadConnector(LoadConnector):
     def __init__(
-        self, workspace: str, export_path_str: str, batch_size: int = INDEX_BATCH_SIZE
+        self,
+        workspace: str,
+        export_path_str: str,
+        channels: list[str] | None = None,
+        batch_size: int = INDEX_BATCH_SIZE,
     ) -> None:
         self.workspace = workspace
+        self.channels = channels
         self.export_path_str = export_path_str
         self.batch_size = batch_size
 
@@ -245,10 +267,12 @@ class SlackLoadConnector(LoadConnector):
         export_path = Path(self.export_path_str)
 
         with open(export_path / "channels.json") as f:
-            channels = json.load(f)
+            all_channels = json.load(f)
+
+        filtered_channels = _filter_channels(all_channels, self.channels)
 
         document_batch: dict[str, Document] = {}
-        for channel_info in channels:
+        for channel_info in filtered_channels:
             channel_dir_path = export_path / cast(str, channel_info["name"])
             channel_file_paths = [
                 channel_dir_path / file_name
@@ -275,8 +299,14 @@ class SlackLoadConnector(LoadConnector):
 
 
 class SlackPollConnector(PollConnector):
-    def __init__(self, workspace: str, batch_size: int = INDEX_BATCH_SIZE) -> None:
+    def __init__(
+        self,
+        workspace: str,
+        channels: list[str] | None = None,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
         self.workspace = workspace
+        self.channels = channels
         self.batch_size = batch_size
         self.client: WebClient | None = None
 
@@ -295,6 +325,7 @@ class SlackPollConnector(PollConnector):
         for document in get_all_docs(
             client=self.client,
             workspace=self.workspace,
+            channels=self.channels,
             # NOTE: need to impute to `None` instead of using 0.0, since Slack will
             # throw an error if we use 0.0 on an account without infinite data
             # retention

web/src/app/admin/connectors/slack/page.tsx

@@ -14,7 +14,10 @@ import {
 } from "@/lib/types";
 import { deleteCredential, linkCredential } from "@/lib/credential";
 import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
-import { TextFormField } from "@/components/admin/connectors/Field";
+import {
+  TextFormField,
+  TextArrayFieldBuilder,
+} from "@/components/admin/connectors/Field";
 import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
 import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
 
@@ -152,6 +155,14 @@ const MainSection = () => {
                   getValue: (connector) =>
                     connector.connector_specific_config.workspace,
                 },
+                {
+                  header: "Channels",
+                  key: "channels",
+                  getValue: (connector) =>
+                    connector.connector_specific_config.channels
+                      ? connector.connector_specific_config.channels.join(", ")
+                      : "All channels",
+                },
               ]}
               onUpdate={() =>
                 mutate("/api/manage/admin/connector/indexing-status")
@@ -170,21 +181,40 @@ const MainSection = () => {
       <div className="border-solid border-gray-600 border rounded-md p-6 mt-4">
         <h2 className="font-bold mb-3">Connect to a New Workspace</h2>
         <ConnectorForm<SlackConfig>
-          nameBuilder={(values) => `SlackConnector-${values.workspace}`}
+          nameBuilder={(values) =>
+            values.channels
+              ? `SlackConnector-${values.workspace}-${values.channels.join(
+                  "_"
+                )}`
+              : `SlackConnector-${values.workspace}`
+          }
           source="slack"
           inputType="poll"
           formBody={
             <>
-              <TextFormField name="workspace" label="Workspace:" />
+              <TextFormField name="workspace" label="Workspace" />
             </>
           }
+          formBodyBuilder={TextArrayFieldBuilder({
+            name: "channels",
+            label: "Channels:",
+            subtext:
+              "Specify 0 or more channels to index. For example, specifying the channel " +
+              "'support' will cause us to only index all content " +
+              "within the '#support' channel. " +
+              "If no channels are specified, all channels in your workspace will be indexed.",
+          })}
           validationSchema={Yup.object().shape({
             workspace: Yup.string().required(
               "Please enter the workspace to index"
             ),
+            channels: Yup.array()
+              .of(Yup.string().required("Channel names must be strings"))
+              .required(),
           })}
           initialValues={{
             workspace: "",
+            channels: [],
           }}
           refreshFreq={10 * 60} // 10 minutes
           onSubmit={async (isSuccess, responseJson) => {

web/src/components/admin/connectors/ConnectorForm.tsx

@@ -50,7 +50,7 @@ interface BaseProps<T extends Yup.AnyObject> {
   source: ValidSources;
   inputType: ValidInputTypes;
   credentialId?: number;
-  // If both are specified, uses formBody
+  // If both are specified, will render formBody and then formBodyBuilder
   formBody?: JSX.Element | null;
   formBodyBuilder?: FormBodyBuilder<T>;
   validationSchema: Yup.ObjectSchema<T>;
@@ -127,7 +127,8 @@ export function ConnectorForm<T extends Yup.AnyObject>({
       >
         {({ isSubmitting, values }) => (
           <Form>
-            {formBody ? formBody : formBodyBuilder && formBodyBuilder(values)}
+            {formBody && formBody}
+            {formBodyBuilder && formBodyBuilder(values)}
             <div className="flex">
               <button
                 type="submit"

web/src/lib/types.ts

@@ -67,6 +67,7 @@ export interface ProductboardConfig {}
 
 export interface SlackConfig {
   workspace: string;
+  channels?: string[];
 }
 
 export interface SlabConfig {