fix: Fix error in which channel names would not have the leading "#" removed (#4664)

* Fix failing entrypoint into slack connector

* Pre-filter channel names upon instantiation of slack connector class

* Add decrypt script

* Add slack connector tests

* Fix mypy errors on decrypt.py

* Add property to SlackConnector class

* Add some basic tests

* Move location of tests

* Change name of env token

* Add secrets for Slack

* Add more parameterized cases

* Change env variable name

* Change names

* Update channel names

* Edit tests

* Modify tests

* Only import type in __main__

* Fix tests to actually test connectors

* Pass parameter to fixture directly

.github/workflows/pr-python-connector-tests.yml

@@ -12,7 +12,7 @@ env:
   # AWS
   AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
   AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}
-  
+
   # Confluence
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_TEST_SPACE: ${{ secrets.CONFLUENCE_TEST_SPACE }}
@@ -20,10 +20,12 @@ env:
   CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }}
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
+
   # Jira
   JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
   JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
 
+  # Gong
   GONG_ACCESS_KEY: ${{ secrets.GONG_ACCESS_KEY }}
   GONG_ACCESS_KEY_SECRET: ${{ secrets.GONG_ACCESS_KEY_SECRET }}
 
@@ -33,39 +35,52 @@ env:
   GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }}
   GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR }}
   GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR }}
+
   # Slab
   SLAB_BOT_TOKEN: ${{ secrets.SLAB_BOT_TOKEN }}
+
   # Zendesk
   ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }}
   ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }}
   ZENDESK_TOKEN: ${{ secrets.ZENDESK_TOKEN }}
+
   # Salesforce
   SF_USERNAME: ${{ secrets.SF_USERNAME }}
   SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
   SF_SECURITY_TOKEN: ${{ secrets.SF_SECURITY_TOKEN }}
+
   # Airtable
   AIRTABLE_TEST_BASE_ID: ${{ secrets.AIRTABLE_TEST_BASE_ID }}
   AIRTABLE_TEST_TABLE_ID: ${{ secrets.AIRTABLE_TEST_TABLE_ID }}
   AIRTABLE_TEST_TABLE_NAME: ${{ secrets.AIRTABLE_TEST_TABLE_NAME }}
   AIRTABLE_ACCESS_TOKEN: ${{ secrets.AIRTABLE_ACCESS_TOKEN }}
+
   # Sharepoint
   SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
   SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
   SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ secrets.SHAREPOINT_CLIENT_DIRECTORY_ID }}
   SHAREPOINT_SITE: ${{ secrets.SHAREPOINT_SITE }}
+
   # Github
   ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}
+
   # Gitlab
   GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }}
+
   # Gitbook
   GITBOOK_SPACE_ID: ${{ secrets.GITBOOK_SPACE_ID }}
   GITBOOK_API_KEY: ${{ secrets.GITBOOK_API_KEY }}
+
   # Notion
   NOTION_INTEGRATION_TOKEN: ${{ secrets.NOTION_INTEGRATION_TOKEN }}
+
   # Highspot
   HIGHSPOT_KEY: ${{ secrets.HIGHSPOT_KEY }}
   HIGHSPOT_SECRET: ${{ secrets.HIGHSPOT_SECRET }}
 
+  # Slack
+  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
 jobs:
   connectors-check:
     # See https://runs-on.com/runners/linux/

backend/onyx/connectors/slack/connector.py

@@ -541,6 +541,16 @@ class SlackConnector(
         self.delay_lock: str | None = None  # the redis key for the shared lock
         self.delay_key: str | None = None  # the redis key for the shared delay
 
+    @property
+    def channels(self) -> list[str] | None:
+        return self._channels
+
+    @channels.setter
+    def channels(self, channels: list[str] | None) -> None:
+        self._channels = (
+            [channel.removeprefix("#") for channel in channels] if channels else None
+        )
+
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         raise NotImplementedError("Use set_credentials_provider with this connector.")
 
@@ -850,12 +860,22 @@ class SlackConnector(
 if __name__ == "__main__":
     import os
     import time
+    from onyx.connectors.credentials_provider import OnyxStaticCredentialsProvider
+    from shared_configs.contextvars import get_current_tenant_id
 
     slack_channel = os.environ.get("SLACK_CHANNEL")
     connector = SlackConnector(
         channels=[slack_channel] if slack_channel else None,
     )
-    connector.load_credentials({"slack_bot_token": os.environ["SLACK_BOT_TOKEN"]})
+
+    provider = OnyxStaticCredentialsProvider(
+        tenant_id=get_current_tenant_id(),
+        connector_name="slack",
+        credential_json={
+            "slack_bot_token": os.environ["SLACK_BOT_TOKEN"],
+        },
+    )
+    connector.set_credentials_provider(provider)
 
     current = time.time()
     one_day_ago = current - 24 * 60 * 60  # 1 day

backend/scripts/decrypt.py

@@ -0,0 +1,48 @@
+import binascii
+import json
+import sys
+
+from onyx.utils.encryption import decrypt_bytes_to_string
+
+
+def decrypt_raw_credential(encrypted_value: str) -> None:
+    """Decrypt and display a raw encrypted credential value
+
+    Args:
+        encrypted_value: The hex encoded encrypted credential value
+    """
+    try:
+        # If string starts with 'x', remove it as it's just a prefix indicating hex
+        if encrypted_value.startswith("x"):
+            encrypted_value = encrypted_value[1:]
+        elif encrypted_value.startswith("\\x"):
+            encrypted_value = encrypted_value[2:]
+
+        # Convert hex string to bytes
+        encrypted_bytes = binascii.unhexlify(encrypted_value)
+
+        # Decrypt the bytes
+        decrypted_str = decrypt_bytes_to_string(encrypted_bytes)
+
+        # Parse and pretty print the decrypted JSON
+        decrypted_json = json.loads(decrypted_str)
+        print("Decrypted credential value:")
+        print(json.dumps(decrypted_json, indent=2))
+
+    except binascii.Error:
+        print("Error: Invalid hex encoded string")
+
+    except json.JSONDecodeError as e:
+        print(f"Decrypted raw value (not JSON): {e}")
+
+    except Exception as e:
+        print(f"Error decrypting value: {e}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python decrypt.py <hex_encoded_encrypted_value>")
+        sys.exit(1)
+
+    encrypted_value = sys.argv[1]
+    decrypt_raw_credential(encrypted_value)

backend/tests/daily/connectors/slack/test_slack_connector.py

@@ -0,0 +1,148 @@
+import os
+import time
+from collections.abc import Generator
+from unittest.mock import MagicMock
+
+import pytest
+from pytest import FixtureRequest
+from slack_sdk import WebClient
+
+from onyx.connectors.credentials_provider import OnyxStaticCredentialsProvider
+from onyx.connectors.models import Document
+from onyx.connectors.models import TextSection
+from onyx.connectors.slack.connector import SlackConnector
+from shared_configs.contextvars import get_current_tenant_id
+from tests.daily.connectors.utils import load_everything_from_checkpoint_connector
+
+
+@pytest.fixture
+def mock_slack_client() -> MagicMock:
+    mock = MagicMock(spec=WebClient)
+    return mock
+
+
+@pytest.fixture
+def slack_connector(
+    request: FixtureRequest,
+    mock_slack_client: MagicMock,
+    slack_credentials_provider: OnyxStaticCredentialsProvider,
+) -> Generator[SlackConnector]:
+    channel: str | None = request.param if hasattr(request, "param") else None
+    connector = SlackConnector(
+        channels=[channel] if channel else None,
+        channel_regex_enabled=False,
+    )
+    connector.client = mock_slack_client
+    connector.set_credentials_provider(credentials_provider=slack_credentials_provider)
+    yield connector
+
+
+@pytest.fixture
+def slack_credentials_provider() -> OnyxStaticCredentialsProvider:
+    CI_ENV_VAR = "SLACK_BOT_TOKEN"
+    LOCAL_ENV_VAR = "DANSWER_BOT_SLACK_BOT_TOKEN"
+
+    slack_bot_token = os.environ.get(CI_ENV_VAR, os.environ.get(LOCAL_ENV_VAR))
+    if not slack_bot_token:
+        raise RuntimeError(
+            f"No slack credentials found; either set the {CI_ENV_VAR} env-var or the {LOCAL_ENV_VAR} env-var"
+        )
+
+    return OnyxStaticCredentialsProvider(
+        tenant_id=get_current_tenant_id(),
+        connector_name="slack",
+        credential_json={
+            "slack_bot_token": slack_bot_token,
+        },
+    )
+
+
+def test_validate_slack_connector_settings(
+    slack_connector: SlackConnector,
+) -> None:
+    slack_connector.validate_connector_settings()
+
+
+@pytest.mark.parametrize(
+    "slack_connector,expected_messages",
+    [
+        ["general", set()],
+        ["#general", set()],
+        [
+            "daily-connector-test-channel",
+            set(
+                [
+                    "Hello, world!",
+                    "",
+                    "Reply!",
+                    "Testing again...",
+                ]
+            ),
+        ],
+        [
+            "#daily-connector-test-channel",
+            set(
+                [
+                    "Hello, world!",
+                    "",
+                    "Reply!",
+                    "Testing again...",
+                ]
+            ),
+        ],
+    ],
+    indirect=["slack_connector"],
+)
+def test_indexing_channels_with_message_count(
+    slack_connector: SlackConnector,
+    expected_messages: set[str],
+) -> None:
+    if not slack_connector.client:
+        raise RuntimeError("Web client must be defined")
+
+    docs = load_everything_from_checkpoint_connector(
+        connector=slack_connector,
+        start=0.0,
+        end=time.time(),
+    )
+
+    messages: list[str] = []
+
+    for doc_or_error in docs:
+        if not isinstance(doc_or_error, Document):
+            raise RuntimeError(doc_or_error)
+        messages.extend(
+            section.text
+            for section in doc_or_error.sections
+            if isinstance(section, TextSection)
+        )
+
+    actual_messages = set(messages)
+    assert expected_messages == actual_messages
+
+
+@pytest.mark.parametrize(
+    "slack_connector",
+    [
+        # w/o hashtag
+        "doesnt-exist",
+        # w/ hashtag
+        "#doesnt-exist",
+    ],
+    indirect=True,
+)
+def test_indexing_channels_that_dont_exist(
+    slack_connector: SlackConnector,
+) -> None:
+    if not slack_connector.client:
+        raise RuntimeError("Web client must be defined")
+
+    with pytest.raises(
+        ValueError,
+        match=r"Channel '.*' not found in workspace.*",
+    ):
+        load_everything_from_checkpoint_connector(
+            connector=slack_connector,
+            start=0.0,
+            end=time.time(),
+        )