Gitlab Connector (#931)

web/src/app/admin/connectors/gitlab/page.tsx

@@ -0,0 +1,253 @@
+"use client";
+
+import * as Yup from "yup";
+import { GitlabIcon, TrashIcon } from "@/components/icons/icons";
+import { TextFormField } from "@/components/admin/connectors/Field";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import useSWR, { useSWRConfig } from "swr";
+import { fetcher } from "@/lib/fetcher";
+import {
+  GitlabConfig,
+  GitlabCredentialJson,
+  Credential,
+  ConnectorIndexingStatus,
+} from "@/lib/types";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { LoadingAnimation } from "@/components/Loading";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { usePublicCredentials } from "@/lib/hooks";
+import { Card, Divider, Text, Title } from "@tremor/react";
+import { AdminPageTitle } from "@/components/admin/Title";
+
+const Main = () => {
+  const { mutate } = useSWRConfig();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    fetcher
+  );
+
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    error: isCredentialsError,
+    refreshCredentials,
+  } = usePublicCredentials();
+
+  if (
+    (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+    (!credentialsData && isCredentialsLoading)
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const gitlabConnectorIndexingStatuses: ConnectorIndexingStatus<
+    GitlabConfig,
+    GitlabCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "gitlab"
+  );
+  const gitlabCredential: Credential<GitlabCredentialJson> | undefined =
+    credentialsData.find(
+      (credential) => credential.credential_json?.gitlab_access_token
+    );
+
+  return (
+    <>
+      <Title className="mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide your access token
+      </Title>
+      {gitlabCredential ? (
+        <>
+          {" "}
+          <div className="flex mb-1 text-sm">
+            <p className="my-auto">Existing Access Token: </p>
+            <p className="ml-1 italic my-auto">
+              {gitlabCredential.credential_json.gitlab_access_token}
+            </p>{" "}
+            <button
+              className="ml-1 hover:bg-hover rounded p-1"
+              onClick={async () => {
+                await adminDeleteCredential(gitlabCredential.id);
+                refreshCredentials();
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <Text>
+            If you don&apos;t have an access token, read the guide{" "}
+            <a
+              className="text-blue-500"
+              href="https://docs.danswer.dev/connectors/gitlab"
+              target="_blank"
+            >
+              here
+            </a>{" "}
+            on how to get one from Gitlab.
+          </Text>
+          <Card className="mt-4">
+            <CredentialForm<GitlabCredentialJson>
+              formBody={
+                <>
+                  <Text>
+                    If you are using GitLab Cloud, keep the default value below</Text>
+                  <TextFormField
+                    name="gitlab_url"
+                    label="GitLab URL:"
+                    type="text"
+                    placeholder="https://gitlab.com"
+                  />
+
+                  <TextFormField
+                    name="gitlab_access_token"
+                    label="Access Token:"
+                    type="password"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                gitlab_url: Yup.string().default("https://gitlab.com"),
+                gitlab_access_token: Yup.string().required(
+                  "Please enter the access token for Gitlab"
+                ),
+              })}
+              initialValues={{
+                gitlab_access_token: "",
+                gitlab_url: "https://gitlab.com"
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  refreshCredentials();
+                }
+              }}
+            />
+          </Card>
+        </>
+      )}
+
+      <Title className="mb-2 mt-6 ml-auto mr-auto">
+        Step 2: Which repositories do you want to make searchable?
+      </Title>
+
+      {gitlabConnectorIndexingStatuses.length > 0 && (
+        <>
+          <Text className="mb-2">
+            We pull the latest Pull Requests from each project listed below
+            every <b>10</b> minutes.
+          </Text>
+          <div className="mb-2">
+            <ConnectorsTable<GitlabConfig, GitlabCredentialJson>
+              connectorIndexingStatuses={gitlabConnectorIndexingStatuses}
+              liveCredential={gitlabCredential}
+              getCredential={(credential) =>
+                credential.credential_json.gitlab_access_token
+              }
+              onCredentialLink={async (connectorId) => {
+                if (gitlabCredential) {
+                  await linkCredential(connectorId, gitlabCredential.id);
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+              specialColumns={[
+                {
+                  header: "Project",
+                  key: "project",
+                  getValue: (ccPairStatus) => {
+                    const connectorConfig =
+                      ccPairStatus.connector.connector_specific_config;
+                    return `${connectorConfig.project_owner}/${connectorConfig.project_name}`;
+                  },
+                },
+              ]}
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+            />
+          </div>
+          <Divider />
+        </>
+      )}
+
+      {gitlabCredential ? (
+        <Card className="mt-4">
+          <h2 className="font-bold mb-3">Connect to a New Project</h2>
+          <ConnectorForm<GitlabConfig>
+            nameBuilder={(values) =>
+              `GitlabConnector-${values.project_owner}/${values.project_name}`
+            }
+            ccPairNameBuilder={(values) =>
+              `${values.project_owner}/${values.project_name}`
+            }
+            source="gitlab"
+            inputType="poll"
+            formBody={
+              <>
+                <TextFormField name="project_owner" label="Project Owner:" />
+                <TextFormField name="project_name" label="Project Name:" />
+              </>
+            }
+            validationSchema={Yup.object().shape({
+              project_owner: Yup.string().required(
+                "Please enter the owner of the project to index e.g. danswer-ai"
+              ),
+              project_name: Yup.string().required(
+                "Please enter the name of the project to index e.g. danswer "
+              ),
+              include_mrs: Yup.boolean().required(),
+              include_issues: Yup.boolean().required(),
+            })}
+            initialValues={{
+              project_owner: "",
+              project_name: "",
+              include_mrs: true,
+              include_issues: true,
+            }}
+            refreshFreq={10 * 60} // 10 minutes
+            credentialId={gitlabCredential.id}
+          />
+        </Card>
+      ) : (
+        <Text>
+          Please provide your access token in Step 1 first! Once done with that,
+          you can then specify which Gitlab repositories you want to make
+          searchable.
+        </Text>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="container mx-auto">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+
+      <AdminPageTitle
+        icon={<GitlabIcon size={32} />}
+        title="Gitlab MRs + Issues"
+      />
+
+      <Main />
+    </div>
+  );
+}

web/src/components/admin/connectors/ConnectorTitle.tsx

@@ -3,6 +3,7 @@ import {
   ConfluenceConfig,
   Connector,
   GithubConfig,
+  GitlabConfig,
   GoogleDriveConfig,
   JiraConfig,
   SlackConfig,
@@ -38,7 +39,13 @@ export const ConnectorTitle = ({
       "Repo",
       `${typedConnector.connector_specific_config.repo_owner}/${typedConnector.connector_specific_config.repo_name}`
     );
-  } else if (connector.source === "confluence") {
+  } else if (connector.source === "gitlab") {
+    const typedConnector = connector as Connector<GitlabConfig>;
+    additionalMetadata.set(
+      "Repo",
+      `${typedConnector.connector_specific_config.project_owner}/${typedConnector.connector_specific_config.project_name}`
+    );
+  }  else if (connector.source === "confluence") {
     const typedConnector = connector as Connector<ConfluenceConfig>;
     additionalMetadata.set(
       "Wiki URL",

web/src/components/icons/icons.tsx

@@ -329,7 +329,19 @@ export const SlackIcon = ({
     </div>
   );
 };
-
+export const GitlabIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <div
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+    >
+      <Image src="/Gitlab.png" alt="Logo" width="96" height="96" />
+    </div>
+  );
+};
 export const GithubIcon = ({
   size = 16,
   className = defaultTailwindCSS,
@@ -344,6 +356,7 @@ export const GithubIcon = ({
   );
 };
 
+
 export const GoogleDriveIcon = ({
   size = 16,
   className = defaultTailwindCSS,

web/src/lib/sources.ts

@@ -4,6 +4,7 @@ import {
   Document360Icon,
   FileIcon,
   GithubIcon,
+  GitlabIcon,
   GlobeIcon,
   GongIcon,
   GoogleDriveIcon,
@@ -60,6 +61,11 @@ const SOURCE_METADATA_MAP: SourceMap = {
     displayName: "Github",
     category: SourceCategory.AppConnection,
   },
+  gitlab :{
+    icon:GitlabIcon,
+    displayName:"Gitlab",
+    category:SourceCategory.AppConnection,
+  },
   confluence: {
     icon: ConfluenceIcon,
     displayName: "Confluence",

web/src/lib/types.ts

@@ -12,6 +12,7 @@ export interface User {
 export type ValidSources =
   | "web"
   | "github"
+  | "gitlab"
   | "slack"
   | "google_drive"
   | "bookstack"
@@ -77,6 +78,14 @@ export interface GithubConfig {
   include_issues: boolean;
 }
 
+export interface GitlabConfig {
+  project_owner: string;
+  project_name: string;
+  include_mrs: boolean;
+  include_issues: boolean;
+}
+
+
 export interface GoogleDriveConfig {
   folder_paths?: string[];
   include_shared?: boolean;
@@ -189,6 +198,11 @@ export interface GithubCredentialJson {
   github_access_token: string;
 }
 
+export interface GitlabCredentialJson {
+  gitlab_url:string,
+  gitlab_access_token: string;
+}
+
 export interface BookstackCredentialJson {
   bookstack_base_url: string;
   bookstack_api_token_id: string;