From 90828008e1d330d5caa29555c9aef6e021449eea Mon Sep 17 00:00:00 2001 From: nlp8899 <142526738+nlp8899@users.noreply.github.com> Date: Wed, 11 Oct 2023 23:10:01 -0400 Subject: [PATCH] Document360 Connector (#552) --- backend/danswer/configs/constants.py | 1 + .../connectors/document360/__init__.py | 0 .../connectors/document360/connector.py | 171 ++++++++++++ backend/danswer/connectors/factory.py | 2 + web/public/Document360.png | Bin 0 -> 11137 bytes .../app/admin/connectors/document360/page.tsx | 244 ++++++++++++++++++ web/src/app/admin/layout.tsx | 2 +- web/src/components/admin/Layout.tsx | 10 + web/src/components/icons/icons.tsx | 15 ++ web/src/components/search/Filters.tsx | 1 + web/src/components/source.tsx | 7 + web/src/lib/types.ts | 14 +- 12 files changed, 465 insertions(+), 2 deletions(-) create mode 100644 backend/danswer/connectors/document360/__init__.py create mode 100644 backend/danswer/connectors/document360/connector.py create mode 100644 web/public/Document360.png create mode 100644 web/src/app/admin/connectors/document360/page.tsx diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index 319170a8b..977682f87 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -60,6 +60,7 @@ class DocumentSource(str, Enum): ZULIP = "zulip" LINEAR = "linear" HUBSPOT = "hubspot" + DOCUMENT360 = "document360" GONG = "gong" GOOGLE_SITES = "google_sites" diff --git a/backend/danswer/connectors/document360/__init__.py b/backend/danswer/connectors/document360/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/danswer/connectors/document360/connector.py b/backend/danswer/connectors/document360/connector.py new file mode 100644 index 000000000..9d39010bc --- /dev/null +++ b/backend/danswer/connectors/document360/connector.py @@ -0,0 +1,171 @@ +from datetime import datetime +from typing import Any +from typing import List +from typing import Optional + +import requests +from bs4 import BeautifulSoup + +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.interfaces import GenerateDocumentsOutput +from danswer.connectors.interfaces import LoadConnector +from danswer.connectors.interfaces import PollConnector +from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import ConnectorMissingCredentialError +from danswer.connectors.models import Document +from danswer.connectors.models import Section + +DOCUMENT360_BASE_URL = "https://preview.portal.document360.io/" +DOCUMENT360_API_BASE_URL = "https://apihub.document360.io/v2" + + +class Document360Connector(LoadConnector, PollConnector): + def __init__( + self, + workspace: str, + categories: List[str] | None = None, + batch_size: int = INDEX_BATCH_SIZE, + portal_id: Optional[str] = None, + api_token: Optional[str] = None, + ) -> None: + self.portal_id = portal_id + self.workspace = workspace + self.categories = categories + self.batch_size = batch_size + self.api_token = api_token + + def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]: + self.api_token = credentials.get("document360_api_token") + self.portal_id = credentials.get("portal_id") + return None + + def _make_request(self, endpoint: str, params: Optional[dict] = None) -> Any: + if not self.api_token: + raise ConnectorMissingCredentialError("Document360") + + headers = {"accept": "application/json", "api_token": self.api_token} + + response = requests.get( + f"{DOCUMENT360_API_BASE_URL}/{endpoint}", headers=headers, params=params + ) + response.raise_for_status() + + return response.json()["data"] + + def _get_workspace_id_by_name(self) -> str: + projects = self._make_request("ProjectVersions") + workspace_id = next( + ( + project["id"] + for project in projects + if project["version_code_name"] == self.workspace + ), + None, + ) + if workspace_id is None: + raise ConnectorMissingCredentialError("Document360") + + return workspace_id + + def _get_articles_with_category(self, workspace_id: str) -> Any: + all_categories = self._make_request( + f"ProjectVersions/{workspace_id}/categories" + ) + articles_with_category = [] + + for category in all_categories: + if self.categories is None or category["name"] in self.categories: + for article in category["articles"]: + articles_with_category.append( + {"id": article["id"], "category_name": category["name"]} + ) + for child_category in category["child_categories"]: + for article in child_category["articles"]: + articles_with_category.append( + { + "id": article["id"], + "category_name": child_category["name"], + } + ) + return articles_with_category + + def _process_articles( + self, start: datetime | None = None, end: datetime | None = None + ) -> GenerateDocumentsOutput: + if self.api_token is None: + raise ConnectorMissingCredentialError("Document360") + + workspace_id = self._get_workspace_id_by_name() + articles = self._get_articles_with_category(workspace_id) + + doc_batch: List[Document] = [] + + for article in articles: + article_details = self._make_request( + f"Articles/{article['id']}", {"langCode": "en"} + ) + + updated_at = datetime.strptime( + article_details["modified_at"], "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=None) + if start is not None and updated_at < start: + continue + if end is not None and updated_at > end: + continue + + doc_link = f"{DOCUMENT360_BASE_URL}/{self.portal_id}/document/v1/view/{article['id']}" + + html_content = article_details["html_content"] + soup = BeautifulSoup(html_content, "html.parser") + article_content = soup.get_text() + doc_text = ( + f"workspace: {self.workspace}\n" + f"category: {article['category_name']}\n" + f"article: {article_details['title']} - " + f"{article_details.get('description', '')} - " + f"{article_content}" + ) + + document = Document( + id=article_details["id"], + sections=[Section(link=doc_link, text=doc_text)], + source=DocumentSource.DOCUMENT360, + semantic_identifier=article_details["title"], + metadata={}, + ) + + doc_batch.append(document) + + if len(doc_batch) >= self.batch_size: + yield doc_batch + doc_batch = [] + + if doc_batch: + yield doc_batch + + def load_from_state(self) -> GenerateDocumentsOutput: + return self._process_articles() + + def poll_source( + self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch + ) -> GenerateDocumentsOutput: + start_datetime = datetime.fromtimestamp(start) + end_datetime = datetime.fromtimestamp(end) + return self._process_articles(start_datetime, end_datetime) + + +if __name__ == "__main__": + import time + + document360_connector = Document360Connector("Your Workspace", ["Your categories"]) + document360_connector.load_credentials( + {"portal_id": "Your Portal ID", "document360_api_token": "Your API Token"} + ) + + current = time.time() + one_day_ago = current - 24 * 60 * 60 # 1 days + latest_docs = document360_connector.poll_source(one_day_ago, current) + + for doc in latest_docs: + print(doc) diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index 2d3af2003..c9824062e 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -4,6 +4,7 @@ from typing import Type from danswer.configs.constants import DocumentSource from danswer.connectors.bookstack.connector import BookstackConnector from danswer.connectors.confluence.connector import ConfluenceConnector +from danswer.connectors.document360.connector import Document360Connector from danswer.connectors.danswer_jira.connector import JiraConnector from danswer.connectors.file.connector import LocalFileConnector from danswer.connectors.github.connector import GithubConnector @@ -54,6 +55,7 @@ def identify_connector_class( DocumentSource.GURU: GuruConnector, DocumentSource.LINEAR: LinearConnector, DocumentSource.HUBSPOT: HubSpotConnector, + DocumentSource.DOCUMENT360: Document360Connector, DocumentSource.GONG: GongConnector, DocumentSource.GOOGLE_SITES: GoogleSitesConnector, } diff --git a/web/public/Document360.png b/web/public/Document360.png new file mode 100644 index 0000000000000000000000000000000000000000..abeac4a39bba7bc0a1fba38ed17c3aad0c7a9404 GIT binary patch literal 11137 zcmd6Nc{tST`~NJ)jBH~GS(A{Eu}_91MM5-{ecuXU#Edn~Fr}>FAlX{9*hjLY@HVy+ z(J3ZVkx{3dCd)Vy$5P*CI^XMieZIfz`u+dCT&`i>&vQS|`@Wz1zF+t2ey%znw-wzc zzYPY1iP{}KOoYJ@ZZH^}fDr`Wz!!eDfDeH)R*qIMSW~X>raub&zccVC(Gdnq)PTXL zbQo+Md`0~W1|#EOFs>gAW_BG0lO>gPoG=Fu1Ox1C55s=)e{Oc(F96@5W9=M|pg*Dz zlG5nBOHGw9n3js&VXIT|Z|A2HuKSQLtS+qvsROC%;t7i>fx^u`ETkuYw_QyM%07R0b;$pj2PWziShdQZcKRs%_Uob|{iA9- zlhgJi*-F=st*w$^assWeQffAo$kF$6TuH22e>~$mjX5VM*le(}=pqS2T3i1}x;ZK$ zBGQ-hhkOE25tgii&m%3lBqQlW<4ex1ty^^Efbf^{`#gIug(2S+H?SZgN12nt>ERBh zQ(IVzQ|JB((`zi?(zV@Q_2C8ZzcD;+e^!8Lxa!GGO~1q5%CfL=`;;i)X0yjD^Y1FR zzSoG>y7iTwI)}iEb@?4os=8y=8Q8{v*sTV6ABlHQ65(Zxk;W+1y7+q70&)QrIa2UJ zgHyzvo$ze0de6~4M1Jl`zjJcFdmX|1%yu8gTSI#f&2$K_iSo4lB>eAwJt6Fx?4!vN z5-u_e>5(rt*9w6bBVJ^bAIUa7%x-Q;Wz>*S&XPkD59=Jzf#^Bg|1~y8Fh$^bX8Do& zW>ma%J915nXV00Xvvi|0k7N1on8c`BARtzKs|^ci?vNHM9O-{&Qxu9@u4)&I->E$| z(v)`WZ3W7n+x3xF(q#QWbIxYZoGt7kY*N&#K(jxW7EVKVX74dhYs0Ygqa+qQT6(sU zIl4{F5AN5aBasb)T|!=!gUj|HF$ z5=*q*xYxDa2*sNHA{+!Ki|t-D=q{4HeNEgiuT3m&^lO#b%yq%)r0=MlOr=A+CN;c{ zT*r2cw5OBytaQ<}mSlAJep8(!2CI-J(AQsK88;Zj6;Ea~VFD2+snOerG}$lY!&Me8 z8n+mn`Al)FOBXuAouw07U~X3jxksf&Q`N~Yu2L1QvLeDUALSZ$ipY10xdq?K|M1Yy z&fD4I7CClKk?U>^U4TB(^>5AcjRpX(e<`^D71 zT-9+S#wmJqR+*iHFYFW%-qQ24a=GUjke+X30QR92^RF3oG|RNtUeHTU%V<}l_*2TL zzP|F6Es!{MnfCRefwv0bTGh*C#7>ruQ-%d?`f9m#y@|@2G_SnQLFKBF%`3J$qiTHw znR8X?Q2jT#nS-C5Bup4dHkso@pplexBpNut>jdoD9?}Vyd%68$YN5$snXS(=)Dzk7 zclvYJHL1f>efw4fg^NzzG1t0>&>S4S^_nfRsmYtEb6Ct#vLRjo6FR@|F0lrNiH8PLiiE7H1^$B);W=3XUH*&`FO)O9@Hl@x9r(|>BKmoT(xGevO!AJk z`}dSymuaG)AipTiGf2_T#9w?S)^G1jDNh%&Va|-@!o-pG)PBe;cR7#pJXM`FyJz#p ztMB_z2TBUR++bV5{;s-ySWfNyZj#Z=t*no z6g?NJxfVrpvNc7N7sf=&Jt6-yB?iw$bZaQWzXftC_DB zJ!`Vw(_zKg#o=B}OlQ@w7GXPh=b2Je#});i7qjNI>)%yt$u9TA`^41AnIILc&7&3N z3G~|=TK+3xhcsGTlj`3KCL`lfBl}$6-4NZ><*k5Vl&^Wj^SpgvKu`X_BK8K^-bQLs zpR;+@Bkx&3faOl~fg@|1;ib=@Wm+SmvDU`!s6~IOdiJyyl*Lw+W{F?Fd0%k33NL~e zk4H0X=fsoC#5ByK6DSqZhJopM+#n$J)>~wZ&dgHm%Khz7cS!L1(YEOm71D!}Iq&zU zeV>$^EPMmKIp{L={kOza@Sc?x(LYajZ>ai*CApsd#9f9mzjPk5REpOmHE6qYx69~+ zCV!R~C7-qCPJk31J|%K!e46~`fQv6XZ5 zJNy6~zu;XDaEIA%n*EW`Sm3O)ktrg$czQ4t^8Bk?vfI*sY-6S=?PxiOf8_)iUW&u_2N82}hORHlwd{s{j$4|q%;`teI4&j-*Qy#k20;|w>68KI$Zg7V zZA+=c%?49{WSdrK_V>^)xIP*DaZusIVUB1{r1QJ@=xb(%NYnP!S^`ytyfoi^nep>z z&u-2L)Et1$8(;!SAtv+YL2aIbv=Pyih-#lSR=9a2VqtzjC7?pu79MpoE~lq@mYeve7@qc)4iNO3+XX(7o`YSjO0P!GaQuI$4u>qikl7QhiZiEv1_qAr|X>r zM@xO5P_l}@=C3>x`tkxM_-|S7A;WaW<4#|}j2xCKz0=J!IJiTdA{Xik%q(|W_H!Wb z>lwB;-S8g|b?uXw<+@f_c7EUx7uG+9n&^L%ZHsqxV&jj^Sr%Wmr!7;D z0K+}`fI1}q#6n2c6J;tnWam`9jcvnxUW*0+tBCQ|_`QXflXLl#RY_*Al>8fiD6`3r zpMGAkl@N;C35Tug;0?n=v3n@jB5Sv~&VQ;oFR~SVeL3Dyu%c44f1<-!!@9=)q-oVL zM~cXdOCGbn=UqVhUvRyhbN{?*P+t>h^Z@WHd#Wj3`wTmP*`-e2 zqKax&5l`4dJ}pcvccZbjLn7XYa;4~O4Ql7i+Gb&b++Que>*9%J2mdna)A z>@@<7++_V|(#T6iCzx*zhfhaH9k?~`@WsoKg>#6wZea=r3?olUIxgBTufmjlo1Em_DQzymEVJ!lr7^*-P2gf*`9tadv!nK&;#6@)V*jN9ne$xuCfs;jq2#vmqmjVCZ2VT$Ohz zyi6LL4iUKU6vwC35xjiU4mS{t9~pvGCq8q2)cV)bJOk7J+U{N=CK-|udhYdy0|$-E zQ1)+I9@F#hl1@Capj=UI#<1lGhlkC#>EtLKdb>lo=u!9!f$v~V)yQ69&ncN1o7JWc zB{!M>7OV+^jG(#b{RnZ-YlYQ;DDDbu=9xqGd%Yzq;a{>JcVe8dd$XjktAMlMA?O7@ zGV4$c-@#)L4BtN1Ih!;3GcGOTXTip2b6_FhfMM-HjBfKn z@=uh#m++(tSYG?r<6fbK-bi)x&b#Zn0u5J`Zlg@#KR+Hg*g^x5)(Qaazu5cb|GZb{ zprrW+iWR|Bkd%-^;F57+AW0|#hfK&U# zc7Ym2-Qs1epATO%Vr!0b*b8$3oMh(BiIj%|Yi=(0;`+g$A^xBOT5llOS330_IC`$E zmxq7CykOsksUyMA8~UBJs@K3ZonjS9Bp@nUg<(#)l2TOoC5-bC%KdB0aWS0s0$U+J zSA3{i86etmX()C-Qw6M(Z}?@^mLO3}VVku`H3YIU<(){RjW0ZQZTcoRy_ zWwyZP+e~l6px{IT&y=nfm^lBGNd`E!8-!5DIUG?57~y*eJchqV+oicSo9bh)-%&IF zD*qT;1R&9Yg2L+F(=0>&D6wjvRCH~z{BG^BB-$#>w`yzFTI`}uCHd2_&Vf|*h((NF zH-B$!PzGD|+hL%2^n3JEg|i>`q>Vt^Aq8sYQ_*@KfKOWEjBv&vIJ5&a0m9hX!J6*U zM#LG(3f!bLVQi0uNHUy{Nh^bb--d5I+YQ3?*2M(koDF`Ltv};aRABlaqrt&u*9*V2 zK$Y-W?2pkKW<&l+Ucuh4NzDe{l7wsHj^JF@maGZPFt)e%bQ*(FMKD<&ue2t~FreIv z5J3o2yHP%+g=9J9x0 zO7*RUB&w0LdA@YDi!zA>XFej0An#&0@#mG zi``xV(H|BsN!rWX#o141y}dwvX8bf$l)j8y++C&;FG>m|_c=N7Kq1_os*Wm6A_6DqL&l?)#nsH)ftn& zlR1~Uv#;Ad)KM}zT5Pmik*Z<;~%U?Ys{?U4(@SP zz_k4+!n)KJxkd#By4-8B)Tzz`Wf;jQlB7ZrA4%Ni=uNCQv9M|Hj5{-DPSlHbNRw7{ zTx;+qS#U~7`9@-3DLkTE8P*DG>&Ugq9~ktcbCZCDFG)?NTV!lBj!PyG58~g3HBU=F z2;I!_-d38Jn*}290!*eKgsoRIbXv_5;9(YkblB!}d|m;>{mG;8haAElnQP_1N0C3M zpDVf_9QdqzNT@<^4#Z5P2)z6Eq1c$Yov)4peG5E=tr0kHpo`e=V#l%rdWgP2-FtZs z+>*0Xl-}I0_SvT{bXJHZOI~-~`qoQF-o0xBK*;n6yh0f)rPCw25ca@at3qURJI|%n z(lGW?`q5fYRCD9d*wN6FZk-*L6ilcmYyYhmumggd`<%QYd?W~W9;wObtUT$BhW=t) zf;!l$RsH}+!+pdzMM-~&r3~-%e{^LW&mVGzqxK$FO7kpJuhRIvAYdEMd}nQn;+OE) zwXDO{(O@0V1fl~dtIO}GzJ|4dTK0#F(t#~$PBs+&W@Uau6hy1Xca^}7D@PEY`Ps#u z^z(G!C%YzvYY5Yj;6&RwbMJLIaSiEOQ^}kL`D|lArZ|Izo2_Zq{2+P>8e*#&-F^dY zP`@M)lacLmZod_V*3Md zm`LI7=LH|+0m2nys=hA;AUfprGkUsh&aOJFr7XVCS>Vk4hd{YnM|!$pmCZxHYG0)s zaG-S|&88dY;s#8P2f3`6=2$fc{4uYeqnPq&| zdZy&8aA!-sMzdeFijolENPy2`%mI$FF)F~lKKSn4(Rec=ALXvU)Q2hnB!Lo{0&%k# zIv`)+9fGmRU*G3)X6)a`#05r)?@;F>mLmJ_8- zgp%^Y9&pc=NCnKEeSm6@a~3_*L+mcG#|OFi^QzG1rW0_;5S;a!svKFmq0wD{kgKQ7 zFXhet{8sAQccAt-fh>*Fqg)a65T5*FMD1dg8Bl(dn5IM3Lflu(>{vCerZe@0n)#6j z^A-+xwnvx|*tkuF^phFtOaEc&lYsO;u6#!m%PzL4wax6Hhrz+)XI*qp`nFov)qb&C z66G*Oz6z$Y(hF)qb;H#}e*g$IlLVbr`(y{_B|$y_bN>;-kpaA`DF3-F=ZLVMFm4)t zC%Z+3mkJJQ*bZJTh$1h6S@ceoUUupiU4UsX?P3LK!Uq9nbY7Jhees+83*eyo!gzIl z!c8Y-I*XnZm30)VY)rge`SFK#3n+WH0xmMWRC~(lr*({9G(p-xEe~vI+D}{RTY+fb z=cr!@)bvbn$9|yjKkS#PXnB*un`R)!0Sw2L9Yj0<%fY@<|7o)R>naGZsHfY`_-6m< z;+1eZ6R2&OA92gFDi>k70EOR6+&8m)%E?Gz695TOP!8VtgtFc8lbj*=zL*jodshrd z!THnoMapt)KV}7?NP%^Ua9LJJ^UGCNXTZZ55$}9X*ROiqr z;3J+aAHRokPpI`WDW~+trxIuw;338*s0ELXGnLpa>)U07JXA;>ZVVYa@` z;n~nP6HfU-8@V9%a3m<$z2~sofR==wisJWBiUd7HHvz05^4Yee(_io^Yk`2W9$=Sr z_^FXR=df=_wW-Ab%96STBLTO{r!Ei=4V0b_eKX_eQD3}DlDgyi%6DT#9&qT^LfZF} z`bJ;*c~Ub$$ZBakvI2^?N7-Rzy|z?SY{=rbp`M4sRCHiE7J$Yb@xr9AIUUh9xivem zf%^dWeGCfj(AOSI<;70?0|ho2vN#l`{{D8 z>C@>s-8K&yeS%8Wg`|=qPJNL^Je=1})oMyJn#7W{<~Pg= zS_46qQvDY8b3gM(N&fe9)b3(N#$Ax*@t8FjfYm^nz?#jk`ZRB#7u2j)#-lR^90%si z%i4f5@|Q|s=qwv>Mk=8F(Bi=8D@n^cQ%TC|0Ckf2NSC@Q@qOGVfjTS08K`qe2Su(e zzd|?zj}p)r934X_V453W4~Xr`1|%}q+2!6>TBcB~yD$1i{0TSIB;YG7hYAp!Pjt-_ zak=I9)guKPfW>OkK;@MV!~vDT*X%i+^qDJ;=T%6+=oh%|bc>{zKy>I;D-OZEBys0s zK1){v2>NiUo}%!zrVerieGA6v;dmU84Y#c>^To2>8*m!Y`lb z7N#m{icV7RHtQUcmU!HOp@I)MP9;?qyr#i4N(R(`EWj#Eg-2oDXm+Sn>FjWwUW$_0{T;`QIl%=HmDxy=ThW|V8? z{MHdb%#Jrf=_GLA`1QF9qBDN)BHfVBbGMeh*O+T*!0*FlC32u9FY@P&UjbmsuP0vv z)O4i}4RooN=q`l$+e>lHiX^{4S>Q2J@!C8+NC@bNP2?OJyz;;m9NVLy;t+V6i9>ZQ z^hQM8-4WBC2;RdMR;f8DlYD;8Tz6FkdJ&j@t)t%p@aahmE@+dTq{Yx&5hugr3qTL! zTW6DHgAd^8gE%O9_np}p-JYcmVSJ}qNX-(zRomcaCwi^$izDPojgkBLhUQ6~9{6!R z@koU<8mzBS)cOsYphP(a^3HcNL@pVDA z^^HymvQpaAXw8^U+<0?ahUaB!Bcd484vx}*7Dl+29013>SZJg6z+t%^xCnGNdB*`O z{Oz!oFd3c^XFD`Z4P>~H-;F>h_mbn|`^$*2fJjxZ5#_ng+C|IoBAGt7-sEmXtbY7a zs|ldG8Lrsb;RUQhXf8XcPJ=1P0iGudqJ!gZu$loViEB-uV(1N2eNJhXWXLd~46ti@ zZW;=;!K66utgn8PzQ!-RP|1cMwcdXG%v&j@ASoFX32KuD$w~|YztQ4@B}@qX+;OnI zzm4Eu)h|RxUaxdyH@2#XozD3r#xag4VIPqEw=+%53=U3=tB_vYl@$}O@XmoYeJ}o7 zh8=6ruz(aEYeItgKVpti?+9t1LL-s}#I#M|zOcwFrNbFL+jYp#x7U4sP;rGI=msK5 z4OhnfnB4py-}8-6T@V?oozP*{jK8PvYEf9Y8+JPi5j8yYZH8?%Fbuk({yoG&9k&sH z$h)9YjKYj~M!D^U&;z^oD{TiW|@=<=VmAn3WCr^|^05G8`A=6oDS;pTGP(~^|S5|l>J7IjHLEt?g0xmSA zjX&deVz(8I6Imxyolv7FpR3E|pcDuEZ^d3$ekLOdg9->ulAk%p1gP-Dbdmt50W;WE zRreTvcDld=t)L`ocV7Z%Ij!xS)jK`7-wx0`Ex>VK?5dMCSLse+i4?;sM~7kq=XMec z+JBj6qYpG~1W9Hyb3GpeXY=-&!Wv%;eJf&Hb^O{-97OB|&14MEhm%bY45-p*SHVA- z7%m*w0_VY*wPgB2@W8gG7;zn!7;^`?vq|ni@<|Xt1OdFz<2Z1(pr`DFs)#z|4|qG| zIREN*@EW~;+C}61TUqY(-t4>gKsS^?@&VB)AU*E}pk$2?S4bDorcnl*gm`=B5d(}D zCK<0iHpqWGdLK04D|gj78RNg{0f(Rh*-Guw@qoQu9TYQo`QMR6;`HE@qN7|lPhL^6 zCS+ZLSCOLguLgIsbbH3AG1U98eL^)?8xTLtw8!2BqVwJYF678H&3;ADx_RU4 z8R&_w>#qi|5a_AdL4&kn_(`B38Ks0=P6kbHN6=3UfnPk?88SGkHs9xj(RS_5RyrJJ2sC#D*jN*(g7C>nvu1gDQtheF z&#Pxwf()F`E#ibz#gT)3{7E}OuYU`89q1Hu0l`r%TytMw8nvnSRQ*ASCpuYZEy(4b zJKdqa1f9L;;av70>%H^y{8Qej<29`E;=waNn(2RoX4KkA<`8QVSi{>51+EsH0=ojeyhK0{bAdOc!&Km-eQ#XC zx;8+L$OOrqKUo|ADLJt8C0;CGGGh2zk1!;mc&m`{LR-E@4p{jv&>$j86?FV8DaxU; z%80I`ay{T3LR0^x_CMe`iX0YT6ceh-zWW-ujl+dwg@KWUuB89rHk($=af9*;* zS$Hk!h175SI|phHa-RGn%!#5Q17`Cm0oZn-HCaxndkFwGN4ar;c|!y2b(s;i5@x<& z1M~>~vg71{t1q*80I0q_4lY&*-Q{laJ9}Wdlry3dxo=g+s@ffBL!E!=V->at`<_#7 z`}S%QZ5cC*;hXQ3lsjX5_3tVez65t=DHwXhE~%2xL3g1y6?mm}4uAyN9SXt@H=KW- zMt3SEC!L!!e}#DsFUDrDUI3NJfx8Ojpni-3?g*O^>S;qX^Vj#&%oy`<+c4JIByyPHcjVWkx2A$8wZWw>+mwMWI6m^it?A} z*GN-@FLIw?kr~A*O!MDLaG5F&JA*I;i?lB7@&A)~30%(z_>@sN9so*!rxB4tDTw%e zfB-BwZKXsl5Lf*8YSAjAXY16Z;^{C1o!MR}!4m04GVrj}B* zsg>|mc(`|UzLBJGkq00(r>K^Q_u}TpVAO9T^mi%ZunBNG!PI&FKEkB^{;^kQfv~+G zV+y{vZdY%2oih(>QT0>v6lOIFon%GT{YY~E6)JxlKQ5!wa)>-RVhF-nG2%)8XqC&3 z-E);5+jQXI{*`9k@hLmDYYX#`di6dT9b+GGXlWA?g6YcJ3AyqS?MLeoE?AR0(?XN* zNvAz7_B2+JmfciP5AhB-Uf7u;PARQ3aQ4KhG`a=xE7ceZq>KJF{H9|sS^Sop#Zk6l z3&Qk{eFl_z*QO>?=s*GP9e0)2Uh30~Vjte)P>M;qb@`%~Yv*gpW8>Cz1liJmCG^P3 zv^H&p6|TnU-=$uyrQ;S>^9&Ga_SD4zu{$0S{^3{UeX>2Q+2^{7oFBDb;CGb|^k`P= z0v%^t=hDiO#kdvW_!7xMpeMa_& z4p}{~3;>T!|Kss#qTT-}DK>(n7ZDm6>ie&1_@==b;|z3-bmPti#6|?|1Dn{PN=^VG OV0K52A8t70m-=6wdq5!o literal 0 HcmV?d00001 diff --git a/web/src/app/admin/connectors/document360/page.tsx b/web/src/app/admin/connectors/document360/page.tsx new file mode 100644 index 000000000..bf9266d51 --- /dev/null +++ b/web/src/app/admin/connectors/document360/page.tsx @@ -0,0 +1,244 @@ +"use client"; + +import * as Yup from "yup"; +import { TrashIcon, Document360Icon } from "@/components/icons/icons"; // Make sure you have a Document360 icon +import { fetcher } from "@/lib/fetcher"; +import useSWR, { useSWRConfig } from "swr"; +import { LoadingAnimation } from "@/components/Loading"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { + Document360Config, + Document360CredentialJson, + ConnectorIndexingStatus, + Credential, +} from "@/lib/types"; // Modify or create these types as required +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { + TextFormField, + TextArrayFieldBuilder, +} from "@/components/admin/connectors/Field"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { usePublicCredentials } from "@/lib/hooks"; + +const MainSection = () => { + const { mutate } = useSWRConfig(); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: isConnectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + fetcher + ); + + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: isCredentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) + ) { + return ; + } + + if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { + return
Failed to load connectors
; + } + + if (isCredentialsError || !credentialsData) { + return
Failed to load credentials
; + } + + const document360ConnectorIndexingStatuses: ConnectorIndexingStatus< + Document360Config, + Document360CredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "document360" + ); + + const document360Credential: Credential | undefined = + credentialsData.find((credential) => credential.credential_json?.document360_api_token); + + return ( + <> +

+ Step 1: Provide Credentials +

+ {document360Credential ? ( + <> +
+

Existing Document360 API Token:

+

+ {document360Credential.credential_json.document360_api_token} +

+ +
+ + ) : ( + <> +

+ To use the Document360 connector, you must first provide the API token + and portal ID corresponding to your Document360 setup. For more details, + see the official Document360 documentation. +

+
+ + formBody={ + <> + + + + } + validationSchema={Yup.object().shape({ + document360_api_token: Yup.string().required("Please enter your Document360 API token"), + portal_id: Yup.string().required("Please enter your portal ID"), + })} + initialValues={{ + document360_api_token: "", + portal_id: "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + } + }} + /> +
+ + )} + +

+ Step 2: Which categories do you want to make searchable? +

+ + {document360ConnectorIndexingStatuses.length > 0 && ( + <> +

+ We index the latest articles from each workspace listed below regularly. +

+
+ + connectorIndexingStatuses={document360ConnectorIndexingStatuses} + liveCredential={document360Credential} + getCredential={(credential) => + credential.credential_json.document360_api_token + } + specialColumns={[ + { + header: "Workspace", + key: "workspace", + getValue: (ccPairStatus) => + ccPairStatus.connector.connector_specific_config.workspace, + }, + { + header: "Categories", + key: "categories", + getValue: (ccPairStatus) => + ccPairStatus.connector.connector_specific_config.categories && + ccPairStatus.connector.connector_specific_config.categories.length > 0 + ? ccPairStatus.connector.connector_specific_config.categories.join(", ") + : "", + }, + ]} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + onCredentialLink={async (connectorId) => { + if (document360Credential) { + await linkCredential(connectorId, document360Credential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + /> +
+ + )} + + {document360Credential ? ( +
+

Connect to a New Workspace

+ + nameBuilder={(values) => + values.categories + ? `Document360Connector-${values.workspace}-${values.categories.join("_")}` + : `Document360Connector-${values.workspace}` + } + source="document360" + inputType="poll" + formBody={ + <> + + + } + formBodyBuilder={TextArrayFieldBuilder({ + name: "categories", + label: "Categories:", + subtext: + "Specify 0 or more categories to index. For instance, specifying the category " + + "'Help' will cause us to only index all content " + + "within the 'Help' category. " + + "If no categories are specified, all categories in your workspace will be indexed.", + })} + validationSchema={Yup.object().shape({ + workspace: Yup.string().required( + "Please enter the workspace to index" + ), + categories: Yup.array() + .of(Yup.string().required("Category names must be strings")) + .required(), + })} + initialValues={{ + workspace: "", + categories: [], + }} + refreshFreq={10 * 60} // 10 minutes + credentialId={document360Credential.id} + /> +
+ ) : ( +

+ Please provide your Document360 API token and portal ID in Step 1 first! Once done with + that, you can then specify which Document360 categories you want to make + searchable. +

+ )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+
+ +

Document360

+
+ +
+ ); +} diff --git a/web/src/app/admin/layout.tsx b/web/src/app/admin/layout.tsx index 026139fe6..43d22001c 100644 --- a/web/src/app/admin/layout.tsx +++ b/web/src/app/admin/layout.tsx @@ -6,4 +6,4 @@ export default async function AdminLayout({ children: React.ReactNode; }) { return await Layout({ children }); -} +} \ No newline at end of file diff --git a/web/src/components/admin/Layout.tsx b/web/src/components/admin/Layout.tsx index c5dfaeed2..02a790ce3 100644 --- a/web/src/components/admin/Layout.tsx +++ b/web/src/components/admin/Layout.tsx @@ -22,6 +22,7 @@ import { HubSpotIcon, BookmarkIcon, CPUIcon, + Document360Icon, GoogleSitesIcon, } from "@/components/icons/icons"; import { getAuthDisabledSS, getCurrentUserSS } from "@/lib/userSS"; @@ -210,6 +211,15 @@ export async function Layout({ children }: { children: React.ReactNode }) { ), link: "/admin/connectors/hubspot", }, + { + name: ( +
+ +
Document360
+
+ ), + link: "/admin/connectors/document360", + }, ], }, { diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index f3be56a73..f8217df56 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -43,6 +43,7 @@ import gongIcon from "../../../public/Gong.png"; import zulipIcon from "../../../public/Zulip.png"; import linearIcon from "../../../public/Linear.png"; import hubSpotIcon from "../../../public/HubSpot.png"; +import document360Icon from "../../../public/Document360.png"; import googleSitesIcon from "../../../public/GoogleSites.png"; interface IconProps { @@ -452,6 +453,20 @@ export const HubSpotIcon = ({ ); }; +export const Document360Icon = ({ + size = 16, + className = defaultTailwindCSS, +}: IconProps) => { + return ( +
+ Logo +
+ ); +}; + export const GoogleSitesIcon = ({ size = 16, className = defaultTailwindCSS, diff --git a/web/src/components/search/Filters.tsx b/web/src/components/search/Filters.tsx index ec5be9b17..737c496d8 100644 --- a/web/src/components/search/Filters.tsx +++ b/web/src/components/search/Filters.tsx @@ -29,6 +29,7 @@ const sources: Source[] = [ { displayName: "Zulip", internalName: "zulip" }, { displayName: "Linear", internalName: "linear" }, { displayName: "HubSpot", internalName: "hubspot" }, + { displayName: "Document360", internalName: "document360" }, { displayName: "Google Sites", internalName: "google_sites" }, ]; diff --git a/web/src/components/source.tsx b/web/src/components/source.tsx index 88744a28a..145aa630a 100644 --- a/web/src/components/source.tsx +++ b/web/src/components/source.tsx @@ -16,6 +16,7 @@ import { SlackIcon, ZulipIcon, HubSpotIcon, + Document360Icon, GoogleSitesIcon, } from "./icons/icons"; @@ -123,6 +124,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => { displayName: "HubSpot", adminPageLink: "/admin/connectors/hubspot", }; + case "document360": + return { + icon: Document360Icon, + displayName: "Document360", + adminPageLink: "/admin/connectors/document360", + }; case "google_sites": return { icon: GoogleSitesIcon, diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 6264fa198..860646e5c 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -23,8 +23,10 @@ export type ValidSources = | "zulip" | "linear" | "hubspot" - | "file" + | "document360" + | "file"; | "google_sites"; + export type ValidInputTypes = "load_state" | "poll" | "event"; export type ValidStatuses = | "success" @@ -115,6 +117,11 @@ export interface NotionConfig {} export interface HubSpotConfig {} +export interface Document360Config { + workspace: string; + categories?: string[]; +} + export interface GoogleSitesConfig { zip_path: string; base_url: string; @@ -227,6 +234,11 @@ export interface HubSpotCredentialJson { hubspot_access_token: string; } +export interface Document360CredentialJson { + portal_id: string; + document360_api_token: string; +} + // DELETION export interface DeletionAttemptSnapshot {