highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-10-09 20:55:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

Pagination Hook (#3494)

Browse Source 
* Backend changes for pagination hook + Paginated users table

* Frontend changes for pagination hook

* Fix invited users endpoint

* Fix layout shift & add enter to submit user invites

* mypy

* Cleanup

* Resolve PR concerns & remove UserStatus

* Fix errors

---------

Co-authored-by: hagen-danswer <hagen@danswer.ai>
This commit is contained in:
skylares
2025-01-03 17:32:55 -05:00
committed by GitHub
parent 66f9124135
commit c191e23256
36 changed files with 1237 additions and 644 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backend/ee/onyx/server/reporting/usage_export_generation.py
View File

@@ -13,9 +13,8 @@ from ee.onyx.db.usage_export import get_all_empty_chat_message_entries
from ee.onyx.db.usage_export import write_usage_report
from ee.onyx.server.reporting.usage_export_models import UsageReportMetadata
from ee.onyx.server.reporting.usage_export_models import UserSkeleton
from onyx.auth.schemas import UserStatus
from onyx.configs.constants import FileOrigin
from onyx.db.users import list_users
from onyx.db.users import get_all_users
from onyx.file_store.constants import MAX_IN_MEMORY_SIZE
from onyx.file_store.file_store import FileStore
from onyx.file_store.file_store import get_default_file_store
@@ -84,15 +83,15 @@ def generate_user_report(
max_size=MAX_IN_MEMORY_SIZE, mode="w+"
) as temp_file:
csvwriter = csv.writer(temp_file, delimiter=",")
csvwriter.writerow(["user_id", "status"])
csvwriter.writerow(["user_id", "is_active"])
users = list_users(db_session)
users = get_all_users(db_session)
for user in users:
user_skeleton = UserSkeleton(
user_id=str(user.id),
status=UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED,
is_active=user.is_active,
)
csvwriter.writerow([user_skeleton.user_id, user_skeleton.status])
csvwriter.writerow([user_skeleton.user_id, user_skeleton.is_active])
temp_file.seek(0)
file_store.save_file(

4
backend/ee/onyx/server/reporting/usage_export_models.py
View File

@@ -4,8 +4,6 @@ from uuid import UUID
from pydantic import BaseModel
from onyx.auth.schemas import UserStatus
class FlowType(str, Enum):
CHAT = "chat"
@@ -22,7 +20,7 @@ class ChatMessageSkeleton(BaseModel):
class UserSkeleton(BaseModel):
user_id: str
status: UserStatus
is_active: bool
class UsageReportMetadata(BaseModel):

6
backend/onyx/auth/schemas.py
View File

@@ -33,12 +33,6 @@ class UserRole(str, Enum):
]
class UserStatus(str, Enum):
LIVE = "live"
INVITED = "invited"
DEACTIVATED = "deactivated"
class UserRead(schemas.BaseUser[uuid.UUID]):
role: UserRole

103
backend/onyx/db/users.py
View File

@@ -1,4 +1,5 @@
from collections.abc import Sequence
from typing import Any
from uuid import UUID
from fastapi import HTTPException
@@ -6,10 +7,14 @@ from fastapi_users.password import PasswordHelper
from sqlalchemy import func
from sqlalchemy import select
from sqlalchemy.orm import Session
from sqlalchemy.sql import expression
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.elements import KeyedColumnElement
from onyx.auth.invited_users import get_invited_users
from onyx.auth.invited_users import write_invited_users
from onyx.auth.schemas import UserRole
from onyx.db.api_key import DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN
from onyx.db.models import DocumentSet__User
from onyx.db.models import Persona__User
from onyx.db.models import SamlAccount
@@ -90,8 +95,10 @@ def validate_user_role_update(requested_role: UserRole, current_role: UserRole)
)
def list_users(
db_session: Session, email_filter_string: str = "", include_external: bool = False
def get_all_users(
db_session: Session,
email_filter_string: str | None = None,
include_external: bool = False,
) -> Sequence[User]:
"""List all users. No pagination as of now, as the # of users
is assumed to be relatively small (<< 1 million)"""
@@ -102,7 +109,7 @@ def list_users(
if not include_external:
where_clause.append(User.role != UserRole.EXT_PERM_USER)
if email_filter_string:
if email_filter_string is not None:
where_clause.append(User.email.ilike(f"%{email_filter_string}%")) # type: ignore
stmt = stmt.where(*where_clause)
@@ -110,13 +117,101 @@ def list_users(
return db_session.scalars(stmt).unique().all()
def _get_accepted_user_where_clause(
email_filter_string: str | None = None,
roles_filter: list[UserRole] = [],
include_external: bool = False,
is_active_filter: bool | None = None,
) -> list[ColumnElement[bool]]:
"""
Generates a SQLAlchemy where clause for filtering users based on the provided parameters.
This is used to build the filters for the function that retrieves the users for the users table in the admin panel.
Parameters:
- email_filter_string: A substring to filter user emails. Only users whose emails contain this substring will be included.
- is_active_filter: When True, only active users will be included. When False, only inactive users will be included.
- roles_filter: A list of user roles to filter by. Only users with roles in this list will be included.
- include_external: If False, external permissioned users will be excluded.
Returns:
- list: A list of conditions to be used in a SQLAlchemy query to filter users.
"""
# Access table columns directly via __table__.c to get proper SQLAlchemy column types
# This ensures type checking works correctly for SQL operations like ilike, endswith, and is_
email_col: KeyedColumnElement[Any] = User.__table__.c.email
is_active_col: KeyedColumnElement[Any] = User.__table__.c.is_active
where_clause: list[ColumnElement[bool]] = [
expression.not_(email_col.endswith(DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN))
]
if not include_external:
where_clause.append(User.role != UserRole.EXT_PERM_USER)
if email_filter_string is not None:
where_clause.append(email_col.ilike(f"%{email_filter_string}%"))
if roles_filter:
where_clause.append(User.role.in_(roles_filter))
if is_active_filter is not None:
where_clause.append(is_active_col.is_(is_active_filter))
return where_clause
def get_page_of_filtered_users(
db_session: Session,
page_size: int,
page_num: int,
email_filter_string: str | None = None,
is_active_filter: bool | None = None,
roles_filter: list[UserRole] = [],
include_external: bool = False,
) -> Sequence[User]:
users_stmt = select(User)
where_clause = _get_accepted_user_where_clause(
email_filter_string=email_filter_string,
roles_filter=roles_filter,
include_external=include_external,
is_active_filter=is_active_filter,
)
# Apply pagination
users_stmt = users_stmt.offset((page_num) * page_size).limit(page_size)
# Apply filtering
users_stmt = users_stmt.where(*where_clause)
return db_session.scalars(users_stmt).unique().all()
def get_total_filtered_users_count(
db_session: Session,
email_filter_string: str | None = None,
is_active_filter: bool | None = None,
roles_filter: list[UserRole] = [],
include_external: bool = False,
) -> int:
where_clause = _get_accepted_user_where_clause(
email_filter_string=email_filter_string,
roles_filter=roles_filter,
include_external=include_external,
is_active_filter=is_active_filter,
)
total_count_stmt = select(func.count()).select_from(User)
# Apply filtering
total_count_stmt = total_count_stmt.where(*where_clause)
return db_session.scalar(total_count_stmt) or 0
def get_user_by_email(email: str, db_session: Session) -> User | None:
user = (
db_session.query(User)
.filter(func.lower(User.email) == func.lower(email))
.first()
)
return user

16
backend/onyx/server/documents/cc_pair.py
View File

@@ -1,4 +1,3 @@
import math
from datetime import datetime
from http import HTTPStatus
@@ -48,7 +47,8 @@ from onyx.server.documents.models import CCStatusUpdateRequest
from onyx.server.documents.models import ConnectorCredentialPairIdentifier
from onyx.server.documents.models import ConnectorCredentialPairMetadata
from onyx.server.documents.models import DocumentSyncStatus
from onyx.server.documents.models import PaginatedIndexAttempts
from onyx.server.documents.models import IndexAttemptSnapshot
from onyx.server.documents.models import PaginatedReturn
from onyx.server.models import StatusResponse
from onyx.utils.logger import setup_logger
from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
@@ -64,7 +64,7 @@ def get_cc_pair_index_attempts(
page_size: int = Query(10, ge=1, le=1000),
user: User | None = Depends(current_curator_or_admin_user),
db_session: Session = Depends(get_session),
) -> PaginatedIndexAttempts:
) -> PaginatedReturn[IndexAttemptSnapshot]:
cc_pair = get_connector_credential_pair_from_id(
cc_pair_id, db_session, user, get_editable=False
)
@@ -82,10 +82,12 @@ def get_cc_pair_index_attempts(
page=page,
page_size=page_size,
)
return PaginatedIndexAttempts.from_models(
index_attempt_models=index_attempts,
page=page,
total_pages=math.ceil(total_count / page_size),
return PaginatedReturn(
items=[
IndexAttemptSnapshot.from_index_attempt_db_model(index_attempt)
for index_attempt in index_attempts
],
total_items=total_count,
)

35
backend/onyx/server/documents/models.py
View File

@@ -1,5 +1,7 @@
from datetime import datetime
from typing import Any
from typing import Generic
from typing import TypeVar
from uuid import UUID
from pydantic import BaseModel
@@ -19,6 +21,8 @@ from onyx.db.models import IndexAttempt
from onyx.db.models import IndexAttemptError as DbIndexAttemptError
from onyx.db.models import IndexingStatus
from onyx.db.models import TaskStatus
from onyx.server.models import FullUserSnapshot
from onyx.server.models import InvitedUserSnapshot
from onyx.server.utils import mask_credential_dict
@@ -201,28 +205,21 @@ class IndexAttemptError(BaseModel):
)
class PaginatedIndexAttempts(BaseModel):
index_attempts: list[IndexAttemptSnapshot]
page: int
total_pages: int
@classmethod
def from_models(
cls,
index_attempt_models: list[IndexAttempt],
page: int,
total_pages: int,
) -> "PaginatedIndexAttempts":
return cls(
index_attempts=[
IndexAttemptSnapshot.from_index_attempt_db_model(index_attempt_model)
for index_attempt_model in index_attempt_models
],
page=page,
total_pages=total_pages,
# These are the types currently supported by the pagination hook
# More api endpoints can be refactored and be added here for use with the pagination hook
PaginatedType = TypeVar(
"PaginatedType",
IndexAttemptSnapshot,
FullUserSnapshot,
InvitedUserSnapshot,
)
class PaginatedReturn(BaseModel, Generic[PaginatedType]):
items: list[PaginatedType]
total_items: int
class CCPairFullInfo(BaseModel):
id: int
name: str

80
backend/onyx/server/manage/users.py
View File

@@ -10,6 +10,7 @@ from fastapi import APIRouter
from fastapi import Body
from fastapi import Depends
from fastapi import HTTPException
from fastapi import Query
from fastapi import Request
from psycopg2.errors import UniqueViolation
from pydantic import BaseModel
@@ -27,7 +28,6 @@ from onyx.auth.invited_users import write_invited_users
from onyx.auth.noauth_user import fetch_no_auth_user
from onyx.auth.noauth_user import set_no_auth_user_preferences
from onyx.auth.schemas import UserRole
from onyx.auth.schemas import UserStatus
from onyx.auth.users import anonymous_user_enabled
from onyx.auth.users import current_admin_user
from onyx.auth.users import current_curator_or_admin_user
@@ -45,10 +45,13 @@ from onyx.db.engine import get_session
from onyx.db.models import AccessToken
from onyx.db.models import User
from onyx.db.users import delete_user_from_db
from onyx.db.users import get_all_users
from onyx.db.users import get_page_of_filtered_users
from onyx.db.users import get_total_filtered_users_count
from onyx.db.users import get_user_by_email
from onyx.db.users import list_users
from onyx.db.users import validate_user_role_update
from onyx.key_value_store.factory import get_kv_store
from onyx.server.documents.models import PaginatedReturn
from onyx.server.manage.models import AllUsersResponse
from onyx.server.manage.models import AutoScrollRequest
from onyx.server.manage.models import UserByEmail
@@ -65,10 +68,8 @@ from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
from shared_configs.configs import MULTI_TENANT
logger = setup_logger()
router = APIRouter()
USERS_PAGE_SIZE = 10
@@ -113,21 +114,68 @@ def set_user_role(
db_session.commit()
@router.get("/manage/users/accepted")
def list_accepted_users(
q: str | None = Query(default=None),
page_num: int = Query(0, ge=0),
page_size: int = Query(10, ge=1, le=1000),
roles: list[UserRole] = Query(default=[]),
is_active: bool | None = Query(default=None),
_: User | None = Depends(current_admin_user),
db_session: Session = Depends(get_session),
) -> PaginatedReturn[FullUserSnapshot]:
filtered_accepted_users = get_page_of_filtered_users(
db_session=db_session,
page_size=page_size,
page_num=page_num,
email_filter_string=q,
is_active_filter=is_active,
roles_filter=roles,
)
total_accepted_users_count = get_total_filtered_users_count(
db_session=db_session,
email_filter_string=q,
is_active_filter=is_active,
roles_filter=roles,
)
if not filtered_accepted_users:
logger.info("No users found")
return PaginatedReturn(
items=[],
total_items=0,
)
return PaginatedReturn(
items=[
FullUserSnapshot.from_user_model(user) for user in filtered_accepted_users
],
total_items=total_accepted_users_count,
)
@router.get("/manage/users/invited")
def list_invited_users(
_: User | None = Depends(current_admin_user),
) -> list[InvitedUserSnapshot]:
invited_emails = get_invited_users()
return [InvitedUserSnapshot(email=email) for email in invited_emails]
@router.get("/manage/users")
def list_all_users(
q: str | None = None,
accepted_page: int | None = None,
slack_users_page: int | None = None,
invited_page: int | None = None,
user: User | None = Depends(current_curator_or_admin_user),
_: User | None = Depends(current_curator_or_admin_user),
db_session: Session = Depends(get_session),
) -> AllUsersResponse:
if not q:
q = ""
users = [
user
for user in list_users(db_session, email_filter_string=q)
for user in get_all_users(db_session, email_filter_string=q)
if not is_api_key_email_address(user.email)
]
@@ -154,9 +202,7 @@ def list_all_users(
id=user.id,
email=user.email,
role=user.role,
status=(
UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED
),
is_active=user.is_active,
)
for user in accepted_users
],
@@ -165,9 +211,7 @@ def list_all_users(
id=user.id,
email=user.email,
role=user.role,
status=(
UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED
),
is_active=user.is_active,
)
for user in slack_users
],
@@ -184,7 +228,7 @@ def list_all_users(
id=user.id,
email=user.email,
role=user.role,
status=UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED,
is_active=user.is_active,
)
for user in accepted_users
][accepted_page * USERS_PAGE_SIZE : (accepted_page + 1) * USERS_PAGE_SIZE],
@@ -193,7 +237,7 @@ def list_all_users(
id=user.id,
email=user.email,
role=user.role,
status=UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED,
is_active=user.is_active,
)
for user in slack_users
][
@@ -412,7 +456,7 @@ def list_all_users_basic_info(
_: User | None = Depends(current_user),
db_session: Session = Depends(get_session),
) -> list[MinimalUserSnapshot]:
users = list_users(db_session)
users = get_all_users(db_session)
return [MinimalUserSnapshot(id=user.id, email=user.email) for user in users]

13
backend/onyx/server/models.py
View File

@@ -6,7 +6,7 @@ from uuid import UUID
from pydantic import BaseModel
from onyx.auth.schemas import UserRole
from onyx.auth.schemas import UserStatus
from onyx.db.models import User
DataT = TypeVar("DataT")
@@ -35,7 +35,16 @@ class FullUserSnapshot(BaseModel):
id: UUID
email: str
role: UserRole
status: UserStatus
is_active: bool
@classmethod
def from_user_model(cls, user: User) -> "FullUserSnapshot":
return cls(
id=user.id,
email=user.email,
role=user.role,
is_active=user.is_active,
)
class InvitedUserSnapshot(BaseModel):

8
backend/scripts/chat_feedback_dump.py
View File

@@ -43,17 +43,11 @@ logger = getLogger(__name__)
# GLOBAL_CURATOR = "global_curator"
# class UserStatus(str, Enum):
# LIVE = "live"
# INVITED = "invited"
# DEACTIVATED = "deactivated"
# class FullUserSnapshot(BaseModel):
# id: UUID
# email: str
# role: UserRole
# status: UserStatus
# is_active: bool
# class InvitedUserSnapshot(BaseModel):

160
backend/tests/integration/common_utils/managers/user.py
View File

@@ -2,17 +2,17 @@ from copy import deepcopy
from urllib.parse import urlencode
from uuid import uuid4
import pytest
import requests
from requests import HTTPError
from onyx.db.models import UserRole
from onyx.server.manage.models import AllUsersResponse
from onyx.auth.schemas import UserRole
from onyx.server.documents.models import PaginatedReturn
from onyx.server.models import FullUserSnapshot
from onyx.server.models import InvitedUserSnapshot
from tests.integration.common_utils.constants import API_SERVER_URL
from tests.integration.common_utils.constants import GENERAL_HEADERS
from tests.integration.common_utils.test_models import DATestUser
DOMAIN = "test.com"
DEFAULT_PASSWORD = "TestPassword123!"
@@ -26,6 +26,7 @@ class UserManager:
def create(
name: str | None = None,
email: str | None = None,
is_first_user: bool = False,
) -> DATestUser:
if name is None:
name = f"test{str(uuid4())}"
@@ -47,11 +48,15 @@ class UserManager:
)
response.raise_for_status()
role = UserRole.ADMIN if is_first_user else UserRole.BASIC
test_user = DATestUser(
id=response.json()["id"],
email=email,
password=password,
headers=deepcopy(GENERAL_HEADERS),
role=role,
is_active=True,
)
print(f"Created user {test_user.email}")
@@ -89,53 +94,148 @@ class UserManager:
return test_user
@staticmethod
def verify_role(user_to_verify: DATestUser, target_role: UserRole) -> bool:
def is_role(
user_to_verify: DATestUser,
target_role: UserRole,
) -> bool:
response = requests.get(
url=f"{API_SERVER_URL}/me",
headers=user_to_verify.headers,
)
if user_to_verify.is_active is False:
with pytest.raises(HTTPError):
response.raise_for_status()
return target_role == UserRole(response.json().get("role", ""))
return user_to_verify.role == target_role
else:
response.raise_for_status()
role_from_response = response.json().get("role", None)
if role_from_response is None:
return user_to_verify.role == target_role
return target_role == UserRole(role_from_response)
@staticmethod
def set_role(
user_to_set: DATestUser,
target_role: UserRole,
user_to_perform_action: DATestUser | None = None,
) -> None:
if user_to_perform_action is None:
user_to_perform_action = user_to_set
user_performing_action: DATestUser,
) -> DATestUser:
response = requests.patch(
url=f"{API_SERVER_URL}/manage/set-user-role",
json={"user_email": user_to_set.email, "new_role": target_role.value},
headers=user_to_perform_action.headers,
headers=user_performing_action.headers,
)
response.raise_for_status()
new_user_updated_role = DATestUser(
id=user_to_set.id,
email=user_to_set.email,
password=user_to_set.password,
headers=user_to_set.headers,
role=target_role,
is_active=user_to_set.is_active,
)
return new_user_updated_role
# TODO: Add a way to check invited status
@staticmethod
def verify(
user: DATestUser, user_to_perform_action: DATestUser | None = None
) -> None:
if user_to_perform_action is None:
user_to_perform_action = user
def is_status(user_to_verify: DATestUser, target_status: bool) -> bool:
response = requests.get(
url=f"{API_SERVER_URL}/manage/users",
headers=user_to_perform_action.headers
if user_to_perform_action
url=f"{API_SERVER_URL}/me",
headers=user_to_verify.headers,
)
if target_status is False:
with pytest.raises(HTTPError):
response.raise_for_status()
else:
response.raise_for_status()
is_active = response.json().get("is_active", None)
if is_active is None:
return user_to_verify.is_active == target_status
return target_status == is_active
@staticmethod
def set_status(
user_to_set: DATestUser,
target_status: bool,
user_performing_action: DATestUser,
) -> DATestUser:
if target_status is True:
url_substring = "activate"
elif target_status is False:
url_substring = "deactivate"
response = requests.patch(
url=f"{API_SERVER_URL}/manage/admin/{url_substring}-user",
json={"user_email": user_to_set.email},
headers=user_performing_action.headers,
)
response.raise_for_status()
new_user_updated_status = DATestUser(
id=user_to_set.id,
email=user_to_set.email,
password=user_to_set.password,
headers=user_to_set.headers,
role=user_to_set.role,
is_active=target_status,
)
return new_user_updated_status
@staticmethod
def create_test_users(
user_performing_action: DATestUser,
user_name_prefix: str,
count: int,
role: UserRole = UserRole.BASIC,
is_active: bool | None = None,
) -> list[DATestUser]:
users_list = []
for i in range(1, count + 1):
user = UserManager.create(name=f"{user_name_prefix}_{i}")
if role != UserRole.BASIC:
user = UserManager.set_role(user, role, user_performing_action)
if is_active is not None:
user = UserManager.set_status(user, is_active, user_performing_action)
users_list.append(user)
return users_list
@staticmethod
def get_user_page(
page_num: int = 0,
page_size: int = 10,
search_query: str | None = None,
role_filter: list[UserRole] | None = None,
is_active_filter: bool | None = None,
user_performing_action: DATestUser | None = None,
) -> PaginatedReturn[FullUserSnapshot]:
query_params = {
"page_num": page_num,
"page_size": page_size,
"q": search_query if search_query else None,
"roles": [role.value for role in role_filter] if role_filter else None,
"is_active": is_active_filter if is_active_filter is not None else None,
}
# Remove None values
query_params = {
key: value for key, value in query_params.items() if value is not None
}
response = requests.get(
url=f"{API_SERVER_URL}/manage/users/accepted?{urlencode(query_params, doseq=True)}",
headers=user_performing_action.headers
if user_performing_action
else GENERAL_HEADERS,
)
response.raise_for_status()
data = response.json()
all_users = AllUsersResponse(
accepted=[FullUserSnapshot(**user) for user in data["accepted"]],
invited=[InvitedUserSnapshot(**user) for user in data["invited"]],
slack_users=[FullUserSnapshot(**user) for user in data["slack_users"]],
accepted_pages=data["accepted_pages"],
invited_pages=data["invited_pages"],
slack_users_pages=data["slack_users_pages"],
paginated_result = PaginatedReturn(
items=[FullUserSnapshot(**user) for user in data["items"]],
total_items=data["total_items"],
)
for accepted_user in all_users.accepted:
if accepted_user.email == user.email and accepted_user.id == user.id:
return
raise ValueError(f"User {user.email} not found")
return paginated_result

2
backend/tests/integration/common_utils/test_models.py
View File

@@ -37,6 +37,8 @@ class DATestUser(BaseModel):
email: str
password: str
headers: dict
role: UserRole
is_active: bool
class DATestPersonaCategory(BaseModel):

4
backend/tests/integration/multitenant_tests/syncing/test_search_permissions.py
View File

@@ -16,12 +16,12 @@ def test_multi_tenant_access_control(reset_multitenant: None) -> None:
# Create Tenant 1 and its Admin User
TenantManager.create("tenant_dev1", "test1@test.com", "Data Plane Registration")
test_user1: DATestUser = UserManager.create(name="test1", email="test1@test.com")
assert UserManager.verify_role(test_user1, UserRole.ADMIN)
assert UserManager.is_role(test_user1, UserRole.ADMIN)
# Create Tenant 2 and its Admin User
TenantManager.create("tenant_dev2", "test2@test.com", "Data Plane Registration")
test_user2: DATestUser = UserManager.create(name="test2", email="test2@test.com")
assert UserManager.verify_role(test_user2, UserRole.ADMIN)
assert UserManager.is_role(test_user2, UserRole.ADMIN)
# Create connectors for Tenant 1
cc_pair_1: DATestCCPair = CCPairManager.create_from_scratch(

2
backend/tests/integration/multitenant_tests/tenants/test_tenant_creation.py
View File

@@ -14,7 +14,7 @@ def test_tenant_creation(reset_multitenant: None) -> None:
TenantManager.create("tenant_dev", "test@test.com", "Data Plane Registration")
test_user: DATestUser = UserManager.create(name="test", email="test@test.com")
assert UserManager.verify_role(test_user, UserRole.ADMIN)
assert UserManager.is_role(test_user, UserRole.ADMIN)
test_credential = CredentialManager.create(
name="admin_test_credential",

3
backend/tests/integration/openai_assistants_api/conftest.py
View File

@@ -10,6 +10,7 @@ from tests.integration.common_utils.managers.llm_provider import LLMProviderMana
from tests.integration.common_utils.managers.user import build_email
from tests.integration.common_utils.managers.user import DEFAULT_PASSWORD
from tests.integration.common_utils.managers.user import UserManager
from tests.integration.common_utils.managers.user import UserRole
from tests.integration.common_utils.test_models import DATestLLMProvider
from tests.integration.common_utils.test_models import DATestUser
@@ -30,6 +31,8 @@ def admin_user() -> DATestUser | None:
email=build_email("admin_user"),
password=DEFAULT_PASSWORD,
headers=GENERAL_HEADERS,
role=UserRole.ADMIN,
is_active=True,
)
)
except Exception:

20
backend/tests/integration/tests/permissions/test_user_role_permissions.py
View File

@@ -13,51 +13,51 @@ from tests.integration.common_utils.managers.user_group import UserGroupManager
def test_user_role_setting_permissions(reset: None) -> None:
# Creating an admin user (first user created is automatically an admin)
admin_user: DATestUser = UserManager.create(name="admin_user")
assert UserManager.verify_role(admin_user, UserRole.ADMIN)
assert UserManager.is_role(admin_user, UserRole.ADMIN)
# Creating a basic user
basic_user: DATestUser = UserManager.create(name="basic_user")
assert UserManager.verify_role(basic_user, UserRole.BASIC)
assert UserManager.is_role(basic_user, UserRole.BASIC)
# Creating a curator
curator: DATestUser = UserManager.create(name="curator")
assert UserManager.verify_role(curator, UserRole.BASIC)
assert UserManager.is_role(curator, UserRole.BASIC)
# Creating a curator without adding to a group should not work
with pytest.raises(HTTPError):
UserManager.set_role(
user_to_set=curator,
target_role=UserRole.CURATOR,
user_to_perform_action=admin_user,
user_performing_action=admin_user,
)
global_curator: DATestUser = UserManager.create(name="global_curator")
assert UserManager.verify_role(global_curator, UserRole.BASIC)
assert UserManager.is_role(global_curator, UserRole.BASIC)
# Setting the role of a global curator should not work for a basic user
with pytest.raises(HTTPError):
UserManager.set_role(
user_to_set=global_curator,
target_role=UserRole.GLOBAL_CURATOR,
user_to_perform_action=basic_user,
user_performing_action=basic_user,
)
# Setting the role of a global curator should work for an admin user
UserManager.set_role(
user_to_set=global_curator,
target_role=UserRole.GLOBAL_CURATOR,
user_to_perform_action=admin_user,
user_performing_action=admin_user,
)
assert UserManager.verify_role(global_curator, UserRole.GLOBAL_CURATOR)
assert UserManager.is_role(global_curator, UserRole.GLOBAL_CURATOR)
# Setting the role of a global curator should not work for an invalid curator
with pytest.raises(HTTPError):
UserManager.set_role(
user_to_set=global_curator,
target_role=UserRole.BASIC,
user_to_perform_action=global_curator,
user_performing_action=global_curator,
)
assert UserManager.verify_role(global_curator, UserRole.GLOBAL_CURATOR)
assert UserManager.is_role(global_curator, UserRole.GLOBAL_CURATOR)
# Creating a user group
user_group_1 = UserGroupManager.create(

12
backend/tests/integration/tests/permissions/test_whole_curator_flow.py
View File

@@ -15,7 +15,7 @@ from tests.integration.common_utils.managers.user_group import UserGroupManager
def test_whole_curator_flow(reset: None) -> None:
# Creating an admin user (first user created is automatically an admin)
admin_user: DATestUser = UserManager.create(name="admin_user")
assert UserManager.verify_role(admin_user, UserRole.ADMIN)
assert UserManager.is_role(admin_user, UserRole.ADMIN)
# Creating a curator
curator: DATestUser = UserManager.create(name="curator")
@@ -36,7 +36,7 @@ def test_whole_curator_flow(reset: None) -> None:
user_to_set_as_curator=curator,
user_performing_action=admin_user,
)
assert UserManager.verify_role(curator, UserRole.CURATOR)
assert UserManager.is_role(curator, UserRole.CURATOR)
# Creating a credential as curator
test_credential = CredentialManager.create(
@@ -92,19 +92,19 @@ def test_whole_curator_flow(reset: None) -> None:
def test_global_curator_flow(reset: None) -> None:
# Creating an admin user (first user created is automatically an admin)
admin_user: DATestUser = UserManager.create(name="admin_user")
assert UserManager.verify_role(admin_user, UserRole.ADMIN)
assert UserManager.is_role(admin_user, UserRole.ADMIN)
# Creating a user
global_curator: DATestUser = UserManager.create(name="global_curator")
assert UserManager.verify_role(global_curator, UserRole.BASIC)
assert UserManager.is_role(global_curator, UserRole.BASIC)
# Set the user to a global curator
UserManager.set_role(
user_to_set=global_curator,
target_role=UserRole.GLOBAL_CURATOR,
user_to_perform_action=admin_user,
user_performing_action=admin_user,
)
assert UserManager.verify_role(global_curator, UserRole.GLOBAL_CURATOR)
assert UserManager.is_role(global_curator, UserRole.GLOBAL_CURATOR)
# Creating a user group containing the global curator
user_group_1 = UserGroupManager.create(

168
backend/tests/integration/tests/users/test_user_pagination.py Normal file
View File

@@ -0,0 +1,168 @@
from onyx.auth.schemas import UserRole
from onyx.server.models import FullUserSnapshot
from tests.integration.common_utils.managers.user import UserManager
from tests.integration.common_utils.test_models import DATestUser
# Gets a page of users from the db that match the given parameters and then
# compares that returned page to the list of users passed into the function
# to verify that the pagination and filtering works as expected.
def _verify_user_pagination(
users: list[DATestUser],
page_size: int = 5,
search_query: str | None = None,
role_filter: list[UserRole] | None = None,
is_active_filter: bool | None = None,
user_performing_action: DATestUser | None = None,
) -> None:
retrieved_users: list[FullUserSnapshot] = []
for i in range(0, len(users), page_size):
paginated_result = UserManager.get_user_page(
page_num=i // page_size,
page_size=page_size,
search_query=search_query,
role_filter=role_filter,
is_active_filter=is_active_filter,
user_performing_action=user_performing_action,
)
# Verify that the total items is equal to the length of the users list
assert paginated_result.total_items == len(users)
# Verify that the number of items in the page is equal to the page size
assert len(paginated_result.items) == page_size
# Add the retrieved users to the list of retrieved users
retrieved_users.extend(paginated_result.items)
# Create a set of all the expected emails
all_expected_emails = set([user.email for user in users])
# Create a set of all the retrieved emails
all_retrieved_emails = set([user.email for user in retrieved_users])
# Verify that the set of retrieved emails is equal to the set of expected emails
assert all_expected_emails == all_retrieved_emails
def _verify_user_role_and_status(users: list) -> None:
for user in users:
assert UserManager.is_role(user, user.role)
assert UserManager.is_status(user, user.is_active)
def test_user_pagination(reset: None) -> None:
# Create an admin user to perform actions
user_performing_action: DATestUser = UserManager.create(
name="admin_performing_action",
is_first_user=True,
)
# Create 9 admin users
admin_users: list[DATestUser] = UserManager.create_test_users(
user_name_prefix="admin",
count=9,
role=UserRole.ADMIN,
user_performing_action=user_performing_action,
)
# Add the user_performing_action to the list of admins
admin_users.append(user_performing_action)
# Create 20 basic users
basic_users: list[DATestUser] = UserManager.create_test_users(
user_name_prefix="basic",
count=10,
role=UserRole.BASIC,
user_performing_action=user_performing_action,
)
# Create 10 global curators
global_curators: list[DATestUser] = UserManager.create_test_users(
user_name_prefix="global_curator",
count=10,
role=UserRole.GLOBAL_CURATOR,
user_performing_action=user_performing_action,
)
# Create 10 inactive admins
inactive_admins: list[DATestUser] = UserManager.create_test_users(
user_name_prefix="inactive_admin",
count=10,
role=UserRole.ADMIN,
is_active=False,
user_performing_action=user_performing_action,
)
# Create 10 global curator users with an email containing "search"
searchable_curators: list[DATestUser] = UserManager.create_test_users(
user_name_prefix="search_curator",
count=10,
role=UserRole.GLOBAL_CURATOR,
user_performing_action=user_performing_action,
)
# Combine all the users lists into the all_users list
all_users: list[DATestUser] = (
admin_users
+ basic_users
+ global_curators
+ inactive_admins
+ searchable_curators
)
_verify_user_role_and_status(all_users)
# Verify pagination
_verify_user_pagination(
users=all_users,
user_performing_action=user_performing_action,
)
# Verify filtering by role
_verify_user_pagination(
users=admin_users + inactive_admins,
role_filter=[UserRole.ADMIN],
user_performing_action=user_performing_action,
)
# Verify filtering by status
_verify_user_pagination(
users=inactive_admins,
is_active_filter=False,
user_performing_action=user_performing_action,
)
# Verify filtering by search query
_verify_user_pagination(
users=searchable_curators,
search_query="search",
user_performing_action=user_performing_action,
)
# Verify filtering by role and status
_verify_user_pagination(
users=inactive_admins,
role_filter=[UserRole.ADMIN],
is_active_filter=False,
user_performing_action=user_performing_action,
)
# Verify filtering by role and search query
_verify_user_pagination(
users=searchable_curators,
role_filter=[UserRole.GLOBAL_CURATOR],
search_query="search",
user_performing_action=user_performing_action,
)
# Verify filtering by role and status and search query
_verify_user_pagination(
users=inactive_admins,
role_filter=[UserRole.ADMIN],
is_active_filter=False,
search_query="inactive_ad",
user_performing_action=user_performing_action,
)
# Verify filtering by multiple roles (admin and global curator)
_verify_user_pagination(
users=admin_users + global_curators + inactive_admins + searchable_curators,
role_filter=[UserRole.ADMIN, UserRole.GLOBAL_CURATOR],
user_performing_action=user_performing_action,
)

208
web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
View File

@@ -1,6 +1,6 @@
"use client";
import { useCallback, useEffect, useRef, useState } from "react";
import { useState } from "react";
import {
Table,
TableHead,
@@ -11,7 +11,8 @@ import {
} from "@/components/ui/table";
import Text from "@/components/ui/text";
import { Callout } from "@/components/ui/callout";
import { CCPairFullInfo, PaginatedIndexAttempts } from "./types";
import { CCPairFullInfo } from "./types";
import { IndexAttemptSnapshot } from "@/lib/types";
import { IndexAttemptStatus } from "@/components/Status";
import { PageSelector } from "@/components/PageSelector";
import { ThreeDotsLoader } from "@/components/Loading";
@@ -22,191 +23,49 @@ import { ErrorCallout } from "@/components/ErrorCallout";
import { InfoIcon, SearchIcon } from "@/components/icons/icons";
import Link from "next/link";
import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
import { useRouter } from "next/navigation";
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { FiInfo } from "react-icons/fi";
import usePaginatedFetch from "@/hooks/usePaginatedFetch";
// This is the number of index attempts to display per page
const NUM_IN_PAGE = 8;
// This is the number of pages to fetch at a time
const BATCH_SIZE = 8;
const ITEMS_PER_PAGE = 8;
const PAGES_PER_BATCH = 8;
export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
const [indexAttemptTracePopupId, setIndexAttemptTracePopupId] = useState<
number | null
>(null);
const totalPages = Math.ceil(ccPair.number_of_index_attempts / NUM_IN_PAGE);
const router = useRouter();
const [page, setPage] = useState(() => {
if (typeof window !== "undefined") {
const urlParams = new URLSearchParams(window.location.search);
return parseInt(urlParams.get("page") || "1", 10);
}
return 1;
const {
currentPageData: pageOfIndexAttempts,
isLoading,
error,
currentPage,
totalPages,
goToPage,
} = usePaginatedFetch<IndexAttemptSnapshot>({
itemsPerPage: ITEMS_PER_PAGE,
pagesPerBatch: PAGES_PER_BATCH,
endpoint: `${buildCCPairInfoUrl(ccPair.id)}/index-attempts`,
});
const [currentPageData, setCurrentPageData] =
useState<PaginatedIndexAttempts | null>(null);
const [currentPageError, setCurrentPageError] = useState<Error | null>(null);
const [isCurrentPageLoading, setIsCurrentPageLoading] = useState(false);
// This is a cache of the data for each "batch" which is a set of pages
const [cachedBatches, setCachedBatches] = useState<{
[key: number]: PaginatedIndexAttempts[];
}>({});
// This is a set of the batches that are currently being fetched
// we use it to avoid duplicate requests
const ongoingRequestsRef = useRef<Set<number>>(new Set());
const batchRetrievalUrlBuilder = useCallback(
(batchNum: number) => {
return `${buildCCPairInfoUrl(
ccPair.id
)}/index-attempts?page=${batchNum}&page_size=${BATCH_SIZE * NUM_IN_PAGE}`;
},
[ccPair.id]
);
// This fetches and caches the data for a given batch number
const fetchBatchData = useCallback(
async (batchNum: number) => {
if (ongoingRequestsRef.current.has(batchNum)) return;
ongoingRequestsRef.current.add(batchNum);
try {
const response = await fetch(batchRetrievalUrlBuilder(batchNum + 1));
if (!response.ok) {
throw new Error("Failed to fetch data");
}
const data = await response.json();
const newBatchData: PaginatedIndexAttempts[] = [];
for (let i = 0; i < BATCH_SIZE; i++) {
const startIndex = i * NUM_IN_PAGE;
const endIndex = startIndex + NUM_IN_PAGE;
const pageIndexAttempts = data.index_attempts.slice(
startIndex,
endIndex
);
newBatchData.push({
...data,
index_attempts: pageIndexAttempts,
});
}
setCachedBatches((prev) => ({
...prev,
[batchNum]: newBatchData,
}));
} catch (error) {
setCurrentPageError(
error instanceof Error ? error : new Error("An error occurred")
);
} finally {
ongoingRequestsRef.current.delete(batchNum);
}
},
[
ongoingRequestsRef,
setCachedBatches,
setCurrentPageError,
batchRetrievalUrlBuilder,
]
);
// This fetches and caches the data for the current batch and the next and previous batches
useEffect(() => {
const batchNum = Math.floor((page - 1) / BATCH_SIZE);
if (!cachedBatches[batchNum]) {
setIsCurrentPageLoading(true);
fetchBatchData(batchNum);
} else {
setIsCurrentPageLoading(false);
}
const nextBatchNum = Math.max(
Math.min(batchNum + 1, Math.ceil(totalPages / BATCH_SIZE) - 1),
0
);
if (!cachedBatches[nextBatchNum]) {
fetchBatchData(nextBatchNum);
}
const prevBatchNum = Math.max(batchNum - 1, 0);
if (!cachedBatches[prevBatchNum]) {
fetchBatchData(prevBatchNum);
}
// Always fetch the first batch if it's not cached
if (!cachedBatches[0]) {
fetchBatchData(0);
}
}, [ccPair.id, page, cachedBatches, totalPages, fetchBatchData]);
// This updates the data on the current page
useEffect(() => {
const batchNum = Math.floor((page - 1) / BATCH_SIZE);
const batchPageNum = (page - 1) % BATCH_SIZE;
if (cachedBatches[batchNum] && cachedBatches[batchNum][batchPageNum]) {
setCurrentPageData(cachedBatches[batchNum][batchPageNum]);
setIsCurrentPageLoading(false);
} else {
setIsCurrentPageLoading(true);
}
}, [page, cachedBatches]);
useEffect(() => {
const interval = setInterval(() => {
const batchNum = Math.floor((page - 1) / BATCH_SIZE);
fetchBatchData(batchNum); // Re-fetch the current batch data
}, 5000); // Refresh every 5 seconds
return () => clearInterval(interval); // Cleanup on unmount
}, [page, fetchBatchData]); // Dependencies to ensure correct batch is fetched
// This updates the page number and manages the URL
const updatePage = (newPage: number) => {
setPage(newPage);
router.replace(`/admin/connector/${ccPair.id}?page=${newPage}`, {
scroll: false,
});
window.scrollTo({
top: 0,
left: 0,
behavior: "smooth",
});
};
if (isCurrentPageLoading || !currentPageData) {
if (isLoading || !pageOfIndexAttempts) {
return <ThreeDotsLoader />;
}
if (currentPageError) {
if (error) {
return (
<ErrorCallout
errorTitle={`Failed to fetch info on Connector with ID ${ccPair.id}`}
errorMsg={currentPageError?.toString() || "Unknown error"}
errorMsg={error?.toString() || "Unknown error"}
/>
);
}
// if no indexing attempts have been scheduled yet, let the user know why
if (
Object.keys(cachedBatches).length === 0 ||
Object.values(cachedBatches).every((batch) =>
batch.every((page) => page.index_attempts.length === 0)
)
) {
if (!pageOfIndexAttempts?.length) {
return (
<Callout
className="mt-4"
@@ -219,18 +78,16 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
);
}
// This is the index attempt that the user wants to view the trace for
const indexAttemptToDisplayTraceFor = currentPageData?.index_attempts?.find(
const indexAttemptToDisplayTraceFor = pageOfIndexAttempts?.find(
(indexAttempt) => indexAttempt.id === indexAttemptTracePopupId
);
return (
<>
{indexAttemptToDisplayTraceFor &&
indexAttemptToDisplayTraceFor.full_exception_trace && (
{indexAttemptToDisplayTraceFor?.full_exception_trace && (
<ExceptionTraceModal
onOutsideClick={() => setIndexAttemptTracePopupId(null)}
exceptionTrace={indexAttemptToDisplayTraceFor.full_exception_trace!}
exceptionTrace={indexAttemptToDisplayTraceFor.full_exception_trace}
/>
)}
@@ -239,20 +96,20 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
<TableRow>
<TableHead>Time Started</TableHead>
<TableHead>Status</TableHead>
<TableHead>New Documents</TableHead>
<TableHead>New Doc Cnt</TableHead>
<TableHead>
<div className="w-fit">
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<span className="cursor-help flex items-center">
New + Modified Documents
Total Doc Cnt
<InfoIcon className="ml-1 w-4 h-4" />
</span>
</TooltipTrigger>
<TooltipContent>
Total number of documents inserted or updated in the index
during this indexing attempt
Total number of documents replaced in the index during
this indexing attempt
</TooltipContent>
</Tooltip>
</TooltipProvider>
@@ -262,7 +119,7 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
</TableRow>
</TableHeader>
<TableBody>
{currentPageData.index_attempts.map((indexAttempt) => {
{pageOfIndexAttempts.map((indexAttempt) => {
const docsPerMinute =
getDocsProcessedPerMinute(indexAttempt)?.toFixed(2);
return (
@@ -322,8 +179,7 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
</Text>
)}
{(indexAttempt.status === "failed" ||
indexAttempt.status === "canceled") &&
{indexAttempt.status === "failed" &&
indexAttempt.error_msg && (
<Text className="flex flex-wrap whitespace-normal">
{indexAttempt.error_msg}
@@ -352,8 +208,8 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
<div className="mx-auto">
<PageSelector
totalPages={totalPages}
currentPage={page}
onPageChange={updatePage}
currentPage={currentPage}
onPageChange={goToPage}
/>
</div>
</div>

7
web/src/app/admin/token-rate-limits/TokenRateLimitTables.tsx
View File

@@ -176,7 +176,10 @@ export const GenericTokenRateLimitTable = ({
responseMapper?: (data: any) => TokenRateLimitDisplay[];
isAdmin?: boolean;
}) => {
const { data, isLoading, error } = useSWR(fetchUrl, errorHandlingFetcher);
const { data, isLoading, error } = useSWR<TokenRateLimitDisplay[]>(
fetchUrl,
errorHandlingFetcher
);
if (isLoading) {
return <ThreeDotsLoader />;
@@ -193,7 +196,7 @@ export const GenericTokenRateLimitTable = ({
return (
<TokenRateLimitTable
tokenRateLimits={processedData}
tokenRateLimits={processedData ?? []}
fetchUrl={fetchUrl}
title={title}
description={description}

132
web/src/app/admin/users/page.tsx
View File

@@ -1,5 +1,5 @@
"use client";
import { useEffect, useState } from "react";
import { useState } from "react";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Button } from "@/components/ui/button";
@@ -8,7 +8,7 @@ import SignedUpUserTable from "@/components/admin/users/SignedUpUserTable";
import { SearchBar } from "@/components/search/SearchBar";
import { FiPlusSquare } from "react-icons/fi";
import { Modal } from "@/components/Modal";
import { LoadingAnimation } from "@/components/Loading";
import { ThreeDotsLoader } from "@/components/Loading";
import { AdminPageTitle } from "@/components/admin/Title";
import { usePopup, PopupSpec } from "@/components/admin/connectors/Popup";
import { UsersIcon } from "@/components/icons/icons";
@@ -16,9 +16,8 @@ import { errorHandlingFetcher } from "@/lib/fetcher";
import useSWR, { mutate } from "swr";
import { ErrorCallout } from "@/components/ErrorCallout";
import BulkAdd from "@/components/admin/users/BulkAdd";
import { UsersResponse } from "@/lib/users/interfaces";
import SlackUserTable from "@/components/admin/users/SlackUserTable";
import Text from "@/components/ui/text";
import { InvitedUserSnapshot } from "@/lib/types";
const UsersTables = ({
q,
@@ -27,21 +26,13 @@ const UsersTables = ({
q: string;
setPopup: (spec: PopupSpec) => void;
}) => {
const [invitedPage, setInvitedPage] = useState(1);
const [acceptedPage, setAcceptedPage] = useState(1);
const [slackUsersPage, setSlackUsersPage] = useState(1);
const [usersData, setUsersData] = useState<UsersResponse | undefined>(
undefined
);
const [domainsData, setDomainsData] = useState<string[] | undefined>(
undefined
);
const { data, error, mutate } = useSWR<UsersResponse>(
`/api/manage/users?q=${encodeURIComponent(q)}&accepted_page=${
acceptedPage - 1
}&invited_page=${invitedPage - 1}&slack_users_page=${slackUsersPage - 1}`,
const {
data: invitedUsers,
error: invitedUsersError,
isLoading: invitedUsersLoading,
mutate: invitedUsersMutate,
} = useSWR<InvitedUserSnapshot[]>(
"/api/manage/users/invited",
errorHandlingFetcher
);
@@ -50,33 +41,9 @@ const UsersTables = ({
errorHandlingFetcher
);
useEffect(() => {
if (data) {
setUsersData(data);
}
}, [data]);
useEffect(() => {
if (validDomains) {
setDomainsData(validDomains);
}
}, [validDomains]);
const activeData = data ?? usersData;
const activeDomains = validDomains ?? domainsData;
// Show loading animation only during the initial data fetch
if (!activeData || !activeDomains) {
return <LoadingAnimation text="Loading" />;
}
if (error) {
return (
<ErrorCallout
errorTitle="Error loading users"
errorMsg={error?.info?.detail}
/>
);
if (!validDomains) {
return <ThreeDotsLoader />;
}
if (domainsError) {
@@ -88,90 +55,43 @@ const UsersTables = ({
);
}
const {
accepted,
invited,
accepted_pages,
invited_pages,
slack_users,
slack_users_pages,
} = activeData;
const finalInvited = invited.filter(
(user) => !accepted.some((u) => u.email === user.email)
);
return (
<Tabs defaultValue="invited">
<Tabs defaultValue="current">
<TabsList>
<TabsTrigger value="invited">Invited Users</TabsTrigger>
<TabsTrigger value="current">Current Users</TabsTrigger>
<TabsTrigger value="onyxbot">OnyxBot Users</TabsTrigger>
<TabsTrigger value="invited">Invited Users</TabsTrigger>
</TabsList>
<TabsContent value="invited">
<Card>
<CardHeader>
<CardTitle>Invited Users</CardTitle>
</CardHeader>
<CardContent>
{finalInvited.length > 0 ? (
<InvitedUserTable
users={finalInvited}
setPopup={setPopup}
currentPage={invitedPage}
onPageChange={setInvitedPage}
totalPages={invited_pages}
mutate={mutate}
/>
) : (
<p>Users that have been invited will show up here</p>
)}
</CardContent>
</Card>
</TabsContent>
<TabsContent value="current">
<Card>
<CardHeader>
<CardTitle>Current Users</CardTitle>
</CardHeader>
<CardContent>
{accepted.length > 0 ? (
<SignedUpUserTable
users={accepted}
invitedUsers={invitedUsers || []}
setPopup={setPopup}
currentPage={acceptedPage}
onPageChange={setAcceptedPage}
totalPages={accepted_pages}
mutate={mutate}
q={q}
invitedUsersMutate={invitedUsersMutate}
/>
) : (
<p>Users that have an account will show up here</p>
)}
</CardContent>
</Card>
</TabsContent>
<TabsContent value="onyxbot">
<TabsContent value="invited">
<Card>
<CardHeader>
<CardTitle>OnyxBot Users</CardTitle>
<CardTitle>Invited Users</CardTitle>
</CardHeader>
<CardContent>
{slack_users.length > 0 ? (
<SlackUserTable
<InvitedUserTable
users={invitedUsers || []}
setPopup={setPopup}
currentPage={slackUsersPage}
onPageChange={setSlackUsersPage}
totalPages={slack_users_pages}
invitedUsers={finalInvited}
slackusers={slack_users}
mutate={mutate}
mutate={invitedUsersMutate}
error={invitedUsersError}
isLoading={invitedUsersLoading}
q={q}
/>
) : (
<p>Slack-only users will show up here</p>
)}
</CardContent>
</Card>
</TabsContent>
@@ -187,7 +107,6 @@ const SearchableTables = () => {
return (
<div>
{popup}
<div className="flex flex-col gap-y-4">
<div className="flex gap-x-4">
<AddUserButton setPopup={setPopup} />
@@ -257,7 +176,6 @@ const Page = () => {
return (
<div className="mx-auto container">
<AdminPageTitle title="Manage Users" icon={<UsersIcon size={32} />} />
<SearchableTables />
</div>
);

1
web/src/app/ee/admin/groups/page.tsx
View File

@@ -15,7 +15,6 @@ import { AdminPageTitle } from "@/components/admin/Title";
import { Button } from "@/components/ui/button";
import { useUser } from "@/components/user/UserProvider";
import { Separator } from "@/components/ui/separator";
const Main = () => {
const { popup, setPopup } = usePopup();

2
web/src/components/admin/connectors/Popup.tsx
View File

@@ -7,7 +7,7 @@ export interface PopupSpec {
export const Popup: React.FC<PopupSpec> = ({ message, type }) => (
<div
className={`fixed bottom-4 left-4 p-4 rounded-md shadow-lg text-white z-[100] ${
className={`fixed bottom-4 left-4 p-4 rounded-md shadow-lg text-white z-[10000] ${
type === "success" ? "bg-green-500" : "bg-error"
}`}
>

17
web/src/components/admin/users/BulkAdd.tsx
View File

@@ -1,7 +1,6 @@
"use client";
import { withFormik, FormikProps, FormikErrors, Form, Field } from "formik";
import { Button } from "@/components/ui/button";
const WHITESPACE_SPLIT = /\s+/;
@@ -30,9 +29,21 @@ const AddUserFormRenderer = ({
touched,
errors,
isSubmitting,
handleSubmit,
}: FormikProps<FormValues>) => (
<Form className="w-full">
<Field id="emails" name="emails" as="textarea" className="w-full p-4" />
<Form className="w-full" onSubmit={handleSubmit}>
<Field
id="emails"
name="emails"
as="textarea"
className="w-full p-4"
onKeyDown={(e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter") {
e.preventDefault();
handleSubmit();
}
}}
/>
{touched.emails && errors.emails && (
<div className="text-error text-sm">{errors.emails}</div>
)}

71
web/src/components/admin/users/InvitedUserTable.tsx
View File

@@ -1,3 +1,4 @@
import { useState } from "react";
import { PopupSpec } from "@/components/admin/connectors/Popup";
import {
Table,
@@ -6,29 +7,63 @@ import {
TableBody,
TableCell,
} from "@/components/ui/table";
import CenteredPageSelector from "./CenteredPageSelector";
import { type PageSelectorProps } from "@/components/PageSelector";
import { type User } from "@/lib/types";
import { ThreeDotsLoader } from "@/components/Loading";
import { InvitedUserSnapshot } from "@/lib/types";
import { TableHeader } from "@/components/ui/table";
import { InviteUserButton } from "./buttons/InviteUserButton";
import { ErrorCallout } from "@/components/ErrorCallout";
import { FetchError } from "@/lib/fetcher";
const USERS_PER_PAGE = 10;
interface Props {
users: Array<User>;
users: InvitedUserSnapshot[];
setPopup: (spec: PopupSpec) => void;
mutate: () => void;
error: FetchError | null;
isLoading: boolean;
q: string;
}
const InvitedUserTable = ({
users,
setPopup,
currentPage,
totalPages,
onPageChange,
mutate,
}: Props & PageSelectorProps) => {
if (!users.length) return null;
error,
isLoading,
q,
}: Props) => {
const [currentPageNum, setCurrentPageNum] = useState<number>(1);
if (!users.length)
return <p>Users that have been invited will show up here</p>;
const totalPages = Math.ceil(users.length / USERS_PER_PAGE);
// Filter users based on the search query
const filteredUsers = q
? users.filter((user) => user.email.includes(q))
: users;
// Get the current page of users
const currentPageOfUsers = filteredUsers.slice(
(currentPageNum - 1) * USERS_PER_PAGE,
currentPageNum * USERS_PER_PAGE
);
if (isLoading) {
return <ThreeDotsLoader />;
}
if (error) {
return (
<ErrorCallout
errorTitle="Error loading users"
errorMsg={error?.info?.detail}
/>
);
}
return (
<>
@@ -42,7 +77,8 @@ const InvitedUserTable = ({
</TableRow>
</TableHeader>
<TableBody>
{users.map((user) => (
{currentPageOfUsers.length ? (
currentPageOfUsers.map((user) => (
<TableRow key={user.email}>
<TableCell>{user.email}</TableCell>
<TableCell>
@@ -56,14 +92,21 @@ const InvitedUserTable = ({
</div>
</TableCell>
</TableRow>
))}
))
) : (
<TableRow>
<TableCell colSpan={2} className="h-24 text-center">
{`No users found matching "${q}"`}
</TableCell>
</TableRow>
)}
</TableBody>
</Table>
{totalPages > 1 ? (
<CenteredPageSelector
currentPage={currentPage}
currentPage={currentPageNum}
totalPages={totalPages}
onPageChange={onPageChange}
onPageChange={setCurrentPageNum}
/>
) : null}
</>

308
web/src/components/admin/users/SignedUpUserTable.tsx
View File

@@ -1,6 +1,11 @@
import { type User, UserStatus, UserRole } from "@/lib/types";
import {
type User,
UserRole,
InvitedUserSnapshot,
USER_ROLE_LABELS,
} from "@/lib/types";
import { useState } from "react";
import CenteredPageSelector from "./CenteredPageSelector";
import { type PageSelectorProps } from "@/components/PageSelector";
import { PopupSpec } from "@/components/admin/connectors/Popup";
import {
Table,
@@ -10,33 +15,76 @@ import {
TableCell,
} from "@/components/ui/table";
import { TableHeader } from "@/components/ui/table";
import { UserRoleDropdown } from "./buttons/UserRoleDropdown";
import { DeleteUserButton } from "./buttons/DeleteUserButton";
import { DeactivaterButton } from "./buttons/DeactivaterButton";
import UserRoleDropdown from "./buttons/UserRoleDropdown";
import DeleteUserButton from "./buttons/DeleteUserButton";
import DeactivateUserButton from "./buttons/DeactivateUserButton";
import usePaginatedFetch from "@/hooks/usePaginatedFetch";
import { ThreeDotsLoader } from "@/components/Loading";
import { ErrorCallout } from "@/components/ErrorCallout";
import { InviteUserButton } from "./buttons/InviteUserButton";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
const ITEMS_PER_PAGE = 10;
const PAGES_PER_BATCH = 2;
import { useUser } from "@/components/user/UserProvider";
import { LeaveOrganizationButton } from "./buttons/LeaveOrganizationButton";
import { NEXT_PUBLIC_CLOUD_ENABLED } from "@/lib/constants";
interface Props {
users: Array<User>;
invitedUsers: InvitedUserSnapshot[];
setPopup: (spec: PopupSpec) => void;
mutate: () => void;
q: string;
invitedUsersMutate: () => void;
}
const SignedUpUserTable = ({
users,
invitedUsers,
setPopup,
q = "",
invitedUsersMutate,
}: Props) => {
const [filters, setFilters] = useState<{
is_active?: boolean;
roles?: UserRole[];
}>({});
const [selectedRoles, setSelectedRoles] = useState<UserRole[]>([]);
const {
currentPageData: pageOfUsers,
isLoading,
error,
currentPage,
totalPages,
onPageChange,
mutate,
}: Props & PageSelectorProps) => {
goToPage,
refresh,
} = usePaginatedFetch<User>({
itemsPerPage: ITEMS_PER_PAGE,
pagesPerBatch: PAGES_PER_BATCH,
endpoint: "/api/manage/users/accepted",
query: q,
filter: filters,
});
const { user: currentUser } = useUser();
if (!users.length) return null;
if (error) {
return (
<ErrorCallout
errorTitle="Error loading users"
errorMsg={error?.message}
/>
);
}
const handlePopup = (message: string, type: "success" | "error") => {
if (type === "success") mutate();
if (type === "success") refresh();
setPopup({ message, type });
};
@@ -45,15 +93,149 @@ const SignedUpUserTable = ({
const onRoleChangeError = (errorMsg: string) =>
handlePopup(`Unable to update user role - ${errorMsg}`, "error");
const toggleRole = (roleEnum: UserRole) => {
setFilters((prev) => {
const currentRoles = prev.roles || [];
const newRoles = currentRoles.includes(roleEnum)
? currentRoles.filter((r) => r !== roleEnum) // Remove role if already selected
: [...currentRoles, roleEnum]; // Add role if not selected
setSelectedRoles(newRoles); // Update selected roles state
return {
...prev,
roles: newRoles,
};
});
};
const removeRole = (roleEnum: UserRole) => {
setSelectedRoles((prev) => prev.filter((role) => role !== roleEnum)); // Remove role from selected roles
toggleRole(roleEnum); // Deselect the role in filters
};
// --------------
// Render Functions
// --------------
const renderFilters = () => (
<>
<div className="flex items-center gap-4 py-4">
<Select
value={filters.is_active?.toString() || "all"}
onValueChange={(selectedStatus) =>
setFilters((prev) => {
if (selectedStatus === "all") {
const { is_active, ...rest } = prev;
return rest;
}
return {
...prev,
is_active: selectedStatus === "true",
};
})
}
>
<SelectTrigger className="w-[260px] h-[34px] bg-neutral">
<SelectValue />
</SelectTrigger>
<SelectContent className="bg-neutral-50">
<SelectItem value="all">All Status</SelectItem>
<SelectItem value="true">Active</SelectItem>
<SelectItem value="false">Inactive</SelectItem>
</SelectContent>
</Select>
<Select value="roles">
<SelectTrigger className="w-[260px] h-[34px] bg-neutral">
<SelectValue>
{filters.roles?.length
? `${filters.roles.length} role(s) selected`
: "All Roles"}
</SelectValue>
</SelectTrigger>
<SelectContent className="bg-neutral-50">
{Object.entries(USER_ROLE_LABELS)
.filter(([role]) => role !== UserRole.EXT_PERM_USER)
.map(([role, label]) => (
<div
key={role}
className="flex items-center space-x-2 px-2 py-1.5 cursor-pointer hover:bg-gray-200"
onClick={() => toggleRole(role as UserRole)}
>
<input
type="checkbox"
checked={filters.roles?.includes(role as UserRole) || false}
onChange={(e) => e.stopPropagation()}
/>
<label className="text-sm font-normal">{label}</label>
</div>
))}
</SelectContent>
</Select>
</div>
<div className="flex gap-2 py-1">
{selectedRoles.map((role) => (
<button
key={role}
className="border border-neutral-300 bg-neutral p-1 rounded text-sm hover:bg-neutral-200"
onClick={() => removeRole(role)}
style={{ padding: "2px 8px" }}
>
<span>{USER_ROLE_LABELS[role]}</span>
<span className="ml-3">&times;</span>
</button>
))}
</div>
</>
);
const renderUserRoleDropdown = (user: User) => {
if (user.role === UserRole.SLACK_USER) {
return <p className="ml-2">Slack User</p>;
}
return (
<UserRoleDropdown
user={user}
onSuccess={onRoleChangeSuccess}
onError={onRoleChangeError}
/>
);
};
const renderActionButtons = (user: User) => {
if (user.role === UserRole.SLACK_USER) {
return (
<InviteUserButton
user={user}
invited={invitedUsers.map((u) => u.email).includes(user.email)}
setPopup={setPopup}
mutate={[refresh, invitedUsersMutate]}
/>
);
}
return NEXT_PUBLIC_CLOUD_ENABLED && user.id === currentUser?.id ? (
<LeaveOrganizationButton
user={user}
setPopup={setPopup}
mutate={refresh}
/>
) : (
<>
<DeactivateUserButton
user={user}
deactivate={user.is_active}
setPopup={setPopup}
mutate={refresh}
/>
{!user.is_active && (
<DeleteUserButton user={user} setPopup={setPopup} mutate={refresh} />
)}
</>
);
};
return (
<>
{totalPages > 1 ? (
<CenteredPageSelector
currentPage={currentPage}
totalPages={totalPages}
onPageChange={onPageChange}
/>
) : null}
{renderFilters()}
<Table className="overflow-visible">
<TableHeader>
<TableRow>
@@ -67,56 +249,52 @@ const SignedUpUserTable = ({
</TableHead>
</TableRow>
</TableHeader>
{isLoading ? (
<TableBody>
{users
// Dont want to show external permissioned users because it's scary
.filter((user) => user.role !== UserRole.EXT_PERM_USER)
.map((user) => (
<TableRow key={user.id}>
<TableCell>{user.email}</TableCell>
<TableCell className="w-40 ">
<UserRoleDropdown
user={user}
onSuccess={onRoleChangeSuccess}
onError={onRoleChangeError}
/>
</TableCell>
<TableCell className="text-center">
<i>{user.status === "live" ? "Active" : "Inactive"}</i>
</TableCell>
<TableCell>
<div className="flex justify-end gap-x-2">
{NEXT_PUBLIC_CLOUD_ENABLED &&
user.id === currentUser?.id ? (
<LeaveOrganizationButton
user={user}
setPopup={setPopup}
mutate={mutate}
/>
) : (
<>
<DeactivaterButton
user={user}
deactivate={user.status === UserStatus.live}
setPopup={setPopup}
mutate={mutate}
/>
{user.status == UserStatus.deactivated && (
<DeleteUserButton
user={user}
setPopup={setPopup}
mutate={mutate}
/>
)}
</>
)}
</div>
<TableRow>
<TableCell colSpan={4} className="text-center">
<ThreeDotsLoader />
</TableCell>
</TableRow>
))}
</TableBody>
) : (
<TableBody>
{!pageOfUsers?.length ? (
<TableRow>
<TableCell colSpan={4} className="text-center">
<p className="pt-4 pb-4">
{filters.roles?.length || filters.is_active !== undefined
? "No users found matching your filters"
: `No users found matching "${q}"`}
</p>
</TableCell>
</TableRow>
) : (
pageOfUsers.map((user) => (
<TableRow key={user.id}>
<TableCell>{user.email}</TableCell>
<TableCell className="w-[180px]">
{renderUserRoleDropdown(user)}
</TableCell>
<TableCell className="text-center w-[140px]">
<i>{user.is_active ? "Active" : "Inactive"}</i>
</TableCell>
<TableCell className="text-right w-[200px]">
{renderActionButtons(user)}
</TableCell>
</TableRow>
))
)}
</TableBody>
)}
</Table>
{totalPages > 1 && (
<CenteredPageSelector
currentPage={currentPage}
totalPages={totalPages}
onPageChange={goToPage}
/>
)}
</>
);
};

78
web/src/components/admin/users/SlackUserTable.tsx
View File

@@ -1,78 +0,0 @@
import React from "react";
import { User } from "@/lib/types";
import {
Table,
TableCell,
TableBody,
TableHead,
TableHeader,
TableRow,
} from "@/components/ui/table";
import { PopupSpec } from "../connectors/Popup";
import { InviteUserButton } from "./buttons/InviteUserButton";
import { PageSelectorProps } from "@/components/PageSelector";
import CenteredPageSelector from "./CenteredPageSelector";
interface SlackUserTableProps {
invitedUsers: User[];
slackusers: User[];
mutate: () => void;
setPopup: (spec: PopupSpec) => void;
}
const SlackUserTable: React.FC<SlackUserTableProps & PageSelectorProps> = ({
invitedUsers,
slackusers,
mutate,
currentPage,
totalPages,
onPageChange,
setPopup,
}) => {
return (
<>
{totalPages > 1 ? (
<CenteredPageSelector
currentPage={currentPage}
totalPages={totalPages}
onPageChange={onPageChange}
/>
) : null}
<Table>
<TableHeader>
<TableRow>
<TableHead>Email</TableHead>
<TableHead className="text-center">Status</TableHead>
<TableHead>
<div className="flex">
<div className="ml-auto">Actions</div>
</div>
</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{slackusers.map((user) => (
<TableRow key={user.id}>
<TableCell>{user.email}</TableCell>
<TableCell className="text-center">
<i>{user.status === "live" ? "Active" : "Inactive"}</i>
</TableCell>
<TableCell className="flex justify-end">
<InviteUserButton
user={user}
invited={invitedUsers
.map((u) => u.email)
.includes(user.email)}
setPopup={setPopup}
mutate={mutate}
/>
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</>
);
};
export default SlackUserTable;

14
web/src/components/admin/users/UserStatusButtons.tsx
View File

@@ -1,24 +1,12 @@
import {
type User,
UserStatus,
UserRole,
USER_ROLE_LABELS,
INVALID_ROLE_HOVER_TEXT,
} from "@/lib/types";
import CenteredPageSelector from "./CenteredPageSelector";
import { type PageSelectorProps } from "@/components/PageSelector";
import { HidableSection } from "@/app/admin/assistants/HidableSection";
import { PopupSpec } from "@/components/admin/connectors/Popup";
import userMutationFetcher from "@/lib/admin/users/userMutationFetcher";
import useSWRMutation from "swr/mutation";
import {
Table,
TableHead,
TableRow,
TableBody,
TableCell,
} from "@/components/ui/table";
import {
Select,
SelectContent,
@@ -30,8 +18,6 @@ import { Button } from "@/components/ui/button";
import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal";
import { useState } from "react";
import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
import { TableHeader } from "@/components/ui/table";
export const InviteUserButton = ({
user,

8
web/src/components/admin/users/buttons/DeactivaterButton.tsx → web/src/components/admin/users/buttons/DeactivateUserButton.tsx
View File

@@ -1,10 +1,10 @@
import { type User } from "@/lib/types";
import { PopupSpec } from "@/components/admin/connectors/Popup";
import userMutationFetcher from "@/lib/admin/users/userMutationFetcher";
import useSWRMutation from "swr/mutation";
import { Button } from "@/components/ui/button";
import useSWRMutation from "swr/mutation";
import userMutationFetcher from "@/lib/admin/users/userMutationFetcher";
export const DeactivaterButton = ({
const DeactivateUserButton = ({
user,
deactivate,
setPopup,
@@ -43,3 +43,5 @@ export const DeactivaterButton = ({
</Button>
);
};
export default DeactivateUserButton;

4
web/src/components/admin/users/buttons/DeleteUserButton.tsx
View File

@@ -6,7 +6,7 @@ import { Button } from "@/components/ui/button";
import { useState } from "react";
import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
export const DeleteUserButton = ({
const DeleteUserButton = ({
user,
setPopup,
mutate,
@@ -59,3 +59,5 @@ export const DeleteUserButton = ({
</>
);
};
export default DeleteUserButton;

29
web/src/components/admin/users/buttons/InviteUserButton.tsx
View File

@@ -1,4 +1,7 @@
import { type User } from "@/lib/types";
import {
type InvitedUserSnapshot,
type AcceptedUserSnapshot,
} from "@/lib/types";
import { PopupSpec } from "@/components/admin/connectors/Popup";
import useSWRMutation from "swr/mutation";
@@ -12,10 +15,10 @@ export const InviteUserButton = ({
setPopup,
mutate,
}: {
user: User;
user: AcceptedUserSnapshot | InvitedUserSnapshot;
invited: boolean;
setPopup: (spec: PopupSpec) => void;
mutate: () => void;
mutate: (() => void) | (() => void)[];
}) => {
const { trigger: inviteTrigger, isMutating: isInviting } = useSWRMutation(
"/api/manage/admin/users",
@@ -35,17 +38,23 @@ export const InviteUserButton = ({
{
onSuccess: () => {
setShowInviteModal(false);
if (typeof mutate === "function") {
mutate();
} else {
mutate.forEach((fn) => fn());
}
setPopup({
message: "User invited successfully!",
type: "success",
});
},
onError: (errorMsg) =>
onError: (errorMsg) => {
setShowInviteModal(false);
setPopup({
message: `Unable to invite user - ${errorMsg}`,
type: "error",
}),
});
},
}
);
@@ -67,17 +76,23 @@ export const InviteUserButton = ({
{
onSuccess: () => {
setShowInviteModal(false);
if (typeof mutate === "function") {
mutate();
} else {
mutate.forEach((fn) => fn());
}
setPopup({
message: "User uninvited successfully!",
type: "success",
});
},
onError: (errorMsg) =>
onError: (errorMsg) => {
setShowInviteModal(false);
setPopup({
message: `Unable to uninvite user - ${errorMsg}`,
type: "error",
}),
});
},
}
);

4
web/src/components/admin/users/buttons/UserRoleDropdown.tsx
View File

@@ -18,7 +18,7 @@ import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal";
import { useState } from "react";
import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
export const UserRoleDropdown = ({
const UserRoleDropdown = ({
user,
onSuccess,
onError,
@@ -121,3 +121,5 @@ export const UserRoleDropdown = ({
</>
);
};
export default UserRoleDropdown;

248
web/src/hooks/usePaginatedFetch.tsx Normal file
View File

@@ -0,0 +1,248 @@
import { useCallback, useEffect, useState, useRef, useMemo } from "react";
import { usePathname, useRouter, useSearchParams } from "next/navigation";
import {
IndexAttemptSnapshot,
AcceptedUserSnapshot,
InvitedUserSnapshot,
} from "@/lib/types";
import { errorHandlingFetcher } from "@/lib/fetcher";
type PaginatedType =
| IndexAttemptSnapshot
| AcceptedUserSnapshot
| InvitedUserSnapshot;
interface PaginatedApiResponse<T extends PaginatedType> {
items: T[];
total_items: number;
}
interface PaginationConfig {
itemsPerPage: number;
pagesPerBatch: number;
endpoint: string;
query?: string;
filter?: Record<string, string | boolean | number | string[]>;
refreshIntervalInMs?: number;
}
interface PaginatedHookReturnData<T extends PaginatedType> {
currentPageData: T[] | null;
isLoading: boolean;
error: Error | null;
currentPage: number;
totalPages: number;
goToPage: (page: number) => void;
refresh: () => Promise<void>;
}
function usePaginatedFetch<T extends PaginatedType>({
itemsPerPage,
pagesPerBatch,
endpoint,
query,
filter,
refreshIntervalInMs = 5000,
}: PaginationConfig): PaginatedHookReturnData<T> {
const router = useRouter();
const currentPath = usePathname();
const searchParams = useSearchParams();
// State to initialize and hold the current page number
const [currentPage, setCurrentPage] = useState(() =>
parseInt(searchParams?.get("page") || "1", 10)
);
const [currentPageData, setCurrentPageData] = useState<T[] | null>(null);
const [error, setError] = useState<Error | null>(null);
const [isLoading, setIsLoading] = useState<boolean>(false);
const [totalItems, setTotalItems] = useState<number>(0);
const [cachedBatches, setCachedBatches] = useState<{ [key: number]: T[][] }>(
{}
);
// Tracks ongoing requests to avoid duplicate requests, uses ref to persist across renders
const ongoingRequestsRef = useRef<Set<number>>(new Set());
const totalPages = useMemo(() => {
if (totalItems === 0) return 1;
return Math.ceil(totalItems / itemsPerPage);
}, [totalItems, itemsPerPage]);
// Calculates which batch we're in, and which page within that batch
const batchAndPageIndices = useMemo(() => {
const batchNum = Math.floor((currentPage - 1) / pagesPerBatch);
const batchPageNum = (currentPage - 1) % pagesPerBatch;
return { batchNum, batchPageNum };
}, [currentPage, pagesPerBatch]);
// Fetches a batch of data and stores it in the cache
const fetchBatchData = useCallback(
async (batchNum: number) => {
// Prevents duplicate requests
if (ongoingRequestsRef.current.has(batchNum)) {
return;
}
ongoingRequestsRef.current.add(batchNum);
try {
// Build query params
const params = new URLSearchParams({
page_num: batchNum.toString(),
page_size: (pagesPerBatch * itemsPerPage).toString(),
});
if (query) params.set("q", query);
if (filter) {
for (const [key, value] of Object.entries(filter)) {
if (Array.isArray(value)) {
value.forEach((str) => params.append(key, str));
} else {
params.set(key, value.toString());
}
}
}
const url = `${endpoint}?${params.toString()}`;
const responseData =
await errorHandlingFetcher<PaginatedApiResponse<T>>(url);
// Validate response data structure
if (
!Array.isArray(
responseData.items || typeof responseData.total_items !== "number"
)
) {
throw new Error(
"Sorry, we encountered an issue with the data format. Please try again or contact support if the problem persists."
);
}
setTotalItems(responseData.total_items);
// Splits a batch into pages
const pagesInBatch = Array.from({ length: pagesPerBatch }, (_, i) => {
const startIndex = i * itemsPerPage;
return responseData.items.slice(
startIndex,
startIndex + itemsPerPage
);
});
setCachedBatches((prev) => ({
...prev,
[batchNum]: pagesInBatch,
}));
} catch (error) {
setError(error instanceof Error ? error : new Error(String(error)));
} finally {
ongoingRequestsRef.current.delete(batchNum);
}
},
[endpoint, pagesPerBatch, itemsPerPage, query, filter]
);
// Updates the URL with the current page number
const updatePageUrl = useCallback(
(page: number) => {
if (currentPath) {
const params = new URLSearchParams(searchParams);
params.set("page", page.toString());
router.replace(`${currentPath}?${params.toString()}`, {
scroll: false,
});
}
},
[currentPath, router, searchParams]
);
// Updates the current page
const goToPage = useCallback(
(newPage: number) => {
setCurrentPage(newPage);
updatePageUrl(newPage);
},
[updatePageUrl]
);
// Loads the current and adjacent batches
useEffect(() => {
const { batchNum } = batchAndPageIndices;
const nextBatchNum = batchNum + 1;
const prevBatchNum = Math.max(batchNum - 1, 0);
if (!cachedBatches[batchNum]) {
setIsLoading(true);
fetchBatchData(batchNum);
}
// Possible total number of items including the next batch
const totalItemsIncludingNextBatch =
nextBatchNum * pagesPerBatch * itemsPerPage;
// Preload next batch if we're not on the last batch
if (
totalItemsIncludingNextBatch <= totalItems &&
!cachedBatches[nextBatchNum]
) {
fetchBatchData(nextBatchNum);
}
// Load previous batch if missing
if (!cachedBatches[prevBatchNum]) {
fetchBatchData(prevBatchNum);
}
// Ensure first batch is always loaded
if (!cachedBatches[0]) {
fetchBatchData(0);
}
}, [currentPage, cachedBatches, totalPages, pagesPerBatch, fetchBatchData]);
// Updates current page data from the cache
useEffect(() => {
const { batchNum, batchPageNum } = batchAndPageIndices;
if (cachedBatches[batchNum] && cachedBatches[batchNum][batchPageNum]) {
setCurrentPageData(cachedBatches[batchNum][batchPageNum]);
setIsLoading(false);
}
}, [currentPage, cachedBatches, pagesPerBatch]);
// Implements periodic refresh
useEffect(() => {
if (!refreshIntervalInMs) return;
const interval = setInterval(() => {
const { batchNum } = batchAndPageIndices;
fetchBatchData(batchNum);
}, refreshIntervalInMs);
return () => clearInterval(interval);
}, [currentPage, pagesPerBatch, refreshIntervalInMs, fetchBatchData]);
// Manually refreshes the current batch
const refresh = useCallback(async () => {
const { batchNum } = batchAndPageIndices;
await fetchBatchData(batchNum);
}, [currentPage, pagesPerBatch, fetchBatchData]);
// Cache invalidation
useEffect(() => {
setCachedBatches({});
setTotalItems(0);
goToPage(1);
setError(null);
}, [currentPath, query, filter]);
return {
currentPage,
currentPageData,
totalPages,
goToPage,
refresh,
isLoading,
error,
};
}
export default usePaginatedFetch;

2
web/src/lib/fetcher.ts
View File

@@ -19,7 +19,7 @@ const DEFAULT_AUTH_ERROR_MSG =
const DEFAULT_ERROR_MSG = "An error occurred while fetching the data.";
export const errorHandlingFetcher = async (url: string) => {
export const errorHandlingFetcher = async <T>(url: string): Promise<T> => {
const res = await fetch(url);
if (res.status === 403) {

4
web/src/lib/hooks.ts
View File

@@ -13,7 +13,7 @@ import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelect
import { SourceMetadata } from "./search/interfaces";
import { destructureValue, structureValue } from "./llm/utils";
import { ChatSession } from "@/app/chat/interfaces";
import { UsersResponse } from "./users/interfaces";
import { AllUsersResponse } from "./types";
import { Credential } from "./connectors/credentials";
import { SettingsContext } from "@/components/settings/SettingsProvider";
import { PersonaCategory } from "@/app/admin/assistants/interfaces";
@@ -145,7 +145,7 @@ export function useFilters(): FilterManager {
export const useUsers = () => {
const url = "/api/manage/users";
const swrResponse = useSWR<UsersResponse>(url, errorHandlingFetcher);
const swrResponse = useSWR<AllUsersResponse>(url, errorHandlingFetcher);
return {
...swrResponse,

33
web/src/lib/types.ts
View File

@@ -12,12 +12,6 @@ interface UserPreferences {
auto_scroll: boolean | null;
}
export enum UserStatus {
live = "live",
invited = "invited",
deactivated = "deactivated",
}
export enum UserRole {
LIMITED = "limited",
BASIC = "basic",
@@ -51,12 +45,11 @@ export const INVALID_ROLE_HOVER_TEXT: Partial<Record<UserRole, string>> = {
export interface User {
id: string;
email: string;
is_active: string;
is_superuser: string;
is_verified: string;
is_active: boolean;
is_superuser: boolean;
is_verified: boolean;
role: UserRole;
preferences: UserPreferences;
status: UserStatus;
current_token_created_at?: Date;
current_token_expiry_length?: number;
oidc_expiry?: Date;
@@ -65,6 +58,26 @@ export interface User {
is_anonymous_user?: boolean;
}
export interface AllUsersResponse {
accepted: User[];
invited: User[];
slack_users: User[];
accepted_pages: number;
invited_pages: number;
slack_users_pages: number;
}
export interface AcceptedUserSnapshot {
id: string;
email: string;
role: UserRole;
is_active: boolean;
}
export interface InvitedUserSnapshot {
email: string;
}
export interface MinimalUserSnapshot {
id: string;
email: string;

10
web/src/lib/users/interfaces.ts
View File

@@ -1,10 +0,0 @@
import { User } from "../types";
export interface UsersResponse {
accepted: User[];
invited: User[];
slack_users: User[];
accepted_pages: number;
invited_pages: number;
slack_users_pages: number;
}