Adding Document Sets (#477)

Adds:
- name for connector credential pairs + frontend changes to start populating this field
- document set table migration
- during indexing, document sets are now checked and inserted into Vespa
- background job to check if document sets need to be synced
- document set management APIs
- document set management dashboard in the UI

backend/alembic/versions/57b53544726e_add_document_set_tables.py

@@ -0,0 +1,59 @@
+"""Add document set tables
+
+Revision ID: 57b53544726e
+Revises: 800f48024ae9
+Create Date: 2023-09-20 16:59:39.097177
+
+"""
+from alembic import op
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "57b53544726e"
+down_revision = "800f48024ae9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "document_set",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("description", sa.String(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column("is_up_to_date", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+    op.create_table(
+        "document_set__connector_credential_pair",
+        sa.Column("document_set_id", sa.Integer(), nullable=False),
+        sa.Column("connector_credential_pair_id", sa.Integer(), nullable=False),
+        sa.Column("is_current", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["connector_credential_pair_id"],
+            ["connector_credential_pair.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["document_set_id"],
+            ["document_set.id"],
+        ),
+        sa.PrimaryKeyConstraint(
+            "document_set_id", "connector_credential_pair_id", "is_current"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("document_set__connector_credential_pair")
+    op.drop_table("document_set")

backend/alembic/versions/800f48024ae9_add_id_to_connectorcredentialpair.py

@@ -0,0 +1,60 @@
+"""Add ID to ConnectorCredentialPair
+
+Revision ID: 800f48024ae9
+Revises: 767f1c2a00eb
+Create Date: 2023-09-19 16:13:42.299715
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.schema import Sequence, CreateSequence
+
+# revision identifiers, used by Alembic.
+revision = "800f48024ae9"
+down_revision = "767f1c2a00eb"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    sequence = Sequence("connector_credential_pair_id_seq")
+    op.execute(CreateSequence(sequence))  # type: ignore
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column(
+            "id", sa.Integer(), nullable=True, server_default=sequence.next_value()
+        ),
+    )
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column("name", sa.String(), nullable=True),
+    )
+
+    # fill in IDs for existing rows
+    op.execute(
+        "UPDATE connector_credential_pair SET id = nextval('connector_credential_pair_id_seq') WHERE id IS NULL"
+    )
+    op.alter_column("connector_credential_pair", "id", nullable=False)
+
+    op.create_unique_constraint(
+        "connector_credential_pair__name__key", "connector_credential_pair", ["name"]
+    )
+    op.create_unique_constraint(
+        "connector_credential_pair__id__key", "connector_credential_pair", ["id"]
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "connector_credential_pair__name__key",
+        "connector_credential_pair",
+        type_="unique",
+    )
+    op.drop_constraint(
+        "connector_credential_pair__id__key",
+        "connector_credential_pair",
+        type_="unique",
+    )
+    op.drop_column("connector_credential_pair", "name")
+    op.drop_column("connector_credential_pair", "id")
+    op.execute("DROP SEQUENCE connector_credential_pair_id_seq")