From b61bd3da2f4fcc2582de3004939682b744a974a2 Mon Sep 17 00:00:00 2001 From: Luke Wulf Date: Thu, 27 Jun 2024 17:49:08 -0700 Subject: [PATCH] merging two columns into one nullable column, num_days --- .gitignore | 1 - .../6dffd0cbb64f_add_persona_filtering_columns.py | 6 ------ backend/danswer/chat/load_yamls.py | 1 - backend/danswer/chat/personas.yaml | 4 ---- backend/danswer/db/models.py | 2 -- backend/danswer/db/persona.py | 4 ---- backend/danswer/db/slack_bot_config.py | 1 - backend/danswer/search/models.py | 2 +- backend/danswer/server/features/persona/models.py | 3 --- backend/package-lock.json | 6 ------ web/package-lock.json | 14 +++++++------- web/src/app/admin/assistants/AssistantEditor.tsx | 4 +--- web/src/app/admin/assistants/interfaces.ts | 2 +- web/src/app/admin/assistants/lib.ts | 4 ---- 14 files changed, 10 insertions(+), 44 deletions(-) delete mode 100644 backend/package-lock.json diff --git a/.gitignore b/.gitignore index 485fdd3d9..c6e7e06e7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ .mypy_cache .idea /backend/apicache/ -/backend/package-lock.json /deployment/data/nginx/app.conf .vscode/launch.json *.sw? diff --git a/backend/alembic/versions/6dffd0cbb64f_add_persona_filtering_columns.py b/backend/alembic/versions/6dffd0cbb64f_add_persona_filtering_columns.py index be78847d3..81e118d1f 100644 --- a/backend/alembic/versions/6dffd0cbb64f_add_persona_filtering_columns.py +++ b/backend/alembic/versions/6dffd0cbb64f_add_persona_filtering_columns.py @@ -17,14 +17,8 @@ depends_on: None = None def upgrade() -> None: - op.add_column( - "persona", sa.Column("use_recent_documents", sa.Boolean(), nullable=True) - ) - op.execute("UPDATE persona SET use_recent_documents = false") - op.alter_column("persona", "use_recent_documents", nullable=False) op.add_column("persona", sa.Column("num_days", sa.Float(), nullable=True)) def downgrade() -> None: - op.drop_column("persona", "use_recent_documents") op.drop_column("persona", "num_days") diff --git a/backend/danswer/chat/load_yamls.py b/backend/danswer/chat/load_yamls.py index 193d3e6f2..8215e225f 100644 --- a/backend/danswer/chat/load_yamls.py +++ b/backend/danswer/chat/load_yamls.py @@ -85,7 +85,6 @@ def load_personas_from_yaml( num_chunks=persona.get("num_chunks") if persona.get("num_chunks") is not None else default_chunks, - use_recent_documents=persona.get("use_recent_documents", False), num_days=persona.get("num_days"), llm_relevance_filter=persona.get("llm_relevance_filter"), starter_messages=persona.get("starter_messages"), diff --git a/backend/danswer/chat/personas.yaml b/backend/danswer/chat/personas.yaml index 88a023494..312e1b8e7 100644 --- a/backend/danswer/chat/personas.yaml +++ b/backend/danswer/chat/personas.yaml @@ -15,8 +15,6 @@ personas: # Remove the field to set to the system default number of chunks/tokens to pass to Gen AI # Each chunk is 512 tokens long num_chunks: 10 - # Flag to limit all searches to most recent documents by last_updated_time - use_recent_documents: false # Specifies how young a document should be to be included in search context # If a documents last_updated_time is over the current time + num_days it is excluded # Field unused if use_recent_documents is set to false @@ -52,7 +50,6 @@ personas: prompts: - "OnlyLLM" num_chunks: 0 - use_recent_documents: false llm_relevance_filter: true llm_filter_extraction: true recency_bias: "auto" @@ -66,7 +63,6 @@ personas: prompts: - "Paraphrase" num_chunks: 10 - use_recent_documents: false llm_relevance_filter: true llm_filter_extraction: true recency_bias: "auto" diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py index dbf216b3b..a85e513c1 100644 --- a/backend/danswer/db/models.py +++ b/backend/danswer/db/models.py @@ -971,8 +971,6 @@ class Persona(Base): ) # Number of chunks to pass to the LLM for generation. num_chunks: Mapped[float | None] = mapped_column(Float, nullable=True) - # True if persona should only use context from recent documents, false by default - use_recent_documents: Mapped[bool] = mapped_column(Boolean) # Number of days old for last_updated documents, only used if use_recent_documents is true num_days: Mapped[float | None] = mapped_column(Float, nullable=True) # Pass every chunk through LLM for evaluation, fairly expensive diff --git a/backend/danswer/db/persona.py b/backend/danswer/db/persona.py index 31acd422b..39a103a17 100644 --- a/backend/danswer/db/persona.py +++ b/backend/danswer/db/persona.py @@ -82,7 +82,6 @@ def create_update_persona( name=create_persona_request.name, description=create_persona_request.description, num_chunks=create_persona_request.num_chunks, - use_recent_documents=create_persona_request.use_recent_documents, num_days=create_persona_request.num_days, llm_relevance_filter=create_persona_request.llm_relevance_filter, llm_filter_extraction=create_persona_request.llm_filter_extraction, @@ -329,7 +328,6 @@ def upsert_persona( name: str, description: str, num_chunks: float, - use_recent_documents: bool, num_days: float | None, llm_relevance_filter: bool, llm_filter_extraction: bool, @@ -369,7 +367,6 @@ def upsert_persona( persona.name = name persona.description = description persona.num_chunks = num_chunks - persona.use_recent_documents = use_recent_documents persona.num_days = num_days persona.llm_relevance_filter = llm_relevance_filter persona.llm_filter_extraction = llm_filter_extraction @@ -402,7 +399,6 @@ def upsert_persona( name=name, description=description, num_chunks=num_chunks, - use_recent_documents=use_recent_documents, num_days=num_days, llm_relevance_filter=llm_relevance_filter, llm_filter_extraction=llm_filter_extraction, diff --git a/backend/danswer/db/slack_bot_config.py b/backend/danswer/db/slack_bot_config.py index 8e3adf190..ff2aa51ef 100644 --- a/backend/danswer/db/slack_bot_config.py +++ b/backend/danswer/db/slack_bot_config.py @@ -58,7 +58,6 @@ def create_slack_bot_persona( name=persona_name, description="", num_chunks=num_chunks, - use_recent_documents=False, num_days=None, llm_relevance_filter=True, llm_filter_extraction=True, diff --git a/backend/danswer/search/models.py b/backend/danswer/search/models.py index d2fe0d475..709922f61 100644 --- a/backend/danswer/search/models.py +++ b/backend/danswer/search/models.py @@ -48,7 +48,7 @@ class BaseFilters(BaseModel): @classmethod def from_persona(cls, persona: Persona) -> "BaseFilters | None": - if persona.use_recent_documents and persona.num_days is not None: + if persona.num_days is not None: from_date = datetime.now(timezone.utc) - timedelta(days=persona.num_days) return cls(time_cutoff=from_date) else: diff --git a/backend/danswer/server/features/persona/models.py b/backend/danswer/server/features/persona/models.py index 0606dadc3..cfaf40b71 100644 --- a/backend/danswer/server/features/persona/models.py +++ b/backend/danswer/server/features/persona/models.py @@ -19,7 +19,6 @@ class CreatePersonaRequest(BaseModel): name: str description: str num_chunks: float - use_recent_documents: bool num_days: float llm_relevance_filter: bool is_public: bool @@ -46,7 +45,6 @@ class PersonaSnapshot(BaseModel): display_priority: int | None description: str num_chunks: float | None - use_recent_documents: bool num_days: float | None llm_relevance_filter: bool llm_filter_extraction: bool @@ -84,7 +82,6 @@ class PersonaSnapshot(BaseModel): display_priority=persona.display_priority, description=persona.description, num_chunks=persona.num_chunks, - use_recent_documents=persona.use_recent_documents, num_days=persona.num_days, llm_relevance_filter=persona.llm_relevance_filter, llm_filter_extraction=persona.llm_filter_extraction, diff --git a/backend/package-lock.json b/backend/package-lock.json deleted file mode 100644 index ad0a0ec6b..000000000 --- a/backend/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "backend", - "lockfileVersion": 2, - "requires": true, - "packages": {} -} diff --git a/web/package-lock.json b/web/package-lock.json index 472946465..0e19baab4 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -2424,11 +2424,11 @@ } }, "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", "dependencies": { - "fill-range": "^7.1.1" + "fill-range": "^7.0.1" }, "engines": { "node": ">=8" @@ -3896,9 +3896,9 @@ } }, "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", "dependencies": { "to-regex-range": "^5.0.1" }, diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx index 8b89f3e9d..fb0835152 100644 --- a/web/src/app/admin/assistants/AssistantEditor.tsx +++ b/web/src/app/admin/assistants/AssistantEditor.tsx @@ -176,7 +176,7 @@ export function AssistantEditor({ existingPersona?.document_sets?.map((documentSet) => documentSet.id) ?? ([] as number[]), num_chunks: existingPersona?.num_chunks ?? null, - recent_documents_enabled: existingPersona?.use_recent_documents ?? false, + recent_documents_enabled: existingPersona?.num_days != null, num_days: existingPersona?.num_days ?? null, include_citations: existingPersona?.prompts[0]?.include_citations ?? true, llm_relevance_filter: existingPersona?.llm_relevance_filter ?? false, @@ -318,7 +318,6 @@ export function AssistantEditor({ existingPromptId: existingPrompt?.id, ...values, num_chunks: numChunks, - use_recent_documents: useRecentDocuments, num_days: numDays, users: user && !checkUserIsNoAuthUser(user.id) ? [user.id] : undefined, @@ -329,7 +328,6 @@ export function AssistantEditor({ [promptResponse, personaResponse] = await createPersona({ ...values, num_chunks: numChunks, - use_recent_documents: useRecentDocuments, num_days: numDays, users: user && !checkUserIsNoAuthUser(user.id) ? [user.id] : undefined, diff --git a/web/src/app/admin/assistants/interfaces.ts b/web/src/app/admin/assistants/interfaces.ts index ec59b0dd7..d7791d316 100644 --- a/web/src/app/admin/assistants/interfaces.ts +++ b/web/src/app/admin/assistants/interfaces.ts @@ -36,7 +36,7 @@ export interface Persona { llm_model_version_override?: string; starter_messages: StarterMessage[] | null; use_recent_documents: boolean; - num_days?: number; + num_days: number | null; default_persona: boolean; users: MinimalUserSnapshot[]; groups: number[]; diff --git a/web/src/app/admin/assistants/lib.ts b/web/src/app/admin/assistants/lib.ts index 21c4962cd..00bd4b70a 100644 --- a/web/src/app/admin/assistants/lib.ts +++ b/web/src/app/admin/assistants/lib.ts @@ -7,7 +7,6 @@ interface PersonaCreationRequest { task_prompt: string; document_set_ids: number[]; num_chunks: number | null; - use_recent_documents: boolean; num_days: number | null; include_citations: boolean; is_public: boolean; @@ -29,7 +28,6 @@ interface PersonaUpdateRequest { task_prompt: string; document_set_ids: number[]; num_chunks: number | null; - use_recent_documents: boolean; num_days: number | null; include_citations: boolean; is_public: boolean; @@ -109,7 +107,6 @@ function buildPersonaAPIBody( description, document_set_ids, num_chunks, - use_recent_documents, num_days, llm_relevance_filter, is_public, @@ -122,7 +119,6 @@ function buildPersonaAPIBody( name, description, num_chunks, - use_recent_documents, num_days, llm_relevance_filter, llm_filter_extraction: false,