Reenable OpenAI Tokenizer (#3062)

* k

* clean up test embeddings

* nit

* minor update to ensure consistency

* minor organizational update

* minor updates

---------

Co-authored-by: pablodanswer <pablo@danswer.ai>

web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx

@@ -11,6 +11,7 @@ import {
   LLM_PROVIDERS_ADMIN_URL,
 } from "../../configuration/llm/constants";
 import { mutate } from "swr";
+import { testEmbedding } from "../pages/utils";
 
 export function ChangeCredentialsModal({
   provider,
@@ -112,16 +113,15 @@ export function ChangeCredentialsModal({
     const normalizedProviderType = provider.provider_type
       .toLowerCase()
       .split(" ")[0];
+
     try {
-      const testResponse = await fetch("/api/admin/embedding/test-embedding", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          provider_type: normalizedProviderType,
-          api_key: apiKey,
-          api_url: apiUrl,
-          model_name: modelName,
-        }),
+      const testResponse = await testEmbedding({
+        provider_type: normalizedProviderType,
+        modelName,
+        apiKey,
+        apiUrl,
+        apiVersion: null,
+        deploymentName: null,
       });
 
       if (!testResponse.ok) {

web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx

@@ -110,20 +110,27 @@ export function ProviderCreationModal({
     setErrorMsg("");
     try {
       const customConfig = Object.fromEntries(values.custom_config);
+      const providerType = values.provider_type.toLowerCase().split(" ")[0];
+      const isOpenAI = providerType === "openai";
+
+      const testModelName =
+        isOpenAI || isAzure ? "text-embedding-3-small" : values.model_name;
+
+      const testEmbeddingPayload = {
+        provider_type: providerType,
+        api_key: values.api_key,
+        api_url: values.api_url,
+        model_name: testModelName,
+        api_version: values.api_version,
+        deployment_name: values.deployment_name,
+      };
 
       const initialResponse = await fetch(
         "/api/admin/embedding/test-embedding",
         {
           method: "POST",
           headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({
-            provider_type: values.provider_type.toLowerCase().split(" ")[0],
-            api_key: values.api_key,
-            api_url: values.api_url,
-            model_name: values.model_name,
-            api_version: values.api_version,
-            deployment_name: values.deployment_name,
-          }),
+          body: JSON.stringify(testEmbeddingPayload),
         }
       );
 

web/src/app/admin/embeddings/pages/utils.ts

@@ -8,3 +8,37 @@ export const deleteSearchSettings = async (search_settings_id: number) => {
   });
   return response;
 };
+
+export const testEmbedding = async ({
+  provider_type,
+  modelName,
+  apiKey,
+  apiUrl,
+  apiVersion,
+  deploymentName,
+}: {
+  provider_type: string;
+  modelName: string;
+  apiKey: string | null;
+  apiUrl: string | null;
+  apiVersion: string | null;
+  deploymentName: string | null;
+}) => {
+  const testModelName =
+    provider_type === "openai" ? "text-embedding-3-small" : modelName;
+
+  const testResponse = await fetch("/api/admin/embedding/test-embedding", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      provider_type: provider_type,
+      api_key: apiKey,
+      api_url: apiUrl,
+      model_name: testModelName,
+      api_version: apiVersion,
+      deployment_name: deploymentName,
+    }),
+  });
+
+  return testResponse;
+};