diff --git a/deployment/cloud_kubernetes/hpa/workers_hpa.yaml b/deployment/cloud_kubernetes/hpa/workers_hpa.yaml deleted file mode 100644 index 5e35da9f088..00000000000 --- a/deployment/cloud_kubernetes/hpa/workers_hpa.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: celery-worker-heavy-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: celery-worker-heavy - minReplicas: 1 - maxReplicas: 5 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 60 ---- -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: celery-worker-light-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: celery-worker-light - minReplicas: 1 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 ---- -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: celery-worker-indexing-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: celery-worker-indexing - minReplicas: 1 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 ---- -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: celery-worker-monitoring-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: celery-worker-indexing - minReplicas: 1 - maxReplicas: 4 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 diff --git a/deployment/cloud_kubernetes/keda/celery-worker-auth.yaml b/deployment/cloud_kubernetes/keda/celery-worker-auth.yaml deleted file mode 100644 index 210957a3264..00000000000 --- a/deployment/cloud_kubernetes/keda/celery-worker-auth.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: celery-worker-auth - namespace: onyx -spec: - secretTargetRef: - - parameter: host - name: keda-redis-secret - key: host - - parameter: password - name: keda-redis-secret - key: password diff --git a/deployment/cloud_kubernetes/keda/celery-worker-indexing-scaledobject.yaml b/deployment/cloud_kubernetes/keda/celery-worker-indexing-scaledobject.yaml deleted file mode 100644 index 70854c12e7d..00000000000 --- a/deployment/cloud_kubernetes/keda/celery-worker-indexing-scaledobject.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: celery-worker-indexing-scaledobject - namespace: onyx - labels: - app: celery-worker-indexing -spec: - scaleTargetRef: - name: celery-worker-indexing - minReplicaCount: 1 - maxReplicaCount: 30 - triggers: - - type: redis - metadata: - sslEnabled: "true" - port: "6379" - enableTLS: "true" - listName: connector_indexing - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - - type: redis - metadata: - sslEnabled: "true" - port: "6379" - enableTLS: "true" - listName: connector_indexing:2 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - sslEnabled: "true" - port: "6379" - enableTLS: "true" - listName: connector_indexing:3 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: cpu - metadata: - type: Utilization - value: "70" - - - type: memory - metadata: - type: Utilization - value: "70" diff --git a/deployment/cloud_kubernetes/keda/celery-worker-light-scaledobject.yaml b/deployment/cloud_kubernetes/keda/celery-worker-light-scaledobject.yaml deleted file mode 100644 index 160a51f9531..00000000000 --- a/deployment/cloud_kubernetes/keda/celery-worker-light-scaledobject.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: celery-worker-light-scaledobject - namespace: onyx - labels: - app: celery-worker-light -spec: - scaleTargetRef: - name: celery-worker-light - minReplicaCount: 5 - maxReplicaCount: 20 - triggers: - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: vespa_metadata_sync - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: vespa_metadata_sync:2 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: vespa_metadata_sync:3 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: connector_deletion - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: connector_deletion:2 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth diff --git a/deployment/cloud_kubernetes/keda/celery-worker-primary-scaledobject.yaml b/deployment/cloud_kubernetes/keda/celery-worker-primary-scaledobject.yaml deleted file mode 100644 index 1977465f976..00000000000 --- a/deployment/cloud_kubernetes/keda/celery-worker-primary-scaledobject.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: celery-worker-primary-scaledobject - namespace: onyx - labels: - app: celery-worker-primary -spec: - scaleTargetRef: - name: celery-worker-primary - pollingInterval: 15 # Check every 15 seconds - cooldownPeriod: 30 # Wait 30 seconds before scaling down - minReplicaCount: 4 - maxReplicaCount: 4 - triggers: - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: celery - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: celery:1 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: celery:2 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: celery:3 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: periodic_tasks - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth - - type: redis - metadata: - port: "6379" - enableTLS: "true" - listName: periodic_tasks:2 - listLength: "1" - databaseIndex: "15" - authenticationRef: - name: celery-worker-auth diff --git a/deployment/cloud_kubernetes/keda/indexing-model-server-scaledobject.yaml b/deployment/cloud_kubernetes/keda/indexing-model-server-scaledobject.yaml deleted file mode 100644 index 6ed6bdc6de1..00000000000 --- a/deployment/cloud_kubernetes/keda/indexing-model-server-scaledobject.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: indexing-model-server-scaledobject - namespace: onyx - labels: - app: indexing-model-server -spec: - scaleTargetRef: - name: indexing-model-server-deployment - pollingInterval: 15 # Check every 15 seconds - cooldownPeriod: 30 # Wait 30 seconds before scaling down - minReplicaCount: 10 - maxReplicaCount: 10 - triggers: - - type: cpu - metadata: - type: Utilization - value: "70" diff --git a/deployment/cloud_kubernetes/keda/keda-redis-secret.yaml b/deployment/cloud_kubernetes/keda/keda-redis-secret.yaml deleted file mode 100644 index fd799638cee..00000000000 --- a/deployment/cloud_kubernetes/keda/keda-redis-secret.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: keda-redis-secret - namespace: onyx -type: Opaque -data: - host: { base64 encoded host here } - password: { base64 encoded password here } diff --git a/deployment/cloud_kubernetes/workers/beat.yaml b/deployment/cloud_kubernetes/workers/beat.yaml deleted file mode 100644 index 0b6ca72afd0..00000000000 --- a/deployment/cloud_kubernetes/workers/beat.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-beat -spec: - replicas: 1 - selector: - matchLabels: - app: celery-beat - template: - metadata: - labels: - app: celery-beat - spec: - containers: - - name: celery-beat - image: onyxdotapp/onyx-backend-cloud:v0.14.0-cloud.beta.21 - imagePullPolicy: IfNotPresent - command: - [ - "celery", - "-A", - "onyx.background.celery.versioned_apps.beat", - "beat", - "--loglevel=INFO", - ] - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: onyx-secrets - key: redis_password - - name: ONYX_VERSION - value: "v0.11.0-cloud.beta.8" - envFrom: - - configMapRef: - name: env-configmap - resources: - requests: - cpu: "250m" - memory: "512Mi" - limits: - cpu: "500m" - memory: "1Gi" diff --git a/deployment/cloud_kubernetes/workers/heavy_worker.yaml b/deployment/cloud_kubernetes/workers/heavy_worker.yaml deleted file mode 100644 index c74a1cdf411..00000000000 --- a/deployment/cloud_kubernetes/workers/heavy_worker.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-worker-heavy -spec: - replicas: 2 - selector: - matchLabels: - app: celery-worker-heavy - template: - metadata: - labels: - app: celery-worker-heavy - spec: - containers: - - name: celery-worker-heavy - image: onyxdotapp/onyx-backend-cloud:v0.14.0-cloud.beta.21 - imagePullPolicy: IfNotPresent - command: - [ - "celery", - "-A", - "onyx.background.celery.versioned_apps.heavy", - "worker", - "--loglevel=INFO", - "--hostname=heavy@%n", - "-Q", - "connector_pruning,connector_doc_permissions_sync,connector_external_group_sync", - ] - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: onyx-secrets - key: redis_password - - name: ONYX_VERSION - value: "v0.11.0-cloud.beta.8" - envFrom: - - configMapRef: - name: env-configmap - volumeMounts: - - name: vespa-certificates - mountPath: "/app/certs" - readOnly: true - resources: - requests: - cpu: "1000m" - memory: "2Gi" - limits: - cpu: "2000m" - memory: "4Gi" - volumes: - - name: vespa-certificates - secret: - secretName: vespa-certificates - items: - - key: cert.pem - path: cert.pem - - key: key.pem - path: key.pem diff --git a/deployment/cloud_kubernetes/workers/indexing_worker.yaml b/deployment/cloud_kubernetes/workers/indexing_worker.yaml deleted file mode 100644 index 894389dafac..00000000000 --- a/deployment/cloud_kubernetes/workers/indexing_worker.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-worker-indexing -spec: - replicas: 1 - selector: - matchLabels: - app: celery-worker-indexing - template: - metadata: - labels: - app: celery-worker-indexing - spec: - containers: - - name: celery-worker-indexing - image: onyxdotapp/onyx-backend-cloud:v0.14.0-cloud.beta.21 - imagePullPolicy: IfNotPresent - command: - [ - "celery", - "-A", - "onyx.background.celery.versioned_apps.indexing", - "worker", - "--loglevel=INFO", - "--hostname=indexing@%n", - "-Q", - "connector_indexing", - "--prefetch-multiplier=1", - "--concurrency=10", - ] - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: onyx-secrets - key: redis_password - - name: ONYX_VERSION - value: "v0.11.0-cloud.beta.8" - envFrom: - - configMapRef: - name: env-configmap - volumeMounts: - - name: vespa-certificates - mountPath: "/app/certs" - readOnly: true - resources: - requests: - cpu: "500m" - memory: "4Gi" - limits: - cpu: "1000m" - memory: "8Gi" - volumes: - - name: vespa-certificates - secret: - secretName: vespa-certificates - items: - - key: cert.pem - path: cert.pem - - key: key.pem - path: key.pem diff --git a/deployment/cloud_kubernetes/workers/light_worker.yaml b/deployment/cloud_kubernetes/workers/light_worker.yaml deleted file mode 100644 index c7d76782147..00000000000 --- a/deployment/cloud_kubernetes/workers/light_worker.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-worker-light -spec: - replicas: 1 - selector: - matchLabels: - app: celery-worker-light - template: - metadata: - labels: - app: celery-worker-light - spec: - containers: - - name: celery-worker-light - image: onyxdotapp/onyx-backend-cloud:v0.14.0-cloud.beta.21 - imagePullPolicy: IfNotPresent - command: - [ - "celery", - "-A", - "onyx.background.celery.versioned_apps.light", - "worker", - "--loglevel=INFO", - "--hostname=light@%n", - "-Q", - "vespa_metadata_sync,connector_deletion,doc_permissions_upsert", - "--prefetch-multiplier=1", - "--concurrency=10", - ] - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: onyx-secrets - key: redis_password - - name: ONYX_VERSION - value: "v0.11.0-cloud.beta.8" - envFrom: - - configMapRef: - name: env-configmap - volumeMounts: - - name: vespa-certificates - mountPath: "/app/certs" - readOnly: true - resources: - requests: - cpu: "500m" - memory: "1Gi" - limits: - cpu: "1000m" - memory: "2Gi" - volumes: - - name: vespa-certificates - secret: - secretName: vespa-certificates - items: - - key: cert.pem - path: cert.pem - - key: key.pem - path: key.pem diff --git a/deployment/cloud_kubernetes/workers/monitoring.yaml b/deployment/cloud_kubernetes/workers/monitoring.yaml deleted file mode 100644 index 6ccb7c99745..00000000000 --- a/deployment/cloud_kubernetes/workers/monitoring.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-worker-monitoring -spec: - replicas: 2 - selector: - matchLabels: - app: celery-worker-monitoring - template: - metadata: - labels: - app: celery-worker-monitoring - spec: - containers: - - name: celery-worker-monitoring - image: onyxdotapp/onyx-backend-cloud:v0.14.0-cloud.beta.21 - imagePullPolicy: IfNotPresent - command: - [ - "celery", - "-A", - "onyx.background.celery.versioned_apps.monitoring", - "worker", - "--loglevel=INFO", - "--hostname=monitoring@%n", - "-Q", - "monitoring", - "--prefetch-multiplier=8", - "--concurrency=8", - ] - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: onyx-secrets - key: redis_password - - name: ONYX_VERSION - value: "v0.11.0-cloud.beta.8" - envFrom: - - configMapRef: - name: env-configmap - volumeMounts: - - name: vespa-certificates - mountPath: "/app/certs" - readOnly: true - resources: - requests: - cpu: "1000m" - memory: "1Gi" - limits: - cpu: "1000m" - memory: "1Gi" - volumes: - - name: vespa-certificates - secret: - secretName: vespa-certificates - items: - - key: cert.pem - path: cert.pem - - key: key.pem - path: key.pem diff --git a/deployment/cloud_kubernetes/workers/primary.yaml b/deployment/cloud_kubernetes/workers/primary.yaml deleted file mode 100644 index 09159f30c3e..00000000000 --- a/deployment/cloud_kubernetes/workers/primary.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-worker-primary -spec: - replicas: 1 - selector: - matchLabels: - app: celery-worker-primary - template: - metadata: - labels: - app: celery-worker-primary - spec: - containers: - - name: celery-worker-primary - image: onyxdotapp/onyx-backend-cloud:v0.14.0-cloud.beta.21 - imagePullPolicy: IfNotPresent - command: - [ - "celery", - "-A", - "onyx.background.celery.versioned_apps.primary", - "worker", - "--loglevel=INFO", - "--hostname=primary@%n", - "-Q", - "celery,periodic_tasks", - "--prefetch-multiplier=1", - "--concurrency=10", - ] - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: onyx-secrets - key: redis_password - - name: ONYX_VERSION - value: "v0.11.0-cloud.beta.8" - envFrom: - - configMapRef: - name: env-configmap - volumeMounts: - - name: vespa-certificates - mountPath: "/app/certs" - readOnly: true - resources: - requests: - cpu: "500m" - memory: "1Gi" - limits: - cpu: "1000m" - memory: "2Gi" - volumes: - - name: vespa-certificates - secret: - secretName: vespa-certificates - items: - - key: cert.pem - path: cert.pem - - key: key.pem - path: key.pem