kubernetes/apps/ai/llmkube/models/bge-embedding-m3.yaml
1
---
2
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/model_v1alpha1.json
3
apiVersion: inference.llmkube.dev/v1alpha1
4
kind: Model
5
metadata:
6
name: bge-embedding-m3
7
spec:
8
format: gguf
9
hardware:
10
accelerator: vulkan
11
gpu:
12
enabled: true
13
layers: 99
14
vendor: amd
15
resourceClaims:
16
- name: gpu
17
resourceClaimTemplateName: llmkube-models-gpu
18
quantization: Q8_0
19
source: https://huggingface.co/gpustack/bge-m3-GGUF/resolve/main/bge-m3-Q8_0.gguf
20
---
21
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/inferenceservice_v1alpha1.json
22
apiVersion: inference.llmkube.dev/v1alpha1
23
kind: InferenceService
24
metadata:
25
name: bge-embedding-m3
26
spec:
27
modelRef: bge-embedding-m3
28
runtime: llamacpp
29
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
30
replicas: 1
31
contextSize: 8192
32
parallelSlots: 2
33
batchSize: 8192
34
uBatchSize: 4096
35
resources:
36
cpu: 100m
37
memory: 2Gi
38
extraArgs:
39
- --embedding
40
- --pooling
41
- cls
42
- --threads
43
- "4"
44
- --cont-batching
45
- --cache-ram
46
- "0"
47
- --sleep-idle-seconds
48
- "300"
49
- --alias
50
- bge-m3
51
- --metrics
52
endpoint:
53
port: 8080
54
type: ClusterIP
55
probeOverrides:
56
startup:
57
httpGet:
58
path: /health
59
port: 8080
60
periodSeconds: 10
61
failureThreshold: 60
62
liveness:
63
httpGet:
64
path: /health
65
port: 8080
66
periodSeconds: 30
67
failureThreshold: 3
68
readiness:
69
httpGet:
70
path: /health
71
port: 8080
72
periodSeconds: 10
73
failureThreshold: 3