kubernetes/apps/ai/llmkube/models/bge-reranker-v2-m3.yaml
1
---
2
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/model_v1alpha1.json
3
apiVersion: inference.llmkube.dev/v1alpha1
4
kind: Model
5
metadata:
6
name: bge-reranker-v2-m3
7
spec:
8
format: gguf
9
hardware:
10
accelerator: vulkan
11
gpu:
12
enabled: true
13
layers: 99
14
vendor: amd
15
resourceClaims:
16
- name: gpu
17
resourceClaimTemplateName: llmkube-models-gpu
18
quantization: Q8_0
19
source: https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF/resolve/main/bge-reranker-v2-m3-Q8_0.gguf
20
---
21
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/inferenceservice_v1alpha1.json
22
apiVersion: inference.llmkube.dev/v1alpha1
23
kind: InferenceService
24
metadata:
25
name: bge-reranker-v2-m3
26
spec:
27
modelRef: bge-reranker-v2-m3
28
runtime: llamacpp
29
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731@sha256:0262343f64f030e8efff15ec05ee8180b69b7f6745de2caa3af5148e0dcb8aba
30
replicas: 1
31
contextSize: 4096
32
parallelSlots: 2
33
batchSize: 1024
34
uBatchSize: 1024
35
resources:
36
cpu: 100m
37
memory: 2Gi
38
extraArgs:
39
- --reranking
40
- --pooling
41
- rank
42
- --embedding
43
- --threads
44
- "4"
45
- --cache-ram
46
- "0"
47
- --no-mmap
48
- --sleep-idle-seconds
49
- "300"
50
- --alias
51
- bge-reranker-v2-m3
52
- --metrics
53
endpoint:
54
port: 8080
55
type: ClusterIP
56
probeOverrides:
57
startup:
58
httpGet:
59
path: /health
60
port: 8080
61
periodSeconds: 10
62
failureThreshold: 60
63
liveness:
64
httpGet:
65
path: /health
66
port: 8080
67
periodSeconds: 30
68
failureThreshold: 3
69
readiness:
70
httpGet:
71
path: /health
72
port: 8080
73
periodSeconds: 10
74
failureThreshold: 3