nebulous 0.1.86

A globally distributed container orchestrator
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

kind: Container
metadata:
  name: vllm-qwen
  namespace: test
  labels:
    type: inference
platform: runpod
accelerators:
  - "1:L4"
image: "vllm/vllm-openai:latest"
command: |
  env && python3 -m vllm.entrypoints.openai.api_server \
    --model $MODEL \
    --port 8000 \
    --host 0.0.0.0 \
    --gpu-memory-utilization 0.8 \
    --max-model-len 2048 \
    --max-num-seqs 1
env:
  - key: MODEL
    value: Qwen/Qwen2.5-0.5B-Instruct
  - key: HF_TOKEN
    secret_name: HF_TOKEN
restart: Always
proxy_port: 8000