vllm-cpp-sys 0.0.1

Raw FFI bindings and native build integration for the stable vllm.cpp C API
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

// vllm.cpp original (vt runtime, inventory deviation ยง9.1); no upstream mirror.
#pragma once

#include <cstddef>
#include <initializer_list>

#include "vt/backend.h"
#include "vt/tensor.h"

namespace vt {

// Per-step bump allocator over one backend slab. Activations live here and
// the whole arena rewinds each engine step. Persistent state (weights, KV
// pages, persistent batch) allocates directly from the Backend instead.
class StepArena {
 public:
  StepArena(Device device, size_t capacity_bytes);
  ~StepArena();
  StepArena(const StepArena&) = delete;
  StepArena& operator=(const StepArena&) = delete;

  Tensor Alloc(DType dtype, std::initializer_list<int64_t> shape);
  void Reset();

  size_t Used() const { return used_; }
  size_t HighWater() const { return high_water_; }
  size_t Capacity() const { return capacity_; }

 private:
  Device device_;
  char* base_ = nullptr;
  size_t capacity_ = 0;
  size_t used_ = 0;
  size_t high_water_ = 0;
};

}  // namespace vt