vyre-runtime 0.6.1

Persistent megakernel + io_uring zero-copy streaming runtime for vyre - GPU as VIR0 bytecode interpreter
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

use super::{
    MegakernelLaunchCacheStats, MegakernelLaunchPolicy, MegakernelLaunchRecommendation,
    MegakernelLaunchRequest,
};
use rustc_hash::FxHashMap;
use std::cell::RefCell;

const LAUNCH_RECOMMENDATION_CACHE_CAP: usize = 128;

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(super) struct LaunchRecommendationCacheKey {
    pub(super) policy: MegakernelLaunchPolicy,
    pub(super) request: MegakernelLaunchRequest,
}

pub(super) struct LaunchRecommendationCache {
    pub(super) entries: FxHashMap<LaunchRecommendationCacheKey, LaunchRecommendationCacheEntry>,
    clock: u64,
    pub(super) hits: u64,
    pub(super) misses: u64,
}

pub(super) struct LaunchRecommendationCacheEntry {
    recommendation: MegakernelLaunchRecommendation,
    last_seen: u64,
}

impl LaunchRecommendationCache {
    pub(super) fn get(
        &mut self,
        key: &LaunchRecommendationCacheKey,
    ) -> Option<MegakernelLaunchRecommendation> {
        if self.clock == u64::MAX {
            self.clock = 0;
            for entry in self.entries.values_mut() {
                entry.last_seen = 0;
            }
        }
        let Some(entry) = self.entries.get_mut(key) else {
            self.misses = self.misses.checked_add(1).unwrap_or_else(|| {
                panic!("megakernel launch-cache miss counter overflowed u64. Fix: reset launch-cache telemetry before counters reach u64::MAX.")
            });
            return None;
        };
        self.clock += 1;
        entry.last_seen = self.clock;
        self.hits = self.hits.checked_add(1).unwrap_or_else(|| {
            panic!("megakernel launch-cache hit counter overflowed u64. Fix: reset launch-cache telemetry before counters reach u64::MAX.")
        });
        Some(entry.recommendation)
    }

    pub(super) fn insert(
        &mut self,
        key: LaunchRecommendationCacheKey,
        value: MegakernelLaunchRecommendation,
    ) {
        let tick = self.next_tick();
        self.entries.insert(
            key,
            LaunchRecommendationCacheEntry {
                recommendation: value,
                last_seen: tick,
            },
        );
        while self.entries.len() > LAUNCH_RECOMMENDATION_CACHE_CAP {
            let Some(evicted) = self
                .entries
                .iter()
                .filter(|(candidate, _)| **candidate != key)
                .min_by_key(|(_, entry)| entry.last_seen)
                .map(|(candidate, _)| *candidate)
            else {
                break;
            };
            self.entries.remove(&evicted);
        }
    }

    pub(super) fn stats(&self) -> MegakernelLaunchCacheStats {
        MegakernelLaunchCacheStats {
            entries: self.entries.len(),
            hits: self.hits,
            misses: self.misses,
        }
    }

    pub(super) fn clear(&mut self) {
        self.entries.clear();
        self.clock = 0;
        self.hits = 0;
        self.misses = 0;
    }

    fn next_tick(&mut self) -> u64 {
        if self.clock == u64::MAX {
            self.clock = 0;
            for entry in self.entries.values_mut() {
                entry.last_seen = 0;
            }
        }
        self.clock += 1;
        self.clock
    }
}

impl Default for LaunchRecommendationCache {
    fn default() -> Self {
        Self {
            entries: FxHashMap::with_capacity_and_hasher(
                LAUNCH_RECOMMENDATION_CACHE_CAP,
                Default::default(),
            ),
            clock: 0,
            hits: 0,
            misses: 0,
        }
    }
}

thread_local! {
    pub(super) static LAUNCH_RECOMMENDATION_CACHE: RefCell<LaunchRecommendationCache> =
        RefCell::new(LaunchRecommendationCache::default());
}