dynamo-llm 1.0.2

Dynamo LLM Library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::cmp::Ordering;
use std::sync::Weak;
use tokio::sync::oneshot;

use crate::block_manager::block::{
    BlockMetadata, ImmutableBlock, MutableBlock, locality::LocalityProvider,
};
use crate::block_manager::pool::BlockPoolError;
use crate::block_manager::storage::Storage;

/// Higher priority offloads are done first.
/// If two offloads have the same priority, the one that was requested first is done first.
#[derive(PartialEq, Eq)]
pub struct OffloadRequestKey {
    pub priority: u64,
    pub timestamp: u64,
}

impl PartialOrd for OffloadRequestKey {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for OffloadRequestKey {
    fn cmp(&self, other: &Self) -> Ordering {
        other
            .priority
            .cmp(&self.priority)
            .then(self.timestamp.cmp(&other.timestamp))
    }
}

/// Data needed to offload a block.
/// While the block is in the offload queue, we hold a weak reference to it.
/// This way, we don't prevent the block from being reused if needed.
pub struct OffloadRequest<S: Storage, L: LocalityProvider, M: BlockMetadata> {
    pub key: OffloadRequestKey,
    pub block: Weak<MutableBlock<S, L, M>>,
    pub sequence_hash: u64,
}

impl<S: Storage, L: LocalityProvider, M: BlockMetadata> PartialOrd for OffloadRequest<S, L, M> {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

/// Order offload requests by priority, high to low.
impl<S: Storage, L: LocalityProvider, M: BlockMetadata> Ord for OffloadRequest<S, L, M> {
    fn cmp(&self, other: &Self) -> Ordering {
        self.key.cmp(&other.key)
    }
}

/// Equality is based on sequence hash, priority, and location.
impl<S: Storage, L: LocalityProvider, M: BlockMetadata> PartialEq for OffloadRequest<S, L, M> {
    fn eq(&self, other: &Self) -> bool {
        self.key == other.key
    }
}

impl<S: Storage, L: LocalityProvider, M: BlockMetadata> Eq for OffloadRequest<S, L, M> {}

pub type BlockResult<Target, Locality, Metadata> =
    Result<Vec<ImmutableBlock<Target, Locality, Metadata>>, BlockPoolError>;

pub type ResponseSender<Target, Locality, Metadata> =
    oneshot::Sender<Result<Vec<ImmutableBlock<Target, Locality, Metadata>>, BlockPoolError>>;

/// Data needed for onboarding.
/// Unlike offloading, we need a means to return the resulting blocks to the caller.
pub struct OnboardRequest<
    Source: Storage,
    Target: Storage,
    Locality: LocalityProvider,
    M: BlockMetadata,
> {
    pub blocks: Vec<ImmutableBlock<Source, Locality, M>>,
    pub response_tx: ResponseSender<Target, Locality, M>,
    pub targets: Option<Vec<MutableBlock<Target, Locality, M>>>,
}

impl<Source: Storage, Target: Storage, Locality: LocalityProvider, M: BlockMetadata>
    OnboardRequest<Source, Target, Locality, M>
{
    pub fn new(
        blocks: Vec<ImmutableBlock<Source, Locality, M>>,
        response_tx: ResponseSender<Target, Locality, M>,
        targets: Option<Vec<MutableBlock<Target, Locality, M>>>,
    ) -> Self {
        Self {
            blocks,
            response_tx,
            targets,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_offload_request_key_ordering() {
        let key1 = OffloadRequestKey {
            priority: 1,
            timestamp: 1,
        };

        let key2 = OffloadRequestKey {
            priority: 2,
            timestamp: 2,
        };

        assert!(key2 < key1);

        let key3 = OffloadRequestKey {
            priority: 2,
            timestamp: 3,
        };

        assert!(key2 < key3);
    }
}