1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
use futures::FutureExt;
use futures::{poll, SinkExt};
use log::{error, warn};
use std::future::Future;
use std::hash::Hash;
use std::ops::{Deref, DerefMut};
use std::pin::Pin;
use std::task::{Context, Poll};
use std::thread;
use std::time::Duration;
use tokio::spawn;
use tokio::sync::mpsc;

use dynamic_pool::{DynamicPool, DynamicPoolItem, DynamicReset};
use futures::future::{join_all, try_join_all, JoinAll};
use tokio::task::{spawn_blocking, JoinError, JoinHandle, JoinSet};
use tokio::time::error::Elapsed;

use super::sender::ShutdownHandleShardSender;
use super::types::GracefulShutdownFuture;
use super::{shard, Commit, ServiceHandleMessage, ServiceHandleShardSender, ShardStats, TakenData};

use crate::{ServiceData, ShardError, ShardShutdownStats};

struct ServiceHandleShardSenderVec<Key, Data>(Vec<ServiceHandleShardSender<Key, Data>>);
impl<Key, Data> DynamicReset for ServiceHandleShardSenderVec<Key, Data> {
    fn reset(&mut self) {}
}

impl<Key, Data> Deref for ServiceHandleShardSenderVec<Key, Data> {
    type Target = Vec<ServiceHandleShardSender<Key, Data>>;
    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl<Key, Data> DerefMut for ServiceHandleShardSenderVec<Key, Data> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.0
    }
}

pub struct ServiceHandle<Key, Data> {
    pool: DynamicPool<ServiceHandleShardSenderVec<Key, Data>>,
    shards: DynamicPoolItem<ServiceHandleShardSenderVec<Key, Data>>,
}

impl<Key: Send + 'static, Data: ServiceData> ServiceHandle<Key, Data> {
    pub(super) fn with_senders_and_pool_capacity(
        senders: Vec<mpsc::Sender<ServiceHandleMessage<Key, Data>>>,
        handle_pool_capacity: usize,
    ) -> Self {
        let shards: Vec<_> = senders
            .into_iter()
            .map(ServiceHandleShardSender::from_sender)
            .collect();

        assert!(
            !shards.is_empty(),
            "Somehow, a ServiceHandle was tried to be constructed that holds 0 shards."
        );

        let pool = DynamicPool::new(handle_pool_capacity, handle_pool_capacity, move || {
            ServiceHandleShardSenderVec(shards.clone())
        });
        let shards = pool.take();

        ServiceHandle { pool, shards }
    }
}

impl<Key: Send + Hash, Data: ServiceData> ServiceHandle<Key, Data> {
    #[inline]
    pub fn handle(&self) -> Self {
        self.clone()
    }

    #[inline]
    fn select_shard(&mut self, key: &Key) -> &mut ServiceHandleShardSender<Key, Data> {
        let shard_len = self.shards.len();
        // When operating in single shard mode, we can circumvent having to hash the key,
        // as there is only one shard that will be able to service it anyways.
        let shard_idx = if shard_len == 1 {
            0
        } else {
            let key_hash = fxhash::hash(&key);
            key_hash % shard_len
        };

        // safety: index is bounds checked above.
        unsafe { self.shards.get_unchecked_mut(shard_idx) }
    }

    #[inline]
    pub fn execute<F, T>(
        &mut self,
        key: Key,
        func: F,
    ) -> impl Future<Output = Result<T, ShardError>> + '_
    where
        F: FnOnce(&Data) -> T + Send + 'static,
        T: Send + 'static,
    {
        self.select_shard(&key).execute(key, func)
    }

    #[inline]
    pub fn execute_if_cached<F, T>(
        &mut self,
        key: Key,
        func: F,
    ) -> impl Future<Output = Result<Option<T>, ShardError>> + '_
    where
        F: FnOnce(&Data) -> T + Send + 'static,
        T: Send + 'static,
    {
        self.select_shard(&key).execute_if_cached(key, func)
    }

    #[inline]
    pub fn take_data(
        &mut self,
        key: Key,
    ) -> impl Future<Output = Result<Option<TakenData<Key, Data>>, ShardError>> + '_ {
        self.select_shard(&key).take_data(key)
    }

    pub async fn get_shard_stats(&mut self) -> Result<Vec<ShardStats>, ShardError> {
        try_join_all(
            self.shards
                .iter_mut()
                .map(|s| s.get_shard_stats())
                .collect::<Vec<_>>(),
        )
        .await
    }
}

impl<Key, Data> Clone for ServiceHandle<Key, Data> {
    fn clone(&self) -> Self {
        let shards = self.pool.take();
        let pool = self.pool.clone();
        Self { shards, pool }
    }
}

pub struct MutableServiceHandle<Key, Data>(ServiceHandle<Key, Data>);

impl<Key: Send + Hash, Data: ServiceData> MutableServiceHandle<Key, Data> {
    pub(super) fn from_service_handle(service_handle: ServiceHandle<Key, Data>) -> Self {
        Self(service_handle)
    }

    pub fn into_immutable_handle(self) -> ServiceHandle<Key, Data> {
        self.0
    }

    #[inline]
    pub fn handle(&self) -> Self {
        self.clone()
    }

    #[inline]
    pub fn execute_mut<F, T>(
        &mut self,
        key: Key,
        func: F,
    ) -> impl Future<Output = Result<T, ShardError>> + '_
    where
        F: FnOnce(&mut Data) -> Commit<T> + Send + 'static,
        T: Send + 'static,
    {
        self.0.select_shard(&key).execute_mut(key, func)
    }
}

impl<Key, Data> Clone for MutableServiceHandle<Key, Data> {
    fn clone(&self) -> Self {
        Self(self.0.clone())
    }
}

impl<Key, Data> Deref for MutableServiceHandle<Key, Data> {
    type Target = ServiceHandle<Key, Data>;
    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl<Key, Data> DerefMut for MutableServiceHandle<Key, Data> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.0
    }
}

struct ShutdownHandleInner {
    shard_join_handles: Vec<JoinHandle<()>>,
    graceful_shutdown_future: GracefulShutdownFuture,
}

pub struct ShutdownHandle {
    inner: Option<ShutdownHandleInner>,
}

impl ShutdownHandle {
    pub(super) fn with_senders_and_join_handles<Key, Data>(
        senders: Vec<mpsc::Sender<ServiceHandleMessage<Key, Data>>>,
        shard_join_handles: Vec<JoinHandle<()>>,
    ) -> Self
    where
        Key: Send + 'static,
        Data: ServiceData,
    {
        let shards: Vec<_> = senders
            .into_iter()
            .map(ShutdownHandleShardSender::from_sender)
            .collect();

        assert!(
            !shards.is_empty(),
            "Somehow, a ShutdownHandle tried to be constructed that holds 0 shards."
        );

        let graceful_shutdown_future =
            Box::pin(try_join_all(shards.into_iter().map(|s| s.shutdown())));

        ShutdownHandle {
            inner: Some(ShutdownHandleInner {
                shard_join_handles,
                graceful_shutdown_future,
            }),
        }
    }
}

impl ShutdownHandle {
    /// Begin gracefully shutting down, returning a `GracefulShutdownHandle` that can be used to interact with the
    /// shutdown further.
    ///
    /// **Note**: Dropping the `GracefulShutdownHandle` will cause a hard shutdown.
    pub fn gracefully_shutdown(mut self) -> GracefulShutdownHandle {
        let inner = self
            .inner
            .take()
            .expect("invariant: missing shutdown handle inner");

        GracefulShutdownHandle {
            inner: Some(GracefulShutdownHandleInner {
                shard_join_handles: inner.shard_join_handles,
                graceful_shutdown_join_handle: tokio::spawn(async move {
                    let result = inner.graceful_shutdown_future.await;

                    match result {
                        Ok(shutdown_stats_vec) => {
                            ShutdownResult::GracefullyShutdown(shutdown_stats_vec)
                        }
                        Err(_) => {
                            error!("one or more shards were already shutdown during the graceful shutdown. performing hard shutdown");
                            ShutdownResult::HardShutdown
                        }
                    }
                }),
            }),
        }
    }
}

impl Drop for ShutdownHandle {
    fn drop(&mut self) {
        let Some(inner) = self.inner.take() else {
            return;
        };

        warn!("ShutdownHandle was dropped before a shutdown was initiated; hard shutting down immediately.");
        for handle in inner.shard_join_handles {
            handle.abort();
        }
    }
}

struct GracefulShutdownHandleInner {
    shard_join_handles: Vec<JoinHandle<()>>,
    graceful_shutdown_join_handle: JoinHandle<ShutdownResult>,
}

pub struct GracefulShutdownHandle {
    inner: Option<GracefulShutdownHandleInner>,
}

impl GracefulShutdownHandle {
    /// Awaits the graceful shutdown, returning a `ShutdownResult::GracefullyShutdown`
    /// once it completes. A `ShutdownResult::HardShutdown` may be returned if the
    /// shutdown errors.
    pub async fn join(&mut self) -> ShutdownResult {
        match self.inner.as_mut() {
            Some(inner) => {
                let handle = &mut inner.graceful_shutdown_join_handle;
                let result = handle.await.expect("shutdown task has panicked");
                self.inner = None;
                result
            }
            None => ShutdownResult::AlreadyShutdown,
        }
    }

    /// Wait for the graceful shutdown to finish, returning a `ShutdownResult::GracefullyShutdown` if it does.
    /// If the graceful shutdown *does not* finish before the provided timeout, a hard shutdown is performed
    /// (and a `ShutdownResult::HardShutdown` is returned).
    pub async fn hard_shutdown_after(mut self, timeout: std::time::Duration) -> ShutdownResult {
        match tokio::time::timeout(timeout, self.join()).await {
            Ok(shutdown_result) => shutdown_result,
            Err(_) => self.hard_shutdown_immediatey(),
        }
    }

    /// Immediately finish shutdown, returning a `ShutdownResult::GracefullyShutdown` result if the
    /// graceful shutdown has finished, or a `ShutdownResult::HardShutdown` result if it has not
    /// (meaning that data was lost).
    pub fn hard_shutdown_immediatey(mut self) -> ShutdownResult {
        let inner = self
            .inner
            .take()
            .expect("invariant: missing graceful shutdown join handle");

        match inner.graceful_shutdown_join_handle.now_or_never() {
            Some(result) => result.expect("shutdown task panicked"),
            None => {
                for handle in inner.shard_join_handles {
                    handle.abort();
                }
                ShutdownResult::HardShutdown
            }
        }
    }
}

impl Drop for GracefulShutdownHandle {
    fn drop(&mut self) {
        let Some(inner) = self.inner.take() else {
            return;
        };
        warn!("GracefulShutdownHandle was dropped before shutdown was complete; hard shutting down immediately.");
        inner.graceful_shutdown_join_handle.abort();
        // We leave this in a separate if because it is expected behaviour for the shard join handles
        // to be present in the event of an error / hard shutdown.
        for handle in inner.shard_join_handles {
            handle.abort();
        }
    }
}

#[derive(Debug)]
pub enum ShutdownResult {
    /// Thingvellir was gracefully shut down and all data was committed.
    GracefullyShutdown(Vec<ShardShutdownStats>),

    /// Thingvellir was hard shut down and uncommitted data was lost.
    HardShutdown,

    /// Joined on the graceful shutdown handle more than one time, and thingvellir was already shutdown.
    AlreadyShutdown,
}