apalis-file-storage 0.1.0-rc.9

A test-friendly single process file-based storage backend for apalis
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

use std::{
    pin::Pin,
    task::{Context, Poll},
};

use futures_channel::mpsc::SendError;
use futures_sink::Sink;
use serde::{Serialize, de::Deserialize};
use serde_json::Value;

use apalis_core::task::{
    Task,
    task_id::{RandomId, TaskId},
};

use crate::{
    JsonMapMetadata, JsonStorage,
    util::{TaskKey, TaskWithMeta},
};

impl<Args> Sink<Task<Value, JsonMapMetadata, RandomId>> for JsonStorage<Args>
where
    Args: Unpin + Serialize + for<'de> Deserialize<'de>,
{
    type Error = SendError;

    fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
        Poll::Ready(Ok(()))
    }

    fn start_send(
        self: Pin<&mut Self>,
        item: Task<Value, JsonMapMetadata, RandomId>,
    ) -> Result<(), Self::Error> {
        let this = Pin::get_mut(self);

        let task_id = item
            .parts
            .task_id
            .clone()
            .unwrap_or(TaskId::new(RandomId::default()));

        let queue = std::any::type_name::<Args>().to_owned();
        let idempotency_key = item.parts.idempotency_key.clone();

        // Prevent duplicates already stored in tasks
        if let Some(ref key) = idempotency_key {
            let tasks = this.tasks.read().unwrap();

            let exists = tasks.values().any(|task| {
                task.idempotency_key
                    .as_ref()
                    .map(|existing| existing == key)
                    .unwrap_or(false)
            });

            if exists {
                return Ok(());
            }
        }

        let task_key = TaskKey {
            task_id,
            queue,
            status: apalis_core::task::status::Status::Pending,
        };

        this.tasks.write().unwrap().insert(
            task_key,
            TaskWithMeta {
                args: item.args,
                ctx: item.parts.ctx,
                result: None,
                idempotency_key,
            },
        );

        Ok(())
    }

    fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
        let this = Pin::get_mut(self);

        let tasks: Vec<_> = this.buffer.drain(..).collect();

        for task in tasks {
            let task_id = task
                .parts
                .task_id
                .clone()
                .unwrap_or(TaskId::new(RandomId::default()));

            let key = TaskKey {
                task_id,
                queue: std::any::type_name::<Args>().to_owned(),
                status: apalis_core::task::status::Status::Pending,
            };

            this.insert(
                &key,
                TaskWithMeta {
                    args: task.args,
                    ctx: task.parts.ctx,
                    result: None,
                    idempotency_key: task.parts.idempotency_key.clone(),
                },
            )
            .unwrap();
        }

        Poll::Ready(Ok(()))
    }

    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
        Sink::<Task<Value, JsonMapMetadata, RandomId>>::poll_flush(self, cx)
    }
}