1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
#![doc(html_root_url = "https://docs.rs/slog-retry/0.1.1/slog-retry/")]
#![warn(missing_docs)]

//! A slog adapter for retrying on errors.
//!
//! The [slog](https://crates.io/crates/slog) infrastructure is very powerful and can be bent to
//! many scenarios. Many of the loggers there may fail, for example because they log over the
//! network ‒ like [slog-json](https://crates.io/crates/slog-json) logging into a TCP stream.
//!
//! The basic framework allows for three options:
//!
//! * Handle the error manually, which is uncomfortable.
//! * Ignore the error, but then all the future records are lost.
//! * Fuse the drain, making it explode on the first error and killing the whole application.
//!
//! This crate brings an adapter that initializes the drain anew each time an error happens, adding
//! the ability to recover from errors.
//!
//! # Warning
//!
//! The adapter blocks the current thread on reconnects. Therefore, you want to wrap it inside
//! [slog-async](https://crates.io/crates/slog-async) and not it directly as the root drain.
//!
//! # Examples
//!
//! ```rust,no_run
//! #[macro_use]
//! extern crate slog;
//! extern crate slog_async;
//! extern crate slog_json;
//! extern crate slog_retry;
//! 
//! use std::net::TcpStream;
//! 
//! use slog::Drain;
//! 
//! fn main() {
//!     let retry = slog_retry::Retry::new(|| -> Result<_, std::io::Error> {
//!             let connection = TcpStream::connect("127.0.0.1:1234")?;
//!             Ok(slog_json::Json::default(connection))
//!         }, None, true)
//!         // Kill the application if the initial connection fails
//!         .unwrap()
//!         // Ignore if it isn't possible to log some of the messages, we'll try again
//!         .ignore_res();
//!     let async = slog_async::Async::default(retry)
//!         .fuse();
//!     let root = slog::Logger::root(async, o!());
//!     info!(root, "Everything is set up");
//! }
//! ```

extern crate failure;
extern crate slog;

use std::cell::{Cell, RefCell, RefMut};
use std::fmt::{Debug, Display, Formatter, Result as FmtResult};
use std::iter;
use std::thread;
use std::time::Duration;

use failure::Fail;
use slog::{Drain, OwnedKVList, Record};

/// An error when the retry adaptor fails.
///
/// It wasn't possible to log the record (or initialize it when starting). Usually that means it
/// wasn't possible to create the drain at all or that each newly created drain failed.
#[derive(Debug)]
pub struct Error<FactoryError: Fail + Debug, SlaveError: Fail + Debug> {
    /// The last error during creation of a new drain, if any.
    pub factory: Option<FactoryError>,
    /// The last error during logging attempt, if any.
    pub slave: Option<SlaveError>,
}

impl<FactoryError, SlaveError> Fail for Error<FactoryError, SlaveError>
where
    FactoryError: Fail + Debug,
    SlaveError: Fail + Debug,
{
    fn cause(&self) -> Option<&Fail> {
        if let Some(ref slave) = self.slave {
            return Some(slave);
        }
        if let Some(ref fact) = self.factory {
            return Some(fact);
        }
        None
    }
}

impl<FactoryError, SlaveError> Display for Error<FactoryError, SlaveError>
where
    FactoryError: Fail + Debug,
    SlaveError: Fail + Debug,
{
    fn fmt(&self, fmt: &mut Formatter) -> FmtResult {
        let factory = self.factory
            .as_ref()
            .map(|f| format!("{}", f))
            .unwrap_or_else(|| "()".to_owned());
        let slave = self.slave
            .as_ref()
            .map(|s| format!("{}", s))
            .unwrap_or_else(|| "()".to_owned());
        write!(
            fmt,
            "Failed to reconnect the logging drain: {}/{}",
            factory, slave
        )
    }
}

/// A retry strategy.
///
/// The iterator describes how long to wait before reconnection attempts. Once the iterator runs
/// out of items, the adapter gives up trying to reconnect. Therefore, it specifies both the
/// waiting intervals and number of retries.
pub type Strategy = Box<Iterator<Item = Duration>>;

/// A constructor of a new instance of a retry strategy.
///
/// Every time the adapter needs to retry connection, it calls the constructor and gets a fresh
/// retry strategy.
pub type NewStrategy = Box<Fn() -> Strategy + Send>;

/// The retry adapter.
///
/// This wraps another drain and forwards log records into that. However, if the drain returns an
/// error, it discards it and tries to create a new one and log the message into it.
///
/// It uses the [retry strategy](type.Strategy.html) to decide how long to wait before retrying and
/// how many times. If the retry strategy runs out of items, it gives up, returns an error and the
/// log record is lost.
///
/// However, it is not destroyed by the error and if it is called to log another record, it tries
/// to reconnect again (using a fresh instance of the strategy).
///
/// # Warning
///
/// This adapter is *synchronous* and *blocks* during the retry attempts. Unless you provide a
/// retry strategy with a single zero item, you don't want to use it directly. Wrap it inside
/// [slog-async](https://crates.io/crates/slog-async), where it'll only slow down the logging
/// thread and the channel into that thread will be used as a buffer for messages waiting to be
/// written after the reconnect.
pub struct Retry<Slave, Factory> {
    slave: RefCell<Option<Slave>>,
    factory: Factory,
    strategy: NewStrategy,
    initialized: Cell<bool>,
}

impl<Slave, FactoryError, Factory> Retry<Slave, Factory>
where
    Slave: Drain,
    FactoryError: Fail + Debug,
    Slave::Err: Fail + Debug,
    Factory: Fn() -> Result<Slave, FactoryError>,
{
    /// Creates a new retry adapter.
    ///
    /// # Parameters
    ///
    /// * `factory`: A factory function that is used to produce new instance of the slave drain on
    ///   every (re)connection attempt.
    /// * `strategy`: A reconnect strategy, describing how long to wait between attempts and how
    ///   many attempts to make. If set to `None` a default strategy with 4 increasingly delayed
    ///   attemps is used.
    /// * `connect_now`: Should a connection be made right away. If it is set to `true`, it may
    ///   block (it uses the reconnect strategy provided) and it may return an error. If set to
    ///   `false`, the connection is made on the first logged message. No matter if connecting now
    ///   or later, the first connection attempt is without waiting.
    pub fn new(
        factory: Factory,
        strategy: Option<NewStrategy>,
        connect_now: bool,
    ) -> Result<Self, Error<FactoryError, Slave::Err>> {
        let result = Self {
            slave: RefCell::new(None),
            factory,
            strategy: strategy.unwrap_or_else(|| Box::new(|| default_new_strategy())),
            initialized: Cell::new(false),
        };
        if connect_now {
            result
                .init(&mut result.slave.borrow_mut(), &mut (result.strategy)())
                .map_err(|factory| {
                    Error {
                        factory,
                        slave: None,
                    }
                })?;
        }
        Ok(result)
    }
    fn init(
        &self,
        slave: &mut RefMut<Option<Slave>>,
        strategy: &mut Strategy,
    ) -> Result<(), Option<FactoryError>> {
        let prefix: Strategy = if self.initialized.get() {
            Box::new(iter::empty())
        } else {
            self.initialized.set(true);
            Box::new(iter::once(Duration::from_secs(0)))
        };
        let mut last_err = None;
        for sleep in prefix.chain(strategy) {
            thread::sleep(sleep);
            match (self.factory)() {
                Ok(ok) => {
                    **slave = Some(ok);
                    return Ok(());
                },
                Err(err) => last_err = Some(err),
            }
        }
        Err(last_err)
    }
}

impl<Slave, FactoryError, Factory> Drain for Retry<Slave, Factory>
where
    Slave: Drain,
    FactoryError: Fail + Debug,
    Slave::Err: Fail + Debug,
    Factory: Fn() -> Result<Slave, FactoryError>,
{
    type Ok = Slave::Ok;
    type Err = Error<FactoryError, Slave::Err>;
    fn log(&self, record: &Record, values: &OwnedKVList) -> Result<Self::Ok, Self::Err> {
        let mut borrowed = self.slave.borrow_mut();
        let mut slave_err = None;

        if let Some(ref slave) = *borrowed {
            match slave.log(record, values) {
                Ok(ok) => return Ok(ok),
                Err(err) => slave_err = Some(err),
            }
        }
        // By now there was no slave to start with or it failed, so we recreate it.
        borrowed.take();

        // Try creating a new one and retry with that.
        let mut strategy = (self.strategy)();
        loop {
            match self.init(&mut borrowed, &mut strategy) {
                Err(factory) =>
                    return Err(Error {
                        factory,
                        slave: slave_err,
                    }),
                Ok(()) => match borrowed.as_ref().unwrap().log(record, values) {
                    Ok(ok) => return Ok(ok),
                    Err(err) => {
                        slave_err = Some(err);
                        borrowed.take();
                    },
                },
            }
        }
    }
}

fn default_new_strategy() -> Strategy {
    let iterator = (1..5).map(Duration::from_secs);
    Box::new(iterator)
}