1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
// License: see LICENSE file at root directory of `master` branch

//! # Bi

use {
    core::{
        sync::atomic::{self, AtomicUsize},
        time::Duration,
    },
    std::{
        io::{Error, ErrorKind, Read},
        sync::Arc,
        thread,
        time::SystemTime,
    },

    crate::{Bytes, IoResult, Options},
};

/// # Atomic Ordering
pub (crate) const ATOMIC_ORDERING: atomic::Ordering = atomic::Ordering::Relaxed;

const DELAY: Duration = Duration::from_millis(10);

/// # Bi
///
/// This struct helps with controlling multiple tasks reading data into memory.
///
/// For examples, your program parses files. Naturally it scans for files on the system, then reads them into memory, and does the job. Now, if
/// you want to limit total memory it uses at a time to 256 MiB, you can do this:
///
/// ```
/// use {
///     core::time::Duration,
///     std::{
///         io::{self, Read},
///         sync::Arc,
///         thread,
///     },
///
///     bi::{Bi, Options},
/// };
///
/// let bi = Arc::new(Bi::from(Options {
///     limit: 1024 * 1024 * 256,
///     buf_size: 1024 * 64,
///     wait_timeout: Duration::from_secs(10),
/// }));
///
/// let threads = (0..100).map(|_| {
///     let bi = bi.clone();
///     thread::spawn(move || {
///         // For demonstration, we use io::repeat()
///         let bytes = bi.read(&mut io::repeat(0).take(100), 100)?;
///
///         // Do something with bytes...
///
///         io::Result::Ok(())
///     })
/// }).collect::<Vec<_>>();
///
/// for t in threads {
///     if let Err(err) = t.join().unwrap() {
///         eprintln!("{}", err);
///     }
/// }
/// ```
#[derive(Debug)]
pub struct Bi {
    options: Options,
    counter: Arc<AtomicUsize>,
}

impl Bi {

    /// # Reads into new [`Bytes`][struct:Bytes]
    ///
    /// Capacity is used to request memory upfront. It would help if you provide a good value. If capacity is lower than real data, the function
    /// will request more memory.
    ///
    /// [struct:Bytes]: struct.Bytes.html
    pub fn read<R>(&self, src: &mut R, capacity: usize) -> IoResult<Bytes> where R: Read {
        let mut buf = self.make_buf()?;

        let mut bytes = Vec::with_capacity(match self.reserve(capacity) {
            Ok(capacity) => capacity,
            Err(err) => {
                if self.counter.fetch_update(ATOMIC_ORDERING, ATOMIC_ORDERING, |c| Some(c.saturating_sub(self.options.buf_size))).is_err() {
                    // Ignore it
                }
                return Err(err);
            },
        });

        let remove_size_read_from_counter = |bytes: &Vec<_>| self.counter.fetch_update(
            ATOMIC_ORDERING, ATOMIC_ORDERING, |c| Some(c.saturating_sub(bytes.len().max(capacity)).saturating_sub(self.options.buf_size)),
        ).map_err(|_| Error::new(ErrorKind::Other, __!()));

        loop {
            match src.read(&mut buf) {
                Ok(0) => break,
                Ok(size) => {
                    let required = match bytes.len().checked_add(size) {
                        Some(new) => new.saturating_sub(bytes.capacity()),
                        None => {
                            remove_size_read_from_counter(&bytes)?;
                            return Err(Error::new(ErrorKind::Other, __!("Out of usize")));
                        },
                    };
                    let start_time = SystemTime::now();
                    loop {
                        match match required {
                            0 => Ok(0),
                            required => self.counter.fetch_update(ATOMIC_ORDERING, ATOMIC_ORDERING, |c| match c.checked_add(required) {
                                Some(new) if new <= self.options.limit => Some(new),
                                _ => None,
                            }),
                        } {
                            Ok(_) => break bytes.extend(&buf[..size]),
                            Err(_) => if let Err(err) = self.sleep(start_time) {
                                remove_size_read_from_counter(&bytes)?;
                                return Err(err);
                            },
                        };
                    }
                },
                Err(err) => {
                    remove_size_read_from_counter(&bytes)?;
                    return Err(err);
                },
            };
        }

        Ok(Bytes::new(bytes, self.counter.clone()))
    }

    /// # Reserves some space, waits for it if necessary
    fn reserve(&self, capacity: usize) -> IoResult<usize> {
        if capacity == 0 {
            return Ok(capacity);
        }

        let start_time = SystemTime::now();
        loop {
            match self.counter.fetch_update(ATOMIC_ORDERING, ATOMIC_ORDERING, |current| match current.checked_add(capacity) {
                Some(new) if new <= self.options.limit => Some(new),
                _ => None,
            }) {
                Ok(_) => return Ok(capacity),
                Err(_) => self.sleep(start_time)?,
            };
        }
    }

    /// # Makes new buffer
    fn make_buf(&self) -> IoResult<Vec<u8>> {
        const ZEROS: &[u8] = &[0; 1024];

        self.reserve(self.options.buf_size)?;

        let mut result = Vec::with_capacity(self.options.buf_size);

        while result.len() < self.options.buf_size {
            result.extend(&ZEROS[..ZEROS.len().min(self.options.buf_size.saturating_sub(result.len()))]);
        }

        Ok(result)
    }

    /// # Sleeps
    fn sleep(&self, start_time: SystemTime) -> IoResult<()> {
        let duration = SystemTime::now().duration_since(start_time).map_err(|e| Error::new(ErrorKind::Other, e))?;
        if duration < self.options.wait_timeout {
            thread::sleep(DELAY);
            Ok(())
        } else {
            Err(Error::new(ErrorKind::TimedOut, "Timed out"))
        }
    }

}

impl From<Options> for Bi {

    fn from(options: Options) -> Self {
        Self {
            options,
            counter: Arc::new(AtomicUsize::new(0)),
        }
    }

}