syntaxdot 0.2.0

Neural sequence labeler
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

//! Iterators over data sets.

use syntaxdot_tokenizers::Tokenize;

use crate::encoders::NamedEncoder;
use crate::error::SyntaxDotError;
use crate::tensor::Tensors;

pub(crate) mod tensor_iter;

pub use conll::ConlluDataSet;

mod conll;
pub(crate) mod sentence_iter;

/// A set of training/validation data.
///
/// A `DataSet` provides an iterator over the batches in that
/// dataset.
pub trait DataSet<'a> {
    type Iter: Iterator<Item = Result<Tensors, SyntaxDotError>>;

    /// Get an iterator over the dataset batches.
    ///
    /// The sequence inputs are encoded with the given `vectorizer`,
    /// the sequence labels using the `encoder`.
    ///
    /// Sentences longer than `max_len` are skipped. If you want to
    /// include all sentences, `max_len` should be `None`.
    ///
    /// If `shuffle_buffer_size` is not `None`, the given number of
    /// sentences will be accumulated and then shuffled.
    ///
    /// If `encoders` is not `None`, output tensors will be created
    /// for the labels in the data set.
    #[allow(clippy::too_many_arguments)]
    fn batches(
        self,
        tokenizer: &'a dyn Tokenize,
        encoders: Option<&'a [NamedEncoder]>,
        batch_size: usize,
        max_len: Option<SequenceLength>,
        shuffle_buffer_size: Option<usize>,
    ) -> Result<Self::Iter, SyntaxDotError>;
}

/// The length of a sequence.
///
/// This enum can be used to express the (maximum) length of a
/// sentence in tokens or in pieces.
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum SequenceLength {
    Tokens(usize),
    Pieces(usize),
}