pub fn build_sequence<T: Tokenizer + ?Sized>(
tok: &T,
state: &Value,
q: &InternalQ,
max_len: usize,
head_max_len: usize,
option_order: Option<&[usize]>,
truncate_left: bool,
) -> Result<(Vec<u32>, Vec<usize>)>Expand description
Build one token sequence + option marker positions.
Format: [CLS] <type> question: <instructions> [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP].
Ported verbatim from build_sequence in laya/common.py.