trustformers_training::sequence_parallelism

Struct SequenceParallelismConfig

pub struct SequenceParallelismConfig {
    pub sequence_parallel_size: usize,
    pub max_sequence_length_per_device: usize,
    pub overlap_size: usize,
    pub attention_communication_opt: bool,
    pub communication_pattern: SequenceCommunicationPattern,
    pub splitting_strategy: SequenceSplittingStrategy,
    pub sync_gradients: bool,
    pub memory_optimization: SequenceMemoryOptimization,
    pub use_checkpointing: bool,
}

Expand description

Sequence Parallelism Configuration

Sequence parallelism distributes long sequences across multiple devices, enabling the processing of sequences that are too long to fit on a single device. This is particularly useful for very long document processing, DNA sequences, or other sequential data that exceeds memory limits.

Fields§

§sequence_parallel_size: usize

Number of devices for sequence parallelism

§max_sequence_length_per_device: usize

Maximum sequence length per device

§overlap_size: usize

Overlap size between adjacent sequence chunks

§attention_communication_opt: bool

Whether to use attention communication optimization

§communication_pattern: SequenceCommunicationPattern

Communication pattern for sequence parallelism

§splitting_strategy: SequenceSplittingStrategy

Sequence splitting strategy

§sync_gradients: bool

Whether to use gradient synchronization across sequence chunks

§memory_optimization: SequenceMemoryOptimization

Memory optimization for long sequences

§use_checkpointing: bool

Whether to use checkpointing for sequence chunks

SequenceParallelismConfig

Struct SequenceParallelismConfig Copy item path

Fields§

Trait Implementations§

impl Clone for SequenceParallelismConfig

fn clone(&self) -> SequenceParallelismConfig

fn clone_from(&mut self, source: &Self)

impl Debug for SequenceParallelismConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl Default for SequenceParallelismConfig

fn default() -> Self

impl<'de> Deserialize<'de> for SequenceParallelismConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where __D: Deserializer<'de>,

impl Serialize for SequenceParallelismConfig

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where __S: Serializer,

Auto Trait Implementations§

impl Freeze for SequenceParallelismConfig

impl RefUnwindSafe for SequenceParallelismConfig

impl Send for SequenceParallelismConfig

impl Sync for SequenceParallelismConfig

impl Unpin for SequenceParallelismConfig

impl UnsafeUnpin for SequenceParallelismConfig

impl UnwindSafe for SequenceParallelismConfig

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> ConfigSerializable for Twhere T: Serialize + for<'de> Deserialize<'de>,

fn to_json(&self) -> Result<String, TrustformersError>

fn from_json(json: &str) -> Result<T, TrustformersError>

fn save_to_file(&self, path: &Path) -> Result<(), TrustformersError>

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>where Self: Sized,

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

impl<T> ToOwned for Twhere T: Clone,

type Owned = T

fn to_owned(&self) -> T

fn clone_into(&self, target: &mut T)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<V, T> VZip<V> for Twhere V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>where S: Into<Dispatch>,

fn with_current_subscriber(self) -> WithDispatch<Self>

impl<T> DeserializeOwned for Twhere T: for<'de> Deserialize<'de>,

Struct SequenceParallelismConfig

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where __D: Deserializer<'de>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T> ConfigSerializable for T
where T: Serialize + for<'de> Deserialize<'de>,

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>
where Self: Sized,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,