gguf_rs_lib/tensor/
data.rs

1//! Tensor data storage and management
2
3use crate::error::{GGUFError, Result};
4#[cfg(feature = "std")]
5use serde::{Deserialize, Serialize};
6#[cfg(feature = "std")]
7use std::sync::Arc;
8
9#[cfg(not(feature = "std"))]
10extern crate alloc;
11#[cfg(not(feature = "std"))]
12use alloc::{
13    format,
14    string::{String, ToString},
15    sync::Arc,
16    vec,
17    vec::Vec,
18};
19
20// Import core modules for no_std compatibility
21#[cfg(not(feature = "std"))]
22use core::{fmt, mem};
23
24fn checked_data_range(
25    start: usize,
26    length: usize,
27    available: usize,
28    description: &str,
29) -> Result<core::ops::Range<usize>> {
30    let end = start.checked_add(length).ok_or_else(|| {
31        GGUFError::InvalidTensorData(format!(
32            "{description} start {start} plus length {length} overflows usize"
33        ))
34    })?;
35
36    if end > available {
37        return Err(GGUFError::InvalidTensorData(format!(
38            "{description} {start}..{end} exceeds data length {available}"
39        )));
40    }
41
42    Ok(start..end)
43}
44
45/// Container for tensor data with different storage backends
46#[derive(Debug, Clone, Default)]
47pub enum TensorData {
48    /// Owned byte vector (most common case)
49    Owned(Vec<u8>),
50
51    /// Reference to borrowed data (zero-copy when possible)
52    Borrowed(&'static [u8]),
53
54    /// Shared reference-counted data
55    Shared(Arc<Vec<u8>>),
56
57    /// Memory-mapped data (when using mmap feature)
58    #[cfg(feature = "mmap")]
59    Mapped {
60        /// The memory map
61        mmap: Arc<memmap2::Mmap>,
62        /// Offset within the map
63        offset: usize,
64        /// Length of the data
65        length: usize,
66    },
67
68    /// Empty/uninitialized data
69    #[default]
70    Empty,
71}
72
73/// Information about tensor data storage
74#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct TensorDataInfo {
77    /// Size of the data in bytes
78    pub size_bytes: usize,
79    /// Storage type
80    pub storage_type: TensorStorageType,
81    /// Whether the data is aligned
82    pub is_aligned: bool,
83    /// Alignment boundary (if applicable)
84    pub alignment: Option<usize>,
85}
86
87/// Types of tensor storage
88#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
90pub enum TensorStorageType {
91    /// Owned in-memory storage
92    Owned,
93    /// Borrowed from external source
94    Borrowed,
95    /// Shared reference-counted storage
96    Shared,
97    /// Memory-mapped file storage
98    Mapped,
99    /// Empty/uninitialized
100    Empty,
101}
102
103impl TensorData {
104    /// Create new owned tensor data
105    pub fn new_owned(data: Vec<u8>) -> Self {
106        Self::Owned(data)
107    }
108
109    /// Create new borrowed tensor data
110    pub fn new_borrowed(data: &'static [u8]) -> Self {
111        Self::Borrowed(data)
112    }
113
114    /// Create new shared tensor data
115    pub fn new_shared(data: Vec<u8>) -> Self {
116        Self::Shared(Arc::new(data))
117    }
118
119    /// Create empty tensor data
120    pub fn empty() -> Self {
121        Self::Empty
122    }
123
124    /// Create tensor data with specified size filled with zeros
125    pub fn zeros(size: usize) -> Self {
126        Self::Owned(vec![0u8; size])
127    }
128
129    /// Fallibly create tensor data with the specified size filled with zeros.
130    pub fn try_zeros(size: usize) -> Result<Self> {
131        Self::try_filled(size, 0)
132    }
133
134    /// Create tensor data with specified size filled with a value
135    pub fn filled(size: usize, value: u8) -> Self {
136        Self::Owned(vec![value; size])
137    }
138
139    /// Fallibly create tensor data with the specified size and fill value.
140    pub fn try_filled(size: usize, value: u8) -> Result<Self> {
141        let mut data = Vec::new();
142        data.try_reserve_exact(size).map_err(|_| {
143            GGUFError::InvalidTensorData(format!(
144                "Unable to allocate tensor data buffer of {} bytes",
145                size
146            ))
147        })?;
148        data.resize(size, value);
149        Ok(Self::Owned(data))
150    }
151
152    /// Create memory-mapped tensor data
153    #[cfg(feature = "mmap")]
154    pub fn new_mapped(mmap: Arc<memmap2::Mmap>, offset: usize, length: usize) -> Result<Self> {
155        checked_data_range(offset, length, mmap.len(), "Mapped region")?;
156
157        Ok(Self::Mapped { mmap, offset, length })
158    }
159
160    /// Get the length of the tensor data in bytes
161    pub fn len(&self) -> usize {
162        match self {
163            TensorData::Owned(data) => data.len(),
164            TensorData::Borrowed(data) => data.len(),
165            TensorData::Shared(data) => data.len(),
166            #[cfg(feature = "mmap")]
167            TensorData::Mapped { length, .. } => *length,
168            TensorData::Empty => 0,
169        }
170    }
171
172    /// Check if the tensor data is empty
173    pub fn is_empty(&self) -> bool {
174        match self {
175            TensorData::Owned(data) => data.is_empty(),
176            TensorData::Borrowed(data) => data.is_empty(),
177            TensorData::Shared(data) => data.is_empty(),
178            #[cfg(feature = "mmap")]
179            TensorData::Mapped { length, .. } => *length == 0,
180            TensorData::Empty => true,
181        }
182    }
183
184    /// Get a slice of the tensor data.
185    ///
186    /// A `Mapped` value constructed directly with an invalid range yields an
187    /// empty slice. Use [`Self::try_as_slice`] when invalid mapped data must be
188    /// distinguished from valid empty data.
189    pub fn as_slice(&self) -> &[u8] {
190        self.try_as_slice().unwrap_or(&[])
191    }
192
193    /// Get a checked slice of the tensor data.
194    ///
195    /// Unlike [`Self::as_slice`], this reports an invalid directly constructed
196    /// memory-mapped range as [`GGUFError::InvalidTensorData`].
197    pub fn try_as_slice(&self) -> Result<&[u8]> {
198        match self {
199            TensorData::Owned(data) => Ok(data),
200            TensorData::Borrowed(data) => Ok(data),
201            TensorData::Shared(data) => Ok(data),
202            #[cfg(feature = "mmap")]
203            TensorData::Mapped { mmap, offset, length } => {
204                let range = checked_data_range(*offset, *length, mmap.len(), "Mapped region")?;
205                mmap.get(range).ok_or_else(|| {
206                    GGUFError::InvalidTensorData(
207                        "Mapped region could not be read from the memory map".to_string(),
208                    )
209                })
210            }
211            TensorData::Empty => Ok(&[]),
212        }
213    }
214
215    /// Get mutable access to the data (only for owned data)
216    pub fn as_mut_slice(&mut self) -> Option<&mut [u8]> {
217        match self {
218            TensorData::Owned(data) => Some(data),
219            _ => None,
220        }
221    }
222
223    /// Convert to owned data (cloning if necessary)
224    pub fn to_owned(&self) -> Vec<u8> {
225        self.as_slice().to_vec()
226    }
227
228    /// Convert to owned data while reporting invalid mapped ranges and
229    /// allocation failure.
230    pub fn try_to_owned(&self) -> Result<Vec<u8>> {
231        let source = self.try_as_slice()?;
232        let mut owned = Vec::new();
233        owned.try_reserve_exact(source.len()).map_err(|_| {
234            GGUFError::InvalidTensorData(format!(
235                "Unable to allocate owned tensor buffer of {} bytes",
236                source.len()
237            ))
238        })?;
239        owned.extend_from_slice(source);
240        Ok(owned)
241    }
242
243    /// Take ownership of the data if possible, otherwise clone
244    pub fn into_owned(self) -> Vec<u8> {
245        match self {
246            TensorData::Owned(data) => data,
247            TensorData::Borrowed(data) => data.to_vec(),
248            TensorData::Shared(data) => {
249                // Try to unwrap if this is the only reference
250                Arc::try_unwrap(data).unwrap_or_else(|shared| (*shared).clone())
251            }
252            #[cfg(feature = "mmap")]
253            TensorData::Mapped { mmap, offset, length } => {
254                let range = checked_data_range(offset, length, mmap.len(), "Mapped region");
255                range.ok().and_then(|range| mmap.get(range)).unwrap_or(&[]).to_vec()
256            }
257            TensorData::Empty => Vec::new(),
258        }
259    }
260
261    /// Get the storage type
262    pub fn storage_type(&self) -> TensorStorageType {
263        match self {
264            TensorData::Owned(_) => TensorStorageType::Owned,
265            TensorData::Borrowed(_) => TensorStorageType::Borrowed,
266            TensorData::Shared(_) => TensorStorageType::Shared,
267            #[cfg(feature = "mmap")]
268            TensorData::Mapped { .. } => TensorStorageType::Mapped,
269            TensorData::Empty => TensorStorageType::Empty,
270        }
271    }
272
273    /// Get storage information
274    pub fn storage_info(&self) -> TensorDataInfo {
275        TensorDataInfo {
276            size_bytes: self.len(),
277            storage_type: self.storage_type(),
278            is_aligned: self.is_aligned(),
279            alignment: self.alignment(),
280        }
281    }
282
283    /// Check if the data is properly aligned
284    pub fn is_aligned(&self) -> bool {
285        #[cfg(feature = "std")]
286        {
287            self.is_aligned_to(std::mem::align_of::<u64>())
288        }
289        #[cfg(not(feature = "std"))]
290        {
291            self.is_aligned_to(mem::align_of::<u64>())
292        }
293    }
294
295    /// Check if the data is aligned to a specific boundary
296    pub fn is_aligned_to(&self, alignment: usize) -> bool {
297        if alignment <= 1 {
298            return true;
299        }
300
301        let ptr = self.as_slice().as_ptr() as usize;
302        ptr.is_multiple_of(alignment)
303    }
304
305    /// Get the alignment of the data pointer
306    pub fn alignment(&self) -> Option<usize> {
307        let ptr = self.as_slice().as_ptr() as usize;
308
309        if ptr == 0 {
310            return None;
311        }
312
313        // Find the largest power of 2 that divides the address
314        let mut alignment = 1;
315        let mut test_ptr = ptr;
316
317        while test_ptr.is_multiple_of(2) && alignment < 4096 {
318            alignment *= 2;
319            test_ptr /= 2;
320        }
321
322        Some(alignment)
323    }
324
325    /// Create a slice of the tensor data
326    pub fn slice(&self, start: usize, length: usize) -> Result<TensorData> {
327        let range = checked_data_range(start, length, self.len(), "Slice")?;
328
329        match self {
330            TensorData::Owned(data) => {
331                data.get(range).map(|data| TensorData::Owned(data.to_vec())).ok_or_else(|| {
332                    GGUFError::InvalidTensorData(
333                        "Slice could not be read from owned tensor data".to_string(),
334                    )
335                })
336            }
337            TensorData::Borrowed(data) => {
338                data.get(range).map(TensorData::Borrowed).ok_or_else(|| {
339                    GGUFError::InvalidTensorData(
340                        "Slice could not be read from borrowed tensor data".to_string(),
341                    )
342                })
343            }
344            TensorData::Shared(data) => {
345                data.get(range).map(|data| TensorData::Owned(data.to_vec())).ok_or_else(|| {
346                    GGUFError::InvalidTensorData(
347                        "Slice could not be read from shared tensor data".to_string(),
348                    )
349                })
350            }
351            #[cfg(feature = "mmap")]
352            TensorData::Mapped { mmap, offset, .. } => {
353                let mapped_offset = offset.checked_add(start).ok_or_else(|| {
354                    GGUFError::InvalidTensorData(format!(
355                        "Mapped slice offset {offset} plus start {start} overflows usize"
356                    ))
357                })?;
358                TensorData::new_mapped(mmap.clone(), mapped_offset, length)
359            }
360            TensorData::Empty => {
361                if start == 0 && length == 0 {
362                    Ok(TensorData::Empty)
363                } else {
364                    Err(GGUFError::InvalidTensorData("Cannot slice empty tensor data".to_string()))
365                }
366            }
367        }
368    }
369
370    /// Concatenate with another tensor data
371    pub fn concat(&self, other: &TensorData) -> TensorData {
372        if self.is_empty() {
373            return other.clone();
374        }
375        if other.is_empty() {
376            return self.clone();
377        }
378
379        let mut result = self.to_owned();
380        result.extend_from_slice(other.as_slice());
381        TensorData::Owned(result)
382    }
383
384    /// Check if two tensor data instances have the same content
385    pub fn content_equals(&self, other: &TensorData) -> bool {
386        self.as_slice() == other.as_slice()
387    }
388
389    /// Get a hexadecimal representation of the first few bytes (for debugging)
390    pub fn hex_preview(&self, max_bytes: usize) -> String {
391        let data = self.as_slice();
392        let preview_len = max_bytes.min(data.len());
393
394        if preview_len == 0 {
395            return "[]".to_string();
396        }
397
398        let hex: String = data
399            .iter()
400            .take(preview_len)
401            .map(|b| format!("{:02x}", b))
402            .collect::<Vec<_>>()
403            .join(" ");
404
405        if data.len() > max_bytes {
406            format!("[{} ... ({} more bytes)]", hex, data.len() - preview_len)
407        } else {
408            format!("[{}]", hex)
409        }
410    }
411
412    /// Validate the tensor data for basic consistency
413    pub fn validate(&self) -> Result<()> {
414        self.try_as_slice().map(|_| ())
415    }
416
417    /// Calculate a small non-cryptographic checksum for caller-side comparison.
418    ///
419    /// This value is not collision resistant and does not establish integrity
420    /// or authenticity without a separately trusted reference value.
421    pub fn checksum(&self) -> u32 {
422        let data = self.as_slice();
423        let mut checksum = 0u32;
424
425        for (i, &byte) in data.iter().enumerate() {
426            checksum = checksum.wrapping_add((byte as u32) << (i % 24));
427            checksum = checksum.wrapping_mul(0x9e37_79b9); // Mixing constant
428        }
429
430        checksum
431    }
432
433    /// Get memory usage information
434    pub fn memory_usage(&self) -> TensorMemoryUsage {
435        match self {
436            TensorData::Owned(data) => TensorMemoryUsage {
437                allocated_bytes: data.capacity(),
438                used_bytes: data.len(),
439                is_shared: false,
440                is_mapped: false,
441            },
442            TensorData::Borrowed(_) => TensorMemoryUsage {
443                allocated_bytes: 0, // We don't own the memory
444                used_bytes: self.len(),
445                is_shared: false,
446                is_mapped: false,
447            },
448            TensorData::Shared(data) => TensorMemoryUsage {
449                allocated_bytes: data.capacity(),
450                used_bytes: data.len(),
451                is_shared: true,
452                is_mapped: false,
453            },
454            #[cfg(feature = "mmap")]
455            TensorData::Mapped { length, .. } => TensorMemoryUsage {
456                allocated_bytes: 0, // Memory mapped
457                used_bytes: *length,
458                is_shared: true,
459                is_mapped: true,
460            },
461            TensorData::Empty => TensorMemoryUsage {
462                allocated_bytes: 0,
463                used_bytes: 0,
464                is_shared: false,
465                is_mapped: false,
466            },
467        }
468    }
469}
470
471/// Memory usage information for tensor data
472#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
473#[derive(Debug, Clone, PartialEq, Eq)]
474pub struct TensorMemoryUsage {
475    /// Allocated memory in bytes
476    pub allocated_bytes: usize,
477    /// Used memory in bytes
478    pub used_bytes: usize,
479    /// Whether the memory is shared
480    pub is_shared: bool,
481    /// Whether the memory is mapped from a file
482    pub is_mapped: bool,
483}
484
485impl TensorMemoryUsage {
486    /// Calculate the memory efficiency (used / allocated)
487    pub fn efficiency(&self) -> f32 {
488        if self.allocated_bytes == 0 {
489            1.0 // Perfect efficiency for borrowed/mapped data
490        } else {
491            self.used_bytes as f32 / self.allocated_bytes as f32
492        }
493    }
494
495    /// Get overhead in bytes
496    pub fn overhead_bytes(&self) -> usize {
497        self.allocated_bytes.saturating_sub(self.used_bytes)
498    }
499}
500
501impl PartialEq for TensorData {
502    fn eq(&self, other: &Self) -> bool {
503        self.content_equals(other)
504    }
505}
506
507#[cfg(feature = "std")]
508impl std::fmt::Display for TensorData {
509    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
510        write!(
511            f,
512            "TensorData({:?}, {} bytes, {})",
513            self.storage_type(),
514            self.len(),
515            self.hex_preview(8)
516        )
517    }
518}
519
520#[cfg(not(feature = "std"))]
521impl fmt::Display for TensorData {
522    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
523        match self {
524            TensorData::Owned(data) => write!(f, "Owned({} bytes)", data.len()),
525            TensorData::Borrowed(data) => write!(f, "Borrowed({} bytes)", data.len()),
526            TensorData::Shared(data) => write!(f, "Shared({} bytes)", data.len()),
527            TensorData::Empty => write!(f, "Empty"),
528            #[cfg(feature = "mmap")]
529            TensorData::Mapped { mmap, .. } => write!(f, "Mapped({} bytes)", mmap.len()),
530        }
531    }
532}
533
534#[cfg(feature = "std")]
535impl std::fmt::Display for TensorStorageType {
536    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
537        match self {
538            TensorStorageType::Owned => write!(f, "Owned"),
539            TensorStorageType::Borrowed => write!(f, "Borrowed"),
540            TensorStorageType::Shared => write!(f, "Shared"),
541            TensorStorageType::Mapped => write!(f, "Mapped"),
542            TensorStorageType::Empty => write!(f, "Empty"),
543        }
544    }
545}
546
547#[cfg(not(feature = "std"))]
548impl fmt::Display for TensorStorageType {
549    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
550        match self {
551            TensorStorageType::Owned => write!(f, "Owned"),
552            TensorStorageType::Borrowed => write!(f, "Borrowed"),
553            TensorStorageType::Shared => write!(f, "Shared"),
554            TensorStorageType::Mapped => write!(f, "Mapped"),
555            TensorStorageType::Empty => write!(f, "Empty"),
556        }
557    }
558}
559
560#[cfg(feature = "std")]
561impl std::fmt::Display for TensorDataInfo {
562    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
563        write!(
564            f,
565            "TensorDataInfo {{ type: {}, size: {} bytes, aligned: {} }}",
566            self.storage_type, self.size_bytes, self.is_aligned
567        )
568    }
569}
570
571#[cfg(not(feature = "std"))]
572impl fmt::Display for TensorDataInfo {
573    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
574        write!(
575            f,
576            "TensorDataInfo {{ storage: {}, size: {} bytes, alignment: {:?} }}",
577            self.storage_type, self.size_bytes, self.alignment
578        )
579    }
580}
581
582#[cfg(all(test, feature = "std"))]
583mod tests {
584    use super::*;
585
586    #[test]
587    fn test_tensor_data_owned() {
588        let data = vec![1, 2, 3, 4, 5];
589        let tensor_data = TensorData::new_owned(data.clone());
590
591        assert_eq!(tensor_data.len(), 5);
592        assert!(!tensor_data.is_empty());
593        assert_eq!(tensor_data.as_slice(), &data);
594        assert_eq!(tensor_data.storage_type(), TensorStorageType::Owned);
595    }
596
597    #[test]
598    fn test_tensor_data_empty() {
599        let tensor_data = TensorData::empty();
600
601        assert_eq!(tensor_data.len(), 0);
602        assert!(tensor_data.is_empty());
603        assert_eq!(tensor_data.as_slice(), &[] as &[u8]);
604        assert_eq!(tensor_data.storage_type(), TensorStorageType::Empty);
605    }
606
607    #[test]
608    fn test_tensor_data_zeros() {
609        let tensor_data = TensorData::zeros(10);
610
611        assert_eq!(tensor_data.len(), 10);
612        assert_eq!(tensor_data.as_slice(), &vec![0u8; 10]);
613    }
614
615    #[test]
616    fn test_tensor_data_filled() {
617        let tensor_data = TensorData::filled(5, 42);
618
619        assert_eq!(tensor_data.len(), 5);
620        assert_eq!(tensor_data.as_slice(), &vec![42u8; 5]);
621    }
622
623    #[test]
624    fn test_fallible_tensor_data_allocation() {
625        assert_eq!(TensorData::try_zeros(4).unwrap().as_slice(), &[0; 4]);
626        assert_eq!(TensorData::try_filled(3, 7).unwrap().as_slice(), &[7; 3]);
627        assert!(TensorData::try_zeros(usize::MAX).is_err());
628    }
629
630    #[test]
631    fn test_tensor_data_shared() {
632        let data = vec![1, 2, 3, 4];
633        let tensor_data = TensorData::new_shared(data.clone());
634
635        assert_eq!(tensor_data.len(), 4);
636        assert_eq!(tensor_data.as_slice(), &data);
637        assert_eq!(tensor_data.storage_type(), TensorStorageType::Shared);
638    }
639
640    #[test]
641    fn test_tensor_data_to_owned() {
642        let data = vec![1, 2, 3, 4];
643        let tensor_data = TensorData::new_shared(data.clone());
644        let owned = tensor_data.to_owned();
645
646        assert_eq!(owned, data);
647    }
648
649    #[test]
650    fn test_tensor_data_into_owned() {
651        let data = vec![1, 2, 3, 4];
652        let tensor_data = TensorData::new_owned(data.clone());
653        let owned = tensor_data.into_owned();
654
655        assert_eq!(owned, data);
656    }
657
658    #[test]
659    fn test_tensor_data_slice() {
660        let data = vec![1, 2, 3, 4, 5];
661        let tensor_data = TensorData::new_owned(data);
662
663        let slice = tensor_data.slice(1, 3).unwrap();
664        assert_eq!(slice.as_slice(), &[2, 3, 4]);
665
666        // Test bounds checking
667        assert!(tensor_data.slice(3, 5).is_err());
668    }
669
670    #[test]
671    fn test_tensor_data_slice_rejects_overflow_and_out_of_range() {
672        let tensor_data = TensorData::new_owned(vec![1, 2, 3]);
673
674        assert!(matches!(tensor_data.slice(1, usize::MAX), Err(GGUFError::InvalidTensorData(_))));
675        assert!(matches!(tensor_data.slice(usize::MAX, 0), Err(GGUFError::InvalidTensorData(_))));
676        assert_eq!(tensor_data.as_slice(), &[1, 2, 3]);
677    }
678
679    #[cfg(feature = "mmap")]
680    #[test]
681    fn test_mapped_tensor_data_rejects_invalid_ranges_without_panicking() {
682        use std::io::Write;
683
684        let mut temp_file = tempfile::NamedTempFile::new().unwrap();
685        temp_file.write_all(&[1, 2, 3, 4]).unwrap();
686        temp_file.flush().unwrap();
687        let mmap = Arc::new(unsafe { memmap2::Mmap::map(temp_file.as_file()).unwrap() });
688
689        assert!(matches!(
690            TensorData::new_mapped(mmap.clone(), 1, usize::MAX),
691            Err(GGUFError::InvalidTensorData(_))
692        ));
693        assert!(matches!(
694            TensorData::new_mapped(mmap.clone(), usize::MAX, 0),
695            Err(GGUFError::InvalidTensorData(_))
696        ));
697
698        let invalid = TensorData::Mapped { mmap, offset: usize::MAX, length: 1 };
699        assert!(matches!(invalid.try_as_slice(), Err(GGUFError::InvalidTensorData(_))));
700        assert!(matches!(invalid.try_to_owned(), Err(GGUFError::InvalidTensorData(_))));
701        assert!(invalid.as_slice().is_empty());
702        assert!(matches!(invalid.validate(), Err(GGUFError::InvalidTensorData(_))));
703        assert!(matches!(invalid.slice(0, 1), Err(GGUFError::InvalidTensorData(_))));
704        assert!(invalid.into_owned().is_empty());
705    }
706
707    #[test]
708    fn test_tensor_data_concat() {
709        let data1 = TensorData::new_owned(vec![1, 2, 3]);
710        let data2 = TensorData::new_owned(vec![4, 5, 6]);
711
712        let concatenated = data1.concat(&data2);
713        assert_eq!(concatenated.as_slice(), &[1, 2, 3, 4, 5, 6]);
714    }
715
716    #[test]
717    fn test_tensor_data_content_equals() {
718        let data1 = TensorData::new_owned(vec![1, 2, 3]);
719        let data2 = TensorData::new_shared(vec![1, 2, 3]);
720        let data3 = TensorData::new_owned(vec![1, 2, 4]);
721
722        assert!(data1.content_equals(&data2));
723        assert!(!data1.content_equals(&data3));
724    }
725
726    #[test]
727    fn test_tensor_data_hex_preview() {
728        let data = TensorData::new_owned(vec![0xde, 0xad, 0xbe, 0xef]);
729        let preview = data.hex_preview(4);
730        assert!(preview.contains("de"));
731        assert!(preview.contains("ad"));
732        assert!(preview.contains("be"));
733        assert!(preview.contains("ef"));
734    }
735
736    #[test]
737    fn test_tensor_data_checksum() {
738        let data1 = TensorData::new_owned(vec![1, 2, 3, 4]);
739        let data2 = TensorData::new_owned(vec![1, 2, 3, 4]);
740        let data3 = TensorData::new_owned(vec![1, 2, 3, 5]);
741
742        assert_eq!(data1.checksum(), data2.checksum());
743        assert_ne!(data1.checksum(), data3.checksum());
744    }
745
746    #[test]
747    fn test_tensor_data_validation() {
748        let valid_data = TensorData::new_owned(vec![1, 2, 3]);
749        assert!(valid_data.validate().is_ok());
750
751        let empty_data = TensorData::empty();
752        assert!(empty_data.validate().is_ok());
753    }
754
755    #[test]
756    fn test_tensor_data_memory_usage() {
757        let data = TensorData::new_owned(vec![1, 2, 3, 4, 5]);
758        let usage = data.memory_usage();
759
760        assert!(usage.used_bytes >= 5);
761        assert!(usage.allocated_bytes >= usage.used_bytes);
762        assert!(!usage.is_shared);
763        assert!(!usage.is_mapped);
764        assert!(usage.efficiency() > 0.0);
765    }
766
767    #[test]
768    fn test_tensor_storage_info() {
769        let data = TensorData::new_owned(vec![1, 2, 3, 4]);
770        let info = data.storage_info();
771
772        assert_eq!(info.size_bytes, 4);
773        assert_eq!(info.storage_type, TensorStorageType::Owned);
774    }
775
776    #[test]
777    fn test_memory_usage_efficiency() {
778        let usage = TensorMemoryUsage {
779            allocated_bytes: 100,
780            used_bytes: 80,
781            is_shared: false,
782            is_mapped: false,
783        };
784
785        assert_eq!(usage.efficiency(), 0.8);
786        assert_eq!(usage.overhead_bytes(), 20);
787    }
788
789    #[test]
790    fn test_tensor_data_display() {
791        let data = TensorData::new_owned(vec![1, 2, 3]);
792        let display_str = format!("{}", data);
793
794        assert!(display_str.contains("TensorData"));
795        assert!(display_str.contains("Owned"));
796        assert!(display_str.contains("3 bytes"));
797    }
798
799    #[test]
800    fn test_tensor_data_equality() {
801        let data1 = TensorData::new_owned(vec![1, 2, 3]);
802        let data2 = TensorData::new_shared(vec![1, 2, 3]);
803        let data3 = TensorData::new_owned(vec![1, 2, 4]);
804
805        assert_eq!(data1, data2); // Content equality
806        assert_ne!(data1, data3);
807    }
808
809    #[test]
810    fn test_tensor_data_mutable_access() {
811        let mut data = TensorData::new_owned(vec![1, 2, 3]);
812
813        if let Some(slice) = data.as_mut_slice() {
814            slice[0] = 42;
815        }
816
817        assert_eq!(data.as_slice()[0], 42);
818
819        // Shared data should not provide mutable access
820        let mut shared_data = TensorData::new_shared(vec![1, 2, 3]);
821        assert!(shared_data.as_mut_slice().is_none());
822    }
823}
gguf_rs_lib/tensor/data.rs

gguf_rs_lib/tensor/
data.rs
