1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

use std::fs::File;
use std::io;
use std::io::Seek;

use byteorder::{LittleEndian, ReadBytesExt};
use libc;
use libc::{MAP_ANON, MAP_HUGETLB, MAP_POPULATE, MAP_SHARED};
use mmap;

/// Represents a consecutive region of physical memory pinned in memory.
pub struct DevMem {
    mapping: mmap::MemoryMap,
}

const MAP_HUGE_SHIFT: usize = 26;
const MAP_HUGE_2MB: i32 = 21 << MAP_HUGE_SHIFT;
const MAP_HUGE_1GB: i32 = 30 << MAP_HUGE_SHIFT;

pub const FOUR_KIB: usize = 4 * 1024;
pub const TWO_MIB: usize = 2 * 1024 * 1024;
pub const ONE_GIB: usize = 1024 * 1024 * 1024;
const PAGESIZE: u64 = FOUR_KIB as u64;

/// Function to read the pagemap in Linux.
/// See also https://www.kernel.org/doc/Documentation/vm/pagemap.txt.
fn read_pagemap(virtual_page: u64) -> io::Result<u64> {
    assert!(virtual_page % PAGESIZE == 0);

    let mut f = File::open("/proc/self/pagemap")?;

    // The pagemap contains one 64-bit value for each virtual page:
    const PAGEMAP_ENTRY_SIZE: u64 = 8;
    let start = (virtual_page / PAGESIZE) * PAGEMAP_ENTRY_SIZE;
    f.seek(io::SeekFrom::Start(start))?;
    let value = f.read_u64::<LittleEndian>()?;

    // Sanity check that the page is not swapped:
    let present_bit = 1 << 63;
    assert!(value & present_bit > 0);

    // Get the physical address by multiplying the PFN bits with the page size
    let pfn_mask: u64 = (1 << 55) - 1;
    Ok((value & pfn_mask) * PAGESIZE)
}

#[derive(Debug)]
pub enum AllocError {
    Map,
    Pin,
}

impl From<mmap::MapError> for AllocError {
    fn from(_e: mmap::MapError) -> Self {
        AllocError::Map
    }
}

impl DevMem {
    /// Allocates a chunk of consecutive physical, pinned memory.
    /// This should be usable by devices that do DMA.
    pub fn alloc(size: usize) -> Result<DevMem, AllocError> {
        assert!(size == FOUR_KIB || size == TWO_MIB || size == ONE_GIB);

        let mut non_standard_flags = MAP_SHARED | MAP_ANON | MAP_POPULATE;
        match size {
            TWO_MIB => non_standard_flags |= MAP_HUGETLB | MAP_HUGE_2MB,
            ONE_GIB => non_standard_flags |= MAP_HUGETLB | MAP_HUGE_1GB,
            _ => (),
        }

        let flags = [
            mmap::MapOption::MapNonStandardFlags(non_standard_flags),
            mmap::MapOption::MapReadable,
            mmap::MapOption::MapWritable,
        ];
        let res = mmap::MemoryMap::new(size, &flags)?;

        // Make sure memory is not swapped:
        let lock_ret = unsafe { libc::mlock(res.data() as *const libc::c_void, res.len()) };
        if lock_ret == -1 {
            return Err(AllocError::Pin);
        }
        assert!(lock_ret == 0);

        Ok(DevMem { mapping: res })
    }

    /// Returns the physical address of the memory region.
    pub fn physical_address(&self) -> u64 {
        read_pagemap(self.virtual_address() as u64).unwrap()
    }

    /// Returns the virtual address of the memory region.
    pub fn virtual_address(&self) -> usize {
        self.as_mut_ptr() as usize
    }

    /// Returns a pointer to the memory region.
    pub fn as_mut_ptr(&self) -> *mut u8 {
        self.mapping.data()
    }

    pub fn as_slice(&self) -> &[u8] {
        unsafe { core::slice::from_raw_parts(self.mapping.data(), self.len()) }
    }

    /// Returns the size of the memory region.
    pub fn len(&self) -> usize {
        self.mapping.len()
    }
}

#[cfg(test)]
mod tests {
    use crate::mem::*;

    #[test]
    fn alloc_1page() {
        let res = DevMem::alloc(FOUR_KIB);
        match res {
            Err(e) => {
                panic!("Can not allocate: {:?}", e);
            }
            Ok(_f) => (),
        }
    }

    #[test]
    fn alloc_2mib() {
        let res = DevMem::alloc(TWO_MIB);
        match res {
            Err(e) => {
                panic!("Can not allocate: {:?}", e);
            }
            Ok(r) => assert!(r.physical_address() % TWO_MIB as u64 == 0),
        }
    }

    #[test]
    #[ignore]
    fn alloc_1gib() {
        let res = DevMem::alloc(ONE_GIB);
        match res {
            Err(e) => {
                panic!("Can not allocate: {:?}", e);
            }
            Ok(r) => assert!((r.physical_address() % ONE_GIB as u64) == 0),
        }
    }
}