lamellar 0.3.1

Lamellar is an asynchronous tasking runtime for HPC systems developed in RUST.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214

extern crate libc;

use std::ffi::CString;

pub(crate) fn rofi_init(provider: &str) -> Result<(), &'static str> {
    let c_str = CString::new(provider).unwrap();
    let retval = unsafe { rofisys::rofi_init(c_str.as_ptr() as *mut _) as i32 };
    if retval == 0 {
        Ok(())
    } else {
        Err("unable to initialize rofi")
    }
}

pub(crate) fn rofi_finit() -> Result<(), &'static str> {
    let retval = unsafe { rofisys::rofi_finit() as i32 };
    if retval == 0 {
        Ok(())
    } else {
        Err("unable to finit rofi")
    }
}

pub(crate) fn rofi_get_size() -> usize {
    unsafe { rofisys::rofi_get_size() as usize }
}

pub(crate) fn rofi_get_id() -> usize {
    unsafe { rofisys::rofi_get_id() as usize }
}

pub(crate) fn rofi_barrier() {
    unsafe { rofisys::rofi_barrier() };
}

pub(crate) fn rofi_alloc(size: usize) -> *mut u8 {
    let mut base_ptr: *mut u8 = std::ptr::null_mut();
    let base_ptr_ptr = (&mut base_ptr as *mut _) as *mut *mut std::ffi::c_void;
    // let mut key = 0u64;
    // let key_ptr = &mut key as *mut c_ulonglong;
    // println!("rofi_alloc");
    unsafe {
        if rofisys::rofi_alloc(size, 0x0, base_ptr_ptr) != 0 {
            panic!("unable to allocate memory region");
        }
    }
    //println!("[{:?}] ({:?}:{:?}) rofi_alloc addr: {:x} size {:?}",rofi_get_id(),file!(),line!(),base_ptr as usize, size);

    // let _rofi_slice = unsafe { std::slice::from_raw_parts(base_ptr as *const u8, size) };
    // rofi_barrier();
    base_ptr
}

#[allow(dead_code)]
pub(crate) fn rofi_release(addr: usize) {
    let base_ptr = addr as *mut u8;
    // println!("rofi_release");
    unsafe {
        if rofisys::rofi_release(base_ptr as *mut std::ffi::c_void) != 0 {
            panic!("unable to release memory region");
        }
    }
    //println!("[{:?}] ({:?}:{:?}) rofi_release addr: {:x} ",rofi_get_id(),file!(),line!(),base_ptr as usize);
}

pub(crate) fn rofi_local_addr(remote_pe: usize, remote_addr: usize) -> usize {
    let addr = unsafe {
        // println!("{:x} {:?} {:?} {:?}",remote_addr,(remote_addr as *mut u8) as *mut std::ffi::c_void,remote_pe,remote_pe as u32);
        rofisys::rofi_get_local_addr_from_remote_addr(
            (remote_addr as *mut u8) as *mut std::ffi::c_void,
            remote_pe as u32,
        ) as usize
    };
    if addr == 0 {
        panic!("unable to locate local memory addr");
    }
    addr
}

// pub(crate) fn rofi_start_addr(id: u64) -> usize {
//     let mut addr_ptr: *mut u8 = std::ptr::null_mut();
//     let addr_ptr_ptr = (&mut addr_ptr as *mut _) as *mut *mut std::ffi::c_void;
//     unsafe {
//         if rofisys::rofi_start_addr(id as c_ulonglong, addr_ptr_ptr) != 0 {
//             panic!("unable to locate remote memory addr");
//         }
//     }
//     addr_ptr as usize
// }

pub(crate) fn rofi_remote_addr(pe: usize, local_addr: usize) -> usize {
    let addr = unsafe {
        rofisys::rofi_get_remote_addr((local_addr as *mut u8) as *mut std::ffi::c_void, pe as u32)
            as usize
    };
    if addr == 0 {
        panic!("unable to locate local memory addr");
    }
    addr
}

// data is a reference, user must ensure lifetime is valid until underlying put is complete, thus is unsafe
pub(crate) unsafe fn rofi_put<T>(src: &[T], dst: usize, pe: usize) -> Result<(), i32> {
    let src_addr = src.as_ptr() as *mut std::ffi::c_void;
    let size = src.len() * std::mem::size_of::<T>();
    let mut ret = rofisys::rofi_put(dst as *mut std::ffi::c_void, src_addr, size, pe as u32, 0);
    //FI_EAGAIN should this be handled here, at c-rofi layer, or application layer?
    while ret == -11 {
        std::thread::yield_now();
        ret = rofisys::rofi_put(dst as *mut std::ffi::c_void, src_addr, size, pe as u32, 0);
        //println!("[{:?}] ({:?}:{:?}) rofi_put src_addr {:?} dst_addr 0x{:x} pe {:?} {:?}",rofi_get_id(),file!(),line!(),src_addr,dst,pe,ret);
    }
    if ret == 0 {
        Ok(())
    } else {
        Err(ret)
    }
}

#[allow(dead_code)]
pub(crate) fn rofi_iput<T>(src: &[T], dst: usize, pe: usize) -> Result<(), i32> {
    let src_addr = src.as_ptr() as *mut std::ffi::c_void;
    let size = src.len() * std::mem::size_of::<T>();
    let mut ret = unsafe {
        rofisys::rofi_iput(dst as *mut std::ffi::c_void, src_addr, size, pe as u32, 0)
    };
    //FI_EAGAIN should this be handled here, at c-rofi layer, or application layer?
    // println!("putting {:?} to {:?} @ {:x}",src,pe,dst);
    while ret == -11 {
        std::thread::yield_now();
        ret = unsafe {
            rofisys::rofi_iput(dst as *mut std::ffi::c_void, src_addr, size, pe as u32, 0)
        };
        //println!("[{:?}] ({:?}:{:?}) rofi_iput src_addr {:?} dst_addr 0x{:x} pe {:?} {:?}",rofi_get_id(),file!(),line!(),src_addr,dst,pe,ret);
    }
    if ret == 0 {
        Ok(())
    } else {
        Err(ret)
    }
}
#[allow(dead_code)]
pub(crate) unsafe fn rofi_get<T>(src: usize, dst: &mut [T], pe: usize) -> Result<(), i32> {
    let src_addr = src as *mut std::ffi::c_void;
    let dst_addr = dst.as_ptr() as *mut std::ffi::c_void;
    let size = dst.len() * std::mem::size_of::<T>();
    let mut ret = rofisys::rofi_get(dst_addr, src_addr, size, pe as u32, 0);
    while ret == -11 {
        std::thread::yield_now();
        ret = rofisys::rofi_get(dst_addr, src_addr, size, pe as u32, 0);
        //println!("[{:?}] ({:?}:{:?}) rofi_get src_addr {:?} dst_addr{:?} pe {:?} {:?}",rofi_get_id(),file!(),line!(),src_addr, dst_addr,pe, ret);
    }
    if ret == 0 {
        Ok(())
    } else {
        Err(ret)
    }
}

#[allow(dead_code)]
pub(crate) fn rofi_iget<T>(src: usize, dst: &mut [T], pe: usize) -> Result<(), i32> {
    let src_addr = src as *mut std::ffi::c_void;
    let dst_addr = dst.as_ptr() as *mut std::ffi::c_void;
    let size = dst.len() * std::mem::size_of::<T>();
    //println!("[{:?}] ({:?}{:?}) rofi_iget src: {:x} dst: {:?} pe: {:?} size: {:?}",rofi_get_id(),file!(),line!(),src, dst.as_ptr(),pe, size);
    let mut ret = unsafe { rofisys::rofi_iget(dst_addr, src_addr, size, pe as u32, 0) };
    while ret == -11 {
        std::thread::yield_now();
        ret = unsafe { rofisys::rofi_iget(dst_addr, src_addr, size, pe as u32, 0) };
        // println!("[{:?}] ({:?}:{:?}) rofi_get src_addr {:?} dst_addr{:?} pe {:?} {:?}",rofi_get_id(),file!(),line!(),src_addr, dst_addr,pe, ret);
    }
    if ret == 0 {
        Ok(())
    } else {
        Err(ret)
    }
}

#[allow(dead_code)]
pub(crate) fn rofi_isend<T: std::fmt::Debug>(src: &[T], pe: usize) -> Result<(), i32> {
    // println!("sending {:?}", src);
    let src_addr = src.as_ptr() as *mut std::ffi::c_void;
    let size = src.len() * std::mem::size_of::<T>();
    // println!("sending {:?} {:?}", src, size);
    let mut ret = unsafe { rofisys::rofi_isend(pe as u64, src_addr, size, 0) };
    while ret == -11 {
        std::thread::yield_now();
        ret = unsafe { rofisys::rofi_isend(pe as u64, src_addr, size, 0) };
        // println!("src_addr {:?} {:?}",src.as_ptr(),ret);
    }
    if ret == 0 {
        Ok(())
    } else {
        Err(ret)
    }
}

#[allow(dead_code)]
pub(crate) fn rofi_irecv<T>(dst: &mut [T], pe: usize) -> Result<(), i32> {
    let dst_addr = dst.as_ptr() as *mut std::ffi::c_void;
    let size = dst.len() * std::mem::size_of::<T>();
    // println!("waiting to receive {:?} from {:?}", size, pe);
    let mut ret = unsafe { rofisys::rofi_irecv(pe as u64, dst_addr, size, 0) };
    while ret == -11 {
        std::thread::yield_now();
        ret = unsafe { rofisys::rofi_irecv(pe as u64, dst_addr, size, 0) };
        // println!("recv ret {:?}", ret);
    }
    if ret == 0 {
        Ok(())
    } else {
        Err(ret)
    }
}