rofisys 0.5.0

This system crate provides Rust language bindings (via the use of Bindgen) to the Rust-OFI library.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

#ifndef ROFI_INTERNAL_H
#define ROFI_INTERNAL_H

#include <pthread.h>
#include <stdatomic.h>

#include <pthread.h>
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_endpoint.h>

#include <uthash.h>

#ifdef __OFI_PROV_CXI__
    #include <stdbool.h>
    #include <rdma/fi_cxi_ext.h>
#endif

#ifndef ROFI_FI_VERSION
#define ROFI_FI_VERSION FI_VERSION(1, 15)
#endif

typedef struct rofi_transport_t rofi_transport_t;

#include "context.h"
#include "mr.h"
#include "rt.h"
#include "rofi.h"

#define ROFI_STATUS_NONE 0
#define ROFI_STATUS_START 1
#define ROFI_STATUS_ACTIVE 2
#define ROFI_STATUS_TERM 3
#define ROFI_STATUS_END 4
#define ROFI_STATUS_ERR 5

#define ROFI_SERVER 1
#define ROFI_CLIENT 2

#define MAX_HOSTNAME_LEN 256
#define KILO 1024UL
#define MEGA 1024UL * KILO
#define GIGA 1024UL * MEGA

#define ROFI_HEAP_NOTALLOCATED 0
#define ROFI_HEAP_ALLOCATED 1
#define ROFI_HEAP_REGISTERED 2
#define ROFI_HEAP_INVALID 3

#define ROFI_ASYNC 0x1
#define ROFI_SYNC 0x2

typedef struct {
    unsigned long status;
    unsigned int nodes;
    unsigned int nid;
    int addrlen;
    unsigned long PageSize;
    uint64_t max_message_size;
    uint64_t inject_size;
} rofi_desc_t;

typedef struct rofi_prov_names_t {
    char **names;
    int num;
} rofi_names_t;

struct rofi_transport_t {
    struct fi_info *info;
    struct fid_fabric *fabric;
    struct fid_domain *domain;
    struct fid_av *av;
    struct fid_ep *ep;
    struct fid_eq *eq;
    struct fid_cntr *put_cntr;
    struct fid_cntr *get_cntr;
    struct fid_cntr *send_cntr;
    struct fid_cntr *recv_cntr;
    struct fid_cq *cq;
    fi_addr_t *remote_addrs;
    _Atomic uint64_t pending_put_cntr;
    _Atomic uint64_t pending_get_cntr;
    _Atomic uint64_t pending_send_cntr;
    _Atomic uint64_t pending_recv_cntr;
    uint64_t error_cnt;
    rofi_desc_t desc;
    rofi_mr_desc *mr;
    uint64_t global_barrier_id;
    uint64_t *global_barrier_buf;
    uint64_t *sub_alloc_barrier_buf;
    struct fi_rma_iov *sub_alloc_buf;
    pthread_mutex_t lock;
    pthread_rwlock_t mr_lock;
    uint64_t fi_collective;
};

extern rofi_transport_t rofi;

int rofi_init_internal(char *, char *);
int rofi_finit_internal(void);
unsigned int rofi_get_size_internal(void);
unsigned int rofi_get_id_internal(void);
int rofi_flush_internal(void);
void rofi_barrier_internal(void);
int rofi_put_internal(void *, void *, size_t, unsigned int, unsigned long);
int rofi_get_internal(void *, void *, size_t, unsigned int, unsigned long);
int rofi_send_internal(unsigned int, void *, size_t, unsigned long);
int rofi_recv_internal(void *, size_t, unsigned long);
void *rofi_alloc_internal(size_t, unsigned long);
void *rofi_sub_alloc_internal(size_t, unsigned long, uint64_t *, uint64_t);
int rofi_release_internal(void *);
int rofi_sub_release_internal(void *, uint64_t *, uint64_t);
int rofi_wait_internal(void);
void *rofi_get_remote_addr_internal(void *, unsigned int);
void *rofi_get_local_addr_from_remote_addr_internal(void *, unsigned int);
int rofi_has_atomics_internal(void);
int rofi_query_atomic_internal(rofi_datatype_t, rofi_atomic_op_t);
int rofi_query_fetch_atomic_internal(rofi_datatype_t, rofi_atomic_op_t);
int rofi_query_compare_atomic_internal(rofi_datatype_t, rofi_atomic_op_t);
ssize_t rofi_atomic_op_internal(void *, const void *, size_t, rofi_datatype_t, rofi_atomic_op_t, unsigned int);
ssize_t rofi_atomic_fetch_internal(void *, const void *, void *, size_t, rofi_datatype_t, rofi_atomic_op_t, unsigned int);
ssize_t rofi_compare_atomic_internal(void *, const void *, const void *, void *, size_t, rofi_datatype_t, rofi_atomic_op_t, unsigned int);

// message-based barrier used once by CXI at end of rofi_init_internal
// Note currently not exposed in the rofi API (i.e., in rofi.h or api.c)
#ifdef __OFI_PROV_CXI__
void rofi_barrier_p2p_internal(void);
#endif


#endif