1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
/**
* Copyright (C) Mellanox Technologies Ltd. 2020. ALL RIGHTS RESERVED.
*
* See file LICENSE for terms.
*/
#ifndef UCP_RKEY_H_
#define UCP_RKEY_H_
#include "ucp_types.h"
#include <ucp/core/ucp_context.h>
#include <ucp/proto/proto_select.h>
/**
* Rkey proto index
*/
enum {
UCP_RKEY_BASIC_PROTO,
UCP_RKEY_SW_PROTO
};
typedef uint8_t ucp_rkey_proto_index_t;
/**
* UCT remote key along with component handle which should be used to release it.
*
*/
typedef struct ucp_tl_rkey {
uct_rkey_bundle_t rkey;
uct_component_h cmpt;
} ucp_tl_rkey_t;
/**
* Rkey flags
*/
enum {
UCP_RKEY_DESC_FLAG_POOL = UCS_BIT(0) /* Descriptor was allocated from pool
and must be retuned to pool, not free */
};
/**
* Rkey configuration key
*/
struct ucp_rkey_config_key {
/* Which *remote* MDs have valid memory handles */
ucp_md_map_t md_map;
/* Endpoint configuration index */
ucp_worker_cfg_index_t ep_cfg_index;
/* Remove system device id */
ucs_sys_device_t sys_dev;
/* Remote memory type */
ucs_memory_type_t mem_type;
};
/**
* Rkey configuration
*/
typedef struct {
/* Configuration key */
ucp_rkey_config_key_t key;
/* Put-short thresholds */
ucp_proto_select_short_t put_short;
/* Remote system topology distance of each lane from the remote memory
* buffer. The number of valid entries is according to the number of lanes
* defined by the configuration at index "key.ep_cfg_index".
*/
ucs_sys_dev_distance_t lanes_distance[UCP_MAX_LANES];
/* Protocol selection data */
ucp_proto_select_t proto_select;
} ucp_rkey_config_t;
/**
* Remote memory key structure.
* Contains remote keys for UCT MDs.
* md_map specifies which MDs from the current context are present in the array.
* The array itself contains only the MDs specified in md_map, without gaps.
*/
typedef struct ucp_rkey {
union {
/* Cached values for the most recent endpoint configuration */
struct {
uint8_t flags; /* Rkey flags */
uint8_t mem_type; /* Memory type of remote key memory */
int8_t max_put_short; /* Cached value of max_put_short */
ucp_worker_cfg_index_t ep_cfg_index; /* EP configuration relevant for the cache */
ucp_lane_index_t rma_lane; /* Lane to use for RMAs */
ucp_lane_index_t amo_lane; /* Lane to use for AMOs */
ucp_rkey_proto_index_t amo_proto_index; /* Protocol for AMOs */
ucp_rkey_proto_index_t rma_proto_index; /* Protocol for RMAs */
uct_rkey_t rma_rkey; /* Key to use for RMAs */
uct_rkey_t amo_rkey; /* Key to use for AMOs */
} cache;
struct {
uint8_t flags; /* Rkey flags */
uint8_t mem_type; /* Memory type of remote key memory */
ucp_worker_cfg_index_t cfg_index; /* Rkey configuration index */
};
};
#if ENABLE_PARAMS_CHECK
ucp_ep_h ep;
#endif
ucp_md_map_t md_map; /* Which *remote* MDs have valid memory handles */
ucp_tl_rkey_t tl_rkey[0]; /* UCT rkey for every remote MD */
} ucp_rkey_t;
#define UCP_RKEY_AMO_PROTO(_amo_proto_index) ucp_amo_proto_list[_amo_proto_index]
#define UCP_RKEY_RMA_PROTO(_rma_proto_index) ucp_rma_proto_list[_rma_proto_index]
#define UCP_RKEY_RESOLVE_NOCHECK(_rkey, _ep, _op_type) \
({ \
ucs_status_t _status_nc = UCS_OK; \
if (ucs_unlikely((_ep)->cfg_index != (_rkey)->cache.ep_cfg_index)) { \
ucp_rkey_resolve_inner(_rkey, _ep); \
} \
if (ucs_unlikely((_rkey)->cache._op_type##_lane == UCP_NULL_LANE)) { \
ucs_error("remote memory is unreachable " \
"(remote md_map 0x%" PRIx64")", \
(_rkey)->md_map); \
_status_nc = UCS_ERR_UNREACHABLE; \
} \
_status_nc; \
})
#if ENABLE_PARAMS_CHECK
#define UCP_RKEY_RESOLVE(_rkey, _ep, _op_type) \
({ \
ucs_status_t _status; \
if ((_rkey)->ep != (_ep)) { \
ucs_error("cannot use a remote key on a different endpoint than it was unpacked on"); \
_status = UCS_ERR_INVALID_PARAM; \
} else { \
_status = UCP_RKEY_RESOLVE_NOCHECK(_rkey, _ep, _op_type); \
} \
_status; \
})
#else
#define UCP_RKEY_RESOLVE UCP_RKEY_RESOLVE_NOCHECK
#endif
void ucp_rkey_resolve_inner(ucp_rkey_h rkey, ucp_ep_h ep);
ucp_lane_index_t ucp_rkey_find_rma_lane(ucp_context_h context,
const ucp_ep_config_t *config,
ucs_memory_type_t mem_type,
const ucp_lane_index_t *lanes,
ucp_rkey_h rkey,
ucp_lane_map_t ignore,
uct_rkey_t *uct_rkey_p);
size_t ucp_rkey_packed_size(ucp_context_h context, ucp_md_map_t md_map,
ucs_sys_device_t sys_dev,
ucp_sys_dev_map_t sys_dev_map);
void ucp_rkey_packed_copy(ucp_context_h context, ucp_md_map_t md_map,
ucs_memory_type_t mem_type, void *buffer,
const void *uct_rkeys[]);
ssize_t
ucp_rkey_pack_uct(ucp_context_h context, ucp_md_map_t md_map,
const uct_mem_h *memh, const ucp_memory_info_t *mem_info,
ucp_sys_dev_map_t sys_dev_map,
const ucs_sys_dev_distance_t *sys_distance, void *buffer);
ucs_status_t ucp_ep_rkey_unpack_internal(ucp_ep_h ep, const void *buffer,
size_t length, ucp_rkey_h *rkey_p);
void ucp_rkey_dump_packed(const void *buffer, size_t length,
ucs_string_buffer_t *strb);
void ucp_rkey_config_dump_brief(const ucp_rkey_config_key_t *rkey_config_key,
ucs_string_buffer_t *strb);
void ucp_rkey_proto_select_dump(ucp_worker_h worker,
ucp_worker_cfg_index_t rkey_cfg_index,
ucs_string_buffer_t *strb);
#endif