1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
|
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2001-2002 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _SYS_RSM_RSMNDI_H
#define _SYS_RSM_RSMNDI_H
#pragma ident "%Z%%M% %I% %E% SMI"
#ifdef __cplusplus
extern "C" {
#endif
#include <sys/types.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <thread.h>
#include <synch.h>
#include <sys/rsm/rsm_common.h>
#include <sys/rsm/rsmapi_common.h>
/*
* This structure defines the functions implemented in rsmlib
* that the NDI library can call.
*/
typedef struct {
int version;
#define RSM_LIB_FUNCS_VERSION 1
int (* rsm_get_hwaddr)(
rsmapi_controller_handle_t handle,
rsm_node_id_t nodeid,
rsm_addr_t *hwaddrp);
int (* rsm_get_nodeid)(
rsmapi_controller_handle_t handle,
rsm_addr_t hwaddr,
rsm_node_id_t *nodeidp);
} rsm_lib_funcs_t;
/* Library attributes - set by specific NDI libraries */
typedef struct {
boolean_t rsm_putget_map_reqd; /* put/get require mapping */
boolean_t rsm_scatgath_map_reqd; /* putv/getv require mapping */
} rsm_ndlib_attr_t;
/* The opaque barrier handle used by the RSMNDI plugin for the barrier calls */
typedef struct rsm_barrier *rsm_barrier_handle_t;
typedef struct {
/*
* structure revision number:
*/
int rsm_version;
/*
* import side memory segment operations
*/
int (* rsm_memseg_import_connect)
(rsmapi_controller_handle_t controller,
rsm_node_id_t node_id,
rsm_memseg_id_t segment_id,
rsm_permission_t perm,
rsm_memseg_import_handle_t *im_memseg);
int (* rsm_memseg_import_disconnect)
(rsm_memseg_import_handle_t im_memseg);
/*
* import side memory segment operations (read access functions):
*/
int (* rsm_memseg_import_get8)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint8_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_get16)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint16_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_get32)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint32_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_get64)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint64_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_get)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
void *dst_addr,
size_t length);
/*
* import side memory segment operations (read access functions):
*/
int (* rsm_memseg_import_put8)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint8_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_put16)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint16_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_put32)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint32_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_put64)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
uint64_t *datap,
ulong_t rep_cnt,
boolean_t swap);
int (* rsm_memseg_import_put)
(rsm_memseg_import_handle_t im_memseg,
off_t offset,
void *src_addr,
size_t length);
/*
* import side memory segment operations (barriers):
*/
int (* rsm_memseg_import_init_barrier)
(rsm_memseg_import_handle_t im_memseg,
rsm_barrier_type_t type,
rsm_barrier_handle_t barrier);
int (* rsm_memseg_import_open_barrier)(rsm_barrier_handle_t barrier);
int (* rsm_memseg_import_order_barrier)(rsm_barrier_handle_t barrier);
int (* rsm_memseg_import_close_barrier)(rsm_barrier_handle_t barrier);
int (* rsm_memseg_import_destroy_barrier)(rsm_barrier_handle_t barrier);
int (* rsm_memseg_import_get_mode)
(rsm_memseg_import_handle_t im_memseg,
rsm_barrier_mode_t *mode);
int (* rsm_memseg_import_set_mode)
(rsm_memseg_import_handle_t im_memseg,
rsm_barrier_mode_t mode);
/*
* import side memory segment data transfer operations.
*/
int (* rsm_memseg_import_putv)(rsm_scat_gath_t *sg_io);
int (* rsm_memseg_import_getv)(rsm_scat_gath_t *sg_io);
int (* rsm_create_localmemory_handle)
(rsmapi_controller_handle_t controller,
rsm_localmemory_handle_t *local_handle_p,
caddr_t local_vaddr, size_t len);
int (* rsm_free_localmemory_handle)
(rsm_localmemory_handle_t local_handle);
int (* rsm_register_lib_funcs)
(rsm_lib_funcs_t *libfuncs);
int (* rsm_get_lib_attr)
(rsm_ndlib_attr_t **libattr);
int (* rsm_closedevice)
(rsmapi_controller_handle_t controller);
} rsm_segops_t;
#define RSM_LIB_VERSION 1
/* library internal controller attribute structure */
typedef struct {
/* following fields should be identical to rsmapi_controller_attr_t */
uint_t attr_direct_access_sizes;
uint_t attr_atomic_sizes;
size_t attr_page_size;
size_t attr_max_export_segment_size;
size_t attr_tot_export_segment_size;
ulong_t attr_max_export_segments;
size_t attr_max_import_map_size;
size_t attr_tot_import_map_size;
ulong_t attr_max_import_segments;
/* following fields are for internal use */
rsm_addr_t attr_controller_addr;
} rsm_int_controller_attr_t;
typedef struct rsm_controller {
void *cntr_privdata;
struct rsm_controller *cntr_next;
int cntr_fd;
int cntr_refcnt;
int cntr_unit;
char *cntr_name; /* generic type eg. sci */
rsm_segops_t *cntr_segops;
struct rsmqueue *cntr_rqlist; /* list of receive queues */
rsm_int_controller_attr_t cntr_attr;
rsm_ndlib_attr_t *cntr_lib_attr;
mutex_t cntr_lock;
cond_t cntr_cv;
} rsm_controller_t;
typedef enum {
EXPORT_CREATE = 0x1,
EXPORT_BIND,
EXPORT_PUBLISH,
IMPORT_CONNECT,
IMPORT_DISCONNECT,
IMPORT_MAP,
IMPORT_UNMAP
} rsm_seg_state_t;
typedef struct {
void *rsmseg_privdata;
rsm_segops_t *rsmseg_ops;
rsm_seg_state_t rsmseg_state;
caddr_t rsmseg_vaddr; /* base address of segment */
size_t rsmseg_size; /* size of segment */
size_t rsmseg_maplen; /* length of mapped region */
rsm_node_id_t rsmseg_nodeid;
rsm_memseg_id_t rsmseg_keyid;
int rsmseg_fd;
int rsmseg_pollfd_refcnt;
rsm_permission_t rsmseg_perm;
rsm_controller_t *rsmseg_controller;
rsm_barrier_mode_t rsmseg_barmode;
void *rsmseg_data;
uint16_t *rsmseg_bar;
uint16_t rsmseg_gnum; /* generation number */
int rsmseg_type;
mutex_t rsmseg_lock;
rsmapi_barrier_t *rsmseg_barrier; /* used in put/get routines */
offset_t rsmseg_mapoffset; /* seg offset where mmapped */
uint32_t rsmseg_flags;
minor_t rsmseg_rnum; /* resource number of the segment */
} rsmseg_handle_t;
/*
* defines for rsmseg_flags
*/
#define RSM_IMPLICIT_MAP 0x00000001 /* segment mapped implicitly */
/* This is a template for all barrier implementations */
typedef struct {
rsmseg_handle_t *rsmbar_seg;
uint16_t rsmbar_gen; /* generation number */
void *rsmbar_privdata;
} rsmbar_handle_t;
/*
* These macros set and get the private data pointer in the opaque barrier
* structure for Network plugins.
*/
#define RSMNDI_BARRIER_SETPRIV(HANDLE, ADDR) \
((rsmbar_handle_t *)HANDLE)->rsmbar_privdata = (void *)ADDR;
#define RSMNDI_BARRIER_GETPRIV(HANDLE) \
((rsmbar_handle_t *)HANDLE)->rsmbar_privdata
#define RSMNDI_BARRIER_GETSEG(HANDLE) \
((rsmbar_handle_t *)HANDLE)->rsmbar_seg
#define RSMNDI_BARRIER_GETUNIT(HANDLE) \
((rsmbar_handle_t *)HANDLE)->rsmbar_seg->rsmseg_controller->cntr_unit
/*
* These macros set and get the private data pointer in the opaque segment
* structure for Network plugins.
*/
#define RSMNDI_SEG_SETPRIV(HANDLE, ADDR) \
((rsmseg_handle_t *)HANDLE)->rsmseg_privdata = (void *)ADDR;
#define RSMNDI_SEG_GETPRIV(HANDLE) \
((rsmseg_handle_t *)HANDLE)->rsmseg_privdata
/*
* Get the controller unit number from a opaque segment structure.
*/
#define RSMNDI_SEG_GETUNIT(HANDLE) \
((rsmseg_handle_t *)HANDLE)->rsmseg_controller->cntr_unit
/*
* These macros set and get the private data pointer in the opaque controller
* structure for Network plugins.
*/
#define RSMNDI_CNTRLR_SETPRIV(HANDLE, ADDR) \
((rsm_controller_t *)HANDLE)->cntr_privdata = (void *)ADDR;
#define RSMNDI_CNTRLR_GETPRIV(HANDLE) \
((rsm_controller_t *)HANDLE)->cntr_privdata
/*
* Get the controller unit number from a opaque controller structure.
*/
#define RSMNDI_CNTRLR_GETUNIT(HANDLE) \
((rsm_controller_t *)HANDLE)->cntr_unit
/*
* This macro returns an address inside a segment given the segment handle
* and a byte offset.
*/
#define RSMNDI_GET_MAPADDR(HANDLE, OFFSET) \
(((rsmseg_handle_t *)HANDLE)->rsmseg_vaddr + OFFSET)
#ifdef __cplusplus
}
#endif
#endif /* _SYS_RSM_RSMNDI_H */
|