1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _ISER_RESOURCE_H
#define _ISER_RESOURCE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <sys/types.h>
#include <sys/ib/ibtl/ibti.h>
#include <sys/ib/ibtl/ibtl_types.h>
#include <sys/iscsi_protocol.h>
#define ISER_CACHE_NAMELEN 31 /* KMEM_CACHE_NAMELEN */
/* Default message lengths */
#define ISER_MAX_CTRLPDU_LEN 0x4000
#define ISER_MAX_TEXTPDU_LEN 0x4000
/* Default data buffer length */
#define ISER_DEFAULT_BUFLEN 0x20000
/*
* iser_resource.h
* Definitions and functions related to set up buffer allocation from
* IBT memory regions and managment of work requessts.
*/
struct iser_hca_s;
/*
* Memory regions
*/
typedef struct iser_mr_s {
ibt_mr_hdl_t is_mrhdl;
ib_vaddr_t is_mrva;
ib_memlen_t is_mrlen;
ibt_lkey_t is_mrlkey;
ibt_rkey_t is_mrrkey;
avl_node_t is_avl_ln;
} iser_mr_t;
typedef struct iser_vmem_mr_pool_s {
iser_hca_t *ivmp_hca;
ibt_mr_flags_t ivmp_mr_flags;
ib_memlen_t ivmp_chunksize;
vmem_t *ivmp_vmem;
uint64_t ivmp_total_size;
uint64_t ivmp_max_total_size;
avl_tree_t ivmp_mr_list;
kmutex_t ivmp_mutex;
} iser_vmem_mr_pool_t;
#define ISER_MR_QUANTSIZE 0x400
#define ISER_MIN_CHUNKSIZE 0x100000 /* 1MB */
#ifdef _LP64
#define ISER_BUF_MR_CHUNKSIZE 0x8000000 /* 128MB */
#define ISER_BUF_POOL_MAX 0x40000000 /* 1GB */
#else
/* Memory is very limited on 32-bit kernels */
#define ISER_BUF_MR_CHUNKSIZE 0x400000 /* 4MB */
#define ISER_BUF_POOL_MAX 0x4000000 /* 64MB */
#endif
#define ISER_BUF_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE | \
IBT_MR_ENABLE_REMOTE_READ | IBT_MR_ENABLE_REMOTE_WRITE
#ifdef _LP64
#define ISER_MSG_MR_CHUNKSIZE 0x2000000 /* 32MB */
#define ISER_MSG_POOL_MAX 0x10000000 /* 256MB */
#else
#define ISER_MSG_MR_CHUNKSIZE 0x100000 /* 1MB */
#define ISER_MSG_POOL_MAX 0x2000000 /* 32MB */
#endif
#define ISER_MSG_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE
iser_vmem_mr_pool_t *iser_vmem_create(const char *name, iser_hca_t *hca,
ib_memlen_t chunksize, uint64_t max_total_size,
ibt_mr_flags_t arena_mr_flags);
void iser_vmem_destroy(iser_vmem_mr_pool_t *vmr_pool);
void *iser_vmem_alloc(iser_vmem_mr_pool_t *vmr_pool, size_t size);
void iser_vmem_free(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size);
idm_status_t iser_vmem_mr(iser_vmem_mr_pool_t *vmr_pool,
void *vaddr, size_t size, iser_mr_t *mr);
/*
* iSER work request structure encodes an iSER Send Queue work request
* context, with pointers to relevant resources related to the work request.
* We hold a pointer to either an IDM PDU handle, an iSER message handle
* or an IDM buffer handle. These are allocated from a kmem_cache when
* we post send WR's, and freed back when the completion is polled.
*/
typedef enum {
ISER_WR_SEND,
ISER_WR_RDMAW,
ISER_WR_RDMAR,
ISER_WR_UNDEFINED
} iser_wr_type_t;
typedef struct iser_wr_s {
iser_wr_type_t iw_type;
struct iser_msg_s *iw_msg;
struct idm_buf_s *iw_buf;
struct idm_pdu_s *iw_pdu;
} iser_wr_t;
int iser_wr_cache_constructor(void *mr, void *arg, int flags);
void iser_wr_cache_destructor(void *mr, void *arg);
iser_wr_t *iser_wr_get();
void iser_wr_free(iser_wr_t *iser_wr);
/*
* iSER message structure for iSCSI Control PDUs, constructor and
* destructor routines, and utility routines for allocating and
* freeing message handles.
*/
typedef struct iser_msg_s {
struct iser_msg_s *nextp; /* for building lists */
kmem_cache_t *cache; /* back pointer for cleanup */
ibt_wr_ds_t msg_ds; /* SGEs for hdr and text */
ibt_mr_hdl_t mrhdl[2]; /* MR handles for each SGE */
} iser_msg_t;
int iser_msg_cache_constructor(void *mr, void *arg, int flags);
void iser_msg_cache_destructor(void *mr, void *arg);
iser_msg_t *iser_msg_get(iser_hca_t *hca, int num, int *ret);
void iser_msg_free(iser_msg_t *msg);
/*
* iSER data buffer structure for iSER RDMA operations, constructor and
* destructor routines, and utility routines for allocating and freeing
* buffer handles.
*/
typedef struct iser_buf_s {
kmem_cache_t *cache; /* back pointer for cleanup */
void *buf; /* buffer */
uint64_t buflen;
iser_mr_t *iser_mr; /* MR handle for this buffer */
ibt_wr_ds_t buf_ds; /* SGE for this buffer */
ibt_send_wr_t buf_wr; /* DEBUG, copy of wr from request */
ibt_wc_t buf_wc; /* DEBUG, copy of wc from completion */
timespec_t buf_constructed;
timespec_t buf_destructed;
} iser_buf_t;
int iser_buf_cache_constructor(void *mr, void *arg, int flags);
void iser_buf_cache_destructor(void *mr, void *arg);
void iser_init_hca_caches(struct iser_hca_s *hca);
void iser_fini_hca_caches(struct iser_hca_s *hca);
/* Routines to register in-place memory passed on an existing idb */
int iser_reg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb);
void iser_dereg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb);
#ifdef __cplusplus
}
#endif
#endif /* _ISER_RESOURCE_H */
|
