1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
|
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _ISER_IB_H
#define _ISER_IB_H
#ifdef __cplusplus
extern "C" {
#endif
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/ib/ibtl/ibti.h>
#include <sys/iscsi_protocol.h>
/*
* iser_ib.h
* Definitions and macros related to iSER InfiniBand transport,
* and the use of the Solaris IBTI (InfiniBand Transport Interface).
*/
struct iser_state_s;
extern struct iser_state_s *iser_state;
extern ddi_taskq_t *iser_taskq;
/*
* iser_hca_s holds all the information about the Infinband HCAs in use.
*/
typedef struct iser_hca_s {
list_node_t hca_node;
boolean_t hca_failed;
ibt_clnt_hdl_t hca_clnt_hdl;
ibt_hca_hdl_t hca_hdl;
ibt_hca_attr_t hca_attr;
ibt_pd_hdl_t hca_pdhdl;
ib_guid_t hca_guid;
uint_t hca_num_ports;
ibt_hca_portinfo_t *hca_port_info;
uint_t hca_port_info_sz;
/* Per PD (per HCA) message and data buffer caches */
struct iser_vmem_mr_pool_s *hca_msg_pool; /* Use iser_msg_cache */
kmem_cache_t *iser_msg_cache;
struct iser_vmem_mr_pool_s *hca_buf_pool; /* Use iser_buf_cache */
kmem_cache_t *iser_buf_cache;
} iser_hca_t;
/* RQ low water mark percentage */
#define ISER_IB_RQ_LWM_PCT 80
/* Maximum number of WRs to post on the RQ at a time */
#define ISER_IB_RQ_POST_MAX 64
/* Maximum number of SCQ WCs to poll at a time */
#define ISER_IB_SCQ_POLL_MAX 8
/*
* iser_qp_t encodes data related to a Queue Pair (QP) in use by
* iSER. Each QP consists of two Work Queues (WQs), one Send Queue
* (SQ) and on Receive Queue (RQ). Most of the data in the QP
* handle relates to monitoring the posted depth of the RQ.
*
* Note that we are explicitly using slightly less than a power-of-2
* number for our queue sizes. The HCA driver will round up for us,
* and this affords us some headroom.
*/
#ifdef _LP64
#define ISER_IB_RECVQ_SIZE 400
#else
/* Memory is very limited on 32-bit kernels */
#define ISER_IB_RECVQ_SIZE 100
#endif
#define ISER_IB_SENDQ_SIZE 2000
#define ISER_IB_SGLIST_SIZE 1
#define ISER_IB_DEFAULT_IRD 2
#define ISER_IB_DEFAULT_ORD 4
typedef struct iser_qp_s {
kmutex_t qp_lock;
uint_t sq_size;
uint_t rq_size;
uint32_t rq_depth;
uint32_t rq_level;
uint32_t rq_min_post_level;
uint32_t rq_lwm;
boolean_t rq_taskqpending;
} iser_qp_t;
/*
* iSER RC channel information
*/
typedef struct iser_chan_s {
kmutex_t ic_chan_lock;
/* IBT channel handle */
ibt_channel_hdl_t ic_chanhdl;
/* local and remote IP addresses and port numbers */
ibt_ip_addr_t ic_localip;
ibt_ip_addr_t ic_remoteip;
in_port_t ic_lport;
in_port_t ic_rport;
/*
* The local HCA GUID, the service ID, Destination GID, Source GID
* the primary hca port on which the channel is connected is
* stored in ic_ibt_path
*/
ibt_path_info_t ic_ibt_path;
/*
* Information related to the HCA handle and the queues.
*/
iser_hca_t *ic_hca;
ibt_cq_hdl_t ic_sendcq;
ibt_cq_hdl_t ic_recvcq;
uint_t ic_sendcq_sz;
uint_t ic_recvcq_sz;
iser_qp_t ic_qp;
/* Used to track the number of WRs posted on the SQ */
kmutex_t ic_sq_post_lock;
uint_t ic_sq_post_count;
uint_t ic_sq_max_post_count;
/*
* To help identify the channel end point and some connection
* specifics, maintain a pointer to the connection on which
* this channel originated
*/
struct iser_conn_s *ic_conn;
} iser_chan_t;
int iser_ib_init(void);
int iser_ib_fini(void);
int iser_ib_register_service(idm_svc_t *idm_svc);
int iser_ib_bind_service(idm_svc_t *idm_svc);
void iser_ib_unbind_service(idm_svc_t *idm_svc);
void iser_ib_deregister_service(idm_svc_t *idm_svc);
void iser_ib_conv_sockaddr2ibtaddr(idm_sockaddr_t *saddr,
ibt_ip_addr_t *ibt_addr);
void iser_ib_conv_ibtaddr2sockaddr(struct sockaddr_storage *ss,
ibt_ip_addr_t *ibt_addr, in_port_t port);
int iser_ib_get_paths(
ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip, ibt_path_info_t *path,
ibt_path_ip_src_t *path_src_ip);
iser_chan_t *iser_ib_alloc_channel_pathlookup(
ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip);
iser_chan_t *iser_ib_alloc_channel_nopathlookup(
ib_guid_t hca_guid, uint8_t hca_port);
iser_chan_t *iser_ib_alloc_rc_channel(iser_hca_t *hca, uint8_t hca_port);
int iser_ib_open_rc_channel(iser_chan_t *chan);
void iser_ib_close_rc_channel(iser_chan_t *chan);
void iser_ib_free_rc_channel(iser_chan_t *chan);
int iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl);
void iser_ib_post_recv(ibt_channel_hdl_t chanhdl);
void iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg);
void iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg);
void iser_ib_async_handler(void *clntp, ibt_hca_hdl_t hdl,
ibt_async_code_t code, ibt_async_event_t *event);
#ifdef __cplusplus
}
#endif
#endif /* _ISER_IB_H */
|
