1 /*
2 * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4 * Copyright (c) 2004 Intel Corporation. All rights reserved.
5 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
6 * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
9 *
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
15 *
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
18 * conditions are met:
19 *
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer.
23 *
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
28 *
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 * SOFTWARE.
37 */
38
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/module.h>
42 #include <linux/string.h>
43 #include <linux/slab.h>
44
45 #include <rdma/ib_verbs.h>
46 #include <rdma/ib_cache.h>
47 #include <rdma/ib_addr.h>
48
ib_rate_to_mult(enum ib_rate rate)49 int ib_rate_to_mult(enum ib_rate rate)
50 {
51 switch (rate) {
52 case IB_RATE_2_5_GBPS: return 1;
53 case IB_RATE_5_GBPS: return 2;
54 case IB_RATE_10_GBPS: return 4;
55 case IB_RATE_20_GBPS: return 8;
56 case IB_RATE_30_GBPS: return 12;
57 case IB_RATE_40_GBPS: return 16;
58 case IB_RATE_60_GBPS: return 24;
59 case IB_RATE_80_GBPS: return 32;
60 case IB_RATE_120_GBPS: return 48;
61 default: return -1;
62 }
63 }
64 EXPORT_SYMBOL(ib_rate_to_mult);
65
mult_to_ib_rate(int mult)66 enum ib_rate mult_to_ib_rate(int mult)
67 {
68 switch (mult) {
69 case 1: return IB_RATE_2_5_GBPS;
70 case 2: return IB_RATE_5_GBPS;
71 case 4: return IB_RATE_10_GBPS;
72 case 8: return IB_RATE_20_GBPS;
73 case 12: return IB_RATE_30_GBPS;
74 case 16: return IB_RATE_40_GBPS;
75 case 24: return IB_RATE_60_GBPS;
76 case 32: return IB_RATE_80_GBPS;
77 case 48: return IB_RATE_120_GBPS;
78 default: return IB_RATE_PORT_CURRENT;
79 }
80 }
81 EXPORT_SYMBOL(mult_to_ib_rate);
82
ib_rate_to_mbps(enum ib_rate rate)83 int ib_rate_to_mbps(enum ib_rate rate)
84 {
85 switch (rate) {
86 case IB_RATE_2_5_GBPS: return 2500;
87 case IB_RATE_5_GBPS: return 5000;
88 case IB_RATE_10_GBPS: return 10000;
89 case IB_RATE_20_GBPS: return 20000;
90 case IB_RATE_30_GBPS: return 30000;
91 case IB_RATE_40_GBPS: return 40000;
92 case IB_RATE_60_GBPS: return 60000;
93 case IB_RATE_80_GBPS: return 80000;
94 case IB_RATE_120_GBPS: return 120000;
95 case IB_RATE_14_GBPS: return 14062;
96 case IB_RATE_56_GBPS: return 56250;
97 case IB_RATE_112_GBPS: return 112500;
98 case IB_RATE_168_GBPS: return 168750;
99 case IB_RATE_25_GBPS: return 25781;
100 case IB_RATE_100_GBPS: return 103125;
101 case IB_RATE_200_GBPS: return 206250;
102 case IB_RATE_300_GBPS: return 309375;
103 default: return -1;
104 }
105 }
106 EXPORT_SYMBOL(ib_rate_to_mbps);
107
108 enum rdma_transport_type
rdma_node_get_transport(enum rdma_node_type node_type)109 rdma_node_get_transport(enum rdma_node_type node_type)
110 {
111 switch (node_type) {
112 case RDMA_NODE_IB_CA:
113 case RDMA_NODE_IB_SWITCH:
114 case RDMA_NODE_IB_ROUTER:
115 return RDMA_TRANSPORT_IB;
116 case RDMA_NODE_RNIC:
117 return RDMA_TRANSPORT_IWARP;
118 case RDMA_NODE_MIC:
119 return RDMA_TRANSPORT_SCIF;
120 default:
121 BUG();
122 return 0;
123 }
124 }
125 EXPORT_SYMBOL(rdma_node_get_transport);
126
rdma_port_get_link_layer(struct ib_device * device,u8 port_num)127 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
128 {
129 if (device->get_link_layer)
130 return device->get_link_layer(device, port_num);
131
132 switch (rdma_node_get_transport(device->node_type)) {
133 case RDMA_TRANSPORT_IB:
134 return IB_LINK_LAYER_INFINIBAND;
135 case RDMA_TRANSPORT_IWARP:
136 return IB_LINK_LAYER_ETHERNET;
137 case RDMA_TRANSPORT_SCIF:
138 return IB_LINK_LAYER_SCIF;
139 default:
140 return IB_LINK_LAYER_UNSPECIFIED;
141 }
142 }
143 EXPORT_SYMBOL(rdma_port_get_link_layer);
144
145 /* Protection domains */
146
ib_alloc_pd(struct ib_device * device)147 struct ib_pd *ib_alloc_pd(struct ib_device *device)
148 {
149 struct ib_pd *pd;
150
151 pd = device->alloc_pd(device, NULL, NULL);
152
153 if (!IS_ERR(pd)) {
154 pd->device = device;
155 pd->uobject = NULL;
156 atomic_set(&pd->usecnt, 0);
157 }
158
159 return pd;
160 }
161 EXPORT_SYMBOL(ib_alloc_pd);
162
ib_dealloc_pd(struct ib_pd * pd)163 int ib_dealloc_pd(struct ib_pd *pd)
164 {
165 if (atomic_read(&pd->usecnt))
166 return -EBUSY;
167
168 return pd->device->dealloc_pd(pd);
169 }
170 EXPORT_SYMBOL(ib_dealloc_pd);
171
172 /* Address handles */
173
ib_create_ah(struct ib_pd * pd,struct ib_ah_attr * ah_attr)174 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
175 {
176 struct ib_ah *ah;
177
178 ah = pd->device->create_ah(pd, ah_attr);
179
180 if (!IS_ERR(ah)) {
181 ah->device = pd->device;
182 ah->pd = pd;
183 ah->uobject = NULL;
184 atomic_inc(&pd->usecnt);
185 }
186
187 return ah;
188 }
189 EXPORT_SYMBOL(ib_create_ah);
190
ib_init_ah_from_wc(struct ib_device * device,u8 port_num,struct ib_wc * wc,struct ib_grh * grh,struct ib_ah_attr * ah_attr)191 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
192 struct ib_grh *grh, struct ib_ah_attr *ah_attr)
193 {
194 u32 flow_class;
195 u16 gid_index;
196 int ret;
197 int is_eth = (rdma_port_get_link_layer(device, port_num) ==
198 IB_LINK_LAYER_ETHERNET);
199
200 memset(ah_attr, 0, sizeof *ah_attr);
201 if (is_eth) {
202 if (!(wc->wc_flags & IB_WC_GRH))
203 return -EPROTOTYPE;
204
205 if (wc->wc_flags & IB_WC_WITH_SMAC &&
206 wc->wc_flags & IB_WC_WITH_VLAN) {
207 memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
208 ah_attr->vlan_id = wc->vlan_id;
209 } else {
210 ret = rdma_addr_find_dmac_by_grh(&grh->dgid, &grh->sgid,
211 ah_attr->dmac, &ah_attr->vlan_id);
212 if (ret)
213 return ret;
214 }
215 } else {
216 ah_attr->vlan_id = 0xffff;
217 }
218
219
220 ah_attr->dlid = wc->slid;
221 ah_attr->sl = wc->sl;
222 ah_attr->src_path_bits = wc->dlid_path_bits;
223 ah_attr->port_num = port_num;
224
225 if (wc->wc_flags & IB_WC_GRH) {
226 ah_attr->ah_flags = IB_AH_GRH;
227 ah_attr->grh.dgid = grh->sgid;
228
229 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
230 &gid_index);
231 if (ret)
232 return ret;
233
234 ah_attr->grh.sgid_index = (u8) gid_index;
235 flow_class = be32_to_cpu(grh->version_tclass_flow);
236 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
237 ah_attr->grh.hop_limit = 0xFF;
238 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
239 }
240 return 0;
241 }
242 EXPORT_SYMBOL(ib_init_ah_from_wc);
243
ib_create_ah_from_wc(struct ib_pd * pd,struct ib_wc * wc,struct ib_grh * grh,u8 port_num)244 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
245 struct ib_grh *grh, u8 port_num)
246 {
247 struct ib_ah_attr ah_attr;
248 int ret;
249
250 ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
251 if (ret)
252 return ERR_PTR(ret);
253
254 return ib_create_ah(pd, &ah_attr);
255 }
256 EXPORT_SYMBOL(ib_create_ah_from_wc);
257
ib_modify_ah(struct ib_ah * ah,struct ib_ah_attr * ah_attr)258 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
259 {
260 return ah->device->modify_ah ?
261 ah->device->modify_ah(ah, ah_attr) :
262 -ENOSYS;
263 }
264 EXPORT_SYMBOL(ib_modify_ah);
265
ib_query_ah(struct ib_ah * ah,struct ib_ah_attr * ah_attr)266 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
267 {
268 return ah->device->query_ah ?
269 ah->device->query_ah(ah, ah_attr) :
270 -ENOSYS;
271 }
272 EXPORT_SYMBOL(ib_query_ah);
273
ib_destroy_ah(struct ib_ah * ah)274 int ib_destroy_ah(struct ib_ah *ah)
275 {
276 struct ib_pd *pd;
277 int ret;
278
279 pd = ah->pd;
280 ret = ah->device->destroy_ah(ah);
281 if (!ret)
282 atomic_dec(&pd->usecnt);
283
284 return ret;
285 }
286 EXPORT_SYMBOL(ib_destroy_ah);
287
288 /* Shared receive queues */
289
ib_create_srq(struct ib_pd * pd,struct ib_srq_init_attr * srq_init_attr)290 struct ib_srq *ib_create_srq(struct ib_pd *pd,
291 struct ib_srq_init_attr *srq_init_attr)
292 {
293 struct ib_srq *srq;
294
295 if (!pd->device->create_srq)
296 return ERR_PTR(-ENOSYS);
297
298 srq = pd->device->create_srq(pd, srq_init_attr, NULL);
299
300 if (!IS_ERR(srq)) {
301 srq->device = pd->device;
302 srq->pd = pd;
303 srq->uobject = NULL;
304 srq->event_handler = srq_init_attr->event_handler;
305 srq->srq_context = srq_init_attr->srq_context;
306 srq->srq_type = srq_init_attr->srq_type;
307 if (srq->srq_type == IB_SRQT_XRC) {
308 srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
309 srq->ext.xrc.cq = srq_init_attr->ext.xrc.cq;
310 atomic_inc(&srq->ext.xrc.xrcd->usecnt);
311 atomic_inc(&srq->ext.xrc.cq->usecnt);
312 }
313 atomic_inc(&pd->usecnt);
314 atomic_set(&srq->usecnt, 0);
315 }
316
317 return srq;
318 }
319 EXPORT_SYMBOL(ib_create_srq);
320
ib_modify_srq(struct ib_srq * srq,struct ib_srq_attr * srq_attr,enum ib_srq_attr_mask srq_attr_mask)321 int ib_modify_srq(struct ib_srq *srq,
322 struct ib_srq_attr *srq_attr,
323 enum ib_srq_attr_mask srq_attr_mask)
324 {
325 return srq->device->modify_srq ?
326 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
327 -ENOSYS;
328 }
329 EXPORT_SYMBOL(ib_modify_srq);
330
ib_query_srq(struct ib_srq * srq,struct ib_srq_attr * srq_attr)331 int ib_query_srq(struct ib_srq *srq,
332 struct ib_srq_attr *srq_attr)
333 {
334 return srq->device->query_srq ?
335 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
336 }
337 EXPORT_SYMBOL(ib_query_srq);
338
ib_query_values(struct ib_device * device,int q_values,struct ib_device_values * values)339 int ib_query_values(struct ib_device *device,
340 int q_values, struct ib_device_values *values)
341 {
342 return device->query_values ?
343 device->query_values(device, q_values, values) : -ENOSYS;
344 }
345 EXPORT_SYMBOL(ib_query_values);
346
ib_destroy_srq(struct ib_srq * srq)347 int ib_destroy_srq(struct ib_srq *srq)
348 {
349 struct ib_pd *pd;
350 enum ib_srq_type srq_type;
351 struct ib_xrcd *uninitialized_var(xrcd);
352 struct ib_cq *uninitialized_var(cq);
353 int ret;
354
355 if (atomic_read(&srq->usecnt))
356 return -EBUSY;
357
358 pd = srq->pd;
359 srq_type = srq->srq_type;
360 if (srq_type == IB_SRQT_XRC) {
361 xrcd = srq->ext.xrc.xrcd;
362 cq = srq->ext.xrc.cq;
363 }
364
365 ret = srq->device->destroy_srq(srq);
366 if (!ret) {
367 atomic_dec(&pd->usecnt);
368 if (srq_type == IB_SRQT_XRC) {
369 atomic_dec(&xrcd->usecnt);
370 atomic_dec(&cq->usecnt);
371 }
372 }
373
374 return ret;
375 }
376 EXPORT_SYMBOL(ib_destroy_srq);
377
378 /* Queue pairs */
379
__ib_shared_qp_event_handler(struct ib_event * event,void * context)380 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
381 {
382 struct ib_qp *qp = context;
383 unsigned long flags;
384
385 /* The code below must be synced with deletions of existing qps (ib_close_qp) --
386 * because a qp from the list may be closed during the scan, resulting in a kernel Oops.
387 */
388 spin_lock_irqsave(&qp->device->event_handler_lock, flags);
389 list_for_each_entry(event->element.qp, &qp->open_list, open_list)
390 if (event->element.qp->event_handler)
391 event->element.qp->event_handler(event, event->element.qp->qp_context);
392 spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
393 }
394
__ib_insert_xrcd_qp(struct ib_xrcd * xrcd,struct ib_qp * qp)395 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
396 {
397 mutex_lock(&xrcd->tgt_qp_mutex);
398 list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
399 mutex_unlock(&xrcd->tgt_qp_mutex);
400 }
401
__ib_open_qp(struct ib_qp * real_qp,void (* event_handler)(struct ib_event *,void *),void * qp_context)402 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
403 void (*event_handler)(struct ib_event *, void *),
404 void *qp_context)
405 {
406 struct ib_qp *qp;
407 unsigned long flags;
408
409 qp = kzalloc(sizeof *qp, GFP_KERNEL);
410 if (!qp)
411 return ERR_PTR(-ENOMEM);
412
413 qp->real_qp = real_qp;
414 atomic_inc(&real_qp->usecnt);
415 qp->device = real_qp->device;
416 qp->event_handler = event_handler;
417 qp->qp_context = qp_context;
418 qp->qp_num = real_qp->qp_num;
419 qp->qp_type = real_qp->qp_type;
420
421 spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
422 list_add(&qp->open_list, &real_qp->open_list);
423 spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
424
425 return qp;
426 }
427
ib_open_qp(struct ib_xrcd * xrcd,struct ib_qp_open_attr * qp_open_attr)428 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
429 struct ib_qp_open_attr *qp_open_attr)
430 {
431 struct ib_qp *qp, *real_qp;
432
433 if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
434 return ERR_PTR(-EINVAL);
435
436 qp = ERR_PTR(-EINVAL);
437 mutex_lock(&xrcd->tgt_qp_mutex);
438 list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
439 if (real_qp->qp_num == qp_open_attr->qp_num) {
440 qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
441 qp_open_attr->qp_context);
442 break;
443 }
444 }
445 mutex_unlock(&xrcd->tgt_qp_mutex);
446 return qp;
447 }
448 EXPORT_SYMBOL(ib_open_qp);
449
ib_create_qp(struct ib_pd * pd,struct ib_qp_init_attr * qp_init_attr)450 struct ib_qp *ib_create_qp(struct ib_pd *pd,
451 struct ib_qp_init_attr *qp_init_attr)
452 {
453 struct ib_qp *qp, *real_qp;
454 struct ib_device *device;
455
456 device = pd ? pd->device : qp_init_attr->xrcd->device;
457 qp = device->create_qp(pd, qp_init_attr, NULL);
458
459 if (!IS_ERR(qp)) {
460 qp->device = device;
461 qp->real_qp = qp;
462 qp->uobject = NULL;
463 qp->qp_type = qp_init_attr->qp_type;
464
465 atomic_set(&qp->usecnt, 0);
466 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
467 qp->event_handler = __ib_shared_qp_event_handler;
468 qp->qp_context = qp;
469 qp->pd = NULL;
470 qp->send_cq = qp->recv_cq = NULL;
471 qp->srq = NULL;
472 qp->xrcd = qp_init_attr->xrcd;
473 atomic_inc(&qp_init_attr->xrcd->usecnt);
474 INIT_LIST_HEAD(&qp->open_list);
475
476 real_qp = qp;
477 qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
478 qp_init_attr->qp_context);
479 if (!IS_ERR(qp))
480 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
481 else
482 real_qp->device->destroy_qp(real_qp);
483 } else {
484 qp->event_handler = qp_init_attr->event_handler;
485 qp->qp_context = qp_init_attr->qp_context;
486 if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
487 qp->recv_cq = NULL;
488 qp->srq = NULL;
489 } else {
490 qp->recv_cq = qp_init_attr->recv_cq;
491 atomic_inc(&qp_init_attr->recv_cq->usecnt);
492 qp->srq = qp_init_attr->srq;
493 if (qp->srq)
494 atomic_inc(&qp_init_attr->srq->usecnt);
495 }
496
497 qp->pd = pd;
498 qp->send_cq = qp_init_attr->send_cq;
499 qp->xrcd = NULL;
500
501 atomic_inc(&pd->usecnt);
502 atomic_inc(&qp_init_attr->send_cq->usecnt);
503 }
504 }
505
506 return qp;
507 }
508 EXPORT_SYMBOL(ib_create_qp);
509
510 static const struct {
511 int valid;
512 enum ib_qp_attr_mask req_param[IB_QPT_MAX];
513 enum ib_qp_attr_mask req_param_add_eth[IB_QPT_MAX];
514 enum ib_qp_attr_mask opt_param[IB_QPT_MAX];
515 enum ib_qp_attr_mask opt_param_add_eth[IB_QPT_MAX];
516 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
517 [IB_QPS_RESET] = {
518 [IB_QPS_RESET] = { .valid = 1 },
519 [IB_QPS_INIT] = {
520 .valid = 1,
521 .req_param = {
522 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
523 IB_QP_PORT |
524 IB_QP_QKEY),
525 [IB_QPT_RAW_PACKET] = IB_QP_PORT,
526 [IB_QPT_UC] = (IB_QP_PKEY_INDEX |
527 IB_QP_PORT |
528 IB_QP_ACCESS_FLAGS),
529 [IB_QPT_RC] = (IB_QP_PKEY_INDEX |
530 IB_QP_PORT |
531 IB_QP_ACCESS_FLAGS),
532 [IB_QPT_DC_INI] = (IB_QP_PKEY_INDEX |
533 IB_QP_PORT |
534 IB_QP_ACCESS_FLAGS |
535 IB_QP_DC_KEY),
536 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX |
537 IB_QP_PORT |
538 IB_QP_ACCESS_FLAGS),
539 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX |
540 IB_QP_PORT |
541 IB_QP_ACCESS_FLAGS),
542 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
543 IB_QP_QKEY),
544 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
545 IB_QP_QKEY),
546 },
547 .opt_param = {
548 [IB_QPT_UD] = IB_QP_GROUP_RSS,
549 [IB_QPT_RAW_PACKET] = IB_QP_GROUP_RSS
550 }
551 },
552 },
553 [IB_QPS_INIT] = {
554 [IB_QPS_RESET] = { .valid = 1 },
555 [IB_QPS_ERR] = { .valid = 1 },
556 [IB_QPS_INIT] = {
557 .valid = 1,
558 .opt_param = {
559 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
560 IB_QP_PORT |
561 IB_QP_QKEY),
562 [IB_QPT_UC] = (IB_QP_PKEY_INDEX |
563 IB_QP_PORT |
564 IB_QP_ACCESS_FLAGS),
565 [IB_QPT_RC] = (IB_QP_PKEY_INDEX |
566 IB_QP_PORT |
567 IB_QP_ACCESS_FLAGS),
568 [IB_QPT_DC_INI] = (IB_QP_PKEY_INDEX |
569 IB_QP_PORT |
570 IB_QP_ACCESS_FLAGS),
571 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX |
572 IB_QP_PORT |
573 IB_QP_ACCESS_FLAGS),
574 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX |
575 IB_QP_PORT |
576 IB_QP_ACCESS_FLAGS),
577 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
578 IB_QP_QKEY),
579 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
580 IB_QP_QKEY),
581 }
582 },
583 [IB_QPS_RTR] = {
584 .valid = 1,
585 .req_param = {
586 [IB_QPT_UC] = (IB_QP_AV |
587 IB_QP_PATH_MTU |
588 IB_QP_DEST_QPN |
589 IB_QP_RQ_PSN),
590 [IB_QPT_RC] = (IB_QP_AV |
591 IB_QP_PATH_MTU |
592 IB_QP_DEST_QPN |
593 IB_QP_RQ_PSN |
594 IB_QP_MAX_DEST_RD_ATOMIC |
595 IB_QP_MIN_RNR_TIMER),
596 [IB_QPT_DC_INI] = (IB_QP_PATH_MTU |
597 IB_QP_MAX_DEST_RD_ATOMIC |
598 IB_QP_MIN_RNR_TIMER),
599 [IB_QPT_XRC_INI] = (IB_QP_AV |
600 IB_QP_PATH_MTU |
601 IB_QP_DEST_QPN |
602 IB_QP_RQ_PSN),
603 [IB_QPT_XRC_TGT] = (IB_QP_AV |
604 IB_QP_PATH_MTU |
605 IB_QP_DEST_QPN |
606 IB_QP_RQ_PSN |
607 IB_QP_MAX_DEST_RD_ATOMIC |
608 IB_QP_MIN_RNR_TIMER),
609 },
610 .req_param_add_eth = {
611 [IB_QPT_RC] = (IB_QP_SMAC),
612 [IB_QPT_UC] = (IB_QP_SMAC),
613 [IB_QPT_XRC_INI] = (IB_QP_SMAC),
614 [IB_QPT_XRC_TGT] = (IB_QP_SMAC)
615 },
616 .opt_param = {
617 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
618 IB_QP_QKEY),
619 [IB_QPT_UC] = (IB_QP_ALT_PATH |
620 IB_QP_ACCESS_FLAGS |
621 IB_QP_PKEY_INDEX),
622 [IB_QPT_RC] = (IB_QP_ALT_PATH |
623 IB_QP_ACCESS_FLAGS |
624 IB_QP_PKEY_INDEX),
625 [IB_QPT_DC_INI] = (IB_QP_ALT_PATH |
626 IB_QP_ACCESS_FLAGS |
627 IB_QP_PKEY_INDEX),
628 [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH |
629 IB_QP_ACCESS_FLAGS |
630 IB_QP_PKEY_INDEX),
631 [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH |
632 IB_QP_ACCESS_FLAGS |
633 IB_QP_PKEY_INDEX),
634 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
635 IB_QP_QKEY),
636 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
637 IB_QP_QKEY),
638 [IB_QPT_RAW_PACKET] = IB_QP_AV,
639 },
640 .opt_param_add_eth = {
641 [IB_QPT_RC] = (IB_QP_ALT_SMAC |
642 IB_QP_VID |
643 IB_QP_ALT_VID),
644 [IB_QPT_UC] = (IB_QP_ALT_SMAC |
645 IB_QP_VID |
646 IB_QP_ALT_VID),
647 [IB_QPT_XRC_INI] = (IB_QP_ALT_SMAC |
648 IB_QP_VID |
649 IB_QP_ALT_VID),
650 [IB_QPT_XRC_TGT] = (IB_QP_ALT_SMAC |
651 IB_QP_VID |
652 IB_QP_ALT_VID)
653 }
654 }
655 },
656 [IB_QPS_RTR] = {
657 [IB_QPS_RESET] = { .valid = 1 },
658 [IB_QPS_ERR] = { .valid = 1 },
659 [IB_QPS_RTS] = {
660 .valid = 1,
661 .req_param = {
662 [IB_QPT_UD] = IB_QP_SQ_PSN,
663 [IB_QPT_UC] = IB_QP_SQ_PSN,
664 [IB_QPT_RC] = (IB_QP_TIMEOUT |
665 IB_QP_RETRY_CNT |
666 IB_QP_RNR_RETRY |
667 IB_QP_SQ_PSN |
668 IB_QP_MAX_QP_RD_ATOMIC),
669 [IB_QPT_DC_INI] = (IB_QP_TIMEOUT |
670 IB_QP_RETRY_CNT |
671 IB_QP_RNR_RETRY |
672 IB_QP_MAX_QP_RD_ATOMIC),
673 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT |
674 IB_QP_RETRY_CNT |
675 IB_QP_RNR_RETRY |
676 IB_QP_SQ_PSN |
677 IB_QP_MAX_QP_RD_ATOMIC),
678 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT |
679 IB_QP_SQ_PSN),
680 [IB_QPT_SMI] = IB_QP_SQ_PSN,
681 [IB_QPT_GSI] = IB_QP_SQ_PSN,
682 },
683 .opt_param = {
684 [IB_QPT_UD] = (IB_QP_CUR_STATE |
685 IB_QP_QKEY),
686 [IB_QPT_UC] = (IB_QP_CUR_STATE |
687 IB_QP_ALT_PATH |
688 IB_QP_ACCESS_FLAGS |
689 IB_QP_PATH_MIG_STATE),
690 [IB_QPT_RC] = (IB_QP_CUR_STATE |
691 IB_QP_ALT_PATH |
692 IB_QP_ACCESS_FLAGS |
693 IB_QP_MIN_RNR_TIMER |
694 IB_QP_PATH_MIG_STATE),
695 [IB_QPT_DC_INI] = (IB_QP_CUR_STATE |
696 IB_QP_ALT_PATH |
697 IB_QP_ACCESS_FLAGS |
698 IB_QP_MIN_RNR_TIMER |
699 IB_QP_PATH_MIG_STATE),
700 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
701 IB_QP_ALT_PATH |
702 IB_QP_ACCESS_FLAGS |
703 IB_QP_PATH_MIG_STATE),
704 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
705 IB_QP_ALT_PATH |
706 IB_QP_ACCESS_FLAGS |
707 IB_QP_MIN_RNR_TIMER |
708 IB_QP_PATH_MIG_STATE),
709 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
710 IB_QP_QKEY),
711 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
712 IB_QP_QKEY),
713 }
714 }
715 },
716 [IB_QPS_RTS] = {
717 [IB_QPS_RESET] = { .valid = 1 },
718 [IB_QPS_ERR] = { .valid = 1 },
719 [IB_QPS_RTS] = {
720 .valid = 1,
721 .opt_param = {
722 [IB_QPT_UD] = (IB_QP_CUR_STATE |
723 IB_QP_QKEY),
724 [IB_QPT_UC] = (IB_QP_CUR_STATE |
725 IB_QP_ACCESS_FLAGS |
726 IB_QP_ALT_PATH |
727 IB_QP_PATH_MIG_STATE),
728 [IB_QPT_RC] = (IB_QP_CUR_STATE |
729 IB_QP_ACCESS_FLAGS |
730 IB_QP_ALT_PATH |
731 IB_QP_PATH_MIG_STATE |
732 IB_QP_MIN_RNR_TIMER),
733 [IB_QPT_DC_INI] = (IB_QP_CUR_STATE |
734 IB_QP_ACCESS_FLAGS |
735 IB_QP_ALT_PATH |
736 IB_QP_PATH_MIG_STATE |
737 IB_QP_MIN_RNR_TIMER),
738 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
739 IB_QP_ACCESS_FLAGS |
740 IB_QP_ALT_PATH |
741 IB_QP_PATH_MIG_STATE),
742 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
743 IB_QP_ACCESS_FLAGS |
744 IB_QP_ALT_PATH |
745 IB_QP_PATH_MIG_STATE |
746 IB_QP_MIN_RNR_TIMER),
747 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
748 IB_QP_QKEY),
749 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
750 IB_QP_QKEY),
751 }
752 },
753 [IB_QPS_SQD] = {
754 .valid = 1,
755 .opt_param = {
756 [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY,
757 [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
758 [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
759 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
760 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
761 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
762 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
763 }
764 },
765 },
766 [IB_QPS_SQD] = {
767 [IB_QPS_RESET] = { .valid = 1 },
768 [IB_QPS_ERR] = { .valid = 1 },
769 [IB_QPS_RTS] = {
770 .valid = 1,
771 .opt_param = {
772 [IB_QPT_UD] = (IB_QP_CUR_STATE |
773 IB_QP_QKEY),
774 [IB_QPT_UC] = (IB_QP_CUR_STATE |
775 IB_QP_ALT_PATH |
776 IB_QP_ACCESS_FLAGS |
777 IB_QP_PATH_MIG_STATE),
778 [IB_QPT_RC] = (IB_QP_CUR_STATE |
779 IB_QP_ALT_PATH |
780 IB_QP_ACCESS_FLAGS |
781 IB_QP_MIN_RNR_TIMER |
782 IB_QP_PATH_MIG_STATE),
783 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
784 IB_QP_ALT_PATH |
785 IB_QP_ACCESS_FLAGS |
786 IB_QP_PATH_MIG_STATE),
787 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
788 IB_QP_ALT_PATH |
789 IB_QP_ACCESS_FLAGS |
790 IB_QP_MIN_RNR_TIMER |
791 IB_QP_PATH_MIG_STATE),
792 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
793 IB_QP_QKEY),
794 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
795 IB_QP_QKEY),
796 }
797 },
798 [IB_QPS_SQD] = {
799 .valid = 1,
800 .opt_param = {
801 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
802 IB_QP_QKEY),
803 [IB_QPT_UC] = (IB_QP_AV |
804 IB_QP_ALT_PATH |
805 IB_QP_ACCESS_FLAGS |
806 IB_QP_PKEY_INDEX |
807 IB_QP_PATH_MIG_STATE),
808 [IB_QPT_RC] = (IB_QP_PORT |
809 IB_QP_AV |
810 IB_QP_TIMEOUT |
811 IB_QP_RETRY_CNT |
812 IB_QP_RNR_RETRY |
813 IB_QP_MAX_QP_RD_ATOMIC |
814 IB_QP_MAX_DEST_RD_ATOMIC |
815 IB_QP_ALT_PATH |
816 IB_QP_ACCESS_FLAGS |
817 IB_QP_PKEY_INDEX |
818 IB_QP_MIN_RNR_TIMER |
819 IB_QP_PATH_MIG_STATE),
820 [IB_QPT_XRC_INI] = (IB_QP_PORT |
821 IB_QP_AV |
822 IB_QP_TIMEOUT |
823 IB_QP_RETRY_CNT |
824 IB_QP_RNR_RETRY |
825 IB_QP_MAX_QP_RD_ATOMIC |
826 IB_QP_ALT_PATH |
827 IB_QP_ACCESS_FLAGS |
828 IB_QP_PKEY_INDEX |
829 IB_QP_PATH_MIG_STATE),
830 [IB_QPT_XRC_TGT] = (IB_QP_PORT |
831 IB_QP_AV |
832 IB_QP_TIMEOUT |
833 IB_QP_MAX_DEST_RD_ATOMIC |
834 IB_QP_ALT_PATH |
835 IB_QP_ACCESS_FLAGS |
836 IB_QP_PKEY_INDEX |
837 IB_QP_MIN_RNR_TIMER |
838 IB_QP_PATH_MIG_STATE),
839 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
840 IB_QP_QKEY),
841 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
842 IB_QP_QKEY),
843 }
844 }
845 },
846 [IB_QPS_SQE] = {
847 [IB_QPS_RESET] = { .valid = 1 },
848 [IB_QPS_ERR] = { .valid = 1 },
849 [IB_QPS_RTS] = {
850 .valid = 1,
851 .opt_param = {
852 [IB_QPT_UD] = (IB_QP_CUR_STATE |
853 IB_QP_QKEY),
854 [IB_QPT_UC] = (IB_QP_CUR_STATE |
855 IB_QP_ACCESS_FLAGS),
856 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
857 IB_QP_QKEY),
858 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
859 IB_QP_QKEY),
860 }
861 }
862 },
863 [IB_QPS_ERR] = {
864 [IB_QPS_RESET] = { .valid = 1 },
865 [IB_QPS_ERR] = { .valid = 1 }
866 }
867 };
868
ib_modify_qp_is_ok(enum ib_qp_state cur_state,enum ib_qp_state next_state,enum ib_qp_type type,enum ib_qp_attr_mask mask,enum rdma_link_layer ll)869 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
870 enum ib_qp_type type, enum ib_qp_attr_mask mask,
871 enum rdma_link_layer ll)
872 {
873 enum ib_qp_attr_mask req_param, opt_param;
874
875 if (cur_state < 0 || cur_state > IB_QPS_ERR ||
876 next_state < 0 || next_state > IB_QPS_ERR)
877 return 0;
878
879 if (mask & IB_QP_CUR_STATE &&
880 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
881 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
882 return 0;
883
884 if (!qp_state_table[cur_state][next_state].valid)
885 return 0;
886
887 req_param = qp_state_table[cur_state][next_state].req_param[type];
888 opt_param = qp_state_table[cur_state][next_state].opt_param[type];
889
890 if (ll == IB_LINK_LAYER_ETHERNET) {
891 req_param |= qp_state_table[cur_state][next_state].
892 req_param_add_eth[type];
893 opt_param |= qp_state_table[cur_state][next_state].
894 opt_param_add_eth[type];
895 }
896
897 if ((mask & req_param) != req_param)
898 return 0;
899
900 if (mask & ~(req_param | opt_param | IB_QP_STATE))
901 return 0;
902
903 return 1;
904 }
905 EXPORT_SYMBOL(ib_modify_qp_is_ok);
906
ib_modify_qp(struct ib_qp * qp,struct ib_qp_attr * qp_attr,int qp_attr_mask)907 int ib_modify_qp(struct ib_qp *qp,
908 struct ib_qp_attr *qp_attr,
909 int qp_attr_mask)
910 {
911 int ret;
912
913 ret = qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
914 if (!ret && (qp_attr_mask & IB_QP_PORT))
915 qp->port_num = qp_attr->port_num;
916
917 return ret;
918 }
919 EXPORT_SYMBOL(ib_modify_qp);
920
ib_query_qp(struct ib_qp * qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)921 int ib_query_qp(struct ib_qp *qp,
922 struct ib_qp_attr *qp_attr,
923 int qp_attr_mask,
924 struct ib_qp_init_attr *qp_init_attr)
925 {
926 return qp->device->query_qp ?
927 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
928 -ENOSYS;
929 }
930 EXPORT_SYMBOL(ib_query_qp);
931
ib_close_qp(struct ib_qp * qp)932 int ib_close_qp(struct ib_qp *qp)
933 {
934 struct ib_qp *real_qp;
935 unsigned long flags;
936
937 real_qp = qp->real_qp;
938 if (real_qp == qp)
939 return -EINVAL;
940
941 spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
942 list_del(&qp->open_list);
943 spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
944
945 atomic_dec(&real_qp->usecnt);
946 kfree(qp);
947
948 return 0;
949 }
950 EXPORT_SYMBOL(ib_close_qp);
951
__ib_destroy_shared_qp(struct ib_qp * qp)952 static int __ib_destroy_shared_qp(struct ib_qp *qp)
953 {
954 struct ib_xrcd *xrcd;
955 struct ib_qp *real_qp;
956 int ret;
957
958 real_qp = qp->real_qp;
959 xrcd = real_qp->xrcd;
960
961 mutex_lock(&xrcd->tgt_qp_mutex);
962 ib_close_qp(qp);
963 if (atomic_read(&real_qp->usecnt) == 0)
964 list_del(&real_qp->xrcd_list);
965 else
966 real_qp = NULL;
967 mutex_unlock(&xrcd->tgt_qp_mutex);
968
969 if (real_qp) {
970 ret = ib_destroy_qp(real_qp);
971 if (!ret)
972 atomic_dec(&xrcd->usecnt);
973 else
974 __ib_insert_xrcd_qp(xrcd, real_qp);
975 }
976
977 return 0;
978 }
979
ib_destroy_qp(struct ib_qp * qp)980 int ib_destroy_qp(struct ib_qp *qp)
981 {
982 struct ib_pd *pd;
983 struct ib_cq *scq, *rcq;
984 struct ib_srq *srq;
985 int ret;
986
987 if (atomic_read(&qp->usecnt))
988 return -EBUSY;
989
990 if (qp->real_qp != qp)
991 return __ib_destroy_shared_qp(qp);
992
993 pd = qp->pd;
994 scq = qp->send_cq;
995 rcq = qp->recv_cq;
996 srq = qp->srq;
997
998 ret = qp->device->destroy_qp(qp);
999 if (!ret) {
1000 if (pd)
1001 atomic_dec(&pd->usecnt);
1002 if (scq)
1003 atomic_dec(&scq->usecnt);
1004 if (rcq)
1005 atomic_dec(&rcq->usecnt);
1006 if (srq)
1007 atomic_dec(&srq->usecnt);
1008 }
1009
1010 return ret;
1011 }
1012 EXPORT_SYMBOL(ib_destroy_qp);
1013
1014 /* Completion queues */
1015
ib_create_cq(struct ib_device * device,ib_comp_handler comp_handler,void (* event_handler)(struct ib_event *,void *),void * cq_context,int cqe,int comp_vector)1016 struct ib_cq *ib_create_cq(struct ib_device *device,
1017 ib_comp_handler comp_handler,
1018 void (*event_handler)(struct ib_event *, void *),
1019 void *cq_context, int cqe, int comp_vector)
1020 {
1021 struct ib_cq *cq;
1022 struct ib_cq_init_attr attr = {
1023 .cqe = cqe,
1024 .comp_vector = comp_vector,
1025 .flags = 0,
1026 };
1027
1028 cq = device->create_cq(device, &attr, NULL, NULL);
1029
1030 if (!IS_ERR(cq)) {
1031 cq->device = device;
1032 cq->uobject = NULL;
1033 cq->comp_handler = comp_handler;
1034 cq->event_handler = event_handler;
1035 cq->cq_context = cq_context;
1036 atomic_set(&cq->usecnt, 0);
1037 }
1038
1039 return cq;
1040 }
1041 EXPORT_SYMBOL(ib_create_cq);
1042
ib_modify_cq(struct ib_cq * cq,struct ib_cq_attr * cq_attr,int cq_attr_mask)1043 int ib_modify_cq(struct ib_cq *cq,
1044 struct ib_cq_attr *cq_attr,
1045 int cq_attr_mask)
1046 {
1047 return cq->device->modify_cq ?
1048 cq->device->modify_cq(cq, cq_attr, cq_attr_mask) : -ENOSYS;
1049 }
1050 EXPORT_SYMBOL(ib_modify_cq);
1051
ib_destroy_cq(struct ib_cq * cq)1052 int ib_destroy_cq(struct ib_cq *cq)
1053 {
1054 if (atomic_read(&cq->usecnt))
1055 return -EBUSY;
1056
1057 return cq->device->destroy_cq(cq);
1058 }
1059 EXPORT_SYMBOL(ib_destroy_cq);
1060
ib_resize_cq(struct ib_cq * cq,int cqe)1061 int ib_resize_cq(struct ib_cq *cq, int cqe)
1062 {
1063 return cq->device->resize_cq ?
1064 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
1065 }
1066 EXPORT_SYMBOL(ib_resize_cq);
1067
1068 /* Memory regions */
1069
ib_get_dma_mr(struct ib_pd * pd,int mr_access_flags)1070 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
1071 {
1072 struct ib_mr *mr;
1073 int err;
1074
1075 err = ib_check_mr_access(mr_access_flags);
1076 if (err)
1077 return ERR_PTR(err);
1078
1079 mr = pd->device->get_dma_mr(pd, mr_access_flags);
1080
1081 if (!IS_ERR(mr)) {
1082 mr->device = pd->device;
1083 mr->pd = pd;
1084 mr->uobject = NULL;
1085 atomic_inc(&pd->usecnt);
1086 atomic_set(&mr->usecnt, 0);
1087 }
1088
1089 return mr;
1090 }
1091 EXPORT_SYMBOL(ib_get_dma_mr);
1092
ib_reg_phys_mr(struct ib_pd * pd,struct ib_phys_buf * phys_buf_array,int num_phys_buf,int mr_access_flags,u64 * iova_start)1093 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
1094 struct ib_phys_buf *phys_buf_array,
1095 int num_phys_buf,
1096 int mr_access_flags,
1097 u64 *iova_start)
1098 {
1099 struct ib_mr *mr;
1100 int err;
1101
1102 err = ib_check_mr_access(mr_access_flags);
1103 if (err)
1104 return ERR_PTR(err);
1105
1106 if (!pd->device->reg_phys_mr)
1107 return ERR_PTR(-ENOSYS);
1108
1109 mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
1110 mr_access_flags, iova_start);
1111
1112 if (!IS_ERR(mr)) {
1113 mr->device = pd->device;
1114 mr->pd = pd;
1115 mr->uobject = NULL;
1116 atomic_inc(&pd->usecnt);
1117 atomic_set(&mr->usecnt, 0);
1118 }
1119
1120 return mr;
1121 }
1122 EXPORT_SYMBOL(ib_reg_phys_mr);
1123
ib_rereg_phys_mr(struct ib_mr * mr,int mr_rereg_mask,struct ib_pd * pd,struct ib_phys_buf * phys_buf_array,int num_phys_buf,int mr_access_flags,u64 * iova_start)1124 int ib_rereg_phys_mr(struct ib_mr *mr,
1125 int mr_rereg_mask,
1126 struct ib_pd *pd,
1127 struct ib_phys_buf *phys_buf_array,
1128 int num_phys_buf,
1129 int mr_access_flags,
1130 u64 *iova_start)
1131 {
1132 struct ib_pd *old_pd;
1133 int ret;
1134
1135 ret = ib_check_mr_access(mr_access_flags);
1136 if (ret)
1137 return ret;
1138
1139 if (!mr->device->rereg_phys_mr)
1140 return -ENOSYS;
1141
1142 if (atomic_read(&mr->usecnt))
1143 return -EBUSY;
1144
1145 old_pd = mr->pd;
1146
1147 ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
1148 phys_buf_array, num_phys_buf,
1149 mr_access_flags, iova_start);
1150
1151 if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
1152 atomic_dec(&old_pd->usecnt);
1153 atomic_inc(&pd->usecnt);
1154 }
1155
1156 return ret;
1157 }
1158 EXPORT_SYMBOL(ib_rereg_phys_mr);
1159
ib_query_mr(struct ib_mr * mr,struct ib_mr_attr * mr_attr)1160 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
1161 {
1162 return mr->device->query_mr ?
1163 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
1164 }
1165 EXPORT_SYMBOL(ib_query_mr);
1166
ib_dereg_mr(struct ib_mr * mr)1167 int ib_dereg_mr(struct ib_mr *mr)
1168 {
1169 struct ib_pd *pd;
1170 int ret;
1171
1172 if (atomic_read(&mr->usecnt))
1173 return -EBUSY;
1174
1175 pd = mr->pd;
1176 ret = mr->device->dereg_mr(mr);
1177 if (!ret)
1178 atomic_dec(&pd->usecnt);
1179
1180 return ret;
1181 }
1182 EXPORT_SYMBOL(ib_dereg_mr);
1183
ib_create_mr(struct ib_pd * pd,struct ib_mr_init_attr * mr_init_attr)1184 struct ib_mr *ib_create_mr(struct ib_pd *pd,
1185 struct ib_mr_init_attr *mr_init_attr)
1186 {
1187 struct ib_mr *mr;
1188
1189 if (!pd->device->create_mr)
1190 return ERR_PTR(-ENOSYS);
1191
1192 mr = pd->device->create_mr(pd, mr_init_attr);
1193
1194 if (!IS_ERR(mr)) {
1195 mr->device = pd->device;
1196 mr->pd = pd;
1197 mr->uobject = NULL;
1198 atomic_inc(&pd->usecnt);
1199 atomic_set(&mr->usecnt, 0);
1200 }
1201
1202 return mr;
1203 }
1204 EXPORT_SYMBOL(ib_create_mr);
1205
ib_destroy_mr(struct ib_mr * mr)1206 int ib_destroy_mr(struct ib_mr *mr)
1207 {
1208 struct ib_pd *pd;
1209 int ret;
1210
1211 if (atomic_read(&mr->usecnt))
1212 return -EBUSY;
1213
1214 pd = mr->pd;
1215 ret = mr->device->destroy_mr(mr);
1216 if (!ret)
1217 atomic_dec(&pd->usecnt);
1218
1219 return ret;
1220 }
1221 EXPORT_SYMBOL(ib_destroy_mr);
1222
ib_alloc_fast_reg_mr(struct ib_pd * pd,int max_page_list_len)1223 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1224 {
1225 struct ib_mr *mr;
1226
1227 if (!pd->device->alloc_fast_reg_mr)
1228 return ERR_PTR(-ENOSYS);
1229
1230 mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
1231
1232 if (!IS_ERR(mr)) {
1233 mr->device = pd->device;
1234 mr->pd = pd;
1235 mr->uobject = NULL;
1236 atomic_inc(&pd->usecnt);
1237 atomic_set(&mr->usecnt, 0);
1238 }
1239
1240 return mr;
1241 }
1242 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
1243
ib_alloc_fast_reg_page_list(struct ib_device * device,int max_page_list_len)1244 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
1245 int max_page_list_len)
1246 {
1247 struct ib_fast_reg_page_list *page_list;
1248
1249 if (!device->alloc_fast_reg_page_list)
1250 return ERR_PTR(-ENOSYS);
1251
1252 page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
1253
1254 if (!IS_ERR(page_list)) {
1255 page_list->device = device;
1256 page_list->max_page_list_len = max_page_list_len;
1257 }
1258
1259 return page_list;
1260 }
1261 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
1262
ib_free_fast_reg_page_list(struct ib_fast_reg_page_list * page_list)1263 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1264 {
1265 page_list->device->free_fast_reg_page_list(page_list);
1266 }
1267 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
1268
1269 /* Memory windows */
1270
ib_alloc_mw(struct ib_pd * pd,enum ib_mw_type type)1271 struct ib_mw *ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
1272 {
1273 struct ib_mw *mw;
1274
1275 if (!pd->device->alloc_mw)
1276 return ERR_PTR(-ENOSYS);
1277
1278 mw = pd->device->alloc_mw(pd, type);
1279 if (!IS_ERR(mw)) {
1280 mw->device = pd->device;
1281 mw->pd = pd;
1282 mw->uobject = NULL;
1283 mw->type = type;
1284 atomic_inc(&pd->usecnt);
1285 }
1286
1287 return mw;
1288 }
1289 EXPORT_SYMBOL(ib_alloc_mw);
1290
ib_dealloc_mw(struct ib_mw * mw)1291 int ib_dealloc_mw(struct ib_mw *mw)
1292 {
1293 struct ib_pd *pd;
1294 int ret;
1295
1296 pd = mw->pd;
1297 ret = mw->device->dealloc_mw(mw);
1298 if (!ret)
1299 atomic_dec(&pd->usecnt);
1300
1301 return ret;
1302 }
1303 EXPORT_SYMBOL(ib_dealloc_mw);
1304
1305 /* "Fast" memory regions */
1306
ib_alloc_fmr(struct ib_pd * pd,int mr_access_flags,struct ib_fmr_attr * fmr_attr)1307 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1308 int mr_access_flags,
1309 struct ib_fmr_attr *fmr_attr)
1310 {
1311 struct ib_fmr *fmr;
1312
1313 if (!pd->device->alloc_fmr)
1314 return ERR_PTR(-ENOSYS);
1315
1316 fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1317 if (!IS_ERR(fmr)) {
1318 fmr->device = pd->device;
1319 fmr->pd = pd;
1320 atomic_inc(&pd->usecnt);
1321 }
1322
1323 return fmr;
1324 }
1325 EXPORT_SYMBOL(ib_alloc_fmr);
1326
ib_unmap_fmr(struct list_head * fmr_list)1327 int ib_unmap_fmr(struct list_head *fmr_list)
1328 {
1329 struct ib_fmr *fmr;
1330
1331 if (list_empty(fmr_list))
1332 return 0;
1333
1334 fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1335 return fmr->device->unmap_fmr(fmr_list);
1336 }
1337 EXPORT_SYMBOL(ib_unmap_fmr);
1338
ib_dealloc_fmr(struct ib_fmr * fmr)1339 int ib_dealloc_fmr(struct ib_fmr *fmr)
1340 {
1341 struct ib_pd *pd;
1342 int ret;
1343
1344 pd = fmr->pd;
1345 ret = fmr->device->dealloc_fmr(fmr);
1346 if (!ret)
1347 atomic_dec(&pd->usecnt);
1348
1349 return ret;
1350 }
1351 EXPORT_SYMBOL(ib_dealloc_fmr);
1352
1353 /* Multicast groups */
1354
ib_attach_mcast(struct ib_qp * qp,union ib_gid * gid,u16 lid)1355 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1356 {
1357 int ret;
1358
1359 if (!qp->device->attach_mcast)
1360 return -ENOSYS;
1361
1362 switch (rdma_node_get_transport(qp->device->node_type)) {
1363 case RDMA_TRANSPORT_IB:
1364 if ((gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) &&
1365 qp->qp_type != IB_QPT_RAW_PACKET)
1366 return -EINVAL;
1367 break;
1368 case RDMA_TRANSPORT_IWARP:
1369 case RDMA_TRANSPORT_SCIF:
1370 if (qp->qp_type != IB_QPT_RAW_PACKET)
1371 return -EINVAL;
1372 break;
1373 }
1374
1375 ret = qp->device->attach_mcast(qp, gid, lid);
1376 if (!ret)
1377 atomic_inc(&qp->usecnt);
1378 return ret;
1379 }
1380 EXPORT_SYMBOL(ib_attach_mcast);
1381
ib_detach_mcast(struct ib_qp * qp,union ib_gid * gid,u16 lid)1382 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1383 {
1384 int ret;
1385
1386 if (!qp->device->detach_mcast)
1387 return -ENOSYS;
1388
1389 switch (rdma_node_get_transport(qp->device->node_type)) {
1390 case RDMA_TRANSPORT_IB:
1391 if ((gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) &&
1392 qp->qp_type != IB_QPT_RAW_PACKET)
1393 return -EINVAL;
1394 break;
1395 case RDMA_TRANSPORT_IWARP:
1396 case RDMA_TRANSPORT_SCIF:
1397
1398 if (qp->qp_type != IB_QPT_RAW_PACKET)
1399 return -EINVAL;
1400 break;
1401 }
1402
1403 ret = qp->device->detach_mcast(qp, gid, lid);
1404 if (!ret)
1405 atomic_dec(&qp->usecnt);
1406 return ret;
1407 }
1408 EXPORT_SYMBOL(ib_detach_mcast);
1409
ib_alloc_xrcd(struct ib_device * device)1410 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1411 {
1412 struct ib_xrcd *xrcd;
1413
1414 if (!device->alloc_xrcd)
1415 return ERR_PTR(-ENOSYS);
1416
1417 xrcd = device->alloc_xrcd(device, NULL, NULL);
1418 if (!IS_ERR(xrcd)) {
1419 xrcd->device = device;
1420 xrcd->inode = NULL;
1421 atomic_set(&xrcd->usecnt, 0);
1422 mutex_init(&xrcd->tgt_qp_mutex);
1423 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1424 }
1425
1426 return xrcd;
1427 }
1428 EXPORT_SYMBOL(ib_alloc_xrcd);
1429
ib_dealloc_xrcd(struct ib_xrcd * xrcd)1430 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1431 {
1432 struct ib_qp *qp;
1433 int ret;
1434
1435 if (atomic_read(&xrcd->usecnt))
1436 return -EBUSY;
1437
1438 while (!list_empty(&xrcd->tgt_qp_list)) {
1439 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1440 ret = ib_destroy_qp(qp);
1441 if (ret)
1442 return ret;
1443 }
1444
1445 return xrcd->device->dealloc_xrcd(xrcd);
1446 }
1447 EXPORT_SYMBOL(ib_dealloc_xrcd);
1448
ib_create_flow(struct ib_qp * qp,struct ib_flow_attr * flow_attr,int domain)1449 struct ib_flow *ib_create_flow(struct ib_qp *qp,
1450 struct ib_flow_attr *flow_attr,
1451 int domain)
1452 {
1453 struct ib_flow *flow_id;
1454 if (!qp->device->create_flow)
1455 return ERR_PTR(-ENOSYS);
1456
1457 flow_id = qp->device->create_flow(qp, flow_attr, domain);
1458 if (!IS_ERR(flow_id))
1459 atomic_inc(&qp->usecnt);
1460 return flow_id;
1461 }
1462 EXPORT_SYMBOL(ib_create_flow);
1463
ib_destroy_flow(struct ib_flow * flow_id)1464 int ib_destroy_flow(struct ib_flow *flow_id)
1465 {
1466 int err;
1467 struct ib_qp *qp;
1468
1469 if (!flow_id)
1470 return -EINVAL;
1471 qp = flow_id->qp;
1472 if (!qp->device->destroy_flow)
1473 return -ENOSYS;
1474 err = qp->device->destroy_flow(flow_id);
1475 if (!err)
1476 atomic_dec(&qp->usecnt);
1477 return err;
1478 }
1479 EXPORT_SYMBOL(ib_destroy_flow);
1480
ib_create_dct(struct ib_pd * pd,struct ib_dct_init_attr * attr,struct ib_udata * udata)1481 struct ib_dct *ib_create_dct(struct ib_pd *pd, struct ib_dct_init_attr *attr,
1482 struct ib_udata *udata)
1483 {
1484 struct ib_dct *dct;
1485
1486 if (!pd->device->exp_create_dct)
1487 return ERR_PTR(-ENOSYS);
1488
1489 dct = pd->device->exp_create_dct(pd, attr, udata);
1490 if (!IS_ERR(dct)) {
1491 dct->pd = pd;
1492 dct->srq = attr->srq;
1493 dct->cq = attr->cq;
1494 atomic_inc(&dct->srq->usecnt);
1495 atomic_inc(&dct->cq->usecnt);
1496 atomic_inc(&dct->pd->usecnt);
1497 }
1498
1499 return dct;
1500 }
1501 EXPORT_SYMBOL(ib_create_dct);
1502
ib_destroy_dct(struct ib_dct * dct)1503 int ib_destroy_dct(struct ib_dct *dct)
1504 {
1505 int err;
1506
1507 if (!dct->device->exp_destroy_dct)
1508 return -ENOSYS;
1509
1510 err = dct->device->exp_destroy_dct(dct);
1511 if (!err) {
1512 atomic_dec(&dct->srq->usecnt);
1513 atomic_dec(&dct->cq->usecnt);
1514 atomic_dec(&dct->pd->usecnt);
1515 }
1516
1517 return err;
1518 }
1519 EXPORT_SYMBOL(ib_destroy_dct);
1520
ib_query_dct(struct ib_dct * dct,struct ib_dct_attr * attr)1521 int ib_query_dct(struct ib_dct *dct, struct ib_dct_attr *attr)
1522 {
1523 if (!dct->device->exp_query_dct)
1524 return -ENOSYS;
1525
1526 return dct->device->exp_query_dct(dct, attr);
1527 }
1528 EXPORT_SYMBOL(ib_query_dct);
1529
ib_check_mr_status(struct ib_mr * mr,u32 check_mask,struct ib_mr_status * mr_status)1530 int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
1531 struct ib_mr_status *mr_status)
1532 {
1533 return mr->device->check_mr_status ?
1534 mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS;
1535 }
1536 EXPORT_SYMBOL(ib_check_mr_status);
1537