1 /*
2 * Copyright (c) 2005 Voltaire Inc. All rights reserved.
3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36 #include <linux/completion.h>
37 #include <linux/in.h>
38 #include <linux/in6.h>
39 #include <linux/mutex.h>
40 #include <linux/random.h>
41 #include <linux/idr.h>
42 #include <linux/inetdevice.h>
43 #include <linux/slab.h>
44 #include <linux/module.h>
45 #include <linux/string.h>
46 #include <net/route.h>
47
48 #include <net/tcp.h>
49 #include <net/ipv6.h>
50
51 #include <rdma/rdma_cm.h>
52 #include <rdma/rdma_cm_ib.h>
53 #include <rdma/ib_cache.h>
54 #include <rdma/ib_cm.h>
55 #include <rdma/ib_sa.h>
56 #include <rdma/iw_cm.h>
57
58 MODULE_AUTHOR("Sean Hefty");
59 MODULE_DESCRIPTION("Generic RDMA CM Agent");
60 MODULE_LICENSE("Dual BSD/GPL");
61
62 #define CMA_CM_RESPONSE_TIMEOUT 20
63 #define CMA_MAX_CM_RETRIES 15
64 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
65 #define CMA_IBOE_PACKET_LIFETIME 18
66
67 static int cma_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
68 module_param_named(cma_response_timeout, cma_response_timeout, int, 0644);
69 MODULE_PARM_DESC(cma_response_timeout, "CMA_CM_RESPONSE_TIMEOUT (default=20)");
70
71 static int def_prec2sl = 3;
72 module_param_named(def_prec2sl, def_prec2sl, int, 0644);
73 MODULE_PARM_DESC(def_prec2sl, "Default value for SL priority with RoCE. Valid values 0 - 7");
74
75 static int unify_tcp_port_space = 1;
76 module_param(unify_tcp_port_space, int, 0644);
77 MODULE_PARM_DESC(unify_tcp_port_space, "Unify the host TCP and RDMA port "
78 "space allocation (default=1)");
79
80 static int debug_level = 0;
81 #define cma_pr(level, priv, format, arg...) \
82 printk(level "CMA: %p: %s: " format, ((struct rdma_id_priv *) priv) , __func__, ## arg)
83
84 #define cma_dbg(priv, format, arg...) \
85 do { if (debug_level) cma_pr(KERN_DEBUG, priv, format, ## arg); } while (0)
86
87 #define cma_warn(priv, format, arg...) \
88 cma_pr(KERN_WARNING, priv, format, ## arg)
89
90 #define CMA_GID_FMT "%2.2x%2.2x:%2.2x%2.2x"
91 #define CMA_GID_RAW_ARG(gid) ((u8 *)(gid))[12],\
92 ((u8 *)(gid))[13],\
93 ((u8 *)(gid))[14],\
94 ((u8 *)(gid))[15]
95
96 #define CMA_GID_ARG(gid) CMA_GID_RAW_ARG((gid).raw)
97 #define cma_debug_path(priv, pfx, p) \
98 cma_dbg(priv, pfx "sgid=" CMA_GID_FMT ",dgid=" \
99 CMA_GID_FMT "\n", CMA_GID_ARG(p.sgid), \
100 CMA_GID_ARG(p.dgid))
101
102 #define cma_debug_gid(priv, g) \
103 cma_dbg(priv, "gid=" CMA_GID_FMT "\n", CMA_GID_ARG(g)
104
105 module_param_named(debug_level, debug_level, int, 0644);
106 MODULE_PARM_DESC(debug_level, "debug level default=0");
107
108 static void cma_add_one(struct ib_device *device);
109 static void cma_remove_one(struct ib_device *device);
110
111 static struct ib_client cma_client = {
112 .name = "cma",
113 .add = cma_add_one,
114 .remove = cma_remove_one
115 };
116
117 static struct ib_sa_client sa_client;
118 static struct rdma_addr_client addr_client;
119 static LIST_HEAD(dev_list);
120 static LIST_HEAD(listen_any_list);
121 static DEFINE_MUTEX(lock);
122 static struct workqueue_struct *cma_wq;
123 static struct workqueue_struct *cma_free_wq;
124 static DEFINE_IDR(sdp_ps);
125 static DEFINE_IDR(tcp_ps);
126 static DEFINE_IDR(udp_ps);
127 static DEFINE_IDR(ipoib_ps);
128 static DEFINE_IDR(ib_ps);
129
130 struct cma_device {
131 struct list_head list;
132 struct ib_device *device;
133 struct completion comp;
134 atomic_t refcount;
135 struct list_head id_list;
136 };
137
138 struct rdma_bind_list {
139 struct idr *ps;
140 struct hlist_head owners;
141 unsigned short port;
142 };
143
144 enum {
145 CMA_OPTION_AFONLY,
146 };
147
148 /*
149 * Device removal can occur at anytime, so we need extra handling to
150 * serialize notifying the user of device removal with other callbacks.
151 * We do this by disabling removal notification while a callback is in process,
152 * and reporting it after the callback completes.
153 */
154 struct rdma_id_private {
155 struct rdma_cm_id id;
156
157 struct rdma_bind_list *bind_list;
158 struct socket *sock;
159 struct hlist_node node;
160 struct list_head list; /* listen_any_list or cma_device.list */
161 struct list_head listen_list; /* per device listens */
162 struct cma_device *cma_dev;
163 struct list_head mc_list;
164
165 int internal_id;
166 enum rdma_cm_state state;
167 spinlock_t lock;
168 spinlock_t cm_lock;
169 struct mutex qp_mutex;
170
171 struct completion comp;
172 atomic_t refcount;
173 struct mutex handler_mutex;
174 struct work_struct work; /* garbage coll */
175
176 int backlog;
177 int timeout_ms;
178 struct ib_sa_query *query;
179 int query_id;
180 union {
181 struct ib_cm_id *ib;
182 struct iw_cm_id *iw;
183 } cm_id;
184
185 u32 seq_num;
186 u32 qkey;
187 u32 qp_num;
188 pid_t owner;
189 u32 options;
190 u8 srq;
191 u8 tos;
192 u8 reuseaddr;
193 u8 afonly;
194 int qp_timeout;
195 /* cache for mc record params */
196 struct ib_sa_mcmember_rec rec;
197 int is_valid_rec;
198 };
199
200 struct cma_multicast {
201 struct rdma_id_private *id_priv;
202 union {
203 struct ib_sa_multicast *ib;
204 } multicast;
205 struct list_head list;
206 void *context;
207 struct sockaddr_storage addr;
208 struct kref mcref;
209 };
210
211 struct cma_work {
212 struct work_struct work;
213 struct rdma_id_private *id;
214 enum rdma_cm_state old_state;
215 enum rdma_cm_state new_state;
216 struct rdma_cm_event event;
217 };
218
219 struct cma_ndev_work {
220 struct work_struct work;
221 struct rdma_id_private *id;
222 struct rdma_cm_event event;
223 };
224
225 struct iboe_mcast_work {
226 struct work_struct work;
227 struct rdma_id_private *id;
228 struct cma_multicast *mc;
229 };
230
231 union cma_ip_addr {
232 struct in6_addr ip6;
233 struct {
234 __be32 pad[3];
235 __be32 addr;
236 } ip4;
237 };
238
239 struct cma_hdr {
240 u8 cma_version;
241 u8 ip_version; /* IP version: 7:4 */
242 __be16 port;
243 union cma_ip_addr src_addr;
244 union cma_ip_addr dst_addr;
245 };
246
247 struct sdp_hh {
248 u8 bsdh[16];
249 u8 sdp_version; /* Major version: 7:4 */
250 u8 ip_version; /* IP version: 7:4 */
251 u8 sdp_specific1[10];
252 __be16 port;
253 __be16 sdp_specific2;
254 union cma_ip_addr src_addr;
255 union cma_ip_addr dst_addr;
256 };
257
258 struct sdp_hah {
259 u8 bsdh[16];
260 u8 sdp_version;
261 };
262
263 #define CMA_VERSION 0x00
264 #define SDP_MAJ_VERSION 0x2
265
cma_comp(struct rdma_id_private * id_priv,enum rdma_cm_state comp)266 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
267 {
268 unsigned long flags;
269 int ret;
270
271 spin_lock_irqsave(&id_priv->lock, flags);
272 ret = (id_priv->state == comp);
273 spin_unlock_irqrestore(&id_priv->lock, flags);
274 return ret;
275 }
276
cma_comp_exch(struct rdma_id_private * id_priv,enum rdma_cm_state comp,enum rdma_cm_state exch)277 static int cma_comp_exch(struct rdma_id_private *id_priv,
278 enum rdma_cm_state comp, enum rdma_cm_state exch)
279 {
280 unsigned long flags;
281 int ret;
282
283 spin_lock_irqsave(&id_priv->lock, flags);
284 if ((ret = (id_priv->state == comp)))
285 id_priv->state = exch;
286 spin_unlock_irqrestore(&id_priv->lock, flags);
287 return ret;
288 }
289
cma_exch(struct rdma_id_private * id_priv,enum rdma_cm_state exch)290 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
291 enum rdma_cm_state exch)
292 {
293 unsigned long flags;
294 enum rdma_cm_state old;
295
296 spin_lock_irqsave(&id_priv->lock, flags);
297 old = id_priv->state;
298 id_priv->state = exch;
299 spin_unlock_irqrestore(&id_priv->lock, flags);
300 return old;
301 }
302
cma_get_ip_ver(struct cma_hdr * hdr)303 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
304 {
305 return hdr->ip_version >> 4;
306 }
307
cma_set_ip_ver(struct cma_hdr * hdr,u8 ip_ver)308 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
309 {
310 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
311 }
312
sdp_get_majv(u8 sdp_version)313 static inline u8 sdp_get_majv(u8 sdp_version)
314 {
315 return sdp_version >> 4;
316 }
317
sdp_get_ip_ver(struct sdp_hh * hh)318 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
319 {
320 return hh->ip_version >> 4;
321 }
322
sdp_set_ip_ver(struct sdp_hh * hh,u8 ip_ver)323 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
324 {
325 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
326 }
327
cma_attach_to_dev(struct rdma_id_private * id_priv,struct cma_device * cma_dev)328 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
329 struct cma_device *cma_dev)
330 {
331 atomic_inc(&cma_dev->refcount);
332 id_priv->cma_dev = cma_dev;
333 id_priv->id.device = cma_dev->device;
334 id_priv->id.route.addr.dev_addr.transport =
335 rdma_node_get_transport(cma_dev->device->node_type);
336 list_add_tail(&id_priv->list, &cma_dev->id_list);
337 }
338
cma_deref_dev(struct cma_device * cma_dev)339 static inline void cma_deref_dev(struct cma_device *cma_dev)
340 {
341 if (atomic_dec_and_test(&cma_dev->refcount))
342 complete(&cma_dev->comp);
343 }
344
release_mc(struct kref * kref)345 static inline void release_mc(struct kref *kref)
346 {
347 struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
348
349 kfree(mc->multicast.ib);
350 kfree(mc);
351 }
352
cma_release_dev(struct rdma_id_private * id_priv)353 static void cma_release_dev(struct rdma_id_private *id_priv)
354 {
355 mutex_lock(&lock);
356 list_del(&id_priv->list);
357 cma_deref_dev(id_priv->cma_dev);
358 id_priv->cma_dev = NULL;
359 mutex_unlock(&lock);
360 }
361
cma_set_qkey(struct rdma_id_private * id_priv)362 static int cma_set_qkey(struct rdma_id_private *id_priv)
363 {
364 struct ib_sa_mcmember_rec rec;
365 int ret = 0;
366
367 if (id_priv->qkey)
368 return 0;
369
370 switch (id_priv->id.ps) {
371 case RDMA_PS_UDP:
372 id_priv->qkey = RDMA_UDP_QKEY;
373 break;
374 case RDMA_PS_IPOIB:
375 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
376 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
377 id_priv->id.port_num, &rec.mgid,
378 &rec);
379 if (!ret)
380 id_priv->qkey = be32_to_cpu(rec.qkey);
381 break;
382 default:
383 break;
384 }
385 return ret;
386 }
387
find_gid_port(struct ib_device * device,union ib_gid * gid,u8 port_num)388 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
389 {
390 int i;
391 int err;
392 struct ib_port_attr props;
393 union ib_gid tmp;
394
395 err = ib_query_port(device, port_num, &props);
396 if (err)
397 return 1;
398
399 for (i = 0; i < props.gid_tbl_len; ++i) {
400 err = ib_query_gid(device, port_num, i, &tmp);
401 if (err)
402 return 1;
403 if (!memcmp(&tmp, gid, sizeof tmp))
404 return 0;
405 }
406
407 return -EAGAIN;
408 }
409
cma_acquire_dev(struct rdma_id_private * id_priv)410 static int cma_acquire_dev(struct rdma_id_private *id_priv)
411 {
412 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
413 struct cma_device *cma_dev;
414 union ib_gid gid, iboe_gid;
415 int ret = -ENODEV;
416 u8 port;
417 enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
418 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
419
420 if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
421 id_priv->id.ps == RDMA_PS_IPOIB)
422 return -EINVAL;
423
424 mutex_lock(&lock);
425 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
426 &iboe_gid);
427
428 memcpy(&gid, dev_addr->src_dev_addr +
429 rdma_addr_gid_offset(dev_addr), sizeof gid);
430 list_for_each_entry(cma_dev, &dev_list, list) {
431 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
432 if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
433 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
434 rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
435 ret = find_gid_port(cma_dev->device, &iboe_gid, port);
436 else
437 ret = find_gid_port(cma_dev->device, &gid, port);
438
439 if (!ret) {
440 id_priv->id.port_num = port;
441 goto out;
442 } else if (ret == 1)
443 break;
444 }
445 }
446 }
447
448 out:
449 if (!ret)
450 cma_attach_to_dev(id_priv, cma_dev);
451
452 mutex_unlock(&lock);
453 return ret;
454 }
455
cma_deref_id(struct rdma_id_private * id_priv)456 static void cma_deref_id(struct rdma_id_private *id_priv)
457 {
458 if (atomic_dec_and_test(&id_priv->refcount))
459 complete(&id_priv->comp);
460 }
461
cma_disable_callback(struct rdma_id_private * id_priv,enum rdma_cm_state state)462 static int cma_disable_callback(struct rdma_id_private *id_priv,
463 enum rdma_cm_state state)
464 {
465 mutex_lock(&id_priv->handler_mutex);
466 if (id_priv->state != state) {
467 mutex_unlock(&id_priv->handler_mutex);
468 return -EINVAL;
469 }
470 return 0;
471 }
472
rdma_create_id(rdma_cm_event_handler event_handler,void * context,enum rdma_port_space ps,enum ib_qp_type qp_type)473 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
474 void *context, enum rdma_port_space ps,
475 enum ib_qp_type qp_type)
476 {
477 struct rdma_id_private *id_priv;
478
479 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
480 if (!id_priv)
481 return ERR_PTR(-ENOMEM);
482
483 id_priv->owner = curthread->td_proc->p_pid;
484 id_priv->state = RDMA_CM_IDLE;
485 id_priv->id.context = context;
486 id_priv->id.event_handler = event_handler;
487 id_priv->id.ps = ps;
488 id_priv->id.qp_type = qp_type;
489 spin_lock_init(&id_priv->lock);
490 spin_lock_init(&id_priv->cm_lock);
491 mutex_init(&id_priv->qp_mutex);
492 init_completion(&id_priv->comp);
493 atomic_set(&id_priv->refcount, 1);
494 mutex_init(&id_priv->handler_mutex);
495 INIT_LIST_HEAD(&id_priv->listen_list);
496 INIT_LIST_HEAD(&id_priv->mc_list);
497 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
498
499 return &id_priv->id;
500 }
501 EXPORT_SYMBOL(rdma_create_id);
502
cma_init_ud_qp(struct rdma_id_private * id_priv,struct ib_qp * qp)503 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
504 {
505 struct ib_qp_attr qp_attr;
506 int qp_attr_mask, ret;
507
508 qp_attr.qp_state = IB_QPS_INIT;
509 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
510 if (ret)
511 return ret;
512
513 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
514 if (ret)
515 return ret;
516
517 qp_attr.qp_state = IB_QPS_RTR;
518 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
519 if (ret)
520 return ret;
521
522 qp_attr.qp_state = IB_QPS_RTS;
523 qp_attr.sq_psn = 0;
524 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
525
526 return ret;
527 }
528
cma_init_conn_qp(struct rdma_id_private * id_priv,struct ib_qp * qp)529 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
530 {
531 struct ib_qp_attr qp_attr;
532 int qp_attr_mask, ret;
533
534 qp_attr.qp_state = IB_QPS_INIT;
535 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
536 if (ret)
537 return ret;
538
539 return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
540 }
541
rdma_create_qp(struct rdma_cm_id * id,struct ib_pd * pd,struct ib_qp_init_attr * qp_init_attr)542 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
543 struct ib_qp_init_attr *qp_init_attr)
544 {
545 struct rdma_id_private *id_priv;
546 struct ib_qp *qp;
547 int ret;
548
549 id_priv = container_of(id, struct rdma_id_private, id);
550 if (id->device != pd->device)
551 return -EINVAL;
552
553 qp = ib_create_qp(pd, qp_init_attr);
554 if (IS_ERR(qp))
555 return PTR_ERR(qp);
556
557 if (id->qp_type == IB_QPT_UD)
558 ret = cma_init_ud_qp(id_priv, qp);
559 else
560 ret = cma_init_conn_qp(id_priv, qp);
561 if (ret)
562 goto err;
563
564 id->qp = qp;
565 id_priv->qp_num = qp->qp_num;
566 id_priv->srq = (qp->srq != NULL);
567 return 0;
568 err:
569 ib_destroy_qp(qp);
570 return ret;
571 }
572 EXPORT_SYMBOL(rdma_create_qp);
573
rdma_destroy_qp(struct rdma_cm_id * id)574 void rdma_destroy_qp(struct rdma_cm_id *id)
575 {
576 struct rdma_id_private *id_priv;
577
578 id_priv = container_of(id, struct rdma_id_private, id);
579 mutex_lock(&id_priv->qp_mutex);
580 ib_destroy_qp(id_priv->id.qp);
581 id_priv->id.qp = NULL;
582 mutex_unlock(&id_priv->qp_mutex);
583 }
584 EXPORT_SYMBOL(rdma_destroy_qp);
585
cma_modify_qp_rtr(struct rdma_id_private * id_priv,struct rdma_conn_param * conn_param)586 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
587 struct rdma_conn_param *conn_param)
588 {
589 struct ib_qp_attr qp_attr;
590 int qp_attr_mask, ret;
591 union ib_gid sgid;
592
593 mutex_lock(&id_priv->qp_mutex);
594 if (!id_priv->id.qp) {
595 ret = 0;
596 goto out;
597 }
598
599 /* Need to update QP attributes from default values. */
600 qp_attr.qp_state = IB_QPS_INIT;
601 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
602 if (ret)
603 goto out;
604
605 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
606 if (ret)
607 goto out;
608
609 qp_attr.qp_state = IB_QPS_RTR;
610 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
611 if (ret)
612 goto out;
613 ret = ib_query_gid(id_priv->id.device, id_priv->id.port_num,
614 qp_attr.ah_attr.grh.sgid_index, &sgid);
615 if (ret)
616 goto out;
617
618 if (rdma_node_get_transport(id_priv->cma_dev->device->node_type)
619 == RDMA_TRANSPORT_IB &&
620 rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)
621 == IB_LINK_LAYER_ETHERNET) {
622 ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr.smac, NULL);
623
624 if (ret)
625 goto out;
626 }
627
628 if (conn_param)
629 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
630 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
631 out:
632 mutex_unlock(&id_priv->qp_mutex);
633 return ret;
634 }
635
cma_modify_qp_rts(struct rdma_id_private * id_priv,struct rdma_conn_param * conn_param)636 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
637 struct rdma_conn_param *conn_param)
638 {
639 struct ib_qp_attr qp_attr;
640 int qp_attr_mask, ret;
641
642 mutex_lock(&id_priv->qp_mutex);
643 if (!id_priv->id.qp) {
644 ret = 0;
645 goto out;
646 }
647
648 qp_attr.qp_state = IB_QPS_RTS;
649 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
650 if (ret)
651 goto out;
652
653 if (conn_param)
654 qp_attr.max_rd_atomic = conn_param->initiator_depth;
655
656 if (id_priv->qp_timeout && id_priv->id.qp->qp_type == IB_QPT_RC) {
657 qp_attr.timeout = id_priv->qp_timeout;
658 qp_attr_mask |= IB_QP_TIMEOUT;
659 }
660
661 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
662 out:
663 mutex_unlock(&id_priv->qp_mutex);
664 return ret;
665 }
666
cma_modify_qp_err(struct rdma_id_private * id_priv)667 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
668 {
669 struct ib_qp_attr qp_attr;
670 int ret;
671
672 mutex_lock(&id_priv->qp_mutex);
673 if (!id_priv->id.qp) {
674 ret = 0;
675 goto out;
676 }
677
678 qp_attr.qp_state = IB_QPS_ERR;
679 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
680 out:
681 mutex_unlock(&id_priv->qp_mutex);
682 return ret;
683 }
684
cma_ib_init_qp_attr(struct rdma_id_private * id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)685 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
686 struct ib_qp_attr *qp_attr, int *qp_attr_mask)
687 {
688 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
689 int ret;
690 u16 pkey;
691
692 if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
693 IB_LINK_LAYER_INFINIBAND)
694 pkey = ib_addr_get_pkey(dev_addr);
695 else
696 pkey = 0xffff;
697
698 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
699 pkey, &qp_attr->pkey_index);
700 if (ret)
701 return ret;
702
703 qp_attr->port_num = id_priv->id.port_num;
704 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
705
706 if (id_priv->id.qp_type == IB_QPT_UD) {
707 ret = cma_set_qkey(id_priv);
708 if (ret)
709 return ret;
710
711 qp_attr->qkey = id_priv->qkey;
712 *qp_attr_mask |= IB_QP_QKEY;
713 } else {
714 qp_attr->qp_access_flags = 0;
715 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
716 }
717 return 0;
718 }
719
rdma_init_qp_attr(struct rdma_cm_id * id,struct ib_qp_attr * qp_attr,int * qp_attr_mask)720 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
721 int *qp_attr_mask)
722 {
723 struct rdma_id_private *id_priv;
724 int ret = 0;
725
726 id_priv = container_of(id, struct rdma_id_private, id);
727 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
728 case RDMA_TRANSPORT_IB:
729 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
730 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
731 else
732 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
733 qp_attr_mask);
734 if (qp_attr->qp_state == IB_QPS_RTR)
735 qp_attr->rq_psn = id_priv->seq_num;
736 break;
737 case RDMA_TRANSPORT_IWARP:
738 case RDMA_TRANSPORT_SCIF:
739 if (!id_priv->cm_id.iw) {
740 qp_attr->qp_access_flags = 0;
741 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
742 } else
743 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
744 qp_attr_mask);
745 break;
746 default:
747 ret = -ENOSYS;
748 break;
749 }
750
751 return ret;
752 }
753 EXPORT_SYMBOL(rdma_init_qp_attr);
754
cma_zero_addr(struct sockaddr * addr)755 static inline int cma_zero_addr(struct sockaddr *addr)
756 {
757 struct in6_addr *ip6;
758
759 if (addr->sa_family == AF_INET)
760 return ipv4_is_zeronet(
761 ((struct sockaddr_in *)addr)->sin_addr.s_addr);
762 else {
763 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
764 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
765 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
766 }
767 }
768
cma_loopback_addr(struct sockaddr * addr)769 static inline int cma_loopback_addr(struct sockaddr *addr)
770 {
771 if (addr->sa_family == AF_INET)
772 return ipv4_is_loopback(
773 ((struct sockaddr_in *) addr)->sin_addr.s_addr);
774 else
775 return ipv6_addr_loopback(
776 &((struct sockaddr_in6 *) addr)->sin6_addr);
777 }
778
cma_any_addr(struct sockaddr * addr)779 static inline int cma_any_addr(struct sockaddr *addr)
780 {
781 return cma_zero_addr(addr) || cma_loopback_addr(addr);
782 }
783
cma_addr_cmp(struct sockaddr * src,struct sockaddr * dst)784 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
785 {
786 if (src->sa_family != dst->sa_family)
787 return -1;
788
789 switch (src->sa_family) {
790 case AF_INET:
791 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
792 ((struct sockaddr_in *) dst)->sin_addr.s_addr;
793 default:
794 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
795 &((struct sockaddr_in6 *) dst)->sin6_addr);
796 }
797 }
798
cma_port(struct sockaddr * addr)799 static inline __be16 cma_port(struct sockaddr *addr)
800 {
801 if (addr->sa_family == AF_INET)
802 return ((struct sockaddr_in *) addr)->sin_port;
803 else
804 return ((struct sockaddr_in6 *) addr)->sin6_port;
805 }
806
cma_any_port(struct sockaddr * addr)807 static inline int cma_any_port(struct sockaddr *addr)
808 {
809 return !cma_port(addr);
810 }
811
cma_get_net_info(void * hdr,enum rdma_port_space ps,u8 * ip_ver,__be16 * port,union cma_ip_addr ** src,union cma_ip_addr ** dst)812 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
813 u8 *ip_ver, __be16 *port,
814 union cma_ip_addr **src, union cma_ip_addr **dst)
815 {
816 switch (ps) {
817 case RDMA_PS_SDP:
818 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
819 SDP_MAJ_VERSION)
820 return -EINVAL;
821
822 *ip_ver = sdp_get_ip_ver(hdr);
823 *port = ((struct sdp_hh *) hdr)->port;
824 *src = &((struct sdp_hh *) hdr)->src_addr;
825 *dst = &((struct sdp_hh *) hdr)->dst_addr;
826 break;
827 default:
828 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
829 return -EINVAL;
830
831 *ip_ver = cma_get_ip_ver(hdr);
832 *port = ((struct cma_hdr *) hdr)->port;
833 *src = &((struct cma_hdr *) hdr)->src_addr;
834 *dst = &((struct cma_hdr *) hdr)->dst_addr;
835 break;
836 }
837
838 if (*ip_ver != 4 && *ip_ver != 6)
839 return -EINVAL;
840 return 0;
841 }
842
cma_save_net_info(struct rdma_addr * addr,struct rdma_addr * listen_addr,u8 ip_ver,__be16 port,union cma_ip_addr * src,union cma_ip_addr * dst)843 static void cma_save_net_info(struct rdma_addr *addr,
844 struct rdma_addr *listen_addr,
845 u8 ip_ver, __be16 port,
846 union cma_ip_addr *src, union cma_ip_addr *dst)
847 {
848 struct sockaddr_in *listen4, *ip4;
849 struct sockaddr_in6 *listen6, *ip6;
850
851 switch (ip_ver) {
852 case 4:
853 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
854 ip4 = (struct sockaddr_in *) &addr->src_addr;
855 ip4->sin_family = listen4->sin_family;
856 ip4->sin_addr.s_addr = dst->ip4.addr;
857 ip4->sin_port = listen4->sin_port;
858
859 ip4 = (struct sockaddr_in *) &addr->dst_addr;
860 ip4->sin_family = listen4->sin_family;
861 ip4->sin_addr.s_addr = src->ip4.addr;
862 ip4->sin_port = port;
863 break;
864 case 6:
865 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
866 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
867 ip6->sin6_family = listen6->sin6_family;
868 ip6->sin6_addr = dst->ip6;
869 ip6->sin6_port = listen6->sin6_port;
870
871 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
872 ip6->sin6_family = listen6->sin6_family;
873 ip6->sin6_addr = src->ip6;
874 ip6->sin6_port = port;
875 break;
876 default:
877 break;
878 }
879 }
880
cma_user_data_offset(enum rdma_port_space ps)881 static inline int cma_user_data_offset(enum rdma_port_space ps)
882 {
883 switch (ps) {
884 case RDMA_PS_SDP:
885 return 0;
886 default:
887 return sizeof(struct cma_hdr);
888 }
889 }
890
cma_cancel_route(struct rdma_id_private * id_priv)891 static void cma_cancel_route(struct rdma_id_private *id_priv)
892 {
893 switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
894 case IB_LINK_LAYER_INFINIBAND:
895 if (id_priv->query)
896 ib_sa_cancel_query(id_priv->query_id, id_priv->query);
897 break;
898 default:
899 break;
900 }
901 }
902
cma_cancel_listens(struct rdma_id_private * id_priv)903 static void cma_cancel_listens(struct rdma_id_private *id_priv)
904 {
905 struct rdma_id_private *dev_id_priv;
906
907 /*
908 * Remove from listen_any_list to prevent added devices from spawning
909 * additional listen requests.
910 */
911 mutex_lock(&lock);
912 list_del(&id_priv->list);
913
914 while (!list_empty(&id_priv->listen_list)) {
915 dev_id_priv = list_entry(id_priv->listen_list.next,
916 struct rdma_id_private, listen_list);
917 /* sync with device removal to avoid duplicate destruction */
918 list_del_init(&dev_id_priv->list);
919 list_del(&dev_id_priv->listen_list);
920 mutex_unlock(&lock);
921
922 rdma_destroy_id(&dev_id_priv->id);
923 mutex_lock(&lock);
924 }
925 mutex_unlock(&lock);
926 }
927
cma_cancel_operation(struct rdma_id_private * id_priv,enum rdma_cm_state state)928 static void cma_cancel_operation(struct rdma_id_private *id_priv,
929 enum rdma_cm_state state)
930 {
931 switch (state) {
932 case RDMA_CM_ADDR_QUERY:
933 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
934 break;
935 case RDMA_CM_ROUTE_QUERY:
936 cma_cancel_route(id_priv);
937 break;
938 case RDMA_CM_LISTEN:
939 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
940 && !id_priv->cma_dev)
941 cma_cancel_listens(id_priv);
942 break;
943 default:
944 break;
945 }
946 }
947
cma_release_port(struct rdma_id_private * id_priv)948 static void cma_release_port(struct rdma_id_private *id_priv)
949 {
950 struct rdma_bind_list *bind_list;
951
952 mutex_lock(&lock);
953 bind_list = id_priv->bind_list;
954 if (!bind_list) {
955 mutex_unlock(&lock);
956 return;
957 }
958 hlist_del(&id_priv->node);
959 id_priv->bind_list = NULL;
960 if (hlist_empty(&bind_list->owners)) {
961 idr_remove(bind_list->ps, bind_list->port);
962 kfree(bind_list);
963 }
964 mutex_unlock(&lock);
965 if (id_priv->sock)
966 sock_release(id_priv->sock);
967 }
968
cma_leave_mc_groups(struct rdma_id_private * id_priv)969 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
970 {
971 struct cma_multicast *mc;
972
973 while (!list_empty(&id_priv->mc_list)) {
974 mc = container_of(id_priv->mc_list.next,
975 struct cma_multicast, list);
976 list_del(&mc->list);
977 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
978 case IB_LINK_LAYER_INFINIBAND:
979 ib_sa_free_multicast(mc->multicast.ib);
980 kfree(mc);
981 break;
982 case IB_LINK_LAYER_ETHERNET:
983 kref_put(&mc->mcref, release_mc);
984 break;
985 default:
986 break;
987 }
988 }
989 }
__rdma_free(struct work_struct * work)990 static void __rdma_free(struct work_struct *work)
991 {
992 struct rdma_id_private *id_priv;
993 id_priv = container_of(work, struct rdma_id_private, work);
994
995 wait_for_completion(&id_priv->comp);
996
997 if (id_priv->internal_id)
998 cma_deref_id(id_priv->id.context);
999
1000 kfree(id_priv->id.route.path_rec);
1001 kfree(id_priv);
1002 }
1003
rdma_destroy_id(struct rdma_cm_id * id)1004 void rdma_destroy_id(struct rdma_cm_id *id)
1005 {
1006 struct rdma_id_private *id_priv;
1007 enum rdma_cm_state state;
1008 unsigned long flags;
1009 struct ib_cm_id *ib;
1010
1011 id_priv = container_of(id, struct rdma_id_private, id);
1012 state = cma_exch(id_priv, RDMA_CM_DESTROYING);
1013 cma_cancel_operation(id_priv, state);
1014
1015 /*
1016 * Wait for any active callback to finish. New callbacks will find
1017 * the id_priv state set to destroying and abort.
1018 */
1019 mutex_lock(&id_priv->handler_mutex);
1020 mutex_unlock(&id_priv->handler_mutex);
1021
1022 if (id_priv->cma_dev) {
1023 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
1024 case RDMA_TRANSPORT_IB:
1025 spin_lock_irqsave(&id_priv->cm_lock, flags);
1026 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) {
1027 ib = id_priv->cm_id.ib;
1028 id_priv->cm_id.ib = NULL;
1029 spin_unlock_irqrestore(&id_priv->cm_lock, flags);
1030 ib_destroy_cm_id(ib);
1031 } else
1032 spin_unlock_irqrestore(&id_priv->cm_lock, flags);
1033 break;
1034 case RDMA_TRANSPORT_IWARP:
1035 case RDMA_TRANSPORT_SCIF:
1036 if (id_priv->cm_id.iw)
1037 iw_destroy_cm_id(id_priv->cm_id.iw);
1038 break;
1039 default:
1040 break;
1041 }
1042 cma_leave_mc_groups(id_priv);
1043 cma_release_dev(id_priv);
1044 }
1045
1046 cma_release_port(id_priv);
1047 cma_deref_id(id_priv);
1048 INIT_WORK(&id_priv->work, __rdma_free);
1049 queue_work(cma_free_wq, &id_priv->work);
1050 }
1051 EXPORT_SYMBOL(rdma_destroy_id);
1052
cma_rep_recv(struct rdma_id_private * id_priv)1053 static int cma_rep_recv(struct rdma_id_private *id_priv)
1054 {
1055 int ret;
1056
1057 ret = cma_modify_qp_rtr(id_priv, NULL);
1058 if (ret)
1059 goto reject;
1060
1061 ret = cma_modify_qp_rts(id_priv, NULL);
1062 if (ret)
1063 goto reject;
1064
1065 cma_dbg(id_priv, "sending RTU\n");
1066 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1067 if (ret)
1068 goto reject;
1069
1070 return 0;
1071 reject:
1072 cma_modify_qp_err(id_priv);
1073 cma_dbg(id_priv, "sending REJ\n");
1074 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1075 NULL, 0, NULL, 0);
1076 return ret;
1077 }
1078
cma_verify_rep(struct rdma_id_private * id_priv,void * data)1079 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
1080 {
1081 if (id_priv->id.ps == RDMA_PS_SDP &&
1082 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
1083 SDP_MAJ_VERSION)
1084 return -EINVAL;
1085
1086 return 0;
1087 }
1088
cma_set_rep_event_data(struct rdma_cm_event * event,struct ib_cm_rep_event_param * rep_data,void * private_data)1089 static void cma_set_rep_event_data(struct rdma_cm_event *event,
1090 struct ib_cm_rep_event_param *rep_data,
1091 void *private_data)
1092 {
1093 event->param.conn.private_data = private_data;
1094 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1095 event->param.conn.responder_resources = rep_data->responder_resources;
1096 event->param.conn.initiator_depth = rep_data->initiator_depth;
1097 event->param.conn.flow_control = rep_data->flow_control;
1098 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1099 event->param.conn.srq = rep_data->srq;
1100 event->param.conn.qp_num = rep_data->remote_qpn;
1101 }
1102
cma_ib_handler(struct ib_cm_id * cm_id,struct ib_cm_event * ib_event)1103 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1104 {
1105 struct rdma_id_private *id_priv = cm_id->context;
1106 struct rdma_cm_event event;
1107 int ret = 0;
1108
1109 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1110 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
1111 (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1112 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
1113 return 0;
1114 memset(&event, 0, sizeof event);
1115 switch (ib_event->event) {
1116 case IB_CM_REQ_ERROR:
1117 case IB_CM_REP_ERROR:
1118 event.event = RDMA_CM_EVENT_UNREACHABLE;
1119 event.status = -ETIMEDOUT;
1120 break;
1121 case IB_CM_REP_RECEIVED:
1122 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1123 if (event.status)
1124 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1125 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1126 event.status = cma_rep_recv(id_priv);
1127 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1128 RDMA_CM_EVENT_ESTABLISHED;
1129 } else
1130 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1131 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1132 ib_event->private_data);
1133 break;
1134 case IB_CM_RTU_RECEIVED:
1135 case IB_CM_USER_ESTABLISHED:
1136 event.event = RDMA_CM_EVENT_ESTABLISHED;
1137 break;
1138 case IB_CM_DREQ_ERROR:
1139 event.status = -ETIMEDOUT; /* fall through */
1140 case IB_CM_DREQ_RECEIVED:
1141 case IB_CM_DREP_RECEIVED:
1142 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1143 RDMA_CM_DISCONNECT))
1144 goto out;
1145 event.event = RDMA_CM_EVENT_DISCONNECTED;
1146 break;
1147 case IB_CM_TIMEWAIT_EXIT:
1148 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1149 break;
1150 case IB_CM_MRA_RECEIVED:
1151 /* ignore event */
1152 goto out;
1153 case IB_CM_REJ_RECEIVED:
1154 cma_modify_qp_err(id_priv);
1155 event.status = ib_event->param.rej_rcvd.reason;
1156 event.event = RDMA_CM_EVENT_REJECTED;
1157 event.param.conn.private_data = ib_event->private_data;
1158 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1159 break;
1160 default:
1161 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1162 ib_event->event);
1163 goto out;
1164 }
1165
1166 ret = id_priv->id.event_handler(&id_priv->id, &event);
1167 if (ret) {
1168 /* Destroy the CM ID by returning a non-zero value. */
1169 id_priv->cm_id.ib = NULL;
1170 cma_exch(id_priv, RDMA_CM_DESTROYING);
1171 mutex_unlock(&id_priv->handler_mutex);
1172 rdma_destroy_id(&id_priv->id);
1173 return ret;
1174 }
1175 out:
1176 mutex_unlock(&id_priv->handler_mutex);
1177 return ret;
1178 }
1179
cma_new_conn_id(struct rdma_cm_id * listen_id,struct ib_cm_event * ib_event)1180 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1181 struct ib_cm_event *ib_event)
1182 {
1183 struct rdma_id_private *id_priv;
1184 struct rdma_cm_id *id;
1185 struct rdma_route *rt;
1186 union cma_ip_addr *src, *dst;
1187 __be16 port;
1188 u8 ip_ver;
1189 int ret;
1190
1191 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1192 &ip_ver, &port, &src, &dst))
1193 return NULL;
1194
1195 id = rdma_create_id(listen_id->event_handler, listen_id->context,
1196 listen_id->ps, ib_event->param.req_rcvd.qp_type);
1197 if (IS_ERR(id))
1198 return NULL;
1199
1200 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1201 ip_ver, port, src, dst);
1202
1203 rt = &id->route;
1204 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1205 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1206 GFP_KERNEL);
1207 if (!rt->path_rec)
1208 goto err;
1209
1210 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1211 if (rt->num_paths == 2)
1212 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1213
1214 if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
1215 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1216 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1217 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
1218 } else {
1219 ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr,
1220 &rt->addr.dev_addr, NULL);
1221 if (ret)
1222 goto err;
1223 }
1224 rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1225
1226 id_priv = container_of(id, struct rdma_id_private, id);
1227 id_priv->state = RDMA_CM_CONNECT;
1228 return id_priv;
1229
1230 err:
1231 rdma_destroy_id(id);
1232 return NULL;
1233 }
1234
cma_new_udp_id(struct rdma_cm_id * listen_id,struct ib_cm_event * ib_event)1235 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1236 struct ib_cm_event *ib_event)
1237 {
1238 struct rdma_id_private *id_priv;
1239 struct rdma_cm_id *id;
1240 union cma_ip_addr *src, *dst;
1241 __be16 port;
1242 u8 ip_ver;
1243 int ret;
1244
1245 id = rdma_create_id(listen_id->event_handler, listen_id->context,
1246 listen_id->ps, IB_QPT_UD);
1247 if (IS_ERR(id))
1248 return NULL;
1249
1250
1251 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1252 &ip_ver, &port, &src, &dst))
1253 goto err;
1254
1255 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1256 ip_ver, port, src, dst);
1257
1258 if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1259 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
1260 &id->route.addr.dev_addr, NULL);
1261 if (ret)
1262 goto err;
1263 }
1264
1265 id_priv = container_of(id, struct rdma_id_private, id);
1266 id_priv->state = RDMA_CM_CONNECT;
1267 return id_priv;
1268 err:
1269 rdma_destroy_id(id);
1270 return NULL;
1271 }
1272
cma_set_req_event_data(struct rdma_cm_event * event,struct ib_cm_req_event_param * req_data,void * private_data,int offset)1273 static void cma_set_req_event_data(struct rdma_cm_event *event,
1274 struct ib_cm_req_event_param *req_data,
1275 void *private_data, int offset)
1276 {
1277 event->param.conn.private_data = private_data + offset;
1278 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1279 event->param.conn.responder_resources = req_data->responder_resources;
1280 event->param.conn.initiator_depth = req_data->initiator_depth;
1281 event->param.conn.flow_control = req_data->flow_control;
1282 event->param.conn.retry_count = req_data->retry_count;
1283 event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1284 event->param.conn.srq = req_data->srq;
1285 event->param.conn.qp_num = req_data->remote_qpn;
1286 }
1287
cma_check_req_qp_type(struct rdma_cm_id * id,struct ib_cm_event * ib_event)1288 static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
1289 {
1290 return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
1291 (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
1292 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
1293 (id->qp_type == IB_QPT_UD)) ||
1294 (!id->qp_type));
1295 }
1296
cma_req_handler(struct ib_cm_id * cm_id,struct ib_cm_event * ib_event)1297 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1298 {
1299 struct rdma_id_private *listen_id, *conn_id;
1300 struct rdma_cm_event event;
1301 int offset, ret;
1302 u8 smac[ETH_ALEN];
1303 u8 alt_smac[ETH_ALEN];
1304 u8 *psmac = smac;
1305 u8 *palt_smac = alt_smac;
1306 int is_iboe = ((rdma_node_get_transport(cm_id->device->node_type) ==
1307 RDMA_TRANSPORT_IB) &&
1308 (rdma_port_get_link_layer(cm_id->device,
1309 ib_event->param.req_rcvd.port) ==
1310 IB_LINK_LAYER_ETHERNET));
1311 int is_sidr = 0;
1312
1313 listen_id = cm_id->context;
1314 if (!cma_check_req_qp_type(&listen_id->id, ib_event))
1315 return -EINVAL;
1316
1317 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1318 return -ECONNABORTED;
1319
1320 memset(&event, 0, sizeof event);
1321 offset = cma_user_data_offset(listen_id->id.ps);
1322 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1323 if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
1324 is_sidr = 1;
1325 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1326 event.param.ud.private_data = ib_event->private_data + offset;
1327 event.param.ud.private_data_len =
1328 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1329 } else {
1330 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1331 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1332 ib_event->private_data, offset);
1333 }
1334 if (!conn_id) {
1335 ret = -ENOMEM;
1336 goto err1;
1337 }
1338
1339 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1340 ret = cma_acquire_dev(conn_id);
1341 if (ret)
1342 goto err2;
1343
1344 conn_id->cm_id.ib = cm_id;
1345 cm_id->context = conn_id;
1346 cm_id->cm_handler = cma_ib_handler;
1347
1348 /*
1349 * Protect against the user destroying conn_id from another thread
1350 * until we're done accessing it.
1351 */
1352 atomic_inc(&conn_id->refcount);
1353 ret = conn_id->id.event_handler(&conn_id->id, &event);
1354 if (ret)
1355 goto err3;
1356
1357 if (is_iboe && !is_sidr) {
1358 if (ib_event->param.req_rcvd.primary_path != NULL)
1359 rdma_addr_find_smac_by_sgid(
1360 &ib_event->param.req_rcvd.primary_path->sgid,
1361 psmac, NULL);
1362 else
1363 psmac = NULL;
1364 if (ib_event->param.req_rcvd.alternate_path != NULL)
1365 rdma_addr_find_smac_by_sgid(
1366 &ib_event->param.req_rcvd.alternate_path->sgid,
1367 palt_smac, NULL);
1368 else
1369 palt_smac = NULL;
1370 }
1371 /*
1372 * Acquire mutex to prevent user executing rdma_destroy_id()
1373 * while we're accessing the cm_id.
1374 */
1375 mutex_lock(&lock);
1376 if (is_iboe && !is_sidr)
1377 ib_update_cm_av(cm_id, psmac, palt_smac);
1378 if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD)) {
1379 cma_dbg(container_of(&conn_id->id, struct rdma_id_private, id), "sending MRA\n");
1380 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1381 }
1382 mutex_unlock(&lock);
1383 mutex_unlock(&conn_id->handler_mutex);
1384 mutex_unlock(&listen_id->handler_mutex);
1385 cma_deref_id(conn_id);
1386 return 0;
1387
1388 err3:
1389 cma_deref_id(conn_id);
1390 /* Destroy the CM ID by returning a non-zero value. */
1391 conn_id->cm_id.ib = NULL;
1392 err2:
1393 cma_exch(conn_id, RDMA_CM_DESTROYING);
1394 mutex_unlock(&conn_id->handler_mutex);
1395 err1:
1396 mutex_unlock(&listen_id->handler_mutex);
1397 if (conn_id)
1398 rdma_destroy_id(&conn_id->id);
1399 return ret;
1400 }
1401
cma_get_service_id(enum rdma_port_space ps,struct sockaddr * addr)1402 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1403 {
1404 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1405 }
1406
cma_set_compare_data(enum rdma_port_space ps,struct sockaddr * addr,struct ib_cm_compare_data * compare)1407 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1408 struct ib_cm_compare_data *compare)
1409 {
1410 struct cma_hdr *cma_data, *cma_mask;
1411 struct sdp_hh *sdp_data, *sdp_mask;
1412 __be32 ip4_addr;
1413 struct in6_addr ip6_addr;
1414
1415 memset(compare, 0, sizeof *compare);
1416 cma_data = (void *) compare->data;
1417 cma_mask = (void *) compare->mask;
1418 sdp_data = (void *) compare->data;
1419 sdp_mask = (void *) compare->mask;
1420
1421 switch (addr->sa_family) {
1422 case AF_INET:
1423 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1424 if (ps == RDMA_PS_SDP) {
1425 sdp_set_ip_ver(sdp_data, 4);
1426 sdp_set_ip_ver(sdp_mask, 0xF);
1427 if (!cma_any_addr(addr)) {
1428 sdp_data->dst_addr.ip4.addr = ip4_addr;
1429 sdp_mask->dst_addr.ip4.addr = htonl(~0);
1430 }
1431 } else {
1432 cma_set_ip_ver(cma_data, 4);
1433 cma_set_ip_ver(cma_mask, 0xF);
1434 if (!cma_any_addr(addr)) {
1435 cma_data->dst_addr.ip4.addr = ip4_addr;
1436 cma_mask->dst_addr.ip4.addr = htonl(~0);
1437 }
1438 }
1439 break;
1440 case AF_INET6:
1441 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1442 if (ps == RDMA_PS_SDP) {
1443 sdp_set_ip_ver(sdp_data, 6);
1444 sdp_set_ip_ver(sdp_mask, 0xF);
1445 if (!cma_any_addr(addr)) {
1446 sdp_data->dst_addr.ip6 = ip6_addr;
1447 memset(&sdp_mask->dst_addr.ip6, 0xFF,
1448 sizeof(sdp_mask->dst_addr.ip6));
1449 }
1450 } else {
1451 cma_set_ip_ver(cma_data, 6);
1452 cma_set_ip_ver(cma_mask, 0xF);
1453 if (!cma_any_addr(addr)) {
1454 cma_data->dst_addr.ip6 = ip6_addr;
1455 memset(&cma_mask->dst_addr.ip6, 0xFF,
1456 sizeof(cma_mask->dst_addr.ip6));
1457 }
1458 }
1459 break;
1460 default:
1461 break;
1462 }
1463 }
1464
cma_iw_handler(struct iw_cm_id * iw_id,struct iw_cm_event * iw_event)1465 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1466 {
1467 struct rdma_id_private *id_priv = iw_id->context;
1468 struct rdma_cm_event event;
1469 struct sockaddr_in *sin;
1470 int ret = 0;
1471
1472 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1473 return 0;
1474
1475 memset(&event, 0, sizeof event);
1476 switch (iw_event->event) {
1477 case IW_CM_EVENT_CLOSE:
1478 event.event = RDMA_CM_EVENT_DISCONNECTED;
1479 break;
1480 case IW_CM_EVENT_CONNECT_REPLY:
1481 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1482 *sin = iw_event->local_addr;
1483 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1484 *sin = iw_event->remote_addr;
1485 switch ((int)iw_event->status) {
1486 case 0:
1487 event.event = RDMA_CM_EVENT_ESTABLISHED;
1488 event.param.conn.initiator_depth = iw_event->ird;
1489 event.param.conn.responder_resources = iw_event->ord;
1490 break;
1491 case -ECONNRESET:
1492 case -ECONNREFUSED:
1493 event.event = RDMA_CM_EVENT_REJECTED;
1494 break;
1495 case -ETIMEDOUT:
1496 event.event = RDMA_CM_EVENT_UNREACHABLE;
1497 break;
1498 default:
1499 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1500 break;
1501 }
1502 break;
1503 case IW_CM_EVENT_ESTABLISHED:
1504 event.event = RDMA_CM_EVENT_ESTABLISHED;
1505 event.param.conn.initiator_depth = iw_event->ird;
1506 event.param.conn.responder_resources = iw_event->ord;
1507 break;
1508 default:
1509 BUG_ON(1);
1510 }
1511
1512 event.status = iw_event->status;
1513 event.param.conn.private_data = iw_event->private_data;
1514 event.param.conn.private_data_len = iw_event->private_data_len;
1515 ret = id_priv->id.event_handler(&id_priv->id, &event);
1516 if (ret) {
1517 /* Destroy the CM ID by returning a non-zero value. */
1518 id_priv->cm_id.iw = NULL;
1519 cma_exch(id_priv, RDMA_CM_DESTROYING);
1520 mutex_unlock(&id_priv->handler_mutex);
1521 rdma_destroy_id(&id_priv->id);
1522 return ret;
1523 }
1524
1525 mutex_unlock(&id_priv->handler_mutex);
1526 return ret;
1527 }
1528
iw_conn_req_handler(struct iw_cm_id * cm_id,struct iw_cm_event * iw_event)1529 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1530 struct iw_cm_event *iw_event)
1531 {
1532 struct rdma_cm_id *new_cm_id;
1533 struct rdma_id_private *listen_id, *conn_id;
1534 struct sockaddr_in *sin;
1535 struct net_device *dev = NULL;
1536 struct rdma_cm_event event;
1537 int ret;
1538 struct ib_device_attr attr;
1539
1540 listen_id = cm_id->context;
1541 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1542 return -ECONNABORTED;
1543
1544 /* Create a new RDMA id for the new IW CM ID */
1545 new_cm_id = rdma_create_id(listen_id->id.event_handler,
1546 listen_id->id.context,
1547 RDMA_PS_TCP, IB_QPT_RC);
1548 if (IS_ERR(new_cm_id)) {
1549 ret = -ENOMEM;
1550 goto out;
1551 }
1552 conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1553 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1554 conn_id->state = RDMA_CM_CONNECT;
1555
1556 dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1557 if (!dev) {
1558 ret = -EADDRNOTAVAIL;
1559 mutex_unlock(&conn_id->handler_mutex);
1560 rdma_destroy_id(new_cm_id);
1561 goto out;
1562 }
1563 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1564 if (ret) {
1565 mutex_unlock(&conn_id->handler_mutex);
1566 rdma_destroy_id(new_cm_id);
1567 goto out;
1568 }
1569
1570 ret = cma_acquire_dev(conn_id);
1571 if (ret) {
1572 mutex_unlock(&conn_id->handler_mutex);
1573 rdma_destroy_id(new_cm_id);
1574 goto out;
1575 }
1576
1577 conn_id->cm_id.iw = cm_id;
1578 cm_id->context = conn_id;
1579 cm_id->cm_handler = cma_iw_handler;
1580
1581 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1582 *sin = iw_event->local_addr;
1583 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1584 *sin = iw_event->remote_addr;
1585
1586 ret = ib_query_device(conn_id->id.device, &attr);
1587 if (ret) {
1588 mutex_unlock(&conn_id->handler_mutex);
1589 rdma_destroy_id(new_cm_id);
1590 goto out;
1591 }
1592
1593 memset(&event, 0, sizeof event);
1594 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1595 event.param.conn.private_data = iw_event->private_data;
1596 event.param.conn.private_data_len = iw_event->private_data_len;
1597 event.param.conn.initiator_depth = iw_event->ird;
1598 event.param.conn.responder_resources = iw_event->ord;
1599
1600 /*
1601 * Protect against the user destroying conn_id from another thread
1602 * until we're done accessing it.
1603 */
1604 atomic_inc(&conn_id->refcount);
1605 ret = conn_id->id.event_handler(&conn_id->id, &event);
1606 if (ret) {
1607 /* User wants to destroy the CM ID */
1608 conn_id->cm_id.iw = NULL;
1609 cma_exch(conn_id, RDMA_CM_DESTROYING);
1610 mutex_unlock(&conn_id->handler_mutex);
1611 cma_deref_id(conn_id);
1612 rdma_destroy_id(&conn_id->id);
1613 goto out;
1614 }
1615
1616 mutex_unlock(&conn_id->handler_mutex);
1617 cma_deref_id(conn_id);
1618
1619 out:
1620 if (dev)
1621 dev_put(dev);
1622 mutex_unlock(&listen_id->handler_mutex);
1623 return ret;
1624 }
1625
cma_ib_listen(struct rdma_id_private * id_priv)1626 static int cma_ib_listen(struct rdma_id_private *id_priv)
1627 {
1628 struct ib_cm_compare_data compare_data;
1629 struct sockaddr *addr;
1630 struct ib_cm_id *id;
1631 __be64 svc_id;
1632 int ret;
1633
1634 id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
1635 if (IS_ERR(id))
1636 return PTR_ERR(id);
1637
1638 id_priv->cm_id.ib = id;
1639
1640 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
1641 svc_id = cma_get_service_id(id_priv->id.ps, addr);
1642 if (cma_any_addr(addr) && !id_priv->afonly)
1643 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1644 else {
1645 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1646 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1647 }
1648
1649 if (ret) {
1650 ib_destroy_cm_id(id_priv->cm_id.ib);
1651 id_priv->cm_id.ib = NULL;
1652 }
1653
1654 return ret;
1655 }
1656
cma_iw_listen(struct rdma_id_private * id_priv,int backlog)1657 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1658 {
1659 int ret;
1660 struct sockaddr_in *sin;
1661 struct iw_cm_id *id;
1662
1663 id = iw_create_cm_id(id_priv->id.device,
1664 id_priv->sock,
1665 iw_conn_req_handler,
1666 id_priv);
1667 if (IS_ERR(id))
1668 return PTR_ERR(id);
1669
1670 id_priv->cm_id.iw = id;
1671
1672 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1673 id_priv->cm_id.iw->local_addr = *sin;
1674
1675 ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1676
1677 if (ret) {
1678 iw_destroy_cm_id(id_priv->cm_id.iw);
1679 id_priv->cm_id.iw = NULL;
1680 }
1681
1682 return ret;
1683 }
1684
cma_listen_handler(struct rdma_cm_id * id,struct rdma_cm_event * event)1685 static int cma_listen_handler(struct rdma_cm_id *id,
1686 struct rdma_cm_event *event)
1687 {
1688 struct rdma_id_private *id_priv = id->context;
1689
1690 id->context = id_priv->id.context;
1691 id->event_handler = id_priv->id.event_handler;
1692 return id_priv->id.event_handler(id, event);
1693 }
1694
cma_listen_on_dev(struct rdma_id_private * id_priv,struct cma_device * cma_dev)1695 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1696 struct cma_device *cma_dev)
1697 {
1698 struct rdma_id_private *dev_id_priv;
1699 struct rdma_cm_id *id;
1700 int ret;
1701
1702 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
1703 id_priv->id.qp_type);
1704 if (IS_ERR(id))
1705 return;
1706
1707 dev_id_priv = container_of(id, struct rdma_id_private, id);
1708
1709 dev_id_priv->state = RDMA_CM_ADDR_BOUND;
1710 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1711 ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
1712
1713 cma_attach_to_dev(dev_id_priv, cma_dev);
1714 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1715 atomic_inc(&id_priv->refcount);
1716 dev_id_priv->internal_id = 1;
1717 dev_id_priv->afonly = id_priv->afonly;
1718
1719 ret = rdma_listen(id, id_priv->backlog);
1720 if (ret)
1721 cma_warn(id_priv, "cma_listen_on_dev, error %d, listening on device %s\n", ret, cma_dev->device->name);
1722 }
1723
cma_listen_on_all(struct rdma_id_private * id_priv)1724 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1725 {
1726 struct cma_device *cma_dev;
1727
1728 mutex_lock(&lock);
1729 list_add_tail(&id_priv->list, &listen_any_list);
1730 list_for_each_entry(cma_dev, &dev_list, list)
1731 cma_listen_on_dev(id_priv, cma_dev);
1732 mutex_unlock(&lock);
1733 }
1734
rdma_set_service_type(struct rdma_cm_id * id,int tos)1735 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1736 {
1737 struct rdma_id_private *id_priv;
1738
1739 id_priv = container_of(id, struct rdma_id_private, id);
1740 id_priv->tos = (u8) tos;
1741 }
1742 EXPORT_SYMBOL(rdma_set_service_type);
1743
rdma_set_timeout(struct rdma_cm_id * id,int timeout)1744 void rdma_set_timeout(struct rdma_cm_id *id, int timeout)
1745 {
1746 struct rdma_id_private *id_priv;
1747
1748 id_priv = container_of(id, struct rdma_id_private, id);
1749 id_priv->qp_timeout = (u8) timeout;
1750 }
1751 EXPORT_SYMBOL(rdma_set_timeout);
1752
cma_query_handler(int status,struct ib_sa_path_rec * path_rec,void * context)1753 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1754 void *context)
1755 {
1756 struct cma_work *work = context;
1757 struct rdma_route *route;
1758
1759 route = &work->id->id.route;
1760
1761 if (!status) {
1762 route->num_paths = 1;
1763 *route->path_rec = *path_rec;
1764 } else {
1765 work->old_state = RDMA_CM_ROUTE_QUERY;
1766 work->new_state = RDMA_CM_ADDR_RESOLVED;
1767 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1768 work->event.status = status;
1769 }
1770
1771 queue_work(cma_wq, &work->work);
1772 }
1773
cma_query_ib_route(struct rdma_id_private * id_priv,int timeout_ms,struct cma_work * work)1774 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1775 struct cma_work *work)
1776 {
1777 struct rdma_addr *addr = &id_priv->id.route.addr;
1778 struct ib_sa_path_rec path_rec;
1779 ib_sa_comp_mask comp_mask;
1780 struct sockaddr_in6 *sin6;
1781
1782 memset(&path_rec, 0, sizeof path_rec);
1783 rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1784 rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1785 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1786 path_rec.numb_path = 1;
1787 path_rec.reversible = 1;
1788 path_rec.service_id = cma_get_service_id(id_priv->id.ps,
1789 (struct sockaddr *) &addr->dst_addr);
1790
1791 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1792 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1793 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1794
1795 if (addr->src_addr.ss_family == AF_INET) {
1796 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1797 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1798 } else {
1799 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1800 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1801 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1802 }
1803
1804 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1805 id_priv->id.port_num, &path_rec,
1806 comp_mask, timeout_ms,
1807 GFP_KERNEL, cma_query_handler,
1808 work, &id_priv->query);
1809
1810 return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1811 }
1812
cma_work_handler(struct work_struct * _work)1813 static void cma_work_handler(struct work_struct *_work)
1814 {
1815 struct cma_work *work = container_of(_work, struct cma_work, work);
1816 struct rdma_id_private *id_priv = work->id;
1817 int destroy = 0;
1818
1819 mutex_lock(&id_priv->handler_mutex);
1820 if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1821 goto out;
1822
1823 if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1824 cma_exch(id_priv, RDMA_CM_DESTROYING);
1825 destroy = 1;
1826 }
1827 out:
1828 mutex_unlock(&id_priv->handler_mutex);
1829 cma_deref_id(id_priv);
1830 if (destroy)
1831 rdma_destroy_id(&id_priv->id);
1832 kfree(work);
1833 }
1834
cma_ndev_work_handler(struct work_struct * _work)1835 static void cma_ndev_work_handler(struct work_struct *_work)
1836 {
1837 struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1838 struct rdma_id_private *id_priv = work->id;
1839 int destroy = 0;
1840
1841 mutex_lock(&id_priv->handler_mutex);
1842 if (id_priv->state == RDMA_CM_DESTROYING ||
1843 id_priv->state == RDMA_CM_DEVICE_REMOVAL)
1844 goto out;
1845
1846 if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1847 cma_exch(id_priv, RDMA_CM_DESTROYING);
1848 destroy = 1;
1849 }
1850
1851 out:
1852 mutex_unlock(&id_priv->handler_mutex);
1853 cma_deref_id(id_priv);
1854 if (destroy)
1855 rdma_destroy_id(&id_priv->id);
1856 kfree(work);
1857 }
1858
cma_resolve_ib_route(struct rdma_id_private * id_priv,int timeout_ms)1859 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1860 {
1861 struct rdma_route *route = &id_priv->id.route;
1862 struct cma_work *work;
1863 int ret;
1864
1865 work = kzalloc(sizeof *work, GFP_KERNEL);
1866 if (!work)
1867 return -ENOMEM;
1868
1869 work->id = id_priv;
1870 INIT_WORK(&work->work, cma_work_handler);
1871 work->old_state = RDMA_CM_ROUTE_QUERY;
1872 work->new_state = RDMA_CM_ROUTE_RESOLVED;
1873 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1874
1875 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1876 if (!route->path_rec) {
1877 ret = -ENOMEM;
1878 goto err1;
1879 }
1880
1881 ret = cma_query_ib_route(id_priv, timeout_ms, work);
1882 if (ret)
1883 goto err2;
1884
1885 return 0;
1886 err2:
1887 kfree(route->path_rec);
1888 route->path_rec = NULL;
1889 err1:
1890 kfree(work);
1891 return ret;
1892 }
1893
rdma_set_ib_paths(struct rdma_cm_id * id,struct ib_sa_path_rec * path_rec,int num_paths)1894 int rdma_set_ib_paths(struct rdma_cm_id *id,
1895 struct ib_sa_path_rec *path_rec, int num_paths)
1896 {
1897 struct rdma_id_private *id_priv;
1898 int ret;
1899
1900 id_priv = container_of(id, struct rdma_id_private, id);
1901 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1902 RDMA_CM_ROUTE_RESOLVED))
1903 return -EINVAL;
1904
1905 id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1906 GFP_KERNEL);
1907 if (!id->route.path_rec) {
1908 ret = -ENOMEM;
1909 goto err;
1910 }
1911
1912 id->route.num_paths = num_paths;
1913 return 0;
1914 err:
1915 cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
1916 return ret;
1917 }
1918 EXPORT_SYMBOL(rdma_set_ib_paths);
1919
cma_resolve_iw_route(struct rdma_id_private * id_priv,int timeout_ms)1920 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1921 {
1922 struct cma_work *work;
1923
1924 work = kzalloc(sizeof *work, GFP_KERNEL);
1925 if (!work)
1926 return -ENOMEM;
1927
1928 work->id = id_priv;
1929 INIT_WORK(&work->work, cma_work_handler);
1930 work->old_state = RDMA_CM_ROUTE_QUERY;
1931 work->new_state = RDMA_CM_ROUTE_RESOLVED;
1932 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1933 queue_work(cma_wq, &work->work);
1934 return 0;
1935 }
1936
tos_to_sl(u8 tos)1937 static u8 tos_to_sl(u8 tos)
1938 {
1939 return def_prec2sl & 7;
1940 }
1941
cma_resolve_iboe_route(struct rdma_id_private * id_priv)1942 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1943 {
1944 struct rdma_route *route = &id_priv->id.route;
1945 struct rdma_addr *addr = &route->addr;
1946 struct cma_work *work;
1947 int ret;
1948 struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
1949 struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
1950 struct net_device *ndev = NULL;
1951
1952
1953 if (src_addr->sin_family != dst_addr->sin_family)
1954 return -EINVAL;
1955
1956 work = kzalloc(sizeof *work, GFP_KERNEL);
1957 if (!work)
1958 return -ENOMEM;
1959
1960 work->id = id_priv;
1961 INIT_WORK(&work->work, cma_work_handler);
1962
1963 route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1964 if (!route->path_rec) {
1965 ret = -ENOMEM;
1966 goto err1;
1967 }
1968
1969 route->num_paths = 1;
1970
1971 if (addr->dev_addr.bound_dev_if)
1972 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1973 if (!ndev) {
1974 ret = -ENODEV;
1975 goto err2;
1976 }
1977
1978 route->path_rec->vlan_id = rdma_vlan_dev_vlan_id(ndev);
1979 memcpy(route->path_rec->dmac, addr->dev_addr.dst_dev_addr, ETH_ALEN);
1980 memcpy(route->path_rec->smac, IF_LLADDR(ndev), ndev->if_addrlen);
1981
1982
1983 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
1984 &route->path_rec->sgid);
1985 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
1986 &route->path_rec->dgid);
1987
1988 route->path_rec->hop_limit = 1;
1989 route->path_rec->reversible = 1;
1990 route->path_rec->pkey = cpu_to_be16(0xffff);
1991 route->path_rec->mtu_selector = IB_SA_EQ;
1992 route->path_rec->sl = tos_to_sl(id_priv->tos);
1993
1994 route->path_rec->mtu = iboe_get_mtu(ndev->if_mtu);
1995 route->path_rec->rate_selector = IB_SA_EQ;
1996 route->path_rec->rate = iboe_get_rate(ndev);
1997 dev_put(ndev);
1998 route->path_rec->packet_life_time_selector = IB_SA_EQ;
1999 route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
2000 if (!route->path_rec->mtu) {
2001 ret = -EINVAL;
2002 goto err2;
2003 }
2004
2005 work->old_state = RDMA_CM_ROUTE_QUERY;
2006 work->new_state = RDMA_CM_ROUTE_RESOLVED;
2007 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
2008 work->event.status = 0;
2009
2010 queue_work(cma_wq, &work->work);
2011
2012 return 0;
2013
2014 err2:
2015 kfree(route->path_rec);
2016 route->path_rec = NULL;
2017 err1:
2018 kfree(work);
2019 return ret;
2020 }
2021
rdma_resolve_route(struct rdma_cm_id * id,int timeout_ms)2022 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
2023 {
2024 struct rdma_id_private *id_priv;
2025 int ret;
2026
2027 id_priv = container_of(id, struct rdma_id_private, id);
2028 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
2029 return -EINVAL;
2030
2031 atomic_inc(&id_priv->refcount);
2032 switch (rdma_node_get_transport(id->device->node_type)) {
2033 case RDMA_TRANSPORT_IB:
2034 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
2035 case IB_LINK_LAYER_INFINIBAND:
2036 ret = cma_resolve_ib_route(id_priv, timeout_ms);
2037 break;
2038 case IB_LINK_LAYER_ETHERNET:
2039 ret = cma_resolve_iboe_route(id_priv);
2040 break;
2041 default:
2042 ret = -ENOSYS;
2043 }
2044 break;
2045 case RDMA_TRANSPORT_IWARP:
2046 case RDMA_TRANSPORT_SCIF:
2047 ret = cma_resolve_iw_route(id_priv, timeout_ms);
2048 break;
2049 default:
2050 ret = -ENOSYS;
2051 break;
2052 }
2053 if (ret)
2054 goto err;
2055
2056 return 0;
2057 err:
2058 cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
2059 cma_deref_id(id_priv);
2060 return ret;
2061 }
2062 EXPORT_SYMBOL(rdma_resolve_route);
2063
rdma_enable_apm(struct rdma_cm_id * id,enum alt_path_type alt_type)2064 int rdma_enable_apm(struct rdma_cm_id *id, enum alt_path_type alt_type)
2065 {
2066 /* APM is not supported yet */
2067 return -EINVAL;
2068 }
2069 EXPORT_SYMBOL(rdma_enable_apm);
2070
cma_bind_loopback(struct rdma_id_private * id_priv)2071 static int cma_bind_loopback(struct rdma_id_private *id_priv)
2072 {
2073 struct cma_device *cma_dev;
2074 struct ib_port_attr port_attr;
2075 union ib_gid gid;
2076 u16 pkey;
2077 int ret;
2078 u8 p;
2079
2080 mutex_lock(&lock);
2081 if (list_empty(&dev_list)) {
2082 ret = -ENODEV;
2083 goto out;
2084 }
2085 list_for_each_entry(cma_dev, &dev_list, list)
2086 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
2087 if (!ib_query_port(cma_dev->device, p, &port_attr) &&
2088 port_attr.state == IB_PORT_ACTIVE)
2089 goto port_found;
2090
2091 p = 1;
2092 cma_dev = list_entry(dev_list.next, struct cma_device, list);
2093
2094 port_found:
2095 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
2096 if (ret)
2097 goto out;
2098
2099 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
2100 if (ret)
2101 goto out;
2102
2103 id_priv->id.route.addr.dev_addr.dev_type =
2104 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
2105 ARPHRD_INFINIBAND : ARPHRD_ETHER;
2106
2107 rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2108 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
2109 id_priv->id.port_num = p;
2110 cma_attach_to_dev(id_priv, cma_dev);
2111 out:
2112 mutex_unlock(&lock);
2113 return ret;
2114 }
2115
addr_handler(int status,struct sockaddr * src_addr,struct rdma_dev_addr * dev_addr,void * context)2116 static void addr_handler(int status, struct sockaddr *src_addr,
2117 struct rdma_dev_addr *dev_addr, void *context)
2118 {
2119 struct rdma_id_private *id_priv = context;
2120 struct rdma_cm_event event;
2121
2122 memset(&event, 0, sizeof event);
2123 mutex_lock(&id_priv->handler_mutex);
2124 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
2125 RDMA_CM_ADDR_RESOLVED))
2126 goto out;
2127
2128 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
2129 ip_addr_size(src_addr));
2130 if (!status && !id_priv->cma_dev)
2131 status = cma_acquire_dev(id_priv);
2132
2133 if (status) {
2134 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2135 RDMA_CM_ADDR_BOUND))
2136 goto out;
2137 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2138 event.status = status;
2139 } else
2140 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2141
2142 if (id_priv->id.event_handler(&id_priv->id, &event)) {
2143 cma_exch(id_priv, RDMA_CM_DESTROYING);
2144 mutex_unlock(&id_priv->handler_mutex);
2145 cma_deref_id(id_priv);
2146 rdma_destroy_id(&id_priv->id);
2147 return;
2148 }
2149 out:
2150 mutex_unlock(&id_priv->handler_mutex);
2151 cma_deref_id(id_priv);
2152 }
2153
cma_resolve_loopback(struct rdma_id_private * id_priv)2154 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
2155 {
2156 struct cma_work *work;
2157 struct sockaddr *src, *dst;
2158 union ib_gid gid;
2159 int ret;
2160
2161 work = kzalloc(sizeof *work, GFP_KERNEL);
2162 if (!work)
2163 return -ENOMEM;
2164
2165 if (!id_priv->cma_dev) {
2166 ret = cma_bind_loopback(id_priv);
2167 if (ret)
2168 goto err;
2169 }
2170
2171 rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2172 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2173
2174 src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2175 if (cma_zero_addr(src)) {
2176 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
2177 if ((src->sa_family = dst->sa_family) == AF_INET) {
2178 ((struct sockaddr_in *)src)->sin_addr =
2179 ((struct sockaddr_in *)dst)->sin_addr;
2180 } else {
2181 ((struct sockaddr_in6 *)src)->sin6_addr =
2182 ((struct sockaddr_in6 *)dst)->sin6_addr;
2183 }
2184 }
2185
2186 work->id = id_priv;
2187 INIT_WORK(&work->work, cma_work_handler);
2188 work->old_state = RDMA_CM_ADDR_QUERY;
2189 work->new_state = RDMA_CM_ADDR_RESOLVED;
2190 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2191 queue_work(cma_wq, &work->work);
2192 return 0;
2193 err:
2194 kfree(work);
2195 return ret;
2196 }
2197
cma_resolve_scif(struct rdma_id_private * id_priv)2198 static int cma_resolve_scif(struct rdma_id_private *id_priv)
2199 {
2200 struct cma_work *work;
2201
2202 work = kzalloc(sizeof *work, GFP_KERNEL);
2203 if (!work)
2204 return -ENOMEM;
2205
2206 /* we probably can leave it empty here */
2207
2208 work->id = id_priv;
2209 INIT_WORK(&work->work, cma_work_handler);
2210 work->old_state = RDMA_CM_ADDR_QUERY;
2211 work->new_state = RDMA_CM_ADDR_RESOLVED;
2212 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2213 queue_work(cma_wq, &work->work);
2214 return 0;
2215 }
2216
cma_bind_addr(struct rdma_cm_id * id,struct sockaddr * src_addr,struct sockaddr * dst_addr)2217 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2218 struct sockaddr *dst_addr)
2219 {
2220 if (!src_addr || !src_addr->sa_family) {
2221 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2222 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2223 ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2224 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2225 }
2226 }
2227 if (!cma_any_addr(src_addr))
2228 return rdma_bind_addr(id, src_addr);
2229 else {
2230 struct sockaddr_in addr_in;
2231
2232 memset(&addr_in, 0, sizeof addr_in);
2233 addr_in.sin_family = dst_addr->sa_family;
2234 addr_in.sin_len = sizeof addr_in;
2235 return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
2236 }
2237
2238 }
2239
rdma_resolve_addr(struct rdma_cm_id * id,struct sockaddr * src_addr,struct sockaddr * dst_addr,int timeout_ms)2240 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2241 struct sockaddr *dst_addr, int timeout_ms)
2242 {
2243 struct rdma_id_private *id_priv;
2244 int ret;
2245
2246 id_priv = container_of(id, struct rdma_id_private, id);
2247 if (id_priv->state == RDMA_CM_IDLE) {
2248 ret = cma_bind_addr(id, src_addr, dst_addr);
2249 if (ret)
2250 return ret;
2251 }
2252
2253 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
2254 return -EINVAL;
2255
2256 atomic_inc(&id_priv->refcount);
2257 memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
2258 if (cma_any_addr(dst_addr))
2259 ret = cma_resolve_loopback(id_priv);
2260 else if (id_priv->id.device &&
2261 rdma_node_get_transport(id_priv->id.device->node_type) == RDMA_TRANSPORT_SCIF)
2262 ret = cma_resolve_scif(id_priv);
2263 else
2264 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
2265 dst_addr, &id->route.addr.dev_addr,
2266 timeout_ms, addr_handler, id_priv);
2267 if (ret)
2268 goto err;
2269
2270 return 0;
2271 err:
2272 cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
2273 cma_deref_id(id_priv);
2274 return ret;
2275 }
2276 EXPORT_SYMBOL(rdma_resolve_addr);
2277
rdma_set_reuseaddr(struct rdma_cm_id * id,int reuse)2278 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
2279 {
2280 struct rdma_id_private *id_priv;
2281 unsigned long flags;
2282 int ret;
2283
2284 id_priv = container_of(id, struct rdma_id_private, id);
2285 spin_lock_irqsave(&id_priv->lock, flags);
2286 if (id_priv->state == RDMA_CM_IDLE) {
2287 id_priv->reuseaddr = reuse;
2288 ret = 0;
2289 } else {
2290 ret = -EINVAL;
2291 }
2292 spin_unlock_irqrestore(&id_priv->lock, flags);
2293 return ret;
2294 }
2295 EXPORT_SYMBOL(rdma_set_reuseaddr);
2296
rdma_set_afonly(struct rdma_cm_id * id,int afonly)2297 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
2298 {
2299 struct rdma_id_private *id_priv;
2300 unsigned long flags;
2301 int ret;
2302
2303 id_priv = container_of(id, struct rdma_id_private, id);
2304 spin_lock_irqsave(&id_priv->lock, flags);
2305 if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
2306 id_priv->options |= (1 << CMA_OPTION_AFONLY);
2307 id_priv->afonly = afonly;
2308 ret = 0;
2309 } else {
2310 ret = -EINVAL;
2311 }
2312 spin_unlock_irqrestore(&id_priv->lock, flags);
2313 return ret;
2314 }
2315 EXPORT_SYMBOL(rdma_set_afonly);
2316
cma_bind_port(struct rdma_bind_list * bind_list,struct rdma_id_private * id_priv)2317 static void cma_bind_port(struct rdma_bind_list *bind_list,
2318 struct rdma_id_private *id_priv)
2319 {
2320 struct sockaddr_in *sin;
2321
2322 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
2323 sin->sin_port = htons(bind_list->port);
2324 id_priv->bind_list = bind_list;
2325 hlist_add_head(&id_priv->node, &bind_list->owners);
2326 }
2327
cma_alloc_port(struct idr * ps,struct rdma_id_private * id_priv,unsigned short snum)2328 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2329 unsigned short snum)
2330 {
2331 struct rdma_bind_list *bind_list;
2332 int port, ret;
2333
2334 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2335 if (!bind_list)
2336 return -ENOMEM;
2337
2338 do {
2339 ret = idr_get_new_above(ps, bind_list, snum, &port);
2340 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
2341
2342 if (ret)
2343 goto err1;
2344
2345 if (port != snum) {
2346 ret = -EADDRNOTAVAIL;
2347 goto err2;
2348 }
2349
2350 bind_list->ps = ps;
2351 bind_list->port = (unsigned short) port;
2352 cma_bind_port(bind_list, id_priv);
2353 return 0;
2354 err2:
2355 idr_remove(ps, port);
2356 err1:
2357 kfree(bind_list);
2358 return ret;
2359 }
2360
cma_alloc_any_port(struct idr * ps,struct rdma_id_private * id_priv)2361 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2362 {
2363 static unsigned int last_used_port;
2364 int low, high, remaining;
2365 unsigned int rover;
2366
2367 inet_get_local_port_range(&low, &high);
2368 remaining = (high - low) + 1;
2369 rover = random() % remaining + low;
2370 retry:
2371 if (last_used_port != rover &&
2372 !idr_find(ps, (unsigned short) rover)) {
2373 int ret = cma_alloc_port(ps, id_priv, rover);
2374 /*
2375 * Remember previously used port number in order to avoid
2376 * re-using same port immediately after it is closed.
2377 */
2378 if (!ret)
2379 last_used_port = rover;
2380 if (ret != -EADDRNOTAVAIL)
2381 return ret;
2382 }
2383 if (--remaining) {
2384 rover++;
2385 if ((rover < low) || (rover > high))
2386 rover = low;
2387 goto retry;
2388 }
2389 return -EADDRNOTAVAIL;
2390 }
2391
2392 /*
2393 * Check that the requested port is available. This is called when trying to
2394 * bind to a specific port, or when trying to listen on a bound port. In
2395 * the latter case, the provided id_priv may already be on the bind_list, but
2396 * we still need to check that it's okay to start listening.
2397 */
cma_check_port(struct rdma_bind_list * bind_list,struct rdma_id_private * id_priv,uint8_t reuseaddr)2398 static int cma_check_port(struct rdma_bind_list *bind_list,
2399 struct rdma_id_private *id_priv, uint8_t reuseaddr)
2400 {
2401 struct rdma_id_private *cur_id;
2402 struct sockaddr *addr, *cur_addr;
2403
2404 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2405 hlist_for_each_entry(cur_id, &bind_list->owners, node) {
2406 if (id_priv == cur_id)
2407 continue;
2408
2409 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
2410 cur_id->reuseaddr)
2411 continue;
2412
2413 cur_addr = (struct sockaddr *) &cur_id->id.route.addr.src_addr;
2414 if (id_priv->afonly && cur_id->afonly &&
2415 (addr->sa_family != cur_addr->sa_family))
2416 continue;
2417
2418 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
2419 return -EADDRNOTAVAIL;
2420
2421 if (!cma_addr_cmp(addr, cur_addr))
2422 return -EADDRINUSE;
2423 }
2424 return 0;
2425 }
2426
cma_use_port(struct idr * ps,struct rdma_id_private * id_priv)2427 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2428 {
2429 struct rdma_bind_list *bind_list;
2430 unsigned short snum;
2431 int ret;
2432
2433 snum = ntohs(cma_port((struct sockaddr *) &id_priv->id.route.addr.src_addr));
2434
2435 bind_list = idr_find(ps, snum);
2436 if (!bind_list) {
2437 ret = cma_alloc_port(ps, id_priv, snum);
2438 } else {
2439 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
2440 if (!ret)
2441 cma_bind_port(bind_list, id_priv);
2442 }
2443 return ret;
2444 }
2445
cma_bind_listen(struct rdma_id_private * id_priv)2446 static int cma_bind_listen(struct rdma_id_private *id_priv)
2447 {
2448 struct rdma_bind_list *bind_list = id_priv->bind_list;
2449 int ret = 0;
2450
2451 mutex_lock(&lock);
2452 if (bind_list->owners.first->next)
2453 ret = cma_check_port(bind_list, id_priv, 0);
2454 mutex_unlock(&lock);
2455 return ret;
2456 }
2457
cma_get_tcp_port(struct rdma_id_private * id_priv)2458 static int cma_get_tcp_port(struct rdma_id_private *id_priv)
2459 {
2460 int ret;
2461 int size;
2462 struct socket *sock;
2463
2464 ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock);
2465 if (ret)
2466 return ret;
2467 #ifdef __linux__
2468 ret = sock->ops->bind(sock,
2469 (struct sockaddr *) &id_priv->id.route.addr.src_addr,
2470 ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
2471 #else
2472 ret = -sobind(sock,
2473 (struct sockaddr *)&id_priv->id.route.addr.src_addr,
2474 curthread);
2475 #endif
2476 if (ret) {
2477 sock_release(sock);
2478 return ret;
2479 }
2480
2481 size = ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr);
2482 ret = sock_getname(sock,
2483 (struct sockaddr *) &id_priv->id.route.addr.src_addr,
2484 &size, 0);
2485 if (ret) {
2486 sock_release(sock);
2487 return ret;
2488 }
2489
2490 id_priv->sock = sock;
2491 return 0;
2492 }
2493
cma_get_port(struct rdma_id_private * id_priv)2494 static int cma_get_port(struct rdma_id_private *id_priv)
2495 {
2496 struct idr *ps;
2497 int ret;
2498
2499 switch (id_priv->id.ps) {
2500 case RDMA_PS_SDP:
2501 ps = &sdp_ps;
2502 break;
2503 case RDMA_PS_TCP:
2504 ps = &tcp_ps;
2505 if (unify_tcp_port_space) {
2506 ret = cma_get_tcp_port(id_priv);
2507 if (ret)
2508 goto out;
2509 }
2510 break;
2511 case RDMA_PS_UDP:
2512 ps = &udp_ps;
2513 break;
2514 case RDMA_PS_IPOIB:
2515 ps = &ipoib_ps;
2516 break;
2517 case RDMA_PS_IB:
2518 ps = &ib_ps;
2519 break;
2520 default:
2521 return -EPROTONOSUPPORT;
2522 }
2523
2524 mutex_lock(&lock);
2525 if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
2526 ret = cma_alloc_any_port(ps, id_priv);
2527 else
2528 ret = cma_use_port(ps, id_priv);
2529 mutex_unlock(&lock);
2530 out:
2531 return ret;
2532 }
2533
cma_check_linklocal(struct rdma_dev_addr * dev_addr,struct sockaddr * addr)2534 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2535 struct sockaddr *addr)
2536 {
2537 #if defined(INET6)
2538 struct sockaddr_in6 *sin6;
2539
2540 if (addr->sa_family != AF_INET6)
2541 return 0;
2542
2543 sin6 = (struct sockaddr_in6 *) addr;
2544 if (IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr) &&
2545 !sin6->sin6_scope_id)
2546 return -EINVAL;
2547
2548 dev_addr->bound_dev_if = sin6->sin6_scope_id;
2549 #endif
2550 return 0;
2551 }
2552
rdma_listen(struct rdma_cm_id * id,int backlog)2553 int rdma_listen(struct rdma_cm_id *id, int backlog)
2554 {
2555 struct rdma_id_private *id_priv;
2556 int ret;
2557
2558 id_priv = container_of(id, struct rdma_id_private, id);
2559 if (id_priv->state == RDMA_CM_IDLE) {
2560 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET;
2561 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr);
2562 if (ret)
2563 return ret;
2564 }
2565
2566 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
2567 return -EINVAL;
2568
2569 if (id_priv->reuseaddr) {
2570 ret = cma_bind_listen(id_priv);
2571 if (ret)
2572 goto err;
2573 }
2574
2575 id_priv->backlog = backlog;
2576 if (id->device) {
2577 switch (rdma_node_get_transport(id->device->node_type)) {
2578 case RDMA_TRANSPORT_IB:
2579 ret = cma_ib_listen(id_priv);
2580 if (ret)
2581 goto err;
2582 break;
2583 case RDMA_TRANSPORT_IWARP:
2584 case RDMA_TRANSPORT_SCIF:
2585 ret = cma_iw_listen(id_priv, backlog);
2586 if (ret)
2587 goto err;
2588 break;
2589 default:
2590 ret = -ENOSYS;
2591 goto err;
2592 }
2593 } else
2594 cma_listen_on_all(id_priv);
2595
2596 return 0;
2597 err:
2598 id_priv->backlog = 0;
2599 cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
2600 return ret;
2601 }
2602 EXPORT_SYMBOL(rdma_listen);
2603
rdma_bind_addr(struct rdma_cm_id * id,struct sockaddr * addr)2604 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2605 {
2606 struct rdma_id_private *id_priv;
2607 int ret;
2608 #if defined(INET6)
2609 int ipv6only;
2610 size_t var_size = sizeof(int);
2611 #endif
2612
2613 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
2614 return -EAFNOSUPPORT;
2615
2616 id_priv = container_of(id, struct rdma_id_private, id);
2617 if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
2618 return -EINVAL;
2619
2620 ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2621 if (ret)
2622 goto err1;
2623
2624 memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2625 if (!cma_any_addr(addr)) {
2626 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr, NULL);
2627 if (ret)
2628 goto err1;
2629
2630 ret = cma_acquire_dev(id_priv);
2631 if (ret)
2632 goto err1;
2633 }
2634
2635 if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
2636 if (addr->sa_family == AF_INET)
2637 id_priv->afonly = 1;
2638 #if defined(INET6)
2639 else if (addr->sa_family == AF_INET6)
2640 id_priv->afonly = kernel_sysctlbyname(&thread0, "net.inet6.ip6.v6only",
2641 &ipv6only, &var_size, NULL, 0, NULL, 0);
2642 #endif
2643 }
2644 ret = cma_get_port(id_priv);
2645 if (ret)
2646 goto err2;
2647
2648 return 0;
2649 err2:
2650 if (id_priv->cma_dev)
2651 cma_release_dev(id_priv);
2652 err1:
2653 cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
2654 return ret;
2655 }
2656 EXPORT_SYMBOL(rdma_bind_addr);
2657
cma_format_hdr(void * hdr,enum rdma_port_space ps,struct rdma_route * route)2658 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2659 struct rdma_route *route)
2660 {
2661 struct cma_hdr *cma_hdr;
2662 struct sdp_hh *sdp_hdr;
2663
2664 if (route->addr.src_addr.ss_family == AF_INET) {
2665 struct sockaddr_in *src4, *dst4;
2666
2667 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2668 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2669
2670 switch (ps) {
2671 case RDMA_PS_SDP:
2672 sdp_hdr = hdr;
2673 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2674 return -EINVAL;
2675 sdp_set_ip_ver(sdp_hdr, 4);
2676 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2677 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2678 sdp_hdr->port = src4->sin_port;
2679 break;
2680 default:
2681 cma_hdr = hdr;
2682 cma_hdr->cma_version = CMA_VERSION;
2683 cma_set_ip_ver(cma_hdr, 4);
2684 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2685 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2686 cma_hdr->port = src4->sin_port;
2687 break;
2688 }
2689 } else {
2690 struct sockaddr_in6 *src6, *dst6;
2691
2692 src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
2693 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
2694
2695 switch (ps) {
2696 case RDMA_PS_SDP:
2697 sdp_hdr = hdr;
2698 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2699 return -EINVAL;
2700 sdp_set_ip_ver(sdp_hdr, 6);
2701 sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2702 sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2703 sdp_hdr->port = src6->sin6_port;
2704 break;
2705 default:
2706 cma_hdr = hdr;
2707 cma_hdr->cma_version = CMA_VERSION;
2708 cma_set_ip_ver(cma_hdr, 6);
2709 cma_hdr->src_addr.ip6 = src6->sin6_addr;
2710 cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2711 cma_hdr->port = src6->sin6_port;
2712 break;
2713 }
2714 }
2715 return 0;
2716 }
2717
cma_sidr_rep_handler(struct ib_cm_id * cm_id,struct ib_cm_event * ib_event)2718 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2719 struct ib_cm_event *ib_event)
2720 {
2721 struct rdma_id_private *id_priv = cm_id->context;
2722 struct rdma_cm_event event;
2723 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2724 int ret = 0;
2725
2726 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
2727 return 0;
2728
2729 memset(&event, 0, sizeof event);
2730 switch (ib_event->event) {
2731 case IB_CM_SIDR_REQ_ERROR:
2732 event.event = RDMA_CM_EVENT_UNREACHABLE;
2733 event.status = -ETIMEDOUT;
2734 break;
2735 case IB_CM_SIDR_REP_RECEIVED:
2736 event.param.ud.private_data = ib_event->private_data;
2737 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2738 if (rep->status != IB_SIDR_SUCCESS) {
2739 event.event = RDMA_CM_EVENT_UNREACHABLE;
2740 event.status = ib_event->param.sidr_rep_rcvd.status;
2741 break;
2742 }
2743 ret = cma_set_qkey(id_priv);
2744 if (ret) {
2745 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2746 event.status = -EINVAL;
2747 break;
2748 }
2749 if (id_priv->qkey != rep->qkey) {
2750 event.event = RDMA_CM_EVENT_UNREACHABLE;
2751 event.status = -EINVAL;
2752 break;
2753 }
2754 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2755 id_priv->id.route.path_rec,
2756 &event.param.ud.ah_attr);
2757 event.param.ud.qp_num = rep->qpn;
2758 event.param.ud.qkey = rep->qkey;
2759 event.event = RDMA_CM_EVENT_ESTABLISHED;
2760 event.status = 0;
2761 break;
2762 default:
2763 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2764 ib_event->event);
2765 goto out;
2766 }
2767
2768 ret = id_priv->id.event_handler(&id_priv->id, &event);
2769 if (ret) {
2770 /* Destroy the CM ID by returning a non-zero value. */
2771 id_priv->cm_id.ib = NULL;
2772 cma_exch(id_priv, RDMA_CM_DESTROYING);
2773 mutex_unlock(&id_priv->handler_mutex);
2774 rdma_destroy_id(&id_priv->id);
2775 return ret;
2776 }
2777 out:
2778 mutex_unlock(&id_priv->handler_mutex);
2779 return ret;
2780 }
2781
cma_resolve_ib_udp(struct rdma_id_private * id_priv,struct rdma_conn_param * conn_param)2782 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2783 struct rdma_conn_param *conn_param)
2784 {
2785 struct ib_cm_sidr_req_param req;
2786 struct rdma_route *route;
2787 struct ib_cm_id *id;
2788 int ret;
2789
2790 req.private_data_len = sizeof(struct cma_hdr) +
2791 conn_param->private_data_len;
2792 if (req.private_data_len < conn_param->private_data_len)
2793 return -EINVAL;
2794
2795 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2796 if (!req.private_data)
2797 return -ENOMEM;
2798
2799 if (conn_param->private_data && conn_param->private_data_len)
2800 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2801 conn_param->private_data, conn_param->private_data_len);
2802
2803 route = &id_priv->id.route;
2804 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2805 if (ret)
2806 goto out;
2807
2808 id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
2809 id_priv);
2810 if (IS_ERR(id)) {
2811 ret = PTR_ERR(id);
2812 goto out;
2813 }
2814 id_priv->cm_id.ib = id;
2815
2816 req.path = route->path_rec;
2817 req.service_id = cma_get_service_id(id_priv->id.ps,
2818 (struct sockaddr *) &route->addr.dst_addr);
2819 req.timeout_ms = 1 << (cma_response_timeout - 8);
2820 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2821
2822 cma_dbg(id_priv, "sending SIDR\n");
2823 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2824 if (ret) {
2825 ib_destroy_cm_id(id_priv->cm_id.ib);
2826 id_priv->cm_id.ib = NULL;
2827 }
2828 out:
2829 kfree(req.private_data);
2830 return ret;
2831 }
2832
cma_connect_ib(struct rdma_id_private * id_priv,struct rdma_conn_param * conn_param)2833 static int cma_connect_ib(struct rdma_id_private *id_priv,
2834 struct rdma_conn_param *conn_param)
2835 {
2836 struct ib_cm_req_param req;
2837 struct rdma_route *route;
2838 void *private_data;
2839 struct ib_cm_id *id;
2840 int offset, ret;
2841
2842 memset(&req, 0, sizeof req);
2843 offset = cma_user_data_offset(id_priv->id.ps);
2844 req.private_data_len = offset + conn_param->private_data_len;
2845 if (req.private_data_len < conn_param->private_data_len)
2846 return -EINVAL;
2847
2848 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2849 if (!private_data)
2850 return -ENOMEM;
2851
2852 if (conn_param->private_data && conn_param->private_data_len)
2853 memcpy(private_data + offset, conn_param->private_data,
2854 conn_param->private_data_len);
2855
2856 id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
2857 if (IS_ERR(id)) {
2858 ret = PTR_ERR(id);
2859 goto out;
2860 }
2861 id_priv->cm_id.ib = id;
2862
2863 route = &id_priv->id.route;
2864 ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2865 if (ret)
2866 goto out;
2867 req.private_data = private_data;
2868
2869 req.primary_path = &route->path_rec[0];
2870 if (route->num_paths == 2)
2871 req.alternate_path = &route->path_rec[1];
2872
2873 req.service_id = cma_get_service_id(id_priv->id.ps,
2874 (struct sockaddr *) &route->addr.dst_addr);
2875 req.qp_num = id_priv->qp_num;
2876 req.qp_type = id_priv->id.qp_type;
2877 req.starting_psn = id_priv->seq_num;
2878 req.responder_resources = conn_param->responder_resources;
2879 req.initiator_depth = conn_param->initiator_depth;
2880 req.flow_control = conn_param->flow_control;
2881 req.retry_count = min_t(u8, 7, conn_param->retry_count);
2882 req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2883 req.remote_cm_response_timeout = cma_response_timeout;
2884 req.local_cm_response_timeout = cma_response_timeout;
2885 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2886 req.srq = id_priv->srq ? 1 : 0;
2887
2888 cma_dbg(id_priv, "sending REQ\n");
2889 ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2890 out:
2891 if (ret && !IS_ERR(id)) {
2892 ib_destroy_cm_id(id);
2893 id_priv->cm_id.ib = NULL;
2894 }
2895
2896 kfree(private_data);
2897 return ret;
2898 }
2899
cma_connect_iw(struct rdma_id_private * id_priv,struct rdma_conn_param * conn_param)2900 static int cma_connect_iw(struct rdma_id_private *id_priv,
2901 struct rdma_conn_param *conn_param)
2902 {
2903 struct iw_cm_id *cm_id;
2904 struct sockaddr_in* sin;
2905 int ret;
2906 struct iw_cm_conn_param iw_param;
2907
2908 cm_id = iw_create_cm_id(id_priv->id.device, id_priv->sock,
2909 cma_iw_handler, id_priv);
2910 if (IS_ERR(cm_id))
2911 return PTR_ERR(cm_id);
2912
2913 id_priv->cm_id.iw = cm_id;
2914
2915 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2916 cm_id->local_addr = *sin;
2917
2918 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2919 cm_id->remote_addr = *sin;
2920
2921 ret = cma_modify_qp_rtr(id_priv, conn_param);
2922 if (ret)
2923 goto out;
2924
2925 if (conn_param) {
2926 iw_param.ord = conn_param->initiator_depth;
2927 iw_param.ird = conn_param->responder_resources;
2928 iw_param.private_data = conn_param->private_data;
2929 iw_param.private_data_len = conn_param->private_data_len;
2930 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
2931 } else {
2932 memset(&iw_param, 0, sizeof iw_param);
2933 iw_param.qpn = id_priv->qp_num;
2934 }
2935 ret = iw_cm_connect(cm_id, &iw_param);
2936 out:
2937 if (ret) {
2938 iw_destroy_cm_id(cm_id);
2939 id_priv->cm_id.iw = NULL;
2940 }
2941 return ret;
2942 }
2943
rdma_connect(struct rdma_cm_id * id,struct rdma_conn_param * conn_param)2944 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2945 {
2946 struct rdma_id_private *id_priv;
2947 int ret;
2948
2949 id_priv = container_of(id, struct rdma_id_private, id);
2950 if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
2951 return -EINVAL;
2952
2953 if (!id->qp) {
2954 id_priv->qp_num = conn_param->qp_num;
2955 id_priv->srq = conn_param->srq;
2956 }
2957
2958 switch (rdma_node_get_transport(id->device->node_type)) {
2959 case RDMA_TRANSPORT_IB:
2960 if (id->qp_type == IB_QPT_UD)
2961 ret = cma_resolve_ib_udp(id_priv, conn_param);
2962 else
2963 ret = cma_connect_ib(id_priv, conn_param);
2964 break;
2965 case RDMA_TRANSPORT_IWARP:
2966 case RDMA_TRANSPORT_SCIF:
2967 ret = cma_connect_iw(id_priv, conn_param);
2968 break;
2969 default:
2970 ret = -ENOSYS;
2971 break;
2972 }
2973 if (ret)
2974 goto err;
2975
2976 return 0;
2977 err:
2978 cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
2979 return ret;
2980 }
2981 EXPORT_SYMBOL(rdma_connect);
2982
cma_accept_ib(struct rdma_id_private * id_priv,struct rdma_conn_param * conn_param)2983 static int cma_accept_ib(struct rdma_id_private *id_priv,
2984 struct rdma_conn_param *conn_param)
2985 {
2986 struct ib_cm_rep_param rep;
2987 int ret;
2988
2989 ret = cma_modify_qp_rtr(id_priv, conn_param);
2990 if (ret)
2991 goto out;
2992
2993 ret = cma_modify_qp_rts(id_priv, conn_param);
2994 if (ret)
2995 goto out;
2996
2997 memset(&rep, 0, sizeof rep);
2998 rep.qp_num = id_priv->qp_num;
2999 rep.starting_psn = id_priv->seq_num;
3000 rep.private_data = conn_param->private_data;
3001 rep.private_data_len = conn_param->private_data_len;
3002 rep.responder_resources = conn_param->responder_resources;
3003 rep.initiator_depth = conn_param->initiator_depth;
3004 rep.failover_accepted = 0;
3005 rep.flow_control = conn_param->flow_control;
3006 rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
3007 rep.srq = id_priv->srq ? 1 : 0;
3008 cma_dbg(id_priv, "sending REP\n");
3009 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
3010 out:
3011 return ret;
3012 }
3013
cma_accept_iw(struct rdma_id_private * id_priv,struct rdma_conn_param * conn_param)3014 static int cma_accept_iw(struct rdma_id_private *id_priv,
3015 struct rdma_conn_param *conn_param)
3016 {
3017 struct iw_cm_conn_param iw_param;
3018 int ret;
3019
3020 if (!conn_param)
3021 return -EINVAL;
3022
3023 ret = cma_modify_qp_rtr(id_priv, conn_param);
3024 if (ret)
3025 return ret;
3026
3027 iw_param.ord = conn_param->initiator_depth;
3028 iw_param.ird = conn_param->responder_resources;
3029 iw_param.private_data = conn_param->private_data;
3030 iw_param.private_data_len = conn_param->private_data_len;
3031 if (id_priv->id.qp) {
3032 iw_param.qpn = id_priv->qp_num;
3033 } else
3034 iw_param.qpn = conn_param->qp_num;
3035
3036 return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
3037 }
3038
cma_send_sidr_rep(struct rdma_id_private * id_priv,enum ib_cm_sidr_status status,const void * private_data,int private_data_len)3039 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
3040 enum ib_cm_sidr_status status,
3041 const void *private_data, int private_data_len)
3042 {
3043 struct ib_cm_sidr_rep_param rep;
3044 int ret;
3045
3046 memset(&rep, 0, sizeof rep);
3047 rep.status = status;
3048 if (status == IB_SIDR_SUCCESS) {
3049 ret = cma_set_qkey(id_priv);
3050 if (ret)
3051 return ret;
3052 rep.qp_num = id_priv->qp_num;
3053 rep.qkey = id_priv->qkey;
3054 }
3055 rep.private_data = private_data;
3056 rep.private_data_len = private_data_len;
3057
3058 cma_dbg(id_priv, "sending SIDR\n");
3059 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
3060 }
3061
rdma_accept(struct rdma_cm_id * id,struct rdma_conn_param * conn_param)3062 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
3063 {
3064 struct rdma_id_private *id_priv;
3065 int ret;
3066
3067 id_priv = container_of(id, struct rdma_id_private, id);
3068
3069 id_priv->owner = curthread->td_proc->p_pid;
3070 if (!cma_comp(id_priv, RDMA_CM_CONNECT))
3071 return -EINVAL;
3072
3073 if (!id->qp && conn_param) {
3074 id_priv->qp_num = conn_param->qp_num;
3075 id_priv->srq = conn_param->srq;
3076 }
3077
3078 switch (rdma_node_get_transport(id->device->node_type)) {
3079 case RDMA_TRANSPORT_IB:
3080 if (id->qp_type == IB_QPT_UD) {
3081 if (conn_param)
3082 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
3083 conn_param->private_data,
3084 conn_param->private_data_len);
3085 else
3086 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
3087 NULL, 0);
3088 } else {
3089 if (conn_param)
3090 ret = cma_accept_ib(id_priv, conn_param);
3091 else
3092 ret = cma_rep_recv(id_priv);
3093 }
3094 break;
3095 case RDMA_TRANSPORT_IWARP:
3096 case RDMA_TRANSPORT_SCIF:
3097 ret = cma_accept_iw(id_priv, conn_param);
3098 break;
3099 default:
3100 ret = -ENOSYS;
3101 break;
3102 }
3103
3104 if (ret)
3105 goto reject;
3106
3107 return 0;
3108 reject:
3109 cma_modify_qp_err(id_priv);
3110 rdma_reject(id, NULL, 0);
3111 return ret;
3112 }
3113 EXPORT_SYMBOL(rdma_accept);
3114
rdma_notify(struct rdma_cm_id * id,enum ib_event_type event)3115 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
3116 {
3117 struct rdma_id_private *id_priv;
3118 int ret;
3119
3120 id_priv = container_of(id, struct rdma_id_private, id);
3121 if (!id_priv->cm_id.ib)
3122 return -EINVAL;
3123
3124 switch (id->device->node_type) {
3125 case RDMA_NODE_IB_CA:
3126 ret = ib_cm_notify(id_priv->cm_id.ib, event);
3127 break;
3128 default:
3129 ret = 0;
3130 break;
3131 }
3132 return ret;
3133 }
3134 EXPORT_SYMBOL(rdma_notify);
3135
rdma_reject(struct rdma_cm_id * id,const void * private_data,u8 private_data_len)3136 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
3137 u8 private_data_len)
3138 {
3139 struct rdma_id_private *id_priv;
3140 int ret;
3141
3142 id_priv = container_of(id, struct rdma_id_private, id);
3143 if (!id_priv->cm_id.ib)
3144 return -EINVAL;
3145
3146 switch (rdma_node_get_transport(id->device->node_type)) {
3147 case RDMA_TRANSPORT_IB:
3148 if (id->qp_type == IB_QPT_UD)
3149 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
3150 private_data, private_data_len);
3151 else {
3152 cma_dbg(id_priv, "sending REJ\n");
3153 ret = ib_send_cm_rej(id_priv->cm_id.ib,
3154 IB_CM_REJ_CONSUMER_DEFINED, NULL,
3155 0, private_data, private_data_len);
3156 }
3157 break;
3158 case RDMA_TRANSPORT_IWARP:
3159 case RDMA_TRANSPORT_SCIF:
3160 ret = iw_cm_reject(id_priv->cm_id.iw,
3161 private_data, private_data_len);
3162 break;
3163 default:
3164 ret = -ENOSYS;
3165 break;
3166 }
3167 return ret;
3168 }
3169 EXPORT_SYMBOL(rdma_reject);
3170
rdma_disconnect(struct rdma_cm_id * id)3171 int rdma_disconnect(struct rdma_cm_id *id)
3172 {
3173 struct rdma_id_private *id_priv;
3174 int ret;
3175
3176 id_priv = container_of(id, struct rdma_id_private, id);
3177 if (!id_priv->cm_id.ib)
3178 return -EINVAL;
3179
3180 switch (rdma_node_get_transport(id->device->node_type)) {
3181 case RDMA_TRANSPORT_IB:
3182 ret = cma_modify_qp_err(id_priv);
3183 if (ret)
3184 goto out;
3185 /* Initiate or respond to a disconnect. */
3186 cma_dbg(id_priv, "sending DREQ\n");
3187 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
3188 cma_dbg(id_priv, "sending DREP\n");
3189 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
3190 }
3191 break;
3192 case RDMA_TRANSPORT_IWARP:
3193 case RDMA_TRANSPORT_SCIF:
3194 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
3195 break;
3196 default:
3197 ret = -EINVAL;
3198 break;
3199 }
3200 out:
3201 return ret;
3202 }
3203 EXPORT_SYMBOL(rdma_disconnect);
3204
cma_ib_mc_handler(int status,struct ib_sa_multicast * multicast)3205 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
3206 {
3207 struct rdma_id_private *id_priv;
3208 struct cma_multicast *mc = multicast->context;
3209 struct rdma_cm_event event;
3210 struct rdma_dev_addr *dev_addr;
3211 int ret;
3212 struct net_device *ndev = NULL;
3213 u16 vlan;
3214
3215 id_priv = mc->id_priv;
3216 dev_addr = &id_priv->id.route.addr.dev_addr;
3217 if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
3218 cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
3219 return 0;
3220
3221 mutex_lock(&id_priv->qp_mutex);
3222 if (!status && id_priv->id.qp)
3223 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
3224 be16_to_cpu(multicast->rec.mlid));
3225 mutex_unlock(&id_priv->qp_mutex);
3226
3227 memset(&event, 0, sizeof event);
3228 event.status = status;
3229 event.param.ud.private_data = mc->context;
3230 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3231 if (!ndev) {
3232 status = -ENODEV;
3233 } else {
3234 vlan = rdma_vlan_dev_vlan_id(ndev);
3235 dev_put(ndev);
3236 }
3237 if (!status) {
3238 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
3239 ib_init_ah_from_mcmember(id_priv->id.device,
3240 id_priv->id.port_num, &multicast->rec,
3241 &event.param.ud.ah_attr);
3242 event.param.ud.ah_attr.vlan_id = vlan;
3243 event.param.ud.qp_num = 0xFFFFFF;
3244 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
3245 } else {
3246 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
3247
3248 /* mark that the cached record is no longer valid */
3249 if (status != -ENETRESET && status != -EAGAIN) {
3250 spin_lock(&id_priv->lock);
3251 id_priv->is_valid_rec = 0;
3252 spin_unlock(&id_priv->lock);
3253 }
3254 }
3255
3256 ret = id_priv->id.event_handler(&id_priv->id, &event);
3257 if (ret) {
3258 cma_exch(id_priv, RDMA_CM_DESTROYING);
3259 mutex_unlock(&id_priv->handler_mutex);
3260 rdma_destroy_id(&id_priv->id);
3261 return 0;
3262 }
3263
3264 mutex_unlock(&id_priv->handler_mutex);
3265 return 0;
3266 }
3267
cma_set_mgid(struct rdma_id_private * id_priv,struct sockaddr * addr,union ib_gid * mgid)3268 static void cma_set_mgid(struct rdma_id_private *id_priv,
3269 struct sockaddr *addr, union ib_gid *mgid)
3270 {
3271 unsigned char mc_map[MAX_ADDR_LEN];
3272 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3273 struct sockaddr_in *sin = (struct sockaddr_in *) addr;
3274 #if defined(INET6)
3275 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
3276 #endif
3277
3278 if (cma_any_addr(addr)) {
3279 memset(mgid, 0, sizeof *mgid);
3280 #if defined(INET6)
3281 } else if ((addr->sa_family == AF_INET6) &&
3282 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
3283 0xFF10A01B)) {
3284 /* IPv6 address is an SA assigned MGID. */
3285 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3286 } else if (addr->sa_family == AF_INET6) {
3287 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
3288 if (id_priv->id.ps == RDMA_PS_UDP)
3289 mc_map[7] = 0x01; /* Use RDMA CM signature */
3290 *mgid = *(union ib_gid *) (mc_map + 4);
3291 #endif
3292 } else {
3293 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
3294 if (id_priv->id.ps == RDMA_PS_UDP)
3295 mc_map[7] = 0x01; /* Use RDMA CM signature */
3296 *mgid = *(union ib_gid *) (mc_map + 4);
3297 }
3298 }
3299
cma_join_ib_multicast(struct rdma_id_private * id_priv,struct cma_multicast * mc)3300 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
3301 struct cma_multicast *mc)
3302 {
3303 struct ib_sa_mcmember_rec rec;
3304 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3305 ib_sa_comp_mask comp_mask;
3306 int ret = 0;
3307
3308 ib_addr_get_mgid(dev_addr, &id_priv->rec.mgid);
3309
3310 /* cache ipoib bc record */
3311 spin_lock(&id_priv->lock);
3312 if (!id_priv->is_valid_rec)
3313 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
3314 id_priv->id.port_num,
3315 &id_priv->rec.mgid,
3316 &id_priv->rec);
3317 if (ret) {
3318 id_priv->is_valid_rec = 0;
3319 spin_unlock(&id_priv->lock);
3320 return ret;
3321 } else {
3322 rec = id_priv->rec;
3323 id_priv->is_valid_rec = 1;
3324 }
3325 spin_unlock(&id_priv->lock);
3326
3327 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
3328 if (id_priv->id.ps == RDMA_PS_UDP)
3329 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3330 rdma_addr_get_sgid(dev_addr, &rec.port_gid);
3331 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
3332 rec.join_state = 1;
3333
3334 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
3335 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
3336 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
3337 IB_SA_MCMEMBER_REC_FLOW_LABEL |
3338 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
3339
3340 if (id_priv->id.ps == RDMA_PS_IPOIB)
3341 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
3342 IB_SA_MCMEMBER_REC_RATE_SELECTOR |
3343 IB_SA_MCMEMBER_REC_MTU_SELECTOR |
3344 IB_SA_MCMEMBER_REC_MTU |
3345 IB_SA_MCMEMBER_REC_HOP_LIMIT;
3346
3347 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
3348 id_priv->id.port_num, &rec,
3349 comp_mask, GFP_KERNEL,
3350 cma_ib_mc_handler, mc);
3351 return PTR_RET(mc->multicast.ib);
3352 }
3353
iboe_mcast_work_handler(struct work_struct * work)3354 static void iboe_mcast_work_handler(struct work_struct *work)
3355 {
3356 struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
3357 struct cma_multicast *mc = mw->mc;
3358 struct ib_sa_multicast *m = mc->multicast.ib;
3359
3360 mc->multicast.ib->context = mc;
3361 cma_ib_mc_handler(0, m);
3362 kref_put(&mc->mcref, release_mc);
3363 kfree(mw);
3364 }
3365
cma_iboe_set_mgid(struct sockaddr * addr,union ib_gid * mgid)3366 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
3367 {
3368 struct sockaddr_in *sin = (struct sockaddr_in *)addr;
3369 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
3370
3371 if (cma_any_addr(addr)) {
3372 memset(mgid, 0, sizeof *mgid);
3373 } else if (addr->sa_family == AF_INET6) {
3374 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3375 } else {
3376 mgid->raw[0] = 0xff;
3377 mgid->raw[1] = 0x0e;
3378 mgid->raw[2] = 0;
3379 mgid->raw[3] = 0;
3380 mgid->raw[4] = 0;
3381 mgid->raw[5] = 0;
3382 mgid->raw[6] = 0;
3383 mgid->raw[7] = 0;
3384 mgid->raw[8] = 0;
3385 mgid->raw[9] = 0;
3386 mgid->raw[10] = 0xff;
3387 mgid->raw[11] = 0xff;
3388 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
3389 }
3390 }
3391
cma_iboe_join_multicast(struct rdma_id_private * id_priv,struct cma_multicast * mc)3392 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
3393 struct cma_multicast *mc)
3394 {
3395 struct iboe_mcast_work *work;
3396 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3397 int err;
3398 struct sockaddr *addr = (struct sockaddr *)&mc->addr;
3399 struct net_device *ndev = NULL;
3400
3401 if (cma_zero_addr((struct sockaddr *)&mc->addr))
3402 return -EINVAL;
3403
3404 work = kzalloc(sizeof *work, GFP_KERNEL);
3405 if (!work)
3406 return -ENOMEM;
3407
3408 mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
3409 if (!mc->multicast.ib) {
3410 err = -ENOMEM;
3411 goto out1;
3412 }
3413
3414 cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
3415
3416 mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3417 if (id_priv->id.ps == RDMA_PS_UDP)
3418 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3419
3420 if (dev_addr->bound_dev_if)
3421 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3422 if (!ndev) {
3423 err = -ENODEV;
3424 goto out2;
3425 }
3426 mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3427 mc->multicast.ib->rec.hop_limit = 1;
3428 mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->if_mtu);
3429 dev_put(ndev);
3430 if (!mc->multicast.ib->rec.mtu) {
3431 err = -EINVAL;
3432 goto out2;
3433 }
3434 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
3435 &mc->multicast.ib->rec.port_gid);
3436 work->id = id_priv;
3437 work->mc = mc;
3438 INIT_WORK(&work->work, iboe_mcast_work_handler);
3439 kref_get(&mc->mcref);
3440 queue_work(cma_wq, &work->work);
3441
3442 return 0;
3443
3444 out2:
3445 kfree(mc->multicast.ib);
3446 out1:
3447 kfree(work);
3448 return err;
3449 }
3450
rdma_join_multicast(struct rdma_cm_id * id,struct sockaddr * addr,void * context)3451 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3452 void *context)
3453 {
3454 struct rdma_id_private *id_priv;
3455 struct cma_multicast *mc;
3456 int ret;
3457
3458 id_priv = container_of(id, struct rdma_id_private, id);
3459 if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
3460 !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
3461 return -EINVAL;
3462
3463 mc = kmalloc(sizeof *mc, GFP_KERNEL);
3464 if (!mc)
3465 return -ENOMEM;
3466
3467 memcpy(&mc->addr, addr, ip_addr_size(addr));
3468 mc->context = context;
3469 mc->id_priv = id_priv;
3470
3471 spin_lock(&id_priv->lock);
3472 list_add(&mc->list, &id_priv->mc_list);
3473 spin_unlock(&id_priv->lock);
3474
3475 switch (rdma_node_get_transport(id->device->node_type)) {
3476 case RDMA_TRANSPORT_IB:
3477 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3478 case IB_LINK_LAYER_INFINIBAND:
3479 ret = cma_join_ib_multicast(id_priv, mc);
3480 break;
3481 case IB_LINK_LAYER_ETHERNET:
3482 kref_init(&mc->mcref);
3483 ret = cma_iboe_join_multicast(id_priv, mc);
3484 break;
3485 default:
3486 ret = -EINVAL;
3487 }
3488 break;
3489 default:
3490 ret = -ENOSYS;
3491 break;
3492 }
3493
3494 if (ret) {
3495 spin_lock_irq(&id_priv->lock);
3496 list_del(&mc->list);
3497 spin_unlock_irq(&id_priv->lock);
3498 kfree(mc);
3499 }
3500 return ret;
3501 }
3502 EXPORT_SYMBOL(rdma_join_multicast);
3503
rdma_leave_multicast(struct rdma_cm_id * id,struct sockaddr * addr)3504 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3505 {
3506 struct rdma_id_private *id_priv;
3507 struct cma_multicast *mc;
3508
3509 id_priv = container_of(id, struct rdma_id_private, id);
3510 spin_lock_irq(&id_priv->lock);
3511 list_for_each_entry(mc, &id_priv->mc_list, list) {
3512 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
3513 list_del(&mc->list);
3514 spin_unlock_irq(&id_priv->lock);
3515
3516 if (id->qp)
3517 ib_detach_mcast(id->qp,
3518 &mc->multicast.ib->rec.mgid,
3519 be16_to_cpu(mc->multicast.ib->rec.mlid));
3520 if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3521 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3522 case IB_LINK_LAYER_INFINIBAND:
3523 ib_sa_free_multicast(mc->multicast.ib);
3524 kfree(mc);
3525 break;
3526 case IB_LINK_LAYER_ETHERNET:
3527 kref_put(&mc->mcref, release_mc);
3528 break;
3529 default:
3530 break;
3531 }
3532 }
3533 return;
3534 }
3535 }
3536 spin_unlock_irq(&id_priv->lock);
3537 }
3538 EXPORT_SYMBOL(rdma_leave_multicast);
3539
cma_netdev_change(struct net_device * ndev,struct rdma_id_private * id_priv)3540 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3541 {
3542 struct rdma_dev_addr *dev_addr;
3543 struct cma_ndev_work *work;
3544
3545 dev_addr = &id_priv->id.route.addr.dev_addr;
3546
3547 if ((dev_addr->bound_dev_if == ndev->if_index) &&
3548 memcmp(dev_addr->src_dev_addr, IF_LLADDR(ndev), ndev->if_addrlen)) {
3549 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3550 ndev->if_xname, &id_priv->id);
3551 work = kzalloc(sizeof *work, GFP_KERNEL);
3552 if (!work)
3553 return -ENOMEM;
3554
3555 INIT_WORK(&work->work, cma_ndev_work_handler);
3556 work->id = id_priv;
3557 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3558 atomic_inc(&id_priv->refcount);
3559 queue_work(cma_wq, &work->work);
3560 }
3561
3562 return 0;
3563 }
3564
cma_netdev_callback(struct notifier_block * self,unsigned long event,void * ctx)3565 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3566 void *ctx)
3567 {
3568 struct net_device *ndev = (struct net_device *)ctx;
3569 struct cma_device *cma_dev;
3570 struct rdma_id_private *id_priv;
3571 int ret = NOTIFY_DONE;
3572
3573 /* BONDING related, commented out until the bonding is resolved */
3574 #if 0
3575 if (dev_net(ndev) != &init_net)
3576 return NOTIFY_DONE;
3577
3578 if (event != NETDEV_BONDING_FAILOVER)
3579 return NOTIFY_DONE;
3580
3581 if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3582 return NOTIFY_DONE;
3583 #endif
3584 if (event != NETDEV_DOWN && event != NETDEV_UNREGISTER)
3585 return NOTIFY_DONE;
3586
3587 mutex_lock(&lock);
3588 list_for_each_entry(cma_dev, &dev_list, list)
3589 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3590 ret = cma_netdev_change(ndev, id_priv);
3591 if (ret)
3592 goto out;
3593 }
3594
3595 out:
3596 mutex_unlock(&lock);
3597 return ret;
3598 }
3599
3600 static struct notifier_block cma_nb = {
3601 .notifier_call = cma_netdev_callback
3602 };
3603
cma_add_one(struct ib_device * device)3604 static void cma_add_one(struct ib_device *device)
3605 {
3606 struct cma_device *cma_dev;
3607 struct rdma_id_private *id_priv;
3608
3609 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3610 if (!cma_dev)
3611 return;
3612
3613 cma_dev->device = device;
3614
3615 init_completion(&cma_dev->comp);
3616 atomic_set(&cma_dev->refcount, 1);
3617 INIT_LIST_HEAD(&cma_dev->id_list);
3618 ib_set_client_data(device, &cma_client, cma_dev);
3619
3620 mutex_lock(&lock);
3621 list_add_tail(&cma_dev->list, &dev_list);
3622 list_for_each_entry(id_priv, &listen_any_list, list)
3623 cma_listen_on_dev(id_priv, cma_dev);
3624 mutex_unlock(&lock);
3625 }
3626
cma_remove_id_dev(struct rdma_id_private * id_priv)3627 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3628 {
3629 struct rdma_cm_event event;
3630 enum rdma_cm_state state;
3631 int ret = 0;
3632
3633 /* Record that we want to remove the device */
3634 state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
3635 if (state == RDMA_CM_DESTROYING)
3636 return 0;
3637
3638 cma_cancel_operation(id_priv, state);
3639 mutex_lock(&id_priv->handler_mutex);
3640
3641 /* Check for destruction from another callback. */
3642 if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
3643 goto out;
3644
3645 memset(&event, 0, sizeof event);
3646 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3647 ret = id_priv->id.event_handler(&id_priv->id, &event);
3648 out:
3649 mutex_unlock(&id_priv->handler_mutex);
3650 return ret;
3651 }
3652
cma_process_remove(struct cma_device * cma_dev)3653 static void cma_process_remove(struct cma_device *cma_dev)
3654 {
3655 struct rdma_id_private *id_priv;
3656 int ret;
3657
3658 mutex_lock(&lock);
3659 while (!list_empty(&cma_dev->id_list)) {
3660 id_priv = list_entry(cma_dev->id_list.next,
3661 struct rdma_id_private, list);
3662
3663 list_del(&id_priv->listen_list);
3664 list_del_init(&id_priv->list);
3665 atomic_inc(&id_priv->refcount);
3666 mutex_unlock(&lock);
3667
3668 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3669 cma_deref_id(id_priv);
3670 if (ret)
3671 rdma_destroy_id(&id_priv->id);
3672
3673 mutex_lock(&lock);
3674 }
3675 mutex_unlock(&lock);
3676
3677 cma_deref_dev(cma_dev);
3678 wait_for_completion(&cma_dev->comp);
3679 }
3680
cma_remove_one(struct ib_device * device)3681 static void cma_remove_one(struct ib_device *device)
3682 {
3683 struct cma_device *cma_dev;
3684
3685 cma_dev = ib_get_client_data(device, &cma_client);
3686 if (!cma_dev)
3687 return;
3688
3689 mutex_lock(&lock);
3690 list_del(&cma_dev->list);
3691 mutex_unlock(&lock);
3692
3693 cma_process_remove(cma_dev);
3694 kfree(cma_dev);
3695 }
3696
cma_init(void)3697 static int __init cma_init(void)
3698 {
3699 int ret = -ENOMEM;
3700
3701 cma_wq = create_singlethread_workqueue("rdma_cm");
3702 if (!cma_wq)
3703 return -ENOMEM;
3704
3705 cma_free_wq = create_singlethread_workqueue("rdma_cm_fr");
3706 if (!cma_free_wq)
3707 goto err1;
3708
3709 ib_sa_register_client(&sa_client);
3710 rdma_addr_register_client(&addr_client);
3711 register_netdevice_notifier(&cma_nb);
3712
3713 ret = ib_register_client(&cma_client);
3714 if (ret)
3715 goto err;
3716
3717 return 0;
3718
3719 err:
3720 unregister_netdevice_notifier(&cma_nb);
3721 rdma_addr_unregister_client(&addr_client);
3722 ib_sa_unregister_client(&sa_client);
3723
3724 destroy_workqueue(cma_free_wq);
3725 err1:
3726 destroy_workqueue(cma_wq);
3727 return ret;
3728 }
3729
cma_cleanup(void)3730 static void __exit cma_cleanup(void)
3731 {
3732 ib_unregister_client(&cma_client);
3733 unregister_netdevice_notifier(&cma_nb);
3734 rdma_addr_unregister_client(&addr_client);
3735 ib_sa_unregister_client(&sa_client);
3736 flush_workqueue(cma_free_wq);
3737 destroy_workqueue(cma_free_wq);
3738 destroy_workqueue(cma_wq);
3739 idr_destroy(&sdp_ps);
3740 idr_destroy(&tcp_ps);
3741 idr_destroy(&udp_ps);
3742 idr_destroy(&ipoib_ps);
3743 idr_destroy(&ib_ps);
3744 }
3745
3746 module_init(cma_init);
3747 module_exit(cma_cleanup);
3748