1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36 #include <linux/file.h>
37 #include <linux/fs.h>
38 #include <linux/slab.h>
39 #include <linux/moduleparam.h>
40 #include <linux/rbtree.h>
41 #include <linux/lockdep.h>
42 #include <rdma/ib_addr.h>
43
44 #include <asm/uaccess.h>
45 #include <asm/fcntl.h>
46 #include <sys/priv.h>
47
48 #include "uverbs.h"
49
50 static int disable_raw_qp_enforcement;
51 module_param_named(disable_raw_qp_enforcement, disable_raw_qp_enforcement, int,
52 0444);
53 MODULE_PARM_DESC(disable_raw_qp_enforcement, "Disable RAW QP enforcement for "
54 "being opened by root (default: 0)");
55
56 struct uverbs_lock_class {
57 struct lock_class_key key;
58 char name[16];
59 };
60
61 static struct uverbs_lock_class pd_lock_class = { .name = "PD-uobj" };
62 static struct uverbs_lock_class mr_lock_class = { .name = "MR-uobj" };
63 static struct uverbs_lock_class mw_lock_class = { .name = "MW-uobj" };
64 static struct uverbs_lock_class cq_lock_class = { .name = "CQ-uobj" };
65 static struct uverbs_lock_class qp_lock_class = { .name = "QP-uobj" };
66 static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" };
67 static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" };
68 static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" };
69 static struct uverbs_lock_class dct_lock_class = { .name = "DCT-uobj" };
70
uverbs_copy_from_udata(void * dest,struct ib_udata * udata,size_t len)71 static int uverbs_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)
72 {
73 return copy_from_user(dest, udata->inbuf, len) ? -EFAULT : 0;
74 }
75
uverbs_copy_to_udata(struct ib_udata * udata,void * src,size_t len)76 static int uverbs_copy_to_udata(struct ib_udata *udata, void *src, size_t len)
77 {
78 return copy_to_user(udata->outbuf, src, len) ? -EFAULT : 0;
79 }
80
81 static struct ib_udata_ops uverbs_copy = {
82 .copy_from = uverbs_copy_from_udata,
83 .copy_to = uverbs_copy_to_udata
84 };
85
86 #define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \
87 do { \
88 (udata)->ops = &uverbs_copy; \
89 (udata)->inbuf = (void __user *) (ibuf); \
90 (udata)->outbuf = (void __user *) (obuf); \
91 (udata)->inlen = (ilen); \
92 (udata)->outlen = (olen); \
93 } while (0)
94
95 enum uverbs_cmd_type {
96 IB_USER_VERBS_CMD_BASIC,
97 IB_USER_VERBS_CMD_EXTENDED
98 };
99
100 /*
101 * The ib_uobject locking scheme is as follows:
102 *
103 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it
104 * needs to be held during all idr operations. When an object is
105 * looked up, a reference must be taken on the object's kref before
106 * dropping this lock.
107 *
108 * - Each object also has an rwsem. This rwsem must be held for
109 * reading while an operation that uses the object is performed.
110 * For example, while registering an MR, the associated PD's
111 * uobject.mutex must be held for reading. The rwsem must be held
112 * for writing while initializing or destroying an object.
113 *
114 * - In addition, each object has a "live" flag. If this flag is not
115 * set, then lookups of the object will fail even if it is found in
116 * the idr. This handles a reader that blocks and does not acquire
117 * the rwsem until after the object is destroyed. The destroy
118 * operation will set the live flag to 0 and then drop the rwsem;
119 * this will allow the reader to acquire the rwsem, see that the
120 * live flag is 0, and then drop the rwsem and its reference to
121 * object. The underlying storage will not be freed until the last
122 * reference to the object is dropped.
123 */
124
init_uobj(struct ib_uobject * uobj,u64 user_handle,struct ib_ucontext * context,struct uverbs_lock_class * c)125 static void init_uobj(struct ib_uobject *uobj, u64 user_handle,
126 struct ib_ucontext *context, struct uverbs_lock_class *c)
127 {
128 uobj->user_handle = user_handle;
129 uobj->context = context;
130 kref_init(&uobj->ref);
131 init_rwsem(&uobj->mutex);
132 lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name);
133 uobj->live = 0;
134 }
135
release_uobj(struct kref * kref)136 static void release_uobj(struct kref *kref)
137 {
138 kfree(container_of(kref, struct ib_uobject, ref));
139 }
140
put_uobj(struct ib_uobject * uobj)141 static void put_uobj(struct ib_uobject *uobj)
142 {
143 kref_put(&uobj->ref, release_uobj);
144 }
145
put_uobj_read(struct ib_uobject * uobj)146 static void put_uobj_read(struct ib_uobject *uobj)
147 {
148 up_read(&uobj->mutex);
149 put_uobj(uobj);
150 }
151
put_uobj_write(struct ib_uobject * uobj)152 static void put_uobj_write(struct ib_uobject *uobj)
153 {
154 up_write(&uobj->mutex);
155 put_uobj(uobj);
156 }
157
idr_add_uobj(struct idr * idr,struct ib_uobject * uobj)158 static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj)
159 {
160 int ret;
161
162 retry:
163 if (!idr_pre_get(idr, GFP_KERNEL))
164 return -ENOMEM;
165
166 spin_lock(&ib_uverbs_idr_lock);
167 ret = idr_get_new(idr, uobj, &uobj->id);
168 spin_unlock(&ib_uverbs_idr_lock);
169
170 if (ret == -EAGAIN)
171 goto retry;
172
173 return ret;
174 }
175
idr_remove_uobj(struct idr * idr,struct ib_uobject * uobj)176 void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj)
177 {
178 spin_lock(&ib_uverbs_idr_lock);
179 idr_remove(idr, uobj->id);
180 spin_unlock(&ib_uverbs_idr_lock);
181 }
182
__idr_get_uobj(struct idr * idr,int id,struct ib_ucontext * context)183 static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id,
184 struct ib_ucontext *context)
185 {
186 struct ib_uobject *uobj;
187
188 spin_lock(&ib_uverbs_idr_lock);
189 uobj = idr_find(idr, id);
190 if (uobj) {
191 if (uobj->context == context)
192 kref_get(&uobj->ref);
193 else
194 uobj = NULL;
195 }
196 spin_unlock(&ib_uverbs_idr_lock);
197
198 return uobj;
199 }
200
idr_read_uobj(struct idr * idr,int id,struct ib_ucontext * context,int nested)201 static struct ib_uobject *idr_read_uobj(struct idr *idr, int id,
202 struct ib_ucontext *context, int nested)
203 {
204 struct ib_uobject *uobj;
205
206 uobj = __idr_get_uobj(idr, id, context);
207 if (!uobj)
208 return NULL;
209
210 if (nested)
211 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING);
212 else
213 down_read(&uobj->mutex);
214 if (!uobj->live) {
215 put_uobj_read(uobj);
216 return NULL;
217 }
218
219 return uobj;
220 }
221
idr_write_uobj(struct idr * idr,int id,struct ib_ucontext * context)222 static struct ib_uobject *idr_write_uobj(struct idr *idr, int id,
223 struct ib_ucontext *context)
224 {
225 struct ib_uobject *uobj;
226
227 uobj = __idr_get_uobj(idr, id, context);
228 if (!uobj)
229 return NULL;
230
231 down_write(&uobj->mutex);
232 if (!uobj->live) {
233 put_uobj_write(uobj);
234 return NULL;
235 }
236
237 return uobj;
238 }
239
idr_read_obj(struct idr * idr,int id,struct ib_ucontext * context,int nested)240 static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context,
241 int nested)
242 {
243 struct ib_uobject *uobj;
244
245 uobj = idr_read_uobj(idr, id, context, nested);
246 return uobj ? uobj->object : NULL;
247 }
248
idr_read_pd(int pd_handle,struct ib_ucontext * context)249 static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context)
250 {
251 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0);
252 }
253
put_pd_read(struct ib_pd * pd)254 static void put_pd_read(struct ib_pd *pd)
255 {
256 put_uobj_read(pd->uobject);
257 }
258
idr_read_cq(int cq_handle,struct ib_ucontext * context,int nested)259 static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested)
260 {
261 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested);
262 }
263
put_cq_read(struct ib_cq * cq)264 static void put_cq_read(struct ib_cq *cq)
265 {
266 put_uobj_read(cq->uobject);
267 }
268
idr_read_ah(int ah_handle,struct ib_ucontext * context)269 static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context)
270 {
271 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0);
272 }
273
put_ah_read(struct ib_ah * ah)274 static void put_ah_read(struct ib_ah *ah)
275 {
276 put_uobj_read(ah->uobject);
277 }
278
idr_read_qp(int qp_handle,struct ib_ucontext * context)279 static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context)
280 {
281 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0);
282 }
283
idr_write_qp(int qp_handle,struct ib_ucontext * context)284 static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context)
285 {
286 struct ib_uobject *uobj;
287
288 uobj = idr_write_uobj(&ib_uverbs_qp_idr, qp_handle, context);
289 return uobj ? uobj->object : NULL;
290 }
291
put_qp_read(struct ib_qp * qp)292 static void put_qp_read(struct ib_qp *qp)
293 {
294 put_uobj_read(qp->uobject);
295 }
296
put_qp_write(struct ib_qp * qp)297 static void put_qp_write(struct ib_qp *qp)
298 {
299 put_uobj_write(qp->uobject);
300 }
301
idr_read_dct(int dct_handle,struct ib_ucontext * context)302 static struct ib_dct *idr_read_dct(int dct_handle, struct ib_ucontext *context)
303 {
304 return idr_read_obj(&ib_uverbs_dct_idr, dct_handle, context, 0);
305 }
306
put_dct_read(struct ib_dct * dct)307 static void put_dct_read(struct ib_dct *dct)
308 {
309 put_uobj_read(dct->uobject);
310 }
311
idr_read_srq(int srq_handle,struct ib_ucontext * context)312 static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context)
313 {
314 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0);
315 }
316
put_srq_read(struct ib_srq * srq)317 static void put_srq_read(struct ib_srq *srq)
318 {
319 put_uobj_read(srq->uobject);
320 }
321
idr_read_xrcd(int xrcd_handle,struct ib_ucontext * context,struct ib_uobject ** uobj)322 static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context,
323 struct ib_uobject **uobj)
324 {
325 *uobj = idr_read_uobj(&ib_uverbs_xrcd_idr, xrcd_handle, context, 0);
326 return *uobj ? (*uobj)->object : NULL;
327 }
328
put_xrcd_read(struct ib_uobject * uobj)329 static void put_xrcd_read(struct ib_uobject *uobj)
330 {
331 put_uobj_read(uobj);
332 }
333
ib_uverbs_get_context(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)334 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
335 const char __user *buf,
336 int in_len, int out_len)
337 {
338 struct ib_uverbs_get_context cmd;
339 struct ib_uverbs_get_context_resp resp;
340 struct ib_udata udata;
341 struct ib_device *ibdev = file->device->ib_dev;
342 struct ib_ucontext *ucontext;
343 struct file *filp;
344 int ret;
345
346 if (out_len < sizeof resp)
347 return -ENOSPC;
348
349 if (copy_from_user(&cmd, buf, sizeof cmd))
350 return -EFAULT;
351
352 mutex_lock(&file->mutex);
353
354 if (file->ucontext) {
355 ret = -EINVAL;
356 goto err;
357 }
358
359 INIT_UDATA(&udata, buf + sizeof cmd,
360 (unsigned long) cmd.response + sizeof resp,
361 in_len - sizeof cmd, out_len - sizeof resp);
362
363 ucontext = ibdev->alloc_ucontext(ibdev, &udata);
364 if (IS_ERR(ucontext)) {
365 ret = PTR_ERR(ucontext);
366 goto err;
367 }
368
369 ucontext->device = ibdev;
370 INIT_LIST_HEAD(&ucontext->pd_list);
371 INIT_LIST_HEAD(&ucontext->mr_list);
372 INIT_LIST_HEAD(&ucontext->mw_list);
373 INIT_LIST_HEAD(&ucontext->cq_list);
374 INIT_LIST_HEAD(&ucontext->qp_list);
375 INIT_LIST_HEAD(&ucontext->srq_list);
376 INIT_LIST_HEAD(&ucontext->ah_list);
377 INIT_LIST_HEAD(&ucontext->xrcd_list);
378 INIT_LIST_HEAD(&ucontext->rule_list);
379 INIT_LIST_HEAD(&ucontext->dct_list);
380 ucontext->closing = 0;
381 ucontext->peer_mem_private_data = NULL;
382 ucontext->peer_mem_name = NULL;
383
384 resp.num_comp_vectors = file->device->num_comp_vectors;
385
386 ret = get_unused_fd();
387 if (ret < 0)
388 goto err_free;
389 resp.async_fd = ret;
390
391 filp = ib_uverbs_alloc_event_file(file, 1);
392 if (IS_ERR(filp)) {
393 ret = PTR_ERR(filp);
394 goto err_fd;
395 }
396
397 file->async_file = filp->private_data;
398
399 INIT_IB_EVENT_HANDLER(&file->event_handler, file->device->ib_dev,
400 ib_uverbs_event_handler);
401 ret = ib_register_event_handler(&file->event_handler);
402 if (ret)
403 goto err_file;
404
405 if (copy_to_user((void __user *) (unsigned long) cmd.response,
406 &resp, sizeof resp)) {
407 ret = -EFAULT;
408 goto err_file;
409 }
410 kref_get(&file->async_file->ref);
411 kref_get(&file->ref);
412 file->ucontext = ucontext;
413
414 fd_install(resp.async_fd, filp);
415
416 mutex_unlock(&file->mutex);
417
418 return in_len;
419
420 err_file:
421 fput(filp);
422
423 err_fd:
424 put_unused_fd(resp.async_fd);
425
426 err_free:
427 ibdev->dealloc_ucontext(ucontext);
428
429 err:
430 mutex_unlock(&file->mutex);
431 return ret;
432 }
433
ib_uverbs_query_device_assign(struct ib_uverbs_query_device_resp * resp,struct ib_device_attr * attr,struct ib_uverbs_file * file)434 static void ib_uverbs_query_device_assign(
435 struct ib_uverbs_query_device_resp *resp,
436 struct ib_device_attr *attr,
437 struct ib_uverbs_file *file)
438 {
439 memset(resp, 0, sizeof(*resp));
440
441 resp->fw_ver = attr->fw_ver;
442 resp->node_guid = file->device->ib_dev->node_guid;
443 resp->sys_image_guid = attr->sys_image_guid;
444 resp->max_mr_size = attr->max_mr_size;
445 resp->page_size_cap = attr->page_size_cap;
446 resp->vendor_id = attr->vendor_id;
447 resp->vendor_part_id = attr->vendor_part_id;
448 resp->hw_ver = attr->hw_ver;
449 resp->max_qp = attr->max_qp;
450 resp->max_qp_wr = attr->max_qp_wr;
451 resp->device_cap_flags = attr->device_cap_flags;
452 resp->max_sge = attr->max_sge;
453 resp->max_sge_rd = attr->max_sge_rd;
454 resp->max_cq = attr->max_cq;
455 resp->max_cqe = attr->max_cqe;
456 resp->max_mr = attr->max_mr;
457 resp->max_pd = attr->max_pd;
458 resp->max_qp_rd_atom = attr->max_qp_rd_atom;
459 resp->max_ee_rd_atom = attr->max_ee_rd_atom;
460 resp->max_res_rd_atom = attr->max_res_rd_atom;
461 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom;
462 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom;
463 resp->atomic_cap = attr->atomic_cap;
464 resp->max_ee = attr->max_ee;
465 resp->max_rdd = attr->max_rdd;
466 resp->max_mw = attr->max_mw;
467 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp;
468 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp;
469 resp->max_mcast_grp = attr->max_mcast_grp;
470 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach;
471 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
472 resp->max_ah = attr->max_ah;
473 resp->max_fmr = attr->max_fmr;
474 resp->max_map_per_fmr = attr->max_map_per_fmr;
475 resp->max_srq = attr->max_srq;
476 resp->max_srq_wr = attr->max_srq_wr;
477 resp->max_srq_sge = attr->max_srq_sge;
478 resp->max_pkeys = attr->max_pkeys;
479 resp->local_ca_ack_delay = attr->local_ca_ack_delay;
480 resp->phys_port_cnt = file->device->ib_dev->phys_port_cnt;
481 }
482
ib_uverbs_query_device(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)483 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
484 const char __user *buf,
485 int in_len, int out_len)
486 {
487 struct ib_uverbs_query_device cmd;
488 struct ib_uverbs_query_device_resp resp;
489 struct ib_device_attr attr;
490 int ret;
491
492 if (out_len < sizeof resp)
493 return -ENOSPC;
494
495 if (copy_from_user(&cmd, buf, sizeof cmd))
496 return -EFAULT;
497
498 ret = ib_query_device(file->device->ib_dev, &attr);
499 if (ret)
500 return ret;
501
502 ib_uverbs_query_device_assign(&resp, &attr, file);
503
504 if (copy_to_user((void __user *)(unsigned long) cmd.response,
505 &resp, sizeof(resp)))
506 return -EFAULT;
507
508 return in_len;
509 }
510
ib_uverbs_query_port(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)511 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
512 const char __user *buf,
513 int in_len, int out_len)
514 {
515 struct ib_uverbs_query_port cmd;
516 struct ib_uverbs_query_port_resp resp;
517 struct ib_port_attr attr;
518 int ret;
519
520 if (out_len < sizeof resp)
521 return -ENOSPC;
522
523 if (copy_from_user(&cmd, buf, sizeof cmd))
524 return -EFAULT;
525
526 ret = ib_query_port(file->device->ib_dev, cmd.port_num, &attr);
527 if (ret)
528 return ret;
529
530 memset(&resp, 0, sizeof resp);
531
532 resp.state = attr.state;
533 resp.max_mtu = attr.max_mtu;
534 resp.active_mtu = attr.active_mtu;
535 resp.gid_tbl_len = attr.gid_tbl_len;
536 resp.port_cap_flags = attr.port_cap_flags;
537 resp.max_msg_sz = attr.max_msg_sz;
538 resp.bad_pkey_cntr = attr.bad_pkey_cntr;
539 resp.qkey_viol_cntr = attr.qkey_viol_cntr;
540 resp.pkey_tbl_len = attr.pkey_tbl_len;
541 resp.lid = attr.lid;
542 resp.sm_lid = attr.sm_lid;
543 resp.lmc = attr.lmc;
544 resp.max_vl_num = attr.max_vl_num;
545 resp.sm_sl = attr.sm_sl;
546 resp.subnet_timeout = attr.subnet_timeout;
547 resp.init_type_reply = attr.init_type_reply;
548 resp.active_width = attr.active_width;
549 resp.active_speed = attr.active_speed;
550 resp.phys_state = attr.phys_state;
551 resp.link_layer = rdma_port_get_link_layer(file->device->ib_dev,
552 cmd.port_num);
553
554 if (copy_to_user((void __user *) (unsigned long) cmd.response,
555 &resp, sizeof resp))
556 return -EFAULT;
557
558 return in_len;
559 }
560
ib_uverbs_alloc_pd(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)561 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
562 const char __user *buf,
563 int in_len, int out_len)
564 {
565 struct ib_uverbs_alloc_pd cmd;
566 struct ib_uverbs_alloc_pd_resp resp;
567 struct ib_udata udata;
568 struct ib_uobject *uobj;
569 struct ib_pd *pd;
570 int ret;
571
572 if (out_len < sizeof resp)
573 return -ENOSPC;
574
575 if (copy_from_user(&cmd, buf, sizeof cmd))
576 return -EFAULT;
577
578 INIT_UDATA(&udata, buf + sizeof cmd,
579 (unsigned long) cmd.response + sizeof resp,
580 in_len - sizeof cmd, out_len - sizeof resp);
581
582 uobj = kmalloc(sizeof *uobj, GFP_KERNEL);
583 if (!uobj)
584 return -ENOMEM;
585
586 init_uobj(uobj, 0, file->ucontext, &pd_lock_class);
587 down_write(&uobj->mutex);
588
589 pd = file->device->ib_dev->alloc_pd(file->device->ib_dev,
590 file->ucontext, &udata);
591 if (IS_ERR(pd)) {
592 ret = PTR_ERR(pd);
593 goto err;
594 }
595
596 pd->device = file->device->ib_dev;
597 pd->uobject = uobj;
598 atomic_set(&pd->usecnt, 0);
599
600 uobj->object = pd;
601 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj);
602 if (ret)
603 goto err_idr;
604
605 memset(&resp, 0, sizeof resp);
606 resp.pd_handle = uobj->id;
607
608 if (copy_to_user((void __user *) (unsigned long) cmd.response,
609 &resp, sizeof resp)) {
610 ret = -EFAULT;
611 goto err_copy;
612 }
613
614 mutex_lock(&file->mutex);
615 list_add_tail(&uobj->list, &file->ucontext->pd_list);
616 mutex_unlock(&file->mutex);
617
618 uobj->live = 1;
619
620 up_write(&uobj->mutex);
621
622 return in_len;
623
624 err_copy:
625 idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
626
627 err_idr:
628 ib_dealloc_pd(pd);
629
630 err:
631 put_uobj_write(uobj);
632 return ret;
633 }
634
ib_uverbs_dealloc_pd(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)635 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file,
636 const char __user *buf,
637 int in_len, int out_len)
638 {
639 struct ib_uverbs_dealloc_pd cmd;
640 struct ib_uobject *uobj;
641 int ret;
642
643 if (copy_from_user(&cmd, buf, sizeof cmd))
644 return -EFAULT;
645
646 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext);
647 if (!uobj)
648 return -EINVAL;
649
650 ret = ib_dealloc_pd(uobj->object);
651 if (!ret)
652 uobj->live = 0;
653
654 put_uobj_write(uobj);
655
656 if (ret)
657 return ret;
658
659 idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
660
661 mutex_lock(&file->mutex);
662 list_del(&uobj->list);
663 mutex_unlock(&file->mutex);
664
665 put_uobj(uobj);
666
667 return in_len;
668 }
669
670 struct xrcd_table_entry {
671 struct rb_node node;
672 struct ib_xrcd *xrcd;
673 struct inode *inode;
674 };
675
xrcd_table_insert(struct ib_uverbs_device * dev,struct inode * inode,struct ib_xrcd * xrcd)676 static int xrcd_table_insert(struct ib_uverbs_device *dev,
677 struct inode *inode,
678 struct ib_xrcd *xrcd)
679 {
680 struct xrcd_table_entry *entry, *scan;
681 struct rb_node **p = &dev->xrcd_tree.rb_node;
682 struct rb_node *parent = NULL;
683
684 entry = kmalloc(sizeof *entry, GFP_KERNEL);
685 if (!entry)
686 return -ENOMEM;
687
688 entry->xrcd = xrcd;
689 entry->inode = inode;
690
691 while (*p) {
692 parent = *p;
693 scan = rb_entry(parent, struct xrcd_table_entry, node);
694
695 if (inode < scan->inode) {
696 p = &(*p)->rb_left;
697 } else if (inode > scan->inode) {
698 p = &(*p)->rb_right;
699 } else {
700 kfree(entry);
701 return -EEXIST;
702 }
703 }
704
705 rb_link_node(&entry->node, parent, p);
706 rb_insert_color(&entry->node, &dev->xrcd_tree);
707 igrab(inode);
708 return 0;
709 }
710
xrcd_table_search(struct ib_uverbs_device * dev,struct inode * inode)711 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
712 struct inode *inode)
713 {
714 struct xrcd_table_entry *entry;
715 struct rb_node *p = dev->xrcd_tree.rb_node;
716
717 while (p) {
718 entry = rb_entry(p, struct xrcd_table_entry, node);
719
720 if (inode < entry->inode)
721 p = p->rb_left;
722 else if (inode > entry->inode)
723 p = p->rb_right;
724 else
725 return entry;
726 }
727
728 return NULL;
729 }
730
find_xrcd(struct ib_uverbs_device * dev,struct inode * inode)731 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
732 {
733 struct xrcd_table_entry *entry;
734
735 entry = xrcd_table_search(dev, inode);
736 if (!entry)
737 return NULL;
738
739 return entry->xrcd;
740 }
741
xrcd_table_delete(struct ib_uverbs_device * dev,struct inode * inode)742 static void xrcd_table_delete(struct ib_uverbs_device *dev,
743 struct inode *inode)
744 {
745 struct xrcd_table_entry *entry;
746
747 entry = xrcd_table_search(dev, inode);
748 if (entry) {
749 iput(inode);
750 rb_erase(&entry->node, &dev->xrcd_tree);
751 kfree(entry);
752 }
753 }
754
ib_uverbs_open_xrcd(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)755 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
756 const char __user *buf, int in_len,
757 int out_len)
758 {
759 struct ib_uverbs_open_xrcd cmd;
760 struct ib_uverbs_open_xrcd_resp resp;
761 struct ib_udata udata;
762 struct ib_uxrcd_object *obj;
763 struct ib_xrcd *xrcd = NULL;
764 struct fd f = {NULL};
765 struct inode *inode = NULL;
766 int ret = 0;
767 int new_xrcd = 0;
768
769 if (out_len < sizeof resp)
770 return -ENOSPC;
771
772 if (copy_from_user(&cmd, buf, sizeof cmd))
773 return -EFAULT;
774
775 INIT_UDATA(&udata, buf + sizeof cmd,
776 (unsigned long) cmd.response + sizeof resp,
777 in_len - sizeof cmd, out_len - sizeof resp);
778
779 mutex_lock(&file->device->xrcd_tree_mutex);
780
781 if (cmd.fd != -1) {
782 /* search for file descriptor */
783 f = fdget(cmd.fd);
784 if (!f.file) {
785 ret = -EBADF;
786 goto err_tree_mutex_unlock;
787 }
788
789 inode = f.file->f_dentry->d_inode;
790 xrcd = find_xrcd(file->device, inode);
791 if (!xrcd && !(cmd.oflags & O_CREAT)) {
792 /* no file descriptor. Need CREATE flag */
793 ret = -EAGAIN;
794 goto err_tree_mutex_unlock;
795 }
796
797 if (xrcd && cmd.oflags & O_EXCL) {
798 ret = -EINVAL;
799 goto err_tree_mutex_unlock;
800 }
801 }
802
803 obj = kmalloc(sizeof *obj, GFP_KERNEL);
804 if (!obj) {
805 ret = -ENOMEM;
806 goto err_tree_mutex_unlock;
807 }
808
809 init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class);
810
811 down_write(&obj->uobject.mutex);
812
813 if (!xrcd) {
814 xrcd = file->device->ib_dev->alloc_xrcd(file->device->ib_dev,
815 file->ucontext, &udata);
816 if (IS_ERR(xrcd)) {
817 ret = PTR_ERR(xrcd);
818 goto err;
819 }
820
821 xrcd->inode = inode;
822 xrcd->device = file->device->ib_dev;
823 atomic_set(&xrcd->usecnt, 0);
824 mutex_init(&xrcd->tgt_qp_mutex);
825 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
826 new_xrcd = 1;
827 }
828
829 atomic_set(&obj->refcnt, 0);
830 obj->uobject.object = xrcd;
831 ret = idr_add_uobj(&ib_uverbs_xrcd_idr, &obj->uobject);
832 if (ret)
833 goto err_idr;
834
835 memset(&resp, 0, sizeof resp);
836 resp.xrcd_handle = obj->uobject.id;
837
838 if (inode) {
839 if (new_xrcd) {
840 /* create new inode/xrcd table entry */
841 ret = xrcd_table_insert(file->device, inode, xrcd);
842 if (ret)
843 goto err_insert_xrcd;
844 }
845 atomic_inc(&xrcd->usecnt);
846 }
847
848 if (copy_to_user((void __user *) (unsigned long) cmd.response,
849 &resp, sizeof resp)) {
850 ret = -EFAULT;
851 goto err_copy;
852 }
853
854 if (f.file)
855 fdput(f);
856
857 mutex_lock(&file->mutex);
858 list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list);
859 mutex_unlock(&file->mutex);
860
861 obj->uobject.live = 1;
862 up_write(&obj->uobject.mutex);
863
864 mutex_unlock(&file->device->xrcd_tree_mutex);
865 return in_len;
866
867 err_copy:
868 if (inode) {
869 if (new_xrcd)
870 xrcd_table_delete(file->device, inode);
871 atomic_dec(&xrcd->usecnt);
872 }
873
874 err_insert_xrcd:
875 idr_remove_uobj(&ib_uverbs_xrcd_idr, &obj->uobject);
876
877 err_idr:
878 ib_dealloc_xrcd(xrcd);
879
880 err:
881 put_uobj_write(&obj->uobject);
882
883 err_tree_mutex_unlock:
884 if (f.file)
885 fdput(f);
886
887 mutex_unlock(&file->device->xrcd_tree_mutex);
888
889 return ret;
890 }
891
ib_uverbs_close_xrcd(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)892 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file,
893 const char __user *buf, int in_len,
894 int out_len)
895 {
896 struct ib_uverbs_close_xrcd cmd;
897 struct ib_uobject *uobj;
898 struct ib_xrcd *xrcd = NULL;
899 struct inode *inode = NULL;
900 struct ib_uxrcd_object *obj;
901 int live;
902 int ret = 0;
903
904 if (copy_from_user(&cmd, buf, sizeof cmd))
905 return -EFAULT;
906
907 mutex_lock(&file->device->xrcd_tree_mutex);
908 uobj = idr_write_uobj(&ib_uverbs_xrcd_idr, cmd.xrcd_handle, file->ucontext);
909 if (!uobj) {
910 ret = -EINVAL;
911 goto out;
912 }
913
914 xrcd = uobj->object;
915 inode = xrcd->inode;
916 obj = container_of(uobj, struct ib_uxrcd_object, uobject);
917 if (atomic_read(&obj->refcnt)) {
918 put_uobj_write(uobj);
919 ret = -EBUSY;
920 goto out;
921 }
922
923 if (!inode || atomic_dec_and_test(&xrcd->usecnt)) {
924 ret = ib_dealloc_xrcd(uobj->object);
925 if (!ret)
926 uobj->live = 0;
927 }
928
929 live = uobj->live;
930 if (inode && ret)
931 atomic_inc(&xrcd->usecnt);
932
933 put_uobj_write(uobj);
934
935 if (ret)
936 goto out;
937
938 if (inode && !live)
939 xrcd_table_delete(file->device, inode);
940
941 idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj);
942 mutex_lock(&file->mutex);
943 list_del(&uobj->list);
944 mutex_unlock(&file->mutex);
945
946 put_uobj(uobj);
947 ret = in_len;
948
949 out:
950 mutex_unlock(&file->device->xrcd_tree_mutex);
951 return ret;
952 }
953
ib_uverbs_dealloc_xrcd(struct ib_uverbs_device * dev,struct ib_xrcd * xrcd)954 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev,
955 struct ib_xrcd *xrcd)
956 {
957 struct inode *inode;
958
959 inode = xrcd->inode;
960 if (inode && !atomic_dec_and_test(&xrcd->usecnt))
961 return;
962
963 ib_dealloc_xrcd(xrcd);
964
965 if (inode)
966 xrcd_table_delete(dev, inode);
967 }
968
ib_uverbs_reg_mr(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)969 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
970 const char __user *buf, int in_len,
971 int out_len)
972 {
973 struct ib_uverbs_reg_mr cmd;
974 struct ib_uverbs_reg_mr_resp resp;
975 struct ib_udata udata;
976 struct ib_uobject *uobj;
977 struct ib_pd *pd;
978 struct ib_mr *mr;
979 int ret;
980
981 if (out_len < sizeof resp)
982 return -ENOSPC;
983
984 if (copy_from_user(&cmd, buf, sizeof cmd))
985 return -EFAULT;
986
987 INIT_UDATA(&udata, buf + sizeof cmd,
988 (unsigned long) cmd.response + sizeof resp,
989 in_len - sizeof cmd, out_len - sizeof resp);
990
991 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
992 return -EINVAL;
993
994 ret = ib_check_mr_access(cmd.access_flags);
995 if (ret)
996 return ret;
997
998 uobj = kmalloc(sizeof *uobj, GFP_KERNEL);
999 if (!uobj)
1000 return -ENOMEM;
1001
1002 init_uobj(uobj, 0, file->ucontext, &mr_lock_class);
1003 down_write(&uobj->mutex);
1004
1005 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
1006 if (!pd) {
1007 ret = -EINVAL;
1008 goto err_free;
1009 }
1010 /* We first get a new "obj id" to be passed later to reg mr for
1011 further use as mr_id.
1012 */
1013 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj);
1014 if (ret)
1015 goto err_put;
1016
1017 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
1018 cmd.access_flags, &udata, uobj->id);
1019 if (IS_ERR(mr)) {
1020 ret = PTR_ERR(mr);
1021 goto err_remove_uobj;
1022 }
1023
1024 mr->device = pd->device;
1025 mr->pd = pd;
1026 mr->uobject = uobj;
1027 atomic_inc(&pd->usecnt);
1028 atomic_set(&mr->usecnt, 0);
1029
1030 uobj->object = mr;
1031
1032 memset(&resp, 0, sizeof resp);
1033 resp.lkey = mr->lkey;
1034 resp.rkey = mr->rkey;
1035 resp.mr_handle = uobj->id;
1036
1037 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1038 &resp, sizeof resp)) {
1039 ret = -EFAULT;
1040 goto err_copy;
1041 }
1042
1043 put_pd_read(pd);
1044
1045 mutex_lock(&file->mutex);
1046 list_add_tail(&uobj->list, &file->ucontext->mr_list);
1047 mutex_unlock(&file->mutex);
1048
1049 uobj->live = 1;
1050
1051 up_write(&uobj->mutex);
1052
1053 return in_len;
1054
1055 err_copy:
1056 ib_dereg_mr(mr);
1057
1058 err_remove_uobj:
1059 idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
1060
1061 err_put:
1062 put_pd_read(pd);
1063
1064 err_free:
1065 put_uobj_write(uobj);
1066 return ret;
1067 }
1068
ib_uverbs_dereg_mr(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1069 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file,
1070 const char __user *buf, int in_len,
1071 int out_len)
1072 {
1073 struct ib_uverbs_dereg_mr cmd;
1074 struct ib_mr *mr;
1075 struct ib_uobject *uobj;
1076 int ret = -EINVAL;
1077
1078 if (copy_from_user(&cmd, buf, sizeof cmd))
1079 return -EFAULT;
1080
1081 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext);
1082 if (!uobj)
1083 return -EINVAL;
1084
1085 mr = uobj->object;
1086
1087 ret = ib_dereg_mr(mr);
1088 if (!ret)
1089 uobj->live = 0;
1090
1091 put_uobj_write(uobj);
1092
1093 if (ret)
1094 return ret;
1095
1096 idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
1097
1098 mutex_lock(&file->mutex);
1099 list_del(&uobj->list);
1100 mutex_unlock(&file->mutex);
1101
1102 put_uobj(uobj);
1103
1104 return in_len;
1105 }
1106
ib_uverbs_alloc_mw(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1107 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
1108 const char __user *buf, int in_len,
1109 int out_len)
1110 {
1111 struct ib_uverbs_alloc_mw cmd;
1112 struct ib_uverbs_alloc_mw_resp resp;
1113 struct ib_uobject *uobj;
1114 struct ib_pd *pd;
1115 struct ib_mw *mw;
1116 int ret;
1117
1118 if (out_len < sizeof(resp))
1119 return -ENOSPC;
1120
1121 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1122 return -EFAULT;
1123
1124 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL);
1125 if (!uobj)
1126 return -ENOMEM;
1127
1128 init_uobj(uobj, 0, file->ucontext, &mw_lock_class);
1129 down_write(&uobj->mutex);
1130
1131 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
1132 if (!pd) {
1133 ret = -EINVAL;
1134 goto err_free;
1135 }
1136
1137 mw = pd->device->alloc_mw(pd, cmd.mw_type);
1138 if (IS_ERR(mw)) {
1139 ret = PTR_ERR(mw);
1140 goto err_put;
1141 }
1142
1143 mw->device = pd->device;
1144 mw->pd = pd;
1145 mw->uobject = uobj;
1146 atomic_inc(&pd->usecnt);
1147
1148 uobj->object = mw;
1149 ret = idr_add_uobj(&ib_uverbs_mw_idr, uobj);
1150 if (ret)
1151 goto err_unalloc;
1152
1153 memset(&resp, 0, sizeof(resp));
1154 resp.rkey = mw->rkey;
1155 resp.mw_handle = uobj->id;
1156
1157 if (copy_to_user((void __user *)(unsigned long)cmd.response,
1158 &resp, sizeof(resp))) {
1159 ret = -EFAULT;
1160 goto err_copy;
1161 }
1162
1163 put_pd_read(pd);
1164
1165 mutex_lock(&file->mutex);
1166 list_add_tail(&uobj->list, &file->ucontext->mw_list);
1167 mutex_unlock(&file->mutex);
1168
1169 uobj->live = 1;
1170
1171 up_write(&uobj->mutex);
1172
1173 return in_len;
1174
1175 err_copy:
1176 idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
1177
1178 err_unalloc:
1179 ib_dealloc_mw(mw);
1180
1181 err_put:
1182 put_pd_read(pd);
1183
1184 err_free:
1185 put_uobj_write(uobj);
1186 return ret;
1187 }
1188
ib_uverbs_dealloc_mw(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1189 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file,
1190 const char __user *buf, int in_len,
1191 int out_len)
1192 {
1193 struct ib_uverbs_dealloc_mw cmd;
1194 struct ib_mw *mw;
1195 struct ib_uobject *uobj;
1196 int ret = -EINVAL;
1197
1198 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1199 return -EFAULT;
1200
1201 uobj = idr_write_uobj(&ib_uverbs_mw_idr, cmd.mw_handle, file->ucontext);
1202 if (!uobj)
1203 return -EINVAL;
1204
1205 mw = uobj->object;
1206
1207 ret = ib_dealloc_mw(mw);
1208 if (!ret)
1209 uobj->live = 0;
1210
1211 put_uobj_write(uobj);
1212
1213 if (ret)
1214 return ret;
1215
1216 idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
1217
1218 mutex_lock(&file->mutex);
1219 list_del(&uobj->list);
1220 mutex_unlock(&file->mutex);
1221
1222 put_uobj(uobj);
1223
1224 return in_len;
1225 }
1226
ib_uverbs_create_comp_channel(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1227 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file,
1228 const char __user *buf, int in_len,
1229 int out_len)
1230 {
1231 struct ib_uverbs_create_comp_channel cmd;
1232 struct ib_uverbs_create_comp_channel_resp resp;
1233 struct file *filp;
1234 int ret;
1235
1236 if (out_len < sizeof resp)
1237 return -ENOSPC;
1238
1239 if (copy_from_user(&cmd, buf, sizeof cmd))
1240 return -EFAULT;
1241
1242 ret = get_unused_fd();
1243 if (ret < 0)
1244 return ret;
1245 resp.fd = ret;
1246
1247 filp = ib_uverbs_alloc_event_file(file, 0);
1248 if (IS_ERR(filp)) {
1249 put_unused_fd(resp.fd);
1250 return PTR_ERR(filp);
1251 }
1252
1253 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1254 &resp, sizeof resp)) {
1255 put_unused_fd(resp.fd);
1256 fput(filp);
1257 return -EFAULT;
1258 }
1259
1260 fd_install(resp.fd, filp);
1261 return in_len;
1262 }
1263
create_cq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len,void * vcmd,int ex,void __user * response)1264 static ssize_t create_cq(struct ib_uverbs_file *file,
1265 const char __user *buf, int in_len,
1266 int out_len, void *vcmd, int ex,
1267 void __user *response)
1268 {
1269 struct ib_uverbs_create_cq *cmd;
1270 struct ib_uverbs_create_cq_ex *cmd_e;
1271 struct ib_uverbs_create_cq_resp resp;
1272 struct ib_udata udata;
1273 struct ib_ucq_object *obj;
1274 struct ib_uverbs_event_file *ev_file = NULL;
1275 struct ib_cq *cq;
1276 struct ib_cq_init_attr attr;
1277 int cmd_sz;
1278 int ret;
1279
1280 if (out_len < sizeof resp)
1281 return -ENOSPC;
1282
1283 cmd = vcmd;
1284 cmd_e = vcmd;
1285 cmd_sz = ex ? sizeof(*cmd_e) : sizeof(*cmd);
1286 INIT_UDATA(&udata, buf + cmd_sz, response + sizeof(resp),
1287 in_len - sizeof(cmd), out_len - sizeof(resp));
1288
1289 if (cmd->comp_vector >= file->device->num_comp_vectors)
1290 return -EINVAL;
1291
1292 obj = kmalloc(sizeof *obj, GFP_KERNEL);
1293 if (!obj)
1294 return -ENOMEM;
1295
1296 init_uobj(&obj->uobject, cmd->user_handle, file->ucontext,
1297 &cq_lock_class);
1298 down_write(&obj->uobject.mutex);
1299
1300 if (cmd->comp_channel >= 0) {
1301 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel);
1302 if (!ev_file) {
1303 ret = -EINVAL;
1304 goto err;
1305 }
1306 }
1307
1308 obj->uverbs_file = file;
1309 obj->comp_events_reported = 0;
1310 obj->async_events_reported = 0;
1311 INIT_LIST_HEAD(&obj->comp_list);
1312 INIT_LIST_HEAD(&obj->async_list);
1313
1314 memset(&attr, 0, sizeof(attr));
1315 attr.cqe = cmd->cqe;
1316 attr.comp_vector = cmd->comp_vector;
1317 if (ex && (cmd_e->comp_mask & IB_UVERBS_CREATE_CQ_EX_CAP_FLAGS))
1318 attr.flags = cmd_e->create_flags;
1319 cq = file->device->ib_dev->create_cq(file->device->ib_dev, &attr,
1320 file->ucontext, &udata);
1321 if (IS_ERR(cq)) {
1322 ret = PTR_ERR(cq);
1323 goto err_file;
1324 }
1325
1326 cq->device = file->device->ib_dev;
1327 cq->uobject = &obj->uobject;
1328 cq->comp_handler = ib_uverbs_comp_handler;
1329 cq->event_handler = ib_uverbs_cq_event_handler;
1330 cq->cq_context = ev_file;
1331 atomic_set(&cq->usecnt, 0);
1332
1333 obj->uobject.object = cq;
1334 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject);
1335 if (ret)
1336 goto err_free;
1337
1338 memset(&resp, 0, sizeof resp);
1339 resp.cq_handle = obj->uobject.id;
1340 resp.cqe = cq->cqe;
1341
1342 if (copy_to_user(response, &resp, sizeof(resp))) {
1343 ret = -EFAULT;
1344 goto err_copy;
1345 }
1346
1347 mutex_lock(&file->mutex);
1348 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list);
1349 mutex_unlock(&file->mutex);
1350
1351 obj->uobject.live = 1;
1352
1353 up_write(&obj->uobject.mutex);
1354
1355 return in_len;
1356
1357 err_copy:
1358 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject);
1359
1360 err_free:
1361 ib_destroy_cq(cq);
1362
1363 err_file:
1364 if (ev_file)
1365 ib_uverbs_release_ucq(file, ev_file, obj);
1366
1367 err:
1368 put_uobj_write(&obj->uobject);
1369 return ret;
1370 }
1371
ib_uverbs_create_cq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1372 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
1373 const char __user *buf, int in_len,
1374 int out_len)
1375 {
1376 struct ib_uverbs_create_cq cmd;
1377
1378 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1379 return -EFAULT;
1380
1381 return create_cq(file, buf, in_len, out_len, &cmd,
1382 IB_USER_VERBS_CMD_BASIC,
1383 (void __user *) (unsigned long) cmd.response);
1384 }
1385
ib_uverbs_resize_cq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1386 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
1387 const char __user *buf, int in_len,
1388 int out_len)
1389 {
1390 struct ib_uverbs_resize_cq cmd;
1391 struct ib_uverbs_resize_cq_resp resp;
1392 struct ib_udata udata;
1393 struct ib_cq *cq;
1394 int ret = -EINVAL;
1395
1396 if (copy_from_user(&cmd, buf, sizeof cmd))
1397 return -EFAULT;
1398
1399 INIT_UDATA(&udata, buf + sizeof cmd,
1400 (unsigned long) cmd.response + sizeof resp,
1401 in_len - sizeof cmd, out_len - sizeof resp);
1402
1403 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
1404 if (!cq)
1405 return -EINVAL;
1406
1407 ret = cq->device->resize_cq(cq, cmd.cqe, &udata);
1408 if (ret)
1409 goto out;
1410
1411 resp.cqe = cq->cqe;
1412
1413 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1414 &resp, sizeof resp.cqe))
1415 ret = -EFAULT;
1416
1417 out:
1418 put_cq_read(cq);
1419
1420 return ret ? ret : in_len;
1421 }
1422
copy_wc_to_user(void __user * dest,struct ib_wc * wc)1423 static int copy_wc_to_user(void __user *dest, struct ib_wc *wc)
1424 {
1425 struct ib_uverbs_wc tmp;
1426
1427 tmp.wr_id = wc->wr_id;
1428 tmp.status = wc->status;
1429 tmp.opcode = wc->opcode;
1430 tmp.vendor_err = wc->vendor_err;
1431 tmp.byte_len = wc->byte_len;
1432 tmp.ex.imm_data = (__u32 __force) wc->ex.imm_data;
1433 tmp.qp_num = wc->qp->qp_num;
1434 tmp.src_qp = wc->src_qp;
1435 tmp.wc_flags = wc->wc_flags;
1436 tmp.pkey_index = wc->pkey_index;
1437 tmp.slid = wc->slid;
1438 tmp.sl = wc->sl;
1439 tmp.dlid_path_bits = wc->dlid_path_bits;
1440 tmp.port_num = wc->port_num;
1441 tmp.reserved = 0;
1442
1443 if (copy_to_user(dest, &tmp, sizeof tmp))
1444 return -EFAULT;
1445
1446 return 0;
1447 }
1448
ib_uverbs_poll_cq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1449 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
1450 const char __user *buf, int in_len,
1451 int out_len)
1452 {
1453 struct ib_uverbs_poll_cq cmd;
1454 struct ib_uverbs_poll_cq_resp resp;
1455 u8 __user *header_ptr;
1456 u8 __user *data_ptr;
1457 struct ib_cq *cq;
1458 struct ib_wc wc;
1459 int ret;
1460
1461 if (copy_from_user(&cmd, buf, sizeof cmd))
1462 return -EFAULT;
1463
1464 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
1465 if (!cq)
1466 return -EINVAL;
1467
1468 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1469 header_ptr = (void __user *)(unsigned long) cmd.response;
1470 data_ptr = header_ptr + sizeof resp;
1471
1472 memset(&resp, 0, sizeof resp);
1473 while (resp.count < cmd.ne) {
1474 ret = ib_poll_cq(cq, 1, &wc);
1475 if (ret < 0)
1476 goto out_put;
1477 if (!ret)
1478 break;
1479
1480 ret = copy_wc_to_user(data_ptr, &wc);
1481 if (ret)
1482 goto out_put;
1483
1484 data_ptr += sizeof(struct ib_uverbs_wc);
1485 ++resp.count;
1486 }
1487
1488 if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1489 ret = -EFAULT;
1490 goto out_put;
1491 }
1492
1493 ret = in_len;
1494
1495 out_put:
1496 put_cq_read(cq);
1497 return ret;
1498 }
1499
ib_uverbs_req_notify_cq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1500 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file,
1501 const char __user *buf, int in_len,
1502 int out_len)
1503 {
1504 struct ib_uverbs_req_notify_cq cmd;
1505 struct ib_cq *cq;
1506
1507 if (copy_from_user(&cmd, buf, sizeof cmd))
1508 return -EFAULT;
1509
1510 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
1511 if (!cq)
1512 return -EINVAL;
1513
1514 ib_req_notify_cq(cq, cmd.solicited_only ?
1515 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1516
1517 put_cq_read(cq);
1518
1519 return in_len;
1520 }
1521
ib_uverbs_destroy_cq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1522 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
1523 const char __user *buf, int in_len,
1524 int out_len)
1525 {
1526 struct ib_uverbs_destroy_cq cmd;
1527 struct ib_uverbs_destroy_cq_resp resp;
1528 struct ib_uobject *uobj;
1529 struct ib_cq *cq;
1530 struct ib_ucq_object *obj;
1531 struct ib_uverbs_event_file *ev_file;
1532 int ret = -EINVAL;
1533
1534 if (copy_from_user(&cmd, buf, sizeof cmd))
1535 return -EFAULT;
1536
1537 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext);
1538 if (!uobj)
1539 return -EINVAL;
1540 cq = uobj->object;
1541 ev_file = cq->cq_context;
1542 obj = container_of(cq->uobject, struct ib_ucq_object, uobject);
1543
1544 ret = ib_destroy_cq(cq);
1545 if (!ret)
1546 uobj->live = 0;
1547
1548 put_uobj_write(uobj);
1549
1550 if (ret)
1551 return ret;
1552
1553 idr_remove_uobj(&ib_uverbs_cq_idr, uobj);
1554
1555 mutex_lock(&file->mutex);
1556 list_del(&uobj->list);
1557 mutex_unlock(&file->mutex);
1558
1559 ib_uverbs_release_ucq(file, ev_file, obj);
1560
1561 memset(&resp, 0, sizeof resp);
1562 resp.comp_events_reported = obj->comp_events_reported;
1563 resp.async_events_reported = obj->async_events_reported;
1564
1565 put_uobj(uobj);
1566
1567 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1568 &resp, sizeof resp))
1569 return -EFAULT;
1570
1571 return in_len;
1572 }
1573
ib_uverbs_create_qp(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1574 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1575 const char __user *buf, int in_len,
1576 int out_len)
1577 {
1578 void __user *response;
1579 struct ib_udata udata;
1580 struct ib_uqp_object *obj;
1581 struct ib_device *device;
1582 struct ib_pd *pd = NULL;
1583 struct ib_xrcd *xrcd = NULL;
1584 struct ib_uobject *uninitialized_var(xrcd_uobj);
1585 struct ib_cq *scq = NULL, *rcq = NULL;
1586 struct ib_srq *srq = NULL;
1587 struct ib_qp *qp;
1588 struct ib_qp_init_attr attr;
1589 int ret;
1590 union {
1591 struct ib_uverbs_create_qp basic;
1592 } cmd_obj;
1593 struct ib_uverbs_create_qp *cmd;
1594 size_t cmd_size = 0;
1595 union {
1596 struct ib_uverbs_create_qp_resp basic;
1597 } resp_obj;
1598 struct ib_uverbs_create_qp_resp *resp;
1599 size_t resp_size = 0;
1600
1601 cmd_size = sizeof(cmd_obj.basic);
1602 cmd = &cmd_obj.basic;
1603
1604 resp_size = sizeof(resp_obj.basic);
1605 resp = &resp_obj.basic;
1606
1607 if (out_len < resp_size)
1608 return -ENOSPC;
1609
1610 if (copy_from_user(&cmd_obj, buf, cmd_size))
1611 return -EFAULT;
1612
1613 response = (void __user *) (unsigned long) cmd->response;
1614
1615 if (!disable_raw_qp_enforcement &&
1616 cmd->qp_type == IB_QPT_RAW_PACKET && !priv_check(curthread, PRIV_NET_RAW))
1617 return -EPERM;
1618
1619 INIT_UDATA(&udata, buf + cmd_size, response + resp_size,
1620 in_len - cmd_size, out_len - resp_size);
1621
1622 obj = kzalloc(sizeof *obj, GFP_KERNEL);
1623 if (!obj)
1624 return -ENOMEM;
1625
1626 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &qp_lock_class);
1627 down_write(&obj->uevent.uobject.mutex);
1628
1629 if (cmd->qp_type == IB_QPT_XRC_TGT) {
1630 xrcd = idr_read_xrcd(cmd->pd_handle, file->ucontext, &xrcd_uobj);
1631 if (!xrcd) {
1632 ret = -EINVAL;
1633 goto err_put;
1634 }
1635 device = xrcd->device;
1636 } else {
1637 if (cmd->qp_type == IB_QPT_XRC_INI) {
1638 cmd->max_recv_wr = 0;
1639 cmd->max_recv_sge = 0;
1640 } else {
1641 if (cmd->is_srq) {
1642 srq = idr_read_srq(cmd->srq_handle, file->ucontext);
1643 if (!srq || srq->srq_type != IB_SRQT_BASIC) {
1644 ret = -EINVAL;
1645 goto err_put;
1646 }
1647 }
1648
1649 if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1650 rcq = idr_read_cq(cmd->recv_cq_handle, file->ucontext, 0);
1651 if (!rcq) {
1652 ret = -EINVAL;
1653 goto err_put;
1654 }
1655 }
1656 }
1657
1658 scq = idr_read_cq(cmd->send_cq_handle, file->ucontext, !!rcq);
1659 rcq = rcq ?: scq;
1660 pd = idr_read_pd(cmd->pd_handle, file->ucontext);
1661 if (!pd || !scq) {
1662 ret = -EINVAL;
1663 goto err_put;
1664 }
1665
1666 device = pd->device;
1667 }
1668
1669 memset(&attr, 0, sizeof attr);
1670 attr.event_handler = ib_uverbs_qp_event_handler;
1671 attr.qp_context = file;
1672 attr.send_cq = scq;
1673 attr.recv_cq = rcq;
1674 attr.srq = srq;
1675 attr.xrcd = xrcd;
1676 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
1677 attr.qp_type = cmd->qp_type;
1678 attr.create_flags = 0;
1679
1680 attr.cap.max_send_wr = cmd->max_send_wr;
1681 attr.cap.max_recv_wr = cmd->max_recv_wr;
1682 attr.cap.max_send_sge = cmd->max_send_sge;
1683 attr.cap.max_recv_sge = cmd->max_recv_sge;
1684 attr.cap.max_inline_data = cmd->max_inline_data;
1685
1686 obj->uevent.events_reported = 0;
1687 INIT_LIST_HEAD(&obj->uevent.event_list);
1688 INIT_LIST_HEAD(&obj->mcast_list);
1689
1690 if (cmd->qp_type == IB_QPT_XRC_TGT)
1691 qp = ib_create_qp(pd, &attr);
1692 else
1693 qp = device->create_qp(pd, &attr, &udata);
1694
1695 if (IS_ERR(qp)) {
1696 ret = PTR_ERR(qp);
1697 goto err_put;
1698 }
1699
1700 if (cmd->qp_type != IB_QPT_XRC_TGT) {
1701 qp->real_qp = qp;
1702 qp->device = device;
1703 qp->pd = pd;
1704 qp->send_cq = attr.send_cq;
1705 qp->recv_cq = attr.recv_cq;
1706 qp->srq = attr.srq;
1707 qp->event_handler = attr.event_handler;
1708 qp->qp_context = attr.qp_context;
1709 qp->qp_type = attr.qp_type;
1710 atomic_set(&qp->usecnt, 0);
1711 atomic_inc(&pd->usecnt);
1712 atomic_inc(&attr.send_cq->usecnt);
1713 if (attr.recv_cq)
1714 atomic_inc(&attr.recv_cq->usecnt);
1715 if (attr.srq)
1716 atomic_inc(&attr.srq->usecnt);
1717 }
1718 qp->uobject = &obj->uevent.uobject;
1719
1720 obj->uevent.uobject.object = qp;
1721 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
1722 if (ret)
1723 goto err_destroy;
1724
1725 memset(&resp_obj, 0, sizeof(resp_obj));
1726 resp->qpn = qp->qp_num;
1727 resp->qp_handle = obj->uevent.uobject.id;
1728 resp->max_recv_sge = attr.cap.max_recv_sge;
1729 resp->max_send_sge = attr.cap.max_send_sge;
1730 resp->max_recv_wr = attr.cap.max_recv_wr;
1731 resp->max_send_wr = attr.cap.max_send_wr;
1732 resp->max_inline_data = attr.cap.max_inline_data;
1733
1734 if (copy_to_user(response, &resp_obj, resp_size)) {
1735 ret = -EFAULT;
1736 goto err_copy;
1737 }
1738
1739 if (xrcd) {
1740 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1741 atomic_inc(&obj->uxrcd->refcnt);
1742 put_xrcd_read(xrcd_uobj);
1743 }
1744
1745 if (pd)
1746 put_pd_read(pd);
1747 if (scq)
1748 put_cq_read(scq);
1749 if (rcq && rcq != scq)
1750 put_cq_read(rcq);
1751 if (srq)
1752 put_srq_read(srq);
1753
1754 mutex_lock(&file->mutex);
1755 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
1756 mutex_unlock(&file->mutex);
1757
1758 obj->uevent.uobject.live = 1;
1759
1760 up_write(&obj->uevent.uobject.mutex);
1761
1762 return in_len;
1763
1764 err_copy:
1765 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
1766
1767 err_destroy:
1768 ib_destroy_qp(qp);
1769
1770 err_put:
1771 if (xrcd)
1772 put_xrcd_read(xrcd_uobj);
1773 if (pd)
1774 put_pd_read(pd);
1775 if (scq)
1776 put_cq_read(scq);
1777 if (rcq && rcq != scq)
1778 put_cq_read(rcq);
1779 if (srq)
1780 put_srq_read(srq);
1781
1782 put_uobj_write(&obj->uevent.uobject);
1783 return ret;
1784 }
1785
ib_uverbs_open_qp(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1786 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
1787 const char __user *buf, int in_len, int out_len)
1788 {
1789 struct ib_uverbs_open_qp cmd;
1790 struct ib_uverbs_create_qp_resp resp;
1791 struct ib_udata udata;
1792 struct ib_uqp_object *obj;
1793 struct ib_xrcd *xrcd;
1794 struct ib_uobject *uninitialized_var(xrcd_uobj);
1795 struct ib_qp *qp;
1796 struct ib_qp_open_attr attr;
1797 int ret;
1798
1799 if (out_len < sizeof resp)
1800 return -ENOSPC;
1801
1802 if (copy_from_user(&cmd, buf, sizeof cmd))
1803 return -EFAULT;
1804
1805 INIT_UDATA(&udata, buf + sizeof cmd,
1806 (unsigned long) cmd.response + sizeof resp,
1807 in_len - sizeof cmd, out_len - sizeof resp);
1808
1809 obj = kmalloc(sizeof *obj, GFP_KERNEL);
1810 if (!obj)
1811 return -ENOMEM;
1812
1813 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class);
1814 down_write(&obj->uevent.uobject.mutex);
1815
1816 xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj);
1817 if (!xrcd) {
1818 ret = -EINVAL;
1819 goto err_put;
1820 }
1821
1822 attr.event_handler = ib_uverbs_qp_event_handler;
1823 attr.qp_context = file;
1824 attr.qp_num = cmd.qpn;
1825 attr.qp_type = cmd.qp_type;
1826
1827 obj->uevent.events_reported = 0;
1828 INIT_LIST_HEAD(&obj->uevent.event_list);
1829 INIT_LIST_HEAD(&obj->mcast_list);
1830
1831 qp = ib_open_qp(xrcd, &attr);
1832 if (IS_ERR(qp)) {
1833 ret = PTR_ERR(qp);
1834 goto err_put;
1835 }
1836
1837 qp->uobject = &obj->uevent.uobject;
1838
1839 obj->uevent.uobject.object = qp;
1840 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
1841 if (ret)
1842 goto err_destroy;
1843
1844 memset(&resp, 0, sizeof resp);
1845 resp.qpn = qp->qp_num;
1846 resp.qp_handle = obj->uevent.uobject.id;
1847
1848 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1849 &resp, sizeof resp)) {
1850 ret = -EFAULT;
1851 goto err_remove;
1852 }
1853
1854 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1855 atomic_inc(&obj->uxrcd->refcnt);
1856 put_xrcd_read(xrcd_uobj);
1857
1858 mutex_lock(&file->mutex);
1859 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
1860 mutex_unlock(&file->mutex);
1861
1862 obj->uevent.uobject.live = 1;
1863 up_write(&obj->uevent.uobject.mutex);
1864
1865 return in_len;
1866
1867 err_remove:
1868 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
1869
1870 err_destroy:
1871 ib_destroy_qp(qp);
1872
1873 err_put:
1874 put_xrcd_read(xrcd_uobj);
1875 put_uobj_write(&obj->uevent.uobject);
1876 return ret;
1877 }
1878
ib_uverbs_query_qp(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)1879 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
1880 const char __user *buf, int in_len,
1881 int out_len)
1882 {
1883 struct ib_uverbs_query_qp cmd;
1884 struct ib_uverbs_query_qp_resp resp;
1885 struct ib_qp *qp;
1886 struct ib_qp_attr *attr;
1887 struct ib_qp_init_attr *init_attr;
1888 int ret;
1889
1890 if (copy_from_user(&cmd, buf, sizeof cmd))
1891 return -EFAULT;
1892
1893 attr = kmalloc(sizeof *attr, GFP_KERNEL);
1894 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1895 if (!attr || !init_attr) {
1896 ret = -ENOMEM;
1897 goto out;
1898 }
1899
1900 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
1901 if (!qp) {
1902 ret = -EINVAL;
1903 goto out;
1904 }
1905
1906 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1907
1908 put_qp_read(qp);
1909
1910 if (ret)
1911 goto out;
1912
1913 memset(&resp, 0, sizeof resp);
1914
1915 resp.qp_state = attr->qp_state;
1916 resp.cur_qp_state = attr->cur_qp_state;
1917 resp.path_mtu = attr->path_mtu;
1918 resp.path_mig_state = attr->path_mig_state;
1919 resp.qkey = attr->qkey;
1920 resp.rq_psn = attr->rq_psn;
1921 resp.sq_psn = attr->sq_psn;
1922 resp.dest_qp_num = attr->dest_qp_num;
1923 resp.qp_access_flags = attr->qp_access_flags;
1924 resp.pkey_index = attr->pkey_index;
1925 resp.alt_pkey_index = attr->alt_pkey_index;
1926 resp.sq_draining = attr->sq_draining;
1927 resp.max_rd_atomic = attr->max_rd_atomic;
1928 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic;
1929 resp.min_rnr_timer = attr->min_rnr_timer;
1930 resp.port_num = attr->port_num;
1931 resp.timeout = attr->timeout;
1932 resp.retry_cnt = attr->retry_cnt;
1933 resp.rnr_retry = attr->rnr_retry;
1934 resp.alt_port_num = attr->alt_port_num;
1935 resp.alt_timeout = attr->alt_timeout;
1936
1937 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16);
1938 resp.dest.flow_label = attr->ah_attr.grh.flow_label;
1939 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index;
1940 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit;
1941 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class;
1942 resp.dest.dlid = attr->ah_attr.dlid;
1943 resp.dest.sl = attr->ah_attr.sl;
1944 resp.dest.src_path_bits = attr->ah_attr.src_path_bits;
1945 resp.dest.static_rate = attr->ah_attr.static_rate;
1946 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH);
1947 resp.dest.port_num = attr->ah_attr.port_num;
1948
1949 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16);
1950 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label;
1951 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index;
1952 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit;
1953 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class;
1954 resp.alt_dest.dlid = attr->alt_ah_attr.dlid;
1955 resp.alt_dest.sl = attr->alt_ah_attr.sl;
1956 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits;
1957 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate;
1958 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH);
1959 resp.alt_dest.port_num = attr->alt_ah_attr.port_num;
1960
1961 resp.max_send_wr = init_attr->cap.max_send_wr;
1962 resp.max_recv_wr = init_attr->cap.max_recv_wr;
1963 resp.max_send_sge = init_attr->cap.max_send_sge;
1964 resp.max_recv_sge = init_attr->cap.max_recv_sge;
1965 resp.max_inline_data = init_attr->cap.max_inline_data;
1966 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1967
1968 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1969 &resp, sizeof resp))
1970 ret = -EFAULT;
1971
1972 out:
1973 kfree(attr);
1974 kfree(init_attr);
1975
1976 return ret ? ret : in_len;
1977 }
1978
1979 /* Remove ignored fields set in the attribute mask */
modify_qp_mask(enum ib_qp_type qp_type,int mask)1980 static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1981 {
1982 switch (qp_type) {
1983 case IB_QPT_XRC_INI:
1984 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1985 case IB_QPT_XRC_TGT:
1986 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1987 IB_QP_RNR_RETRY);
1988 default:
1989 return mask;
1990 }
1991 }
1992
__uverbs_modify_qp(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len,enum uverbs_cmd_type cmd_type)1993 static ssize_t __uverbs_modify_qp(struct ib_uverbs_file *file,
1994 const char __user *buf, int in_len,
1995 int out_len,
1996 enum uverbs_cmd_type cmd_type)
1997 {
1998 struct ib_uverbs_modify_qp_ex cmd;
1999 struct ib_udata udata;
2000 struct ib_qp *qp;
2001 struct ib_qp_attr *attr;
2002 struct ib_qp_attr_ex *attrx;
2003 int ret;
2004 void *p;
2005 union ib_gid sgid;
2006 union ib_gid *dgid;
2007 u8 port_num;
2008
2009 if (cmd_type == IB_USER_VERBS_CMD_BASIC) {
2010 p = &cmd;
2011 p += sizeof(cmd.comp_mask);
2012 if (copy_from_user(p, buf,
2013 sizeof(struct ib_uverbs_modify_qp)))
2014 return -EFAULT;
2015 } else {
2016 if (copy_from_user(&cmd, buf, sizeof(cmd)))
2017 return -EFAULT;
2018 }
2019
2020 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
2021 out_len);
2022
2023 attrx = kzalloc(sizeof(*attrx), GFP_KERNEL);
2024 if (!attrx)
2025 return -ENOMEM;
2026 attr = (struct ib_qp_attr *)attrx;
2027
2028 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
2029 if (!qp) {
2030 kfree(attrx);
2031 return -EINVAL;
2032 }
2033
2034 attr->qp_state = cmd.qp_state;
2035 attr->cur_qp_state = cmd.cur_qp_state;
2036 attr->path_mtu = cmd.path_mtu;
2037 attr->path_mig_state = cmd.path_mig_state;
2038 attr->qkey = cmd.qkey;
2039 attr->rq_psn = cmd.rq_psn;
2040 attr->sq_psn = cmd.sq_psn;
2041 attr->dest_qp_num = cmd.dest_qp_num;
2042 attr->qp_access_flags = cmd.qp_access_flags;
2043 attr->pkey_index = cmd.pkey_index;
2044 attr->alt_pkey_index = cmd.alt_pkey_index;
2045 attr->en_sqd_async_notify = cmd.en_sqd_async_notify;
2046 attr->max_rd_atomic = cmd.max_rd_atomic;
2047 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic;
2048 attr->min_rnr_timer = cmd.min_rnr_timer;
2049 attr->port_num = cmd.port_num;
2050 attr->timeout = cmd.timeout;
2051 attr->retry_cnt = cmd.retry_cnt;
2052 attr->rnr_retry = cmd.rnr_retry;
2053 attr->alt_port_num = cmd.alt_port_num;
2054 attr->alt_timeout = cmd.alt_timeout;
2055
2056 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16);
2057 attr->ah_attr.grh.flow_label = cmd.dest.flow_label;
2058 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index;
2059 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit;
2060 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class;
2061 attr->ah_attr.dlid = cmd.dest.dlid;
2062 attr->ah_attr.sl = cmd.dest.sl;
2063 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits;
2064 attr->ah_attr.static_rate = cmd.dest.static_rate;
2065 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0;
2066 attr->ah_attr.port_num = cmd.dest.port_num;
2067
2068 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16);
2069 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label;
2070 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index;
2071 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit;
2072 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class;
2073 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid;
2074 attr->alt_ah_attr.sl = cmd.alt_dest.sl;
2075 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits;
2076 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate;
2077 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0;
2078 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num;
2079 port_num = (cmd.attr_mask & IB_QP_PORT) ? cmd.port_num : qp->port_num;
2080 if ((cmd.attr_mask & IB_QP_AV) && port_num &&
2081 (rdma_port_get_link_layer(qp->device, port_num) ==
2082 IB_LINK_LAYER_ETHERNET)) {
2083 ret = ib_query_gid(qp->device, port_num,
2084 attr->ah_attr.grh.sgid_index, &sgid);
2085 if (ret)
2086 goto out;
2087 dgid = &attr->ah_attr.grh.dgid;
2088 if (rdma_link_local_addr((struct in6_addr *)dgid->raw)) {
2089 rdma_get_ll_mac((struct in6_addr *)dgid->raw,
2090 attr->ah_attr.dmac);
2091 rdma_get_ll_mac((struct in6_addr *)sgid.raw,
2092 attr->smac);
2093 attr->vlan_id = rdma_get_vlan_id(&sgid);
2094 } else {
2095 ret = rdma_addr_find_dmac_by_grh(&sgid, dgid,
2096 attr->ah_attr.dmac,
2097 &attr->vlan_id);
2098 if (ret)
2099 goto out;
2100 ret = rdma_addr_find_smac_by_sgid(&sgid, attr->smac,
2101 NULL);
2102 if (ret)
2103 goto out;
2104 }
2105 cmd.attr_mask |= IB_QP_SMAC;
2106 if (attr->vlan_id < 0xFFFF)
2107 cmd.attr_mask |= IB_QP_VID;
2108 }
2109 if (cmd_type == IB_USER_VERBS_CMD_EXTENDED) {
2110 if (cmd.comp_mask & IB_UVERBS_QP_ATTR_DCT_KEY)
2111 attrx->dct_key = cmd.dct_key;
2112 }
2113
2114 if (qp->real_qp == qp) {
2115 ret = qp->device->modify_qp(qp, attr,
2116 modify_qp_mask(qp->qp_type, cmd.attr_mask), &udata);
2117 if (!ret && (cmd.attr_mask & IB_QP_PORT))
2118 qp->port_num = attr->port_num;
2119 } else {
2120 ret = ib_modify_qp(qp, attr, modify_qp_mask(qp->qp_type, cmd.attr_mask));
2121 }
2122
2123 if (ret)
2124 goto out;
2125
2126 ret = in_len;
2127
2128 out:
2129 put_qp_read(qp);
2130 kfree(attrx);
2131
2132 return ret;
2133 }
2134
ib_uverbs_modify_qp(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2135 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
2136 const char __user *buf, int in_len,
2137 int out_len)
2138 {
2139 return __uverbs_modify_qp(file, buf, in_len, out_len,
2140 IB_USER_VERBS_CMD_BASIC);
2141 }
2142
ib_uverbs_destroy_qp(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2143 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
2144 const char __user *buf, int in_len,
2145 int out_len)
2146 {
2147 struct ib_uverbs_destroy_qp cmd;
2148 struct ib_uverbs_destroy_qp_resp resp;
2149 struct ib_uobject *uobj;
2150 struct ib_qp *qp;
2151 struct ib_uqp_object *obj;
2152 int ret = -EINVAL;
2153
2154 if (copy_from_user(&cmd, buf, sizeof cmd))
2155 return -EFAULT;
2156
2157 memset(&resp, 0, sizeof resp);
2158
2159 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext);
2160 if (!uobj)
2161 return -EINVAL;
2162 qp = uobj->object;
2163 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
2164
2165 if (!list_empty(&obj->mcast_list)) {
2166 put_uobj_write(uobj);
2167 return -EBUSY;
2168 }
2169
2170 ret = ib_destroy_qp(qp);
2171 if (!ret)
2172 uobj->live = 0;
2173
2174 put_uobj_write(uobj);
2175
2176 if (ret)
2177 return ret;
2178
2179 if (obj->uxrcd)
2180 atomic_dec(&obj->uxrcd->refcnt);
2181
2182 idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
2183
2184 mutex_lock(&file->mutex);
2185 list_del(&uobj->list);
2186 mutex_unlock(&file->mutex);
2187
2188 ib_uverbs_release_uevent(file, &obj->uevent);
2189
2190 resp.events_reported = obj->uevent.events_reported;
2191
2192 put_uobj(uobj);
2193
2194 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2195 &resp, sizeof resp))
2196 return -EFAULT;
2197
2198 return in_len;
2199 }
2200
ib_uverbs_post_send(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2201 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
2202 const char __user *buf, int in_len,
2203 int out_len)
2204 {
2205 struct ib_uverbs_post_send cmd;
2206 struct ib_uverbs_post_send_resp resp;
2207 struct ib_uverbs_send_wr *user_wr;
2208 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr;
2209 struct ib_qp *qp;
2210 int i, sg_ind;
2211 int is_ud;
2212 ssize_t ret = -EINVAL;
2213
2214 if (copy_from_user(&cmd, buf, sizeof cmd))
2215 return -EFAULT;
2216
2217 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count +
2218 cmd.sge_count * sizeof (struct ib_uverbs_sge))
2219 return -EINVAL;
2220
2221 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr))
2222 return -EINVAL;
2223
2224 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2225 if (!user_wr)
2226 return -ENOMEM;
2227
2228 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
2229 if (!qp)
2230 goto out;
2231
2232 is_ud = qp->qp_type == IB_QPT_UD;
2233 sg_ind = 0;
2234 last = NULL;
2235 for (i = 0; i < cmd.wr_count; ++i) {
2236 if (copy_from_user(user_wr,
2237 buf + sizeof cmd + i * cmd.wqe_size,
2238 cmd.wqe_size)) {
2239 ret = -EFAULT;
2240 goto out_put;
2241 }
2242
2243 if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2244 ret = -EINVAL;
2245 goto out_put;
2246 }
2247
2248 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2249 user_wr->num_sge * sizeof (struct ib_sge),
2250 GFP_KERNEL);
2251 if (!next) {
2252 ret = -ENOMEM;
2253 goto out_put;
2254 }
2255
2256 if (!last)
2257 wr = next;
2258 else
2259 last->next = next;
2260 last = next;
2261
2262 next->next = NULL;
2263 next->wr_id = user_wr->wr_id;
2264 next->num_sge = user_wr->num_sge;
2265 next->opcode = user_wr->opcode;
2266 next->send_flags = user_wr->send_flags;
2267
2268 if (is_ud) {
2269 next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah,
2270 file->ucontext);
2271 if (!next->wr.ud.ah) {
2272 ret = -EINVAL;
2273 goto out_put;
2274 }
2275 next->wr.ud.remote_qpn = user_wr->wr.ud.remote_qpn;
2276 next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey;
2277 } else {
2278 switch (next->opcode) {
2279 case IB_WR_RDMA_WRITE_WITH_IMM:
2280 next->ex.imm_data =
2281 (__be32 __force) user_wr->ex.imm_data;
2282 case IB_WR_RDMA_WRITE:
2283 case IB_WR_RDMA_READ:
2284 next->wr.rdma.remote_addr =
2285 user_wr->wr.rdma.remote_addr;
2286 next->wr.rdma.rkey =
2287 user_wr->wr.rdma.rkey;
2288 break;
2289 case IB_WR_SEND_WITH_IMM:
2290 next->ex.imm_data =
2291 (__be32 __force) user_wr->ex.imm_data;
2292 break;
2293 case IB_WR_SEND_WITH_INV:
2294 next->ex.invalidate_rkey =
2295 user_wr->ex.invalidate_rkey;
2296 break;
2297 case IB_WR_ATOMIC_CMP_AND_SWP:
2298 case IB_WR_ATOMIC_FETCH_AND_ADD:
2299 next->wr.atomic.remote_addr =
2300 user_wr->wr.atomic.remote_addr;
2301 next->wr.atomic.compare_add =
2302 user_wr->wr.atomic.compare_add;
2303 next->wr.atomic.swap = user_wr->wr.atomic.swap;
2304 next->wr.atomic.rkey = user_wr->wr.atomic.rkey;
2305 break;
2306 default:
2307 break;
2308 }
2309 }
2310
2311 if (next->num_sge) {
2312 next->sg_list = (void *) next +
2313 ALIGN(sizeof *next, sizeof (struct ib_sge));
2314 if (copy_from_user(next->sg_list,
2315 buf + sizeof cmd +
2316 cmd.wr_count * cmd.wqe_size +
2317 sg_ind * sizeof (struct ib_sge),
2318 next->num_sge * sizeof (struct ib_sge))) {
2319 ret = -EFAULT;
2320 goto out_put;
2321 }
2322 sg_ind += next->num_sge;
2323 } else
2324 next->sg_list = NULL;
2325 }
2326
2327 resp.bad_wr = 0;
2328 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr);
2329 if (ret)
2330 for (next = wr; next; next = next->next) {
2331 ++resp.bad_wr;
2332 if (next == bad_wr)
2333 break;
2334 }
2335
2336 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2337 &resp, sizeof resp))
2338 ret = -EFAULT;
2339
2340 out_put:
2341 put_qp_read(qp);
2342
2343 while (wr) {
2344 if (is_ud && wr->wr.ud.ah)
2345 put_ah_read(wr->wr.ud.ah);
2346 next = wr->next;
2347 kfree(wr);
2348 wr = next;
2349 }
2350
2351 out:
2352 kfree(user_wr);
2353
2354 return ret ? ret : in_len;
2355 }
2356
ib_uverbs_unmarshall_recv(const char __user * buf,int in_len,u32 wr_count,u32 sge_count,u32 wqe_size)2357 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf,
2358 int in_len,
2359 u32 wr_count,
2360 u32 sge_count,
2361 u32 wqe_size)
2362 {
2363 struct ib_uverbs_recv_wr *user_wr;
2364 struct ib_recv_wr *wr = NULL, *last, *next;
2365 int sg_ind;
2366 int i;
2367 int ret;
2368
2369 if (in_len < wqe_size * wr_count +
2370 sge_count * sizeof (struct ib_uverbs_sge))
2371 return ERR_PTR(-EINVAL);
2372
2373 if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2374 return ERR_PTR(-EINVAL);
2375
2376 user_wr = kmalloc(wqe_size, GFP_KERNEL);
2377 if (!user_wr)
2378 return ERR_PTR(-ENOMEM);
2379
2380 sg_ind = 0;
2381 last = NULL;
2382 for (i = 0; i < wr_count; ++i) {
2383 if (copy_from_user(user_wr, buf + i * wqe_size,
2384 wqe_size)) {
2385 ret = -EFAULT;
2386 goto err;
2387 }
2388
2389 if (user_wr->num_sge + sg_ind > sge_count) {
2390 ret = -EINVAL;
2391 goto err;
2392 }
2393
2394 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2395 user_wr->num_sge * sizeof (struct ib_sge),
2396 GFP_KERNEL);
2397 if (!next) {
2398 ret = -ENOMEM;
2399 goto err;
2400 }
2401
2402 if (!last)
2403 wr = next;
2404 else
2405 last->next = next;
2406 last = next;
2407
2408 next->next = NULL;
2409 next->wr_id = user_wr->wr_id;
2410 next->num_sge = user_wr->num_sge;
2411
2412 if (next->num_sge) {
2413 next->sg_list = (void *) next +
2414 ALIGN(sizeof *next, sizeof (struct ib_sge));
2415 if (copy_from_user(next->sg_list,
2416 buf + wr_count * wqe_size +
2417 sg_ind * sizeof (struct ib_sge),
2418 next->num_sge * sizeof (struct ib_sge))) {
2419 ret = -EFAULT;
2420 goto err;
2421 }
2422 sg_ind += next->num_sge;
2423 } else
2424 next->sg_list = NULL;
2425 }
2426
2427 kfree(user_wr);
2428 return wr;
2429
2430 err:
2431 kfree(user_wr);
2432
2433 while (wr) {
2434 next = wr->next;
2435 kfree(wr);
2436 wr = next;
2437 }
2438
2439 return ERR_PTR(ret);
2440 }
2441
ib_uverbs_post_recv(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2442 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
2443 const char __user *buf, int in_len,
2444 int out_len)
2445 {
2446 struct ib_uverbs_post_recv cmd;
2447 struct ib_uverbs_post_recv_resp resp;
2448 struct ib_recv_wr *wr, *next, *bad_wr;
2449 struct ib_qp *qp;
2450 ssize_t ret = -EINVAL;
2451
2452 if (copy_from_user(&cmd, buf, sizeof cmd))
2453 return -EFAULT;
2454
2455 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2456 in_len - sizeof cmd, cmd.wr_count,
2457 cmd.sge_count, cmd.wqe_size);
2458 if (IS_ERR(wr))
2459 return PTR_ERR(wr);
2460
2461 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
2462 if (!qp)
2463 goto out;
2464
2465 resp.bad_wr = 0;
2466 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr);
2467
2468 put_qp_read(qp);
2469
2470 if (ret)
2471 for (next = wr; next; next = next->next) {
2472 ++resp.bad_wr;
2473 if (next == bad_wr)
2474 break;
2475 }
2476
2477 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2478 &resp, sizeof resp))
2479 ret = -EFAULT;
2480
2481 out:
2482 while (wr) {
2483 next = wr->next;
2484 kfree(wr);
2485 wr = next;
2486 }
2487
2488 return ret ? ret : in_len;
2489 }
2490
ib_uverbs_post_srq_recv(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2491 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file,
2492 const char __user *buf, int in_len,
2493 int out_len)
2494 {
2495 struct ib_uverbs_post_srq_recv cmd;
2496 struct ib_uverbs_post_srq_recv_resp resp;
2497 struct ib_recv_wr *wr, *next, *bad_wr;
2498 struct ib_srq *srq;
2499 ssize_t ret = -EINVAL;
2500
2501 if (copy_from_user(&cmd, buf, sizeof cmd))
2502 return -EFAULT;
2503
2504 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2505 in_len - sizeof cmd, cmd.wr_count,
2506 cmd.sge_count, cmd.wqe_size);
2507 if (IS_ERR(wr))
2508 return PTR_ERR(wr);
2509
2510 srq = idr_read_srq(cmd.srq_handle, file->ucontext);
2511 if (!srq)
2512 goto out;
2513
2514 resp.bad_wr = 0;
2515 ret = srq->device->post_srq_recv(srq, wr, &bad_wr);
2516
2517 put_srq_read(srq);
2518
2519 if (ret)
2520 for (next = wr; next; next = next->next) {
2521 ++resp.bad_wr;
2522 if (next == bad_wr)
2523 break;
2524 }
2525
2526 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2527 &resp, sizeof resp))
2528 ret = -EFAULT;
2529
2530 out:
2531 while (wr) {
2532 next = wr->next;
2533 kfree(wr);
2534 wr = next;
2535 }
2536
2537 return ret ? ret : in_len;
2538 }
2539
ib_uverbs_create_ah(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2540 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
2541 const char __user *buf, int in_len,
2542 int out_len)
2543 {
2544 struct ib_uverbs_create_ah cmd;
2545 struct ib_uverbs_create_ah_resp resp;
2546 struct ib_uobject *uobj;
2547 struct ib_pd *pd;
2548 struct ib_ah *ah;
2549 struct ib_ah_attr attr;
2550 int ret;
2551
2552 if (out_len < sizeof resp)
2553 return -ENOSPC;
2554
2555 if (copy_from_user(&cmd, buf, sizeof cmd))
2556 return -EFAULT;
2557
2558 uobj = kmalloc(sizeof *uobj, GFP_KERNEL);
2559 if (!uobj)
2560 return -ENOMEM;
2561
2562 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class);
2563 down_write(&uobj->mutex);
2564
2565 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
2566 if (!pd) {
2567 ret = -EINVAL;
2568 goto err;
2569 }
2570
2571 attr.dlid = cmd.attr.dlid;
2572 attr.sl = cmd.attr.sl;
2573 attr.src_path_bits = cmd.attr.src_path_bits;
2574 attr.static_rate = cmd.attr.static_rate;
2575 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0;
2576 attr.port_num = cmd.attr.port_num;
2577 attr.grh.flow_label = cmd.attr.grh.flow_label;
2578 attr.grh.sgid_index = cmd.attr.grh.sgid_index;
2579 attr.grh.hop_limit = cmd.attr.grh.hop_limit;
2580 attr.grh.traffic_class = cmd.attr.grh.traffic_class;
2581 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16);
2582
2583 ah = ib_create_ah(pd, &attr);
2584 if (IS_ERR(ah)) {
2585 ret = PTR_ERR(ah);
2586 goto err_put;
2587 }
2588
2589 ah->uobject = uobj;
2590 uobj->object = ah;
2591
2592 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj);
2593 if (ret)
2594 goto err_destroy;
2595
2596 resp.ah_handle = uobj->id;
2597
2598 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2599 &resp, sizeof resp)) {
2600 ret = -EFAULT;
2601 goto err_copy;
2602 }
2603
2604 put_pd_read(pd);
2605
2606 mutex_lock(&file->mutex);
2607 list_add_tail(&uobj->list, &file->ucontext->ah_list);
2608 mutex_unlock(&file->mutex);
2609
2610 uobj->live = 1;
2611
2612 up_write(&uobj->mutex);
2613
2614 return in_len;
2615
2616 err_copy:
2617 idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
2618
2619 err_destroy:
2620 ib_destroy_ah(ah);
2621
2622 err_put:
2623 put_pd_read(pd);
2624
2625 err:
2626 put_uobj_write(uobj);
2627 return ret;
2628 }
2629
ib_uverbs_destroy_ah(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2630 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file,
2631 const char __user *buf, int in_len, int out_len)
2632 {
2633 struct ib_uverbs_destroy_ah cmd;
2634 struct ib_ah *ah;
2635 struct ib_uobject *uobj;
2636 int ret;
2637
2638 if (copy_from_user(&cmd, buf, sizeof cmd))
2639 return -EFAULT;
2640
2641 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext);
2642 if (!uobj)
2643 return -EINVAL;
2644 ah = uobj->object;
2645
2646 ret = ib_destroy_ah(ah);
2647 if (!ret)
2648 uobj->live = 0;
2649
2650 put_uobj_write(uobj);
2651
2652 if (ret)
2653 return ret;
2654
2655 idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
2656
2657 mutex_lock(&file->mutex);
2658 list_del(&uobj->list);
2659 mutex_unlock(&file->mutex);
2660
2661 put_uobj(uobj);
2662
2663 return in_len;
2664 }
2665
ib_uverbs_attach_mcast(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2666 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file,
2667 const char __user *buf, int in_len,
2668 int out_len)
2669 {
2670 struct ib_uverbs_attach_mcast cmd;
2671 struct ib_qp *qp;
2672 struct ib_uqp_object *obj;
2673 struct ib_uverbs_mcast_entry *mcast;
2674 int ret;
2675
2676 if (copy_from_user(&cmd, buf, sizeof cmd))
2677 return -EFAULT;
2678
2679 qp = idr_write_qp(cmd.qp_handle, file->ucontext);
2680 if (!qp)
2681 return -EINVAL;
2682
2683 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2684
2685 list_for_each_entry(mcast, &obj->mcast_list, list)
2686 if (cmd.mlid == mcast->lid &&
2687 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2688 ret = 0;
2689 goto out_put;
2690 }
2691
2692 mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2693 if (!mcast) {
2694 ret = -ENOMEM;
2695 goto out_put;
2696 }
2697
2698 mcast->lid = cmd.mlid;
2699 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2700
2701 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2702 if (!ret)
2703 list_add_tail(&mcast->list, &obj->mcast_list);
2704 else
2705 kfree(mcast);
2706
2707 out_put:
2708 put_qp_write(qp);
2709
2710 return ret ? ret : in_len;
2711 }
2712
ib_uverbs_detach_mcast(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2713 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file,
2714 const char __user *buf, int in_len,
2715 int out_len)
2716 {
2717 struct ib_uverbs_detach_mcast cmd;
2718 struct ib_uqp_object *obj;
2719 struct ib_qp *qp;
2720 struct ib_uverbs_mcast_entry *mcast;
2721 int ret = -EINVAL;
2722
2723 if (copy_from_user(&cmd, buf, sizeof cmd))
2724 return -EFAULT;
2725
2726 qp = idr_write_qp(cmd.qp_handle, file->ucontext);
2727 if (!qp)
2728 return -EINVAL;
2729
2730 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid);
2731 if (ret)
2732 goto out_put;
2733
2734 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2735
2736 list_for_each_entry(mcast, &obj->mcast_list, list)
2737 if (cmd.mlid == mcast->lid &&
2738 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2739 list_del(&mcast->list);
2740 kfree(mcast);
2741 break;
2742 }
2743
2744 out_put:
2745 put_qp_write(qp);
2746
2747 return ret ? ret : in_len;
2748 }
2749
__uverbs_create_xsrq(struct ib_uverbs_file * file,struct ib_uverbs_create_xsrq * cmd,struct ib_udata * udata)2750 static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
2751 struct ib_uverbs_create_xsrq *cmd,
2752 struct ib_udata *udata)
2753 {
2754 struct ib_uverbs_create_srq_resp resp;
2755 struct ib_usrq_object *obj;
2756 struct ib_pd *pd;
2757 struct ib_srq *srq;
2758 struct ib_uobject *uninitialized_var(xrcd_uobj);
2759 struct ib_srq_init_attr attr;
2760 int ret;
2761
2762 obj = kmalloc(sizeof(*obj), GFP_KERNEL);
2763 if (!obj)
2764 return -ENOMEM;
2765
2766 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class);
2767 down_write(&obj->uevent.uobject.mutex);
2768
2769 if (cmd->srq_type == IB_SRQT_XRC) {
2770 attr.ext.xrc.xrcd = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj);
2771 if (!attr.ext.xrc.xrcd) {
2772 ret = -EINVAL;
2773 goto err;
2774 }
2775
2776 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
2777 atomic_inc(&obj->uxrcd->refcnt);
2778
2779 attr.ext.xrc.cq = idr_read_cq(cmd->cq_handle, file->ucontext, 0);
2780 if (!attr.ext.xrc.cq) {
2781 ret = -EINVAL;
2782 goto err_put_xrcd;
2783 }
2784 }
2785
2786 pd = idr_read_pd(cmd->pd_handle, file->ucontext);
2787 if (!pd) {
2788 ret = -EINVAL;
2789 goto err_put_cq;
2790 }
2791
2792 attr.event_handler = ib_uverbs_srq_event_handler;
2793 attr.srq_context = file;
2794 attr.srq_type = cmd->srq_type;
2795 attr.attr.max_wr = cmd->max_wr;
2796 attr.attr.max_sge = cmd->max_sge;
2797 attr.attr.srq_limit = cmd->srq_limit;
2798
2799 obj->uevent.events_reported = 0;
2800 INIT_LIST_HEAD(&obj->uevent.event_list);
2801
2802 srq = pd->device->create_srq(pd, &attr, udata);
2803 if (IS_ERR(srq)) {
2804 ret = PTR_ERR(srq);
2805 goto err_put;
2806 }
2807
2808 srq->device = pd->device;
2809 srq->pd = pd;
2810 srq->srq_type = cmd->srq_type;
2811 srq->uobject = &obj->uevent.uobject;
2812 srq->event_handler = attr.event_handler;
2813 srq->srq_context = attr.srq_context;
2814
2815 if (cmd->srq_type == IB_SRQT_XRC) {
2816 srq->ext.xrc.cq = attr.ext.xrc.cq;
2817 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
2818 atomic_inc(&attr.ext.xrc.cq->usecnt);
2819 atomic_inc(&attr.ext.xrc.xrcd->usecnt);
2820 }
2821
2822 atomic_inc(&pd->usecnt);
2823 atomic_set(&srq->usecnt, 0);
2824
2825 obj->uevent.uobject.object = srq;
2826 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject);
2827 if (ret)
2828 goto err_destroy;
2829
2830 memset(&resp, 0, sizeof resp);
2831 resp.srq_handle = obj->uevent.uobject.id;
2832 resp.max_wr = attr.attr.max_wr;
2833 resp.max_sge = attr.attr.max_sge;
2834 if (cmd->srq_type == IB_SRQT_XRC)
2835 resp.srqn = srq->ext.xrc.srq_num;
2836
2837 if (copy_to_user((void __user *) (unsigned long) cmd->response,
2838 &resp, sizeof resp)) {
2839 ret = -EFAULT;
2840 goto err_copy;
2841 }
2842
2843 if (cmd->srq_type == IB_SRQT_XRC) {
2844 put_uobj_read(xrcd_uobj);
2845 put_cq_read(attr.ext.xrc.cq);
2846 }
2847 put_pd_read(pd);
2848
2849 mutex_lock(&file->mutex);
2850 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->srq_list);
2851 mutex_unlock(&file->mutex);
2852
2853 obj->uevent.uobject.live = 1;
2854
2855 up_write(&obj->uevent.uobject.mutex);
2856
2857 return 0;
2858
2859 err_copy:
2860 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject);
2861
2862 err_destroy:
2863 ib_destroy_srq(srq);
2864
2865 err_put:
2866 put_pd_read(pd);
2867
2868 err_put_cq:
2869 if (cmd->srq_type == IB_SRQT_XRC)
2870 put_cq_read(attr.ext.xrc.cq);
2871
2872 err_put_xrcd:
2873 if (cmd->srq_type == IB_SRQT_XRC) {
2874 atomic_dec(&obj->uxrcd->refcnt);
2875 put_uobj_read(xrcd_uobj);
2876 }
2877
2878 err:
2879 put_uobj_write(&obj->uevent.uobject);
2880 return ret;
2881 }
2882
ib_uverbs_create_srq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2883 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
2884 const char __user *buf, int in_len,
2885 int out_len)
2886 {
2887 struct ib_uverbs_create_srq cmd;
2888 struct ib_uverbs_create_xsrq xcmd;
2889 struct ib_uverbs_create_srq_resp resp;
2890 struct ib_udata udata;
2891 int ret;
2892
2893 if (out_len < sizeof resp)
2894 return -ENOSPC;
2895
2896 if (copy_from_user(&cmd, buf, sizeof cmd))
2897 return -EFAULT;
2898
2899 xcmd.response = cmd.response;
2900 xcmd.user_handle = cmd.user_handle;
2901 xcmd.srq_type = IB_SRQT_BASIC;
2902 xcmd.pd_handle = cmd.pd_handle;
2903 xcmd.max_wr = cmd.max_wr;
2904 xcmd.max_sge = cmd.max_sge;
2905 xcmd.srq_limit = cmd.srq_limit;
2906
2907 INIT_UDATA(&udata, buf + sizeof cmd,
2908 (unsigned long) cmd.response + sizeof resp,
2909 in_len - sizeof cmd, out_len - sizeof resp);
2910
2911 ret = __uverbs_create_xsrq(file, &xcmd, &udata);
2912 if (ret)
2913 return ret;
2914
2915 return in_len;
2916 }
2917
ib_uverbs_create_xsrq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2918 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file,
2919 const char __user *buf, int in_len, int out_len)
2920 {
2921 struct ib_uverbs_create_xsrq cmd;
2922 struct ib_uverbs_create_srq_resp resp;
2923 struct ib_udata udata;
2924 int ret;
2925
2926 if (out_len < sizeof resp)
2927 return -ENOSPC;
2928
2929 if (copy_from_user(&cmd, buf, sizeof cmd))
2930 return -EFAULT;
2931
2932 INIT_UDATA(&udata, buf + sizeof cmd,
2933 (unsigned long) cmd.response + sizeof resp,
2934 in_len - sizeof cmd, out_len - sizeof resp);
2935
2936 ret = __uverbs_create_xsrq(file, &cmd, &udata);
2937 if (ret)
2938 return ret;
2939
2940 return in_len;
2941 }
2942
ib_uverbs_modify_srq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2943 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
2944 const char __user *buf, int in_len,
2945 int out_len)
2946 {
2947 struct ib_uverbs_modify_srq cmd;
2948 struct ib_udata udata;
2949 struct ib_srq *srq;
2950 struct ib_srq_attr attr;
2951 int ret;
2952
2953 if (copy_from_user(&cmd, buf, sizeof cmd))
2954 return -EFAULT;
2955
2956 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
2957 out_len);
2958
2959 srq = idr_read_srq(cmd.srq_handle, file->ucontext);
2960 if (!srq)
2961 return -EINVAL;
2962
2963 attr.max_wr = cmd.max_wr;
2964 attr.srq_limit = cmd.srq_limit;
2965
2966 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata);
2967
2968 put_srq_read(srq);
2969
2970 return ret ? ret : in_len;
2971 }
2972
ib_uverbs_query_srq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)2973 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file,
2974 const char __user *buf,
2975 int in_len, int out_len)
2976 {
2977 struct ib_uverbs_query_srq cmd;
2978 struct ib_uverbs_query_srq_resp resp;
2979 struct ib_srq_attr attr;
2980 struct ib_srq *srq;
2981 int ret;
2982
2983 if (out_len < sizeof resp)
2984 return -ENOSPC;
2985
2986 if (copy_from_user(&cmd, buf, sizeof cmd))
2987 return -EFAULT;
2988
2989 srq = idr_read_srq(cmd.srq_handle, file->ucontext);
2990 if (!srq)
2991 return -EINVAL;
2992
2993 ret = ib_query_srq(srq, &attr);
2994
2995 put_srq_read(srq);
2996
2997 if (ret)
2998 return ret;
2999
3000 memset(&resp, 0, sizeof resp);
3001
3002 resp.max_wr = attr.max_wr;
3003 resp.max_sge = attr.max_sge;
3004 resp.srq_limit = attr.srq_limit;
3005
3006 if (copy_to_user((void __user *) (unsigned long) cmd.response,
3007 &resp, sizeof resp))
3008 return -EFAULT;
3009
3010 return in_len;
3011 }
3012
ib_uverbs_destroy_srq(struct ib_uverbs_file * file,const char __user * buf,int in_len,int out_len)3013 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3014 const char __user *buf, int in_len,
3015 int out_len)
3016 {
3017 struct ib_uverbs_destroy_srq cmd;
3018 struct ib_uverbs_destroy_srq_resp resp;
3019 struct ib_uobject *uobj;
3020 struct ib_srq *srq;
3021 struct ib_uevent_object *obj;
3022 int ret = -EINVAL;
3023 struct ib_usrq_object *us;
3024 enum ib_srq_type srq_type;
3025
3026 if (copy_from_user(&cmd, buf, sizeof cmd))
3027 return -EFAULT;
3028
3029 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext);
3030 if (!uobj)
3031 return -EINVAL;
3032 srq = uobj->object;
3033 obj = container_of(uobj, struct ib_uevent_object, uobject);
3034 srq_type = srq->srq_type;
3035
3036 ret = ib_destroy_srq(srq);
3037 if (!ret)
3038 uobj->live = 0;
3039
3040 put_uobj_write(uobj);
3041
3042 if (ret)
3043 return ret;
3044
3045 if (srq_type == IB_SRQT_XRC) {
3046 us = container_of(obj, struct ib_usrq_object, uevent);
3047 atomic_dec(&us->uxrcd->refcnt);
3048 }
3049
3050 idr_remove_uobj(&ib_uverbs_srq_idr, uobj);
3051
3052 mutex_lock(&file->mutex);
3053 list_del(&uobj->list);
3054 mutex_unlock(&file->mutex);
3055
3056 ib_uverbs_release_uevent(file, obj);
3057
3058 memset(&resp, 0, sizeof resp);
3059 resp.events_reported = obj->events_reported;
3060
3061 put_uobj(uobj);
3062
3063 if (copy_to_user((void __user *) (unsigned long) cmd.response,
3064 &resp, sizeof resp))
3065 ret = -EFAULT;
3066
3067 return ret ? ret : in_len;
3068 }
3069
ib_uverbs_exp_create_dct(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3070 ssize_t ib_uverbs_exp_create_dct(struct ib_uverbs_file *file,
3071 struct ib_udata *ucore,
3072 struct ib_udata *uhw)
3073 {
3074 int in_len = ucore->inlen + uhw->inlen;
3075 int out_len = ucore->outlen + uhw->outlen;
3076 struct ib_uverbs_create_dct cmd;
3077 struct ib_uverbs_create_dct_resp resp;
3078 struct ib_udata udata;
3079 struct ib_udct_object *obj;
3080 struct ib_dct *dct;
3081 int ret;
3082 struct ib_dct_init_attr attr;
3083 struct ib_pd *pd = NULL;
3084 struct ib_cq *cq = NULL;
3085 struct ib_srq *srq = NULL;
3086
3087 if (out_len < sizeof(resp))
3088 return -ENOSPC;
3089
3090 ret = ucore->ops->copy_from(&cmd, ucore, sizeof(cmd));
3091 if (ret)
3092 return ret;
3093
3094 obj = kmalloc(sizeof(*obj), GFP_KERNEL);
3095 if (!obj)
3096 return -ENOMEM;
3097
3098 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext,
3099 &dct_lock_class);
3100 down_write(&obj->uobject.mutex);
3101
3102 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
3103 if (!pd) {
3104 ret = -EINVAL;
3105 goto err_pd;
3106 }
3107
3108 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
3109 if (!cq) {
3110 ret = -EINVAL;
3111 goto err_put;
3112 }
3113
3114 srq = idr_read_srq(cmd.srq_handle, file->ucontext);
3115 if (!srq) {
3116 ret = -EINVAL;
3117 goto err_put;
3118 }
3119
3120 attr.cq = cq;
3121 attr.access_flags = cmd.access_flags;
3122 attr.min_rnr_timer = cmd.min_rnr_timer;
3123 attr.srq = srq;
3124 attr.tclass = cmd.tclass;
3125 attr.flow_label = cmd.flow_label;
3126 attr.dc_key = cmd.dc_key;
3127 attr.mtu = cmd.mtu;
3128 attr.port = cmd.port;
3129 attr.pkey_index = cmd.pkey_index;
3130 attr.gid_index = cmd.gid_index;
3131 attr.hop_limit = cmd.hop_limit;
3132 attr.create_flags = cmd.create_flags;
3133
3134 dct = ib_create_dct(pd, &attr, &udata);
3135 if (IS_ERR(dct)) {
3136 ret = PTR_ERR(dct);
3137 goto err_put;
3138 }
3139
3140 dct->device = file->device->ib_dev;
3141 dct->uobject = &obj->uobject;
3142
3143 obj->uobject.object = dct;
3144 ret = idr_add_uobj(&ib_uverbs_dct_idr, &obj->uobject);
3145 if (ret)
3146 goto err_dct;
3147
3148 memset(&resp, 0, sizeof(resp));
3149 resp.dct_handle = obj->uobject.id;
3150 resp.dctn = dct->dct_num;
3151
3152 ret = ucore->ops->copy_to(ucore, &resp, sizeof(resp));
3153 if (ret)
3154 goto err_copy;
3155
3156 mutex_lock(&file->mutex);
3157 list_add_tail(&obj->uobject.list, &file->ucontext->dct_list);
3158 mutex_unlock(&file->mutex);
3159
3160 obj->uobject.live = 1;
3161
3162 put_srq_read(srq);
3163 put_cq_read(cq);
3164 put_pd_read(pd);
3165
3166 up_write(&obj->uobject.mutex);
3167
3168 return in_len;
3169
3170 err_copy:
3171 idr_remove_uobj(&ib_uverbs_dct_idr, &obj->uobject);
3172
3173 err_dct:
3174 ib_destroy_dct(dct);
3175
3176 err_put:
3177 if (srq)
3178 put_srq_read(srq);
3179
3180 if (cq)
3181 put_cq_read(cq);
3182
3183 put_pd_read(pd);
3184
3185 err_pd:
3186 put_uobj_write(&obj->uobject);
3187 return ret;
3188 }
3189
ib_uverbs_exp_destroy_dct(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3190 ssize_t ib_uverbs_exp_destroy_dct(struct ib_uverbs_file *file,
3191 struct ib_udata *ucore,
3192 struct ib_udata *uhw)
3193 {
3194 int in_len = ucore->inlen + uhw->inlen;
3195 int out_len = ucore->outlen + uhw->outlen;
3196 struct ib_uverbs_destroy_dct cmd;
3197 struct ib_uverbs_destroy_dct_resp resp;
3198 struct ib_uobject *uobj;
3199 struct ib_dct *dct;
3200 struct ib_udct_object *obj;
3201 int ret;
3202
3203 if (out_len < sizeof(resp))
3204 return -ENOSPC;
3205
3206 ret = ucore->ops->copy_from(&cmd, ucore, sizeof(cmd));
3207 if (ret)
3208 return ret;
3209
3210 uobj = idr_write_uobj(&ib_uverbs_dct_idr, cmd.user_handle, file->ucontext);
3211 if (!uobj)
3212 return -EINVAL;
3213
3214 dct = uobj->object;
3215 obj = container_of(dct->uobject, struct ib_udct_object, uobject);
3216
3217 ret = ib_destroy_dct(dct);
3218 if (!ret)
3219 uobj->live = 0;
3220
3221 put_uobj_write(uobj);
3222
3223 if (ret)
3224 return ret;
3225
3226 idr_remove_uobj(&ib_uverbs_dct_idr, uobj);
3227
3228 mutex_lock(&file->mutex);
3229 list_del(&uobj->list);
3230 mutex_unlock(&file->mutex);
3231
3232 memset(&resp, 0, sizeof(resp));
3233
3234 put_uobj(uobj);
3235
3236 ret = ucore->ops->copy_to(ucore, &resp, sizeof(resp));
3237 if (ret)
3238 return ret;
3239
3240 return in_len;
3241 }
3242
ib_uverbs_exp_query_dct(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3243 ssize_t ib_uverbs_exp_query_dct(struct ib_uverbs_file *file,
3244 struct ib_udata *ucore,
3245 struct ib_udata *uhw)
3246 {
3247 int in_len = ucore->inlen + uhw->inlen;
3248 int out_len = ucore->outlen + uhw->outlen;
3249 struct ib_uverbs_query_dct cmd;
3250 struct ib_uverbs_query_dct_resp resp;
3251 struct ib_dct *dct;
3252 struct ib_dct_attr *attr;
3253 int err;
3254
3255 if (out_len < sizeof(resp))
3256 return -ENOSPC;
3257
3258 err = ucore->ops->copy_from(&cmd, ucore, sizeof(cmd));
3259 if (err)
3260 return err;
3261
3262 attr = kmalloc(sizeof(*attr), GFP_KERNEL);
3263 if (!attr) {
3264 err = -ENOMEM;
3265 goto out;
3266 }
3267
3268 dct = idr_read_dct(cmd.dct_handle, file->ucontext);
3269 if (!dct) {
3270 err = -EINVAL;
3271 goto out;
3272 }
3273
3274 err = ib_query_dct(dct, attr);
3275
3276 put_dct_read(dct);
3277
3278 if (err)
3279 goto out;
3280
3281 memset(&resp, 0, sizeof(resp));
3282
3283 resp.dc_key = attr->dc_key;
3284 resp.access_flags = attr->access_flags;
3285 resp.flow_label = attr->flow_label;
3286 resp.key_violations = attr->key_violations;
3287 resp.port = attr->port;
3288 resp.min_rnr_timer = attr->min_rnr_timer;
3289 resp.tclass = attr->tclass;
3290 resp.mtu = attr->mtu;
3291 resp.pkey_index = attr->pkey_index;
3292 resp.gid_index = attr->gid_index;
3293 resp.hop_limit = attr->hop_limit;
3294 resp.state = attr->state;
3295
3296 err = ucore->ops->copy_to(ucore, &resp, sizeof(resp));
3297
3298 out:
3299 kfree(attr);
3300
3301 return err ? err : in_len;
3302 }
3303
3304 /*
3305 * Experimental functions
3306 */
3307
3308 static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" };
3309
kern_spec_to_ib_spec(struct ib_uverbs_flow_spec * kern_spec,union ib_flow_spec * ib_spec)3310 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
3311 union ib_flow_spec *ib_spec)
3312 {
3313 ib_spec->type = kern_spec->type;
3314
3315 switch (ib_spec->type) {
3316 case IB_FLOW_SPEC_ETH:
3317 ib_spec->eth.size = sizeof(struct ib_flow_spec_eth);
3318 memcpy(&ib_spec->eth.val, &kern_spec->eth.val,
3319 sizeof(struct ib_flow_eth_filter));
3320 memcpy(&ib_spec->eth.mask, &kern_spec->eth.mask,
3321 sizeof(struct ib_flow_eth_filter));
3322 break;
3323 case IB_FLOW_SPEC_IB:
3324 ib_spec->ib.size = sizeof(struct ib_flow_spec_ib);
3325 memcpy(&ib_spec->ib.val, &kern_spec->ib.val,
3326 sizeof(struct ib_flow_ib_filter));
3327 memcpy(&ib_spec->ib.mask, &kern_spec->ib.mask,
3328 sizeof(struct ib_flow_ib_filter));
3329 break;
3330 case IB_FLOW_SPEC_IPV4:
3331 ib_spec->ipv4.size = sizeof(struct ib_flow_spec_ipv4);
3332 memcpy(&ib_spec->ipv4.val, &kern_spec->ipv4.val,
3333 sizeof(struct ib_flow_ipv4_filter));
3334 memcpy(&ib_spec->ipv4.mask, &kern_spec->ipv4.mask,
3335 sizeof(struct ib_flow_ipv4_filter));
3336 break;
3337 case IB_FLOW_SPEC_TCP:
3338 case IB_FLOW_SPEC_UDP:
3339 ib_spec->tcp_udp.size = sizeof(struct ib_flow_spec_tcp_udp);
3340 memcpy(&ib_spec->tcp_udp.val, &kern_spec->tcp_udp.val,
3341 sizeof(struct ib_flow_tcp_udp_filter));
3342 memcpy(&ib_spec->tcp_udp.mask, &kern_spec->tcp_udp.mask,
3343 sizeof(struct ib_flow_tcp_udp_filter));
3344 break;
3345 default:
3346 return -EINVAL;
3347 }
3348 return 0;
3349 }
3350
ib_uverbs_ex_create_flow(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3351 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
3352 struct ib_udata *ucore,
3353 struct ib_udata *uhw)
3354 {
3355 struct ib_uverbs_create_flow cmd;
3356 struct ib_uverbs_create_flow_resp resp;
3357 struct ib_uobject *uobj;
3358 struct ib_flow *flow_id;
3359 struct ib_uverbs_flow_attr *kern_flow_attr;
3360 struct ib_flow_attr *flow_attr;
3361 struct ib_qp *qp;
3362 int err = 0;
3363 void *kern_spec;
3364 void *ib_spec;
3365 int i;
3366
3367 if (ucore->outlen < sizeof(resp))
3368 return -ENOSPC;
3369
3370 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3371 if (err)
3372 return err;
3373
3374 ucore->inbuf += sizeof(cmd);
3375 ucore->inlen -= sizeof(cmd);
3376
3377 if (cmd.comp_mask)
3378 return -EINVAL;
3379
3380 if (!priv_check(curthread, PRIV_NET_RAW) && !disable_raw_qp_enforcement)
3381 return -EPERM;
3382
3383 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3384 return -EINVAL;
3385
3386 if (cmd.flow_attr.size > ucore->inlen ||
3387 cmd.flow_attr.size >
3388 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3389 return -EINVAL;
3390
3391 if (cmd.flow_attr.num_of_specs) {
3392 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) +
3393 cmd.flow_attr.size, GFP_KERNEL);
3394 if (!kern_flow_attr)
3395 return -ENOMEM;
3396
3397 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr));
3398 err = ib_copy_from_udata(kern_flow_attr + 1, ucore,
3399 cmd.flow_attr.size);
3400 if (err)
3401 goto err_free_attr;
3402 } else {
3403 kern_flow_attr = &cmd.flow_attr;
3404 }
3405
3406 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL);
3407 if (!uobj) {
3408 err = -ENOMEM;
3409 goto err_free_attr;
3410 }
3411 init_uobj(uobj, 0, file->ucontext, &rule_lock_class);
3412 down_write(&uobj->mutex);
3413
3414 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
3415 if (!qp) {
3416 err = -EINVAL;
3417 goto err_uobj;
3418 }
3419
3420 flow_attr = kmalloc(sizeof(*flow_attr) + cmd.flow_attr.size,
3421 GFP_KERNEL);
3422 if (!flow_attr) {
3423 err = -ENOMEM;
3424 goto err_put;
3425 }
3426
3427 flow_attr->type = kern_flow_attr->type;
3428 flow_attr->priority = kern_flow_attr->priority;
3429 flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3430 flow_attr->port = kern_flow_attr->port;
3431 flow_attr->flags = kern_flow_attr->flags;
3432 flow_attr->size = sizeof(*flow_attr);
3433
3434 kern_spec = kern_flow_attr + 1;
3435 ib_spec = flow_attr + 1;
3436 for (i = 0; i < flow_attr->num_of_specs &&
3437 cmd.flow_attr.size >
3438 offsetof(struct ib_uverbs_flow_spec, reserved) &&
3439 cmd.flow_attr.size >=
3440 ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) {
3441 err = kern_spec_to_ib_spec(kern_spec, ib_spec);
3442 if (err)
3443 goto err_free;
3444 flow_attr->size +=
3445 ((union ib_flow_spec *)ib_spec)->size;
3446 cmd.flow_attr.size -=
3447 ((struct ib_uverbs_flow_spec *)kern_spec)->size;
3448 kern_spec += ((struct ib_uverbs_flow_spec *)kern_spec)->size;
3449 ib_spec += ((union ib_flow_spec *)ib_spec)->size;
3450 }
3451 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3452 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3453 i, cmd.flow_attr.size);
3454 goto err_free;
3455 }
3456 flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER);
3457 if (IS_ERR(flow_id)) {
3458 err = PTR_ERR(flow_id);
3459 goto err_free;
3460 }
3461 flow_id->qp = qp;
3462 flow_id->uobject = uobj;
3463 uobj->object = flow_id;
3464
3465 err = idr_add_uobj(&ib_uverbs_rule_idr, uobj);
3466 if (err)
3467 goto destroy_flow;
3468
3469 memset(&resp, 0, sizeof(resp));
3470 resp.flow_handle = uobj->id;
3471
3472 err = ib_copy_to_udata(ucore,
3473 &resp, sizeof(resp));
3474 if (err)
3475 goto err_copy;
3476
3477 put_qp_read(qp);
3478 mutex_lock(&file->mutex);
3479 list_add_tail(&uobj->list, &file->ucontext->rule_list);
3480 mutex_unlock(&file->mutex);
3481
3482 uobj->live = 1;
3483
3484 up_write(&uobj->mutex);
3485 kfree(flow_attr);
3486 if (cmd.flow_attr.num_of_specs)
3487 kfree(kern_flow_attr);
3488 return 0;
3489 err_copy:
3490 idr_remove_uobj(&ib_uverbs_rule_idr, uobj);
3491 destroy_flow:
3492 ib_destroy_flow(flow_id);
3493 err_free:
3494 kfree(flow_attr);
3495 err_put:
3496 put_qp_read(qp);
3497 err_uobj:
3498 put_uobj_write(uobj);
3499 err_free_attr:
3500 if (cmd.flow_attr.num_of_specs)
3501 kfree(kern_flow_attr);
3502 return err;
3503 }
3504
ib_uverbs_ex_destroy_flow(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3505 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file,
3506 struct ib_udata *ucore,
3507 struct ib_udata *uhw)
3508 {
3509 struct ib_uverbs_destroy_flow cmd;
3510 struct ib_flow *flow_id;
3511 struct ib_uobject *uobj;
3512 int ret;
3513
3514 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3515 if (ret)
3516 return ret;
3517
3518 uobj = idr_write_uobj(&ib_uverbs_rule_idr, cmd.flow_handle,
3519 file->ucontext);
3520 if (!uobj)
3521 return -EINVAL;
3522 flow_id = uobj->object;
3523
3524 ret = ib_destroy_flow(flow_id);
3525 if (!ret)
3526 uobj->live = 0;
3527
3528 put_uobj_write(uobj);
3529
3530 idr_remove_uobj(&ib_uverbs_rule_idr, uobj);
3531
3532 mutex_lock(&file->mutex);
3533 list_del(&uobj->list);
3534 mutex_unlock(&file->mutex);
3535
3536 put_uobj(uobj);
3537
3538 return ret;
3539 }
3540
ib_uverbs_exp_modify_qp(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3541 ssize_t ib_uverbs_exp_modify_qp(struct ib_uverbs_file *file,
3542 struct ib_udata *ucore, struct ib_udata *uhw)
3543 {
3544 const char __user *buf = ucore->inbuf;
3545 int in_len = ucore->inlen + uhw->inlen;
3546 int out_len = ucore->outlen + uhw->outlen;
3547
3548 return __uverbs_modify_qp(file, buf, in_len, out_len,
3549 IB_USER_VERBS_CMD_EXTENDED);
3550 }
3551
3552
ib_uverbs_exp_create_cq(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3553 ssize_t ib_uverbs_exp_create_cq(struct ib_uverbs_file *file,
3554 struct ib_udata *ucore, struct ib_udata *uhw)
3555 {
3556 const char __user *buf = ucore->inbuf;
3557 int in_len = ucore->inlen + uhw->inlen;
3558 int out_len = ucore->outlen + uhw->outlen;
3559 struct ib_uverbs_create_cq_ex cmd;
3560
3561 if (copy_from_user(&cmd, buf, sizeof(cmd)))
3562 return -EFAULT;
3563
3564 return create_cq(file, buf, in_len, out_len, &cmd,
3565 IB_USER_VERBS_CMD_EXTENDED, ucore->outbuf);
3566 }
3567
ib_uverbs_exp_modify_cq(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3568 ssize_t ib_uverbs_exp_modify_cq(struct ib_uverbs_file *file,
3569 struct ib_udata *ucore, struct ib_udata *uhw)
3570 {
3571 const char __user *buf = ucore->inbuf;
3572 int in_len = ucore->inlen + uhw->inlen;
3573 struct ib_uverbs_modify_cq_ex cmd;
3574 struct ib_cq *cq;
3575 struct ib_cq_attr attr;
3576 int ret;
3577
3578 if (copy_from_user(&cmd, buf, sizeof(cmd)))
3579 return -EFAULT;
3580
3581 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
3582 if (!cq)
3583 return -EINVAL;
3584
3585 attr.moderation.cq_count = cmd.cq_count;
3586 attr.moderation.cq_period = cmd.cq_period;
3587 attr.cq_cap_flags = cmd.cq_cap_flags;
3588
3589 ret = ib_modify_cq(cq, &attr, cmd.attr_mask);
3590
3591 put_cq_read(cq);
3592
3593 return ret ? ret : in_len;
3594 }
3595
3596
ib_uverbs_exp_query_device(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3597 ssize_t ib_uverbs_exp_query_device(struct ib_uverbs_file *file,
3598 struct ib_udata *ucore, struct ib_udata *uhw)
3599 {
3600 struct ib_uverbs_exp_query_device_resp resp;
3601 struct ib_exp_device_attr exp_attr;
3602 int ret;
3603
3604 if (ucore->outlen + uhw->outlen < sizeof(resp))
3605 return -ENOSPC;
3606
3607 memset(&resp, 0, sizeof(resp));
3608 memset(&exp_attr, 0, sizeof(exp_attr));
3609 ret = ib_exp_query_device(file->device->ib_dev, &exp_attr);
3610 if (ret)
3611 return ret;
3612
3613 ib_uverbs_query_device_assign(&resp.base, &exp_attr.base, file);
3614
3615 resp.comp_mask = 0;
3616 resp.device_cap_flags2 = 0;
3617
3618 /*
3619 * Handle regular attr fields
3620 */
3621 if (exp_attr.base.comp_mask & IB_DEVICE_ATTR_WITH_TIMESTAMP_MASK) {
3622 resp.timestamp_mask = exp_attr.base.timestamp_mask;
3623 resp.comp_mask |= IB_EXP_DEVICE_ATTR_WITH_TIMESTAMP_MASK;
3624 }
3625
3626 if (exp_attr.base.comp_mask & IB_DEVICE_ATTR_WITH_HCA_CORE_CLOCK) {
3627 resp.hca_core_clock = exp_attr.base.hca_core_clock;
3628 resp.comp_mask |= IB_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK;
3629 }
3630
3631 /*
3632 * Handle experimental attr fields
3633 */
3634 if (exp_attr.exp_comp_mask & IB_EXP_DEVICE_ATTR_CAP_FLAGS2) {
3635 resp.device_cap_flags2 = exp_attr.device_cap_flags2;
3636 resp.comp_mask |= IB_EXP_DEVICE_ATTR_CAP_FLAGS2;
3637 }
3638
3639 if (exp_attr.exp_comp_mask & IB_EXP_DEVICE_ATTR_DC_REQ_RD) {
3640 resp.dc_rd_req = exp_attr.dc_rd_req;
3641 resp.comp_mask |= IB_EXP_DEVICE_ATTR_DC_REQ_RD;
3642 }
3643
3644 if (exp_attr.exp_comp_mask & IB_EXP_DEVICE_ATTR_DC_RES_RD) {
3645 resp.dc_rd_res = exp_attr.dc_rd_res;
3646 resp.comp_mask |= IB_EXP_DEVICE_ATTR_DC_RES_RD;
3647 }
3648
3649 if (exp_attr.exp_comp_mask & IB_EXP_DEVICE_ATTR_INLINE_RECV_SZ) {
3650 resp.inline_recv_sz = exp_attr.inline_recv_sz;
3651 resp.comp_mask |= IB_EXP_DEVICE_ATTR_INLINE_RECV_SZ;
3652 }
3653
3654 if (exp_attr.exp_comp_mask & IB_EXP_DEVICE_ATTR_RSS_TBL_SZ) {
3655 resp.max_rss_tbl_sz = exp_attr.max_rss_tbl_sz;
3656 resp.comp_mask |= IB_EXP_DEVICE_ATTR_RSS_TBL_SZ;
3657 }
3658
3659 if (copy_to_user(ucore->outbuf, &resp, sizeof(resp)))
3660 return -EFAULT;
3661
3662 return ucore->inlen + uhw->inlen;
3663 }
3664
ib_uverbs_exp_create_qp(struct ib_uverbs_file * file,struct ib_udata * ucore,struct ib_udata * uhw)3665 ssize_t ib_uverbs_exp_create_qp(struct ib_uverbs_file *file,
3666 struct ib_udata *ucore, struct ib_udata *uhw)
3667 {
3668 struct ib_uqp_object *obj;
3669 struct ib_device *device;
3670 struct ib_pd *pd = NULL;
3671 struct ib_xrcd *xrcd = NULL;
3672 struct ib_uobject *uninitialized_var(xrcd_uobj);
3673 struct ib_cq *scq = NULL, *rcq = NULL;
3674 struct ib_srq *srq = NULL;
3675 struct ib_qp *qp;
3676 struct ib_exp_qp_init_attr attr;
3677 int ret;
3678 struct ib_uverbs_exp_create_qp cmd_exp;
3679 struct ib_uverbs_exp_create_qp_resp resp_exp;
3680 struct ib_qp *parentqp = NULL;
3681
3682 memset(&cmd_exp, 0, sizeof(cmd_exp));
3683
3684 ret = ucore->ops->copy_from(&cmd_exp, ucore, sizeof(cmd_exp));
3685 if (ret)
3686 return ret;
3687
3688 if (!disable_raw_qp_enforcement &&
3689 cmd_exp.qp_type == IB_QPT_RAW_PACKET && !priv_check(curthread,
3690 PRIV_NET_RAW))
3691 return -EPERM;
3692
3693 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3694 if (!obj)
3695 return -ENOMEM;
3696
3697 init_uobj(&obj->uevent.uobject, cmd_exp.user_handle, file->ucontext,
3698 &qp_lock_class);
3699 down_write(&obj->uevent.uobject.mutex);
3700
3701 if (cmd_exp.qp_type == IB_QPT_XRC_TGT) {
3702 xrcd = idr_read_xrcd(cmd_exp.pd_handle, file->ucontext, &xrcd_uobj);
3703 if (!xrcd) {
3704 ret = -EINVAL;
3705 goto err_put;
3706 }
3707 device = xrcd->device;
3708 } else {
3709 if (cmd_exp.qp_type == IB_QPT_XRC_INI) {
3710 cmd_exp.max_recv_wr = 0;
3711 cmd_exp.max_recv_sge = 0;
3712 } else {
3713 if (cmd_exp.is_srq) {
3714 srq = idr_read_srq(cmd_exp.srq_handle, file->ucontext);
3715 if (!srq || srq->srq_type != IB_SRQT_BASIC) {
3716 ret = -EINVAL;
3717 goto err_put;
3718 }
3719 }
3720
3721 if (cmd_exp.recv_cq_handle != cmd_exp.send_cq_handle) {
3722 rcq = idr_read_cq(cmd_exp.recv_cq_handle, file->ucontext, 0);
3723 if (!rcq) {
3724 ret = -EINVAL;
3725 goto err_put;
3726 }
3727 }
3728 }
3729
3730 scq = idr_read_cq(cmd_exp.send_cq_handle, file->ucontext, !!rcq);
3731 rcq = rcq ?: scq;
3732 pd = idr_read_pd(cmd_exp.pd_handle, file->ucontext);
3733 if (!pd || !scq) {
3734 ret = -EINVAL;
3735 goto err_put;
3736 }
3737
3738 device = pd->device;
3739 }
3740
3741 memset(&attr, 0, sizeof(attr));
3742 attr.event_handler = ib_uverbs_qp_event_handler;
3743 attr.qp_context = file;
3744 attr.send_cq = scq;
3745 attr.recv_cq = rcq;
3746 attr.srq = srq;
3747 attr.xrcd = xrcd;
3748 attr.sq_sig_type = cmd_exp.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
3749 attr.qp_type = cmd_exp.qp_type;
3750 attr.create_flags = 0;
3751
3752 attr.cap.max_send_wr = cmd_exp.max_send_wr;
3753 attr.cap.max_recv_wr = cmd_exp.max_recv_wr;
3754 attr.cap.max_send_sge = cmd_exp.max_send_sge;
3755 attr.cap.max_recv_sge = cmd_exp.max_recv_sge;
3756 attr.cap.max_inline_data = cmd_exp.max_inline_data;
3757
3758 if (cmd_exp.comp_mask & IB_UVERBS_EXP_CREATE_QP_CAP_FLAGS)
3759 attr.create_flags |= cmd_exp.qp_cap_flags &
3760 (IB_QP_CREATE_CROSS_CHANNEL |
3761 IB_QP_CREATE_MANAGED_SEND |
3762 IB_QP_CREATE_MANAGED_RECV);
3763
3764 if (cmd_exp.comp_mask & IB_UVERBS_EXP_CREATE_QP_QPG) {
3765 struct ib_uverbs_qpg *qpg;
3766 if (cmd_exp.qp_type != IB_QPT_RAW_PACKET &&
3767 cmd_exp.qp_type != IB_QPT_UD) {
3768 ret = -EINVAL;
3769 goto err_put;
3770 }
3771 qpg = &cmd_exp.qpg;
3772 switch (qpg->qpg_type) {
3773 case IB_QPG_PARENT:
3774 attr.parent_attrib.rss_child_count =
3775 qpg->parent_attrib.rss_child_count;
3776 attr.parent_attrib.tss_child_count =
3777 qpg->parent_attrib.tss_child_count;
3778 break;
3779 case IB_QPG_CHILD_RX:
3780 case IB_QPG_CHILD_TX:
3781 parentqp = idr_read_qp(qpg->parent_handle,
3782 file->ucontext);
3783 if (!parentqp) {
3784 ret = -EINVAL;
3785 goto err_put;
3786 }
3787 attr.qpg_parent = parentqp;
3788 break;
3789 default:
3790 ret = -EINVAL;
3791 goto err_put;
3792 }
3793 attr.qpg_type = qpg->qpg_type;
3794 }
3795
3796 if (cmd_exp.comp_mask & IB_UVERBS_EXP_CREATE_QP_INL_RECV)
3797 attr.max_inl_recv = cmd_exp.max_inl_recv;
3798
3799 obj->uevent.events_reported = 0;
3800 INIT_LIST_HEAD(&obj->uevent.event_list);
3801 INIT_LIST_HEAD(&obj->mcast_list);
3802
3803 if (cmd_exp.qp_type == IB_QPT_XRC_TGT)
3804 qp = ib_create_qp(pd, (struct ib_qp_init_attr *)&attr);
3805 else
3806 qp = device->exp_create_qp(pd, &attr, uhw);
3807
3808 if (IS_ERR(qp)) {
3809 ret = PTR_ERR(qp);
3810 goto err_put;
3811 }
3812
3813 if (cmd_exp.qp_type != IB_QPT_XRC_TGT) {
3814 qp->real_qp = qp;
3815 qp->device = device;
3816 qp->pd = pd;
3817 qp->send_cq = attr.send_cq;
3818 qp->recv_cq = attr.recv_cq;
3819 qp->srq = attr.srq;
3820 qp->event_handler = attr.event_handler;
3821 qp->qp_context = attr.qp_context;
3822 qp->qp_type = attr.qp_type;
3823 atomic_set(&qp->usecnt, 0);
3824 atomic_inc(&pd->usecnt);
3825 atomic_inc(&attr.send_cq->usecnt);
3826 if (attr.recv_cq)
3827 atomic_inc(&attr.recv_cq->usecnt);
3828 if (attr.srq)
3829 atomic_inc(&attr.srq->usecnt);
3830 }
3831 qp->uobject = &obj->uevent.uobject;
3832
3833 obj->uevent.uobject.object = qp;
3834 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
3835 if (ret)
3836 goto err_destroy;
3837
3838 memset(&resp_exp, 0, sizeof(resp_exp));
3839 resp_exp.qpn = qp->qp_num;
3840 resp_exp.qp_handle = obj->uevent.uobject.id;
3841 resp_exp.max_recv_sge = attr.cap.max_recv_sge;
3842 resp_exp.max_send_sge = attr.cap.max_send_sge;
3843 resp_exp.max_recv_wr = attr.cap.max_recv_wr;
3844 resp_exp.max_send_wr = attr.cap.max_send_wr;
3845 resp_exp.max_inline_data = attr.cap.max_inline_data;
3846
3847 if (cmd_exp.comp_mask & IB_UVERBS_EXP_CREATE_QP_INL_RECV) {
3848 resp_exp.comp_mask |= IB_UVERBS_EXP_CREATE_QP_RESP_INL_RECV;
3849 resp_exp.max_inl_recv = attr.max_inl_recv;
3850 }
3851
3852 ret = ucore->ops->copy_to(ucore, &resp_exp, sizeof(resp_exp));
3853 if (ret)
3854 goto err_copy;
3855
3856 if (xrcd) {
3857 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3858 atomic_inc(&obj->uxrcd->refcnt);
3859 put_xrcd_read(xrcd_uobj);
3860 }
3861
3862 if (pd)
3863 put_pd_read(pd);
3864 if (scq)
3865 put_cq_read(scq);
3866 if (rcq && rcq != scq)
3867 put_cq_read(rcq);
3868 if (srq)
3869 put_srq_read(srq);
3870 if (parentqp)
3871 put_qp_read(parentqp);
3872
3873 mutex_lock(&file->mutex);
3874 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
3875 mutex_unlock(&file->mutex);
3876
3877 obj->uevent.uobject.live = 1;
3878
3879 up_write(&obj->uevent.uobject.mutex);
3880
3881 return ucore->inlen + uhw->inlen;
3882
3883 err_copy:
3884 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
3885
3886 err_destroy:
3887 ib_destroy_qp(qp);
3888
3889 err_put:
3890 if (xrcd)
3891 put_xrcd_read(xrcd_uobj);
3892 if (pd)
3893 put_pd_read(pd);
3894 if (scq)
3895 put_cq_read(scq);
3896 if (rcq && rcq != scq)
3897 put_cq_read(rcq);
3898 if (srq)
3899 put_srq_read(srq);
3900 if (parentqp)
3901 put_qp_read(parentqp);
3902
3903 put_uobj_write(&obj->uevent.uobject);
3904 return ret;
3905 }
3906
ib_exp_query_device(struct ib_device * device,struct ib_exp_device_attr * device_attr)3907 int ib_exp_query_device(struct ib_device *device,
3908 struct ib_exp_device_attr *device_attr)
3909 {
3910 return device->exp_query_device(device, device_attr);
3911 }
3912 EXPORT_SYMBOL(ib_exp_query_device);
3913