1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2018 - 2023 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "irdma_main.h"
36
37 #define IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
38
kc_rdma_flow_label_to_udp_sport(u32 fl)39 static u16 kc_rdma_flow_label_to_udp_sport(u32 fl) {
40 u32 fl_low = fl & 0x03FFF;
41 u32 fl_high = fl & 0xFC000;
42
43 fl_low ^= fl_high >> 14;
44
45 return (u16)(fl_low | IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN);
46 }
47
48 #define IRDMA_GRH_FLOWLABEL_MASK (0x000FFFFF)
49
kc_rdma_calc_flow_label(u32 lqpn,u32 rqpn)50 static u32 kc_rdma_calc_flow_label(u32 lqpn, u32 rqpn) {
51 u64 fl = (u64)lqpn * rqpn;
52
53 fl ^= fl >> 20;
54 fl ^= fl >> 40;
55
56 return (u32)(fl & IRDMA_GRH_FLOWLABEL_MASK);
57 }
58
59 u16
kc_rdma_get_udp_sport(u32 fl,u32 lqpn,u32 rqpn)60 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
61 {
62 if (!fl)
63 fl = kc_rdma_calc_flow_label(lqpn, rqpn);
64 return kc_rdma_flow_label_to_udp_sport(fl);
65 }
66
67 void
irdma_get_dev_fw_str(struct ib_device * dev,char * str,size_t str_len)68 irdma_get_dev_fw_str(struct ib_device *dev,
69 char *str,
70 size_t str_len)
71 {
72 struct irdma_device *iwdev = to_iwdev(dev);
73
74 snprintf(str, str_len, "%u.%u",
75 irdma_fw_major_ver(&iwdev->rf->sc_dev),
76 irdma_fw_minor_ver(&iwdev->rf->sc_dev));
77 }
78
79 int
irdma_add_gid(struct ib_device * device,u8 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr,void ** context)80 irdma_add_gid(struct ib_device *device,
81 u8 port_num,
82 unsigned int index,
83 const union ib_gid *gid,
84 const struct ib_gid_attr *attr,
85 void **context)
86 {
87 return 0;
88 }
89
90 int
irdma_del_gid(struct ib_device * device,u8 port_num,unsigned int index,void ** context)91 irdma_del_gid(struct ib_device *device,
92 u8 port_num,
93 unsigned int index,
94 void **context)
95 {
96 return 0;
97 }
98
99 /**
100 * irdma_alloc_mr - register stag for fast memory registration
101 * @pd: ibpd pointer
102 * @mr_type: memory for stag registrion
103 * @max_num_sg: man number of pages
104 */
105 struct ib_mr *
irdma_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_num_sg)106 irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
107 u32 max_num_sg)
108 {
109 struct irdma_device *iwdev = to_iwdev(pd->device);
110 struct irdma_pble_alloc *palloc;
111 struct irdma_pbl *iwpbl;
112 struct irdma_mr *iwmr;
113 int status;
114 u32 stag;
115 int err_code = -ENOMEM;
116
117 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
118 if (!iwmr)
119 return ERR_PTR(-ENOMEM);
120
121 stag = irdma_create_stag(iwdev);
122 if (!stag) {
123 err_code = -ENOMEM;
124 goto err;
125 }
126
127 iwmr->stag = stag;
128 iwmr->ibmr.rkey = stag;
129 iwmr->ibmr.lkey = stag;
130 iwmr->ibmr.pd = pd;
131 iwmr->ibmr.device = pd->device;
132 iwpbl = &iwmr->iwpbl;
133 iwpbl->iwmr = iwmr;
134 iwmr->type = IRDMA_MEMREG_TYPE_MEM;
135 palloc = &iwpbl->pble_alloc;
136 iwmr->page_cnt = max_num_sg;
137 /* Assume system PAGE_SIZE as the sg page sizes are unknown. */
138 iwmr->len = max_num_sg * PAGE_SIZE;
139 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
140 false);
141 if (status)
142 goto err_get_pble;
143
144 err_code = irdma_hw_alloc_stag(iwdev, iwmr);
145 if (err_code)
146 goto err_alloc_stag;
147
148 iwpbl->pbl_allocated = true;
149
150 return &iwmr->ibmr;
151 err_alloc_stag:
152 irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
153 err_get_pble:
154 irdma_free_stag(iwdev, stag);
155 err:
156 kfree(iwmr);
157
158 return ERR_PTR(err_code);
159 }
160
161 #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
162 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
163
164 /**
165 * irdma_alloc_ucontext - Allocate the user context data structure
166 * @ibdev: ib device pointer
167 * @udata: user data
168 *
169 * This keeps track of all objects associated with a particular
170 * user-mode client.
171 */
172 struct ib_ucontext *
irdma_alloc_ucontext(struct ib_device * ibdev,struct ib_udata * udata)173 irdma_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
174 {
175 struct irdma_device *iwdev = to_iwdev(ibdev);
176 struct irdma_alloc_ucontext_req req = {0};
177 struct irdma_alloc_ucontext_resp uresp = {0};
178 struct irdma_ucontext *ucontext;
179 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
180
181 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||
182 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN)
183 return ERR_PTR(-EINVAL);
184
185 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)))
186 return ERR_PTR(-EINVAL);
187
188 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER)
189 goto ver_error;
190
191 ucontext = kzalloc(sizeof(*ucontext), GFP_KERNEL);
192 if (!ucontext)
193 return ERR_PTR(-ENOMEM);
194
195 ucontext->iwdev = iwdev;
196 ucontext->abi_ver = req.userspace_ver;
197
198 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR)
199 ucontext->use_raw_attrs = true;
200
201 /* GEN_1 legacy support with libi40iw */
202 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {
203 if (uk_attrs->hw_rev != IRDMA_GEN_1) {
204 kfree(ucontext);
205 return ERR_PTR(-EOPNOTSUPP);
206 }
207
208 ucontext->legacy_mode = true;
209 uresp.max_qps = iwdev->rf->max_qp;
210 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
211 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
212 uresp.kernel_ver = req.userspace_ver;
213 if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) {
214 kfree(ucontext);
215 return ERR_PTR(-EFAULT);
216 }
217 } else {
218 u64 bar_off;
219
220 uresp.kernel_ver = IRDMA_ABI_VER;
221 uresp.feature_flags = uk_attrs->feature_flags;
222 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags;
223 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges;
224 uresp.max_hw_inline = uk_attrs->max_hw_inline;
225 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta;
226 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta;
227 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk;
228 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size;
229 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
230 uresp.hw_rev = uk_attrs->hw_rev;
231 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
232 uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
233 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
234
235 bar_off =
236 (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
237
238 spin_lock_init(&ucontext->mmap_tbl_lock);
239 ucontext->db_mmap_entry =
240 irdma_user_mmap_entry_add_hash(ucontext, bar_off,
241 IRDMA_MMAP_IO_NC,
242 &uresp.db_mmap_key);
243 if (!ucontext->db_mmap_entry) {
244 spin_lock_destroy(&ucontext->mmap_tbl_lock);
245 kfree(ucontext);
246 return ERR_PTR(-ENOMEM);
247 }
248
249 if (ib_copy_to_udata(udata, &uresp,
250 min(sizeof(uresp), udata->outlen))) {
251 irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry);
252 spin_lock_destroy(&ucontext->mmap_tbl_lock);
253 kfree(ucontext);
254 return ERR_PTR(-EFAULT);
255 }
256 }
257
258 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
259 spin_lock_init(&ucontext->cq_reg_mem_list_lock);
260 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
261 spin_lock_init(&ucontext->qp_reg_mem_list_lock);
262 INIT_LIST_HEAD(&ucontext->vma_list);
263 mutex_init(&ucontext->vma_list_mutex);
264
265 return &ucontext->ibucontext;
266
267 ver_error:
268 irdma_dev_err(&iwdev->ibdev,
269 "Invalid userspace driver version detected. Detected version %d, should be %d\n",
270 req.userspace_ver, IRDMA_ABI_VER);
271 return ERR_PTR(-EINVAL);
272 }
273
274 /**
275 * irdma_dealloc_ucontext - deallocate the user context data structure
276 * @context: user context created during alloc
277 */
278 int
irdma_dealloc_ucontext(struct ib_ucontext * context)279 irdma_dealloc_ucontext(struct ib_ucontext *context)
280 {
281 struct irdma_ucontext *ucontext = to_ucontext(context);
282
283 irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry);
284 spin_lock_destroy(&ucontext->mmap_tbl_lock);
285 kfree(ucontext);
286
287 return 0;
288 }
289
290 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
291
292 /**
293 * irdma_alloc_pd - allocate protection domain
294 * @ibdev: IB device
295 * @context: user context
296 * @udata: user data
297 */
298 struct ib_pd *
irdma_alloc_pd(struct ib_device * ibdev,struct ib_ucontext * context,struct ib_udata * udata)299 irdma_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata)
300 {
301 struct irdma_pd *iwpd;
302 struct irdma_device *iwdev = to_iwdev(ibdev);
303 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
304 struct irdma_pci_f *rf = iwdev->rf;
305 struct irdma_alloc_pd_resp uresp = {0};
306 struct irdma_sc_pd *sc_pd;
307 u32 pd_id = 0;
308 int err;
309
310 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id,
311 &rf->next_pd);
312 if (err)
313 return ERR_PTR(err);
314
315 iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL);
316 if (!iwpd) {
317 err = -ENOMEM;
318 goto free_res;
319 }
320
321 sc_pd = &iwpd->sc_pd;
322 if (udata) {
323 struct irdma_ucontext *ucontext = to_ucontext(context);
324
325 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
326 uresp.pd_id = pd_id;
327 if (ib_copy_to_udata(udata, &uresp,
328 min(sizeof(uresp), udata->outlen))) {
329 err = -EFAULT;
330 goto error;
331 }
332 } else {
333 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);
334 }
335
336 spin_lock_init(&iwpd->udqp_list_lock);
337 INIT_LIST_HEAD(&iwpd->udqp_list);
338
339 return &iwpd->ibpd;
340
341 error:
342 kfree(iwpd);
343 free_res:
344
345 irdma_free_rsrc(rf, rf->allocated_pds, pd_id);
346
347 return ERR_PTR(err);
348 }
349
350 int
irdma_dealloc_pd(struct ib_pd * ibpd)351 irdma_dealloc_pd(struct ib_pd *ibpd)
352 {
353 struct irdma_pd *iwpd = to_iwpd(ibpd);
354 struct irdma_device *iwdev = to_iwdev(ibpd->device);
355
356 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
357 kfree(iwpd);
358 return 0;
359 }
360
361 /**
362 * irdma_find_qp_update_qs - update QS handle for UD QPs
363 * @rf: RDMA PCI function
364 * @pd: protection domain object
365 * @user_pri: selected user priority
366 */
367 static void
irdma_find_qp_update_qs(struct irdma_pci_f * rf,struct irdma_pd * pd,u8 user_pri)368 irdma_find_qp_update_qs(struct irdma_pci_f *rf,
369 struct irdma_pd *pd, u8 user_pri)
370 {
371 struct irdma_qp *iwqp;
372 struct list_head *tmp_node, *list_node;
373 struct irdma_udqs_work *work;
374 unsigned long flags;
375 bool qs_change;
376
377 spin_lock_irqsave(&pd->udqp_list_lock, flags);
378 list_for_each_safe(list_node, tmp_node, &pd->udqp_list) {
379 qs_change = true;
380 iwqp = list_entry(list_node, struct irdma_qp, ud_list_elem);
381 irdma_qp_add_ref(&iwqp->ibqp);
382 /* check if qs_handle needs to be changed */
383 if (iwqp->sc_qp.qs_handle == iwqp->sc_qp.vsi->qos[user_pri].qs_handle) {
384 if (iwqp->ctx_info.user_pri == user_pri) {
385 /* qs_handle and user_pri don't change */
386 irdma_qp_rem_ref(&iwqp->ibqp);
387 continue;
388 }
389 qs_change = false;
390 }
391 /* perform qp qos change */
392 work = kzalloc(sizeof(*work), GFP_ATOMIC);
393 if (!work) {
394 irdma_qp_rem_ref(&iwqp->ibqp);
395 spin_unlock_irqrestore(&pd->udqp_list_lock, flags);
396 return;
397 }
398 work->iwqp = iwqp;
399 work->user_prio = user_pri;
400 work->qs_change = qs_change;
401 INIT_WORK(&work->work, irdma_udqp_qs_worker);
402 if (qs_change)
403 irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND);
404 queue_work(rf->iwdev->cleanup_wq, &work->work);
405 }
406 spin_unlock_irqrestore(&pd->udqp_list_lock, flags);
407 }
408
409 static void
irdma_fill_ah_info(struct vnet * vnet,struct irdma_ah_info * ah_info,const struct ib_gid_attr * sgid_attr,union irdma_sockaddr * sgid_addr,union irdma_sockaddr * dgid_addr,u8 * dmac,u8 net_type)410 irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info,
411 const struct ib_gid_attr *sgid_attr,
412 union irdma_sockaddr *sgid_addr,
413 union irdma_sockaddr *dgid_addr,
414 u8 *dmac, u8 net_type)
415 {
416 if (net_type == RDMA_NETWORK_IPV4) {
417 ah_info->ipv4_valid = true;
418 ah_info->dest_ip_addr[0] =
419 ntohl(dgid_addr->saddr_in.sin_addr.s_addr);
420 ah_info->src_ip_addr[0] =
421 ntohl(sgid_addr->saddr_in.sin_addr.s_addr);
422 CURVNET_SET_QUIET(vnet);
423 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0],
424 ah_info->dest_ip_addr[0]);
425 CURVNET_RESTORE();
426 if (ipv4_is_multicast(dgid_addr->saddr_in.sin_addr.s_addr)) {
427 irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac);
428 }
429 } else {
430 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
431 dgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
432 irdma_copy_ip_ntohl(ah_info->src_ip_addr,
433 sgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
434 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr,
435 ah_info->dest_ip_addr);
436 if (rdma_is_multicast_addr(&dgid_addr->saddr_in6.sin6_addr)) {
437 irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac);
438 }
439 }
440 }
441
irdma_roce_get_vlan_prio(if_t ndev,u8 prio)442 static inline u8 irdma_roce_get_vlan_prio(if_t ndev, u8 prio)
443 {
444 return prio;
445 }
446
447 static int
irdma_create_ah_vlan_tag(struct irdma_device * iwdev,struct irdma_pd * pd,struct irdma_ah_info * ah_info,const struct ib_gid_attr * sgid_attr,u8 * dmac)448 irdma_create_ah_vlan_tag(struct irdma_device *iwdev,
449 struct irdma_pd *pd,
450 struct irdma_ah_info *ah_info,
451 const struct ib_gid_attr *sgid_attr,
452 u8 *dmac)
453 {
454 u16 vlan_prio;
455
456 if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev))
457 ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev);
458 else
459 ah_info->vlan_tag = VLAN_N_VID;
460
461 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac);
462
463 if (ah_info->dst_arpindex == -1)
464 return -EINVAL;
465
466 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode)
467 ah_info->vlan_tag = 0;
468
469 if (ah_info->vlan_tag < VLAN_N_VID) {
470 ah_info->insert_vlan_tag = true;
471 vlan_prio = (u16)irdma_roce_get_vlan_prio(sgid_attr->ndev,
472 rt_tos2priority(ah_info->tc_tos));
473 ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT;
474 irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio);
475 }
476 if (iwdev->roce_dcqcn_en) {
477 ah_info->tc_tos &= ~ECN_CODE_PT_MASK;
478 ah_info->tc_tos |= ECN_CODE_PT_VAL;
479 }
480
481 return 0;
482 }
483
484 static int
irdma_create_ah_wait(struct irdma_pci_f * rf,struct irdma_sc_ah * sc_ah,bool sleep)485 irdma_create_ah_wait(struct irdma_pci_f *rf,
486 struct irdma_sc_ah *sc_ah, bool sleep)
487 {
488 int ret;
489
490 if (!sleep) {
491 int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms *
492 CQP_TIMEOUT_THRESHOLD;
493 struct irdma_cqp_request *cqp_request =
494 sc_ah->ah_info.cqp_request;
495
496 do {
497 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
498 mdelay(1);
499 } while (!READ_ONCE(cqp_request->request_done) && --cnt);
500
501 if (cnt && !cqp_request->compl_info.op_ret_val) {
502 irdma_put_cqp_request(&rf->cqp, cqp_request);
503 sc_ah->ah_info.ah_valid = true;
504 } else {
505 ret = !cnt ? -ETIMEDOUT : -EINVAL;
506 irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d",
507 ret, cqp_request->compl_info.op_ret_val);
508 irdma_put_cqp_request(&rf->cqp, cqp_request);
509 if (!cnt && !rf->reset) {
510 rf->reset = true;
511 rf->gen_ops.request_reset(rf);
512 }
513 return ret;
514 }
515 }
516
517 return 0;
518 }
519
520 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
521
522
523 void
irdma_ether_copy(u8 * dmac,struct ib_ah_attr * attr)524 irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr)
525 {
526 ether_addr_copy(dmac, attr->dmac);
527 }
528
529 struct ib_ah *
irdma_create_ah_stub(struct ib_pd * ibpd,struct ib_ah_attr * attr,struct ib_udata * udata)530 irdma_create_ah_stub(struct ib_pd *ibpd,
531 struct ib_ah_attr *attr,
532 struct ib_udata *udata)
533 {
534 return ERR_PTR(-ENOSYS);
535 }
536
537 int
irdma_destroy_ah_stub(struct ib_ah * ibah)538 irdma_destroy_ah_stub(struct ib_ah *ibah)
539 {
540 return -ENOSYS;
541 }
542
543 /**
544 * irdma_create_ah - create address handle
545 * @ibpd: ptr to pd
546 * @attr: address handle attributes
547 * @udata: user data
548 *
549 * returns a pointer to an address handle
550 */
551 struct ib_ah *
irdma_create_ah(struct ib_pd * ibpd,struct ib_ah_attr * attr,struct ib_udata * udata)552 irdma_create_ah(struct ib_pd *ibpd,
553 struct ib_ah_attr *attr,
554 struct ib_udata *udata)
555 {
556 struct irdma_pd *pd = to_iwpd(ibpd);
557 struct irdma_device *iwdev = to_iwdev(ibpd->device);
558 struct irdma_ah *ah;
559 union ib_gid sgid;
560 struct ib_gid_attr sgid_attr;
561 struct irdma_pci_f *rf = iwdev->rf;
562 struct irdma_sc_ah *sc_ah;
563 u32 ah_id = 0;
564 struct irdma_ah_info *ah_info;
565 struct irdma_create_ah_resp uresp = {};
566 union irdma_sockaddr sgid_addr, dgid_addr;
567 int err;
568 u8 dmac[ETHER_ADDR_LEN];
569 bool sleep = udata ? true : false;
570
571 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
572 return ERR_PTR(-EINVAL);
573
574 err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
575 rf->max_ah, &ah_id, &rf->next_ah);
576
577 if (err)
578 return ERR_PTR(err);
579
580 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
581 if (!ah) {
582 irdma_free_rsrc(rf, rf->allocated_ahs, ah_id);
583 return ERR_PTR(-ENOMEM);
584 }
585
586 ah->pd = pd;
587 sc_ah = &ah->sc_ah;
588 sc_ah->ah_info.ah_idx = ah_id;
589 sc_ah->ah_info.vsi = &iwdev->vsi;
590 irdma_sc_init_ah(&rf->sc_dev, sc_ah);
591 ah->sgid_index = attr->grh.sgid_index;
592 memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid));
593 rcu_read_lock();
594 err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num,
595 attr->grh.sgid_index, &sgid, &sgid_attr);
596 rcu_read_unlock();
597 if (err) {
598 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
599 "GID lookup at idx=%d with port=%d failed\n",
600 attr->grh.sgid_index, attr->port_num);
601 err = -EINVAL;
602 goto err_gid_l2;
603 }
604 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid);
605 rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
606 ah->av.attrs = *attr;
607 ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
608
609 if (sgid_attr.ndev)
610 dev_put(sgid_attr.ndev);
611
612 ah_info = &sc_ah->ah_info;
613 ah_info->ah_idx = ah_id;
614 ah_info->pd_idx = pd->sc_pd.pd_id;
615
616 ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev));
617 if (attr->ah_flags & IB_AH_GRH) {
618 ah_info->flow_label = attr->grh.flow_label;
619 ah_info->hop_ttl = attr->grh.hop_limit;
620 ah_info->tc_tos = attr->grh.traffic_class;
621 }
622
623 if (udata)
624 ib_resolve_eth_dmac(ibpd->device, attr);
625 irdma_ether_copy(dmac, attr);
626
627 irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr,
628 dmac, ah->av.net_type);
629
630 err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac);
631 if (err)
632 goto err_gid_l2;
633
634 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
635 sleep, NULL, sc_ah);
636 if (err) {
637 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail");
638 goto err_gid_l2;
639 }
640
641 err = irdma_create_ah_wait(rf, sc_ah, sleep);
642 if (err)
643 goto err_gid_l2;
644
645 if (udata) {
646 uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
647 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
648 if (err) {
649 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
650 IRDMA_OP_AH_DESTROY, false, NULL, ah);
651 goto err_gid_l2;
652 }
653 }
654
655 return &ah->ibah;
656 err_gid_l2:
657 kfree(ah);
658 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
659
660 return ERR_PTR(err);
661 }
662
663 /**
664 * irdma_free_qp_rsrc - free up memory resources for qp
665 * @iwqp: qp ptr (user or kernel)
666 */
667 void
irdma_free_qp_rsrc(struct irdma_qp * iwqp)668 irdma_free_qp_rsrc(struct irdma_qp *iwqp)
669 {
670 struct irdma_device *iwdev = iwqp->iwdev;
671 struct irdma_pci_f *rf = iwdev->rf;
672 u32 qp_num = iwqp->ibqp.qp_num;
673
674 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
675 irdma_dealloc_push_page(rf, iwqp);
676 if (iwqp->sc_qp.vsi) {
677 irdma_qp_rem_qos(&iwqp->sc_qp);
678 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
679 iwqp->sc_qp.user_pri);
680 }
681
682 if (qp_num > 2)
683 irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
684 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem);
685 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem);
686 kfree(iwqp->kqp.sig_trk_mem);
687 iwqp->kqp.sig_trk_mem = NULL;
688 kfree(iwqp->kqp.sq_wrid_mem);
689 kfree(iwqp->kqp.rq_wrid_mem);
690 kfree(iwqp->sg_list);
691 kfree(iwqp);
692 }
693
694 /**
695 * irdma_create_qp - create qp
696 * @ibpd: ptr of pd
697 * @init_attr: attributes for qp
698 * @udata: user data for create qp
699 */
700 struct ib_qp *
irdma_create_qp(struct ib_pd * ibpd,struct ib_qp_init_attr * init_attr,struct ib_udata * udata)701 irdma_create_qp(struct ib_pd *ibpd,
702 struct ib_qp_init_attr *init_attr,
703 struct ib_udata *udata)
704 {
705 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx)
706 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd)
707 struct irdma_pd *iwpd = to_iwpd(ibpd);
708 struct irdma_device *iwdev = to_iwdev(ibpd->device);
709 struct irdma_pci_f *rf = iwdev->rf;
710 struct irdma_qp *iwqp;
711 struct irdma_create_qp_resp uresp = {0};
712 u32 qp_num = 0;
713 int ret;
714 int err_code;
715 struct irdma_sc_qp *qp;
716 struct irdma_sc_dev *dev = &rf->sc_dev;
717 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs;
718 struct irdma_qp_init_info init_info = {{0}};
719 struct irdma_qp_host_ctx_info *ctx_info;
720 unsigned long flags;
721
722 err_code = irdma_validate_qp_attrs(init_attr, iwdev);
723 if (err_code)
724 return ERR_PTR(err_code);
725
726 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN ||
727 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN))
728 return ERR_PTR(-EINVAL);
729
730 init_info.vsi = &iwdev->vsi;
731 init_info.qp_uk_init_info.uk_attrs = uk_attrs;
732 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr;
733 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr;
734 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
735 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
736 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
737
738 iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL);
739 if (!iwqp)
740 return ERR_PTR(-ENOMEM);
741
742 iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list),
743 GFP_KERNEL);
744 if (!iwqp->sg_list) {
745 kfree(iwqp);
746 return ERR_PTR(-ENOMEM);
747 }
748
749 qp = &iwqp->sc_qp;
750 qp->qp_uk.back_qp = iwqp;
751 qp->qp_uk.lock = &iwqp->lock;
752 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
753
754 iwqp->iwdev = iwdev;
755 iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE;
756 iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem,
757 iwqp->q2_ctx_mem.size,
758 256);
759 if (!iwqp->q2_ctx_mem.va) {
760 kfree(iwqp->sg_list);
761 kfree(iwqp);
762 return ERR_PTR(-ENOMEM);
763 }
764
765 init_info.q2 = iwqp->q2_ctx_mem.va;
766 init_info.q2_pa = iwqp->q2_ctx_mem.pa;
767 init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE);
768 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE;
769
770 if (init_attr->qp_type == IB_QPT_GSI)
771 qp_num = 1;
772 else
773 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp,
774 &qp_num, &rf->next_qp);
775 if (err_code)
776 goto error;
777
778 iwqp->iwpd = iwpd;
779 iwqp->ibqp.qp_num = qp_num;
780 qp = &iwqp->sc_qp;
781 iwqp->iwscq = to_iwcq(init_attr->send_cq);
782 iwqp->iwrcq = to_iwcq(init_attr->recv_cq);
783 iwqp->host_ctx.va = init_info.host_ctx;
784 iwqp->host_ctx.pa = init_info.host_ctx_pa;
785 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE;
786
787 init_info.pd = &iwpd->sc_pd;
788 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num;
789 if (!rdma_protocol_roce(&iwdev->ibdev, 1))
790 init_info.qp_uk_init_info.first_sq_wq = 1;
791 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
792 init_waitqueue_head(&iwqp->waitq);
793 init_waitqueue_head(&iwqp->mod_qp_waitq);
794
795 spin_lock_init(&iwqp->dwork_flush_lock);
796
797 if (udata) {
798 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver;
799 err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr);
800 } else {
801 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker);
802 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER;
803 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr);
804 }
805
806 if (err_code) {
807 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "setup qp failed\n");
808 goto error;
809 }
810
811 if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
812 if (init_attr->qp_type == IB_QPT_RC) {
813 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC;
814 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
815 IRDMA_WRITE_WITH_IMM |
816 IRDMA_ROCE;
817 } else {
818 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD;
819 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
820 IRDMA_ROCE;
821 }
822 } else {
823 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP;
824 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM;
825 }
826
827 ret = irdma_sc_qp_init(qp, &init_info);
828 if (ret) {
829 err_code = -EPROTO;
830 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_init fail\n");
831 goto error;
832 }
833
834 ctx_info = &iwqp->ctx_info;
835 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
836 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
837
838 if (rdma_protocol_roce(&iwdev->ibdev, 1))
839 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info);
840 else
841 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info);
842
843 err_code = irdma_cqp_create_qp_cmd(iwqp);
844 if (err_code)
845 goto error;
846
847 atomic_set(&iwqp->refcnt, 1);
848 spin_lock_init(&iwqp->lock);
849 spin_lock_init(&iwqp->sc_qp.pfpdu.lock);
850 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
851 rf->qp_table[qp_num] = iwqp;
852
853 if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
854 if (dev->ws_add(&iwdev->vsi, 0)) {
855 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp);
856 err_code = -EINVAL;
857 goto error;
858 }
859
860 irdma_qp_add_qos(&iwqp->sc_qp);
861 spin_lock_irqsave(&iwpd->udqp_list_lock, flags);
862 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD)
863 list_add_tail(&iwqp->ud_list_elem, &iwpd->udqp_list);
864 spin_unlock_irqrestore(&iwpd->udqp_list_lock, flags);
865 }
866
867 if (udata) {
868 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
869 if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) {
870 uresp.lsmm = 1;
871 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;
872 } else {
873 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) {
874 uresp.lsmm = 1;
875 if (qp->qp_uk.start_wqe_idx) {
876 uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
877 uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx;
878 }
879 }
880 }
881 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
882 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
883 uresp.qp_id = qp_num;
884 uresp.qp_caps = qp->qp_uk.qp_caps;
885
886 err_code = ib_copy_to_udata(udata, &uresp,
887 min(sizeof(uresp), udata->outlen));
888 if (err_code) {
889 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n");
890 irdma_destroy_qp(&iwqp->ibqp);
891 return ERR_PTR(err_code);
892 }
893 }
894
895 init_completion(&iwqp->free_qp);
896 return &iwqp->ibqp;
897
898 error:
899 irdma_free_qp_rsrc(iwqp);
900
901 return ERR_PTR(err_code);
902 }
903
904 /**
905 * irdma_destroy_qp - destroy qp
906 * @ibqp: qp's ib pointer also to get to device's qp address
907 * @udata: user data
908 */
909 int
irdma_destroy_qp(struct ib_qp * ibqp)910 irdma_destroy_qp(struct ib_qp *ibqp)
911 {
912 struct irdma_qp *iwqp = to_iwqp(ibqp);
913 struct irdma_device *iwdev = iwqp->iwdev;
914 unsigned long flags;
915
916 if (iwqp->sc_qp.qp_uk.destroy_pending)
917 goto free_rsrc;
918 iwqp->sc_qp.qp_uk.destroy_pending = true;
919
920 spin_lock_irqsave(&iwqp->iwpd->udqp_list_lock, flags);
921 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD)
922 list_del(&iwqp->ud_list_elem);
923 spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags);
924
925 if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE)
926 irdma_modify_qp_to_err(&iwqp->sc_qp);
927
928 if (!iwqp->user_mode) {
929 if (iwqp->iwscq) {
930 irdma_clean_cqes(iwqp, iwqp->iwscq);
931 if (iwqp->iwrcq != iwqp->iwscq)
932 irdma_clean_cqes(iwqp, iwqp->iwrcq);
933 }
934 }
935 irdma_qp_rem_ref(&iwqp->ibqp);
936 wait_for_completion(&iwqp->free_qp);
937 irdma_free_lsmm_rsrc(iwqp);
938 if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp))
939 return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE;
940 free_rsrc:
941 irdma_remove_push_mmap_entries(iwqp);
942 irdma_free_qp_rsrc(iwqp);
943
944 return 0;
945 }
946
947 /**
948 * irdma_create_cq - create cq
949 * @ibcq: CQ allocated
950 * @attr: attributes for cq
951 * @udata: user data
952 */
953 struct ib_cq *
irdma_create_cq(struct ib_device * ibdev,const struct ib_cq_init_attr * attr,struct ib_ucontext * context,struct ib_udata * udata)954 irdma_create_cq(struct ib_device *ibdev,
955 const struct ib_cq_init_attr *attr,
956 struct ib_ucontext *context,
957 struct ib_udata *udata)
958 {
959 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)
960 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)
961 struct irdma_device *iwdev = to_iwdev(ibdev);
962 struct irdma_pci_f *rf = iwdev->rf;
963 struct irdma_cq *iwcq;
964 u32 cq_num = 0;
965 struct irdma_sc_cq *cq;
966 struct irdma_sc_dev *dev = &rf->sc_dev;
967 struct irdma_cq_init_info info = {0};
968 int status;
969 struct irdma_cqp_request *cqp_request;
970 struct cqp_cmds_info *cqp_info;
971 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
972 unsigned long flags;
973 int err_code;
974 int entries = attr->cqe;
975 bool cqe_64byte_ena;
976
977 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
978 if (err_code)
979 return ERR_PTR(err_code);
980
981 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
982 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
983 return ERR_PTR(-EINVAL);
984
985 iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL);
986 if (!iwcq)
987 return ERR_PTR(-ENOMEM);
988 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num,
989 &rf->next_cq);
990 if (err_code)
991 goto error;
992 cq = &iwcq->sc_cq;
993 cq->back_cq = iwcq;
994 atomic_set(&iwcq->refcnt, 1);
995 spin_lock_init(&iwcq->lock);
996 INIT_LIST_HEAD(&iwcq->resize_list);
997 INIT_LIST_HEAD(&iwcq->cmpl_generated);
998 info.dev = dev;
999 ukinfo->cq_size = max(entries, 4);
1000 ukinfo->cq_id = cq_num;
1001 cqe_64byte_ena = (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE) ? true : false;
1002 ukinfo->avoid_mem_cflct = cqe_64byte_ena;
1003 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
1004 atomic_set(&iwcq->armed, 0);
1005 if (attr->comp_vector < rf->ceqs_count)
1006 info.ceq_id = attr->comp_vector;
1007 info.ceq_id_valid = true;
1008 info.ceqe_mask = 1;
1009 info.type = IRDMA_CQ_TYPE_IWARP;
1010 info.vsi = &iwdev->vsi;
1011
1012 if (udata) {
1013 struct irdma_ucontext *ucontext;
1014 struct irdma_create_cq_req req = {0};
1015 struct irdma_cq_mr *cqmr;
1016 struct irdma_pbl *iwpbl;
1017 struct irdma_pbl *iwpbl_shadow;
1018 struct irdma_cq_mr *cqmr_shadow;
1019
1020 iwcq->user_mode = true;
1021 ucontext = to_ucontext(context);
1022
1023 if (ib_copy_from_udata(&req, udata,
1024 min(sizeof(req), udata->inlen))) {
1025 err_code = -EFAULT;
1026 goto cq_free_rsrc;
1027 }
1028
1029 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1030 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
1031 &ucontext->cq_reg_mem_list);
1032 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1033 if (!iwpbl) {
1034 err_code = -EPROTO;
1035 goto cq_free_rsrc;
1036 }
1037 iwcq->iwpbl = iwpbl;
1038 iwcq->cq_mem_size = 0;
1039 cqmr = &iwpbl->cq_mr;
1040
1041 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
1042 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) {
1043 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1044 iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area,
1045 &ucontext->cq_reg_mem_list);
1046 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1047
1048 if (!iwpbl_shadow) {
1049 err_code = -EPROTO;
1050 goto cq_free_rsrc;
1051 }
1052 iwcq->iwpbl_shadow = iwpbl_shadow;
1053 cqmr_shadow = &iwpbl_shadow->cq_mr;
1054 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr;
1055 cqmr->split = true;
1056 } else {
1057 info.shadow_area_pa = cqmr->shadow;
1058 }
1059 if (iwpbl->pbl_allocated) {
1060 info.virtual_map = true;
1061 info.pbl_chunk_size = 1;
1062 info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
1063 } else {
1064 info.cq_base_pa = cqmr->cq_pbl.addr;
1065 }
1066 } else {
1067 /* Kmode allocations */
1068 int rsize;
1069
1070 if (entries < 1 || entries > rf->max_cqe) {
1071 err_code = -EINVAL;
1072 goto cq_free_rsrc;
1073 }
1074
1075 entries++;
1076 if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1077 entries *= 2;
1078 ukinfo->cq_size = entries;
1079
1080 if (cqe_64byte_ena)
1081 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe);
1082 else
1083 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
1084 iwcq->kmem.size = round_up(rsize, IRDMA_HW_PAGE_SIZE);
1085 iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem,
1086 iwcq->kmem.size, IRDMA_HW_PAGE_SIZE);
1087 if (!iwcq->kmem.va) {
1088 err_code = -ENOMEM;
1089 goto cq_free_rsrc;
1090 }
1091
1092 iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3;
1093 iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw,
1094 &iwcq->kmem_shadow,
1095 iwcq->kmem_shadow.size,
1096 64);
1097
1098 if (!iwcq->kmem_shadow.va) {
1099 err_code = -ENOMEM;
1100 goto cq_kmem_free;
1101 }
1102 info.shadow_area_pa = iwcq->kmem_shadow.pa;
1103 ukinfo->shadow_area = iwcq->kmem_shadow.va;
1104 ukinfo->cq_base = iwcq->kmem.va;
1105 info.cq_base_pa = iwcq->kmem.pa;
1106 }
1107
1108 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,
1109 (u32)IRDMA_MAX_CQ_READ_THRESH);
1110 if (irdma_sc_cq_init(cq, &info)) {
1111 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n");
1112 err_code = -EPROTO;
1113 goto cq_kmem_free;
1114 }
1115
1116 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1117 if (!cqp_request) {
1118 err_code = -ENOMEM;
1119 goto cq_kmem_free;
1120 }
1121 cqp_info = &cqp_request->info;
1122 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1123 cqp_info->post_sq = 1;
1124 cqp_info->in.u.cq_create.cq = cq;
1125 cqp_info->in.u.cq_create.check_overflow = true;
1126 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1127 status = irdma_handle_cqp_op(rf, cqp_request);
1128 irdma_put_cqp_request(&rf->cqp, cqp_request);
1129 if (status) {
1130 err_code = -ENOMEM;
1131 goto cq_kmem_free;
1132 }
1133
1134 if (udata) {
1135 struct irdma_create_cq_resp resp = {0};
1136
1137 resp.cq_id = info.cq_uk_init_info.cq_id;
1138 resp.cq_size = info.cq_uk_init_info.cq_size;
1139 if (ib_copy_to_udata(udata, &resp,
1140 min(sizeof(resp), udata->outlen))) {
1141 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy to user data\n");
1142 err_code = -EPROTO;
1143 goto cq_destroy;
1144 }
1145 }
1146
1147 rf->cq_table[cq_num] = iwcq;
1148 init_completion(&iwcq->free_cq);
1149
1150 return &iwcq->ibcq;
1151 cq_destroy:
1152 irdma_cq_wq_destroy(rf, cq);
1153 cq_kmem_free:
1154 if (!iwcq->user_mode) {
1155 irdma_free_dma_mem(dev->hw, &iwcq->kmem);
1156 irdma_free_dma_mem(dev->hw, &iwcq->kmem_shadow);
1157 }
1158 cq_free_rsrc:
1159 irdma_free_rsrc(rf, rf->allocated_cqs, cq_num);
1160 error:
1161 kfree(iwcq);
1162 return ERR_PTR(err_code);
1163 }
1164
1165 /**
1166 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally
1167 * @iwmr: iwmr for IB's user page addresses
1168 * @pbl: ple pointer to save 1 level or 0 level pble
1169 * @level: indicated level 0, 1 or 2
1170 */
1171
1172 void
irdma_copy_user_pgaddrs(struct irdma_mr * iwmr,u64 * pbl,enum irdma_pble_level level)1173 irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
1174 enum irdma_pble_level level)
1175 {
1176 struct ib_umem *region = iwmr->region;
1177 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1178 int chunk_pages, entry, i;
1179 struct scatterlist *sg;
1180 u64 pg_addr = 0;
1181 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
1182 struct irdma_pble_info *pinfo;
1183 u32 idx = 0;
1184 u32 pbl_cnt = 0;
1185
1186 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf;
1187 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
1188 chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size);
1189 if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page)
1190 iwpbl->qp_mr.sq_page = sg_page(sg);
1191 for (i = 0; i < chunk_pages; i++) {
1192 pg_addr = sg_dma_address(sg) + (i * iwmr->page_size);
1193 if ((entry + i) == 0)
1194 *pbl = pg_addr & iwmr->page_msk;
1195 else if (!(pg_addr & ~iwmr->page_msk))
1196 *pbl = pg_addr;
1197 else
1198 continue;
1199 if (++pbl_cnt == palloc->total_cnt)
1200 break;
1201 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx);
1202 }
1203 }
1204 }
1205
1206 /**
1207 * irdma_destroy_ah - Destroy address handle
1208 * @ibah: pointer to address handle
1209 * @ah_flags: destroy flags
1210 */
1211 int
irdma_destroy_ah(struct ib_ah * ibah)1212 irdma_destroy_ah(struct ib_ah *ibah)
1213 {
1214 struct irdma_device *iwdev = to_iwdev(ibah->device);
1215 struct irdma_ah *ah = to_iwah(ibah);
1216
1217 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,
1218 false, NULL, ah);
1219
1220 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
1221 ah->sc_ah.ah_info.ah_idx);
1222
1223 kfree(ah);
1224 return 0;
1225 }
1226
1227 int
irdma_dereg_mr(struct ib_mr * ib_mr)1228 irdma_dereg_mr(struct ib_mr *ib_mr)
1229 {
1230 struct irdma_mr *iwmr = to_iwmr(ib_mr);
1231 struct irdma_device *iwdev = to_iwdev(ib_mr->device);
1232 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1233 int ret;
1234
1235 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
1236 if (iwmr->region) {
1237 struct irdma_ucontext *ucontext;
1238 struct ib_pd *ibpd = ib_mr->pd;
1239
1240 ucontext = to_ucontext(ibpd->uobject->context);
1241 irdma_del_memlist(iwmr, ucontext);
1242 }
1243 goto done;
1244 }
1245
1246 ret = irdma_hwdereg_mr(ib_mr);
1247 if (ret)
1248 return ret;
1249
1250 irdma_free_stag(iwdev, iwmr->stag);
1251 done:
1252 if (iwpbl->pbl_allocated)
1253 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
1254
1255 if (iwmr->region)
1256 ib_umem_release(iwmr->region);
1257
1258 kfree(iwmr);
1259
1260 return 0;
1261 }
1262
1263 /*
1264 * irdma_rereg_user_mr - Re-Register a user memory region @ibmr: ib mem to access iwarp mr pointer @flags: bit mask to
1265 * indicate which of the attr's of MR modified @start: virtual start address @len: length of mr @virt: virtual address
1266 * @new access flags: bit mask of access flags @new_pd: ptr of pd @udata: user data
1267 */
1268 int
irdma_rereg_user_mr(struct ib_mr * ib_mr,int flags,u64 start,u64 len,u64 virt,int new_access,struct ib_pd * new_pd,struct ib_udata * udata)1269 irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len,
1270 u64 virt, int new_access, struct ib_pd *new_pd,
1271 struct ib_udata *udata)
1272 {
1273 struct irdma_device *iwdev = to_iwdev(ib_mr->device);
1274 struct irdma_mr *iwmr = to_iwmr(ib_mr);
1275 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1276 int ret;
1277
1278 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
1279 return -EINVAL;
1280
1281 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
1282 return -EOPNOTSUPP;
1283
1284 ret = irdma_hwdereg_mr(ib_mr);
1285 if (ret)
1286 return ret;
1287
1288 if (flags & IB_MR_REREG_ACCESS)
1289 iwmr->access = new_access;
1290
1291 if (flags & IB_MR_REREG_PD) {
1292 iwmr->ibmr.pd = new_pd;
1293 iwmr->ibmr.device = new_pd->device;
1294 }
1295
1296 if (flags & IB_MR_REREG_TRANS) {
1297 if (iwpbl->pbl_allocated) {
1298 irdma_free_pble(iwdev->rf->pble_rsrc,
1299 &iwpbl->pble_alloc);
1300 iwpbl->pbl_allocated = false;
1301 }
1302 if (iwmr->region) {
1303 ib_umem_release(iwmr->region);
1304 iwmr->region = NULL;
1305 }
1306
1307 ib_mr = irdma_rereg_mr_trans(iwmr, start, len, virt, udata);
1308 if (IS_ERR(ib_mr))
1309 return PTR_ERR(ib_mr);
1310
1311 } else {
1312 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
1313 if (ret)
1314 return ret;
1315 }
1316
1317 return 0;
1318 }
1319
1320 int
kc_irdma_set_roce_cm_info(struct irdma_qp * iwqp,struct ib_qp_attr * attr,u16 * vlan_id)1321 kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr,
1322 u16 *vlan_id)
1323 {
1324 int ret;
1325 union ib_gid sgid;
1326 struct ib_gid_attr sgid_attr;
1327 struct irdma_av *av = &iwqp->roce_ah.av;
1328
1329 ret = ib_get_cached_gid(iwqp->ibqp.device, attr->ah_attr.port_num,
1330 attr->ah_attr.grh.sgid_index, &sgid,
1331 &sgid_attr);
1332 if (ret)
1333 return ret;
1334
1335 if (sgid_attr.ndev) {
1336 *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev);
1337 ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, if_getlladdr(sgid_attr.ndev));
1338 }
1339
1340 av->net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
1341 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid);
1342 dev_put(sgid_attr.ndev);
1343 iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri;
1344
1345 return 0;
1346 }
1347
1348 /**
1349 * irdma_destroy_cq - destroy cq
1350 * @ib_cq: cq pointer
1351 */
1352 int
irdma_destroy_cq(struct ib_cq * ib_cq)1353 irdma_destroy_cq(struct ib_cq *ib_cq)
1354 {
1355 struct irdma_device *iwdev = to_iwdev(ib_cq->device);
1356 struct irdma_cq *iwcq = to_iwcq(ib_cq);
1357 struct irdma_sc_cq *cq = &iwcq->sc_cq;
1358 struct irdma_sc_dev *dev = cq->dev;
1359 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id];
1360 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq);
1361 unsigned long flags;
1362
1363 spin_lock_irqsave(&iwcq->lock, flags);
1364 if (!list_empty(&iwcq->cmpl_generated))
1365 irdma_remove_cmpls_list(iwcq);
1366 if (!list_empty(&iwcq->resize_list))
1367 irdma_process_resize_list(iwcq, iwdev, NULL);
1368 spin_unlock_irqrestore(&iwcq->lock, flags);
1369
1370 irdma_cq_rem_ref(ib_cq);
1371 wait_for_completion(&iwcq->free_cq);
1372
1373 irdma_cq_wq_destroy(iwdev->rf, cq);
1374
1375 spin_lock_irqsave(&iwceq->ce_lock, flags);
1376 irdma_sc_cleanup_ceqes(cq, ceq);
1377 spin_unlock_irqrestore(&iwceq->ce_lock, flags);
1378
1379 irdma_cq_free_rsrc(iwdev->rf, iwcq);
1380 kfree(iwcq);
1381
1382 return 0;
1383 }
1384
1385 /**
1386 * kc_set_loc_seq_num_mss - Set local seq number and mss
1387 * @cm_node: cm node info
1388 */
1389 void
kc_set_loc_seq_num_mss(struct irdma_cm_node * cm_node)1390 kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node)
1391 {
1392 struct timespec ts;
1393
1394 getnanotime(&ts);
1395 cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec;
1396 if (cm_node->iwdev->vsi.mtu > 1500 &&
1397 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd)
1398 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ?
1399 (1500 - IRDMA_MTU_TO_MSS_IPV4) :
1400 (1500 - IRDMA_MTU_TO_MSS_IPV6);
1401 else
1402 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ?
1403 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) :
1404 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6);
1405 }
1406
1407 struct irdma_vma_data {
1408 struct list_head list;
1409 struct vm_area_struct *vma;
1410 struct mutex *vma_list_mutex; /* protect the vma_list */
1411 };
1412
1413 /**
1414 * irdma_vma_open -
1415 * @vma: User VMA
1416 */
1417 static void
irdma_vma_open(struct vm_area_struct * vma)1418 irdma_vma_open(struct vm_area_struct *vma)
1419 {
1420 vma->vm_ops = NULL;
1421 }
1422
1423 /**
1424 * irdma_vma_close - Remove vma data from vma list
1425 * @vma: User VMA
1426 */
1427 static void
irdma_vma_close(struct vm_area_struct * vma)1428 irdma_vma_close(struct vm_area_struct *vma)
1429 {
1430 struct irdma_vma_data *vma_data;
1431
1432 vma_data = vma->vm_private_data;
1433 vma->vm_private_data = NULL;
1434 vma_data->vma = NULL;
1435 mutex_lock(vma_data->vma_list_mutex);
1436 list_del(&vma_data->list);
1437 mutex_unlock(vma_data->vma_list_mutex);
1438 kfree(vma_data);
1439 }
1440
1441 static const struct vm_operations_struct irdma_vm_ops = {
1442 .open = irdma_vma_open,
1443 .close = irdma_vma_close
1444 };
1445
1446 /**
1447 * irdma_set_vma_data - Save vma data in context list
1448 * @vma: User VMA
1449 * @context: ib user context
1450 */
1451 static int
irdma_set_vma_data(struct vm_area_struct * vma,struct irdma_ucontext * context)1452 irdma_set_vma_data(struct vm_area_struct *vma,
1453 struct irdma_ucontext *context)
1454 {
1455 struct list_head *vma_head = &context->vma_list;
1456 struct irdma_vma_data *vma_entry;
1457
1458 vma_entry = kzalloc(sizeof(*vma_entry), GFP_KERNEL);
1459 if (!vma_entry)
1460 return -ENOMEM;
1461
1462 vma->vm_private_data = vma_entry;
1463 vma->vm_ops = &irdma_vm_ops;
1464
1465 vma_entry->vma = vma;
1466 vma_entry->vma_list_mutex = &context->vma_list_mutex;
1467
1468 mutex_lock(&context->vma_list_mutex);
1469 list_add(&vma_entry->list, vma_head);
1470 mutex_unlock(&context->vma_list_mutex);
1471
1472 return 0;
1473 }
1474
1475 /**
1476 * irdma_disassociate_ucontext - Disassociate user context
1477 * @context: ib user context
1478 */
1479 void
irdma_disassociate_ucontext(struct ib_ucontext * context)1480 irdma_disassociate_ucontext(struct ib_ucontext *context)
1481 {
1482 struct irdma_ucontext *ucontext = to_ucontext(context);
1483
1484 struct irdma_vma_data *vma_data, *n;
1485 struct vm_area_struct *vma;
1486
1487 mutex_lock(&ucontext->vma_list_mutex);
1488 list_for_each_entry_safe(vma_data, n, &ucontext->vma_list, list) {
1489 vma = vma_data->vma;
1490 zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE);
1491
1492 vma->vm_ops = NULL;
1493 list_del(&vma_data->list);
1494 kfree(vma_data);
1495 }
1496 mutex_unlock(&ucontext->vma_list_mutex);
1497 }
1498
1499 int
rdma_user_mmap_io(struct ib_ucontext * context,struct vm_area_struct * vma,unsigned long pfn,unsigned long size,pgprot_t prot)1500 rdma_user_mmap_io(struct ib_ucontext *context, struct vm_area_struct *vma,
1501 unsigned long pfn, unsigned long size, pgprot_t prot)
1502 {
1503 if (io_remap_pfn_range(vma,
1504 vma->vm_start,
1505 pfn,
1506 size,
1507 prot))
1508 return -EAGAIN;
1509
1510 return irdma_set_vma_data(vma, to_ucontext(context));
1511 }
1512
1513 struct ib_device *
ib_device_get_by_netdev(if_t netdev,int driver_id)1514 ib_device_get_by_netdev(if_t netdev, int driver_id)
1515 {
1516 struct irdma_device *iwdev;
1517 struct irdma_handler *hdl;
1518 unsigned long flags;
1519
1520 spin_lock_irqsave(&irdma_handler_lock, flags);
1521 list_for_each_entry(hdl, &irdma_handlers, list) {
1522 iwdev = hdl->iwdev;
1523 if (netdev == iwdev->netdev) {
1524 spin_unlock_irqrestore(&irdma_handler_lock,
1525 flags);
1526 return &iwdev->ibdev;
1527 }
1528 }
1529 spin_unlock_irqrestore(&irdma_handler_lock, flags);
1530
1531 return NULL;
1532 }
1533
1534 void
ib_unregister_device_put(struct ib_device * device)1535 ib_unregister_device_put(struct ib_device *device)
1536 {
1537 ib_unregister_device(device);
1538 }
1539
1540 /**
1541 * irdma_query_gid_roce - Query port GID for Roce
1542 * @ibdev: device pointer from stack
1543 * @port: port number
1544 * @index: Entry index
1545 * @gid: Global ID
1546 */
1547 int
irdma_query_gid_roce(struct ib_device * ibdev,u8 port,int index,union ib_gid * gid)1548 irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index,
1549 union ib_gid *gid)
1550 {
1551 int ret;
1552
1553 ret = ib_get_cached_gid(ibdev, port, index, gid, NULL);
1554 if (ret == -EAGAIN) {
1555 memcpy(gid, &zgid, sizeof(*gid));
1556 return 0;
1557 }
1558
1559 return ret;
1560 }
1561
1562 /**
1563 * irdma_modify_port - modify port attributes
1564 * @ibdev: device pointer from stack
1565 * @port: port number for query
1566 * @mask: Property mask
1567 * @props: returning device attributes
1568 */
1569 int
irdma_modify_port(struct ib_device * ibdev,u8 port,int mask,struct ib_port_modify * props)1570 irdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
1571 struct ib_port_modify *props)
1572 {
1573 if (port > 1)
1574 return -EINVAL;
1575
1576 return 0;
1577 }
1578
1579 /**
1580 * irdma_query_pkey - Query partition key
1581 * @ibdev: device pointer from stack
1582 * @port: port number
1583 * @index: index of pkey
1584 * @pkey: pointer to store the pkey
1585 */
1586 int
irdma_query_pkey(struct ib_device * ibdev,u8 port,u16 index,u16 * pkey)1587 irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1588 u16 *pkey)
1589 {
1590 if (index >= IRDMA_PKEY_TBL_SZ)
1591 return -EINVAL;
1592
1593 *pkey = IRDMA_DEFAULT_PKEY;
1594 return 0;
1595 }
1596
1597 int
irdma_roce_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)1598 irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
1599 struct ib_port_immutable *immutable)
1600 {
1601 struct ib_port_attr attr;
1602 int err;
1603
1604 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
1605 err = ib_query_port(ibdev, port_num, &attr);
1606 if (err)
1607 return err;
1608
1609 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
1610 immutable->pkey_tbl_len = attr.pkey_tbl_len;
1611 immutable->gid_tbl_len = attr.gid_tbl_len;
1612
1613 return 0;
1614 }
1615
1616 int
irdma_iw_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)1617 irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num,
1618 struct ib_port_immutable *immutable)
1619 {
1620 struct ib_port_attr attr;
1621 int err;
1622
1623 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1624 err = ib_query_port(ibdev, port_num, &attr);
1625 if (err)
1626 return err;
1627 immutable->gid_tbl_len = 1;
1628
1629 return 0;
1630 }
1631
1632 /**
1633 * irdma_query_port - get port attributes
1634 * @ibdev: device pointer from stack
1635 * @port: port number for query
1636 * @props: returning device attributes
1637 */
1638 int
irdma_query_port(struct ib_device * ibdev,u8 port,struct ib_port_attr * props)1639 irdma_query_port(struct ib_device *ibdev, u8 port,
1640 struct ib_port_attr *props)
1641 {
1642 struct irdma_device *iwdev = to_iwdev(ibdev);
1643 if_t netdev = iwdev->netdev;
1644
1645 /* no need to zero out pros here. done by caller */
1646
1647 props->max_mtu = IB_MTU_4096;
1648 props->active_mtu = ib_mtu_int_to_enum(if_getmtu(netdev));
1649 props->lid = 1;
1650 props->lmc = 0;
1651 props->sm_lid = 0;
1652 props->sm_sl = 0;
1653 if ((if_getlinkstate(netdev) == LINK_STATE_UP) && (if_getdrvflags(netdev) & IFF_DRV_RUNNING)) {
1654 props->state = IB_PORT_ACTIVE;
1655 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
1656 } else {
1657 props->state = IB_PORT_DOWN;
1658 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
1659 }
1660 ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width);
1661
1662 if (rdma_protocol_roce(ibdev, 1)) {
1663 props->gid_tbl_len = 32;
1664 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
1665 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ;
1666 } else {
1667 props->gid_tbl_len = 1;
1668 }
1669 props->qkey_viol_cntr = 0;
1670 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP;
1671 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size;
1672
1673 return 0;
1674 }
1675
1676 static const char *const irdma_hw_stat_names[] = {
1677 /* gen1 - 32-bit */
1678 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
1679 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
1680 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
1681 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
1682 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
1683 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
1684 [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors",
1685 /* gen1 - 64-bit */
1686 [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets",
1687 [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts",
1688 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd",
1689 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts",
1690 [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets",
1691 [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts",
1692 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd",
1693 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts",
1694 [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets",
1695 [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts",
1696 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd",
1697 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts",
1698 [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets",
1699 [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts",
1700 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd",
1701 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts",
1702 [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "InRdmaReads",
1703 [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "InRdmaSends",
1704 [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "InRdmaWrites",
1705 [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "OutRdmaReads",
1706 [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "OutRdmaSends",
1707 [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "OutRdmaWrites",
1708 [IRDMA_HW_STAT_INDEX_RDMAVBND] = "RdmaBnd",
1709 [IRDMA_HW_STAT_INDEX_RDMAVINV] = "RdmaInv",
1710
1711 /* gen2 - 32-bit */
1712 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled",
1713 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored",
1714 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent",
1715 /* gen2 - 64-bit */
1716 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets",
1717 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets",
1718 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets",
1719 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets",
1720 [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP",
1721 [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP",
1722 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd",
1723 [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "RetransSegs",
1724 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "InOptErrors",
1725 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "InProtoErrors",
1726 [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "InSegs",
1727 [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "OutSegs",
1728 };
1729
1730 /**
1731 * irdma_alloc_hw_stats - Allocate a hw stats structure
1732 * @ibdev: device pointer from stack
1733 * @port_num: port number
1734 */
1735 struct rdma_hw_stats *
irdma_alloc_hw_stats(struct ib_device * ibdev,u8 port_num)1736 irdma_alloc_hw_stats(struct ib_device *ibdev,
1737 u8 port_num)
1738 {
1739 struct irdma_device *iwdev = to_iwdev(ibdev);
1740 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
1741
1742 int num_counters = dev->hw_attrs.max_stat_idx;
1743 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
1744
1745 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters,
1746 lifespan);
1747 }
1748
1749 /**
1750 * irdma_get_hw_stats - Populates the rdma_hw_stats structure
1751 * @ibdev: device pointer from stack
1752 * @stats: stats pointer from stack
1753 * @port_num: port number
1754 * @index: which hw counter the stack is requesting we update
1755 */
1756 int
irdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u8 port_num,int index)1757 irdma_get_hw_stats(struct ib_device *ibdev,
1758 struct rdma_hw_stats *stats, u8 port_num,
1759 int index)
1760 {
1761 struct irdma_device *iwdev = to_iwdev(ibdev);
1762 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats;
1763
1764 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)
1765 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true);
1766
1767 memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters);
1768
1769 return stats->num_counters;
1770 }
1771
1772 /**
1773 * irdma_query_gid - Query port GID
1774 * @ibdev: device pointer from stack
1775 * @port: port number
1776 * @index: Entry index
1777 * @gid: Global ID
1778 */
1779 int
irdma_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * gid)1780 irdma_query_gid(struct ib_device *ibdev, u8 port, int index,
1781 union ib_gid *gid)
1782 {
1783 struct irdma_device *iwdev = to_iwdev(ibdev);
1784
1785 memset(gid->raw, 0, sizeof(gid->raw));
1786 ether_addr_copy(gid->raw, if_getlladdr(iwdev->netdev));
1787
1788 return 0;
1789 }
1790
1791 enum rdma_link_layer
irdma_get_link_layer(struct ib_device * ibdev,u8 port_num)1792 irdma_get_link_layer(struct ib_device *ibdev,
1793 u8 port_num)
1794 {
1795 return IB_LINK_LAYER_ETHERNET;
1796 }
1797
1798 inline enum ib_mtu
ib_mtu_int_to_enum(int mtu)1799 ib_mtu_int_to_enum(int mtu)
1800 {
1801 if (mtu >= 4096)
1802 return IB_MTU_4096;
1803 else if (mtu >= 2048)
1804 return IB_MTU_2048;
1805 else if (mtu >= 1024)
1806 return IB_MTU_1024;
1807 else if (mtu >= 512)
1808 return IB_MTU_512;
1809 else
1810 return IB_MTU_256;
1811 }
1812
1813 inline void
kc_set_roce_uverbs_cmd_mask(struct irdma_device * iwdev)1814 kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev)
1815 {
1816 iwdev->ibdev.uverbs_cmd_mask |=
1817 BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) |
1818 BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) |
1819 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) |
1820 BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST);
1821 }
1822
1823 inline void
kc_set_rdma_uverbs_cmd_mask(struct irdma_device * iwdev)1824 kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev)
1825 {
1826 iwdev->ibdev.uverbs_cmd_mask =
1827 BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) |
1828 BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) |
1829 BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) |
1830 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) |
1831 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) |
1832 BIT_ULL(IB_USER_VERBS_CMD_REG_MR) |
1833 BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) |
1834 BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) |
1835 BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1836 BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) |
1837 BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) |
1838 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) |
1839 BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1840 BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) |
1841 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) |
1842 BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
1843 BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) |
1844 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
1845 BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) |
1846 BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);
1847 iwdev->ibdev.uverbs_ex_cmd_mask =
1848 BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) |
1849 BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE);
1850
1851 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)
1852 iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ);
1853 }
1854
1855 int
ib_get_eth_speed(struct ib_device * ibdev,u32 port_num,u8 * speed,u8 * width)1856 ib_get_eth_speed(struct ib_device *ibdev, u32 port_num, u8 *speed, u8 *width)
1857 {
1858 if_t netdev = ibdev->get_netdev(ibdev, port_num);
1859 u32 netdev_speed;
1860
1861 if (!netdev)
1862 return -ENODEV;
1863
1864 netdev_speed = netdev->if_baudrate;
1865 dev_put(netdev);
1866 if (netdev_speed <= SPEED_1000) {
1867 *width = IB_WIDTH_1X;
1868 *speed = IB_SPEED_SDR;
1869 } else if (netdev_speed <= SPEED_10000) {
1870 *width = IB_WIDTH_1X;
1871 *speed = IB_SPEED_FDR10;
1872 } else if (netdev_speed <= SPEED_20000) {
1873 *width = IB_WIDTH_4X;
1874 *speed = IB_SPEED_DDR;
1875 } else if (netdev_speed <= SPEED_25000) {
1876 *width = IB_WIDTH_1X;
1877 *speed = IB_SPEED_EDR;
1878 } else if (netdev_speed <= SPEED_40000) {
1879 *width = IB_WIDTH_4X;
1880 *speed = IB_SPEED_FDR10;
1881 } else {
1882 *width = IB_WIDTH_4X;
1883 *speed = IB_SPEED_EDR;
1884 }
1885
1886 return 0;
1887 }
1888
1889 u64
irdma_mac_to_u64(const u8 * eth_add)1890 irdma_mac_to_u64(const u8 *eth_add)
1891 {
1892 int idx;
1893 u64 u64_eth_add;
1894
1895 for (idx = 0, u64_eth_add = 0; idx < ETHER_ADDR_LEN; idx++)
1896 u64_eth_add = u64_eth_add << 8 | eth_add[idx];
1897
1898 return u64_eth_add;
1899 }
1900