1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2021 - 2023 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "osdep.h"
36 #include "ice_rdma.h"
37 #include "irdma_di_if.h"
38 #include "irdma_main.h"
39 #include <sys/gsb_crc32.h>
40 #include <netinet/in_fib.h>
41 #include <netinet6/in6_fib.h>
42 #include <net/route/nhop.h>
43 #include <net/if_llatbl.h>
44
45 /* additional QP debuging option. Keep false unless needed */
46 bool irdma_upload_context = false;
47
48 inline u32
irdma_rd32(struct irdma_dev_ctx * dev_ctx,u32 reg)49 irdma_rd32(struct irdma_dev_ctx *dev_ctx, u32 reg){
50
51 KASSERT(reg < dev_ctx->mem_bus_space_size,
52 ("irdma: register offset %#jx too large (max is %#jx)",
53 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
54
55 return (bus_space_read_4(dev_ctx->mem_bus_space_tag,
56 dev_ctx->mem_bus_space_handle, reg));
57 }
58
59 inline void
irdma_wr32(struct irdma_dev_ctx * dev_ctx,u32 reg,u32 value)60 irdma_wr32(struct irdma_dev_ctx *dev_ctx, u32 reg, u32 value)
61 {
62
63 KASSERT(reg < dev_ctx->mem_bus_space_size,
64 ("irdma: register offset %#jx too large (max is %#jx)",
65 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
66
67 bus_space_write_4(dev_ctx->mem_bus_space_tag,
68 dev_ctx->mem_bus_space_handle, reg, value);
69 }
70
71 inline u64
irdma_rd64(struct irdma_dev_ctx * dev_ctx,u32 reg)72 irdma_rd64(struct irdma_dev_ctx *dev_ctx, u32 reg){
73
74 KASSERT(reg < dev_ctx->mem_bus_space_size,
75 ("irdma: register offset %#jx too large (max is %#jx)",
76 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
77
78 return (bus_space_read_8(dev_ctx->mem_bus_space_tag,
79 dev_ctx->mem_bus_space_handle, reg));
80 }
81
82 inline void
irdma_wr64(struct irdma_dev_ctx * dev_ctx,u32 reg,u64 value)83 irdma_wr64(struct irdma_dev_ctx *dev_ctx, u32 reg, u64 value)
84 {
85
86 KASSERT(reg < dev_ctx->mem_bus_space_size,
87 ("irdma: register offset %#jx too large (max is %#jx)",
88 (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
89
90 bus_space_write_8(dev_ctx->mem_bus_space_tag,
91 dev_ctx->mem_bus_space_handle, reg, value);
92
93 }
94
95 void
irdma_request_reset(struct irdma_pci_f * rf)96 irdma_request_reset(struct irdma_pci_f *rf)
97 {
98 struct ice_rdma_peer *peer = rf->peer_info;
99 struct ice_rdma_request req = {0};
100
101 req.type = ICE_RDMA_EVENT_RESET;
102
103 printf("%s:%d requesting pf-reset\n", __func__, __LINE__);
104 IRDMA_DI_REQ_HANDLER(peer, &req);
105 }
106
107 int
irdma_register_qset(struct irdma_sc_vsi * vsi,struct irdma_ws_node * tc_node)108 irdma_register_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node)
109 {
110 struct irdma_device *iwdev = vsi->back_vsi;
111 struct ice_rdma_peer *peer = iwdev->rf->peer_info;
112 struct ice_rdma_request req = {0};
113 struct ice_rdma_qset_update *res = &req.res;
114
115 req.type = ICE_RDMA_EVENT_QSET_REGISTER;
116 res->cnt_req = 1;
117 res->res_type = ICE_RDMA_QSET_ALLOC;
118 res->qsets.qs_handle = tc_node->qs_handle;
119 res->qsets.tc = tc_node->traffic_class;
120 res->qsets.vsi_id = vsi->vsi_idx;
121
122 IRDMA_DI_REQ_HANDLER(peer, &req);
123
124 tc_node->l2_sched_node_id = res->qsets.teid;
125 vsi->qos[tc_node->user_pri].l2_sched_node_id =
126 res->qsets.teid;
127
128 return 0;
129 }
130
131 void
irdma_unregister_qset(struct irdma_sc_vsi * vsi,struct irdma_ws_node * tc_node)132 irdma_unregister_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node)
133 {
134 struct irdma_device *iwdev = vsi->back_vsi;
135 struct ice_rdma_peer *peer = iwdev->rf->peer_info;
136 struct ice_rdma_request req = {0};
137 struct ice_rdma_qset_update *res = &req.res;
138
139 req.type = ICE_RDMA_EVENT_QSET_REGISTER;
140 res->res_allocated = 1;
141 res->res_type = ICE_RDMA_QSET_FREE;
142 res->qsets.vsi_id = vsi->vsi_idx;
143 res->qsets.teid = tc_node->l2_sched_node_id;
144 res->qsets.qs_handle = tc_node->qs_handle;
145
146 IRDMA_DI_REQ_HANDLER(peer, &req);
147 }
148
149 void *
hw_to_dev(struct irdma_hw * hw)150 hw_to_dev(struct irdma_hw *hw)
151 {
152 struct irdma_pci_f *rf;
153
154 rf = container_of(hw, struct irdma_pci_f, hw);
155 return rf->pcidev;
156 }
157
158 void
irdma_free_hash_desc(void * desc)159 irdma_free_hash_desc(void *desc)
160 {
161 return;
162 }
163
164 int
irdma_init_hash_desc(void ** desc)165 irdma_init_hash_desc(void **desc)
166 {
167 return 0;
168 }
169
170 int
irdma_ieq_check_mpacrc(void * desc,void * addr,u32 len,u32 val)171 irdma_ieq_check_mpacrc(void *desc,
172 void *addr, u32 len, u32 val)
173 {
174 u32 crc = calculate_crc32c(0xffffffff, addr, len) ^ 0xffffffff;
175 int ret_code = 0;
176
177 if (crc != val) {
178 irdma_pr_err("mpa crc check fail %x %x\n", crc, val);
179 ret_code = -EINVAL;
180 }
181 printf("%s: result crc=%x value=%x\n", __func__, crc, val);
182 return ret_code;
183 }
184
185 static u_int
irdma_add_ipv6_cb(void * arg,struct ifaddr * addr,u_int count __unused)186 irdma_add_ipv6_cb(void *arg, struct ifaddr *addr, u_int count __unused){
187 struct irdma_device *iwdev = arg;
188 struct sockaddr_in6 *sin6;
189 u32 local_ipaddr6[4] = {};
190 char ip6buf[INET6_ADDRSTRLEN];
191 u8 *mac_addr;
192
193 sin6 = (struct sockaddr_in6 *)addr->ifa_addr;
194
195 irdma_copy_ip_ntohl(local_ipaddr6, (u32 *)&sin6->sin6_addr);
196
197 mac_addr = if_getlladdr(addr->ifa_ifp);
198
199 printf("%s:%d IP=%s, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
200 __func__, __LINE__,
201 ip6_sprintf(ip6buf, &sin6->sin6_addr),
202 mac_addr[0], mac_addr[1], mac_addr[2],
203 mac_addr[3], mac_addr[4], mac_addr[5]);
204
205 irdma_manage_arp_cache(iwdev->rf, mac_addr, local_ipaddr6,
206 IRDMA_ARP_ADD);
207 return (0);
208 }
209
210 /**
211 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
212 * @iwdev: irdma device
213 * @ifp: interface network device pointer
214 */
215 static void
irdma_add_ipv6_addr(struct irdma_device * iwdev,struct ifnet * ifp)216 irdma_add_ipv6_addr(struct irdma_device *iwdev, struct ifnet *ifp)
217 {
218 if_addr_rlock(ifp);
219 if_foreach_addr_type(ifp, AF_INET6, irdma_add_ipv6_cb, iwdev);
220 if_addr_runlock(ifp);
221 }
222
223 static u_int
irdma_add_ipv4_cb(void * arg,struct ifaddr * addr,u_int count __unused)224 irdma_add_ipv4_cb(void *arg, struct ifaddr *addr, u_int count __unused){
225 struct irdma_device *iwdev = arg;
226 struct sockaddr_in *sin;
227 u32 ip_addr[4] = {};
228 uint8_t *mac_addr;
229
230 sin = (struct sockaddr_in *)addr->ifa_addr;
231
232 ip_addr[0] = ntohl(sin->sin_addr.s_addr);
233
234 mac_addr = if_getlladdr(addr->ifa_ifp);
235
236 printf("%s:%d IP=%d.%d.%d.%d, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
237 __func__, __LINE__,
238 ip_addr[0] >> 24,
239 (ip_addr[0] >> 16) & 0xFF,
240 (ip_addr[0] >> 8) & 0xFF,
241 ip_addr[0] & 0xFF,
242 mac_addr[0], mac_addr[1], mac_addr[2],
243 mac_addr[3], mac_addr[4], mac_addr[5]);
244
245 irdma_manage_arp_cache(iwdev->rf, mac_addr, ip_addr,
246 IRDMA_ARP_ADD);
247 return (0);
248 }
249
250 /**
251 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
252 * @iwdev: irdma device
253 * @ifp: interface network device pointer
254 */
255 static void
irdma_add_ipv4_addr(struct irdma_device * iwdev,struct ifnet * ifp)256 irdma_add_ipv4_addr(struct irdma_device *iwdev, struct ifnet *ifp)
257 {
258 if_addr_rlock(ifp);
259 if_foreach_addr_type(ifp, AF_INET, irdma_add_ipv4_cb, iwdev);
260 if_addr_runlock(ifp);
261 }
262
263 /**
264 * irdma_add_ip - add ip addresses
265 * @iwdev: irdma device
266 *
267 * Add ipv4/ipv6 addresses to the arp cache
268 */
269 void
irdma_add_ip(struct irdma_device * iwdev)270 irdma_add_ip(struct irdma_device *iwdev)
271 {
272 struct ifnet *ifp = iwdev->netdev;
273 struct ifnet *ifv;
274 struct epoch_tracker et;
275 int i;
276
277 irdma_add_ipv4_addr(iwdev, ifp);
278 irdma_add_ipv6_addr(iwdev, ifp);
279 for (i = 0; if_getvlantrunk(ifp) != NULL && i < VLAN_N_VID; ++i) {
280 NET_EPOCH_ENTER(et);
281 ifv = VLAN_DEVAT(ifp, i);
282 NET_EPOCH_EXIT(et);
283 if (!ifv)
284 continue;
285 irdma_add_ipv4_addr(iwdev, ifv);
286 irdma_add_ipv6_addr(iwdev, ifv);
287 }
288 }
289
290 static void
irdma_ifaddrevent_handler(void * arg,struct ifnet * ifp,struct ifaddr * ifa,int event)291 irdma_ifaddrevent_handler(void *arg, struct ifnet *ifp, struct ifaddr *ifa, int event)
292 {
293 struct irdma_pci_f *rf = arg;
294 struct ifnet *ifv = NULL;
295 struct sockaddr_in *sin;
296 struct epoch_tracker et;
297 int arp_index = 0, i = 0;
298 u32 ip[4] = {};
299
300 if (!ifa || !ifa->ifa_addr || !ifp)
301 return;
302 if (rf->iwdev->netdev != ifp) {
303 for (i = 0; if_getvlantrunk(rf->iwdev->netdev) != NULL && i < VLAN_N_VID; ++i) {
304 NET_EPOCH_ENTER(et);
305 ifv = VLAN_DEVAT(rf->iwdev->netdev, i);
306 NET_EPOCH_EXIT(et);
307 if (ifv == ifp)
308 break;
309 }
310 if (ifv != ifp)
311 return;
312 }
313 sin = (struct sockaddr_in *)ifa->ifa_addr;
314
315 switch (event) {
316 case IFADDR_EVENT_ADD:
317 if (sin->sin_family == AF_INET)
318 irdma_add_ipv4_addr(rf->iwdev, ifp);
319 else if (sin->sin_family == AF_INET6)
320 irdma_add_ipv6_addr(rf->iwdev, ifp);
321 break;
322 case IFADDR_EVENT_DEL:
323 if (sin->sin_family == AF_INET) {
324 ip[0] = ntohl(sin->sin_addr.s_addr);
325 } else if (sin->sin_family == AF_INET6) {
326 irdma_copy_ip_ntohl(ip, (u32 *)&((struct sockaddr_in6 *)sin)->sin6_addr);
327 } else {
328 break;
329 }
330 for_each_set_bit(arp_index, rf->allocated_arps, rf->arp_table_size) {
331 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) {
332 irdma_manage_arp_cache(rf, rf->arp_table[arp_index].mac_addr,
333 rf->arp_table[arp_index].ip_addr,
334 IRDMA_ARP_DELETE);
335 }
336 }
337 break;
338 default:
339 break;
340 }
341 }
342
343 void
irdma_reg_ipaddr_event_cb(struct irdma_pci_f * rf)344 irdma_reg_ipaddr_event_cb(struct irdma_pci_f *rf)
345 {
346 rf->irdma_ifaddr_event = EVENTHANDLER_REGISTER(ifaddr_event_ext,
347 irdma_ifaddrevent_handler,
348 rf,
349 EVENTHANDLER_PRI_ANY);
350 }
351
352 void
irdma_dereg_ipaddr_event_cb(struct irdma_pci_f * rf)353 irdma_dereg_ipaddr_event_cb(struct irdma_pci_f *rf)
354 {
355 EVENTHANDLER_DEREGISTER(ifaddr_event_ext, rf->irdma_ifaddr_event);
356 }
357
358 static int
irdma_get_route_ifp(struct sockaddr * dst_sin,struct ifnet * netdev,struct ifnet ** ifp,struct sockaddr ** nexthop,bool * gateway)359 irdma_get_route_ifp(struct sockaddr *dst_sin, struct ifnet *netdev,
360 struct ifnet **ifp, struct sockaddr **nexthop, bool *gateway)
361 {
362 struct nhop_object *nh;
363
364 if (dst_sin->sa_family == AF_INET6)
365 nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr,
366 ((struct sockaddr_in6 *)dst_sin)->sin6_scope_id, NHR_NONE, 0);
367 else
368 nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0);
369 if (!nh || (nh->nh_ifp != netdev &&
370 rdma_vlan_dev_real_dev(nh->nh_ifp) != netdev))
371 goto rt_not_found;
372 *gateway = (nh->nh_flags & NHF_GATEWAY) ? true : false;
373 *nexthop = (*gateway) ? &nh->gw_sa : dst_sin;
374 *ifp = nh->nh_ifp;
375
376 return 0;
377
378 rt_not_found:
379 pr_err("irdma: route not found\n");
380 return -ENETUNREACH;
381 }
382
383 /**
384 * irdma_get_dst_mac - get destination mac address
385 * @cm_node: connection's node
386 * @dst_sin: destination address information
387 * @dst_mac: mac address array to return
388 */
389 int
irdma_get_dst_mac(struct irdma_cm_node * cm_node,struct sockaddr * dst_sin,u8 * dst_mac)390 irdma_get_dst_mac(struct irdma_cm_node *cm_node, struct sockaddr *dst_sin, u8 *dst_mac)
391 {
392 struct ifnet *netdev = cm_node->iwdev->netdev;
393 #ifdef VIMAGE
394 struct vnet *vnet = irdma_cmid_to_vnet(cm_node->cm_id);
395 #endif
396 struct ifnet *ifp;
397 struct llentry *lle;
398 struct sockaddr *nexthop;
399 struct epoch_tracker et;
400 int err;
401 bool gateway;
402
403 NET_EPOCH_ENTER(et);
404 CURVNET_SET_QUIET(vnet);
405 err = irdma_get_route_ifp(dst_sin, netdev, &ifp, &nexthop, &gateway);
406 if (err)
407 goto get_route_fail;
408
409 if (dst_sin->sa_family == AF_INET) {
410 err = arpresolve(ifp, gateway, NULL, nexthop, dst_mac, NULL, &lle);
411 } else if (dst_sin->sa_family == AF_INET6) {
412 err = nd6_resolve(ifp, LLE_SF(AF_INET6, gateway), NULL, nexthop,
413 dst_mac, NULL, &lle);
414 } else {
415 err = -EPROTONOSUPPORT;
416 }
417
418 get_route_fail:
419 CURVNET_RESTORE();
420 NET_EPOCH_EXIT(et);
421 if (err) {
422 pr_err("failed to resolve neighbor address (err=%d)\n",
423 err);
424 return -ENETUNREACH;
425 }
426
427 return 0;
428 }
429
430 /**
431 * irdma_addr_resolve_neigh - resolve neighbor address
432 * @cm_node: connection's node
433 * @dst_ip: remote ip address
434 * @arpindex: if there is an arp entry
435 */
436 int
irdma_addr_resolve_neigh(struct irdma_cm_node * cm_node,u32 dst_ip,int arpindex)437 irdma_addr_resolve_neigh(struct irdma_cm_node *cm_node,
438 u32 dst_ip, int arpindex)
439 {
440 struct irdma_device *iwdev = cm_node->iwdev;
441 struct sockaddr_in dst_sin = {};
442 int err;
443 u32 ip[4] = {};
444 u8 dst_mac[MAX_ADDR_LEN];
445
446 dst_sin.sin_len = sizeof(dst_sin);
447 dst_sin.sin_family = AF_INET;
448 dst_sin.sin_port = 0;
449 dst_sin.sin_addr.s_addr = htonl(dst_ip);
450
451 err = irdma_get_dst_mac(cm_node, (struct sockaddr *)&dst_sin, dst_mac);
452 if (err)
453 return arpindex;
454
455 ip[0] = dst_ip;
456
457 return irdma_add_arp(iwdev->rf, ip, dst_mac);
458 }
459
460 /**
461 * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address
462 * @cm_node: connection's node
463 * @dest: remote ip address
464 * @arpindex: if there is an arp entry
465 */
466 int
irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node * cm_node,u32 * dest,int arpindex)467 irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node *cm_node,
468 u32 *dest, int arpindex)
469 {
470 struct irdma_device *iwdev = cm_node->iwdev;
471 struct sockaddr_in6 dst_addr = {};
472 int err;
473 u8 dst_mac[MAX_ADDR_LEN];
474
475 dst_addr.sin6_family = AF_INET6;
476 dst_addr.sin6_len = sizeof(dst_addr);
477 dst_addr.sin6_scope_id = if_getindex(iwdev->netdev);
478
479 irdma_copy_ip_htonl(dst_addr.sin6_addr.__u6_addr.__u6_addr32, dest);
480 err = irdma_get_dst_mac(cm_node, (struct sockaddr *)&dst_addr, dst_mac);
481 if (err)
482 return arpindex;
483
484 return irdma_add_arp(iwdev->rf, dest, dst_mac);
485 }
486
487 int
irdma_resolve_neigh_lpb_chk(struct irdma_device * iwdev,struct irdma_cm_node * cm_node,struct irdma_cm_info * cm_info)488 irdma_resolve_neigh_lpb_chk(struct irdma_device *iwdev, struct irdma_cm_node *cm_node,
489 struct irdma_cm_info *cm_info)
490 {
491 #ifdef VIMAGE
492 struct vnet *vnet = irdma_cmid_to_vnet(cm_node->cm_id);
493 #endif
494 int arpindex;
495 int oldarpindex;
496 bool is_lpb = false;
497
498 CURVNET_SET_QUIET(vnet);
499 is_lpb = cm_node->ipv4 ?
500 irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0]) :
501 irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr);
502 CURVNET_RESTORE();
503 if (is_lpb) {
504 cm_node->do_lpb = true;
505 arpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
506 NULL,
507 IRDMA_ARP_RESOLVE);
508 } else {
509 oldarpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
510 NULL,
511 IRDMA_ARP_RESOLVE);
512 if (cm_node->ipv4)
513 arpindex = irdma_addr_resolve_neigh(cm_node,
514 cm_info->rem_addr[0],
515 oldarpindex);
516 else
517 arpindex = irdma_addr_resolve_neigh_ipv6(cm_node,
518 cm_info->rem_addr,
519 oldarpindex);
520 }
521 return arpindex;
522 }
523
524 /**
525 * irdma_add_handler - add a handler to the list
526 * @hdl: handler to be added to the handler list
527 */
528 void
irdma_add_handler(struct irdma_handler * hdl)529 irdma_add_handler(struct irdma_handler *hdl)
530 {
531 unsigned long flags;
532
533 spin_lock_irqsave(&irdma_handler_lock, flags);
534 list_add(&hdl->list, &irdma_handlers);
535 spin_unlock_irqrestore(&irdma_handler_lock, flags);
536 }
537
538 /**
539 * irdma_del_handler - delete a handler from the list
540 * @hdl: handler to be deleted from the handler list
541 */
542 void
irdma_del_handler(struct irdma_handler * hdl)543 irdma_del_handler(struct irdma_handler *hdl)
544 {
545 unsigned long flags;
546
547 spin_lock_irqsave(&irdma_handler_lock, flags);
548 list_del(&hdl->list);
549 spin_unlock_irqrestore(&irdma_handler_lock, flags);
550 }
551
552 /**
553 * irdma_set_rf_user_cfg_params - apply user configurable settings
554 * @rf: RDMA PCI function
555 */
556 void
irdma_set_rf_user_cfg_params(struct irdma_pci_f * rf)557 irdma_set_rf_user_cfg_params(struct irdma_pci_f *rf)
558 {
559 int en_rem_endpoint_trk = 0;
560 int limits_sel = 4;
561
562 rf->en_rem_endpoint_trk = en_rem_endpoint_trk;
563 rf->limits_sel = limits_sel;
564 rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
565 /* Enable DCQCN algorithm by default */
566 rf->dcqcn_ena = true;
567 }
568
569 /**
570 * irdma_sysctl_dcqcn_update - handle dcqcn_ena sysctl update
571 * @arg1: pointer to rf
572 * @arg2: unused
573 * @oidp: sysctl oid structure
574 * @req: sysctl request pointer
575 */
576 static int
irdma_sysctl_dcqcn_update(SYSCTL_HANDLER_ARGS)577 irdma_sysctl_dcqcn_update(SYSCTL_HANDLER_ARGS)
578 {
579 struct irdma_pci_f *rf = (struct irdma_pci_f *)arg1;
580 int ret;
581 u8 dcqcn_ena = rf->dcqcn_ena;
582
583 ret = sysctl_handle_8(oidp, &dcqcn_ena, 0, req);
584 if ((ret) || (req->newptr == NULL))
585 return ret;
586 if (dcqcn_ena == 0)
587 rf->dcqcn_ena = false;
588 else
589 rf->dcqcn_ena = true;
590
591 return 0;
592 }
593
594 enum irdma_cqp_stats_info {
595 IRDMA_CQP_REQ_CMDS = 28,
596 IRDMA_CQP_CMPL_CMDS = 29
597 };
598
599 static int
irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS)600 irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS)
601 {
602 struct irdma_sc_cqp *cqp = (struct irdma_sc_cqp *)arg1;
603 char rslt[192] = "no cqp available yet";
604 int rslt_size = sizeof(rslt) - 1;
605 int option = (int)arg2;
606
607 if (!cqp) {
608 return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
609 }
610
611 snprintf(rslt, sizeof(rslt), "");
612 switch (option) {
613 case IRDMA_CQP_REQ_CMDS:
614 snprintf(rslt, rslt_size, "%lu", cqp->requested_ops);
615 break;
616 case IRDMA_CQP_CMPL_CMDS:
617 snprintf(rslt, rslt_size, "%lu", atomic64_read(&cqp->completed_ops));
618 break;
619 }
620
621 return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
622 }
623
624 struct irdma_sw_stats_tunable_info {
625 u8 op_type;
626 const char name[32];
627 const char desc[32];
628 uintptr_t value;
629 };
630
631 static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = {
632 {IRDMA_OP_CEQ_DESTROY, "ceq_destroy", "ceq_destroy", 0},
633 {IRDMA_OP_AEQ_DESTROY, "aeq_destroy", "aeq_destroy", 0},
634 {IRDMA_OP_DELETE_ARP_CACHE_ENTRY, "delete_arp_cache_entry",
635 "delete_arp_cache_entry", 0},
636 {IRDMA_OP_MANAGE_APBVT_ENTRY, "manage_apbvt_entry",
637 "manage_apbvt_entry", 0},
638 {IRDMA_OP_CEQ_CREATE, "ceq_create", "ceq_create", 0},
639 {IRDMA_OP_AEQ_CREATE, "aeq_create", "aeq_create", 0},
640 {IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY, "manage_qhash_table_entry",
641 "manage_qhash_table_entry", 0},
642 {IRDMA_OP_QP_MODIFY, "qp_modify", "qp_modify", 0},
643 {IRDMA_OP_QP_UPLOAD_CONTEXT, "qp_upload_context", "qp_upload_context",
644 0},
645 {IRDMA_OP_CQ_CREATE, "cq_create", "cq_create", 0},
646 {IRDMA_OP_CQ_DESTROY, "cq_destroy", "cq_destroy", 0},
647 {IRDMA_OP_QP_CREATE, "qp_create", "qp_create", 0},
648 {IRDMA_OP_QP_DESTROY, "qp_destroy", "qp_destroy", 0},
649 {IRDMA_OP_ALLOC_STAG, "alloc_stag", "alloc_stag", 0},
650 {IRDMA_OP_MR_REG_NON_SHARED, "mr_reg_non_shared", "mr_reg_non_shared",
651 0},
652 {IRDMA_OP_DEALLOC_STAG, "dealloc_stag", "dealloc_stag", 0},
653 {IRDMA_OP_MW_ALLOC, "mw_alloc", "mw_alloc", 0},
654 {IRDMA_OP_QP_FLUSH_WQES, "qp_flush_wqes", "qp_flush_wqes", 0},
655 {IRDMA_OP_ADD_ARP_CACHE_ENTRY, "add_arp_cache_entry",
656 "add_arp_cache_entry", 0},
657 {IRDMA_OP_MANAGE_PUSH_PAGE, "manage_push_page", "manage_push_page", 0},
658 {IRDMA_OP_UPDATE_PE_SDS, "update_pe_sds", "update_pe_sds", 0},
659 {IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE, "manage_hmc_pm_func_table",
660 "manage_hmc_pm_func_table", 0},
661 {IRDMA_OP_SUSPEND, "suspend", "suspend", 0},
662 {IRDMA_OP_RESUME, "resume", "resume", 0},
663 {25, "manage_vchnl_req_pble_bp", "manage_vchnl_req_pble_bp", 0},
664 {IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0},
665 {IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0},
666 {IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0},
667 {IRDMA_OP_AH_MODIFY, "ah_modify", "ah_modify", 0},
668 {IRDMA_OP_AH_DESTROY, "ah_destroy", "ah_destroy", 0},
669 {IRDMA_OP_MC_CREATE, "mc_create", "mc_create", 0},
670 {IRDMA_OP_MC_DESTROY, "mc_destroy", "mc_destroy", 0},
671 {IRDMA_OP_MC_MODIFY, "mc_modify", "mc_modify", 0},
672 {IRDMA_OP_STATS_ALLOCATE, "stats_allocate", "stats_allocate", 0},
673 {IRDMA_OP_STATS_FREE, "stats_free", "stats_free", 0},
674 {IRDMA_OP_STATS_GATHER, "stats_gather", "stats_gather", 0},
675 {IRDMA_OP_WS_ADD_NODE, "ws_add_node", "ws_add_node", 0},
676 {IRDMA_OP_WS_MODIFY_NODE, "ws_modify_node", "ws_modify_node", 0},
677 {IRDMA_OP_WS_DELETE_NODE, "ws_delete_node", "ws_delete_node", 0},
678 {IRDMA_OP_WS_FAILOVER_START, "ws_failover_start", "ws_failover_start",
679 0},
680 {IRDMA_OP_WS_FAILOVER_COMPLETE, "ws_failover_complete",
681 "ws_failover_complete", 0},
682 {IRDMA_OP_SET_UP_MAP, "set_up_map", "set_up_map", 0},
683 {IRDMA_OP_GEN_AE, "gen_ae", "gen_ae", 0},
684 {IRDMA_OP_QUERY_RDMA_FEATURES, "query_rdma_features",
685 "query_rdma_features", 0},
686 {IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY, "alloc_local_mac_entry",
687 "alloc_local_mac_entry", 0},
688 {IRDMA_OP_ADD_LOCAL_MAC_ENTRY, "add_local_mac_entry",
689 "add_local_mac_entry", 0},
690 {IRDMA_OP_DELETE_LOCAL_MAC_ENTRY, "delete_local_mac_entry",
691 "delete_local_mac_entry", 0},
692 {IRDMA_OP_CQ_MODIFY, "cq_modify", "cq_modify", 0}
693 };
694
695 static const struct irdma_sw_stats_tunable_info irdma_cmcs_list[] = {
696 {0, "cm_nodes_created", "cm_nodes_created",
697 offsetof(struct irdma_cm_core, stats_nodes_created)},
698 {0, "cm_nodes_destroyed", "cm_nodes_destroyed",
699 offsetof(struct irdma_cm_core, stats_nodes_destroyed)},
700 {0, "cm_listen_created", "cm_listen_created",
701 offsetof(struct irdma_cm_core, stats_listen_created)},
702 {0, "cm_listen_destroyed", "cm_listen_destroyed",
703 offsetof(struct irdma_cm_core, stats_listen_destroyed)},
704 {0, "cm_listen_nodes_created", "cm_listen_nodes_created",
705 offsetof(struct irdma_cm_core, stats_listen_nodes_created)},
706 {0, "cm_listen_nodes_destroyed", "cm_listen_nodes_destroyed",
707 offsetof(struct irdma_cm_core, stats_listen_nodes_destroyed)},
708 {0, "cm_lpbs", "cm_lpbs", offsetof(struct irdma_cm_core, stats_lpbs)},
709 {0, "cm_accepts", "cm_accepts", offsetof(struct irdma_cm_core,
710 stats_accepts)},
711 {0, "cm_rejects", "cm_rejects", offsetof(struct irdma_cm_core,
712 stats_rejects)},
713 {0, "cm_connect_errs", "cm_connect_errs",
714 offsetof(struct irdma_cm_core, stats_connect_errs)},
715 {0, "cm_passive_errs", "cm_passive_errs",
716 offsetof(struct irdma_cm_core, stats_passive_errs)},
717 {0, "cm_pkt_retrans", "cm_pkt_retrans", offsetof(struct irdma_cm_core,
718 stats_pkt_retrans)},
719 {0, "cm_backlog_drops", "cm_backlog_drops",
720 offsetof(struct irdma_cm_core, stats_backlog_drops)},
721 };
722
723 static const struct irdma_sw_stats_tunable_info irdma_ilqs32_list[] = {
724 {0, "ilq_avail_buf_count", "ilq_avail_buf_count",
725 offsetof(struct irdma_puda_rsrc, avail_buf_count)},
726 {0, "ilq_alloc_buf_count", "ilq_alloc_buf_count",
727 offsetof(struct irdma_puda_rsrc, alloc_buf_count)}
728 };
729 static const struct irdma_sw_stats_tunable_info irdma_ilqs_list[] = {
730 {0, "ilq_stats_buf_alloc_fail", "ilq_stats_buf_alloc_fail",
731 offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)},
732 {0, "ilq_stats_pkt_rcvd", "ilq_stats_pkt_rcvd",
733 offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)},
734 {0, "ilq_stats_pkt_sent", "ilq_stats_pkt_sent",
735 offsetof(struct irdma_puda_rsrc, stats_pkt_sent)},
736 {0, "ilq_stats_rcvd_pkt_err", "ilq_stats_rcvd_pkt_err",
737 offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)},
738 {0, "ilq_stats_sent_pkt_q", "ilq_stats_sent_pkt_q",
739 offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)}
740 };
741
742 static const struct irdma_sw_stats_tunable_info irdma_ieqs32_list[] = {
743 {0, "ieq_avail_buf_count", "ieq_avail_buf_count",
744 offsetof(struct irdma_puda_rsrc, avail_buf_count)},
745 {0, "ieq_alloc_buf_count", "ieq_alloc_buf_count",
746 offsetof(struct irdma_puda_rsrc, alloc_buf_count)}
747 };
748 static const struct irdma_sw_stats_tunable_info irdma_ieqs_list[] = {
749 {0, "ieq_stats_buf_alloc_fail", "ieq_stats_buf_alloc_fail",
750 offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)},
751 {0, "ieq_stats_pkt_rcvd", "ieq_stats_pkt_rcvd",
752 offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)},
753 {0, "ieq_stats_pkt_sent", "ieq_stats_pkt_sent",
754 offsetof(struct irdma_puda_rsrc, stats_pkt_sent)},
755 {0, "ieq_stats_rcvd_pkt_err", "ieq_stats_rcvd_pkt_err",
756 offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)},
757 {0, "ieq_stats_sent_pkt_q", "ieq_stats_sent_pkt_q",
758 offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)},
759 {0, "ieq_stats_bad_qp_id", "ieq_stats_bad_qp_id",
760 offsetof(struct irdma_puda_rsrc, stats_bad_qp_id)},
761 {0, "ieq_fpdu_processed", "ieq_fpdu_processed",
762 offsetof(struct irdma_puda_rsrc, fpdu_processed)},
763 {0, "ieq_bad_seq_num", "ieq_bad_seq_num",
764 offsetof(struct irdma_puda_rsrc, bad_seq_num)},
765 {0, "ieq_crc_err", "ieq_crc_err", offsetof(struct irdma_puda_rsrc,
766 crc_err)},
767 {0, "ieq_pmode_count", "ieq_pmode_count",
768 offsetof(struct irdma_puda_rsrc, pmode_count)},
769 {0, "ieq_partials_handled", "ieq_partials_handled",
770 offsetof(struct irdma_puda_rsrc, partials_handled)},
771 };
772
773 /**
774 * irdma_dcqcn_tunables_init - create tunables for dcqcn settings
775 * @rf: RDMA PCI function
776 *
777 * Create DCQCN related sysctls for the driver.
778 * dcqcn_ena is writeable settings and applicable to next QP creation or
779 * context setting.
780 * all other settings are of RDTUN type (read on driver load) and are
781 * applicable only to CQP creation.
782 */
783 void
irdma_dcqcn_tunables_init(struct irdma_pci_f * rf)784 irdma_dcqcn_tunables_init(struct irdma_pci_f *rf)
785 {
786 struct sysctl_oid_list *irdma_sysctl_oid_list;
787
788 irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree);
789
790 SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
791 OID_AUTO, "dcqcn_enable", CTLFLAG_RW | CTLTYPE_U8, rf, 0,
792 irdma_sysctl_dcqcn_update, "A",
793 "enables DCQCN algorithm for RoCEv2 on all ports, default=true");
794
795 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
796 OID_AUTO, "dcqcn_cc_cfg_valid", CTLFLAG_RDTUN,
797 &rf->dcqcn_params.cc_cfg_valid, 0,
798 "set DCQCN parameters to be valid, default=false");
799
800 rf->dcqcn_params.min_dec_factor = 1;
801 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
802 OID_AUTO, "dcqcn_min_dec_factor", CTLFLAG_RDTUN,
803 &rf->dcqcn_params.min_dec_factor, 0,
804 "set minimum percentage factor by which tx rate can be changed for CNP, Range: 1-100, default=1");
805
806 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
807 OID_AUTO, "dcqcn_min_rate_MBps", CTLFLAG_RDTUN,
808 &rf->dcqcn_params.min_rate, 0,
809 "set minimum rate limit value, in MBits per second, default=0");
810
811 rf->dcqcn_params.dcqcn_f = 5;
812 SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
813 OID_AUTO, "dcqcn_F", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_f, 0,
814 "set number of times to stay in each stage of bandwidth recovery, default=5");
815
816 rf->dcqcn_params.dcqcn_t = 0x37;
817 SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
818 OID_AUTO, "dcqcn_T", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_t, 0,
819 "number of us to elapse before increasing the CWND in DCQCN mode, default=0x37");
820
821 rf->dcqcn_params.dcqcn_b = 0x249f0;
822 SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
823 OID_AUTO, "dcqcn_B", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_b, 0,
824 "set number of MSS to add to the congestion window in additive increase mode, default=0x249f0");
825
826 rf->dcqcn_params.rai_factor = 1;
827 SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
828 OID_AUTO, "dcqcn_rai_factor", CTLFLAG_RDTUN,
829 &rf->dcqcn_params.rai_factor, 0,
830 "set number of MSS to add to the congestion window in additive increase mode, default=1");
831
832 rf->dcqcn_params.hai_factor = 5;
833 SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
834 OID_AUTO, "dcqcn_hai_factor", CTLFLAG_RDTUN,
835 &rf->dcqcn_params.hai_factor, 0,
836 "set number of MSS to add to the congestion window in hyperactive increase mode, default=5");
837
838 rf->dcqcn_params.rreduce_mperiod = 50;
839 SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
840 OID_AUTO, "dcqcn_rreduce_mperiod", CTLFLAG_RDTUN,
841 &rf->dcqcn_params.rreduce_mperiod, 0,
842 "set minimum time between 2 consecutive rate reductions for a single flow, default=50");
843 }
844
845 /**
846 * irdma_sysctl_settings - sysctl runtime settings init
847 * @rf: RDMA PCI function
848 */
849 void
irdma_sysctl_settings(struct irdma_pci_f * rf)850 irdma_sysctl_settings(struct irdma_pci_f *rf)
851 {
852 struct sysctl_oid_list *irdma_sysctl_oid_list;
853
854 irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree);
855
856 SYSCTL_ADD_BOOL(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
857 OID_AUTO, "upload_context", CTLFLAG_RWTUN,
858 &irdma_upload_context, 0,
859 "allow for generating QP's upload context, default=0");
860 }
861
862 void
irdma_sw_stats_tunables_init(struct irdma_pci_f * rf)863 irdma_sw_stats_tunables_init(struct irdma_pci_f *rf)
864 {
865 struct sysctl_oid_list *sws_oid_list;
866 struct sysctl_ctx_list *irdma_ctx = &rf->tun_info.irdma_sysctl_ctx;
867 struct irdma_sc_dev *dev = &rf->sc_dev;
868 struct irdma_cm_core *cm_core = &rf->iwdev->cm_core;
869 struct irdma_puda_rsrc *ilq = rf->iwdev->vsi.ilq;
870 struct irdma_puda_rsrc *ieq = rf->iwdev->vsi.ieq;
871 u64 *ll_ptr;
872 u32 *l_ptr;
873 int cqp_stat_cnt = sizeof(irdma_sws_list) / sizeof(struct irdma_sw_stats_tunable_info);
874 int cmcore_stat_cnt = sizeof(irdma_cmcs_list) / sizeof(struct irdma_sw_stats_tunable_info);
875 int ilqs_stat_cnt = sizeof(irdma_ilqs_list) / sizeof(struct irdma_sw_stats_tunable_info);
876 int ilqs32_stat_cnt = sizeof(irdma_ilqs32_list) / sizeof(struct irdma_sw_stats_tunable_info);
877 int ieqs_stat_cnt = sizeof(irdma_ieqs_list) / sizeof(struct irdma_sw_stats_tunable_info);
878 int ieqs32_stat_cnt = sizeof(irdma_ieqs32_list) / sizeof(struct irdma_sw_stats_tunable_info);
879 int i;
880
881 sws_oid_list = SYSCTL_CHILDREN(rf->tun_info.sws_sysctl_tree);
882
883 for (i = 0; i < cqp_stat_cnt; ++i) {
884 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
885 irdma_sws_list[i].name, CTLFLAG_RD,
886 &dev->cqp_cmd_stats[irdma_sws_list[i].op_type],
887 0, irdma_sws_list[i].desc);
888 }
889 SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO,
890 "req_cmds", CTLFLAG_RD | CTLTYPE_STRING,
891 dev->cqp, IRDMA_CQP_REQ_CMDS, irdma_sysctl_cqp_stats, "A",
892 "req_cmds");
893 SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO,
894 "cmpl_cmds", CTLFLAG_RD | CTLTYPE_STRING,
895 dev->cqp, IRDMA_CQP_CMPL_CMDS, irdma_sysctl_cqp_stats, "A",
896 "cmpl_cmds");
897 for (i = 0; i < cmcore_stat_cnt; ++i) {
898 ll_ptr = (u64 *)((uintptr_t)cm_core + irdma_cmcs_list[i].value);
899 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
900 irdma_cmcs_list[i].name, CTLFLAG_RD, ll_ptr,
901 0, irdma_cmcs_list[i].desc);
902 }
903 for (i = 0; ilq && i < ilqs_stat_cnt; ++i) {
904 ll_ptr = (u64 *)((uintptr_t)ilq + irdma_ilqs_list[i].value);
905 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
906 irdma_ilqs_list[i].name, CTLFLAG_RD, ll_ptr,
907 0, irdma_ilqs_list[i].desc);
908 }
909 for (i = 0; ilq && i < ilqs32_stat_cnt; ++i) {
910 l_ptr = (u32 *)((uintptr_t)ilq + irdma_ilqs32_list[i].value);
911 SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO,
912 irdma_ilqs32_list[i].name, CTLFLAG_RD, l_ptr,
913 0, irdma_ilqs32_list[i].desc);
914 }
915 for (i = 0; ieq && i < ieqs_stat_cnt; ++i) {
916 ll_ptr = (u64 *)((uintptr_t)ieq + irdma_ieqs_list[i].value);
917 SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
918 irdma_ieqs_list[i].name, CTLFLAG_RD, ll_ptr,
919 0, irdma_ieqs_list[i].desc);
920 }
921 for (i = 0; ieq && i < ieqs32_stat_cnt; ++i) {
922 l_ptr = (u32 *)((uintptr_t)ieq + irdma_ieqs32_list[i].value);
923 SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO,
924 irdma_ieqs32_list[i].name, CTLFLAG_RD, l_ptr,
925 0, irdma_ieqs32_list[i].desc);
926 }
927 }
928
929 /**
930 * irdma_dmamap_cb - callback for bus_dmamap_load
931 */
932 static void
irdma_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)933 irdma_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
934 {
935 if (error)
936 return;
937 *(bus_addr_t *) arg = segs->ds_addr;
938 return;
939 }
940
941 /**
942 * irdma_allocate_dma_mem - allocate dma memory
943 * @hw: pointer to hw structure
944 * @mem: structure holding memory information
945 * @size: requested size
946 * @alignment: requested alignment
947 */
948 void *
irdma_allocate_dma_mem(struct irdma_hw * hw,struct irdma_dma_mem * mem,u64 size,u32 alignment)949 irdma_allocate_dma_mem(struct irdma_hw *hw, struct irdma_dma_mem *mem,
950 u64 size, u32 alignment)
951 {
952 struct irdma_dev_ctx *dev_ctx = (struct irdma_dev_ctx *)hw->dev_context;
953 device_t dev = dev_ctx->dev;
954 void *va;
955 int ret;
956
957 ret = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
958 alignment, 0, /* alignment, bounds */
959 BUS_SPACE_MAXADDR, /* lowaddr */
960 BUS_SPACE_MAXADDR, /* highaddr */
961 NULL, NULL, /* filter, filterarg */
962 size, /* maxsize */
963 1, /* nsegments */
964 size, /* maxsegsize */
965 BUS_DMA_ALLOCNOW, /* flags */
966 NULL, /* lockfunc */
967 NULL, /* lockfuncarg */
968 &mem->tag);
969 if (ret != 0) {
970 device_printf(dev, "%s: bus_dma_tag_create failed, error %u\n",
971 __func__, ret);
972 goto fail_0;
973 }
974 ret = bus_dmamem_alloc(mem->tag, (void **)&va,
975 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &mem->map);
976 if (ret != 0) {
977 device_printf(dev, "%s: bus_dmamem_alloc failed, error %u\n",
978 __func__, ret);
979 goto fail_1;
980 }
981 ret = bus_dmamap_load(mem->tag, mem->map, va, size,
982 irdma_dmamap_cb, &mem->pa, BUS_DMA_NOWAIT);
983 if (ret != 0) {
984 device_printf(dev, "%s: bus_dmamap_load failed, error %u\n",
985 __func__, ret);
986 goto fail_2;
987 }
988 mem->nseg = 1;
989 mem->size = size;
990 bus_dmamap_sync(mem->tag, mem->map,
991 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
992
993 return va;
994 fail_2:
995 bus_dmamem_free(mem->tag, va, mem->map);
996 fail_1:
997 bus_dma_tag_destroy(mem->tag);
998 fail_0:
999 mem->map = NULL;
1000 mem->tag = NULL;
1001
1002 return NULL;
1003 }
1004
1005 /**
1006 * irdma_free_dma_mem - Memory free helper fn
1007 * @hw: pointer to hw structure
1008 * @mem: ptr to mem struct to free
1009 */
1010 int
irdma_free_dma_mem(struct irdma_hw * hw,struct irdma_dma_mem * mem)1011 irdma_free_dma_mem(struct irdma_hw *hw, struct irdma_dma_mem *mem)
1012 {
1013 if (!mem)
1014 return -EINVAL;
1015 bus_dmamap_sync(mem->tag, mem->map,
1016 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1017 bus_dmamap_unload(mem->tag, mem->map);
1018 if (!mem->va)
1019 return -ENOMEM;
1020 bus_dmamem_free(mem->tag, mem->va, mem->map);
1021 bus_dma_tag_destroy(mem->tag);
1022
1023 mem->va = NULL;
1024
1025 return 0;
1026 }
1027
1028 u_int
if_foreach_addr_type(if_t ifp,int type,if_addr_cb_t cb,void * cb_arg)1029 if_foreach_addr_type(if_t ifp, int type, if_addr_cb_t cb, void *cb_arg){
1030 struct epoch_tracker et;
1031 struct ifaddr *ifa;
1032 u_int count;
1033
1034 MPASS(cb);
1035
1036 count = 0;
1037 NET_EPOCH_ENTER(et);
1038 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1039 if (ifa->ifa_addr->sa_family != type)
1040 continue;
1041 count += (*cb) (cb_arg, ifa, count);
1042 }
1043 NET_EPOCH_EXIT(et);
1044
1045 return (count);
1046 }
1047
1048 int
if_foreach(if_foreach_cb_t cb,void * cb_arg)1049 if_foreach(if_foreach_cb_t cb, void *cb_arg)
1050 {
1051 if_t ifp;
1052 int error;
1053
1054 NET_EPOCH_ASSERT();
1055 MPASS(cb);
1056
1057 error = 0;
1058 CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
1059 error = cb(ifp, cb_arg);
1060 if (error != 0)
1061 break;
1062 }
1063
1064 return (error);
1065 }
1066
1067 if_t
if_iter_start(struct if_iter * iter)1068 if_iter_start(struct if_iter *iter){
1069 if_t ifp;
1070
1071 NET_EPOCH_ASSERT();
1072 bzero(iter, sizeof(*iter));
1073 ifp = CK_STAILQ_FIRST(&V_ifnet);
1074 if (ifp != NULL)
1075 iter->context[0] = CK_STAILQ_NEXT(ifp, if_link);
1076 else
1077 iter->context[0] = NULL;
1078 return (ifp);
1079 }
1080
1081 if_t
if_iter_next(struct if_iter * iter)1082 if_iter_next(struct if_iter *iter){
1083 if_t cur_ifp = iter->context[0];
1084
1085 if (cur_ifp != NULL)
1086 iter->context[0] = CK_STAILQ_NEXT(cur_ifp, if_link);
1087 return (cur_ifp);
1088 }
1089
1090 void
if_iter_finish(struct if_iter * iter)1091 if_iter_finish(struct if_iter *iter)
1092 {
1093 /* NOP */
1094 }
1095
1096
1097 void
irdma_cleanup_dead_qps(struct irdma_sc_vsi * vsi)1098 irdma_cleanup_dead_qps(struct irdma_sc_vsi *vsi)
1099 {
1100 struct irdma_sc_qp *qp = NULL;
1101 struct irdma_qp *iwqp;
1102 struct irdma_pci_f *rf;
1103 u8 i;
1104
1105 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
1106 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
1107 while (qp) {
1108 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_UDA) {
1109 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
1110 continue;
1111 }
1112 iwqp = qp->qp_uk.back_qp;
1113 rf = iwqp->iwdev->rf;
1114 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem);
1115 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem);
1116
1117 kfree(iwqp->kqp.sq_wrid_mem);
1118 kfree(iwqp->kqp.rq_wrid_mem);
1119 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
1120 kfree(iwqp);
1121 }
1122 }
1123 }
1124