1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2015 - 2023 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #ifndef IRDMA_VERBS_H
36 #define IRDMA_VERBS_H
37
38 #define IRDMA_MAX_SAVED_PHY_PGADDR 4
39 #define IRDMA_FLUSH_DELAY_MS 20
40
41 #define IRDMA_PKEY_TBL_SZ 1
42 #define IRDMA_DEFAULT_PKEY 0xFFFF
43
44 #define IRDMA_SHADOW_PGCNT 1
45
46 #define iwdev_to_idev(iwdev) (&(iwdev)->rf->sc_dev)
47
48 struct irdma_ucontext {
49 struct ib_ucontext ibucontext;
50 struct irdma_device *iwdev;
51 struct irdma_user_mmap_entry *db_mmap_entry;
52 DECLARE_HASHTABLE(mmap_hash_tbl, 6);
53 spinlock_t mmap_tbl_lock; /* protect mmap hash table entries */
54 struct list_head cq_reg_mem_list;
55 spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */
56 struct list_head qp_reg_mem_list;
57 spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */
58 /* FIXME: Move to kcompat ideally. Used < 4.20.0 for old diassasscoaite flow */
59 struct list_head vma_list;
60 struct mutex vma_list_mutex; /* protect the vma_list */
61 int abi_ver;
62 bool legacy_mode:1;
63 bool use_raw_attrs:1;
64 };
65
66 struct irdma_pd {
67 struct ib_pd ibpd;
68 struct irdma_sc_pd sc_pd;
69 struct list_head udqp_list;
70 spinlock_t udqp_list_lock;
71 };
72
73 union irdma_sockaddr {
74 struct sockaddr_in saddr_in;
75 struct sockaddr_in6 saddr_in6;
76 };
77
78 struct irdma_av {
79 u8 macaddr[16];
80 struct ib_ah_attr attrs;
81 union irdma_sockaddr sgid_addr;
82 union irdma_sockaddr dgid_addr;
83 u8 net_type;
84 };
85
86 struct irdma_ah {
87 struct ib_ah ibah;
88 struct irdma_sc_ah sc_ah;
89 struct irdma_pd *pd;
90 struct irdma_av av;
91 u8 sgid_index;
92 union ib_gid dgid;
93 };
94
95 struct irdma_hmc_pble {
96 union {
97 u32 idx;
98 dma_addr_t addr;
99 };
100 };
101
102 struct irdma_cq_mr {
103 struct irdma_hmc_pble cq_pbl;
104 dma_addr_t shadow;
105 bool split;
106 };
107
108 struct irdma_qp_mr {
109 struct irdma_hmc_pble sq_pbl;
110 struct irdma_hmc_pble rq_pbl;
111 dma_addr_t shadow;
112 struct page *sq_page;
113 };
114
115 struct irdma_cq_buf {
116 struct irdma_dma_mem kmem_buf;
117 struct irdma_cq_uk cq_uk;
118 struct irdma_hw *hw;
119 struct list_head list;
120 struct work_struct work;
121 };
122
123 struct irdma_pbl {
124 struct list_head list;
125 union {
126 struct irdma_qp_mr qp_mr;
127 struct irdma_cq_mr cq_mr;
128 };
129
130 bool pbl_allocated:1;
131 bool on_list:1;
132 u64 user_base;
133 struct irdma_pble_alloc pble_alloc;
134 struct irdma_mr *iwmr;
135 };
136
137 struct irdma_mr {
138 union {
139 struct ib_mr ibmr;
140 struct ib_mw ibmw;
141 };
142 struct ib_umem *region;
143 int access;
144 u8 is_hwreg;
145 u16 type;
146 u32 page_cnt;
147 u64 page_size;
148 u64 page_msk;
149 u32 npages;
150 u32 stag;
151 u64 len;
152 u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR];
153 struct irdma_pbl iwpbl;
154 };
155
156 struct irdma_cq {
157 struct ib_cq ibcq;
158 struct irdma_sc_cq sc_cq;
159 u16 cq_head;
160 u16 cq_size;
161 u16 cq_num;
162 bool user_mode;
163 atomic_t armed;
164 enum irdma_cmpl_notify last_notify;
165 u32 polled_cmpls;
166 u32 cq_mem_size;
167 struct irdma_dma_mem kmem;
168 struct irdma_dma_mem kmem_shadow;
169 struct completion free_cq;
170 atomic_t refcnt;
171 spinlock_t lock; /* for poll cq */
172 struct irdma_pbl *iwpbl;
173 struct irdma_pbl *iwpbl_shadow;
174 struct list_head resize_list;
175 struct irdma_cq_poll_info cur_cqe;
176 struct list_head cmpl_generated;
177 };
178
179 struct irdma_cmpl_gen {
180 struct list_head list;
181 struct irdma_cq_poll_info cpi;
182 };
183
184 struct disconn_work {
185 struct work_struct work;
186 struct irdma_qp *iwqp;
187 };
188
189 struct if_notify_work {
190 struct work_struct work;
191 struct irdma_device *iwdev;
192 u32 ipaddr[4];
193 u16 vlan_id;
194 bool ipv4:1;
195 bool ifup:1;
196 };
197
198 struct iw_cm_id;
199
200 struct irdma_qp_kmode {
201 struct irdma_dma_mem dma_mem;
202 u32 *sig_trk_mem;
203 struct irdma_sq_uk_wr_trk_info *sq_wrid_mem;
204 u64 *rq_wrid_mem;
205 };
206
207 struct irdma_qp {
208 struct ib_qp ibqp;
209 struct irdma_sc_qp sc_qp;
210 struct irdma_device *iwdev;
211 struct irdma_cq *iwscq;
212 struct irdma_cq *iwrcq;
213 struct irdma_pd *iwpd;
214 struct irdma_user_mmap_entry *push_wqe_mmap_entry;
215 struct irdma_user_mmap_entry *push_db_mmap_entry;
216 struct irdma_qp_host_ctx_info ctx_info;
217 union {
218 struct irdma_iwarp_offload_info iwarp_info;
219 struct irdma_roce_offload_info roce_info;
220 };
221
222 union {
223 struct irdma_tcp_offload_info tcp_info;
224 struct irdma_udp_offload_info udp_info;
225 };
226
227 struct irdma_ah roce_ah;
228 struct list_head teardown_entry;
229 struct list_head ud_list_elem;
230 atomic_t refcnt;
231 struct iw_cm_id *cm_id;
232 struct irdma_cm_node *cm_node;
233 struct delayed_work dwork_flush;
234 struct ib_mr *lsmm_mr;
235 atomic_t hw_mod_qp_pend;
236 enum ib_qp_state ibqp_state;
237 u32 qp_mem_size;
238 u32 last_aeq;
239 int max_send_wr;
240 int max_recv_wr;
241 atomic_t close_timer_started;
242 spinlock_t lock; /* serialize posting WRs to SQ/RQ */
243 spinlock_t dwork_flush_lock; /* protect mod_delayed_work */
244 struct irdma_qp_context *iwqp_context;
245 void *pbl_vbase;
246 dma_addr_t pbl_pbase;
247 struct page *page;
248 u8 iwarp_state;
249 u16 term_sq_flush_code;
250 u16 term_rq_flush_code;
251 u8 hw_iwarp_state;
252 u8 hw_tcp_state;
253 struct irdma_qp_kmode kqp;
254 struct irdma_dma_mem host_ctx;
255 struct timer_list terminate_timer;
256 struct irdma_pbl *iwpbl;
257 struct ib_sge *sg_list;
258 struct irdma_dma_mem q2_ctx_mem;
259 struct irdma_dma_mem ietf_mem;
260 struct completion free_qp;
261 wait_queue_head_t waitq;
262 wait_queue_head_t mod_qp_waitq;
263 u8 rts_ae_rcvd;
264 bool active_conn:1;
265 bool user_mode:1;
266 bool hte_added:1;
267 bool flush_issued:1;
268 bool sig_all:1;
269 bool pau_mode:1;
270 bool suspend_pending:1;
271 };
272
273 struct irdma_udqs_work {
274 struct work_struct work;
275 struct irdma_qp *iwqp;
276 u8 user_prio;
277 bool qs_change:1;
278 };
279
280 enum irdma_mmap_flag {
281 IRDMA_MMAP_IO_NC,
282 IRDMA_MMAP_IO_WC,
283 };
284
285 struct irdma_user_mmap_entry {
286 struct irdma_ucontext *ucontext;
287 struct hlist_node hlist;
288 u64 pgoff_key; /* Used to compute offset (in bytes) returned to user libc's mmap */
289 u64 bar_offset;
290 u8 mmap_flag;
291 };
292
irdma_fw_major_ver(struct irdma_sc_dev * dev)293 static inline u16 irdma_fw_major_ver(struct irdma_sc_dev *dev)
294 {
295 return (u16)FIELD_GET(IRDMA_FW_VER_MAJOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
296 }
297
irdma_fw_minor_ver(struct irdma_sc_dev * dev)298 static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev)
299 {
300 return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
301 }
302
set_ib_wc_op_sq(struct irdma_cq_poll_info * cq_poll_info,struct ib_wc * entry)303 static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
304 struct ib_wc *entry)
305 {
306 struct irdma_sc_qp *qp;
307
308 switch (cq_poll_info->op_type) {
309 case IRDMA_OP_TYPE_RDMA_WRITE:
310 case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
311 entry->opcode = IB_WC_RDMA_WRITE;
312 break;
313 case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
314 case IRDMA_OP_TYPE_RDMA_READ:
315 entry->opcode = IB_WC_RDMA_READ;
316 break;
317 case IRDMA_OP_TYPE_SEND_SOL:
318 case IRDMA_OP_TYPE_SEND_SOL_INV:
319 case IRDMA_OP_TYPE_SEND_INV:
320 case IRDMA_OP_TYPE_SEND:
321 entry->opcode = IB_WC_SEND;
322 break;
323 case IRDMA_OP_TYPE_FAST_REG_NSMR:
324 entry->opcode = IB_WC_REG_MR;
325 break;
326 case IRDMA_OP_TYPE_INV_STAG:
327 entry->opcode = IB_WC_LOCAL_INV;
328 break;
329 default:
330 qp = cq_poll_info->qp_handle;
331 irdma_dev_err(to_ibdev(qp->dev), "Invalid opcode = %d in CQE\n",
332 cq_poll_info->op_type);
333 entry->status = IB_WC_GENERAL_ERR;
334 }
335 }
336
set_ib_wc_op_rq(struct irdma_cq_poll_info * cq_poll_info,struct ib_wc * entry,bool send_imm_support)337 static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
338 struct ib_wc *entry, bool send_imm_support)
339 {
340 /**
341 * iWARP does not support sendImm, so the presence of Imm data
342 * must be WriteImm.
343 */
344 if (!send_imm_support) {
345 entry->opcode = cq_poll_info->imm_valid ?
346 IB_WC_RECV_RDMA_WITH_IMM :
347 IB_WC_RECV;
348 return;
349 }
350 switch (cq_poll_info->op_type) {
351 case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
352 case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
353 entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
354 break;
355 default:
356 entry->opcode = IB_WC_RECV;
357 }
358 }
359
360 /**
361 * irdma_mcast_mac_v4 - Get the multicast MAC for an IP address
362 * @ip_addr: IPv4 address
363 * @mac: pointer to result MAC address
364 *
365 */
irdma_mcast_mac_v4(u32 * ip_addr,u8 * mac)366 static inline void irdma_mcast_mac_v4(u32 *ip_addr, u8 *mac)
367 {
368 u8 *ip = (u8 *)ip_addr;
369 unsigned char mac4[ETHER_ADDR_LEN] = {0x01, 0x00, 0x5E, ip[2] & 0x7F,
370 ip[1], ip[0]};
371
372 ether_addr_copy(mac, mac4);
373 }
374
375 /**
376 * irdma_mcast_mac_v6 - Get the multicast MAC for an IP address
377 * @ip_addr: IPv6 address
378 * @mac: pointer to result MAC address
379 *
380 */
irdma_mcast_mac_v6(u32 * ip_addr,u8 * mac)381 static inline void irdma_mcast_mac_v6(u32 *ip_addr, u8 *mac)
382 {
383 u8 *ip = (u8 *)ip_addr;
384 unsigned char mac6[ETHER_ADDR_LEN] = {0x33, 0x33, ip[3], ip[2], ip[1],
385 ip[0]};
386
387 ether_addr_copy(mac, mac6);
388 }
389
390 struct irdma_user_mmap_entry *
391 irdma_user_mmap_entry_add_hash(struct irdma_ucontext *ucontext, u64 bar_offset,
392 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset);
393 void irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry);
394 int irdma_ib_register_device(struct irdma_device *iwdev);
395 void irdma_ib_unregister_device(struct irdma_device *iwdev);
396 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event);
397 void irdma_generate_flush_completions(struct irdma_qp *iwqp);
398 void irdma_remove_cmpls_list(struct irdma_cq *iwcq);
399 int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info);
400 void irdma_sched_qp_flush_work(struct irdma_qp *iwqp);
401 void irdma_flush_worker(struct work_struct *work);
402 #endif /* IRDMA_VERBS_H */
403