1 /*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #ifndef MTHCA_H
35 #define MTHCA_H
36
37 #include <stddef.h>
38
39 #include <infiniband/driver.h>
40 #include <infiniband/arch.h>
41
42 #ifdef HAVE_VALGRIND_MEMCHECK_H
43
44 # include <valgrind/memcheck.h>
45
46 # if !defined(VALGRIND_MAKE_MEM_DEFINED) || !defined(VALGRIND_MAKE_MEM_UNDEFINED)
47 # warning "Valgrind support requested, but VALGRIND_MAKE_MEM_(UN)DEFINED not available"
48 # endif
49
50 #endif /* HAVE_VALGRIND_MEMCHECK_H */
51
52 #ifndef VALGRIND_MAKE_MEM_DEFINED
53 # define VALGRIND_MAKE_MEM_DEFINED(addr,len)
54 #endif
55
56 #ifndef VALGRIND_MAKE_MEM_UNDEFINED
57 # define VALGRIND_MAKE_MEM_UNDEFINED(addr,len)
58 #endif
59
60 #ifndef rmb
61 # define rmb() mb()
62 #endif
63
64 #ifndef wmb
65 # define wmb() mb()
66 #endif
67
68 #define HIDDEN __attribute__((visibility ("hidden")))
69
70 #define PFX "mthca: "
71
72 enum mthca_hca_type {
73 MTHCA_TAVOR,
74 MTHCA_ARBEL
75 };
76
77 enum {
78 MTHCA_CQ_ENTRY_SIZE = 0x20
79 };
80
81 enum {
82 MTHCA_QP_TABLE_BITS = 8,
83 MTHCA_QP_TABLE_SIZE = 1 << MTHCA_QP_TABLE_BITS,
84 MTHCA_QP_TABLE_MASK = MTHCA_QP_TABLE_SIZE - 1
85 };
86
87 enum {
88 MTHCA_DB_REC_PAGE_SIZE = 4096,
89 MTHCA_DB_REC_PER_PAGE = MTHCA_DB_REC_PAGE_SIZE / 8
90 };
91
92 enum mthca_db_type {
93 MTHCA_DB_TYPE_INVALID = 0x0,
94 MTHCA_DB_TYPE_CQ_SET_CI = 0x1,
95 MTHCA_DB_TYPE_CQ_ARM = 0x2,
96 MTHCA_DB_TYPE_SQ = 0x3,
97 MTHCA_DB_TYPE_RQ = 0x4,
98 MTHCA_DB_TYPE_SRQ = 0x5,
99 MTHCA_DB_TYPE_GROUP_SEP = 0x7
100 };
101
102 enum {
103 MTHCA_OPCODE_NOP = 0x00,
104 MTHCA_OPCODE_RDMA_WRITE = 0x08,
105 MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09,
106 MTHCA_OPCODE_SEND = 0x0a,
107 MTHCA_OPCODE_SEND_IMM = 0x0b,
108 MTHCA_OPCODE_RDMA_READ = 0x10,
109 MTHCA_OPCODE_ATOMIC_CS = 0x11,
110 MTHCA_OPCODE_ATOMIC_FA = 0x12,
111 MTHCA_OPCODE_BIND_MW = 0x18,
112 MTHCA_OPCODE_INVALID = 0xff
113 };
114
115 struct mthca_ah_page;
116
117 struct mthca_device {
118 struct ibv_device ibv_dev;
119 enum mthca_hca_type hca_type;
120 int page_size;
121 };
122
123 struct mthca_db_table;
124
125 struct mthca_context {
126 struct ibv_context ibv_ctx;
127 void *uar;
128 pthread_spinlock_t uar_lock;
129 struct mthca_db_table *db_tab;
130 struct ibv_pd *pd;
131 struct {
132 struct mthca_qp **table;
133 int refcnt;
134 } qp_table[MTHCA_QP_TABLE_SIZE];
135 pthread_mutex_t qp_table_mutex;
136 int num_qps;
137 int qp_table_shift;
138 int qp_table_mask;
139 };
140
141 struct mthca_buf {
142 void *buf;
143 size_t length;
144 };
145
146 struct mthca_pd {
147 struct ibv_pd ibv_pd;
148 struct mthca_ah_page *ah_list;
149 pthread_mutex_t ah_mutex;
150 uint32_t pdn;
151 };
152
153 struct mthca_cq {
154 struct ibv_cq ibv_cq;
155 struct mthca_buf buf;
156 pthread_spinlock_t lock;
157 struct ibv_mr *mr;
158 uint32_t cqn;
159 uint32_t cons_index;
160
161 /* Next fields are mem-free only */
162 int set_ci_db_index;
163 uint32_t *set_ci_db;
164 int arm_db_index;
165 uint32_t *arm_db;
166 int arm_sn;
167 };
168
169 struct mthca_srq {
170 struct ibv_srq ibv_srq;
171 struct mthca_buf buf;
172 void *last;
173 pthread_spinlock_t lock;
174 struct ibv_mr *mr;
175 uint64_t *wrid;
176 uint32_t srqn;
177 int max;
178 int max_gs;
179 int wqe_shift;
180 int first_free;
181 int last_free;
182 int buf_size;
183
184 /* Next fields are mem-free only */
185 int db_index;
186 uint32_t *db;
187 uint16_t counter;
188 };
189
190 struct mthca_wq {
191 pthread_spinlock_t lock;
192 int max;
193 unsigned next_ind;
194 unsigned last_comp;
195 unsigned head;
196 unsigned tail;
197 void *last;
198 int max_gs;
199 int wqe_shift;
200
201 /* Next fields are mem-free only */
202 int db_index;
203 uint32_t *db;
204 };
205
206 struct mthca_qp {
207 struct ibv_qp ibv_qp;
208 struct mthca_buf buf;
209 uint64_t *wrid;
210 int send_wqe_offset;
211 int max_inline_data;
212 int buf_size;
213 struct mthca_wq sq;
214 struct mthca_wq rq;
215 struct ibv_mr *mr;
216 int sq_sig_all;
217 };
218
219 struct mthca_av {
220 uint32_t port_pd;
221 uint8_t reserved1;
222 uint8_t g_slid;
223 uint16_t dlid;
224 uint8_t reserved2;
225 uint8_t gid_index;
226 uint8_t msg_sr;
227 uint8_t hop_limit;
228 uint32_t sl_tclass_flowlabel;
229 uint32_t dgid[4];
230 };
231
232 struct mthca_ah {
233 struct ibv_ah ibv_ah;
234 struct mthca_av *av;
235 struct mthca_ah_page *page;
236 uint32_t key;
237 };
238
align(unsigned long val,unsigned long align)239 static inline unsigned long align(unsigned long val, unsigned long align)
240 {
241 return (val + align - 1) & ~(align - 1);
242 }
243
db_align(uint32_t * db)244 static inline uintptr_t db_align(uint32_t *db)
245 {
246 return (uintptr_t) db & ~((uintptr_t) MTHCA_DB_REC_PAGE_SIZE - 1);
247 }
248
249 #define to_mxxx(xxx, type) \
250 ((struct mthca_##type *) \
251 ((void *) ib##xxx - offsetof(struct mthca_##type, ibv_##xxx)))
252
to_mdev(struct ibv_device * ibdev)253 static inline struct mthca_device *to_mdev(struct ibv_device *ibdev)
254 {
255 return to_mxxx(dev, device);
256 }
257
to_mctx(struct ibv_context * ibctx)258 static inline struct mthca_context *to_mctx(struct ibv_context *ibctx)
259 {
260 return to_mxxx(ctx, context);
261 }
262
to_mpd(struct ibv_pd * ibpd)263 static inline struct mthca_pd *to_mpd(struct ibv_pd *ibpd)
264 {
265 return to_mxxx(pd, pd);
266 }
267
to_mcq(struct ibv_cq * ibcq)268 static inline struct mthca_cq *to_mcq(struct ibv_cq *ibcq)
269 {
270 return to_mxxx(cq, cq);
271 }
272
to_msrq(struct ibv_srq * ibsrq)273 static inline struct mthca_srq *to_msrq(struct ibv_srq *ibsrq)
274 {
275 return to_mxxx(srq, srq);
276 }
277
to_mqp(struct ibv_qp * ibqp)278 static inline struct mthca_qp *to_mqp(struct ibv_qp *ibqp)
279 {
280 return to_mxxx(qp, qp);
281 }
282
to_mah(struct ibv_ah * ibah)283 static inline struct mthca_ah *to_mah(struct ibv_ah *ibah)
284 {
285 return to_mxxx(ah, ah);
286 }
287
mthca_is_memfree(struct ibv_context * ibctx)288 static inline int mthca_is_memfree(struct ibv_context *ibctx)
289 {
290 return to_mdev(ibctx->device)->hca_type == MTHCA_ARBEL;
291 }
292
293 int mthca_alloc_buf(struct mthca_buf *buf, size_t size, int page_size);
294 void mthca_free_buf(struct mthca_buf *buf);
295
296 int mthca_alloc_db(struct mthca_db_table *db_tab, enum mthca_db_type type,
297 uint32_t **db);
298 void mthca_set_db_qn(uint32_t *db, enum mthca_db_type type, uint32_t qn);
299 void mthca_free_db(struct mthca_db_table *db_tab, enum mthca_db_type type, int db_index);
300 struct mthca_db_table *mthca_alloc_db_tab(int uarc_size);
301 void mthca_free_db_tab(struct mthca_db_table *db_tab);
302
303 int mthca_query_device(struct ibv_context *context,
304 struct ibv_device_attr *attr);
305 int mthca_query_port(struct ibv_context *context, uint8_t port,
306 struct ibv_port_attr *attr);
307
308 struct ibv_pd *mthca_alloc_pd(struct ibv_context *context);
309 int mthca_free_pd(struct ibv_pd *pd);
310
311 struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr,
312 size_t length, enum ibv_access_flags access);
313 int mthca_dereg_mr(struct ibv_mr *mr);
314
315 struct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe,
316 struct ibv_comp_channel *channel,
317 int comp_vector);
318 int mthca_resize_cq(struct ibv_cq *cq, int cqe);
319 int mthca_destroy_cq(struct ibv_cq *cq);
320 int mthca_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
321 int mthca_tavor_arm_cq(struct ibv_cq *cq, int solicited);
322 int mthca_arbel_arm_cq(struct ibv_cq *cq, int solicited);
323 void mthca_arbel_cq_event(struct ibv_cq *cq);
324 void __mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq);
325 void mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq);
326 void mthca_cq_resize_copy_cqes(struct mthca_cq *cq, void *buf, int new_cqe);
327 int mthca_alloc_cq_buf(struct mthca_device *dev, struct mthca_buf *buf, int nent);
328
329 struct ibv_srq *mthca_create_srq(struct ibv_pd *pd,
330 struct ibv_srq_init_attr *attr);
331 int mthca_modify_srq(struct ibv_srq *srq,
332 struct ibv_srq_attr *attr,
333 enum ibv_srq_attr_mask mask);
334 int mthca_query_srq(struct ibv_srq *srq,
335 struct ibv_srq_attr *attr);
336 int mthca_destroy_srq(struct ibv_srq *srq);
337 int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
338 struct mthca_srq *srq);
339 void mthca_free_srq_wqe(struct mthca_srq *srq, int ind);
340 int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq,
341 struct ibv_recv_wr *wr,
342 struct ibv_recv_wr **bad_wr);
343 int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq,
344 struct ibv_recv_wr *wr,
345 struct ibv_recv_wr **bad_wr);
346
347 struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
348 int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
349 enum ibv_qp_attr_mask attr_mask,
350 struct ibv_qp_init_attr *init_attr);
351 int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
352 enum ibv_qp_attr_mask attr_mask);
353 int mthca_destroy_qp(struct ibv_qp *qp);
354 void mthca_init_qp_indices(struct mthca_qp *qp);
355 int mthca_tavor_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
356 struct ibv_send_wr **bad_wr);
357 int mthca_tavor_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
358 struct ibv_recv_wr **bad_wr);
359 int mthca_arbel_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
360 struct ibv_send_wr **bad_wr);
361 int mthca_arbel_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
362 struct ibv_recv_wr **bad_wr);
363 int mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
364 enum ibv_qp_type type, struct mthca_qp *qp);
365 struct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn);
366 int mthca_store_qp(struct mthca_context *ctx, uint32_t qpn, struct mthca_qp *qp);
367 void mthca_clear_qp(struct mthca_context *ctx, uint32_t qpn);
368 int mthca_free_err_wqe(struct mthca_qp *qp, int is_send,
369 int index, int *dbd, uint32_t *new_wqe);
370 struct ibv_ah *mthca_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
371 int mthca_destroy_ah(struct ibv_ah *ah);
372 int mthca_alloc_av(struct mthca_pd *pd, struct ibv_ah_attr *attr,
373 struct mthca_ah *ah);
374 void mthca_free_av(struct mthca_ah *ah);
375 int mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
376 int mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
377
378 #endif /* MTHCA_H */
379