1
2 /**************************************************************************
3
4 Copyright (c) 2007, Chelsio Inc.
5 All rights reserved.
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
12
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
28
29 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include "opt_inet.h"
34
35 #ifdef TCP_OFFLOAD
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/bus.h>
40 #include <sys/pciio.h>
41 #include <sys/conf.h>
42 #include <machine/bus.h>
43 #include <machine/resource.h>
44 #include <sys/bus_dma.h>
45 #include <sys/rman.h>
46 #include <sys/ioccom.h>
47 #include <sys/mbuf.h>
48 #include <sys/mutex.h>
49 #include <sys/rwlock.h>
50 #include <sys/linker.h>
51 #include <sys/firmware.h>
52 #include <sys/socket.h>
53 #include <sys/sockio.h>
54 #include <sys/smp.h>
55 #include <sys/sysctl.h>
56 #include <sys/syslog.h>
57 #include <sys/queue.h>
58 #include <sys/taskqueue.h>
59 #include <sys/proc.h>
60 #include <sys/queue.h>
61 #include <sys/libkern.h>
62
63 #include <netinet/in.h>
64 #include <rdma/ib_verbs.h>
65 #include <rdma/ib_umem.h>
66 #include <rdma/ib_user_verbs.h>
67 #include <linux/idr.h>
68 #include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
69
70 #include <cxgb_include.h>
71 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
72 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
73 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
74 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
75 #include <ulp/iw_cxgb/iw_cxgb.h>
76 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
77 #include <ulp/iw_cxgb/iw_cxgb_user.h>
78
79 /*
80 * Get one cq entry from cxio and map it to openib.
81 *
82 * Returns:
83 * 0 cqe returned
84 * -ENOBUFS EMPTY;
85 * -EAGAIN caller must try again
86 * any other neg errno fatal error
87 */
iwch_poll_cq_one(struct iwch_dev * rhp,struct iwch_cq * chp,struct ib_wc * wc)88 static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
89 struct ib_wc *wc)
90 {
91 struct iwch_qp *qhp = NULL;
92 struct t3_cqe cqe, *rd_cqe;
93 struct t3_wq *wq;
94 u32 credit = 0;
95 u8 cqe_flushed;
96 u64 cookie;
97 int ret = 1;
98
99 rd_cqe = cxio_next_cqe(&chp->cq);
100
101 if (!rd_cqe)
102 return 0;
103
104 qhp = get_qhp(rhp, CQE_QPID(*rd_cqe));
105 if (!qhp)
106 wq = NULL;
107 else {
108 mtx_lock(&qhp->lock);
109 wq = &(qhp->wq);
110 }
111 ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie,
112 &credit);
113 if (t3a_device(chp->rhp) && credit) {
114 CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__,
115 credit, chp->cq.cqid);
116 cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit);
117 }
118
119 if (ret) {
120 ret = -EAGAIN;
121 goto out;
122 }
123 ret = 1;
124
125 wc->wr_id = cookie;
126 wc->qp = &qhp->ibqp;
127 wc->vendor_err = CQE_STATUS(cqe);
128
129 CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x",
130 CQE_QPID(cqe), CQE_TYPE(cqe),
131 CQE_OPCODE(cqe), CQE_STATUS(cqe));
132 CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx",
133 CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie);
134
135 if (CQE_TYPE(cqe) == 0) {
136 if (!CQE_STATUS(cqe))
137 wc->byte_len = CQE_LEN(cqe);
138 else
139 wc->byte_len = 0;
140 wc->opcode = IB_WC_RECV;
141 } else {
142 switch (CQE_OPCODE(cqe)) {
143 case T3_RDMA_WRITE:
144 wc->opcode = IB_WC_RDMA_WRITE;
145 break;
146 case T3_READ_REQ:
147 wc->opcode = IB_WC_RDMA_READ;
148 wc->byte_len = CQE_LEN(cqe);
149 break;
150 case T3_SEND:
151 case T3_SEND_WITH_SE:
152 wc->opcode = IB_WC_SEND;
153 break;
154 case T3_BIND_MW:
155 wc->opcode = IB_WC_BIND_MW;
156 break;
157
158 /* these aren't supported yet */
159 case T3_SEND_WITH_INV:
160 case T3_SEND_WITH_SE_INV:
161 case T3_LOCAL_INV:
162 case T3_FAST_REGISTER:
163 default:
164 log(LOG_ERR, "Unexpected opcode %d "
165 "in the CQE received for QPID=0x%0x\n",
166 CQE_OPCODE(cqe), CQE_QPID(cqe));
167 ret = -EINVAL;
168 goto out;
169 }
170 }
171
172 if (cqe_flushed)
173 wc->status = IB_WC_WR_FLUSH_ERR;
174 else {
175
176 switch (CQE_STATUS(cqe)) {
177 case TPT_ERR_SUCCESS:
178 wc->status = IB_WC_SUCCESS;
179 break;
180 case TPT_ERR_STAG:
181 wc->status = IB_WC_LOC_ACCESS_ERR;
182 break;
183 case TPT_ERR_PDID:
184 wc->status = IB_WC_LOC_PROT_ERR;
185 break;
186 case TPT_ERR_QPID:
187 case TPT_ERR_ACCESS:
188 wc->status = IB_WC_LOC_ACCESS_ERR;
189 break;
190 case TPT_ERR_WRAP:
191 wc->status = IB_WC_GENERAL_ERR;
192 break;
193 case TPT_ERR_BOUND:
194 wc->status = IB_WC_LOC_LEN_ERR;
195 break;
196 case TPT_ERR_INVALIDATE_SHARED_MR:
197 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:
198 wc->status = IB_WC_MW_BIND_ERR;
199 break;
200 case TPT_ERR_CRC:
201 case TPT_ERR_MARKER:
202 case TPT_ERR_PDU_LEN_ERR:
203 case TPT_ERR_OUT_OF_RQE:
204 case TPT_ERR_DDP_VERSION:
205 case TPT_ERR_RDMA_VERSION:
206 case TPT_ERR_DDP_QUEUE_NUM:
207 case TPT_ERR_MSN:
208 case TPT_ERR_TBIT:
209 case TPT_ERR_MO:
210 case TPT_ERR_MSN_RANGE:
211 case TPT_ERR_IRD_OVERFLOW:
212 case TPT_ERR_OPCODE:
213 wc->status = IB_WC_FATAL_ERR;
214 break;
215 case TPT_ERR_SWFLUSH:
216 wc->status = IB_WC_WR_FLUSH_ERR;
217 break;
218 default:
219 log(LOG_ERR, "Unexpected cqe_status 0x%x for "
220 "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe));
221 ret = -EINVAL;
222 }
223 }
224 out:
225 if (wq)
226 mtx_unlock(&qhp->lock);
227 return ret;
228 }
229
iwch_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)230 int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
231 {
232 struct iwch_dev *rhp;
233 struct iwch_cq *chp;
234 int npolled;
235 int err = 0;
236
237 chp = to_iwch_cq(ibcq);
238 rhp = chp->rhp;
239
240 mtx_lock(&chp->lock);
241 for (npolled = 0; npolled < num_entries; ++npolled) {
242 #ifdef DEBUG
243 int i=0;
244 #endif
245
246 /*
247 * Because T3 can post CQEs that are _not_ associated
248 * with a WR, we might have to poll again after removing
249 * one of these.
250 */
251 do {
252 err = iwch_poll_cq_one(rhp, chp, wc + npolled);
253 #ifdef DEBUG
254 PANIC_IF(++i > 1000);
255 #endif
256 } while (err == -EAGAIN);
257 if (err <= 0)
258 break;
259 }
260 mtx_unlock(&chp->lock);
261
262 if (err < 0) {
263 return err;
264 } else {
265 return npolled;
266 }
267 }
268 #endif
269