1 /*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/rman.h>
36 #include <sys/smp.h>
37 #include <sys/syslog.h>
38
39 #include <machine/bus.h>
40 #include <machine/resource.h>
41
42 #include <dev/pci/pcireg.h>
43 #include <dev/pci/pcivar.h>
44
45 #include "common/efx.h"
46
47 #include "sfxge.h"
48
49 static int
sfxge_intr_line_filter(void * arg)50 sfxge_intr_line_filter(void *arg)
51 {
52 struct sfxge_evq *evq;
53 struct sfxge_softc *sc;
54 efx_nic_t *enp;
55 struct sfxge_intr *intr;
56 boolean_t fatal;
57 uint32_t qmask;
58
59 evq = (struct sfxge_evq *)arg;
60 sc = evq->sc;
61 enp = sc->enp;
62 intr = &sc->intr;
63
64 KASSERT(intr != NULL, ("intr == NULL"));
65 KASSERT(intr->type == EFX_INTR_LINE,
66 ("intr->type != EFX_INTR_LINE"));
67
68 if (intr->state != SFXGE_INTR_STARTED)
69 return (FILTER_STRAY);
70
71 (void)efx_intr_status_line(enp, &fatal, &qmask);
72
73 if (fatal) {
74 (void) efx_intr_disable(enp);
75 (void) efx_intr_fatal(enp);
76 return (FILTER_HANDLED);
77 }
78
79 if (qmask != 0) {
80 intr->zero_count = 0;
81 return (FILTER_SCHEDULE_THREAD);
82 }
83
84 /* SF bug 15783: If the function is not asserting its IRQ and
85 * we read the queue mask on the cycle before a flag is added
86 * to the mask, this inhibits the function from asserting the
87 * IRQ even though we don't see the flag set. To work around
88 * this, we must re-prime all event queues and report the IRQ
89 * as handled when we see a mask of zero. To allow for shared
90 * IRQs, we don't repeat this if we see a mask of zero twice
91 * or more in a row.
92 */
93 if (intr->zero_count++ == 0) {
94 if (evq->init_state == SFXGE_EVQ_STARTED) {
95 if (efx_ev_qpending(evq->common, evq->read_ptr))
96 return (FILTER_SCHEDULE_THREAD);
97 efx_ev_qprime(evq->common, evq->read_ptr);
98 return (FILTER_HANDLED);
99 }
100 }
101
102 return (FILTER_STRAY);
103 }
104
105 static void
sfxge_intr_line(void * arg)106 sfxge_intr_line(void *arg)
107 {
108 struct sfxge_evq *evq = arg;
109
110 (void)sfxge_ev_qpoll(evq);
111 }
112
113 static void
sfxge_intr_message(void * arg)114 sfxge_intr_message(void *arg)
115 {
116 struct sfxge_evq *evq;
117 struct sfxge_softc *sc;
118 efx_nic_t *enp;
119 struct sfxge_intr *intr;
120 unsigned int index;
121 boolean_t fatal;
122
123 evq = (struct sfxge_evq *)arg;
124 sc = evq->sc;
125 enp = sc->enp;
126 intr = &sc->intr;
127 index = evq->index;
128
129 KASSERT(intr != NULL, ("intr == NULL"));
130 KASSERT(intr->type == EFX_INTR_MESSAGE,
131 ("intr->type != EFX_INTR_MESSAGE"));
132
133 if (__predict_false(intr->state != SFXGE_INTR_STARTED))
134 return;
135
136 (void)efx_intr_status_message(enp, index, &fatal);
137
138 if (fatal) {
139 (void)efx_intr_disable(enp);
140 (void)efx_intr_fatal(enp);
141 return;
142 }
143
144 (void)sfxge_ev_qpoll(evq);
145 }
146
147 static int
sfxge_intr_bus_enable(struct sfxge_softc * sc)148 sfxge_intr_bus_enable(struct sfxge_softc *sc)
149 {
150 struct sfxge_intr *intr;
151 struct sfxge_intr_hdl *table;
152 driver_filter_t *filter;
153 driver_intr_t *handler;
154 int index;
155 int err;
156
157 intr = &sc->intr;
158 table = intr->table;
159
160 switch (intr->type) {
161 case EFX_INTR_MESSAGE:
162 filter = NULL; /* not shared */
163 handler = sfxge_intr_message;
164 break;
165
166 case EFX_INTR_LINE:
167 filter = sfxge_intr_line_filter;
168 handler = sfxge_intr_line;
169 break;
170
171 default:
172 KASSERT(0, ("Invalid interrupt type"));
173 return (EINVAL);
174 }
175
176 /* Try to add the handlers */
177 for (index = 0; index < intr->n_alloc; index++) {
178 if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
179 INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
180 sc->evq[index], &table[index].eih_tag)) != 0) {
181 goto fail;
182 }
183 #ifdef SFXGE_HAVE_DESCRIBE_INTR
184 if (intr->n_alloc > 1)
185 bus_describe_intr(sc->dev, table[index].eih_res,
186 table[index].eih_tag, "%d", index);
187 #endif
188 bus_bind_intr(sc->dev, table[index].eih_res, index);
189
190 }
191
192 return (0);
193
194 fail:
195 /* Remove remaining handlers */
196 while (--index >= 0)
197 bus_teardown_intr(sc->dev, table[index].eih_res,
198 table[index].eih_tag);
199
200 return (err);
201 }
202
203 static void
sfxge_intr_bus_disable(struct sfxge_softc * sc)204 sfxge_intr_bus_disable(struct sfxge_softc *sc)
205 {
206 struct sfxge_intr *intr;
207 struct sfxge_intr_hdl *table;
208 int i;
209
210 intr = &sc->intr;
211 table = intr->table;
212
213 /* Remove all handlers */
214 for (i = 0; i < intr->n_alloc; i++)
215 bus_teardown_intr(sc->dev, table[i].eih_res,
216 table[i].eih_tag);
217 }
218
219 static int
sfxge_intr_alloc(struct sfxge_softc * sc,int count)220 sfxge_intr_alloc(struct sfxge_softc *sc, int count)
221 {
222 device_t dev;
223 struct sfxge_intr_hdl *table;
224 struct sfxge_intr *intr;
225 struct resource *res;
226 int rid;
227 int error;
228 int i;
229
230 dev = sc->dev;
231 intr = &sc->intr;
232 error = 0;
233
234 table = malloc(count * sizeof(struct sfxge_intr_hdl),
235 M_SFXGE, M_WAITOK);
236 intr->table = table;
237
238 for (i = 0; i < count; i++) {
239 rid = i + 1;
240 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
241 RF_SHAREABLE | RF_ACTIVE);
242 if (res == NULL) {
243 device_printf(dev, "Couldn't allocate interrupts for "
244 "message %d\n", rid);
245 error = ENOMEM;
246 break;
247 }
248 table[i].eih_rid = rid;
249 table[i].eih_res = res;
250 }
251
252 if (error != 0) {
253 count = i - 1;
254 for (i = 0; i < count; i++)
255 bus_release_resource(dev, SYS_RES_IRQ,
256 table[i].eih_rid, table[i].eih_res);
257 }
258
259 return (error);
260 }
261
262 static void
sfxge_intr_teardown_msix(struct sfxge_softc * sc)263 sfxge_intr_teardown_msix(struct sfxge_softc *sc)
264 {
265 device_t dev;
266 struct resource *resp;
267 int rid;
268
269 dev = sc->dev;
270 resp = sc->intr.msix_res;
271
272 rid = rman_get_rid(resp);
273 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
274 }
275
276 static int
sfxge_intr_setup_msix(struct sfxge_softc * sc)277 sfxge_intr_setup_msix(struct sfxge_softc *sc)
278 {
279 struct sfxge_intr *intr;
280 struct resource *resp;
281 device_t dev;
282 int count;
283 int rid;
284
285 dev = sc->dev;
286 intr = &sc->intr;
287
288 /* Check if MSI-X is available. */
289 count = pci_msix_count(dev);
290 if (count == 0)
291 return (EINVAL);
292
293 /* Limit the number of interrupts to the number of CPUs. */
294 if (count > mp_ncpus)
295 count = mp_ncpus;
296
297 /* Not very likely these days... */
298 if (count > EFX_MAXRSS)
299 count = EFX_MAXRSS;
300
301 if (sc->max_rss_channels > 0 && count > sc->max_rss_channels)
302 count = sc->max_rss_channels;
303
304 rid = PCIR_BAR(4);
305 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
306 if (resp == NULL)
307 return (ENOMEM);
308
309 if (pci_alloc_msix(dev, &count) != 0) {
310 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
311 return (ENOMEM);
312 }
313
314 /* Allocate interrupt handlers. */
315 if (sfxge_intr_alloc(sc, count) != 0) {
316 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
317 pci_release_msi(dev);
318 return (ENOMEM);
319 }
320
321 intr->type = EFX_INTR_MESSAGE;
322 intr->n_alloc = count;
323 intr->msix_res = resp;
324
325 return (0);
326 }
327
328 static int
sfxge_intr_setup_msi(struct sfxge_softc * sc)329 sfxge_intr_setup_msi(struct sfxge_softc *sc)
330 {
331 struct sfxge_intr_hdl *table;
332 struct sfxge_intr *intr;
333 device_t dev;
334 int count;
335 int error;
336
337 dev = sc->dev;
338 intr = &sc->intr;
339 table = intr->table;
340
341 /*
342 * Check if MSI is available. All messages must be written to
343 * the same address and on x86 this means the IRQs have the
344 * same CPU affinity. So we only ever allocate 1.
345 */
346 count = pci_msi_count(dev) ? 1 : 0;
347 if (count == 0)
348 return (EINVAL);
349
350 if ((error = pci_alloc_msi(dev, &count)) != 0)
351 return (ENOMEM);
352
353 /* Allocate interrupt handler. */
354 if (sfxge_intr_alloc(sc, count) != 0) {
355 pci_release_msi(dev);
356 return (ENOMEM);
357 }
358
359 intr->type = EFX_INTR_MESSAGE;
360 intr->n_alloc = count;
361
362 return (0);
363 }
364
365 static int
sfxge_intr_setup_fixed(struct sfxge_softc * sc)366 sfxge_intr_setup_fixed(struct sfxge_softc *sc)
367 {
368 struct sfxge_intr_hdl *table;
369 struct sfxge_intr *intr;
370 struct resource *res;
371 device_t dev;
372 int rid;
373
374 dev = sc->dev;
375 intr = &sc->intr;
376
377 rid = 0;
378 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
379 RF_SHAREABLE | RF_ACTIVE);
380 if (res == NULL)
381 return (ENOMEM);
382
383 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
384 table[0].eih_rid = rid;
385 table[0].eih_res = res;
386
387 intr->type = EFX_INTR_LINE;
388 intr->n_alloc = 1;
389 intr->table = table;
390
391 return (0);
392 }
393
394 static const char *const __sfxge_err[] = {
395 "",
396 "SRAM out-of-bounds",
397 "Buffer ID out-of-bounds",
398 "Internal memory parity",
399 "Receive buffer ownership",
400 "Transmit buffer ownership",
401 "Receive descriptor ownership",
402 "Transmit descriptor ownership",
403 "Event queue ownership",
404 "Event queue FIFO overflow",
405 "Illegal address",
406 "SRAM parity"
407 };
408
409 void
sfxge_err(efsys_identifier_t * arg,unsigned int code,uint32_t dword0,uint32_t dword1)410 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
411 uint32_t dword1)
412 {
413 struct sfxge_softc *sc = (struct sfxge_softc *)arg;
414 device_t dev = sc->dev;
415
416 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
417 device_get_name(dev), device_get_unit(dev),
418 __sfxge_err[code], dword1, dword0);
419 }
420
421 void
sfxge_intr_stop(struct sfxge_softc * sc)422 sfxge_intr_stop(struct sfxge_softc *sc)
423 {
424 struct sfxge_intr *intr;
425
426 intr = &sc->intr;
427
428 KASSERT(intr->state == SFXGE_INTR_STARTED,
429 ("Interrupts not started"));
430
431 intr->state = SFXGE_INTR_INITIALIZED;
432
433 /* Disable interrupts at the NIC */
434 efx_intr_disable(sc->enp);
435
436 /* Disable interrupts at the bus */
437 sfxge_intr_bus_disable(sc);
438
439 /* Tear down common code interrupt bits. */
440 efx_intr_fini(sc->enp);
441 }
442
443 int
sfxge_intr_start(struct sfxge_softc * sc)444 sfxge_intr_start(struct sfxge_softc *sc)
445 {
446 struct sfxge_intr *intr;
447 efsys_mem_t *esmp;
448 int rc;
449
450 intr = &sc->intr;
451 esmp = &intr->status;
452
453 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
454 ("Interrupts not initialized"));
455
456 /* Zero the memory. */
457 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);
458
459 /* Initialize common code interrupt bits. */
460 (void)efx_intr_init(sc->enp, intr->type, esmp);
461
462 /* Enable interrupts at the bus */
463 if ((rc = sfxge_intr_bus_enable(sc)) != 0)
464 goto fail;
465
466 intr->state = SFXGE_INTR_STARTED;
467
468 /* Enable interrupts at the NIC */
469 efx_intr_enable(sc->enp);
470
471 return (0);
472
473 fail:
474 /* Tear down common code interrupt bits. */
475 efx_intr_fini(sc->enp);
476
477 intr->state = SFXGE_INTR_INITIALIZED;
478
479 return (rc);
480 }
481
482 void
sfxge_intr_fini(struct sfxge_softc * sc)483 sfxge_intr_fini(struct sfxge_softc *sc)
484 {
485 struct sfxge_intr_hdl *table;
486 struct sfxge_intr *intr;
487 efsys_mem_t *esmp;
488 device_t dev;
489 int i;
490
491 dev = sc->dev;
492 intr = &sc->intr;
493 esmp = &intr->status;
494 table = intr->table;
495
496 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
497 ("intr->state != SFXGE_INTR_INITIALIZED"));
498
499 /* Free DMA memory. */
500 sfxge_dma_free(esmp);
501
502 /* Free interrupt handles. */
503 for (i = 0; i < intr->n_alloc; i++)
504 bus_release_resource(dev, SYS_RES_IRQ,
505 table[i].eih_rid, table[i].eih_res);
506
507 if (table[0].eih_rid != 0)
508 pci_release_msi(dev);
509
510 if (intr->msix_res != NULL)
511 sfxge_intr_teardown_msix(sc);
512
513 /* Free the handle table */
514 free(table, M_SFXGE);
515 intr->table = NULL;
516 intr->n_alloc = 0;
517
518 /* Clear the interrupt type */
519 intr->type = EFX_INTR_INVALID;
520
521 intr->state = SFXGE_INTR_UNINITIALIZED;
522 }
523
524 int
sfxge_intr_init(struct sfxge_softc * sc)525 sfxge_intr_init(struct sfxge_softc *sc)
526 {
527 device_t dev;
528 struct sfxge_intr *intr;
529 efsys_mem_t *esmp;
530 int rc;
531
532 dev = sc->dev;
533 intr = &sc->intr;
534 esmp = &intr->status;
535
536 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
537 ("Interrupts already initialized"));
538
539 /* Try to setup MSI-X or MSI interrupts if available. */
540 if ((rc = sfxge_intr_setup_msix(sc)) == 0)
541 device_printf(dev, "Using MSI-X interrupts\n");
542 else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
543 device_printf(dev, "Using MSI interrupts\n");
544 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
545 device_printf(dev, "Using fixed interrupts\n");
546 } else {
547 device_printf(dev, "Couldn't setup interrupts\n");
548 return (ENOMEM);
549 }
550
551 /* Set up DMA for interrupts. */
552 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
553 return (ENOMEM);
554
555 intr->state = SFXGE_INTR_INITIALIZED;
556
557 return (0);
558 }
559