1 /*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35
36 #include "common/efx.h"
37
38 #include "sfxge.h"
39
40 static void
sfxge_ev_qcomplete(struct sfxge_evq * evq,boolean_t eop)41 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
42 {
43 struct sfxge_softc *sc;
44 unsigned int index;
45 struct sfxge_rxq *rxq;
46 struct sfxge_txq *txq;
47
48 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
49
50 sc = evq->sc;
51 index = evq->index;
52 rxq = sc->rxq[index];
53
54 if ((txq = evq->txq) != NULL) {
55 evq->txq = NULL;
56 evq->txqs = &(evq->txq);
57
58 do {
59 struct sfxge_txq *next;
60
61 next = txq->next;
62 txq->next = NULL;
63
64 KASSERT(txq->evq_index == index,
65 ("txq->evq_index != index"));
66
67 if (txq->pending != txq->completed)
68 sfxge_tx_qcomplete(txq, evq);
69
70 txq = next;
71 } while (txq != NULL);
72 }
73
74 if (rxq->pending != rxq->completed)
75 sfxge_rx_qcomplete(rxq, eop);
76 }
77
78 static boolean_t
sfxge_ev_rx(void * arg,uint32_t label,uint32_t id,uint32_t size,uint16_t flags)79 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
80 uint16_t flags)
81 {
82 struct sfxge_evq *evq;
83 struct sfxge_softc *sc;
84 struct sfxge_rxq *rxq;
85 unsigned int expected;
86 struct sfxge_rx_sw_desc *rx_desc;
87
88 evq = arg;
89 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
90
91 sc = evq->sc;
92
93 if (evq->exception)
94 goto done;
95
96 rxq = sc->rxq[label];
97 KASSERT(rxq != NULL, ("rxq == NULL"));
98 KASSERT(evq->index == rxq->index,
99 ("evq->index != rxq->index"));
100
101 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
102 goto done;
103
104 expected = rxq->pending++ & rxq->ptr_mask;
105 if (id != expected) {
106 evq->exception = B_TRUE;
107
108 device_printf(sc->dev, "RX completion out of order"
109 " (id=%#x expected=%#x flags=%#x); resetting\n",
110 id, expected, flags);
111 sfxge_schedule_reset(sc);
112
113 goto done;
114 }
115
116 rx_desc = &rxq->queue[id];
117
118 KASSERT(rx_desc->flags == EFX_DISCARD,
119 ("rx_desc->flags != EFX_DISCARD"));
120 rx_desc->flags = flags;
121
122 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
123 rx_desc->size = (uint16_t)size;
124 prefetch_read_many(rx_desc->mbuf);
125
126 evq->rx_done++;
127
128 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
129 sfxge_ev_qcomplete(evq, B_FALSE);
130
131 done:
132 return (evq->rx_done >= SFXGE_EV_BATCH);
133 }
134
135 static boolean_t
sfxge_ev_exception(void * arg,uint32_t code,uint32_t data)136 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
137 {
138 struct sfxge_evq *evq;
139 struct sfxge_softc *sc;
140
141 evq = (struct sfxge_evq *)arg;
142 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
143
144 sc = evq->sc;
145
146 evq->exception = B_TRUE;
147
148 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
149 device_printf(sc->dev,
150 "hardware exception (code=%u); resetting\n",
151 code);
152 sfxge_schedule_reset(sc);
153 }
154
155 return (B_FALSE);
156 }
157
158 static boolean_t
sfxge_ev_rxq_flush_done(void * arg,uint32_t rxq_index)159 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
160 {
161 struct sfxge_evq *evq;
162 struct sfxge_softc *sc;
163 struct sfxge_rxq *rxq;
164 unsigned int index;
165 unsigned int label;
166 uint16_t magic;
167
168 evq = (struct sfxge_evq *)arg;
169 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
170
171 sc = evq->sc;
172 rxq = sc->rxq[rxq_index];
173
174 KASSERT(rxq != NULL, ("rxq == NULL"));
175
176 /* Resend a software event on the correct queue */
177 index = rxq->index;
178 evq = sc->evq[index];
179
180 label = rxq_index;
181 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
182 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
183 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
184
185 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
186 ("evq not started"));
187 efx_ev_qpost(evq->common, magic);
188
189 return (B_FALSE);
190 }
191
192 static boolean_t
sfxge_ev_rxq_flush_failed(void * arg,uint32_t rxq_index)193 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
194 {
195 struct sfxge_evq *evq;
196 struct sfxge_softc *sc;
197 struct sfxge_rxq *rxq;
198 unsigned int index;
199 unsigned int label;
200 uint16_t magic;
201
202 evq = (struct sfxge_evq *)arg;
203 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
204
205 sc = evq->sc;
206 rxq = sc->rxq[rxq_index];
207
208 KASSERT(rxq != NULL, ("rxq == NULL"));
209
210 /* Resend a software event on the correct queue */
211 index = rxq->index;
212 evq = sc->evq[index];
213
214 label = rxq_index;
215 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
216 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
217 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
218
219 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
220 ("evq not started"));
221 efx_ev_qpost(evq->common, magic);
222
223 return (B_FALSE);
224 }
225
226 static struct sfxge_txq *
sfxge_get_txq_by_label(struct sfxge_evq * evq,enum sfxge_txq_type label)227 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
228 {
229 unsigned int index;
230
231 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
232 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
233 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
234 return (evq->sc->txq[index]);
235 }
236
237 static boolean_t
sfxge_ev_tx(void * arg,uint32_t label,uint32_t id)238 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
239 {
240 struct sfxge_evq *evq;
241 struct sfxge_txq *txq;
242 unsigned int stop;
243 unsigned int delta;
244
245 evq = (struct sfxge_evq *)arg;
246 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
247
248 txq = sfxge_get_txq_by_label(evq, label);
249
250 KASSERT(txq != NULL, ("txq == NULL"));
251 KASSERT(evq->index == txq->evq_index,
252 ("evq->index != txq->evq_index"));
253
254 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
255 goto done;
256
257 stop = (id + 1) & txq->ptr_mask;
258 id = txq->pending & txq->ptr_mask;
259
260 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
261 txq->pending += delta;
262
263 evq->tx_done++;
264
265 if (txq->next == NULL &&
266 evq->txqs != &(txq->next)) {
267 *(evq->txqs) = txq;
268 evq->txqs = &(txq->next);
269 }
270
271 if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
272 sfxge_tx_qcomplete(txq, evq);
273
274 done:
275 return (evq->tx_done >= SFXGE_EV_BATCH);
276 }
277
278 static boolean_t
sfxge_ev_txq_flush_done(void * arg,uint32_t txq_index)279 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
280 {
281 struct sfxge_evq *evq;
282 struct sfxge_softc *sc;
283 struct sfxge_txq *txq;
284 unsigned int label;
285 uint16_t magic;
286
287 evq = (struct sfxge_evq *)arg;
288 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
289
290 sc = evq->sc;
291 txq = sc->txq[txq_index];
292
293 KASSERT(txq != NULL, ("txq == NULL"));
294 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
295 ("txq not initialized"));
296
297 /* Resend a software event on the correct queue */
298 evq = sc->evq[txq->evq_index];
299
300 label = txq->type;
301 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
302 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
303 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
304
305 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
306 ("evq not started"));
307 efx_ev_qpost(evq->common, magic);
308
309 return (B_FALSE);
310 }
311
312 static boolean_t
sfxge_ev_software(void * arg,uint16_t magic)313 sfxge_ev_software(void *arg, uint16_t magic)
314 {
315 struct sfxge_evq *evq;
316 struct sfxge_softc *sc;
317 unsigned int label;
318
319 evq = (struct sfxge_evq *)arg;
320 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
321
322 sc = evq->sc;
323
324 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
325 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
326
327 switch (magic) {
328 case SFXGE_MAGIC_RX_QFLUSH_DONE: {
329 struct sfxge_rxq *rxq = sc->rxq[label];
330
331 KASSERT(rxq != NULL, ("rxq == NULL"));
332 KASSERT(evq->index == rxq->index,
333 ("evq->index != rxq->index"));
334
335 sfxge_rx_qflush_done(rxq);
336 break;
337 }
338 case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
339 struct sfxge_rxq *rxq = sc->rxq[label];
340
341 KASSERT(rxq != NULL, ("rxq == NULL"));
342 KASSERT(evq->index == rxq->index,
343 ("evq->index != rxq->index"));
344
345 sfxge_rx_qflush_failed(rxq);
346 break;
347 }
348 case SFXGE_MAGIC_RX_QREFILL: {
349 struct sfxge_rxq *rxq = sc->rxq[label];
350
351 KASSERT(rxq != NULL, ("rxq == NULL"));
352 KASSERT(evq->index == rxq->index,
353 ("evq->index != rxq->index"));
354
355 sfxge_rx_qrefill(rxq);
356 break;
357 }
358 case SFXGE_MAGIC_TX_QFLUSH_DONE: {
359 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
360
361 KASSERT(txq != NULL, ("txq == NULL"));
362 KASSERT(evq->index == txq->evq_index,
363 ("evq->index != txq->evq_index"));
364
365 sfxge_tx_qflush_done(txq);
366 break;
367 }
368 default:
369 break;
370 }
371
372 return (B_FALSE);
373 }
374
375 static boolean_t
sfxge_ev_sram(void * arg,uint32_t code)376 sfxge_ev_sram(void *arg, uint32_t code)
377 {
378 (void)arg;
379 (void)code;
380
381 switch (code) {
382 case EFX_SRAM_UPDATE:
383 EFSYS_PROBE(sram_update);
384 break;
385
386 case EFX_SRAM_CLEAR:
387 EFSYS_PROBE(sram_clear);
388 break;
389
390 case EFX_SRAM_ILLEGAL_CLEAR:
391 EFSYS_PROBE(sram_illegal_clear);
392 break;
393
394 default:
395 KASSERT(B_FALSE, ("Impossible SRAM event"));
396 break;
397 }
398
399 return (B_FALSE);
400 }
401
402 static boolean_t
sfxge_ev_timer(void * arg,uint32_t index)403 sfxge_ev_timer(void *arg, uint32_t index)
404 {
405 (void)arg;
406 (void)index;
407
408 return (B_FALSE);
409 }
410
411 static boolean_t
sfxge_ev_wake_up(void * arg,uint32_t index)412 sfxge_ev_wake_up(void *arg, uint32_t index)
413 {
414 (void)arg;
415 (void)index;
416
417 return (B_FALSE);
418 }
419
420 #if EFSYS_OPT_QSTATS
421
422 static void
sfxge_ev_stat_update(struct sfxge_softc * sc)423 sfxge_ev_stat_update(struct sfxge_softc *sc)
424 {
425 struct sfxge_evq *evq;
426 unsigned int index;
427 clock_t now;
428
429 SFXGE_ADAPTER_LOCK(sc);
430
431 if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
432 goto out;
433
434 now = ticks;
435 if (now - sc->ev_stats_update_time < hz)
436 goto out;
437
438 sc->ev_stats_update_time = now;
439
440 /* Add event counts from each event queue in turn */
441 for (index = 0; index < sc->evq_count; index++) {
442 evq = sc->evq[index];
443 SFXGE_EVQ_LOCK(evq);
444 efx_ev_qstats_update(evq->common, sc->ev_stats);
445 SFXGE_EVQ_UNLOCK(evq);
446 }
447 out:
448 SFXGE_ADAPTER_UNLOCK(sc);
449 }
450
451 static int
sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)452 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
453 {
454 struct sfxge_softc *sc = arg1;
455 unsigned int id = arg2;
456
457 sfxge_ev_stat_update(sc);
458
459 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
460 }
461
462 static void
sfxge_ev_stat_init(struct sfxge_softc * sc)463 sfxge_ev_stat_init(struct sfxge_softc *sc)
464 {
465 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
466 struct sysctl_oid_list *stat_list;
467 unsigned int id;
468 char name[40];
469
470 stat_list = SYSCTL_CHILDREN(sc->stats_node);
471
472 for (id = 0; id < EV_NQSTATS; id++) {
473 snprintf(name, sizeof(name), "ev_%s",
474 efx_ev_qstat_name(sc->enp, id));
475 SYSCTL_ADD_PROC(
476 ctx, stat_list,
477 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
478 sc, id, sfxge_ev_stat_handler, "Q",
479 "");
480 }
481 }
482
483 #endif /* EFSYS_OPT_QSTATS */
484
485 static void
sfxge_ev_qmoderate(struct sfxge_softc * sc,unsigned int idx,unsigned int us)486 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
487 {
488 struct sfxge_evq *evq;
489 efx_evq_t *eep;
490
491 evq = sc->evq[idx];
492 eep = evq->common;
493
494 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
495 ("evq->init_state != SFXGE_EVQ_STARTED"));
496
497 (void)efx_ev_qmoderate(eep, us);
498 }
499
500 static int
sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)501 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
502 {
503 struct sfxge_softc *sc = arg1;
504 struct sfxge_intr *intr = &sc->intr;
505 unsigned int moderation;
506 int error;
507 unsigned int index;
508
509 SFXGE_ADAPTER_LOCK(sc);
510
511 if (req->newptr != NULL) {
512 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
513 != 0)
514 goto out;
515
516 /* We may not be calling efx_ev_qmoderate() now,
517 * so we have to range-check the value ourselves.
518 */
519 if (moderation >
520 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
521 error = EINVAL;
522 goto out;
523 }
524
525 sc->ev_moderation = moderation;
526 if (intr->state == SFXGE_INTR_STARTED) {
527 for (index = 0; index < sc->evq_count; index++)
528 sfxge_ev_qmoderate(sc, index, moderation);
529 }
530 } else {
531 error = SYSCTL_OUT(req, &sc->ev_moderation,
532 sizeof(sc->ev_moderation));
533 }
534
535 out:
536 SFXGE_ADAPTER_UNLOCK(sc);
537
538 return (error);
539 }
540
541 static boolean_t
sfxge_ev_initialized(void * arg)542 sfxge_ev_initialized(void *arg)
543 {
544 struct sfxge_evq *evq;
545
546 evq = (struct sfxge_evq *)arg;
547 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
548
549 KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
550 ("evq not starting"));
551
552 evq->init_state = SFXGE_EVQ_STARTED;
553
554 return (0);
555 }
556
557 static boolean_t
sfxge_ev_link_change(void * arg,efx_link_mode_t link_mode)558 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
559 {
560 struct sfxge_evq *evq;
561 struct sfxge_softc *sc;
562
563 evq = (struct sfxge_evq *)arg;
564 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
565
566 sc = evq->sc;
567
568 sfxge_mac_link_update(sc, link_mode);
569
570 return (0);
571 }
572
573 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
574 .eec_initialized = sfxge_ev_initialized,
575 .eec_rx = sfxge_ev_rx,
576 .eec_tx = sfxge_ev_tx,
577 .eec_exception = sfxge_ev_exception,
578 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done,
579 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed,
580 .eec_txq_flush_done = sfxge_ev_txq_flush_done,
581 .eec_software = sfxge_ev_software,
582 .eec_sram = sfxge_ev_sram,
583 .eec_wake_up = sfxge_ev_wake_up,
584 .eec_timer = sfxge_ev_timer,
585 .eec_link_change = sfxge_ev_link_change,
586 };
587
588
589 int
sfxge_ev_qpoll(struct sfxge_evq * evq)590 sfxge_ev_qpoll(struct sfxge_evq *evq)
591 {
592 int rc;
593
594 SFXGE_EVQ_LOCK(evq);
595
596 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
597 evq->init_state != SFXGE_EVQ_STARTED)) {
598 rc = EINVAL;
599 goto fail;
600 }
601
602 /* Synchronize the DMA memory for reading */
603 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
604 BUS_DMASYNC_POSTREAD);
605
606 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
607 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
608 KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
609 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
610
611 /* Poll the queue */
612 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
613
614 evq->rx_done = 0;
615 evq->tx_done = 0;
616
617 /* Perform any pending completion processing */
618 sfxge_ev_qcomplete(evq, B_TRUE);
619
620 /* Re-prime the event queue for interrupts */
621 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
622 goto fail;
623
624 SFXGE_EVQ_UNLOCK(evq);
625
626 return (0);
627
628 fail:
629 SFXGE_EVQ_UNLOCK(evq);
630 return (rc);
631 }
632
633 static void
sfxge_ev_qstop(struct sfxge_softc * sc,unsigned int index)634 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
635 {
636 struct sfxge_evq *evq;
637
638 evq = sc->evq[index];
639
640 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
641 ("evq->init_state != SFXGE_EVQ_STARTED"));
642
643 SFXGE_EVQ_LOCK(evq);
644 evq->init_state = SFXGE_EVQ_INITIALIZED;
645 evq->read_ptr = 0;
646 evq->exception = B_FALSE;
647
648 #if EFSYS_OPT_QSTATS
649 /* Add event counts before discarding the common evq state */
650 efx_ev_qstats_update(evq->common, sc->ev_stats);
651 #endif
652
653 efx_ev_qdestroy(evq->common);
654 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
655 EFX_EVQ_NBUFS(evq->entries));
656 SFXGE_EVQ_UNLOCK(evq);
657 }
658
659 static int
sfxge_ev_qstart(struct sfxge_softc * sc,unsigned int index)660 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
661 {
662 struct sfxge_evq *evq;
663 efsys_mem_t *esmp;
664 int count;
665 int rc;
666
667 evq = sc->evq[index];
668 esmp = &evq->mem;
669
670 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
671 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
672
673 /* Clear all events. */
674 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
675
676 /* Program the buffer table. */
677 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
678 EFX_EVQ_NBUFS(evq->entries))) != 0)
679 return (rc);
680
681 /* Create the common code event queue. */
682 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
683 evq->buf_base_id, &evq->common)) != 0)
684 goto fail;
685
686 SFXGE_EVQ_LOCK(evq);
687
688 /* Set the default moderation */
689 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
690
691 /* Prime the event queue for interrupts */
692 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
693 goto fail2;
694
695 evq->init_state = SFXGE_EVQ_STARTING;
696
697 SFXGE_EVQ_UNLOCK(evq);
698
699 /* Wait for the initialization event */
700 count = 0;
701 do {
702 /* Pause for 100 ms */
703 pause("sfxge evq init", hz / 10);
704
705 /* Check to see if the test event has been processed */
706 if (evq->init_state == SFXGE_EVQ_STARTED)
707 goto done;
708
709 } while (++count < 20);
710
711 rc = ETIMEDOUT;
712 goto fail3;
713
714 done:
715 return (0);
716
717 fail3:
718 SFXGE_EVQ_LOCK(evq);
719 evq->init_state = SFXGE_EVQ_INITIALIZED;
720 fail2:
721 SFXGE_EVQ_UNLOCK(evq);
722 efx_ev_qdestroy(evq->common);
723 fail:
724 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
725 EFX_EVQ_NBUFS(evq->entries));
726
727 return (rc);
728 }
729
730 void
sfxge_ev_stop(struct sfxge_softc * sc)731 sfxge_ev_stop(struct sfxge_softc *sc)
732 {
733 struct sfxge_intr *intr;
734 efx_nic_t *enp;
735 int index;
736
737 intr = &sc->intr;
738 enp = sc->enp;
739
740 KASSERT(intr->state == SFXGE_INTR_STARTED,
741 ("Interrupts not started"));
742
743 /* Stop the event queue(s) */
744 index = sc->evq_count;
745 while (--index >= 0)
746 sfxge_ev_qstop(sc, index);
747
748 /* Tear down the event module */
749 efx_ev_fini(enp);
750 }
751
752 int
sfxge_ev_start(struct sfxge_softc * sc)753 sfxge_ev_start(struct sfxge_softc *sc)
754 {
755 struct sfxge_intr *intr;
756 int index;
757 int rc;
758
759 intr = &sc->intr;
760
761 KASSERT(intr->state == SFXGE_INTR_STARTED,
762 ("intr->state != SFXGE_INTR_STARTED"));
763
764 /* Initialize the event module */
765 if ((rc = efx_ev_init(sc->enp)) != 0)
766 return (rc);
767
768 /* Start the event queues */
769 for (index = 0; index < sc->evq_count; index++) {
770 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
771 goto fail;
772 }
773
774 return (0);
775
776 fail:
777 /* Stop the event queue(s) */
778 while (--index >= 0)
779 sfxge_ev_qstop(sc, index);
780
781 /* Tear down the event module */
782 efx_ev_fini(sc->enp);
783
784 return (rc);
785 }
786
787 static void
sfxge_ev_qfini(struct sfxge_softc * sc,unsigned int index)788 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
789 {
790 struct sfxge_evq *evq;
791
792 evq = sc->evq[index];
793
794 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
795 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
796 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
797
798 sfxge_dma_free(&evq->mem);
799
800 sc->evq[index] = NULL;
801
802 SFXGE_EVQ_LOCK_DESTROY(evq);
803
804 free(evq, M_SFXGE);
805 }
806
807 static int
sfxge_ev_qinit(struct sfxge_softc * sc,unsigned int index)808 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
809 {
810 struct sfxge_evq *evq;
811 efsys_mem_t *esmp;
812 int rc;
813
814 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
815
816 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
817 evq->sc = sc;
818 evq->index = index;
819 sc->evq[index] = evq;
820 esmp = &evq->mem;
821
822 /* Build an event queue with room for one event per tx and rx buffer,
823 * plus some extra for link state events and MCDI completions.
824 * There are three tx queues in the first event queue and one in
825 * other.
826 */
827 if (index == 0)
828 evq->entries =
829 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
830 3 * sc->txq_entries +
831 128);
832 else
833 evq->entries =
834 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
835 sc->txq_entries +
836 128);
837
838 /* Initialise TX completion list */
839 evq->txqs = &evq->txq;
840
841 /* Allocate DMA space. */
842 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
843 return (rc);
844
845 /* Allocate buffer table entries. */
846 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
847 &evq->buf_base_id);
848
849 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
850
851 evq->init_state = SFXGE_EVQ_INITIALIZED;
852
853 return (0);
854 }
855
856 void
sfxge_ev_fini(struct sfxge_softc * sc)857 sfxge_ev_fini(struct sfxge_softc *sc)
858 {
859 struct sfxge_intr *intr;
860 int index;
861
862 intr = &sc->intr;
863
864 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
865 ("intr->state != SFXGE_INTR_INITIALIZED"));
866
867 sc->ev_moderation = 0;
868
869 /* Tear down the event queue(s). */
870 index = sc->evq_count;
871 while (--index >= 0)
872 sfxge_ev_qfini(sc, index);
873
874 sc->evq_count = 0;
875 }
876
877 int
sfxge_ev_init(struct sfxge_softc * sc)878 sfxge_ev_init(struct sfxge_softc *sc)
879 {
880 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
881 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
882 struct sfxge_intr *intr;
883 int index;
884 int rc;
885
886 intr = &sc->intr;
887
888 sc->evq_count = intr->n_alloc;
889
890 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
891 ("intr->state != SFXGE_INTR_INITIALIZED"));
892
893 /* Set default interrupt moderation; add a sysctl to
894 * read and change it.
895 */
896 sc->ev_moderation = SFXGE_MODERATION;
897 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
898 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
899 sc, 0, sfxge_int_mod_handler, "IU",
900 "sfxge interrupt moderation (us)");
901
902 /*
903 * Initialize the event queue(s) - one per interrupt.
904 */
905 for (index = 0; index < sc->evq_count; index++) {
906 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
907 goto fail;
908 }
909
910 #if EFSYS_OPT_QSTATS
911 sfxge_ev_stat_init(sc);
912 #endif
913
914 return (0);
915
916 fail:
917 while (--index >= 0)
918 sfxge_ev_qfini(sc, index);
919
920 sc->evq_count = 0;
921 return (rc);
922 }
923