1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2010-2016 Solarflare Communications Inc.
5 * All rights reserved.
6 *
7 * This software was developed in part by Philip Paeps under contract for
8 * Solarflare Communications, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * The views and conclusions contained in the software and documentation are
32 * those of the authors and should not be interpreted as representing official
33 * policies, either expressed or implied, of the FreeBSD Project.
34 */
35
36 #include <sys/cdefs.h>
37 #include <sys/param.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/param.h>
41 #include <sys/queue.h>
42 #include <sys/systm.h>
43 #include <sys/taskqueue.h>
44
45 #include "common/efx.h"
46
47 #include "sfxge.h"
48
49 static void
sfxge_ev_qcomplete(struct sfxge_evq * evq,boolean_t eop)50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
51 {
52 struct sfxge_softc *sc;
53 unsigned int index;
54 struct sfxge_rxq *rxq;
55 struct sfxge_txq *txq;
56
57 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
58
59 sc = evq->sc;
60 index = evq->index;
61 rxq = sc->rxq[index];
62
63 if ((txq = evq->txq) != NULL) {
64 evq->txq = NULL;
65 evq->txqs = &(evq->txq);
66
67 do {
68 struct sfxge_txq *next;
69
70 next = txq->next;
71 txq->next = NULL;
72
73 KASSERT(txq->evq_index == index,
74 ("txq->evq_index != index"));
75
76 if (txq->pending != txq->completed)
77 sfxge_tx_qcomplete(txq, evq);
78
79 txq = next;
80 } while (txq != NULL);
81 }
82
83 if (rxq->pending != rxq->completed)
84 sfxge_rx_qcomplete(rxq, eop);
85 }
86
87 static struct sfxge_rxq *
sfxge_get_rxq_by_label(struct sfxge_evq * evq,uint32_t label)88 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
89 {
90 struct sfxge_rxq *rxq;
91
92 KASSERT(label == 0, ("unexpected rxq label != 0"));
93
94 rxq = evq->sc->rxq[evq->index];
95
96 KASSERT(rxq != NULL, ("rxq == NULL"));
97 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
98
99 return (rxq);
100 }
101
102 static boolean_t
sfxge_ev_rx(void * arg,uint32_t label,uint32_t id,uint32_t size,uint16_t flags)103 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
104 uint16_t flags)
105 {
106 struct sfxge_evq *evq;
107 struct sfxge_softc *sc;
108 struct sfxge_rxq *rxq;
109 unsigned int stop;
110 unsigned int delta;
111 struct sfxge_rx_sw_desc *rx_desc;
112
113 evq = arg;
114 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
115
116 sc = evq->sc;
117
118 if (evq->exception)
119 goto done;
120
121 rxq = sfxge_get_rxq_by_label(evq, label);
122 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
123 goto done;
124
125 stop = (id + 1) & rxq->ptr_mask;
126 id = rxq->pending & rxq->ptr_mask;
127 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
128 rxq->pending += delta;
129
130 if (delta != 1) {
131 if ((delta <= 0) ||
132 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
133 evq->exception = B_TRUE;
134
135 device_printf(sc->dev, "RX completion out of order"
136 " (id=%#x delta=%u flags=%#x); resetting\n",
137 id, delta, flags);
138 sfxge_schedule_reset(sc);
139
140 goto done;
141 }
142 }
143
144 rx_desc = &rxq->queue[id];
145
146 prefetch_read_many(rx_desc->mbuf);
147
148 for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
149 rx_desc = &rxq->queue[id];
150 KASSERT(rx_desc->flags == EFX_DISCARD,
151 ("rx_desc->flags != EFX_DISCARD"));
152 rx_desc->flags = flags;
153
154 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
155 rx_desc->size = (uint16_t)size;
156 }
157
158 evq->rx_done++;
159
160 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
161 sfxge_ev_qcomplete(evq, B_FALSE);
162
163 done:
164 return (evq->rx_done >= SFXGE_EV_BATCH);
165 }
166
167 static boolean_t
sfxge_ev_exception(void * arg,uint32_t code,uint32_t data)168 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
169 {
170 struct sfxge_evq *evq;
171 struct sfxge_softc *sc;
172
173 evq = (struct sfxge_evq *)arg;
174 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
175
176 sc = evq->sc;
177
178 DBGPRINT(sc->dev, "[%d] %s", evq->index,
179 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
180 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
181 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
182 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
183 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
184 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
185 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
186 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
187 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
188 "UNKNOWN");
189
190 evq->exception = B_TRUE;
191
192 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
193 device_printf(sc->dev,
194 "hardware exception (code=%u); resetting\n",
195 code);
196 sfxge_schedule_reset(sc);
197 }
198
199 return (B_FALSE);
200 }
201
202 static boolean_t
sfxge_ev_rxq_flush_done(void * arg,uint32_t rxq_index)203 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
204 {
205 struct sfxge_evq *evq;
206 struct sfxge_softc *sc;
207 struct sfxge_rxq *rxq;
208 unsigned int index;
209 uint16_t magic;
210
211 evq = (struct sfxge_evq *)arg;
212 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
213
214 sc = evq->sc;
215 rxq = sc->rxq[rxq_index];
216
217 KASSERT(rxq != NULL, ("rxq == NULL"));
218
219 /* Resend a software event on the correct queue */
220 index = rxq->index;
221 if (index == evq->index) {
222 sfxge_rx_qflush_done(rxq);
223 return (B_FALSE);
224 }
225
226 evq = sc->evq[index];
227 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
228
229 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
230 ("evq not started"));
231 efx_ev_qpost(evq->common, magic);
232
233 return (B_FALSE);
234 }
235
236 static boolean_t
sfxge_ev_rxq_flush_failed(void * arg,uint32_t rxq_index)237 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
238 {
239 struct sfxge_evq *evq;
240 struct sfxge_softc *sc;
241 struct sfxge_rxq *rxq;
242 unsigned int index;
243 uint16_t magic;
244
245 evq = (struct sfxge_evq *)arg;
246 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
247
248 sc = evq->sc;
249 rxq = sc->rxq[rxq_index];
250
251 KASSERT(rxq != NULL, ("rxq == NULL"));
252
253 /* Resend a software event on the correct queue */
254 index = rxq->index;
255 evq = sc->evq[index];
256 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
257
258 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
259 ("evq not started"));
260 efx_ev_qpost(evq->common, magic);
261
262 return (B_FALSE);
263 }
264
265 static struct sfxge_txq *
sfxge_get_txq_by_label(struct sfxge_evq * evq,enum sfxge_txq_type label)266 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
267 {
268 unsigned int index;
269
270 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) :
271 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
272 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)),
273 ("unexpected txq label"));
274
275 index = (evq->index == 0) ?
276 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc));
277 return (evq->sc->txq[index]);
278 }
279
280 static boolean_t
sfxge_ev_tx(void * arg,uint32_t label,uint32_t id)281 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
282 {
283 struct sfxge_evq *evq;
284 struct sfxge_txq *txq;
285 unsigned int stop;
286 unsigned int delta;
287
288 evq = (struct sfxge_evq *)arg;
289 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
290
291 txq = sfxge_get_txq_by_label(evq, label);
292
293 KASSERT(txq != NULL, ("txq == NULL"));
294 KASSERT(evq->index == txq->evq_index,
295 ("evq->index != txq->evq_index"));
296
297 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
298 goto done;
299
300 stop = (id + 1) & txq->ptr_mask;
301 id = txq->pending & txq->ptr_mask;
302
303 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
304 txq->pending += delta;
305
306 evq->tx_done++;
307
308 if (txq->next == NULL &&
309 evq->txqs != &(txq->next)) {
310 *(evq->txqs) = txq;
311 evq->txqs = &(txq->next);
312 }
313
314 if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
315 sfxge_tx_qcomplete(txq, evq);
316
317 done:
318 return (evq->tx_done >= SFXGE_EV_BATCH);
319 }
320
321 static boolean_t
sfxge_ev_txq_flush_done(void * arg,uint32_t txq_index)322 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
323 {
324 struct sfxge_evq *evq;
325 struct sfxge_softc *sc;
326 struct sfxge_txq *txq;
327 uint16_t magic;
328
329 evq = (struct sfxge_evq *)arg;
330 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
331
332 sc = evq->sc;
333 txq = sc->txq[txq_index];
334
335 KASSERT(txq != NULL, ("txq == NULL"));
336 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
337 ("txq not initialized"));
338
339 if (txq->evq_index == evq->index) {
340 sfxge_tx_qflush_done(txq);
341 return (B_FALSE);
342 }
343
344 /* Resend a software event on the correct queue */
345 evq = sc->evq[txq->evq_index];
346 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
347
348 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
349 ("evq not started"));
350 efx_ev_qpost(evq->common, magic);
351
352 return (B_FALSE);
353 }
354
355 static boolean_t
sfxge_ev_software(void * arg,uint16_t magic)356 sfxge_ev_software(void *arg, uint16_t magic)
357 {
358 struct sfxge_evq *evq;
359 struct sfxge_softc *sc;
360 unsigned int label;
361
362 evq = (struct sfxge_evq *)arg;
363 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
364
365 sc = evq->sc;
366
367 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
368 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
369
370 switch (magic) {
371 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
372 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
373 break;
374
375 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
376 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
377 break;
378
379 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
380 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
381 break;
382
383 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
384 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
385
386 KASSERT(txq != NULL, ("txq == NULL"));
387 KASSERT(evq->index == txq->evq_index,
388 ("evq->index != txq->evq_index"));
389
390 sfxge_tx_qflush_done(txq);
391 break;
392 }
393 default:
394 break;
395 }
396
397 return (B_FALSE);
398 }
399
400 static boolean_t
sfxge_ev_sram(void * arg,uint32_t code)401 sfxge_ev_sram(void *arg, uint32_t code)
402 {
403 (void)arg;
404 (void)code;
405
406 switch (code) {
407 case EFX_SRAM_UPDATE:
408 EFSYS_PROBE(sram_update);
409 break;
410
411 case EFX_SRAM_CLEAR:
412 EFSYS_PROBE(sram_clear);
413 break;
414
415 case EFX_SRAM_ILLEGAL_CLEAR:
416 EFSYS_PROBE(sram_illegal_clear);
417 break;
418
419 default:
420 KASSERT(B_FALSE, ("Impossible SRAM event"));
421 break;
422 }
423
424 return (B_FALSE);
425 }
426
427 static boolean_t
sfxge_ev_timer(void * arg,uint32_t index)428 sfxge_ev_timer(void *arg, uint32_t index)
429 {
430 (void)arg;
431 (void)index;
432
433 return (B_FALSE);
434 }
435
436 static boolean_t
sfxge_ev_wake_up(void * arg,uint32_t index)437 sfxge_ev_wake_up(void *arg, uint32_t index)
438 {
439 (void)arg;
440 (void)index;
441
442 return (B_FALSE);
443 }
444
445 #if EFSYS_OPT_QSTATS
446
447 static void
sfxge_evq_stat_update(struct sfxge_evq * evq)448 sfxge_evq_stat_update(struct sfxge_evq *evq)
449 {
450 clock_t now;
451
452 SFXGE_EVQ_LOCK(evq);
453
454 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED))
455 goto out;
456
457 now = ticks;
458 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz)
459 goto out;
460
461 evq->stats_update_time = now;
462 efx_ev_qstats_update(evq->common, evq->stats);
463
464 out:
465 SFXGE_EVQ_UNLOCK(evq);
466 }
467
468 static int
sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS)469 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS)
470 {
471 struct sfxge_evq *evq = arg1;
472 struct sfxge_softc *sc = evq->sc;
473 unsigned int id = arg2;
474
475 SFXGE_ADAPTER_LOCK(sc);
476
477 sfxge_evq_stat_update(evq);
478
479 SFXGE_ADAPTER_UNLOCK(sc);
480
481 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id])));
482 }
483
484 static int
sfxge_evq_stat_init(struct sfxge_evq * evq)485 sfxge_evq_stat_init(struct sfxge_evq *evq)
486 {
487 struct sfxge_softc *sc = evq->sc;
488 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
489 char name[16];
490 struct sysctl_oid *evq_stats_node;
491 unsigned int id;
492
493 snprintf(name, sizeof(name), "%u", evq->index);
494 evq_stats_node = SYSCTL_ADD_NODE(ctx,
495 SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name,
496 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
497 if (evq_stats_node == NULL)
498 return (ENOMEM);
499
500 for (id = 0; id < EV_NQSTATS; id++) {
501 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node),
502 OID_AUTO, efx_ev_qstat_name(sc->enp, id),
503 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id,
504 sfxge_evq_stat_handler, "Q", "");
505 }
506
507 return (0);
508 }
509
510 static void
sfxge_ev_stat_update(struct sfxge_softc * sc)511 sfxge_ev_stat_update(struct sfxge_softc *sc)
512 {
513 struct sfxge_evq *evq;
514 unsigned int index;
515 clock_t now;
516 unsigned int id;
517
518 SFXGE_ADAPTER_LOCK(sc);
519
520 now = ticks;
521 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz)
522 goto out;
523
524 sc->ev_stats_update_time = now;
525
526 memset(sc->ev_stats, 0, sizeof(sc->ev_stats));
527
528 /* Update and add event counts from each event queue in turn */
529 for (index = 0; index < sc->evq_count; index++) {
530 evq = sc->evq[index];
531 sfxge_evq_stat_update(evq);
532 for (id = 0; id < EV_NQSTATS; id++)
533 sc->ev_stats[id] += evq->stats[id];
534 }
535 out:
536 SFXGE_ADAPTER_UNLOCK(sc);
537 }
538
539 static int
sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)540 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
541 {
542 struct sfxge_softc *sc = arg1;
543 unsigned int id = arg2;
544
545 sfxge_ev_stat_update(sc);
546
547 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
548 }
549
550 static void
sfxge_ev_stat_init(struct sfxge_softc * sc)551 sfxge_ev_stat_init(struct sfxge_softc *sc)
552 {
553 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
554 struct sysctl_oid_list *stat_list;
555 unsigned int id;
556 char name[40];
557
558 stat_list = SYSCTL_CHILDREN(sc->stats_node);
559
560 for (id = 0; id < EV_NQSTATS; id++) {
561 snprintf(name, sizeof(name), "ev_%s",
562 efx_ev_qstat_name(sc->enp, id));
563 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name,
564 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
565 sc, id, sfxge_ev_stat_handler, "Q", "");
566 }
567 }
568
569 #endif /* EFSYS_OPT_QSTATS */
570
571 static void
sfxge_ev_qmoderate(struct sfxge_softc * sc,unsigned int idx,unsigned int us)572 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
573 {
574 struct sfxge_evq *evq;
575 efx_evq_t *eep;
576
577 evq = sc->evq[idx];
578 eep = evq->common;
579
580 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
581 ("evq->init_state != SFXGE_EVQ_STARTED"));
582
583 (void)efx_ev_qmoderate(eep, us);
584 }
585
586 static int
sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)587 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
588 {
589 struct sfxge_softc *sc = arg1;
590 struct sfxge_intr *intr = &sc->intr;
591 unsigned int moderation;
592 int error;
593 unsigned int index;
594
595 SFXGE_ADAPTER_LOCK(sc);
596
597 if (req->newptr != NULL) {
598 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
599 != 0)
600 goto out;
601
602 /* We may not be calling efx_ev_qmoderate() now,
603 * so we have to range-check the value ourselves.
604 */
605 if (moderation >
606 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
607 error = EINVAL;
608 goto out;
609 }
610
611 sc->ev_moderation = moderation;
612 if (intr->state == SFXGE_INTR_STARTED) {
613 for (index = 0; index < sc->evq_count; index++)
614 sfxge_ev_qmoderate(sc, index, moderation);
615 }
616 } else {
617 error = SYSCTL_OUT(req, &sc->ev_moderation,
618 sizeof(sc->ev_moderation));
619 }
620
621 out:
622 SFXGE_ADAPTER_UNLOCK(sc);
623
624 return (error);
625 }
626
627 static boolean_t
sfxge_ev_initialized(void * arg)628 sfxge_ev_initialized(void *arg)
629 {
630 struct sfxge_evq *evq;
631
632 evq = (struct sfxge_evq *)arg;
633 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
634
635 /* Init done events may be duplicated on 7xxx */
636 KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
637 evq->init_state == SFXGE_EVQ_STARTED,
638 ("evq not starting"));
639
640 evq->init_state = SFXGE_EVQ_STARTED;
641
642 return (0);
643 }
644
645 static boolean_t
sfxge_ev_link_change(void * arg,efx_link_mode_t link_mode)646 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
647 {
648 struct sfxge_evq *evq;
649 struct sfxge_softc *sc;
650
651 evq = (struct sfxge_evq *)arg;
652 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
653
654 sc = evq->sc;
655
656 sfxge_mac_link_update(sc, link_mode);
657
658 return (0);
659 }
660
661 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
662 .eec_initialized = sfxge_ev_initialized,
663 .eec_rx = sfxge_ev_rx,
664 .eec_tx = sfxge_ev_tx,
665 .eec_exception = sfxge_ev_exception,
666 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done,
667 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed,
668 .eec_txq_flush_done = sfxge_ev_txq_flush_done,
669 .eec_software = sfxge_ev_software,
670 .eec_sram = sfxge_ev_sram,
671 .eec_wake_up = sfxge_ev_wake_up,
672 .eec_timer = sfxge_ev_timer,
673 .eec_link_change = sfxge_ev_link_change,
674 };
675
676 int
sfxge_ev_qpoll(struct sfxge_evq * evq)677 sfxge_ev_qpoll(struct sfxge_evq *evq)
678 {
679 int rc;
680
681 SFXGE_EVQ_LOCK(evq);
682
683 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
684 evq->init_state != SFXGE_EVQ_STARTED)) {
685 rc = EINVAL;
686 goto fail;
687 }
688
689 /* Synchronize the DMA memory for reading */
690 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
691 BUS_DMASYNC_POSTREAD);
692
693 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
694 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
695 KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
696 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
697
698 /* Poll the queue */
699 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
700
701 evq->rx_done = 0;
702 evq->tx_done = 0;
703
704 /* Perform any pending completion processing */
705 sfxge_ev_qcomplete(evq, B_TRUE);
706
707 /* Re-prime the event queue for interrupts */
708 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
709 goto fail;
710
711 SFXGE_EVQ_UNLOCK(evq);
712
713 return (0);
714
715 fail:
716 SFXGE_EVQ_UNLOCK(evq);
717 return (rc);
718 }
719
720 static void
sfxge_ev_qstop(struct sfxge_softc * sc,unsigned int index)721 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
722 {
723 struct sfxge_evq *evq;
724
725 evq = sc->evq[index];
726
727 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
728 ("evq->init_state != SFXGE_EVQ_STARTED"));
729
730 SFXGE_EVQ_LOCK(evq);
731 evq->init_state = SFXGE_EVQ_INITIALIZED;
732 evq->read_ptr = 0;
733 evq->exception = B_FALSE;
734
735 #if EFSYS_OPT_QSTATS
736 /* Add event counts before discarding the common evq state */
737 efx_ev_qstats_update(evq->common, evq->stats);
738 #endif
739
740 efx_ev_qdestroy(evq->common);
741 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
742 EFX_EVQ_NBUFS(evq->entries));
743 SFXGE_EVQ_UNLOCK(evq);
744 }
745
746 static int
sfxge_ev_qstart(struct sfxge_softc * sc,unsigned int index)747 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
748 {
749 struct sfxge_evq *evq;
750 efsys_mem_t *esmp;
751 int count;
752 int rc;
753
754 evq = sc->evq[index];
755 esmp = &evq->mem;
756
757 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
758 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
759
760 /* Clear all events. */
761 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
762
763 /* Program the buffer table. */
764 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
765 EFX_EVQ_NBUFS(evq->entries))) != 0)
766 return (rc);
767
768 /* Create the common code event queue. */
769 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
770 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
771 &evq->common)) != 0)
772 goto fail;
773
774 SFXGE_EVQ_LOCK(evq);
775
776 /* Prime the event queue for interrupts */
777 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
778 goto fail2;
779
780 evq->init_state = SFXGE_EVQ_STARTING;
781
782 SFXGE_EVQ_UNLOCK(evq);
783
784 /* Wait for the initialization event */
785 count = 0;
786 do {
787 /* Pause for 100 ms */
788 pause("sfxge evq init", hz / 10);
789
790 /* Check to see if the test event has been processed */
791 if (evq->init_state == SFXGE_EVQ_STARTED)
792 goto done;
793
794 } while (++count < 20);
795
796 rc = ETIMEDOUT;
797 goto fail3;
798
799 done:
800 return (0);
801
802 fail3:
803 SFXGE_EVQ_LOCK(evq);
804 evq->init_state = SFXGE_EVQ_INITIALIZED;
805 fail2:
806 SFXGE_EVQ_UNLOCK(evq);
807 efx_ev_qdestroy(evq->common);
808 fail:
809 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
810 EFX_EVQ_NBUFS(evq->entries));
811
812 return (rc);
813 }
814
815 void
sfxge_ev_stop(struct sfxge_softc * sc)816 sfxge_ev_stop(struct sfxge_softc *sc)
817 {
818 struct sfxge_intr *intr;
819 efx_nic_t *enp;
820 int index;
821
822 intr = &sc->intr;
823 enp = sc->enp;
824
825 KASSERT(intr->state == SFXGE_INTR_STARTED,
826 ("Interrupts not started"));
827
828 /* Stop the event queue(s) */
829 index = sc->evq_count;
830 while (--index >= 0)
831 sfxge_ev_qstop(sc, index);
832
833 /* Tear down the event module */
834 efx_ev_fini(enp);
835 }
836
837 int
sfxge_ev_start(struct sfxge_softc * sc)838 sfxge_ev_start(struct sfxge_softc *sc)
839 {
840 struct sfxge_intr *intr;
841 int index;
842 int rc;
843
844 intr = &sc->intr;
845
846 KASSERT(intr->state == SFXGE_INTR_STARTED,
847 ("intr->state != SFXGE_INTR_STARTED"));
848
849 /* Initialize the event module */
850 if ((rc = efx_ev_init(sc->enp)) != 0)
851 return (rc);
852
853 /* Start the event queues */
854 for (index = 0; index < sc->evq_count; index++) {
855 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
856 goto fail;
857 }
858
859 return (0);
860
861 fail:
862 /* Stop the event queue(s) */
863 while (--index >= 0)
864 sfxge_ev_qstop(sc, index);
865
866 /* Tear down the event module */
867 efx_ev_fini(sc->enp);
868
869 return (rc);
870 }
871
872 static void
sfxge_ev_qfini(struct sfxge_softc * sc,unsigned int index)873 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
874 {
875 struct sfxge_evq *evq;
876
877 evq = sc->evq[index];
878
879 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
880 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
881 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
882
883 sfxge_dma_free(&evq->mem);
884
885 sc->evq[index] = NULL;
886
887 SFXGE_EVQ_LOCK_DESTROY(evq);
888
889 free(evq, M_SFXGE);
890 }
891
892 static int
sfxge_ev_qinit(struct sfxge_softc * sc,unsigned int index)893 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
894 {
895 struct sfxge_evq *evq;
896 efsys_mem_t *esmp;
897 int rc;
898
899 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
900
901 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
902 evq->sc = sc;
903 evq->index = index;
904 sc->evq[index] = evq;
905 esmp = &evq->mem;
906
907 /* Build an event queue with room for one event per tx and rx buffer,
908 * plus some extra for link state events and MCDI completions.
909 * There are three tx queues in the first event queue and one in
910 * other.
911 */
912 if (index == 0)
913 evq->entries =
914 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
915 3 * sc->txq_entries +
916 128);
917 else
918 evq->entries =
919 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
920 sc->txq_entries +
921 128);
922
923 /* Initialise TX completion list */
924 evq->txqs = &evq->txq;
925
926 /* Allocate DMA space. */
927 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
928 return (rc);
929
930 /* Allocate buffer table entries. */
931 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
932 &evq->buf_base_id);
933
934 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
935
936 evq->init_state = SFXGE_EVQ_INITIALIZED;
937
938 #if EFSYS_OPT_QSTATS
939 rc = sfxge_evq_stat_init(evq);
940 if (rc != 0)
941 goto fail_evq_stat_init;
942 #endif
943
944 return (0);
945
946 #if EFSYS_OPT_QSTATS
947 fail_evq_stat_init:
948 evq->init_state = SFXGE_EVQ_UNINITIALIZED;
949 SFXGE_EVQ_LOCK_DESTROY(evq);
950 sfxge_dma_free(esmp);
951 sc->evq[index] = NULL;
952 free(evq, M_SFXGE);
953
954 return (rc);
955 #endif
956 }
957
958 void
sfxge_ev_fini(struct sfxge_softc * sc)959 sfxge_ev_fini(struct sfxge_softc *sc)
960 {
961 struct sfxge_intr *intr;
962 int index;
963
964 intr = &sc->intr;
965
966 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
967 ("intr->state != SFXGE_INTR_INITIALIZED"));
968
969 sc->ev_moderation = 0;
970
971 /* Tear down the event queue(s). */
972 index = sc->evq_count;
973 while (--index >= 0)
974 sfxge_ev_qfini(sc, index);
975
976 sc->evq_count = 0;
977 }
978
979 int
sfxge_ev_init(struct sfxge_softc * sc)980 sfxge_ev_init(struct sfxge_softc *sc)
981 {
982 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
983 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
984 struct sfxge_intr *intr;
985 int index;
986 int rc;
987
988 intr = &sc->intr;
989
990 sc->evq_count = intr->n_alloc;
991
992 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
993 ("intr->state != SFXGE_INTR_INITIALIZED"));
994
995 /* Set default interrupt moderation; add a sysctl to
996 * read and change it.
997 */
998 sc->ev_moderation = SFXGE_MODERATION;
999 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1000 OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1001 sc, 0, sfxge_int_mod_handler, "IU",
1002 "sfxge interrupt moderation (us)");
1003
1004 #if EFSYS_OPT_QSTATS
1005 sc->evqs_stats_node = SYSCTL_ADD_NODE(
1006 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node),
1007 OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1008 "Event queues stats");
1009 if (sc->evqs_stats_node == NULL) {
1010 rc = ENOMEM;
1011 goto fail_evqs_stats_node;
1012 }
1013 #endif
1014
1015 /*
1016 * Initialize the event queue(s) - one per interrupt.
1017 */
1018 for (index = 0; index < sc->evq_count; index++) {
1019 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
1020 goto fail;
1021 }
1022
1023 #if EFSYS_OPT_QSTATS
1024 sfxge_ev_stat_init(sc);
1025 #endif
1026
1027 return (0);
1028
1029 fail:
1030 while (--index >= 0)
1031 sfxge_ev_qfini(sc, index);
1032
1033 #if EFSYS_OPT_QSTATS
1034 fail_evqs_stats_node:
1035 #endif
1036 sc->evq_count = 0;
1037 return (rc);
1038 }
1039