693 netmap_txsync_to_host_compat(struct netmap_kring *kring, int flags)  in netmap_txsync_to_host_compat()  argument
696 	netmap_txsync_to_host(kring->na);  in netmap_txsync_to_host_compat()
702 netmap_rxsync_from_host_compat(struct netmap_kring *kring, int flags)  in netmap_rxsync_from_host_compat()  argument
705 	netmap_rxsync_from_host(kring->na, NULL, NULL);  in netmap_rxsync_from_host_compat()
740 	struct netmap_kring *kring;  in netmap_krings_create()  local
762 		kring = &na->tx_rings[i];  in netmap_krings_create()
763 		bzero(kring, sizeof(*kring));  in netmap_krings_create()
764 		kring->na = na;  in netmap_krings_create()
765 		kring->ring_id = i;  in netmap_krings_create()
766 		kring->nkr_num_slots = ndesc;  in netmap_krings_create()
768 			kring->nm_sync = na->nm_txsync;  in netmap_krings_create()
770 			kring->nm_sync = netmap_txsync_to_host_compat;  in netmap_krings_create()
775 		kring->rhead = kring->rcur = kring->nr_hwcur = 0;  in netmap_krings_create()
776 		kring->rtail = kring->nr_hwtail = ndesc - 1;  in netmap_krings_create()
777 		snprintf(kring->name, sizeof(kring->name) - 1, "%s TX%d", na->name, i);  in netmap_krings_create()
779 			kring->name, kring->rhead, kring->rcur, kring->rtail);  in netmap_krings_create()
780 		mtx_init(&kring->q_lock, "nm_txq_lock", NULL, MTX_DEF);  in netmap_krings_create()
781 		init_waitqueue_head(&kring->si);  in netmap_krings_create()
786 		kring = &na->rx_rings[i];  in netmap_krings_create()
787 		bzero(kring, sizeof(*kring));  in netmap_krings_create()
788 		kring->na = na;  in netmap_krings_create()
789 		kring->ring_id = i;  in netmap_krings_create()
790 		kring->nkr_num_slots = ndesc;  in netmap_krings_create()
792 			kring->nm_sync = na->nm_rxsync;  in netmap_krings_create()
794 			kring->nm_sync = netmap_rxsync_from_host_compat;  in netmap_krings_create()
796 		kring->rhead = kring->rcur = kring->nr_hwcur = 0;  in netmap_krings_create()
797 		kring->rtail = kring->nr_hwtail = 0;  in netmap_krings_create()
798 		snprintf(kring->name, sizeof(kring->name) - 1, "%s RX%d", na->name, i);  in netmap_krings_create()
800 			kring->name, kring->rhead, kring->rcur, kring->rtail);  in netmap_krings_create()
801 		mtx_init(&kring->q_lock, "nm_rxq_lock", NULL, MTX_DEF);  in netmap_krings_create()
802 		init_waitqueue_head(&kring->si);  in netmap_krings_create()
831 	struct netmap_kring *kring = na->tx_rings;  in netmap_krings_delete()  local
834 	for ( ; kring != na->tailroom; kring++) {  in netmap_krings_delete()
835 		mtx_destroy(&kring->q_lock);  in netmap_krings_delete()
836 		netmap_knlist_destroy(&kring->si);  in netmap_krings_delete()
1148 netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force)  in netmap_grab_packets()  argument
1150 	u_int const lim = kring->nkr_num_slots - 1;  in netmap_grab_packets()
1151 	u_int const head = kring->ring->head;  in netmap_grab_packets()
1153 	struct netmap_adapter *na = kring->na;  in netmap_grab_packets()
1155 	for (n = kring->nr_hwcur; n != head; n = nm_next(n, lim)) {  in netmap_grab_packets()
1157 		struct netmap_slot *slot = &kring->ring->slot[n];  in netmap_grab_packets()
1184 	struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];  in netmap_sw_to_nic()  local
1185 	struct netmap_slot *rxslot = kring->ring->slot;  in netmap_sw_to_nic()
1186 	u_int i, rxcur = kring->nr_hwcur;  in netmap_sw_to_nic()
1187 	u_int const head = kring->rhead;  in netmap_sw_to_nic()
1188 	u_int const src_lim = kring->nkr_num_slots - 1;  in netmap_sw_to_nic()
1237 	struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings];  in netmap_txsync_to_host()  local
1238 	struct netmap_ring *ring = kring->ring;  in netmap_txsync_to_host()
1239 	u_int const lim = kring->nkr_num_slots - 1;  in netmap_txsync_to_host()
1240 	u_int const head = kring->rhead;  in netmap_txsync_to_host()
1250 	netmap_grab_packets(kring, &q, 1 /* force */);  in netmap_txsync_to_host()
1252 	kring->nr_hwcur = head;  in netmap_txsync_to_host()
1253 	kring->nr_hwtail = head + lim;  in netmap_txsync_to_host()
1254 	if (kring->nr_hwtail > lim)  in netmap_txsync_to_host()
1255 		kring->nr_hwtail -= lim + 1;  in netmap_txsync_to_host()
1256 	nm_txsync_finalize(kring);  in netmap_txsync_to_host()
1278 	struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];  in netmap_rxsync_from_host()  local
1279 	struct netmap_ring *ring = kring->ring;  in netmap_rxsync_from_host()
1281 	u_int const lim = kring->nkr_num_slots - 1;  in netmap_rxsync_from_host()
1282 	u_int const head = kring->rhead;  in netmap_rxsync_from_host()
1284 	struct mbq *q = &kring->rx_queue;  in netmap_rxsync_from_host()
1297 		nm_i = kring->nr_hwtail;  in netmap_rxsync_from_host()
1309 			slot->flags = kring->nkr_slot_flags;  in netmap_rxsync_from_host()
1313 		kring->nr_hwtail = nm_i;  in netmap_rxsync_from_host()
1319 	nm_i = kring->nr_hwcur;  in netmap_rxsync_from_host()
1321 		if (netmap_fwd || kring->ring->flags & NR_FORWARD)  in netmap_rxsync_from_host()
1323 		kring->nr_hwcur = head;  in netmap_rxsync_from_host()
1326 	nm_rxsync_finalize(kring);  in netmap_rxsync_from_host()
1329 	if (kring->rcur == kring->rtail && td) /* no bufs available */  in netmap_rxsync_from_host()
1330 		OS_selrecord(td, &kring->si);  in netmap_rxsync_from_host()
1545 nm_txsync_prologue(struct netmap_kring *kring)  in nm_txsync_prologue()  argument
1547 	struct netmap_ring *ring = kring->ring;  in nm_txsync_prologue()
1550 	u_int n = kring->nkr_num_slots;  in nm_txsync_prologue()
1553 		kring->name,  in nm_txsync_prologue()
1554 		kring->nr_hwcur, kring->nr_hwtail,  in nm_txsync_prologue()
1557 	if (kring->nr_hwcur >= n || kring->rhead >= n ||  in nm_txsync_prologue()
1558 	    kring->rtail >= n ||  kring->nr_hwtail >= n)  in nm_txsync_prologue()
1570 	if (kring->rtail >= kring->rhead) {  in nm_txsync_prologue()
1572 		if (head < kring->rhead || head > kring->rtail)  in nm_txsync_prologue()
1575 		if (cur < head || cur > kring->rtail)  in nm_txsync_prologue()
1579 		if (head > kring->rtail && head < kring->rhead)  in nm_txsync_prologue()
1583 		if (head <= kring->rtail) {  in nm_txsync_prologue()
1585 			if (cur < head || cur > kring->rtail)  in nm_txsync_prologue()
1589 			if (cur > kring->rtail && cur < head)  in nm_txsync_prologue()
1593 	if (ring->tail != kring->rtail) {  in nm_txsync_prologue()
1595 			ring->tail, kring->rtail);  in nm_txsync_prologue()
1596 		ring->tail = kring->rtail;  in nm_txsync_prologue()
1598 	kring->rhead = head;  in nm_txsync_prologue()
1599 	kring->rcur = cur;  in nm_txsync_prologue()
1604 		kring->name,  in nm_txsync_prologue()
1605 		kring->nr_hwcur,  in nm_txsync_prologue()
1606 		kring->rcur, kring->nr_hwtail,  in nm_txsync_prologue()
1624 nm_rxsync_prologue(struct netmap_kring *kring)  in nm_rxsync_prologue()  argument
1626 	struct netmap_ring *ring = kring->ring;  in nm_rxsync_prologue()
1627 	uint32_t const n = kring->nkr_num_slots;  in nm_rxsync_prologue()
1631 		kring->name,  in nm_rxsync_prologue()
1632 		kring->nr_hwcur, kring->nr_hwtail,  in nm_rxsync_prologue()
1641 	cur = kring->rcur = ring->cur;	/* read only once */  in nm_rxsync_prologue()
1642 	head = kring->rhead = ring->head;	/* read only once */  in nm_rxsync_prologue()
1644 	if (kring->nr_hwcur >= n || kring->nr_hwtail >= n)  in nm_rxsync_prologue()
1648 	if (kring->nr_hwtail >= kring->nr_hwcur) {  in nm_rxsync_prologue()
1650 		if (head < kring->nr_hwcur || head > kring->nr_hwtail)  in nm_rxsync_prologue()
1653 		if (cur < head || cur > kring->nr_hwtail)  in nm_rxsync_prologue()
1657 		if (head < kring->nr_hwcur && head > kring->nr_hwtail)  in nm_rxsync_prologue()
1660 		if (head <= kring->nr_hwtail) {  in nm_rxsync_prologue()
1662 			if (cur < head || cur > kring->nr_hwtail)  in nm_rxsync_prologue()
1666 			if (cur < head && cur > kring->nr_hwtail)  in nm_rxsync_prologue()
1670 	if (ring->tail != kring->rtail) {  in nm_rxsync_prologue()
1672 			kring->name,  in nm_rxsync_prologue()
1673 			ring->tail, kring->rtail);  in nm_rxsync_prologue()
1674 		ring->tail = kring->rtail;  in nm_rxsync_prologue()
1680 		kring->nr_hwcur,  in nm_rxsync_prologue()
1681 		kring->rcur, kring->nr_hwtail,  in nm_rxsync_prologue()
1682 		kring->rhead, kring->rcur, ring->tail);  in nm_rxsync_prologue()
1700 netmap_ring_reinit(struct netmap_kring *kring)  in netmap_ring_reinit()  argument
1702 	struct netmap_ring *ring = kring->ring;  in netmap_ring_reinit()
1703 	u_int i, lim = kring->nkr_num_slots - 1;  in netmap_ring_reinit()
1707 	RD(10, "called for %s", kring->name);  in netmap_ring_reinit()
1709 	kring->rhead = ring->head;  in netmap_ring_reinit()
1710 	kring->rcur  = ring->cur;  in netmap_ring_reinit()
1711 	kring->rtail = ring->tail;  in netmap_ring_reinit()
1726 		} else if (len > NETMAP_BUF_SIZE(kring->na)) {  in netmap_ring_reinit()
1734 			kring->name,  in netmap_ring_reinit()
1735 			ring->cur, kring->nr_hwcur,  in netmap_ring_reinit()
1736 			ring->tail, kring->nr_hwtail);  in netmap_ring_reinit()
1737 		ring->head = kring->rhead = kring->nr_hwcur;  in netmap_ring_reinit()
1738 		ring->cur  = kring->rcur  = kring->nr_hwcur;  in netmap_ring_reinit()
1739 		ring->tail = kring->rtail = kring->nr_hwtail;  in netmap_ring_reinit()
2196 			struct netmap_kring *kring = krings + i;  in netmap_ioctl()  local
2197 			if (nm_kr_tryget(kring)) {  in netmap_ioctl()
2204 					    i, kring->ring->cur,  in netmap_ioctl()
2205 					    kring->nr_hwcur);  in netmap_ioctl()
2206 				if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) {  in netmap_ioctl()
2207 					netmap_ring_reinit(kring);  in netmap_ioctl()
2209 					kring->nm_sync(kring, NAF_FORCE_RECLAIM);  in netmap_ioctl()
2213 					    i, kring->ring->cur,  in netmap_ioctl()
2214 					    kring->nr_hwcur);  in netmap_ioctl()
2216 				kring->nm_sync(kring, NAF_FORCE_READ);  in netmap_ioctl()
2219 			nm_kr_put(kring);  in netmap_ioctl()
2288 	struct netmap_kring *kring;  in netmap_poll()  local
2358 		kring = &na->rx_rings[i];  in netmap_poll()
2360 		if (!nm_ring_empty(kring->ring)) {  in netmap_poll()
2366 		kring = &na->tx_rings[i];  in netmap_poll()
2368 		if (!nm_ring_empty(kring->ring)) {  in netmap_poll()
2392 			kring = &na->tx_rings[i];  in netmap_poll()
2393 			if (!want_tx && kring->ring->cur == kring->nr_hwcur)  in netmap_poll()
2396 			if (nm_kr_tryget(kring)) {  in netmap_poll()
2407 			if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) {  in netmap_poll()
2408 				netmap_ring_reinit(kring);  in netmap_poll()
2411 				if (kring->nm_sync(kring, 0))  in netmap_poll()
2421 			found = kring->rcur != kring->rtail;  in netmap_poll()
2422 			nm_kr_put(kring);  in netmap_poll()
2448 			kring = &na->rx_rings[i];  in netmap_poll()
2450 			if (nm_kr_tryget(kring)) {  in netmap_poll()
2463 			if (netmap_fwd ||kring->ring->flags & NR_FORWARD) {  in netmap_poll()
2465 				    kring->nr_hwcur, kring->ring->cur);  in netmap_poll()
2466 				netmap_grab_packets(kring, &q, netmap_fwd);  in netmap_poll()
2469 			if (kring->nm_sync(kring, 0))  in netmap_poll()
2472 					kring->ring->flags & NR_TIMESTAMP) {  in netmap_poll()
2473 				microtime(&kring->ring->ts);  in netmap_poll()
2476 			found = kring->rcur != kring->rtail;  in netmap_poll()
2477 			nm_kr_put(kring);  in netmap_poll()
2487 			kring = &na->rx_rings[na->num_rx_rings];  in netmap_poll()
2489 			    && (netmap_fwd || kring->ring->flags & NR_FORWARD)) {  in netmap_poll()
2491 				if (nm_ring_empty(kring->ring))  in netmap_poll()
2493 				if (!nm_ring_empty(kring->ring))  in netmap_poll()
2538 	struct netmap_kring *kring;  in netmap_notify()  local
2541 		kring = na->tx_rings + n_ring;  in netmap_notify()
2542 		OS_selwakeup(&kring->si, PI_NET);  in netmap_notify()
2550 		kring = na->rx_rings + n_ring;  in netmap_notify()
2551 		OS_selwakeup(&kring->si, PI_NET);  in netmap_notify()
2821 	struct netmap_kring *kring;  in netmap_transmit()  local
2837 	kring = &na->rx_rings[na->num_rx_rings];  in netmap_transmit()
2838 	q = &kring->rx_queue;  in netmap_transmit()
2854         space = kring->nr_hwtail - kring->nr_hwcur;  in netmap_transmit()
2856                 space += kring->nkr_num_slots;  in netmap_transmit()
2857 	if (space + mbq_len(q) >= kring->nkr_num_slots - 1) { // XXX  in netmap_transmit()
2859 			na->name, kring->nr_hwcur, kring->nr_hwtail, mbq_len(q),  in netmap_transmit()
2894 	struct netmap_kring *kring;  in netmap_reset()  local
2913 		kring = na->tx_rings + n;  in netmap_reset()
2915 		new_hwofs = kring->nr_hwcur - new_cur;  in netmap_reset()
2919 		kring = na->rx_rings + n;  in netmap_reset()
2920 		new_hwofs = kring->nr_hwtail - new_cur;  in netmap_reset()
2922 	lim = kring->nkr_num_slots - 1;  in netmap_reset()
2931 		kring->nkr_hwofs, new_hwofs,  in netmap_reset()
2932 		kring->nr_hwtail,  in netmap_reset()
2933 		tx == NR_TX ? lim : kring->nr_hwtail);  in netmap_reset()
2934 	kring->nkr_hwofs = new_hwofs;  in netmap_reset()
2936 		kring->nr_hwtail = kring->nr_hwcur + lim;  in netmap_reset()
2937 		if (kring->nr_hwtail > lim)  in netmap_reset()
2938 			kring->nr_hwtail -= lim + 1;  in netmap_reset()
2957 	return kring->ring->slot;  in netmap_reset()
2979 	struct netmap_kring *kring;  in netmap_common_irq()  local
2990 		kring = na->rx_rings + q;  in netmap_common_irq()
2991 		kring->nr_kflags |= NKR_PENDINTR;	// XXX atomic ?  in netmap_common_irq()
2997 		kring = na->tx_rings + q;  in netmap_common_irq()