142 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
143 static void oce_tx_complete(struct oce_wq *wq, uint32_t wqe_idx,
146 				 struct oce_wq *wq);
562 	struct oce_wq *wq = NULL;  in oce_multiq_start()  local
569 	wq = sc->wq[queue_index];  in oce_multiq_start()
571 	LOCK(&wq->tx_lock);  in oce_multiq_start()
572 	status = oce_multiq_transmit(ifp, m, wq);  in oce_multiq_start()
573 	UNLOCK(&wq->tx_lock);  in oce_multiq_start()
588 		while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)  in oce_multiq_flush()
869 	struct oce_wq *wq = sc->wq[wq_index];  in oce_tx()  local
908 	pd = &wq->pckts[wq->pkt_desc_head];  in oce_tx()
910 	rc = bus_dmamap_load_mbuf_sg(wq->tag,  in oce_tx()
920 		if (num_wqes >= RING_NUM_FREE(wq->ring)) {  in oce_tx()
921 			bus_dmamap_unload(wq->tag, pd->map);  in oce_tx()
924 		atomic_store_rel_int(&wq->pkt_desc_head,  in oce_tx()
925 				     (wq->pkt_desc_head + 1) % \  in oce_tx()
927 		bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);  in oce_tx()
931 		    RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);  in oce_tx()
963 		RING_PUT(wq->ring, 1);  in oce_tx()
964 		atomic_add_int(&wq->ring->num_used, 1);  in oce_tx()
968 			    RING_GET_PRODUCER_ITEM_VA(wq->ring,  in oce_tx()
974 			pd->wqe_idx = wq->ring->pidx;  in oce_tx()
975 			RING_PUT(wq->ring, 1);  in oce_tx()
976 			atomic_add_int(&wq->ring->num_used, 1);  in oce_tx()
980 			    RING_GET_PRODUCER_ITEM_VA(wq->ring,  in oce_tx()
986 			pd->wqe_idx = wq->ring->pidx;  in oce_tx()
987 			RING_PUT(wq->ring, 1);  in oce_tx()
988 			atomic_add_int(&wq->ring->num_used, 1);  in oce_tx()
993 		wq->tx_stats.tx_reqs++;  in oce_tx()
994 		wq->tx_stats.tx_wrbs += num_wqes;  in oce_tx()
995 		wq->tx_stats.tx_bytes += m->m_pkthdr.len;  in oce_tx()
996 		wq->tx_stats.tx_pkts++;  in oce_tx()
998 		bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,  in oce_tx()
1000 		reg_value = (num_wqes << 16) | wq->wq_id;  in oce_tx()
1001 		OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);  in oce_tx()
1029 oce_tx_complete(struct oce_wq *wq, uint32_t wqe_idx, uint32_t status)  in oce_tx_complete()  argument
1032 	POCE_SOFTC sc = (POCE_SOFTC) wq->parent;  in oce_tx_complete()
1035 	pd = &wq->pckts[wq->pkt_desc_tail];  in oce_tx_complete()
1036 	atomic_store_rel_int(&wq->pkt_desc_tail,  in oce_tx_complete()
1037 			     (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);   in oce_tx_complete()
1038 	atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);  in oce_tx_complete()
1039 	bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);  in oce_tx_complete()
1040 	bus_dmamap_unload(wq->tag, pd->map);  in oce_tx_complete()
1048 		if (wq->ring->num_used < (wq->ring->num_items / 2)) {  in oce_tx_complete()
1050 			oce_tx_restart(sc, wq);	  in oce_tx_complete()
1057 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)  in oce_tx_restart()  argument
1064 	if (!drbr_empty(sc->ifp, wq->br))  in oce_tx_restart()
1068 		taskqueue_enqueue_fast(taskqueue_swi, &wq->txtask);  in oce_tx_restart()
1145 	struct oce_wq *wq = arg;  in oce_tx_task()  local
1146 	POCE_SOFTC sc = wq->parent;  in oce_tx_task()
1151 	LOCK(&wq->tx_lock);  in oce_tx_task()
1152 	rc = oce_multiq_transmit(ifp, NULL, wq);  in oce_tx_task()
1155 				"TX[%d] restart failed\n", wq->queue_index);  in oce_tx_task()
1157 	UNLOCK(&wq->tx_lock);  in oce_tx_task()
1185 		LOCK(&sc->wq[def_q]->tx_lock);  in oce_start()
1187 		UNLOCK(&sc->wq[def_q]->tx_lock);  in oce_start()
1190 				sc->wq[def_q]->tx_stats.tx_stops ++;  in oce_start()
1210 	struct oce_wq *wq = (struct oce_wq *)arg;  in oce_wq_handler()  local
1211 	POCE_SOFTC sc = wq->parent;  in oce_wq_handler()
1212 	struct oce_cq *cq = wq->cq;  in oce_wq_handler()
1222 		wq->ring->cidx = cqe->u0.s.wqe_index + 1;  in oce_wq_handler()
1223 		if (wq->ring->cidx >= wq->ring->num_items)  in oce_wq_handler()
1224 			wq->ring->cidx -= wq->ring->num_items;  in oce_wq_handler()
1226 		oce_tx_complete(wq, cqe->u0.s.wqe_index, cqe->u0.s.status);  in oce_wq_handler()
1227 		wq->tx_stats.tx_compl++;  in oce_wq_handler()
1257 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)  in oce_multiq_transmit()  argument
1264 	br  = wq->br;  in oce_multiq_transmit()
1265 	queue_index = wq->queue_index;  in oce_multiq_transmit()
1284 				wq->tx_stats.tx_stops ++;  in oce_multiq_transmit()
2047 		oce_tx_restart(sc, sc->wq[i]);  in oce_local_timer()
2066 	struct oce_wq *wq;  in oce_if_deactivate()  local
2074 		for_all_wq_queues(sc, wq, i) {  in oce_if_deactivate()
2075 			if (wq->ring->num_used) {  in oce_if_deactivate()
2109 	for_all_wq_queues(sc, wq, i)  in oce_if_deactivate()
2110 		oce_drain_wq_cq(wq);  in oce_if_deactivate()
2127 	struct oce_wq *wq;  in oce_if_activate()  local
2142 	for_all_wq_queues(sc, wq, i) {  in oce_if_activate()
2143 		rc = oce_start_wq(wq);  in oce_if_activate()