109 	__hal_ring_t *ring = (__hal_ring_t *) ringh;  in __hal_ring_first_block_address_get()  local
112 	dma_object = __hal_mempool_memblock_dma(ring->mempool, 0);  in __hal_ring_first_block_address_get()
232     __hal_ring_t *ring,  in __hal_ring_rxdblock_link()  argument
241 	vxge_assert((mempoolh != NULL) && (ring != NULL));  in __hal_ring_rxdblock_link()
243 	hldev = (__hal_device_t *) ring->channel.devh;  in __hal_ring_rxdblock_link()
250 	    "from = %d, to = %d", (ptr_t) mempoolh, (ptr_t) ring, from, to);  in __hal_ring_rxdblock_link()
277 	vxge_os_dma_sync(ring->channel.pdev,  in __hal_ring_rxdblock_link()
321 	__hal_ring_t *ring = (__hal_ring_t *) userdata;  in __hal_ring_mempool_item_alloc()  local
324 	vxge_assert((item != NULL) && (ring != NULL));  in __hal_ring_mempool_item_alloc()
326 	hldev = (__hal_device_t *) ring->channel.devh;  in __hal_ring_mempool_item_alloc()
340 	for (i = 0; i < ring->rxds_per_block; i++) {  in __hal_ring_mempool_item_alloc()
347 		u32 dtr_index = item_index * ring->rxds_per_block + i;  in __hal_ring_mempool_item_alloc()
349 		ring->channel.dtr_arr[dtr_index].dtr =  in __hal_ring_mempool_item_alloc()
350 		    ((u8 *) item) + i * ring->rxd_size;  in __hal_ring_mempool_item_alloc()
364 		    ring->channel.dtr_arr[dtr_index].dtr;  in __hal_ring_mempool_item_alloc()
366 		uld_priv = ((u8 *) rxdblock_priv + ring->rxd_priv_size * i);  in __hal_ring_mempool_item_alloc()
369 		    ring->per_rxd_space));  in __hal_ring_mempool_item_alloc()
373 		ring->channel.dtr_arr[dtr_index].uld_priv = (void *)uld_priv;  in __hal_ring_mempool_item_alloc()
374 		ring->channel.dtr_arr[dtr_index].hal_priv = (void *)rxd_priv;  in __hal_ring_mempool_item_alloc()
384 		rxd_priv->db_bytes = ring->rxd_size;  in __hal_ring_mempool_item_alloc()
386 		if (i == (ring->rxds_per_block - 1)) {  in __hal_ring_mempool_item_alloc()
389 			    (ring->rxds_per_block * ring->rxd_size));  in __hal_ring_mempool_item_alloc()
396 		__hal_ring_rxdblock_link(mempoolh, ring, item_index, 0);  in __hal_ring_mempool_item_alloc()
401 		__hal_ring_rxdblock_link(mempoolh, ring, item_index - 1, item_index);  in __hal_ring_mempool_item_alloc()
435 	__hal_ring_t *ring = (__hal_ring_t *) userdata;  in __hal_ring_mempool_item_free()  local
438 	vxge_assert((item != NULL) && (ring != NULL));  in __hal_ring_mempool_item_free()
440 	hldev = (__hal_device_t *) ring->channel.devh;  in __hal_ring_mempool_item_free()
468     __hal_ring_t *ring,  in __hal_ring_initial_replenish()  argument
476 	vxge_assert(ring != NULL);  in __hal_ring_initial_replenish()
478 	hldev = (__hal_device_t *) ring->channel.devh;  in __hal_ring_initial_replenish()
484 	    (ptr_t) ring, reopen);  in __hal_ring_initial_replenish()
486 	while (vxge_hal_ring_rxd_reserve(ring->channel.vph, &rxd, &uld_priv) ==  in __hal_ring_initial_replenish()
489 		if (ring->rxd_init) {  in __hal_ring_initial_replenish()
490 			status = ring->rxd_init(ring->channel.vph,  in __hal_ring_initial_replenish()
494 			    ring->channel.userdata,  in __hal_ring_initial_replenish()
497 				vxge_hal_ring_rxd_free(ring->channel.vph, rxd);  in __hal_ring_initial_replenish()
505 		vxge_hal_ring_rxd_post(ring->channel.vph, rxd);  in __hal_ring_initial_replenish()
527 	__hal_ring_t *ring;  in __hal_ring_create()  local
552 	    &vp->vpath->hldev->header.config.vp_config[vp->vpath->vp_id].ring;  in __hal_ring_create()
559 	ring = (__hal_ring_t *) vxge_hal_channel_allocate(  in __hal_ring_create()
567 	if (ring == NULL) {  in __hal_ring_create()
576 	vp->vpath->ringh = (vxge_hal_ring_h) ring;  in __hal_ring_create()
578 	ring->stats = &vp->vpath->sw_stats->ring_stats;  in __hal_ring_create()
580 	ring->config = config;  in __hal_ring_create()
581 	ring->callback = attr->callback;  in __hal_ring_create()
582 	ring->rxd_init = attr->rxd_init;  in __hal_ring_create()
583 	ring->rxd_term = attr->rxd_term;  in __hal_ring_create()
585 	ring->indicate_max_pkts = config->indicate_max_pkts;  in __hal_ring_create()
586 	ring->buffer_mode = config->buffer_mode;  in __hal_ring_create()
589 	vxge_os_spin_lock_init(&ring->channel.post_lock, hldev->pdev);  in __hal_ring_create()
591 	vxge_os_spin_lock_init_irq(&ring->channel.post_lock, hldev->irqh);  in __hal_ring_create()
594 	ring->rxd_size = vxge_hal_ring_rxd_size_get(config->buffer_mode);  in __hal_ring_create()
595 	ring->rxd_priv_size =  in __hal_ring_create()
597 	ring->per_rxd_space = attr->per_rxd_space;  in __hal_ring_create()
599 	ring->rxd_priv_size =  in __hal_ring_create()
600 	    ((ring->rxd_priv_size + __vxge_os_cacheline_size - 1) /  in __hal_ring_create()
607 	ring->rxds_per_block =  in __hal_ring_create()
611 	ring->rxdblock_priv_size = ring->rxd_priv_size * ring->rxds_per_block;  in __hal_ring_create()
613 	ring->rxd_mem_avail =  in __hal_ring_create()
614 	    ((__hal_vpath_handle_t *) ring->channel.vph)->vpath->rxd_mem_size;  in __hal_ring_create()
616 	ring->db_byte_count = 0;  in __hal_ring_create()
618 	ring->mempool = vxge_hal_mempool_create(  in __hal_ring_create()
622 	    ring->rxdblock_priv_size,  in __hal_ring_create()
623 	    ring->config->ring_length / ring->rxds_per_block,  in __hal_ring_create()
624 	    ring->config->ring_length / ring->rxds_per_block,  in __hal_ring_create()
627 	    ring);  in __hal_ring_create()
629 	if (ring->mempool == NULL) {  in __hal_ring_create()
636 	status = vxge_hal_channel_initialize(&ring->channel);  in __hal_ring_create()
653 	if (ring->rxd_init) {  in __hal_ring_create()
655 		    ring,  in __hal_ring_create()
669 	ring->stats->common_stats.usage_cnt = 0;  in __hal_ring_create()
692 	__hal_ring_t *ring = (__hal_ring_t *) ringh;  in __hal_ring_abort()  local
696 	hldev = (__hal_device_t *) ring->channel.devh;  in __hal_ring_abort()
704 	if (ring->rxd_term) {  in __hal_ring_abort()
705 		__hal_channel_for_each_dtr(&ring->channel, rxdh, i) {  in __hal_ring_abort()
706 			if (!__hal_channel_is_posted_dtr(&ring->channel, i)) {  in __hal_ring_abort()
707 				ring->rxd_term(ring->channel.vph, rxdh,  in __hal_ring_abort()
708 				    VXGE_HAL_RING_ULD_PRIV(ring, rxdh),  in __hal_ring_abort()
710 				    ring->channel.userdata,  in __hal_ring_abort()
717 		__hal_channel_dtr_try_complete(&ring->channel, &rxdh);  in __hal_ring_abort()
721 		__hal_channel_dtr_complete(&ring->channel);  in __hal_ring_abort()
722 		if (ring->rxd_term) {  in __hal_ring_abort()
723 			ring->rxd_term(ring->channel.vph, rxdh,  in __hal_ring_abort()
724 			    VXGE_HAL_RING_ULD_PRIV(ring, rxdh),  in __hal_ring_abort()
726 			    ring->channel.userdata,  in __hal_ring_abort()
729 		__hal_channel_dtr_free(&ring->channel,  in __hal_ring_abort()
747 	__hal_ring_t *ring = (__hal_ring_t *) ringh;  in __hal_ring_reset()  local
750 	__hal_vpath_handle_t *vph = (__hal_vpath_handle_t *) ring->channel.vph;  in __hal_ring_reset()
754 	hldev = (__hal_device_t *) ring->channel.devh;  in __hal_ring_reset()
764 	status = __hal_channel_reset(&ring->channel);  in __hal_ring_reset()
773 	ring->rxd_mem_avail = vph->vpath->rxd_mem_size;  in __hal_ring_reset()
774 	ring->db_byte_count = 0;  in __hal_ring_reset()
777 	if (ring->rxd_init) {  in __hal_ring_reset()
779 		    ring,  in __hal_ring_reset()
806 	__hal_ring_t *ring;  in __hal_ring_delete()  local
818 	ring = (__hal_ring_t *) vp->vpath->ringh;  in __hal_ring_delete()
820 	vxge_assert(ring != NULL);  in __hal_ring_delete()
822 	vxge_assert(ring->channel.pdev);  in __hal_ring_delete()
827 	if (ring->mempool) {  in __hal_ring_delete()
828 		vxge_hal_mempool_destroy(ring->mempool);  in __hal_ring_delete()
831 	vxge_hal_channel_terminate(&ring->channel);  in __hal_ring_delete()
834 	vxge_os_spin_lock_destroy(&ring->channel.post_lock, hldev->pdev);  in __hal_ring_delete()
836 	vxge_os_spin_lock_destroy_irq(&ring->channel.post_lock, hldev->pdev);  in __hal_ring_delete()
839 	vxge_hal_channel_free(&ring->channel);  in __hal_ring_delete()
891 	if (vpath->vp_config->ring.max_frm_len !=  in __hal_ring_frame_length_set()
895 		    vpath->vp_config->ring.max_frm_len +  in __hal_ring_frame_length_set()
943 	__hal_ring_t *ring;  in vxge_hal_ring_rxd_reserve()  local
958 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_rxd_reserve()
960 	vxge_assert(ring != NULL);  in vxge_hal_ring_rxd_reserve()
963 	vxge_os_spin_lock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_reserve()
965 	vxge_os_spin_lock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_reserve()
968 	status = __hal_channel_dtr_reserve(&ring->channel, rxdh);  in vxge_hal_ring_rxd_reserve()
971 	vxge_os_spin_unlock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_reserve()
973 	vxge_os_spin_unlock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_reserve()
982 		*rxd_priv = VXGE_HAL_RING_ULD_PRIV(ring, rxdp);  in vxge_hal_ring_rxd_reserve()
985 		VXGE_HAL_RING_HAL_PRIV(ring, rxdp)->allocated = 1;  in vxge_hal_ring_rxd_reserve()
1018 	__hal_ring_t *ring;  in vxge_hal_ring_rxd_pre_post()  local
1031 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_rxd_pre_post()
1033 	vxge_assert(ring != NULL);  in vxge_hal_ring_rxd_pre_post()
1042 	vxge_os_spin_lock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_pre_post()
1044 	vxge_os_spin_lock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_pre_post()
1054 			rxdp_priv = VXGE_HAL_RING_HAL_PRIV(ring, rxdp);  in vxge_hal_ring_rxd_pre_post()
1057 				index = ring->channel.length;  in vxge_hal_ring_rxd_pre_post()
1061 			prev_rxdh = ring->channel.dtr_arr[index].dtr;  in vxge_hal_ring_rxd_pre_post()
1067 				    ring->rxd_size == rxdh);  in vxge_hal_ring_rxd_pre_post()
1073 	__hal_channel_dtr_post(&ring->channel, VXGE_HAL_RING_RXD_INDEX(rxdh));  in vxge_hal_ring_rxd_pre_post()
1075 	ring->db_byte_count +=  in vxge_hal_ring_rxd_pre_post()
1076 	    VXGE_HAL_RING_HAL_PRIV(ring, rxdh)->db_bytes;  in vxge_hal_ring_rxd_pre_post()
1079 	vxge_os_spin_unlock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_pre_post()
1081 	vxge_os_spin_unlock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_pre_post()
1108 	__hal_ring_t *ring;  in vxge_hal_ring_rxd_post_post()  local
1121 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_rxd_post_post()
1123 	vxge_assert(ring != NULL);  in vxge_hal_ring_rxd_post_post()
1131 	priv = __hal_ring_rxd_priv(ring, rxdp);  in vxge_hal_ring_rxd_post_post()
1132 	vxge_os_dma_sync(ring->channel.pdev,  in vxge_hal_ring_rxd_post_post()
1136 	    ring->rxd_size,  in vxge_hal_ring_rxd_post_post()
1139 	if (ring->stats->common_stats.usage_cnt > 0)  in vxge_hal_ring_rxd_post_post()
1140 		ring->stats->common_stats.usage_cnt--;  in vxge_hal_ring_rxd_post_post()
1169 	__hal_ring_t *ring;  in vxge_hal_ring_rxd_post()  local
1182 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_rxd_post()
1184 	vxge_assert(ring != NULL);  in vxge_hal_ring_rxd_post()
1201 		rxdp_temp1 = VXGE_HAL_RING_HAL_PRIV(ring, rxdp);  in vxge_hal_ring_rxd_post()
1202 		vxge_os_dma_sync(ring->channel.pdev,  in vxge_hal_ring_rxd_post()
1206 		    ring->rxd_size,  in vxge_hal_ring_rxd_post()
1212 	vxge_os_spin_lock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_post()
1214 	vxge_os_spin_lock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_post()
1224 			rxdp_temp2 = VXGE_HAL_RING_HAL_PRIV(ring, rxdp);  in vxge_hal_ring_rxd_post()
1226 			    ring->channel.dtr_arr[VXGE_HAL_RING_RXD_INDEX(rxdp) - 1].dtr;  in vxge_hal_ring_rxd_post()
1230 				vxge_assert((char *) prev_rxdh + ring->rxd_size == rxdh);  in vxge_hal_ring_rxd_post()
1235 	__hal_channel_dtr_post(&ring->channel, VXGE_HAL_RING_RXD_INDEX(rxdh));  in vxge_hal_ring_rxd_post()
1237 	ring->db_byte_count +=  in vxge_hal_ring_rxd_post()
1238 	    VXGE_HAL_RING_HAL_PRIV(ring, rxdp)->db_bytes;  in vxge_hal_ring_rxd_post()
1241 	vxge_os_spin_unlock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_post()
1243 	vxge_os_spin_unlock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_post()
1246 	if (ring->stats->common_stats.usage_cnt > 0)  in vxge_hal_ring_rxd_post()
1247 		ring->stats->common_stats.usage_cnt--;  in vxge_hal_ring_rxd_post()
1300 	__hal_ring_t *ring;  in vxge_hal_ring_rxd_post_post_db()  local
1306 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_rxd_post_post_db()
1315 	vxge_os_spin_lock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_post_post_db()
1317 	vxge_os_spin_lock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_post_post_db()
1320 	if (ring->db_byte_count <= ring->rxd_mem_avail) {  in vxge_hal_ring_rxd_post_post_db()
1321 		__hal_rxd_db_post(vpath_handle, ring->db_byte_count);  in vxge_hal_ring_rxd_post_post_db()
1322 		ring->rxd_mem_avail -= ring->db_byte_count;  in vxge_hal_ring_rxd_post_post_db()
1323 		ring->db_byte_count = 0;  in vxge_hal_ring_rxd_post_post_db()
1325 		__hal_rxd_db_post(vpath_handle, ring->rxd_mem_avail);  in vxge_hal_ring_rxd_post_post_db()
1326 		ring->db_byte_count -= ring->rxd_mem_avail;  in vxge_hal_ring_rxd_post_post_db()
1327 		ring->rxd_mem_avail = 0;  in vxge_hal_ring_rxd_post_post_db()
1331 	vxge_os_spin_unlock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_post_post_db()
1333 	vxge_os_spin_unlock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_post_post_db()
1354 	__hal_ring_t *ring;  in vxge_hal_ring_is_next_rxd_completed()  local
1371 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_is_next_rxd_completed()
1373 	vxge_assert(ring != NULL);  in vxge_hal_ring_is_next_rxd_completed()
1376 	vxge_os_spin_lock(&ring->channel.post_lock);  in vxge_hal_ring_is_next_rxd_completed()
1378 	vxge_os_spin_lock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_is_next_rxd_completed()
1381 	__hal_channel_dtr_try_complete(&ring->channel, &rxdh);  in vxge_hal_ring_is_next_rxd_completed()
1397 	vxge_os_spin_unlock(&ring->channel.post_lock);  in vxge_hal_ring_is_next_rxd_completed()
1399 	vxge_os_spin_unlock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_is_next_rxd_completed()
1448 	__hal_ring_t *ring;  in vxge_hal_ring_rxd_next_completed()  local
1472 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_rxd_next_completed()
1474 	vxge_assert(ring != NULL);  in vxge_hal_ring_rxd_next_completed()
1480 	vxge_os_spin_lock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_next_completed()
1482 	vxge_os_spin_lock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_next_completed()
1485 	__hal_channel_dtr_try_complete(&ring->channel, rxdh);  in vxge_hal_ring_rxd_next_completed()
1499 		priv = __hal_ring_rxd_priv(ring, rxdp);  in vxge_hal_ring_rxd_next_completed()
1500 		vxge_os_dma_sync(ring->channel.pdev,  in vxge_hal_ring_rxd_next_completed()
1518 			if (++ring->cmpl_cnt > ring->indicate_max_pkts) {  in vxge_hal_ring_rxd_next_completed()
1527 				__hal_channel_dtr_complete(&ring->channel);  in vxge_hal_ring_rxd_next_completed()
1529 				*rxd_priv = VXGE_HAL_RING_ULD_PRIV(ring, rxdp);  in vxge_hal_ring_rxd_next_completed()
1531 				ring->rxd_mem_avail +=  in vxge_hal_ring_rxd_next_completed()
1532 				    (VXGE_HAL_RING_HAL_PRIV(ring, rxdp))->db_bytes;  in vxge_hal_ring_rxd_next_completed()
1534 				ring->stats->common_stats.usage_cnt++;  in vxge_hal_ring_rxd_next_completed()
1535 				if (ring->stats->common_stats.usage_max <  in vxge_hal_ring_rxd_next_completed()
1536 				    ring->stats->common_stats.usage_cnt)  in vxge_hal_ring_rxd_next_completed()
1537 					ring->stats->common_stats.usage_max =  in vxge_hal_ring_rxd_next_completed()
1538 					    ring->stats->common_stats.usage_cnt;  in vxge_hal_ring_rxd_next_completed()
1540 				switch (ring->buffer_mode) {  in vxge_hal_ring_rxd_next_completed()
1542 					ring->channel.poll_bytes +=  in vxge_hal_ring_rxd_next_completed()
1547 					ring->channel.poll_bytes +=  in vxge_hal_ring_rxd_next_completed()
1556 					ring->channel.poll_bytes +=  in vxge_hal_ring_rxd_next_completed()
1578 	vxge_os_spin_unlock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_next_completed()
1580 	vxge_os_spin_unlock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_next_completed()
1777 	__hal_ring_t *ring;  in vxge_hal_ring_rxd_free()  local
1792 	ring = (__hal_ring_t *) vp->vpath->ringh;  in vxge_hal_ring_rxd_free()
1794 	vxge_assert(ring != NULL);  in vxge_hal_ring_rxd_free()
1797 	vxge_os_spin_lock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_free()
1799 	vxge_os_spin_lock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_free()
1802 	__hal_channel_dtr_free(&ring->channel, VXGE_HAL_RING_RXD_INDEX(rxdh));  in vxge_hal_ring_rxd_free()
1804 	VXGE_HAL_RING_HAL_PRIV(ring, rxdh)->allocated = 0;  in vxge_hal_ring_rxd_free()
1808 	vxge_os_spin_unlock(&ring->channel.post_lock);  in vxge_hal_ring_rxd_free()
1810 	vxge_os_spin_unlock_irq(&ring->channel.post_lock, flags);  in vxge_hal_ring_rxd_free()