910 			      int ring, u32 cp_int_cntl)  in cayman_cp_int_cntl_setup()  argument
914 	WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));  in cayman_cp_int_cntl_setup()
924 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cayman_fence_ring_emit()  local
925 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cayman_fence_ring_emit()
928 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));  in cayman_fence_ring_emit()
929 	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);  in cayman_fence_ring_emit()
930 	radeon_ring_write(ring, 0);  in cayman_fence_ring_emit()
931 	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));  in cayman_fence_ring_emit()
932 	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);  in cayman_fence_ring_emit()
933 	radeon_ring_write(ring, 0xFFFFFFFF);  in cayman_fence_ring_emit()
934 	radeon_ring_write(ring, 0);  in cayman_fence_ring_emit()
935 	radeon_ring_write(ring, 10); /* poll interval */  in cayman_fence_ring_emit()
937 	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));  in cayman_fence_ring_emit()
938 	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));  in cayman_fence_ring_emit()
939 	radeon_ring_write(ring, addr & 0xffffffff);  in cayman_fence_ring_emit()
940 	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));  in cayman_fence_ring_emit()
941 	radeon_ring_write(ring, fence->seq);  in cayman_fence_ring_emit()
942 	radeon_ring_write(ring, 0);  in cayman_fence_ring_emit()
947 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cayman_ring_ib_execute()  local
950 	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));  in cayman_ring_ib_execute()
951 	radeon_ring_write(ring, 1);  in cayman_ring_ib_execute()
953 	if (ring->rptr_save_reg) {  in cayman_ring_ib_execute()
954 		uint32_t next_rptr = ring->wptr + 3 + 4 + 8;  in cayman_ring_ib_execute()
955 		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));  in cayman_ring_ib_execute()
956 		radeon_ring_write(ring, ((ring->rptr_save_reg -   in cayman_ring_ib_execute()
958 		radeon_ring_write(ring, next_rptr);  in cayman_ring_ib_execute()
961 	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));  in cayman_ring_ib_execute()
962 	radeon_ring_write(ring,  in cayman_ring_ib_execute()
967 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);  in cayman_ring_ib_execute()
968 	radeon_ring_write(ring, ib->length_dw |   in cayman_ring_ib_execute()
972 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));  in cayman_ring_ib_execute()
973 	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);  in cayman_ring_ib_execute()
974 	radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);  in cayman_ring_ib_execute()
975 	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));  in cayman_ring_ib_execute()
976 	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);  in cayman_ring_ib_execute()
977 	radeon_ring_write(ring, 0xFFFFFFFF);  in cayman_ring_ib_execute()
978 	radeon_ring_write(ring, 0);  in cayman_ring_ib_execute()
979 	radeon_ring_write(ring, 10); /* poll interval */  in cayman_ring_ib_execute()
990 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cayman_cp_enable()
1023 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_cp_start()  local
1026 	r = radeon_ring_lock(rdev, ring, 7);  in cayman_cp_start()
1031 	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));  in cayman_cp_start()
1032 	radeon_ring_write(ring, 0x1);  in cayman_cp_start()
1033 	radeon_ring_write(ring, 0x0);  in cayman_cp_start()
1034 	radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);  in cayman_cp_start()
1035 	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));  in cayman_cp_start()
1036 	radeon_ring_write(ring, 0);  in cayman_cp_start()
1037 	radeon_ring_write(ring, 0);  in cayman_cp_start()
1038 	radeon_ring_unlock_commit(rdev, ring);  in cayman_cp_start()
1042 	r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);  in cayman_cp_start()
1049 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in cayman_cp_start()
1050 	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);  in cayman_cp_start()
1053 		radeon_ring_write(ring, cayman_default_state[i]);  in cayman_cp_start()
1055 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in cayman_cp_start()
1056 	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);  in cayman_cp_start()
1059 	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));  in cayman_cp_start()
1060 	radeon_ring_write(ring, 0);  in cayman_cp_start()
1063 	radeon_ring_write(ring, 0xc0026f00);  in cayman_cp_start()
1064 	radeon_ring_write(ring, 0x00000000);  in cayman_cp_start()
1065 	radeon_ring_write(ring, 0x00000000);  in cayman_cp_start()
1066 	radeon_ring_write(ring, 0x00000000);  in cayman_cp_start()
1069 	radeon_ring_write(ring, 0xc0036f00);  in cayman_cp_start()
1070 	radeon_ring_write(ring, 0x00000bc4);  in cayman_cp_start()
1071 	radeon_ring_write(ring, 0xffffffff);  in cayman_cp_start()
1072 	radeon_ring_write(ring, 0xffffffff);  in cayman_cp_start()
1073 	radeon_ring_write(ring, 0xffffffff);  in cayman_cp_start()
1075 	radeon_ring_write(ring, 0xc0026900);  in cayman_cp_start()
1076 	radeon_ring_write(ring, 0x00000316);  in cayman_cp_start()
1077 	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */  in cayman_cp_start()
1078 	radeon_ring_write(ring, 0x00000010); /*  */  in cayman_cp_start()
1080 	radeon_ring_unlock_commit(rdev, ring);  in cayman_cp_start()
1089 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_cp_fini()  local
1091 	radeon_ring_fini(rdev, ring);  in cayman_cp_fini()
1092 	radeon_scratch_free(rdev, ring->rptr_save_reg);  in cayman_cp_fini()
1122 	struct radeon_ring *ring;  in cayman_cp_resume()  local
1154 		ring = &rdev->ring[ridx[i]];  in cayman_cp_resume()
1155 		rb_cntl = drm_order(ring->ring_size / 8);  in cayman_cp_resume()
1170 		ring = &rdev->ring[ridx[i]];  in cayman_cp_resume()
1171 		WREG32(cp_rb_base[i], ring->gpu_addr >> 8);  in cayman_cp_resume()
1176 		ring = &rdev->ring[ridx[i]];  in cayman_cp_resume()
1179 		ring->rptr = ring->wptr = 0;  in cayman_cp_resume()
1180 		WREG32(ring->rptr_reg, ring->rptr);  in cayman_cp_resume()
1181 		WREG32(ring->wptr_reg, ring->wptr);  in cayman_cp_resume()
1189 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;  in cayman_cp_resume()
1190 	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;  in cayman_cp_resume()
1191 	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;  in cayman_cp_resume()
1193 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cayman_cp_resume()
1195 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cayman_cp_resume()
1196 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;  in cayman_cp_resume()
1197 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;  in cayman_cp_resume()
1227 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cayman_dma_ring_ib_execute()  local
1230 		u32 next_rptr = ring->wptr + 4;  in cayman_dma_ring_ib_execute()
1234 		radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));  in cayman_dma_ring_ib_execute()
1235 		radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);  in cayman_dma_ring_ib_execute()
1236 		radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);  in cayman_dma_ring_ib_execute()
1237 		radeon_ring_write(ring, next_rptr);  in cayman_dma_ring_ib_execute()
1243 	while ((ring->wptr & 7) != 5)  in cayman_dma_ring_ib_execute()
1244 		radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));  in cayman_dma_ring_ib_execute()
1245 	radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0));  in cayman_dma_ring_ib_execute()
1246 	radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));  in cayman_dma_ring_ib_execute()
1247 	radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));  in cayman_dma_ring_ib_execute()
1274 	rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;  in cayman_dma_stop()
1275 	rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;  in cayman_dma_stop()
1288 	struct radeon_ring *ring;  in cayman_dma_resume()  local
1302 			ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cayman_dma_resume()
1306 			ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cayman_dma_resume()
1315 		rb_bufsz = drm_order(ring->ring_size / 4);  in cayman_dma_resume()
1335 		WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8);  in cayman_dma_resume()
1348 		ring->wptr = 0;  in cayman_dma_resume()
1349 		WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);  in cayman_dma_resume()
1351 		ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2;  in cayman_dma_resume()
1355 		ring->ready = true;  in cayman_dma_resume()
1357 		r = radeon_ring_test(rdev, ring->idx, ring);  in cayman_dma_resume()
1359 			ring->ready = false;  in cayman_dma_resume()
1379 	radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);  in cayman_dma_fini()
1380 	radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);  in cayman_dma_fini()
1540 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in cayman_dma_is_lockup()  argument
1544 	if (ring->idx == R600_RING_TYPE_DMA_INDEX)  in cayman_dma_is_lockup()
1549 		radeon_ring_lockup_update(ring);  in cayman_dma_is_lockup()
1553 	radeon_ring_force_activity(rdev, ring);  in cayman_dma_is_lockup()
1554 	return radeon_ring_test_lockup(rdev, ring);  in cayman_dma_is_lockup()
1559 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_startup()  local
1659 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in cayman_startup()
1665 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cayman_startup()
1666 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,  in cayman_startup()
1673 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cayman_startup()
1674 	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,  in cayman_startup()
1752 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_init()  local
1801 	ring->ring_obj = NULL;  in cayman_init()
1802 	r600_ring_init(rdev, ring, 1024 * 1024);  in cayman_init()
1804 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cayman_init()
1805 	ring->ring_obj = NULL;  in cayman_init()
1806 	r600_ring_init(rdev, ring, 64 * 1024);  in cayman_init()
1808 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cayman_init()
1809 	ring->ring_obj = NULL;  in cayman_init()
1810 	r600_ring_init(rdev, ring, 64 * 1024);  in cayman_init()
1930 	struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index];  in cayman_vm_set_page()  local
1941 			radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, ndw));  in cayman_vm_set_page()
1942 			radeon_ring_write(ring, pe);  in cayman_vm_set_page()
1943 			radeon_ring_write(ring, upper_32_bits(pe) & 0xff);  in cayman_vm_set_page()
1955 				radeon_ring_write(ring, value);  in cayman_vm_set_page()
1956 				radeon_ring_write(ring, upper_32_bits(value));  in cayman_vm_set_page()
1966 			radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw));  in cayman_vm_set_page()
1967 			radeon_ring_write(ring, pe);  in cayman_vm_set_page()
1968 			radeon_ring_write(ring, upper_32_bits(pe) & 0xff);  in cayman_vm_set_page()
1980 				radeon_ring_write(ring, value);  in cayman_vm_set_page()
1981 				radeon_ring_write(ring, upper_32_bits(value));  in cayman_vm_set_page()
1997 	struct radeon_ring *ring = &rdev->ring[ridx];  in cayman_vm_flush()  local
2002 	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));  in cayman_vm_flush()
2003 	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);  in cayman_vm_flush()
2006 	radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));  in cayman_vm_flush()
2007 	radeon_ring_write(ring, 0x1);  in cayman_vm_flush()
2010 	radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));  in cayman_vm_flush()
2011 	radeon_ring_write(ring, 1 << vm->id);  in cayman_vm_flush()
2014 	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));  in cayman_vm_flush()
2015 	radeon_ring_write(ring, 0x0);  in cayman_vm_flush()
2020 	struct radeon_ring *ring = &rdev->ring[ridx];  in cayman_dma_vm_flush()  local
2025 	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));  in cayman_dma_vm_flush()
2026 	radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));  in cayman_dma_vm_flush()
2027 	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);  in cayman_dma_vm_flush()
2030 	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));  in cayman_dma_vm_flush()
2031 	radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));  in cayman_dma_vm_flush()
2032 	radeon_ring_write(ring, 1);  in cayman_dma_vm_flush()
2035 	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));  in cayman_dma_vm_flush()
2036 	radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));  in cayman_dma_vm_flush()
2037 	radeon_ring_write(ring, 1 << vm->id);  in cayman_dma_vm_flush()