67 	struct ioat_test	*test;  member
86 	struct ioat_test *test;  in ioat_test_transaction_destroy()  local
89 	test = tx->test;  in ioat_test_transaction_destroy()
93 			if (test->testkind == IOAT_TEST_DMA_8K)  in ioat_test_transaction_destroy()
105 test_transaction *ioat_test_transaction_create(struct ioat_test *test,  in ioat_test_transaction_create()  argument
115 	tx->length = test->buffer_size;  in ioat_test_transaction_create()
118 		if (test->testkind == IOAT_TEST_DMA_8K)  in ioat_test_transaction_create()
119 			tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST,  in ioat_test_transaction_create()
122 			tx->buf[i] = contigmalloc(test->buffer_size,  in ioat_test_transaction_create()
149 	struct ioat_test *test;  in ioat_compare_ok()  local
153 	test = tx->test;  in ioat_compare_ok()
159 		if (test->testkind == IOAT_TEST_FILL) {  in ioat_compare_ok()
166 		} else if (test->testkind == IOAT_TEST_DMA) {  in ioat_compare_ok()
169 		} else if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_compare_ok()
170 			if (test->raw_write)  in ioat_compare_ok()
171 				dst = test->raw_vtarget;  in ioat_compare_ok()
182 	struct ioat_test *test;  in ioat_dma_test_callback()  local
188 	test = tx->test;  in ioat_dma_test_callback()
190 	if (test->verify && !ioat_compare_ok(tx)) {  in ioat_dma_test_callback()
192 		atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);  in ioat_dma_test_callback()
193 	} else if (!test->too_late)  in ioat_dma_test_callback()
194 		atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);  in ioat_dma_test_callback()
197 	TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_dma_test_callback()
198 	TAILQ_INSERT_TAIL(&test->free_q, tx, entry);  in ioat_dma_test_callback()
199 	wakeup(&test->free_q);  in ioat_dma_test_callback()
204 ioat_test_prealloc_memory(struct ioat_test *test, int index)  in ioat_test_prealloc_memory()  argument
209 	for (i = 0; i < test->transactions; i++) {  in ioat_test_prealloc_memory()
210 		tx = ioat_test_transaction_create(test, test->chain_depth * 2);  in ioat_test_prealloc_memory()
213 			test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_test_prealloc_memory()
217 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_prealloc_memory()
219 		tx->test = test;  in ioat_test_prealloc_memory()
220 		tx->depth = test->chain_depth;  in ioat_test_prealloc_memory()
226 			for (k = 0; k < test->chain_depth; k++) {  in ioat_test_prealloc_memory()
236 ioat_test_release_memory(struct ioat_test *test)  in ioat_test_release_memory()  argument
240 	TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)  in ioat_test_release_memory()
242 	TAILQ_INIT(&test->free_q);  in ioat_test_release_memory()
244 	TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)  in ioat_test_release_memory()
246 	TAILQ_INIT(&test->pend_q);  in ioat_test_release_memory()
250 ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma)  in ioat_test_submit_1_tx()  argument
262 	while (TAILQ_EMPTY(&test->free_q))  in ioat_test_submit_1_tx()
263 		msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0);  in ioat_test_submit_1_tx()
265 	tx = TAILQ_FIRST(&test->free_q);  in ioat_test_submit_1_tx()
266 	TAILQ_REMOVE(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
267 	TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
270 	if (test->testkind != IOAT_TEST_MEMCPY)  in ioat_test_submit_1_tx()
273 		if (test->testkind == IOAT_TEST_MEMCPY) {  in ioat_test_submit_1_tx()
283 		if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_test_submit_1_tx()
284 			if (test->raw_write)  in ioat_test_submit_1_tx()
285 				dest = test->raw_target;  in ioat_test_submit_1_tx()
287 				src = test->raw_target;  in ioat_test_submit_1_tx()
298 		if (test->testkind == IOAT_TEST_DMA ||  in ioat_test_submit_1_tx()
299 		    test->testkind == IOAT_TEST_RAW_DMA)  in ioat_test_submit_1_tx()
302 		else if (test->testkind == IOAT_TEST_FILL) {  in ioat_test_submit_1_tx()
306 		} else if (test->testkind == IOAT_TEST_DMA_8K) {  in ioat_test_submit_1_tx()
314 		} else if (test->testkind == IOAT_TEST_DMA_8K_PB) {  in ioat_test_submit_1_tx()
322 		} else if (test->testkind == IOAT_TEST_DMA_CRC) {  in ioat_test_submit_1_tx()
329 		} else if (test->testkind == IOAT_TEST_DMA_CRC_COPY) {  in ioat_test_submit_1_tx()
340 	if (test->testkind == IOAT_TEST_MEMCPY)  in ioat_test_submit_1_tx()
350 		atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth);  in ioat_test_submit_1_tx()
352 		TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
353 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
362 	struct ioat_test *test;  in ioat_dma_test()  local
367 	test = arg;  in ioat_dma_test()
368 	memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status));  in ioat_dma_test()
370 	if ((test->testkind == IOAT_TEST_DMA_8K ||  in ioat_dma_test()
371 	    test->testkind == IOAT_TEST_DMA_8K_PB) &&  in ioat_dma_test()
372 	    test->buffer_size != 2 * PAGE_SIZE) {  in ioat_dma_test()
374 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
378 	if (test->buffer_size > 1024 * 1024) {  in ioat_dma_test()
380 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
384 	if (test->chain_depth * 2 > IOAT_MAX_BUFS) {  in ioat_dma_test()
387 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
391 	if (btoc((uint64_t)test->buffer_size * test->chain_depth *  in ioat_dma_test()
392 	    test->transactions) > (physmem / 4)) {  in ioat_dma_test()
395 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
399 	if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) {  in ioat_dma_test()
402 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
406 	if (test->testkind >= IOAT_NUM_TESTKINDS) {  in ioat_dma_test()
408 		    (unsigned)test->testkind);  in ioat_dma_test()
409 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
413 	dmaengine = ioat_get_dmaengine(test->channel_index, M_NOWAIT);  in ioat_dma_test()
416 		test->status[IOAT_TEST_NO_DMA_ENGINE]++;  in ioat_dma_test()
421 	if (test->testkind == IOAT_TEST_FILL &&  in ioat_dma_test()
426 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
430 	if (test->coalesce_period > ioat->intrdelay_max) {  in ioat_dma_test()
433 		    (unsigned)test->coalesce_period);  in ioat_dma_test()
434 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
437 	error = ioat_set_interrupt_coalesce(dmaengine, test->coalesce_period);  in ioat_dma_test()
438 	if (error == ENODEV && test->coalesce_period == 0)  in ioat_dma_test()
442 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
446 	if (test->zero_stats)  in ioat_dma_test()
449 	if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_dma_test()
450 		if (test->raw_is_virtual) {  in ioat_dma_test()
451 			test->raw_vtarget = (void *)test->raw_target;  in ioat_dma_test()
452 			test->raw_target = vtophys(test->raw_vtarget);  in ioat_dma_test()
454 			test->raw_vtarget = pmap_mapdev(test->raw_target,  in ioat_dma_test()
455 			    test->buffer_size);  in ioat_dma_test()
460 	TAILQ_INIT(&test->free_q);  in ioat_dma_test()
461 	TAILQ_INIT(&test->pend_q);  in ioat_dma_test()
463 	if (test->duration == 0)  in ioat_dma_test()
465 		    index, test->transactions);  in ioat_dma_test()
469 	rc = ioat_test_prealloc_memory(test, index);  in ioat_dma_test()
476 	test->too_late = false;  in ioat_dma_test()
478 	end = start + (((sbintime_t)test->duration * hz) / 1000);  in ioat_dma_test()
481 		if (test->duration == 0 && loops >= test->transactions)  in ioat_dma_test()
483 		else if (test->duration != 0 && time_after(ticks, end)) {  in ioat_dma_test()
484 			test->too_late = true;  in ioat_dma_test()
488 		ioat_test_submit_1_tx(test, dmaengine);  in ioat_dma_test()
495 	while (!TAILQ_EMPTY(&test->pend_q))  in ioat_dma_test()
496 		msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz);  in ioat_dma_test()
502 	ioat_test_release_memory(test);  in ioat_dma_test()
504 	if (test->testkind == IOAT_TEST_RAW_DMA && !test->raw_is_virtual)  in ioat_dma_test()
505 		pmap_unmapdev(test->raw_vtarget, test->buffer_size);  in ioat_dma_test()