Lines Matching refs:cb
57 #define DEBUG_LOG(cb, x...) if (krping_debug) krping_printf((cb)->cookie, x) argument
58 #define PRINTF(cb, x...) krping_printf((cb)->cookie, x) argument
245 struct krping_cb *cb = cma_id->context; in krping_cma_event_handler() local
247 DEBUG_LOG(cb, "cma_event type %d cma_id %p (%s)\n", event->event, in krping_cma_event_handler()
248 cma_id, (cma_id == cb->cm_id) ? "parent" : "child"); in krping_cma_event_handler()
252 cb->state = ADDR_RESOLVED; in krping_cma_event_handler()
255 PRINTF(cb, "rdma_resolve_route error %d\n", ret); in krping_cma_event_handler()
256 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
261 cb->state = ROUTE_RESOLVED; in krping_cma_event_handler()
262 cb->child_cm_id = cma_id; in krping_cma_event_handler()
263 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
267 cb->state = CONNECT_REQUEST; in krping_cma_event_handler()
268 cb->child_cm_id = cma_id; in krping_cma_event_handler()
269 DEBUG_LOG(cb, "child cma %p\n", cb->child_cm_id); in krping_cma_event_handler()
270 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
274 DEBUG_LOG(cb, "ESTABLISHED\n"); in krping_cma_event_handler()
275 if (!cb->server) { in krping_cma_event_handler()
276 cb->state = CONNECTED; in krping_cma_event_handler()
278 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
286 PRINTF(cb, "cma event %d, error %d\n", event->event, in krping_cma_event_handler()
288 cb->state = ERROR; in krping_cma_event_handler()
289 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
293 PRINTF(cb, "DISCONNECT EVENT...\n"); in krping_cma_event_handler()
294 cb->state = ERROR; in krping_cma_event_handler()
295 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
299 PRINTF(cb, "cma detected device removal!!!!\n"); in krping_cma_event_handler()
303 PRINTF(cb, "oof bad type!\n"); in krping_cma_event_handler()
304 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
310 static int server_recv(struct krping_cb *cb, struct ib_wc *wc) in server_recv() argument
312 if (wc->byte_len != sizeof(cb->recv_buf)) { in server_recv()
313 PRINTF(cb, "Received bogus data, size %d\n", in server_recv()
318 cb->remote_rkey = ntohl(cb->recv_buf.rkey); in server_recv()
319 cb->remote_addr = ntohll(cb->recv_buf.buf); in server_recv()
320 cb->remote_len = ntohl(cb->recv_buf.size); in server_recv()
321 DEBUG_LOG(cb, "Received rkey %x addr %llx len %d from peer\n", in server_recv()
322 cb->remote_rkey, (unsigned long long)cb->remote_addr, in server_recv()
323 cb->remote_len); in server_recv()
325 if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) in server_recv()
326 cb->state = RDMA_READ_ADV; in server_recv()
328 cb->state = RDMA_WRITE_ADV; in server_recv()
333 static int client_recv(struct krping_cb *cb, struct ib_wc *wc) in client_recv() argument
335 if (wc->byte_len != sizeof(cb->recv_buf)) { in client_recv()
336 PRINTF(cb, "Received bogus data, size %d\n", in client_recv()
341 if (cb->state == RDMA_READ_ADV) in client_recv()
342 cb->state = RDMA_WRITE_ADV; in client_recv()
344 cb->state = RDMA_WRITE_COMPLETE; in client_recv()
351 struct krping_cb *cb = ctx; in krping_cq_event_handler() local
356 BUG_ON(cb->cq != cq); in krping_cq_event_handler()
357 if (cb->state == ERROR) { in krping_cq_event_handler()
358 PRINTF(cb, "cq completion in ERROR state\n"); in krping_cq_event_handler()
361 if (cb->frtest) { in krping_cq_event_handler()
362 PRINTF(cb, "cq completion event in frtest!\n"); in krping_cq_event_handler()
365 if (!cb->wlat && !cb->rlat && !cb->bw) in krping_cq_event_handler()
366 ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); in krping_cq_event_handler()
367 while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { in krping_cq_event_handler()
370 DEBUG_LOG(cb, "cq flushed\n"); in krping_cq_event_handler()
373 PRINTF(cb, "cq completion failed with " in krping_cq_event_handler()
382 DEBUG_LOG(cb, "send completion\n"); in krping_cq_event_handler()
383 cb->stats.send_bytes += cb->send_sgl.length; in krping_cq_event_handler()
384 cb->stats.send_msgs++; in krping_cq_event_handler()
388 DEBUG_LOG(cb, "rdma write completion\n"); in krping_cq_event_handler()
389 cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length; in krping_cq_event_handler()
390 cb->stats.write_msgs++; in krping_cq_event_handler()
391 cb->state = RDMA_WRITE_COMPLETE; in krping_cq_event_handler()
392 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
396 DEBUG_LOG(cb, "rdma read completion\n"); in krping_cq_event_handler()
397 cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length; in krping_cq_event_handler()
398 cb->stats.read_msgs++; in krping_cq_event_handler()
399 cb->state = RDMA_READ_COMPLETE; in krping_cq_event_handler()
400 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
404 DEBUG_LOG(cb, "recv completion\n"); in krping_cq_event_handler()
405 cb->stats.recv_bytes += sizeof(cb->recv_buf); in krping_cq_event_handler()
406 cb->stats.recv_msgs++; in krping_cq_event_handler()
407 if (cb->wlat || cb->rlat || cb->bw) in krping_cq_event_handler()
408 ret = server_recv(cb, &wc); in krping_cq_event_handler()
410 ret = cb->server ? server_recv(cb, &wc) : in krping_cq_event_handler()
411 client_recv(cb, &wc); in krping_cq_event_handler()
413 PRINTF(cb, "recv wc error: %d\n", ret); in krping_cq_event_handler()
417 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); in krping_cq_event_handler()
419 PRINTF(cb, "post recv error: %d\n", in krping_cq_event_handler()
423 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
427 PRINTF(cb, in krping_cq_event_handler()
434 PRINTF(cb, "poll error %d\n", ret); in krping_cq_event_handler()
439 cb->state = ERROR; in krping_cq_event_handler()
440 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
443 static int krping_accept(struct krping_cb *cb) in krping_accept() argument
448 DEBUG_LOG(cb, "accepting client connection request\n"); in krping_accept()
454 ret = rdma_accept(cb->child_cm_id, &conn_param); in krping_accept()
456 PRINTF(cb, "rdma_accept error: %d\n", ret); in krping_accept()
460 if (!cb->wlat && !cb->rlat && !cb->bw) { in krping_accept()
461 wait_event_interruptible(cb->sem, cb->state >= CONNECTED); in krping_accept()
462 if (cb->state == ERROR) { in krping_accept()
463 PRINTF(cb, "wait for CONNECTED state %d\n", in krping_accept()
464 cb->state); in krping_accept()
471 static void krping_setup_wr(struct krping_cb *cb) in krping_setup_wr() argument
473 cb->recv_sgl.addr = cb->recv_dma_addr; in krping_setup_wr()
474 cb->recv_sgl.length = sizeof cb->recv_buf; in krping_setup_wr()
475 if (cb->local_dma_lkey) in krping_setup_wr()
476 cb->recv_sgl.lkey = cb->qp->device->local_dma_lkey; in krping_setup_wr()
477 else if (cb->mem == DMA) in krping_setup_wr()
478 cb->recv_sgl.lkey = cb->dma_mr->lkey; in krping_setup_wr()
480 cb->recv_sgl.lkey = cb->recv_mr->lkey; in krping_setup_wr()
481 cb->rq_wr.sg_list = &cb->recv_sgl; in krping_setup_wr()
482 cb->rq_wr.num_sge = 1; in krping_setup_wr()
484 cb->send_sgl.addr = cb->send_dma_addr; in krping_setup_wr()
485 cb->send_sgl.length = sizeof cb->send_buf; in krping_setup_wr()
486 if (cb->local_dma_lkey) in krping_setup_wr()
487 cb->send_sgl.lkey = cb->qp->device->local_dma_lkey; in krping_setup_wr()
488 else if (cb->mem == DMA) in krping_setup_wr()
489 cb->send_sgl.lkey = cb->dma_mr->lkey; in krping_setup_wr()
491 cb->send_sgl.lkey = cb->send_mr->lkey; in krping_setup_wr()
493 cb->sq_wr.opcode = IB_WR_SEND; in krping_setup_wr()
494 cb->sq_wr.send_flags = IB_SEND_SIGNALED; in krping_setup_wr()
495 cb->sq_wr.sg_list = &cb->send_sgl; in krping_setup_wr()
496 cb->sq_wr.num_sge = 1; in krping_setup_wr()
498 if (cb->server || cb->wlat || cb->rlat || cb->bw) { in krping_setup_wr()
499 cb->rdma_sgl.addr = cb->rdma_dma_addr; in krping_setup_wr()
500 if (cb->mem == MR) in krping_setup_wr()
501 cb->rdma_sgl.lkey = cb->rdma_mr->lkey; in krping_setup_wr()
502 cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; in krping_setup_wr()
503 cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; in krping_setup_wr()
504 cb->rdma_sq_wr.num_sge = 1; in krping_setup_wr()
507 switch(cb->mem) { in krping_setup_wr()
515 cb->fastreg_wr.opcode = IB_WR_FAST_REG_MR; in krping_setup_wr()
516 cb->fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; in krping_setup_wr()
517 cb->fastreg_wr.wr.fast_reg.length = cb->size; in krping_setup_wr()
518 cb->fastreg_wr.wr.fast_reg.page_list = cb->page_list; in krping_setup_wr()
519 cb->fastreg_wr.wr.fast_reg.page_list_len = cb->page_list_len; in krping_setup_wr()
521 cb->invalidate_wr.next = &cb->fastreg_wr; in krping_setup_wr()
522 cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; in krping_setup_wr()
525 cb->bind_attr.wr_id = 0xabbaabba; in krping_setup_wr()
526 cb->bind_attr.send_flags = 0; /* unsignaled */ in krping_setup_wr()
527 cb->bind_attr.length = cb->size; in krping_setup_wr()
534 static int krping_setup_buffers(struct krping_cb *cb) in krping_setup_buffers() argument
540 DEBUG_LOG(cb, "krping_setup_buffers called on cb %p\n", cb); in krping_setup_buffers()
542 cb->recv_dma_addr = dma_map_single(cb->pd->device->dma_device, in krping_setup_buffers()
543 &cb->recv_buf, in krping_setup_buffers()
544 sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); in krping_setup_buffers()
545 pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); in krping_setup_buffers()
546 cb->send_dma_addr = dma_map_single(cb->pd->device->dma_device, in krping_setup_buffers()
547 &cb->send_buf, sizeof(cb->send_buf), in krping_setup_buffers()
549 pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); in krping_setup_buffers()
551 if (cb->mem == DMA) { in krping_setup_buffers()
552 cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| in krping_setup_buffers()
555 if (IS_ERR(cb->dma_mr)) { in krping_setup_buffers()
556 DEBUG_LOG(cb, "reg_dmamr failed\n"); in krping_setup_buffers()
557 ret = PTR_ERR(cb->dma_mr); in krping_setup_buffers()
561 if (!cb->local_dma_lkey) { in krping_setup_buffers()
562 buf.addr = cb->recv_dma_addr; in krping_setup_buffers()
563 buf.size = sizeof cb->recv_buf; in krping_setup_buffers()
564 DEBUG_LOG(cb, "recv buf dma_addr %llx size %d\n", buf.addr, in krping_setup_buffers()
566 iovbase = cb->recv_dma_addr; in krping_setup_buffers()
567 cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, in krping_setup_buffers()
571 if (IS_ERR(cb->recv_mr)) { in krping_setup_buffers()
572 DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); in krping_setup_buffers()
573 ret = PTR_ERR(cb->recv_mr); in krping_setup_buffers()
577 buf.addr = cb->send_dma_addr; in krping_setup_buffers()
578 buf.size = sizeof cb->send_buf; in krping_setup_buffers()
579 DEBUG_LOG(cb, "send buf dma_addr %llx size %d\n", buf.addr, in krping_setup_buffers()
581 iovbase = cb->send_dma_addr; in krping_setup_buffers()
582 cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, in krping_setup_buffers()
585 if (IS_ERR(cb->send_mr)) { in krping_setup_buffers()
586 DEBUG_LOG(cb, "send_buf reg_mr failed\n"); in krping_setup_buffers()
587 ret = PTR_ERR(cb->send_mr); in krping_setup_buffers()
593 cb->rdma_buf = kmalloc(cb->size, GFP_KERNEL); in krping_setup_buffers()
594 if (!cb->rdma_buf) { in krping_setup_buffers()
595 DEBUG_LOG(cb, "rdma_buf malloc failed\n"); in krping_setup_buffers()
600 cb->rdma_dma_addr = dma_map_single(cb->pd->device->dma_device, in krping_setup_buffers()
601 cb->rdma_buf, cb->size, in krping_setup_buffers()
603 pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); in krping_setup_buffers()
604 if (cb->mem != DMA) { in krping_setup_buffers()
605 switch (cb->mem) { in krping_setup_buffers()
607 cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + in krping_setup_buffers()
609 cb->page_list = ib_alloc_fast_reg_page_list( in krping_setup_buffers()
610 cb->pd->device, in krping_setup_buffers()
611 cb->page_list_len); in krping_setup_buffers()
612 if (IS_ERR(cb->page_list)) { in krping_setup_buffers()
613 DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); in krping_setup_buffers()
614 ret = PTR_ERR(cb->page_list); in krping_setup_buffers()
617 cb->fastreg_mr = ib_alloc_fast_reg_mr(cb->pd, in krping_setup_buffers()
618 cb->page_list->max_page_list_len); in krping_setup_buffers()
619 if (IS_ERR(cb->fastreg_mr)) { in krping_setup_buffers()
620 DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); in krping_setup_buffers()
621 ret = PTR_ERR(cb->fastreg_mr); in krping_setup_buffers()
624 DEBUG_LOG(cb, "fastreg rkey 0x%x page_list %p" in krping_setup_buffers()
625 " page_list_len %u\n", cb->fastreg_mr->rkey, in krping_setup_buffers()
626 cb->page_list, cb->page_list_len); in krping_setup_buffers()
629 cb->mw = ib_alloc_mw(cb->pd); in krping_setup_buffers()
630 if (IS_ERR(cb->mw)) { in krping_setup_buffers()
631 DEBUG_LOG(cb, "recv_buf alloc_mw failed\n"); in krping_setup_buffers()
632 ret = PTR_ERR(cb->mw); in krping_setup_buffers()
635 DEBUG_LOG(cb, "mw rkey 0x%x\n", cb->mw->rkey); in krping_setup_buffers()
638 buf.addr = cb->rdma_dma_addr; in krping_setup_buffers()
639 buf.size = cb->size; in krping_setup_buffers()
640 iovbase = cb->rdma_dma_addr; in krping_setup_buffers()
641 cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, in krping_setup_buffers()
645 if (IS_ERR(cb->rdma_mr)) { in krping_setup_buffers()
646 DEBUG_LOG(cb, "rdma_buf reg_mr failed\n"); in krping_setup_buffers()
647 ret = PTR_ERR(cb->rdma_mr); in krping_setup_buffers()
650 DEBUG_LOG(cb, "rdma buf dma_addr %llx size %d mr rkey 0x%x\n", in krping_setup_buffers()
651 buf.addr, (int)buf.size, cb->rdma_mr->rkey); in krping_setup_buffers()
660 if (!cb->server || cb->wlat || cb->rlat || cb->bw) { in krping_setup_buffers()
662 cb->start_buf = kmalloc(cb->size, GFP_KERNEL); in krping_setup_buffers()
663 if (!cb->start_buf) { in krping_setup_buffers()
664 DEBUG_LOG(cb, "start_buf malloc failed\n"); in krping_setup_buffers()
669 cb->start_dma_addr = dma_map_single(cb->pd->device->dma_device, in krping_setup_buffers()
670 cb->start_buf, cb->size, in krping_setup_buffers()
672 pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); in krping_setup_buffers()
674 if (cb->mem == MR || cb->mem == MW) { in krping_setup_buffers()
677 if (cb->wlat || cb->rlat || cb->bw) in krping_setup_buffers()
680 buf.addr = cb->start_dma_addr; in krping_setup_buffers()
681 buf.size = cb->size; in krping_setup_buffers()
682 DEBUG_LOG(cb, "start buf dma_addr %llx size %d\n", in krping_setup_buffers()
684 iovbase = cb->start_dma_addr; in krping_setup_buffers()
685 cb->start_mr = ib_reg_phys_mr(cb->pd, &buf, 1, in krping_setup_buffers()
689 if (IS_ERR(cb->start_mr)) { in krping_setup_buffers()
690 DEBUG_LOG(cb, "start_buf reg_mr failed\n"); in krping_setup_buffers()
691 ret = PTR_ERR(cb->start_mr); in krping_setup_buffers()
697 krping_setup_wr(cb); in krping_setup_buffers()
698 DEBUG_LOG(cb, "allocated & registered buffers...\n"); in krping_setup_buffers()
701 if (cb->fastreg_mr && !IS_ERR(cb->fastreg_mr)) in krping_setup_buffers()
702 ib_dereg_mr(cb->fastreg_mr); in krping_setup_buffers()
703 if (cb->mw && !IS_ERR(cb->mw)) in krping_setup_buffers()
704 ib_dealloc_mw(cb->mw); in krping_setup_buffers()
705 if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) in krping_setup_buffers()
706 ib_dereg_mr(cb->rdma_mr); in krping_setup_buffers()
707 if (cb->page_list && !IS_ERR(cb->page_list)) in krping_setup_buffers()
708 ib_free_fast_reg_page_list(cb->page_list); in krping_setup_buffers()
709 if (cb->dma_mr && !IS_ERR(cb->dma_mr)) in krping_setup_buffers()
710 ib_dereg_mr(cb->dma_mr); in krping_setup_buffers()
711 if (cb->recv_mr && !IS_ERR(cb->recv_mr)) in krping_setup_buffers()
712 ib_dereg_mr(cb->recv_mr); in krping_setup_buffers()
713 if (cb->send_mr && !IS_ERR(cb->send_mr)) in krping_setup_buffers()
714 ib_dereg_mr(cb->send_mr); in krping_setup_buffers()
715 if (cb->rdma_buf) in krping_setup_buffers()
716 kfree(cb->rdma_buf); in krping_setup_buffers()
717 if (cb->start_buf) in krping_setup_buffers()
718 kfree(cb->start_buf); in krping_setup_buffers()
722 static void krping_free_buffers(struct krping_cb *cb) in krping_free_buffers() argument
724 DEBUG_LOG(cb, "krping_free_buffers called on cb %p\n", cb); in krping_free_buffers()
726 if (cb->dma_mr) in krping_free_buffers()
727 ib_dereg_mr(cb->dma_mr); in krping_free_buffers()
728 if (cb->send_mr) in krping_free_buffers()
729 ib_dereg_mr(cb->send_mr); in krping_free_buffers()
730 if (cb->recv_mr) in krping_free_buffers()
731 ib_dereg_mr(cb->recv_mr); in krping_free_buffers()
732 if (cb->rdma_mr) in krping_free_buffers()
733 ib_dereg_mr(cb->rdma_mr); in krping_free_buffers()
734 if (cb->start_mr) in krping_free_buffers()
735 ib_dereg_mr(cb->start_mr); in krping_free_buffers()
736 if (cb->fastreg_mr) in krping_free_buffers()
737 ib_dereg_mr(cb->fastreg_mr); in krping_free_buffers()
738 if (cb->mw) in krping_free_buffers()
739 ib_dealloc_mw(cb->mw); in krping_free_buffers()
741 dma_unmap_single(cb->pd->device->dma_device, in krping_free_buffers()
742 pci_unmap_addr(cb, recv_mapping), in krping_free_buffers()
743 sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); in krping_free_buffers()
744 dma_unmap_single(cb->pd->device->dma_device, in krping_free_buffers()
745 pci_unmap_addr(cb, send_mapping), in krping_free_buffers()
746 sizeof(cb->send_buf), DMA_BIDIRECTIONAL); in krping_free_buffers()
747 dma_unmap_single(cb->pd->device->dma_device, in krping_free_buffers()
748 pci_unmap_addr(cb, rdma_mapping), in krping_free_buffers()
749 cb->size, DMA_BIDIRECTIONAL); in krping_free_buffers()
750 kfree(cb->rdma_buf); in krping_free_buffers()
751 if (cb->start_buf) { in krping_free_buffers()
752 dma_unmap_single(cb->pd->device->dma_device, in krping_free_buffers()
753 pci_unmap_addr(cb, start_mapping), in krping_free_buffers()
754 cb->size, DMA_BIDIRECTIONAL); in krping_free_buffers()
755 kfree(cb->start_buf); in krping_free_buffers()
759 static int krping_create_qp(struct krping_cb *cb) in krping_create_qp() argument
765 init_attr.cap.max_send_wr = cb->txdepth; in krping_create_qp()
770 init_attr.send_cq = cb->cq; in krping_create_qp()
771 init_attr.recv_cq = cb->cq; in krping_create_qp()
774 if (cb->server) { in krping_create_qp()
775 ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); in krping_create_qp()
777 cb->qp = cb->child_cm_id->qp; in krping_create_qp()
779 ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); in krping_create_qp()
781 cb->qp = cb->cm_id->qp; in krping_create_qp()
787 static void krping_free_qp(struct krping_cb *cb) in krping_free_qp() argument
789 ib_destroy_qp(cb->qp); in krping_free_qp()
790 ib_destroy_cq(cb->cq); in krping_free_qp()
791 ib_dealloc_pd(cb->pd); in krping_free_qp()
794 static int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) in krping_setup_qp() argument
797 cb->pd = ib_alloc_pd(cm_id->device); in krping_setup_qp()
798 if (IS_ERR(cb->pd)) { in krping_setup_qp()
799 PRINTF(cb, "ib_alloc_pd failed\n"); in krping_setup_qp()
800 return PTR_ERR(cb->pd); in krping_setup_qp()
802 DEBUG_LOG(cb, "created pd %p\n", cb->pd); in krping_setup_qp()
804 strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); in krping_setup_qp()
806 cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, in krping_setup_qp()
807 cb, cb->txdepth * 2, 0); in krping_setup_qp()
808 if (IS_ERR(cb->cq)) { in krping_setup_qp()
809 PRINTF(cb, "ib_create_cq failed\n"); in krping_setup_qp()
810 ret = PTR_ERR(cb->cq); in krping_setup_qp()
813 DEBUG_LOG(cb, "created cq %p\n", cb->cq); in krping_setup_qp()
815 if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { in krping_setup_qp()
816 ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); in krping_setup_qp()
818 PRINTF(cb, "ib_create_cq failed\n"); in krping_setup_qp()
823 ret = krping_create_qp(cb); in krping_setup_qp()
825 PRINTF(cb, "krping_create_qp failed: %d\n", ret); in krping_setup_qp()
828 DEBUG_LOG(cb, "created qp %p\n", cb->qp); in krping_setup_qp()
831 ib_destroy_cq(cb->cq); in krping_setup_qp()
833 ib_dealloc_pd(cb->pd); in krping_setup_qp()
843 static u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) in krping_rdma_rkey() argument
851 switch (cb->mem) { in krping_rdma_rkey()
853 cb->invalidate_wr.ex.invalidate_rkey = cb->fastreg_mr->rkey; in krping_rdma_rkey()
858 ib_update_fast_reg_key(cb->fastreg_mr, ++cb->key); in krping_rdma_rkey()
859 cb->fastreg_wr.wr.fast_reg.rkey = cb->fastreg_mr->rkey; in krping_rdma_rkey()
864 if (buf == (u64)cb->start_dma_addr) in krping_rdma_rkey()
865 cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_READ; in krping_rdma_rkey()
867 cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; in krping_rdma_rkey()
868 cb->fastreg_wr.wr.fast_reg.iova_start = buf; in krping_rdma_rkey()
870 for (i=0; i < cb->fastreg_wr.wr.fast_reg.page_list_len; in krping_rdma_rkey()
872 cb->page_list->page_list[i] = p; in krping_rdma_rkey()
873 DEBUG_LOG(cb, "page_list[%d] 0x%llx\n", i, p); in krping_rdma_rkey()
876 DEBUG_LOG(cb, "post_inv = %d, fastreg new rkey 0x%x shift %u len %u" in krping_rdma_rkey()
879 cb->fastreg_wr.wr.fast_reg.rkey, in krping_rdma_rkey()
880 cb->fastreg_wr.wr.fast_reg.page_shift, in krping_rdma_rkey()
881 cb->fastreg_wr.wr.fast_reg.length, in krping_rdma_rkey()
882 cb->fastreg_wr.wr.fast_reg.iova_start, in krping_rdma_rkey()
883 cb->fastreg_wr.wr.fast_reg.page_list_len); in krping_rdma_rkey()
886 ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); in krping_rdma_rkey()
888 ret = ib_post_send(cb->qp, &cb->fastreg_wr, &bad_wr); in krping_rdma_rkey()
890 PRINTF(cb, "post send error %d\n", ret); in krping_rdma_rkey()
891 cb->state = ERROR; in krping_rdma_rkey()
893 rkey = cb->fastreg_mr->rkey; in krping_rdma_rkey()
899 if (buf == (u64)cb->start_dma_addr) { in krping_rdma_rkey()
900 cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_READ; in krping_rdma_rkey()
901 cb->bind_attr.mr = cb->start_mr; in krping_rdma_rkey()
903 cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_WRITE; in krping_rdma_rkey()
904 cb->bind_attr.mr = cb->rdma_mr; in krping_rdma_rkey()
906 cb->bind_attr.addr = buf; in krping_rdma_rkey()
907 DEBUG_LOG(cb, "binding mw rkey 0x%x to buf %llx mr rkey 0x%x\n", in krping_rdma_rkey()
908 cb->mw->rkey, buf, cb->bind_attr.mr->rkey); in krping_rdma_rkey()
909 ret = ib_bind_mw(cb->qp, cb->mw, &cb->bind_attr); in krping_rdma_rkey()
911 PRINTF(cb, "bind mw error %d\n", ret); in krping_rdma_rkey()
912 cb->state = ERROR; in krping_rdma_rkey()
914 rkey = cb->mw->rkey; in krping_rdma_rkey()
917 if (buf == (u64)cb->start_dma_addr) in krping_rdma_rkey()
918 rkey = cb->start_mr->rkey; in krping_rdma_rkey()
920 rkey = cb->rdma_mr->rkey; in krping_rdma_rkey()
923 rkey = cb->dma_mr->rkey; in krping_rdma_rkey()
926 PRINTF(cb, "%s:%d case ERROR\n", __func__, __LINE__); in krping_rdma_rkey()
927 cb->state = ERROR; in krping_rdma_rkey()
933 static void krping_format_send(struct krping_cb *cb, u64 buf) in krping_format_send() argument
935 struct krping_rdma_info *info = &cb->send_buf; in krping_format_send()
943 if (!cb->server || cb->wlat || cb->rlat || cb->bw) { in krping_format_send()
944 rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); in krping_format_send()
947 info->size = htonl(cb->size); in krping_format_send()
948 DEBUG_LOG(cb, "RDMA addr %llx rkey %x len %d\n", in krping_format_send()
949 (unsigned long long)buf, rkey, cb->size); in krping_format_send()
953 static void krping_test_server(struct krping_cb *cb) in krping_test_server() argument
960 wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); in krping_test_server()
961 if (cb->state != RDMA_READ_ADV) { in krping_test_server()
962 PRINTF(cb, "wait for RDMA_READ_ADV state %d\n", in krping_test_server()
963 cb->state); in krping_test_server()
967 DEBUG_LOG(cb, "server received sink adv\n"); in krping_test_server()
969 cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; in krping_test_server()
970 cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; in krping_test_server()
971 cb->rdma_sq_wr.sg_list->length = cb->remote_len; in krping_test_server()
972 cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 1); in krping_test_server()
973 cb->rdma_sq_wr.next = NULL; in krping_test_server()
976 if (cb->read_inv) in krping_test_server()
977 cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ_WITH_INV; in krping_test_server()
980 cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; in krping_test_server()
981 if (cb->mem == FASTREG) { in krping_test_server()
986 cb->rdma_sq_wr.next = &inv; in krping_test_server()
989 inv.ex.invalidate_rkey = cb->fastreg_mr->rkey; in krping_test_server()
994 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); in krping_test_server()
996 PRINTF(cb, "post send error %d\n", ret); in krping_test_server()
999 cb->rdma_sq_wr.next = NULL; in krping_test_server()
1001 DEBUG_LOG(cb, "server posted rdma read req \n"); in krping_test_server()
1004 wait_event_interruptible(cb->sem, in krping_test_server()
1005 cb->state >= RDMA_READ_COMPLETE); in krping_test_server()
1006 if (cb->state != RDMA_READ_COMPLETE) { in krping_test_server()
1007 PRINTF(cb, in krping_test_server()
1009 cb->state); in krping_test_server()
1012 DEBUG_LOG(cb, "server received read complete\n"); in krping_test_server()
1015 if (cb->verbose) in krping_test_server()
1016 PRINTF(cb, "server ping data: %s\n", in krping_test_server()
1017 cb->rdma_buf); in krping_test_server()
1020 if (cb->server && cb->server_invalidate) { in krping_test_server()
1021 cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; in krping_test_server()
1022 cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; in krping_test_server()
1023 DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); in krping_test_server()
1025 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_server()
1027 PRINTF(cb, "post send error %d\n", ret); in krping_test_server()
1030 DEBUG_LOG(cb, "server posted go ahead\n"); in krping_test_server()
1033 wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); in krping_test_server()
1034 if (cb->state != RDMA_WRITE_ADV) { in krping_test_server()
1035 PRINTF(cb, in krping_test_server()
1037 cb->state); in krping_test_server()
1040 DEBUG_LOG(cb, "server received sink adv\n"); in krping_test_server()
1043 cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; in krping_test_server()
1044 cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; in krping_test_server()
1045 cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; in krping_test_server()
1046 cb->rdma_sq_wr.sg_list->length = strlen(cb->rdma_buf) + 1; in krping_test_server()
1047 if (cb->local_dma_lkey) in krping_test_server()
1048 cb->rdma_sgl.lkey = cb->qp->device->local_dma_lkey; in krping_test_server()
1050 cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 0); in krping_test_server()
1052 DEBUG_LOG(cb, "rdma write from lkey %x laddr %llx len %d\n", in krping_test_server()
1053 cb->rdma_sq_wr.sg_list->lkey, in krping_test_server()
1054 (unsigned long long)cb->rdma_sq_wr.sg_list->addr, in krping_test_server()
1055 cb->rdma_sq_wr.sg_list->length); in krping_test_server()
1057 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); in krping_test_server()
1059 PRINTF(cb, "post send error %d\n", ret); in krping_test_server()
1064 ret = wait_event_interruptible(cb->sem, cb->state >= in krping_test_server()
1066 if (cb->state != RDMA_WRITE_COMPLETE) { in krping_test_server()
1067 PRINTF(cb, in krping_test_server()
1069 cb->state); in krping_test_server()
1072 DEBUG_LOG(cb, "server rdma write complete \n"); in krping_test_server()
1074 cb->state = CONNECTED; in krping_test_server()
1077 if (cb->server && cb->server_invalidate) { in krping_test_server()
1078 cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; in krping_test_server()
1079 cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; in krping_test_server()
1080 DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); in krping_test_server()
1082 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_server()
1084 PRINTF(cb, "post send error %d\n", ret); in krping_test_server()
1087 DEBUG_LOG(cb, "server posted go ahead\n"); in krping_test_server()
1091 static void rlat_test(struct krping_cb *cb) in rlat_test() argument
1094 int iters = cb->count; in rlat_test()
1102 cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; in rlat_test()
1103 cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; in rlat_test()
1104 cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; in rlat_test()
1105 cb->rdma_sq_wr.sg_list->length = cb->size; in rlat_test()
1108 if (!cb->poll) { in rlat_test()
1109 cb->state = RDMA_READ_ADV; in rlat_test()
1110 ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); in rlat_test()
1114 cb->state = RDMA_READ_ADV; in rlat_test()
1115 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); in rlat_test()
1117 PRINTF(cb, in rlat_test()
1124 if (!cb->poll) { in rlat_test()
1125 wait_event_interruptible(cb->sem, in rlat_test()
1126 cb->state != RDMA_READ_ADV); in rlat_test()
1127 if (cb->state == RDMA_READ_COMPLETE) { in rlat_test()
1129 ib_req_notify_cq(cb->cq, in rlat_test()
1135 ne = ib_poll_cq(cb->cq, 1, &wc); in rlat_test()
1136 if (cb->state == ERROR) { in rlat_test()
1137 PRINTF(cb, in rlat_test()
1145 PRINTF(cb, "poll CQ failed %d\n", ne); in rlat_test()
1148 if (cb->poll && wc.status != IB_WC_SUCCESS) { in rlat_test()
1149 PRINTF(cb, "Completion wth error at %s:\n", in rlat_test()
1150 cb->server ? "server" : "client"); in rlat_test()
1151 PRINTF(cb, "Failed status %d: wr_id %d\n", in rlat_test()
1164 PRINTF(cb, "delta sec %lu delta usec %lu iter %d size %d\n", in rlat_test()
1167 scnt, cb->size); in rlat_test()
1170 static void wlat_test(struct krping_cb *cb) in wlat_test() argument
1173 int iters=cb->count; in wlat_test()
1174 volatile char *poll_buf = (char *) cb->start_buf; in wlat_test()
1175 char *buf = (char *)cb->rdma_buf; in wlat_test()
1190 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in wlat_test()
1195 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in wlat_test()
1200 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in wlat_test()
1205 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in wlat_test()
1211 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in wlat_test()
1214 cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; in wlat_test()
1215 cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; in wlat_test()
1216 cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; in wlat_test()
1217 cb->rdma_sq_wr.sg_list->length = cb->size; in wlat_test()
1225 if (rcnt < iters && !(scnt < 1 && !cb->server)) { in wlat_test()
1228 if (cb->state == ERROR) { in wlat_test()
1229 PRINTF(cb, in wlat_test()
1242 if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { in wlat_test()
1243 PRINTF(cb, in wlat_test()
1263 ne = ib_poll_cq(cb->cq, 1, &wc); in wlat_test()
1270 PRINTF(cb, "poll CQ failed %d\n", ne); in wlat_test()
1274 PRINTF(cb, in wlat_test()
1276 cb->server ? "server" : "client"); in wlat_test()
1277 PRINTF(cb, in wlat_test()
1280 PRINTF(cb, in wlat_test()
1299 PRINTF(cb, in wlat_test()
1304 scnt, cb->size, cycle_iters, in wlat_test()
1314 static void bw_test(struct krping_cb *cb) in bw_test() argument
1317 int iters=cb->count; in bw_test()
1332 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in bw_test()
1337 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in bw_test()
1342 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in bw_test()
1347 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in bw_test()
1353 PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); in bw_test()
1356 cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; in bw_test()
1357 cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; in bw_test()
1358 cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; in bw_test()
1359 cb->rdma_sq_wr.sg_list->length = cb->size; in bw_test()
1366 while (scnt < iters && scnt - ccnt < cb->txdepth) { in bw_test()
1371 if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { in bw_test()
1372 PRINTF(cb, in bw_test()
1392 ne = ib_poll_cq(cb->cq, 1, &wc); in bw_test()
1399 PRINTF(cb, "poll CQ failed %d\n", ne); in bw_test()
1403 PRINTF(cb, in bw_test()
1405 cb->server ? "server" : "client"); in bw_test()
1406 PRINTF(cb, in bw_test()
1425 PRINTF(cb, in bw_test()
1430 scnt, cb->size, cycle_iters, in bw_test()
1440 static void krping_rlat_test_server(struct krping_cb *cb) in krping_rlat_test_server() argument
1447 while (cb->state < RDMA_READ_ADV) { in krping_rlat_test_server()
1448 krping_cq_event_handler(cb->cq, cb); in krping_rlat_test_server()
1452 krping_format_send(cb, cb->start_dma_addr); in krping_rlat_test_server()
1453 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_rlat_test_server()
1455 PRINTF(cb, "post send error %d\n", ret); in krping_rlat_test_server()
1460 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_rlat_test_server()
1462 PRINTF(cb, "poll error %d\n", ret); in krping_rlat_test_server()
1466 PRINTF(cb, "send completiong error %d\n", wc.status); in krping_rlat_test_server()
1470 wait_event_interruptible(cb->sem, cb->state == ERROR); in krping_rlat_test_server()
1473 static void krping_wlat_test_server(struct krping_cb *cb) in krping_wlat_test_server() argument
1480 while (cb->state < RDMA_READ_ADV) { in krping_wlat_test_server()
1481 krping_cq_event_handler(cb->cq, cb); in krping_wlat_test_server()
1485 krping_format_send(cb, cb->start_dma_addr); in krping_wlat_test_server()
1486 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_wlat_test_server()
1488 PRINTF(cb, "post send error %d\n", ret); in krping_wlat_test_server()
1493 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_wlat_test_server()
1495 PRINTF(cb, "poll error %d\n", ret); in krping_wlat_test_server()
1499 PRINTF(cb, "send completiong error %d\n", wc.status); in krping_wlat_test_server()
1503 wlat_test(cb); in krping_wlat_test_server()
1504 wait_event_interruptible(cb->sem, cb->state == ERROR); in krping_wlat_test_server()
1507 static void krping_bw_test_server(struct krping_cb *cb) in krping_bw_test_server() argument
1514 while (cb->state < RDMA_READ_ADV) { in krping_bw_test_server()
1515 krping_cq_event_handler(cb->cq, cb); in krping_bw_test_server()
1519 krping_format_send(cb, cb->start_dma_addr); in krping_bw_test_server()
1520 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_bw_test_server()
1522 PRINTF(cb, "post send error %d\n", ret); in krping_bw_test_server()
1527 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_bw_test_server()
1529 PRINTF(cb, "poll error %d\n", ret); in krping_bw_test_server()
1533 PRINTF(cb, "send completiong error %d\n", wc.status); in krping_bw_test_server()
1537 if (cb->duplex) in krping_bw_test_server()
1538 bw_test(cb); in krping_bw_test_server()
1539 wait_event_interruptible(cb->sem, cb->state == ERROR); in krping_bw_test_server()
1542 static int fastreg_supported(struct krping_cb *cb) in fastreg_supported() argument
1544 struct ib_device *dev = cb->child_cm_id->device; in fastreg_supported()
1550 PRINTF(cb, "ib_query_device failed ret %d\n", ret); in fastreg_supported()
1554 PRINTF(cb, "Fastreg not supported - device_cap_flags 0x%x\n", in fastreg_supported()
1558 DEBUG_LOG(cb, "Fastreg supported - device_cap_flags 0x%x\n", in fastreg_supported()
1563 static int krping_bind_server(struct krping_cb *cb) in krping_bind_server() argument
1571 sin.sin_addr.s_addr = cb->addr.s_addr; in krping_bind_server()
1572 sin.sin_port = cb->port; in krping_bind_server()
1574 ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &sin); in krping_bind_server()
1576 PRINTF(cb, "rdma_bind_addr error %d\n", ret); in krping_bind_server()
1579 DEBUG_LOG(cb, "rdma_bind_addr successful\n"); in krping_bind_server()
1581 DEBUG_LOG(cb, "rdma_listen\n"); in krping_bind_server()
1582 ret = rdma_listen(cb->cm_id, 3); in krping_bind_server()
1584 PRINTF(cb, "rdma_listen failed: %d\n", ret); in krping_bind_server()
1588 wait_event_interruptible(cb->sem, cb->state >= CONNECT_REQUEST); in krping_bind_server()
1589 if (cb->state != CONNECT_REQUEST) { in krping_bind_server()
1590 PRINTF(cb, "wait for CONNECT_REQUEST state %d\n", in krping_bind_server()
1591 cb->state); in krping_bind_server()
1595 if (cb->mem == FASTREG && !fastreg_supported(cb)) in krping_bind_server()
1601 static void krping_run_server(struct krping_cb *cb) in krping_run_server() argument
1606 ret = krping_bind_server(cb); in krping_run_server()
1610 ret = krping_setup_qp(cb, cb->child_cm_id); in krping_run_server()
1612 PRINTF(cb, "setup_qp failed: %d\n", ret); in krping_run_server()
1616 ret = krping_setup_buffers(cb); in krping_run_server()
1618 PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); in krping_run_server()
1622 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); in krping_run_server()
1624 PRINTF(cb, "ib_post_recv failed: %d\n", ret); in krping_run_server()
1628 ret = krping_accept(cb); in krping_run_server()
1630 PRINTF(cb, "connect error %d\n", ret); in krping_run_server()
1634 if (cb->wlat) in krping_run_server()
1635 krping_wlat_test_server(cb); in krping_run_server()
1636 else if (cb->rlat) in krping_run_server()
1637 krping_rlat_test_server(cb); in krping_run_server()
1638 else if (cb->bw) in krping_run_server()
1639 krping_bw_test_server(cb); in krping_run_server()
1641 krping_test_server(cb); in krping_run_server()
1642 rdma_disconnect(cb->child_cm_id); in krping_run_server()
1644 krping_free_buffers(cb); in krping_run_server()
1646 krping_free_qp(cb); in krping_run_server()
1648 rdma_destroy_id(cb->child_cm_id); in krping_run_server()
1651 static void krping_test_client(struct krping_cb *cb) in krping_test_client() argument
1658 for (ping = 0; !cb->count || ping < cb->count; ping++) { in krping_test_client()
1659 cb->state = RDMA_READ_ADV; in krping_test_client()
1662 cc = sprintf(cb->start_buf, "rdma-ping-%d: ", ping); in krping_test_client()
1663 for (i = cc, c = start; i < cb->size; i++) { in krping_test_client()
1664 cb->start_buf[i] = c; in krping_test_client()
1672 cb->start_buf[cb->size - 1] = 0; in krping_test_client()
1674 krping_format_send(cb, cb->start_dma_addr); in krping_test_client()
1675 if (cb->state == ERROR) { in krping_test_client()
1676 PRINTF(cb, "krping_format_send failed\n"); in krping_test_client()
1679 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_client()
1681 PRINTF(cb, "post send error %d\n", ret); in krping_test_client()
1686 wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); in krping_test_client()
1687 if (cb->state != RDMA_WRITE_ADV) { in krping_test_client()
1688 PRINTF(cb, in krping_test_client()
1690 cb->state); in krping_test_client()
1694 krping_format_send(cb, cb->rdma_dma_addr); in krping_test_client()
1695 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_client()
1697 PRINTF(cb, "post send error %d\n", ret); in krping_test_client()
1702 wait_event_interruptible(cb->sem, in krping_test_client()
1703 cb->state >= RDMA_WRITE_COMPLETE); in krping_test_client()
1704 if (cb->state != RDMA_WRITE_COMPLETE) { in krping_test_client()
1705 PRINTF(cb, in krping_test_client()
1707 cb->state); in krping_test_client()
1711 if (cb->validate) in krping_test_client()
1712 if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { in krping_test_client()
1713 PRINTF(cb, "data mismatch!\n"); in krping_test_client()
1717 if (cb->verbose) in krping_test_client()
1718 PRINTF(cb, "ping data: %s\n", cb->rdma_buf); in krping_test_client()
1720 wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); in krping_test_client()
1725 static void krping_rlat_test_client(struct krping_cb *cb) in krping_rlat_test_client() argument
1731 cb->state = RDMA_READ_ADV; in krping_rlat_test_client()
1734 krping_format_send(cb, cb->start_dma_addr); in krping_rlat_test_client()
1735 if (cb->state == ERROR) { in krping_rlat_test_client()
1736 PRINTF(cb, "krping_format_send failed\n"); in krping_rlat_test_client()
1739 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_rlat_test_client()
1741 PRINTF(cb, "post send error %d\n", ret); in krping_rlat_test_client()
1746 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_rlat_test_client()
1748 PRINTF(cb, "poll error %d\n", ret); in krping_rlat_test_client()
1752 PRINTF(cb, "send completion error %d\n", wc.status); in krping_rlat_test_client()
1757 while (cb->state < RDMA_WRITE_ADV) { in krping_rlat_test_client()
1758 krping_cq_event_handler(cb->cq, cb); in krping_rlat_test_client()
1772 cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; in krping_rlat_test_client()
1773 cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; in krping_rlat_test_client()
1774 cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; in krping_rlat_test_client()
1775 cb->rdma_sq_wr.sg_list->length = 0; in krping_rlat_test_client()
1776 cb->rdma_sq_wr.num_sge = 0; in krping_rlat_test_client()
1780 if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { in krping_rlat_test_client()
1781 PRINTF(cb, "Couldn't post send\n"); in krping_rlat_test_client()
1785 ne = ib_poll_cq(cb->cq, 1, &wc); in krping_rlat_test_client()
1788 PRINTF(cb, "poll CQ failed %d\n", ne); in krping_rlat_test_client()
1792 PRINTF(cb, "Completion wth error at %s:\n", in krping_rlat_test_client()
1793 cb->server ? "server" : "client"); in krping_rlat_test_client()
1794 PRINTF(cb, "Failed status %d: wr_id %d\n", in krping_rlat_test_client()
1808 PRINTF(cb, "0B-write-lat iters 100000 usec %llu\n", elapsed); in krping_rlat_test_client()
1812 rlat_test(cb); in krping_rlat_test_client()
1815 static void krping_wlat_test_client(struct krping_cb *cb) in krping_wlat_test_client() argument
1821 cb->state = RDMA_READ_ADV; in krping_wlat_test_client()
1824 krping_format_send(cb, cb->start_dma_addr); in krping_wlat_test_client()
1825 if (cb->state == ERROR) { in krping_wlat_test_client()
1826 PRINTF(cb, "krping_format_send failed\n"); in krping_wlat_test_client()
1829 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_wlat_test_client()
1831 PRINTF(cb, "post send error %d\n", ret); in krping_wlat_test_client()
1836 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_wlat_test_client()
1838 PRINTF(cb, "poll error %d\n", ret); in krping_wlat_test_client()
1842 PRINTF(cb, "send completion error %d\n", wc.status); in krping_wlat_test_client()
1847 while (cb->state < RDMA_WRITE_ADV) { in krping_wlat_test_client()
1848 krping_cq_event_handler(cb->cq, cb); in krping_wlat_test_client()
1851 wlat_test(cb); in krping_wlat_test_client()
1854 static void krping_bw_test_client(struct krping_cb *cb) in krping_bw_test_client() argument
1860 cb->state = RDMA_READ_ADV; in krping_bw_test_client()
1863 krping_format_send(cb, cb->start_dma_addr); in krping_bw_test_client()
1864 if (cb->state == ERROR) { in krping_bw_test_client()
1865 PRINTF(cb, "krping_format_send failed\n"); in krping_bw_test_client()
1868 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_bw_test_client()
1870 PRINTF(cb, "post send error %d\n", ret); in krping_bw_test_client()
1875 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_bw_test_client()
1877 PRINTF(cb, "poll error %d\n", ret); in krping_bw_test_client()
1881 PRINTF(cb, "send completion error %d\n", wc.status); in krping_bw_test_client()
1886 while (cb->state < RDMA_WRITE_ADV) { in krping_bw_test_client()
1887 krping_cq_event_handler(cb->cq, cb); in krping_bw_test_client()
1890 bw_test(cb); in krping_bw_test_client()
1893 static void krping_fr_test(struct krping_cb *cb) in krping_fr_test() argument
1902 int size = cb->size; in krping_fr_test()
1908 pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); in krping_fr_test()
1910 PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); in krping_fr_test()
1914 mr = ib_alloc_fast_reg_mr(cb->pd, plen); in krping_fr_test()
1916 PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); in krping_fr_test()
1936 …DEBUG_LOG(cb, "fr_test: stag index 0x%x plen %u size %u depth %u\n", mr->rkey >> 8, plen, cb->size… in krping_fr_test()
1940 DEBUG_LOG(cb, "fr_test: pausing 1 second! count %u latest size %u plen %u\n", count, size, plen); in krping_fr_test()
1941 wait_event_interruptible(cb->sem, cb->state == ERROR); in krping_fr_test()
1942 if (cb->state == ERROR) in krping_fr_test()
1946 while (scnt < (cb->txdepth>>1)) { in krping_fr_test()
1950 size = arc4random() % cb->size; in krping_fr_test()
1952 size = cb->size; in krping_fr_test()
1956 ret = ib_post_send(cb->qp, &fr, &bad); in krping_fr_test()
1958 PRINTF(cb, "ib_post_send failed %d\n", ret); in krping_fr_test()
1965 ret = ib_poll_cq(cb->cq, 1, &wc); in krping_fr_test()
1967 PRINTF(cb, "ib_poll_cq failed %d\n", ret); in krping_fr_test()
1972 PRINTF(cb, "completion error %u\n", wc.status); in krping_fr_test()
1979 PRINTF(cb, "signal!\n"); in krping_fr_test()
1986 DEBUG_LOG(cb, "sleeping 1 second\n"); in krping_fr_test()
1987 wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); in krping_fr_test()
1989 DEBUG_LOG(cb, "draining the cq...\n"); in krping_fr_test()
1991 ret = ib_poll_cq(cb->cq, 1, &wc); in krping_fr_test()
1993 PRINTF(cb, "ib_poll_cq failed %d\n", ret); in krping_fr_test()
1998 PRINTF(cb, "completion error %u opcode %u\n", wc.status, wc.opcode); in krping_fr_test()
2002 DEBUG_LOG(cb, "fr_test: done!\n"); in krping_fr_test()
2008 static int krping_connect_client(struct krping_cb *cb) in krping_connect_client() argument
2018 ret = rdma_connect(cb->cm_id, &conn_param); in krping_connect_client()
2020 PRINTF(cb, "rdma_connect error %d\n", ret); in krping_connect_client()
2024 wait_event_interruptible(cb->sem, cb->state >= CONNECTED); in krping_connect_client()
2025 if (cb->state == ERROR) { in krping_connect_client()
2026 PRINTF(cb, "wait for CONNECTED state %d\n", cb->state); in krping_connect_client()
2030 DEBUG_LOG(cb, "rdma_connect successful\n"); in krping_connect_client()
2034 static int krping_bind_client(struct krping_cb *cb) in krping_bind_client() argument
2042 sin.sin_addr.s_addr = cb->addr.s_addr; in krping_bind_client()
2043 sin.sin_port = cb->port; in krping_bind_client()
2045 ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &sin, in krping_bind_client()
2048 PRINTF(cb, "rdma_resolve_addr error %d\n", ret); in krping_bind_client()
2052 wait_event_interruptible(cb->sem, cb->state >= ROUTE_RESOLVED); in krping_bind_client()
2053 if (cb->state != ROUTE_RESOLVED) { in krping_bind_client()
2054 PRINTF(cb, in krping_bind_client()
2056 cb->state); in krping_bind_client()
2060 if (cb->mem == FASTREG && !fastreg_supported(cb)) in krping_bind_client()
2063 DEBUG_LOG(cb, "rdma_resolve_addr - rdma_resolve_route successful\n"); in krping_bind_client()
2067 static void krping_run_client(struct krping_cb *cb) in krping_run_client() argument
2072 ret = krping_bind_client(cb); in krping_run_client()
2076 ret = krping_setup_qp(cb, cb->cm_id); in krping_run_client()
2078 PRINTF(cb, "setup_qp failed: %d\n", ret); in krping_run_client()
2082 ret = krping_setup_buffers(cb); in krping_run_client()
2084 PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); in krping_run_client()
2088 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); in krping_run_client()
2090 PRINTF(cb, "ib_post_recv failed: %d\n", ret); in krping_run_client()
2094 ret = krping_connect_client(cb); in krping_run_client()
2096 PRINTF(cb, "connect error %d\n", ret); in krping_run_client()
2100 if (cb->wlat) in krping_run_client()
2101 krping_wlat_test_client(cb); in krping_run_client()
2102 else if (cb->rlat) in krping_run_client()
2103 krping_rlat_test_client(cb); in krping_run_client()
2104 else if (cb->bw) in krping_run_client()
2105 krping_bw_test_client(cb); in krping_run_client()
2106 else if (cb->frtest) in krping_run_client()
2107 krping_fr_test(cb); in krping_run_client()
2109 krping_test_client(cb); in krping_run_client()
2110 rdma_disconnect(cb->cm_id); in krping_run_client()
2112 krping_free_buffers(cb); in krping_run_client()
2114 krping_free_qp(cb); in krping_run_client()
2119 struct krping_cb *cb; in krping_doit() local
2125 cb = kzalloc(sizeof(*cb), GFP_KERNEL); in krping_doit()
2126 if (!cb) in krping_doit()
2130 list_add_tail(&cb->list, &krping_cbs); in krping_doit()
2133 cb->cookie = cookie; in krping_doit()
2134 cb->server = -1; in krping_doit()
2135 cb->state = IDLE; in krping_doit()
2136 cb->size = 64; in krping_doit()
2137 cb->txdepth = RPING_SQ_DEPTH; in krping_doit()
2138 cb->mem = DMA; in krping_doit()
2139 init_waitqueue_head(&cb->sem); in krping_doit()
2145 cb->addr_str = optarg; in krping_doit()
2146 DEBUG_LOG(cb, "ipaddr (%s)\n", optarg); in krping_doit()
2147 if (!inet_aton(optarg, &cb->addr)) { in krping_doit()
2148 PRINTF(cb, "bad addr string %s\n", in krping_doit()
2154 cb->port = htons(optint); in krping_doit()
2155 DEBUG_LOG(cb, "port %d\n", (int)optint); in krping_doit()
2158 cb->poll = 1; in krping_doit()
2159 DEBUG_LOG(cb, "server\n"); in krping_doit()
2162 cb->server = 1; in krping_doit()
2163 DEBUG_LOG(cb, "server\n"); in krping_doit()
2166 cb->server = 0; in krping_doit()
2167 DEBUG_LOG(cb, "client\n"); in krping_doit()
2170 cb->size = optint; in krping_doit()
2171 if ((cb->size < 1) || in krping_doit()
2172 (cb->size > RPING_BUFSIZE)) { in krping_doit()
2173 PRINTF(cb, "Invalid size %d " in krping_doit()
2175 cb->size, RPING_BUFSIZE); in krping_doit()
2178 DEBUG_LOG(cb, "size %d\n", (int)optint); in krping_doit()
2181 cb->count = optint; in krping_doit()
2182 if (cb->count < 0) { in krping_doit()
2183 PRINTF(cb, "Invalid count %d\n", in krping_doit()
2184 cb->count); in krping_doit()
2187 DEBUG_LOG(cb, "count %d\n", (int) cb->count); in krping_doit()
2190 cb->verbose++; in krping_doit()
2191 DEBUG_LOG(cb, "verbose\n"); in krping_doit()
2194 cb->validate++; in krping_doit()
2195 DEBUG_LOG(cb, "validate data\n"); in krping_doit()
2198 cb->wlat++; in krping_doit()
2201 cb->rlat++; in krping_doit()
2204 cb->bw++; in krping_doit()
2207 cb->duplex++; in krping_doit()
2211 cb->mem = DMA; in krping_doit()
2213 cb->mem = FASTREG; in krping_doit()
2215 cb->mem = MW; in krping_doit()
2217 cb->mem = MR; in krping_doit()
2219 PRINTF(cb, "unknown mem mode %s. " in krping_doit()
2227 cb->server_invalidate = 1; in krping_doit()
2230 cb->txdepth = optint; in krping_doit()
2231 DEBUG_LOG(cb, "txdepth %d\n", (int) cb->txdepth); in krping_doit()
2234 cb->local_dma_lkey = 1; in krping_doit()
2235 DEBUG_LOG(cb, "using local dma lkey\n"); in krping_doit()
2238 cb->read_inv = 1; in krping_doit()
2239 DEBUG_LOG(cb, "using read-with-inv\n"); in krping_doit()
2242 cb->frtest = 1; in krping_doit()
2243 DEBUG_LOG(cb, "fast-reg test!\n"); in krping_doit()
2246 PRINTF(cb, "unknown opt %s\n", optarg); in krping_doit()
2254 if (cb->server == -1) { in krping_doit()
2255 PRINTF(cb, "must be either client or server\n"); in krping_doit()
2260 if (cb->server && cb->frtest) { in krping_doit()
2261 PRINTF(cb, "must be client to run frtest\n"); in krping_doit()
2266 if ((cb->frtest + cb->bw + cb->rlat + cb->wlat) > 1) { in krping_doit()
2267 PRINTF(cb, "Pick only one test: fr, bw, rlat, wlat\n"); in krping_doit()
2272 if (cb->server_invalidate && cb->mem != FASTREG) { in krping_doit()
2273 PRINTF(cb, "server_invalidate only valid with fastreg mem_mode\n"); in krping_doit()
2278 if (cb->read_inv && cb->mem != FASTREG) { in krping_doit()
2279 PRINTF(cb, "read_inv only valid with fastreg mem_mode\n"); in krping_doit()
2284 if (cb->mem != MR && (cb->wlat || cb->rlat || cb->bw)) { in krping_doit()
2285 PRINTF(cb, "wlat, rlat, and bw tests only support mem_mode MR\n"); in krping_doit()
2290 cb->cm_id = rdma_create_id(krping_cma_event_handler, cb, RDMA_PS_TCP); in krping_doit()
2291 if (IS_ERR(cb->cm_id)) { in krping_doit()
2292 ret = PTR_ERR(cb->cm_id); in krping_doit()
2293 PRINTF(cb, "rdma_create_id error %d\n", ret); in krping_doit()
2296 DEBUG_LOG(cb, "created cm_id %p\n", cb->cm_id); in krping_doit()
2298 if (cb->server) in krping_doit()
2299 krping_run_server(cb); in krping_doit()
2301 krping_run_client(cb); in krping_doit()
2303 DEBUG_LOG(cb, "destroy cm_id %p\n", cb->cm_id); in krping_doit()
2304 rdma_destroy_id(cb->cm_id); in krping_doit()
2307 list_del(&cb->list); in krping_doit()
2309 kfree(cb); in krping_doit()
2316 struct krping_cb *cb; in krping_walk_cb_list() local
2319 list_for_each_entry(cb, &krping_cbs, list) in krping_walk_cb_list()
2320 (*f)(cb->pd ? &cb->stats : NULL, arg); in krping_walk_cb_list()