1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #if HAVE_CONFIG_H
34 # include <config.h>
35 #endif /* HAVE_CONFIG_H */
36
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <unistd.h>
40 #include <string.h>
41 #include <sys/types.h>
42 #include <sys/socket.h>
43 #include <sys/time.h>
44 #include <netdb.h>
45 #include <getopt.h>
46 #include <arpa/inet.h>
47 #include <time.h>
48
49 #include "pingpong.h"
50
51 enum {
52 PINGPONG_RECV_WRID = 1,
53 PINGPONG_SEND_WRID = 2,
54 };
55
56 static int page_size;
57
58 struct pingpong_context {
59 struct ibv_context *context;
60 struct ibv_comp_channel *channel;
61 struct ibv_pd *pd;
62 struct ibv_mr *mr;
63 struct ibv_cq *cq;
64 struct ibv_qp *qp;
65 struct ibv_ah *ah;
66 void *buf;
67 int size;
68 int rx_depth;
69 int pending;
70 struct ibv_port_attr portinfo;
71 };
72
73 struct pingpong_dest {
74 int lid;
75 int qpn;
76 int psn;
77 union ibv_gid gid;
78 };
79
pp_connect_ctx(struct pingpong_context * ctx,int port,int my_psn,int sl,struct pingpong_dest * dest,int sgid_idx)80 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
81 int sl, struct pingpong_dest *dest, int sgid_idx)
82 {
83 struct ibv_ah_attr ah_attr = {
84 .is_global = 0,
85 .dlid = dest->lid,
86 .sl = sl,
87 .src_path_bits = 0,
88 .port_num = port
89 };
90 struct ibv_qp_attr attr = {
91 .qp_state = IBV_QPS_RTR
92 };
93
94 if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
95 fprintf(stderr, "Failed to modify QP to RTR\n");
96 return 1;
97 }
98
99 attr.qp_state = IBV_QPS_RTS;
100 attr.sq_psn = my_psn;
101
102 if (ibv_modify_qp(ctx->qp, &attr,
103 IBV_QP_STATE |
104 IBV_QP_SQ_PSN)) {
105 fprintf(stderr, "Failed to modify QP to RTS\n");
106 return 1;
107 }
108
109 if (dest->gid.global.interface_id) {
110 ah_attr.is_global = 1;
111 ah_attr.grh.hop_limit = 1;
112 ah_attr.grh.dgid = dest->gid;
113 ah_attr.grh.sgid_index = sgid_idx;
114 }
115
116 ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
117 if (!ctx->ah) {
118 fprintf(stderr, "Failed to create AH\n");
119 return 1;
120 }
121
122 return 0;
123 }
124
pp_client_exch_dest(const char * servername,int port,const struct pingpong_dest * my_dest)125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
126 const struct pingpong_dest *my_dest)
127 {
128 struct addrinfo *res, *t;
129 struct addrinfo hints = {
130 .ai_family = AF_INET,
131 .ai_socktype = SOCK_STREAM
132 };
133 char *service;
134 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
135 int n;
136 int sockfd = -1;
137 struct pingpong_dest *rem_dest = NULL;
138 char gid[33];
139
140 if (asprintf(&service, "%d", port) < 0)
141 return NULL;
142
143 n = getaddrinfo(servername, service, &hints, &res);
144
145 if (n < 0) {
146 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
147 free(service);
148 return NULL;
149 }
150
151 for (t = res; t; t = t->ai_next) {
152 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
153 if (sockfd >= 0) {
154 if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
155 break;
156 close(sockfd);
157 sockfd = -1;
158 }
159 }
160
161 freeaddrinfo(res);
162 free(service);
163
164 if (sockfd < 0) {
165 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
166 return NULL;
167 }
168
169 gid_to_wire_gid(&my_dest->gid, gid);
170 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
171 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
172 fprintf(stderr, "Couldn't send local address\n");
173 goto out;
174 }
175
176 if (read(sockfd, msg, sizeof msg) != sizeof msg) {
177 perror("client read");
178 fprintf(stderr, "Couldn't read remote address\n");
179 goto out;
180 }
181
182 write(sockfd, "done", sizeof "done");
183
184 rem_dest = malloc(sizeof *rem_dest);
185 if (!rem_dest)
186 goto out;
187
188 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
189 wire_gid_to_gid(gid, &rem_dest->gid);
190
191 out:
192 close(sockfd);
193 return rem_dest;
194 }
195
pp_server_exch_dest(struct pingpong_context * ctx,int ib_port,int port,int sl,const struct pingpong_dest * my_dest,int sgid_idx)196 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
197 int ib_port, int port, int sl,
198 const struct pingpong_dest *my_dest,
199 int sgid_idx)
200 {
201 struct addrinfo *res, *t;
202 struct addrinfo hints = {
203 .ai_flags = AI_PASSIVE,
204 .ai_family = AF_INET,
205 .ai_socktype = SOCK_STREAM
206 };
207 char *service;
208 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
209 int n;
210 int sockfd = -1, connfd;
211 struct pingpong_dest *rem_dest = NULL;
212 char gid[33];
213
214 if (asprintf(&service, "%d", port) < 0)
215 return NULL;
216
217 n = getaddrinfo(NULL, service, &hints, &res);
218
219 if (n < 0) {
220 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
221 free(service);
222 return NULL;
223 }
224
225 for (t = res; t; t = t->ai_next) {
226 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
227 if (sockfd >= 0) {
228 n = 1;
229
230 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
231
232 if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
233 break;
234 close(sockfd);
235 sockfd = -1;
236 }
237 }
238
239 freeaddrinfo(res);
240 free(service);
241
242 if (sockfd < 0) {
243 fprintf(stderr, "Couldn't listen to port %d\n", port);
244 return NULL;
245 }
246
247 listen(sockfd, 1);
248 connfd = accept(sockfd, NULL, 0);
249 close(sockfd);
250 if (connfd < 0) {
251 fprintf(stderr, "accept() failed\n");
252 return NULL;
253 }
254
255 n = read(connfd, msg, sizeof msg);
256 if (n != sizeof msg) {
257 perror("server read");
258 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
259 goto out;
260 }
261
262 rem_dest = malloc(sizeof *rem_dest);
263 if (!rem_dest)
264 goto out;
265
266 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
267 wire_gid_to_gid(gid, &rem_dest->gid);
268
269 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) {
270 fprintf(stderr, "Couldn't connect to remote QP\n");
271 free(rem_dest);
272 rem_dest = NULL;
273 goto out;
274 }
275
276 gid_to_wire_gid(&my_dest->gid, gid);
277 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
278 if (write(connfd, msg, sizeof msg) != sizeof msg) {
279 fprintf(stderr, "Couldn't send local address\n");
280 free(rem_dest);
281 rem_dest = NULL;
282 goto out;
283 }
284
285 read(connfd, msg, sizeof msg);
286
287 out:
288 close(connfd);
289 return rem_dest;
290 }
291
pp_init_ctx(struct ibv_device * ib_dev,int size,int rx_depth,int port,int use_event)292 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
293 int rx_depth, int port,
294 int use_event)
295 {
296 struct pingpong_context *ctx;
297
298 ctx = malloc(sizeof *ctx);
299 if (!ctx)
300 return NULL;
301
302 ctx->size = size;
303 ctx->rx_depth = rx_depth;
304
305 ctx->buf = malloc(roundup(size + 40, page_size));
306 if (!ctx->buf) {
307 fprintf(stderr, "Couldn't allocate work buf.\n");
308 return NULL;
309 }
310
311 memset(ctx->buf, 0, size + 40);
312
313 ctx->context = ibv_open_device(ib_dev);
314 if (!ctx->context) {
315 fprintf(stderr, "Couldn't get context for %s\n",
316 ibv_get_device_name(ib_dev));
317 return NULL;
318 }
319
320 if (use_event) {
321 ctx->channel = ibv_create_comp_channel(ctx->context);
322 if (!ctx->channel) {
323 fprintf(stderr, "Couldn't create completion channel\n");
324 return NULL;
325 }
326 } else
327 ctx->channel = NULL;
328
329 ctx->pd = ibv_alloc_pd(ctx->context);
330 if (!ctx->pd) {
331 fprintf(stderr, "Couldn't allocate PD\n");
332 return NULL;
333 }
334
335 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
336 if (!ctx->mr) {
337 fprintf(stderr, "Couldn't register MR\n");
338 return NULL;
339 }
340
341 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
342 ctx->channel, 0);
343 if (!ctx->cq) {
344 fprintf(stderr, "Couldn't create CQ\n");
345 return NULL;
346 }
347
348 {
349 struct ibv_qp_init_attr attr = {
350 .send_cq = ctx->cq,
351 .recv_cq = ctx->cq,
352 .cap = {
353 .max_send_wr = 1,
354 .max_recv_wr = rx_depth,
355 .max_send_sge = 1,
356 .max_recv_sge = 1
357 },
358 .qp_type = IBV_QPT_UD,
359 };
360
361 ctx->qp = ibv_create_qp(ctx->pd, &attr);
362 if (!ctx->qp) {
363 fprintf(stderr, "Couldn't create QP\n");
364 return NULL;
365 }
366 }
367
368 {
369 struct ibv_qp_attr attr = {
370 .qp_state = IBV_QPS_INIT,
371 .pkey_index = 0,
372 .port_num = port,
373 .qkey = 0x11111111
374 };
375
376 if (ibv_modify_qp(ctx->qp, &attr,
377 IBV_QP_STATE |
378 IBV_QP_PKEY_INDEX |
379 IBV_QP_PORT |
380 IBV_QP_QKEY)) {
381 fprintf(stderr, "Failed to modify QP to INIT\n");
382 return NULL;
383 }
384 }
385
386 return ctx;
387 }
388
pp_close_ctx(struct pingpong_context * ctx)389 int pp_close_ctx(struct pingpong_context *ctx)
390 {
391 if (ibv_destroy_qp(ctx->qp)) {
392 fprintf(stderr, "Couldn't destroy QP\n");
393 return 1;
394 }
395
396 if (ibv_destroy_cq(ctx->cq)) {
397 fprintf(stderr, "Couldn't destroy CQ\n");
398 return 1;
399 }
400
401 if (ibv_dereg_mr(ctx->mr)) {
402 fprintf(stderr, "Couldn't deregister MR\n");
403 return 1;
404 }
405
406 if (ibv_destroy_ah(ctx->ah)) {
407 fprintf(stderr, "Couldn't destroy AH\n");
408 return 1;
409 }
410
411 if (ibv_dealloc_pd(ctx->pd)) {
412 fprintf(stderr, "Couldn't deallocate PD\n");
413 return 1;
414 }
415
416 if (ctx->channel) {
417 if (ibv_destroy_comp_channel(ctx->channel)) {
418 fprintf(stderr, "Couldn't destroy completion channel\n");
419 return 1;
420 }
421 }
422
423 if (ibv_close_device(ctx->context)) {
424 fprintf(stderr, "Couldn't release context\n");
425 return 1;
426 }
427
428 free(ctx->buf);
429 free(ctx);
430
431 return 0;
432 }
433
pp_post_recv(struct pingpong_context * ctx,int n)434 static int pp_post_recv(struct pingpong_context *ctx, int n)
435 {
436 struct ibv_sge list = {
437 .addr = (uintptr_t) ctx->buf,
438 .length = ctx->size + 40,
439 .lkey = ctx->mr->lkey
440 };
441 struct ibv_recv_wr wr = {
442 .wr_id = PINGPONG_RECV_WRID,
443 .sg_list = &list,
444 .num_sge = 1,
445 };
446 struct ibv_recv_wr *bad_wr;
447 int i;
448
449 for (i = 0; i < n; ++i)
450 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
451 break;
452
453 return i;
454 }
455
pp_post_send(struct pingpong_context * ctx,uint32_t qpn)456 static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
457 {
458 struct ibv_sge list = {
459 .addr = (uintptr_t) ctx->buf + 40,
460 .length = ctx->size,
461 .lkey = ctx->mr->lkey
462 };
463 struct ibv_send_wr wr = {
464 .wr_id = PINGPONG_SEND_WRID,
465 .sg_list = &list,
466 .num_sge = 1,
467 .opcode = IBV_WR_SEND,
468 .send_flags = IBV_SEND_SIGNALED,
469 .wr = {
470 .ud = {
471 .ah = ctx->ah,
472 .remote_qpn = qpn,
473 .remote_qkey = 0x11111111
474 }
475 }
476 };
477 struct ibv_send_wr *bad_wr;
478
479 return ibv_post_send(ctx->qp, &wr, &bad_wr);
480 }
481
usage(const char * argv0)482 static void usage(const char *argv0)
483 {
484 printf("Usage:\n");
485 printf(" %s start a server and wait for connection\n", argv0);
486 printf(" %s <host> connect to server at <host>\n", argv0);
487 printf("\n");
488 printf("Options:\n");
489 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
490 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
491 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
492 printf(" -s, --size=<size> size of message to exchange (default 1024)\n");
493 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
494 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
495 printf(" -e, --events sleep on CQ events (default poll)\n");
496 printf(" -g, --gid-idx=<gid index> local port gid index\n");
497 }
498
main(int argc,char * argv[])499 int main(int argc, char *argv[])
500 {
501 struct ibv_device **dev_list;
502 struct ibv_device *ib_dev;
503 struct pingpong_context *ctx;
504 struct pingpong_dest my_dest;
505 struct pingpong_dest *rem_dest;
506 struct timeval start, end;
507 char *ib_devname = NULL;
508 char *servername = NULL;
509 int port = 18515;
510 int ib_port = 1;
511 int size = 1024;
512 int rx_depth = 500;
513 int iters = 1000;
514 int use_event = 0;
515 int routs;
516 int rcnt, scnt;
517 int num_cq_events = 0;
518 int sl = 0;
519 int gidx = -1;
520 char gid[33];
521
522 srand48(getpid() * time(NULL));
523
524 while (1) {
525 int c;
526
527 static struct option long_options[] = {
528 { .name = "port", .has_arg = 1, .val = 'p' },
529 { .name = "ib-dev", .has_arg = 1, .val = 'd' },
530 { .name = "ib-port", .has_arg = 1, .val = 'i' },
531 { .name = "size", .has_arg = 1, .val = 's' },
532 { .name = "rx-depth", .has_arg = 1, .val = 'r' },
533 { .name = "iters", .has_arg = 1, .val = 'n' },
534 { .name = "sl", .has_arg = 1, .val = 'l' },
535 { .name = "events", .has_arg = 0, .val = 'e' },
536 { .name = "gid-idx", .has_arg = 1, .val = 'g' },
537 { 0 }
538 };
539
540 c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL);
541 if (c == -1)
542 break;
543
544 switch (c) {
545 case 'p':
546 port = strtol(optarg, NULL, 0);
547 if (port < 0 || port > 65535) {
548 usage(argv[0]);
549 return 1;
550 }
551 break;
552
553 case 'd':
554 ib_devname = strdup(optarg);
555 break;
556
557 case 'i':
558 ib_port = strtol(optarg, NULL, 0);
559 if (ib_port < 0) {
560 usage(argv[0]);
561 return 1;
562 }
563 break;
564
565 case 's':
566 size = strtol(optarg, NULL, 0);
567 break;
568
569 case 'r':
570 rx_depth = strtol(optarg, NULL, 0);
571 break;
572
573 case 'n':
574 iters = strtol(optarg, NULL, 0);
575 break;
576
577 case 'l':
578 sl = strtol(optarg, NULL, 0);
579 break;
580
581 case 'e':
582 ++use_event;
583 break;
584
585 case 'g':
586 gidx = strtol(optarg, NULL, 0);
587 break;
588
589 default:
590 usage(argv[0]);
591 return 1;
592 }
593 }
594
595 if (optind == argc - 1)
596 servername = strdup(argv[optind]);
597 else if (optind < argc) {
598 usage(argv[0]);
599 return 1;
600 }
601
602 page_size = sysconf(_SC_PAGESIZE);
603
604 dev_list = ibv_get_device_list(NULL);
605 if (!dev_list) {
606 perror("Failed to get IB devices list");
607 return 1;
608 }
609
610 if (!ib_devname) {
611 ib_dev = *dev_list;
612 if (!ib_dev) {
613 fprintf(stderr, "No IB devices found\n");
614 return 1;
615 }
616 } else {
617 int i;
618 for (i = 0; dev_list[i]; ++i)
619 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
620 break;
621 ib_dev = dev_list[i];
622 if (!ib_dev) {
623 fprintf(stderr, "IB device %s not found\n", ib_devname);
624 return 1;
625 }
626 }
627
628 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
629 if (!ctx)
630 return 1;
631
632 routs = pp_post_recv(ctx, ctx->rx_depth);
633 if (routs < ctx->rx_depth) {
634 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
635 return 1;
636 }
637
638 if (use_event)
639 if (ibv_req_notify_cq(ctx->cq, 0)) {
640 fprintf(stderr, "Couldn't request CQ notification\n");
641 return 1;
642 }
643
644 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
645 fprintf(stderr, "Couldn't get port info\n");
646 return 1;
647 }
648 my_dest.lid = ctx->portinfo.lid;
649
650 my_dest.qpn = ctx->qp->qp_num;
651 my_dest.psn = lrand48() & 0xffffff;
652
653 if (gidx >= 0) {
654 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
655 fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
656 return 1;
657 }
658 } else
659 memset(&my_dest.gid, 0, sizeof my_dest.gid);
660
661 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
662 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
663 my_dest.lid, my_dest.qpn, my_dest.psn, gid);
664
665 if (servername)
666 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
667 else
668 rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx);
669
670 if (!rem_dest)
671 return 1;
672
673 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
674 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
675 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
676
677 if (servername)
678 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx))
679 return 1;
680
681 ctx->pending = PINGPONG_RECV_WRID;
682
683 if (servername) {
684 if (pp_post_send(ctx, rem_dest->qpn)) {
685 fprintf(stderr, "Couldn't post send\n");
686 return 1;
687 }
688 ctx->pending |= PINGPONG_SEND_WRID;
689 }
690
691 if (gettimeofday(&start, NULL)) {
692 perror("gettimeofday");
693 return 1;
694 }
695
696 rcnt = scnt = 0;
697 while (rcnt < iters || scnt < iters) {
698 if (use_event) {
699 struct ibv_cq *ev_cq;
700 void *ev_ctx;
701
702 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
703 fprintf(stderr, "Failed to get cq_event\n");
704 return 1;
705 }
706
707 ++num_cq_events;
708
709 if (ev_cq != ctx->cq) {
710 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
711 return 1;
712 }
713
714 if (ibv_req_notify_cq(ctx->cq, 0)) {
715 fprintf(stderr, "Couldn't request CQ notification\n");
716 return 1;
717 }
718 }
719
720 {
721 struct ibv_wc wc[2];
722 int ne, i;
723
724 do {
725 ne = ibv_poll_cq(ctx->cq, 2, wc);
726 if (ne < 0) {
727 fprintf(stderr, "poll CQ failed %d\n", ne);
728 return 1;
729 }
730 } while (!use_event && ne < 1);
731
732 for (i = 0; i < ne; ++i) {
733 if (wc[i].status != IBV_WC_SUCCESS) {
734 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
735 ibv_wc_status_str(wc[i].status),
736 wc[i].status, (int) wc[i].wr_id);
737 return 1;
738 }
739
740 switch ((int) wc[i].wr_id) {
741 case PINGPONG_SEND_WRID:
742 ++scnt;
743 break;
744
745 case PINGPONG_RECV_WRID:
746 if (--routs <= 1) {
747 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
748 if (routs < ctx->rx_depth) {
749 fprintf(stderr,
750 "Couldn't post receive (%d)\n",
751 routs);
752 return 1;
753 }
754 }
755
756 ++rcnt;
757 break;
758
759 default:
760 fprintf(stderr, "Completion for unknown wr_id %d\n",
761 (int) wc[i].wr_id);
762 return 1;
763 }
764
765 ctx->pending &= ~(int) wc[i].wr_id;
766 if (scnt < iters && !ctx->pending) {
767 if (pp_post_send(ctx, rem_dest->qpn)) {
768 fprintf(stderr, "Couldn't post send\n");
769 return 1;
770 }
771 ctx->pending = PINGPONG_RECV_WRID |
772 PINGPONG_SEND_WRID;
773 }
774 }
775 }
776 }
777
778 if (gettimeofday(&end, NULL)) {
779 perror("gettimeofday");
780 return 1;
781 }
782
783 {
784 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
785 (end.tv_usec - start.tv_usec);
786 long long bytes = (long long) size * iters * 2;
787
788 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
789 bytes, usec / 1000000., bytes * 8. / usec);
790 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
791 iters, usec / 1000000., usec / iters);
792 }
793
794 ibv_ack_cq_events(ctx->cq, num_cq_events);
795
796 if (pp_close_ctx(ctx))
797 return 1;
798
799 ibv_free_device_list(dev_list);
800 free(rem_dest);
801
802 return 0;
803 }
804