1 #ifndef _SDP_H_
2 #define _SDP_H_
3
4 #define LINUXKPI_PARAM_PREFIX ib_sdp_
5
6 #include "opt_ddb.h"
7 #include "opt_inet.h"
8 #include "opt_ofed.h"
9
10 #include <sys/param.h>
11 #include <sys/systm.h>
12 #include <sys/malloc.h>
13 #include <sys/kernel.h>
14 #include <sys/sysctl.h>
15 #include <sys/mbuf.h>
16 #include <sys/lock.h>
17 #include <sys/rwlock.h>
18 #include <sys/socket.h>
19 #include <sys/socketvar.h>
20 #include <sys/protosw.h>
21 #include <sys/proc.h>
22 #include <sys/jail.h>
23 #include <sys/domain.h>
24
25 #ifdef DDB
26 #include <ddb/ddb.h>
27 #endif
28
29 #include <net/if.h>
30 #include <net/route.h>
31 #include <net/vnet.h>
32
33 #include <netinet/in.h>
34 #include <netinet/in_systm.h>
35 #include <netinet/in_var.h>
36 #include <netinet/in_pcb.h>
37 #include <netinet/tcp.h>
38 #include <netinet/tcp_fsm.h>
39 #include <netinet/tcp_timer.h>
40 #include <netinet/tcp_var.h>
41
42 #include <linux/device.h>
43 #include <linux/err.h>
44 #include <linux/sched.h>
45 #include <linux/workqueue.h>
46 #include <linux/wait.h>
47 #include <linux/module.h>
48 #include <linux/moduleparam.h>
49 #include <linux/pci.h>
50
51 #include <rdma/ib_verbs.h>
52 #include <rdma/rdma_cm.h>
53 #include <rdma/ib_cm.h>
54 #include <rdma/sdp_socket.h>
55 #include <rdma/ib_fmr_pool.h>
56
57 #ifdef SDP_DEBUG
58 #define CONFIG_INFINIBAND_SDP_DEBUG
59 #endif
60
61 #include "sdp_dbg.h"
62
63 #undef LIST_HEAD
64 /* From sys/queue.h */
65 #define LIST_HEAD(name, type) \
66 struct name { \
67 struct type *lh_first; /* first element */ \
68 }
69
70 /* Interval between sucessive polls in the Tx routine when polling is used
71 instead of interrupts (in per-core Tx rings) - should be power of 2 */
72 #define SDP_TX_POLL_MODER 16
73 #define SDP_TX_POLL_TIMEOUT (HZ / 20)
74 #define SDP_NAGLE_TIMEOUT (HZ / 10)
75
76 #define SDP_SRCAVAIL_CANCEL_TIMEOUT (HZ * 5)
77 #define SDP_SRCAVAIL_ADV_TIMEOUT (1 * HZ)
78 #define SDP_SRCAVAIL_PAYLOAD_LEN 1
79
80 #define SDP_RESOLVE_TIMEOUT 1000
81 #define SDP_ROUTE_TIMEOUT 1000
82 #define SDP_RETRY_COUNT 5
83 #define SDP_KEEPALIVE_TIME (120 * 60 * HZ)
84 #define SDP_FIN_WAIT_TIMEOUT (60 * HZ) /* like TCP_FIN_TIMEOUT */
85
86 #define SDP_TX_SIZE 0x40
87 #define SDP_RX_SIZE 0x40
88
89 #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64))
90 #define SDP_FMR_POOL_SIZE 1024
91 #define SDP_FMR_DIRTY_SIZE ( SDP_FMR_POOL_SIZE / 4 )
92
93 #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2))
94
95 /* mb inlined data len - rest will be rx'ed into frags */
96 #define SDP_HEAD_SIZE (sizeof(struct sdp_bsdh))
97
98 /* limit tx payload len, if the sink supports bigger buffers than the source
99 * can handle.
100 * or rx fragment size (limited by sge->length size) */
101 #define SDP_MAX_PACKET (1 << 16)
102 #define SDP_MAX_PAYLOAD (SDP_MAX_PACKET - SDP_HEAD_SIZE)
103
104 #define SDP_MAX_RECV_SGES (SDP_MAX_PACKET / MCLBYTES)
105 #define SDP_MAX_SEND_SGES (SDP_MAX_PACKET / MCLBYTES) + 2
106
107 #define SDP_NUM_WC 4
108
109 #define SDP_DEF_ZCOPY_THRESH 64*1024
110 #define SDP_MIN_ZCOPY_THRESH PAGE_SIZE
111 #define SDP_MAX_ZCOPY_THRESH 1048576
112
113 #define SDP_OP_RECV 0x800000000LL
114 #define SDP_OP_SEND 0x400000000LL
115 #define SDP_OP_RDMA 0x200000000LL
116 #define SDP_OP_NOP 0x100000000LL
117
118 /* how long (in jiffies) to block sender till tx completion*/
119 #define SDP_BZCOPY_POLL_TIMEOUT (HZ / 10)
120
121 #define SDP_AUTO_CONF 0xffff
122 #define AUTO_MOD_DELAY (HZ / 4)
123
124 struct sdp_mb_cb {
125 __u32 seq; /* Starting sequence number */
126 struct bzcopy_state *bz;
127 struct rx_srcavail_state *rx_sa;
128 struct tx_srcavail_state *tx_sa;
129 };
130
131 #define M_PUSH M_PROTO1 /* Do a 'push'. */
132 #define M_URG M_PROTO2 /* Mark as urgent (oob). */
133
134 #define SDP_SKB_CB(__mb) ((struct sdp_mb_cb *)&((__mb)->cb[0]))
135 #define BZCOPY_STATE(mb) (SDP_SKB_CB(mb)->bz)
136 #define RX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->rx_sa)
137 #define TX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->tx_sa)
138
139 #ifndef MIN
140 #define MIN(a, b) (a < b ? a : b)
141 #endif
142
143 #define ring_head(ring) (atomic_read(&(ring).head))
144 #define ring_tail(ring) (atomic_read(&(ring).tail))
145 #define ring_posted(ring) (ring_head(ring) - ring_tail(ring))
146
147 #define rx_ring_posted(ssk) ring_posted(ssk->rx_ring)
148 #ifdef SDP_ZCOPY
149 #define tx_ring_posted(ssk) (ring_posted(ssk->tx_ring) + \
150 (ssk->tx_ring.rdma_inflight ? ssk->tx_ring.rdma_inflight->busy : 0))
151 #else
152 #define tx_ring_posted(ssk) ring_posted(ssk->tx_ring)
153 #endif
154
155 extern int sdp_zcopy_thresh;
156 extern int rcvbuf_initial_size;
157 extern struct workqueue_struct *rx_comp_wq;
158 extern struct ib_client sdp_client;
159
160 enum sdp_mid {
161 SDP_MID_HELLO = 0x0,
162 SDP_MID_HELLO_ACK = 0x1,
163 SDP_MID_DISCONN = 0x2,
164 SDP_MID_ABORT = 0x3,
165 SDP_MID_SENDSM = 0x4,
166 SDP_MID_RDMARDCOMPL = 0x6,
167 SDP_MID_SRCAVAIL_CANCEL = 0x8,
168 SDP_MID_CHRCVBUF = 0xB,
169 SDP_MID_CHRCVBUF_ACK = 0xC,
170 SDP_MID_SINKAVAIL = 0xFD,
171 SDP_MID_SRCAVAIL = 0xFE,
172 SDP_MID_DATA = 0xFF,
173 };
174
175 enum sdp_flags {
176 SDP_OOB_PRES = 1 << 0,
177 SDP_OOB_PEND = 1 << 1,
178 };
179
180 enum {
181 SDP_MIN_TX_CREDITS = 2
182 };
183
184 enum {
185 SDP_ERR_ERROR = -4,
186 SDP_ERR_FAULT = -3,
187 SDP_NEW_SEG = -2,
188 SDP_DO_WAIT_MEM = -1
189 };
190
191 struct sdp_bsdh {
192 u8 mid;
193 u8 flags;
194 __u16 bufs;
195 __u32 len;
196 __u32 mseq;
197 __u32 mseq_ack;
198 } __attribute__((__packed__));
199
200 union cma_ip_addr {
201 struct in6_addr ip6;
202 struct {
203 __u32 pad[3];
204 __u32 addr;
205 } ip4;
206 } __attribute__((__packed__));
207
208 /* TODO: too much? Can I avoid having the src/dst and port here? */
209 struct sdp_hh {
210 struct sdp_bsdh bsdh;
211 u8 majv_minv;
212 u8 ipv_cap;
213 u8 rsvd1;
214 u8 max_adverts;
215 __u32 desremrcvsz;
216 __u32 localrcvsz;
217 __u16 port;
218 __u16 rsvd2;
219 union cma_ip_addr src_addr;
220 union cma_ip_addr dst_addr;
221 u8 rsvd3[IB_CM_REQ_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 48];
222 } __attribute__((__packed__));
223
224 struct sdp_hah {
225 struct sdp_bsdh bsdh;
226 u8 majv_minv;
227 u8 ipv_cap;
228 u8 rsvd1;
229 u8 ext_max_adverts;
230 __u32 actrcvsz;
231 u8 rsvd2[IB_CM_REP_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 8];
232 } __attribute__((__packed__));
233
234 struct sdp_rrch {
235 __u32 len;
236 } __attribute__((__packed__));
237
238 struct sdp_srcah {
239 __u32 len;
240 __u32 rkey;
241 __u64 vaddr;
242 } __attribute__((__packed__));
243
244 struct sdp_buf {
245 struct mbuf *mb;
246 u64 mapping[SDP_MAX_SEND_SGES];
247 } __attribute__((__packed__));
248
249 struct sdp_chrecvbuf {
250 u32 size;
251 } __attribute__((__packed__));
252
253 /* Context used for synchronous zero copy bcopy (BZCOPY) */
254 struct bzcopy_state {
255 unsigned char __user *u_base;
256 int u_len;
257 int left;
258 int page_cnt;
259 int cur_page;
260 int cur_offset;
261 int busy;
262 struct sdp_sock *ssk;
263 struct page **pages;
264 };
265
266 enum rx_sa_flag {
267 RX_SA_ABORTED = 2,
268 };
269
270 enum tx_sa_flag {
271 TX_SA_SENDSM = 0x01,
272 TX_SA_CROSS_SEND = 0x02,
273 TX_SA_INTRRUPTED = 0x04,
274 TX_SA_TIMEDOUT = 0x08,
275 TX_SA_ERROR = 0x10,
276 };
277
278 struct rx_srcavail_state {
279 /* Advertised buffer stuff */
280 u32 mseq;
281 u32 used;
282 u32 reported;
283 u32 len;
284 u32 rkey;
285 u64 vaddr;
286
287 /* Dest buff info */
288 struct ib_umem *umem;
289 struct ib_pool_fmr *fmr;
290
291 /* Utility */
292 u8 busy;
293 enum rx_sa_flag flags;
294 };
295
296 struct tx_srcavail_state {
297 /* Data below 'busy' will be reset */
298 u8 busy;
299
300 struct ib_umem *umem;
301 struct ib_pool_fmr *fmr;
302
303 u32 bytes_sent;
304 u32 bytes_acked;
305
306 enum tx_sa_flag abort_flags;
307 u8 posted;
308
309 u32 mseq;
310 };
311
312 struct sdp_tx_ring {
313 #ifdef SDP_ZCOPY
314 struct rx_srcavail_state *rdma_inflight;
315 #endif
316 struct sdp_buf *buffer;
317 atomic_t head;
318 atomic_t tail;
319 struct ib_cq *cq;
320
321 atomic_t credits;
322 #define tx_credits(ssk) (atomic_read(&ssk->tx_ring.credits))
323
324 struct callout timer;
325 u16 poll_cnt;
326 };
327
328 struct sdp_rx_ring {
329 struct sdp_buf *buffer;
330 atomic_t head;
331 atomic_t tail;
332 struct ib_cq *cq;
333
334 int destroyed;
335 struct rwlock destroyed_lock;
336 };
337
338 struct sdp_device {
339 struct ib_pd *pd;
340 struct ib_mr *mr;
341 struct ib_fmr_pool *fmr_pool;
342 };
343
344 struct sdp_moderation {
345 unsigned long last_moder_packets;
346 unsigned long last_moder_tx_packets;
347 unsigned long last_moder_bytes;
348 unsigned long last_moder_jiffies;
349 int last_moder_time;
350 u16 rx_usecs;
351 u16 rx_frames;
352 u16 tx_usecs;
353 u32 pkt_rate_low;
354 u16 rx_usecs_low;
355 u32 pkt_rate_high;
356 u16 rx_usecs_high;
357 u16 sample_interval;
358 u16 adaptive_rx_coal;
359 u32 msg_enable;
360
361 int moder_cnt;
362 int moder_time;
363 };
364
365 /* These are flags fields. */
366 #define SDP_TIMEWAIT 0x0001 /* In ssk timewait state. */
367 #define SDP_DROPPED 0x0002 /* Socket has been dropped. */
368 #define SDP_SOCKREF 0x0004 /* Holding a sockref for close. */
369 #define SDP_NODELAY 0x0008 /* Disble nagle. */
370 #define SDP_NEEDFIN 0x0010 /* Send a fin on the next tx. */
371 #define SDP_DREQWAIT 0x0020 /* Waiting on DREQ. */
372 #define SDP_DESTROY 0x0040 /* Being destroyed. */
373 #define SDP_DISCON 0x0080 /* rdma_disconnect is owed. */
374
375 /* These are oobflags */
376 #define SDP_HADOOB 0x0001 /* Had OOB data. */
377 #define SDP_HAVEOOB 0x0002 /* Have OOB data. */
378
379 struct sdp_sock {
380 LIST_ENTRY(sdp_sock) list;
381 struct socket *socket;
382 struct rdma_cm_id *id;
383 struct ib_device *ib_device;
384 struct sdp_device *sdp_dev;
385 struct ib_qp *qp;
386 struct ucred *cred;
387 struct callout keep2msl; /* 2msl and keepalive timer. */
388 struct callout nagle_timer; /* timeout waiting for ack */
389 struct ib_ucontext context;
390 in_port_t lport;
391 in_addr_t laddr;
392 in_port_t fport;
393 in_addr_t faddr;
394 int flags;
395 int oobflags; /* protected by rx lock. */
396 int state;
397 int softerror;
398 int recv_bytes; /* Bytes per recv. buf including header */
399 int xmit_size_goal;
400 char iobc;
401
402 struct sdp_rx_ring rx_ring;
403 struct sdp_tx_ring tx_ring;
404 struct rwlock lock;
405 struct mbuf *rx_ctl_q;
406 struct mbuf *rx_ctl_tail;
407
408 int qp_active; /* XXX Flag. */
409 int max_sge;
410 struct work_struct rx_comp_work;
411 #define rcv_nxt(ssk) atomic_read(&(ssk->rcv_nxt))
412 atomic_t rcv_nxt;
413
414 /* SDP specific */
415 atomic_t mseq_ack;
416 #define mseq_ack(ssk) (atomic_read(&ssk->mseq_ack))
417 unsigned max_bufs; /* Initial buffers offered by other side */
418 unsigned min_bufs; /* Low water mark to wake senders */
419
420 unsigned long nagle_last_unacked; /* mseq of lastest unacked packet */
421
422 atomic_t remote_credits;
423 #define remote_credits(ssk) (atomic_read(&ssk->remote_credits))
424 int poll_cq;
425
426 /* SDP slow start */
427 int recv_request_head; /* mark the rx_head when the resize request
428 was recieved */
429 int recv_request; /* XXX flag if request to resize was recieved */
430
431 unsigned long tx_packets;
432 unsigned long rx_packets;
433 unsigned long tx_bytes;
434 unsigned long rx_bytes;
435 struct sdp_moderation auto_mod;
436 struct task shutdown_task;
437 #ifdef SDP_ZCOPY
438 struct tx_srcavail_state *tx_sa;
439 struct rx_srcavail_state *rx_sa;
440 spinlock_t tx_sa_lock;
441 struct delayed_work srcavail_cancel_work;
442 int srcavail_cancel_mseq;
443 /* ZCOPY data: -1:use global; 0:disable zcopy; >0: zcopy threshold */
444 int zcopy_thresh;
445 #endif
446 };
447
448 #define sdp_sk(so) ((struct sdp_sock *)(so->so_pcb))
449
450 #define SDP_RLOCK(ssk) rw_rlock(&(ssk)->lock)
451 #define SDP_WLOCK(ssk) rw_wlock(&(ssk)->lock)
452 #define SDP_RUNLOCK(ssk) rw_runlock(&(ssk)->lock)
453 #define SDP_WUNLOCK(ssk) rw_wunlock(&(ssk)->lock)
454 #define SDP_WLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_WLOCKED)
455 #define SDP_RLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_RLOCKED)
456 #define SDP_LOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_LOCKED)
457
tx_sa_reset(struct tx_srcavail_state * tx_sa)458 static inline void tx_sa_reset(struct tx_srcavail_state *tx_sa)
459 {
460 memset((void *)&tx_sa->busy, 0,
461 sizeof(*tx_sa) - offsetof(typeof(*tx_sa), busy));
462 }
463
rx_ring_unlock(struct sdp_rx_ring * rx_ring)464 static inline void rx_ring_unlock(struct sdp_rx_ring *rx_ring)
465 {
466 rw_runlock(&rx_ring->destroyed_lock);
467 }
468
rx_ring_trylock(struct sdp_rx_ring * rx_ring)469 static inline int rx_ring_trylock(struct sdp_rx_ring *rx_ring)
470 {
471 rw_rlock(&rx_ring->destroyed_lock);
472 if (rx_ring->destroyed) {
473 rx_ring_unlock(rx_ring);
474 return 0;
475 }
476 return 1;
477 }
478
rx_ring_destroy_lock(struct sdp_rx_ring * rx_ring)479 static inline void rx_ring_destroy_lock(struct sdp_rx_ring *rx_ring)
480 {
481 rw_wlock(&rx_ring->destroyed_lock);
482 rx_ring->destroyed = 1;
483 rw_wunlock(&rx_ring->destroyed_lock);
484 }
485
sdp_arm_rx_cq(struct sdp_sock * ssk)486 static inline void sdp_arm_rx_cq(struct sdp_sock *ssk)
487 {
488 sdp_prf(ssk->socket, NULL, "Arming RX cq");
489 sdp_dbg_data(ssk->socket, "Arming RX cq\n");
490
491 ib_req_notify_cq(ssk->rx_ring.cq, IB_CQ_NEXT_COMP);
492 }
493
sdp_arm_tx_cq(struct sdp_sock * ssk)494 static inline void sdp_arm_tx_cq(struct sdp_sock *ssk)
495 {
496 sdp_prf(ssk->socket, NULL, "Arming TX cq");
497 sdp_dbg_data(ssk->socket, "Arming TX cq. credits: %d, posted: %d\n",
498 tx_credits(ssk), tx_ring_posted(ssk));
499
500 ib_req_notify_cq(ssk->tx_ring.cq, IB_CQ_NEXT_COMP);
501 }
502
503 /* return the min of:
504 * - tx credits
505 * - free slots in tx_ring (not including SDP_MIN_TX_CREDITS
506 */
tx_slots_free(struct sdp_sock * ssk)507 static inline int tx_slots_free(struct sdp_sock *ssk)
508 {
509 int min_free;
510
511 min_free = MIN(tx_credits(ssk),
512 SDP_TX_SIZE - tx_ring_posted(ssk));
513 if (min_free < SDP_MIN_TX_CREDITS)
514 return 0;
515
516 return min_free - SDP_MIN_TX_CREDITS;
517 };
518
519 /* utilities */
mid2str(int mid)520 static inline char *mid2str(int mid)
521 {
522 #define ENUM2STR(e) [e] = #e
523 static char *mid2str[] = {
524 ENUM2STR(SDP_MID_HELLO),
525 ENUM2STR(SDP_MID_HELLO_ACK),
526 ENUM2STR(SDP_MID_ABORT),
527 ENUM2STR(SDP_MID_DISCONN),
528 ENUM2STR(SDP_MID_SENDSM),
529 ENUM2STR(SDP_MID_RDMARDCOMPL),
530 ENUM2STR(SDP_MID_SRCAVAIL_CANCEL),
531 ENUM2STR(SDP_MID_CHRCVBUF),
532 ENUM2STR(SDP_MID_CHRCVBUF_ACK),
533 ENUM2STR(SDP_MID_DATA),
534 ENUM2STR(SDP_MID_SRCAVAIL),
535 ENUM2STR(SDP_MID_SINKAVAIL),
536 };
537
538 if (mid >= ARRAY_SIZE(mid2str))
539 return NULL;
540
541 return mid2str[mid];
542 }
543
544 static inline struct mbuf *
sdp_alloc_mb(struct socket * sk,u8 mid,int size,int wait)545 sdp_alloc_mb(struct socket *sk, u8 mid, int size, int wait)
546 {
547 struct sdp_bsdh *h;
548 struct mbuf *mb;
549
550 MGETHDR(mb, wait, MT_DATA);
551 if (mb == NULL)
552 return (NULL);
553 mb->m_pkthdr.len = mb->m_len = sizeof(struct sdp_bsdh);
554 h = mtod(mb, struct sdp_bsdh *);
555 h->mid = mid;
556
557 return mb;
558 }
559 static inline struct mbuf *
sdp_alloc_mb_data(struct socket * sk,int wait)560 sdp_alloc_mb_data(struct socket *sk, int wait)
561 {
562 return sdp_alloc_mb(sk, SDP_MID_DATA, 0, wait);
563 }
564
565 static inline struct mbuf *
sdp_alloc_mb_disconnect(struct socket * sk,int wait)566 sdp_alloc_mb_disconnect(struct socket *sk, int wait)
567 {
568 return sdp_alloc_mb(sk, SDP_MID_DISCONN, 0, wait);
569 }
570
571 static inline void *
mb_put(struct mbuf * mb,int len)572 mb_put(struct mbuf *mb, int len)
573 {
574 uint8_t *data;
575
576 data = mb->m_data;
577 data += mb->m_len;
578 mb->m_len += len;
579 return (void *)data;
580 }
581
582 static inline struct mbuf *
sdp_alloc_mb_chrcvbuf_ack(struct socket * sk,int size,int wait)583 sdp_alloc_mb_chrcvbuf_ack(struct socket *sk, int size, int wait)
584 {
585 struct mbuf *mb;
586 struct sdp_chrecvbuf *resp_size;
587
588 mb = sdp_alloc_mb(sk, SDP_MID_CHRCVBUF_ACK, sizeof(*resp_size), wait);
589 if (mb == NULL)
590 return (NULL);
591 resp_size = (struct sdp_chrecvbuf *)mb_put(mb, sizeof *resp_size);
592 resp_size->size = htonl(size);
593
594 return mb;
595 }
596
597 static inline struct mbuf *
sdp_alloc_mb_srcavail(struct socket * sk,u32 len,u32 rkey,u64 vaddr,int wait)598 sdp_alloc_mb_srcavail(struct socket *sk, u32 len, u32 rkey, u64 vaddr, int wait)
599 {
600 struct mbuf *mb;
601 struct sdp_srcah *srcah;
602
603 mb = sdp_alloc_mb(sk, SDP_MID_SRCAVAIL, sizeof(*srcah), wait);
604 if (mb == NULL)
605 return (NULL);
606 srcah = (struct sdp_srcah *)mb_put(mb, sizeof(*srcah));
607 srcah->len = htonl(len);
608 srcah->rkey = htonl(rkey);
609 srcah->vaddr = cpu_to_be64(vaddr);
610
611 return mb;
612 }
613
614 static inline struct mbuf *
sdp_alloc_mb_srcavail_cancel(struct socket * sk,int wait)615 sdp_alloc_mb_srcavail_cancel(struct socket *sk, int wait)
616 {
617 return sdp_alloc_mb(sk, SDP_MID_SRCAVAIL_CANCEL, 0, wait);
618 }
619
620 static inline struct mbuf *
sdp_alloc_mb_rdmardcompl(struct socket * sk,u32 len,int wait)621 sdp_alloc_mb_rdmardcompl(struct socket *sk, u32 len, int wait)
622 {
623 struct mbuf *mb;
624 struct sdp_rrch *rrch;
625
626 mb = sdp_alloc_mb(sk, SDP_MID_RDMARDCOMPL, sizeof(*rrch), wait);
627 if (mb == NULL)
628 return (NULL);
629 rrch = (struct sdp_rrch *)mb_put(mb, sizeof(*rrch));
630 rrch->len = htonl(len);
631
632 return mb;
633 }
634
635 static inline struct mbuf *
sdp_alloc_mb_sendsm(struct socket * sk,int wait)636 sdp_alloc_mb_sendsm(struct socket *sk, int wait)
637 {
638 return sdp_alloc_mb(sk, SDP_MID_SENDSM, 0, wait);
639 }
sdp_tx_ring_slots_left(struct sdp_sock * ssk)640 static inline int sdp_tx_ring_slots_left(struct sdp_sock *ssk)
641 {
642 return SDP_TX_SIZE - tx_ring_posted(ssk);
643 }
644
credit_update_needed(struct sdp_sock * ssk)645 static inline int credit_update_needed(struct sdp_sock *ssk)
646 {
647 int c;
648
649 c = remote_credits(ssk);
650 if (likely(c > SDP_MIN_TX_CREDITS))
651 c += c/2;
652 return unlikely(c < rx_ring_posted(ssk)) &&
653 likely(tx_credits(ssk) > 0) &&
654 likely(sdp_tx_ring_slots_left(ssk));
655 }
656
657
658 #define SDPSTATS_COUNTER_INC(stat)
659 #define SDPSTATS_COUNTER_ADD(stat, val)
660 #define SDPSTATS_COUNTER_MID_INC(stat, mid)
661 #define SDPSTATS_HIST_LINEAR(stat, size)
662 #define SDPSTATS_HIST(stat, size)
663
664 static inline void
sdp_cleanup_sdp_buf(struct sdp_sock * ssk,struct sdp_buf * sbuf,enum dma_data_direction dir)665 sdp_cleanup_sdp_buf(struct sdp_sock *ssk, struct sdp_buf *sbuf,
666 enum dma_data_direction dir)
667 {
668 struct ib_device *dev;
669 struct mbuf *mb;
670 int i;
671
672 dev = ssk->ib_device;
673 for (i = 0, mb = sbuf->mb; mb != NULL; mb = mb->m_next, i++)
674 ib_dma_unmap_single(dev, sbuf->mapping[i], mb->m_len, dir);
675 }
676
677 /* sdp_main.c */
678 void sdp_set_default_moderation(struct sdp_sock *ssk);
679 void sdp_start_keepalive_timer(struct socket *sk);
680 void sdp_urg(struct sdp_sock *ssk, struct mbuf *mb);
681 void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk);
682 void sdp_abort(struct socket *sk);
683 struct sdp_sock *sdp_notify(struct sdp_sock *ssk, int error);
684
685
686 /* sdp_cma.c */
687 int sdp_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *);
688
689 /* sdp_tx.c */
690 int sdp_tx_ring_create(struct sdp_sock *ssk, struct ib_device *device);
691 void sdp_tx_ring_destroy(struct sdp_sock *ssk);
692 int sdp_xmit_poll(struct sdp_sock *ssk, int force);
693 void sdp_post_send(struct sdp_sock *ssk, struct mbuf *mb);
694 void sdp_post_sends(struct sdp_sock *ssk, int wait);
695 void sdp_post_keepalive(struct sdp_sock *ssk);
696
697 /* sdp_rx.c */
698 void sdp_rx_ring_init(struct sdp_sock *ssk);
699 int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device);
700 void sdp_rx_ring_destroy(struct sdp_sock *ssk);
701 int sdp_resize_buffers(struct sdp_sock *ssk, u32 new_size);
702 int sdp_init_buffers(struct sdp_sock *ssk, u32 new_size);
703 void sdp_do_posts(struct sdp_sock *ssk);
704 void sdp_rx_comp_full(struct sdp_sock *ssk);
705
706 /* sdp_zcopy.c */
707 struct kiocb;
708 int sdp_sendmsg_zcopy(struct kiocb *iocb, struct socket *sk, struct iovec *iov);
709 int sdp_handle_srcavail(struct sdp_sock *ssk, struct sdp_srcah *srcah);
710 void sdp_handle_sendsm(struct sdp_sock *ssk, u32 mseq_ack);
711 void sdp_handle_rdma_read_compl(struct sdp_sock *ssk, u32 mseq_ack,
712 u32 bytes_completed);
713 int sdp_handle_rdma_read_cqe(struct sdp_sock *ssk);
714 int sdp_rdma_to_iovec(struct socket *sk, struct iovec *iov, struct mbuf *mb,
715 unsigned long *used);
716 int sdp_post_rdma_rd_compl(struct sdp_sock *ssk,
717 struct rx_srcavail_state *rx_sa);
718 int sdp_post_sendsm(struct socket *sk);
719 void srcavail_cancel_timeout(struct work_struct *work);
720 void sdp_abort_srcavail(struct socket *sk);
721 void sdp_abort_rdma_read(struct socket *sk);
722 int sdp_process_rx(struct sdp_sock *ssk);
723
724 #endif
725