1
2 /*-
3 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
4 * The Regents of the University of California. All rights reserved.
5 * Copyright (c) 2004 The FreeBSD Foundation. All rights reserved.
6 * Copyright (c) 2004-2008 Robert N. M. Watson. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 4. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * Excerpts taken from tcp_subr.c, tcp_usrreq.c, uipc_socket.c
33 */
34
35 /*
36 *
37 * Copyright (c) 2010 Isilon Systems, Inc.
38 * Copyright (c) 2010 iX Systems, Inc.
39 * Copyright (c) 2010 Panasas, Inc.
40 * All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice unmodified, this list of conditions, and the following
47 * disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 *
52 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
53 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
54 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
55 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
56 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
57 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
58 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
59 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
60 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
61 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
62 *
63 */
64 #include <sys/cdefs.h>
65 __FBSDID("$FreeBSD$");
66
67 #include <sys/param.h>
68 #include <sys/kernel.h>
69 #include <sys/malloc.h>
70
71 #include "sdp.h"
72
73 #include <net/if.h>
74 #include <net/route.h>
75 #include <net/vnet.h>
76 #include <sys/sysctl.h>
77
78 uma_zone_t sdp_zone;
79 struct rwlock sdp_lock;
80 LIST_HEAD(, sdp_sock) sdp_list;
81
82 struct workqueue_struct *rx_comp_wq;
83
84 RW_SYSINIT(sdplockinit, &sdp_lock, "SDP lock");
85 #define SDP_LIST_WLOCK() rw_wlock(&sdp_lock)
86 #define SDP_LIST_RLOCK() rw_rlock(&sdp_lock)
87 #define SDP_LIST_WUNLOCK() rw_wunlock(&sdp_lock)
88 #define SDP_LIST_RUNLOCK() rw_runlock(&sdp_lock)
89 #define SDP_LIST_WLOCK_ASSERT() rw_assert(&sdp_lock, RW_WLOCKED)
90 #define SDP_LIST_RLOCK_ASSERT() rw_assert(&sdp_lock, RW_RLOCKED)
91 #define SDP_LIST_LOCK_ASSERT() rw_assert(&sdp_lock, RW_LOCKED)
92
93 MALLOC_DEFINE(M_SDP, "sdp", "Sockets Direct Protocol");
94
95 static void sdp_stop_keepalive_timer(struct socket *so);
96
97 /*
98 * SDP protocol interface to socket abstraction.
99 */
100 /*
101 * sdp_sendspace and sdp_recvspace are the default send and receive window
102 * sizes, respectively.
103 */
104 u_long sdp_sendspace = 1024*32;
105 u_long sdp_recvspace = 1024*64;
106
107 static int sdp_count;
108
109 /*
110 * Disable async. CMA events for sockets which are being torn down.
111 */
112 static void
sdp_destroy_cma(struct sdp_sock * ssk)113 sdp_destroy_cma(struct sdp_sock *ssk)
114 {
115
116 if (ssk->id == NULL)
117 return;
118 rdma_destroy_id(ssk->id);
119 ssk->id = NULL;
120 }
121
122 static int
sdp_pcbbind(struct sdp_sock * ssk,struct sockaddr * nam,struct ucred * cred)123 sdp_pcbbind(struct sdp_sock *ssk, struct sockaddr *nam, struct ucred *cred)
124 {
125 struct sockaddr_in *sin;
126 struct sockaddr_in null;
127 int error;
128
129 SDP_WLOCK_ASSERT(ssk);
130
131 if (ssk->lport != 0 || ssk->laddr != INADDR_ANY)
132 return (EINVAL);
133 /* rdma_bind_addr handles bind races. */
134 SDP_WUNLOCK(ssk);
135 if (ssk->id == NULL)
136 ssk->id = rdma_create_id(&init_net, sdp_cma_handler, ssk, RDMA_PS_SDP, IB_QPT_RC);
137 if (ssk->id == NULL) {
138 SDP_WLOCK(ssk);
139 return (ENOMEM);
140 }
141 if (nam == NULL) {
142 null.sin_family = AF_INET;
143 null.sin_len = sizeof(null);
144 null.sin_addr.s_addr = INADDR_ANY;
145 null.sin_port = 0;
146 bzero(&null.sin_zero, sizeof(null.sin_zero));
147 nam = (struct sockaddr *)&null;
148 }
149 error = -rdma_bind_addr(ssk->id, nam);
150 SDP_WLOCK(ssk);
151 if (error == 0) {
152 sin = (struct sockaddr_in *)&ssk->id->route.addr.src_addr;
153 ssk->laddr = sin->sin_addr.s_addr;
154 ssk->lport = sin->sin_port;
155 } else
156 sdp_destroy_cma(ssk);
157 return (error);
158 }
159
160 static void
sdp_pcbfree(struct sdp_sock * ssk)161 sdp_pcbfree(struct sdp_sock *ssk)
162 {
163
164 KASSERT(ssk->socket == NULL, ("ssk %p socket still attached", ssk));
165 KASSERT((ssk->flags & SDP_DESTROY) == 0,
166 ("ssk %p already destroyed", ssk));
167
168 sdp_dbg(ssk->socket, "Freeing pcb");
169 SDP_WLOCK_ASSERT(ssk);
170 ssk->flags |= SDP_DESTROY;
171 SDP_WUNLOCK(ssk);
172 SDP_LIST_WLOCK();
173 sdp_count--;
174 LIST_REMOVE(ssk, list);
175 SDP_LIST_WUNLOCK();
176 crfree(ssk->cred);
177 ssk->qp_active = 0;
178 if (ssk->qp) {
179 ib_destroy_qp(ssk->qp);
180 ssk->qp = NULL;
181 }
182 sdp_tx_ring_destroy(ssk);
183 sdp_rx_ring_destroy(ssk);
184 sdp_destroy_cma(ssk);
185 rw_destroy(&ssk->rx_ring.destroyed_lock);
186 rw_destroy(&ssk->lock);
187 uma_zfree(sdp_zone, ssk);
188 }
189
190 /*
191 * Common routines to return a socket address.
192 */
193 static struct sockaddr *
sdp_sockaddr(in_port_t port,struct in_addr * addr_p)194 sdp_sockaddr(in_port_t port, struct in_addr *addr_p)
195 {
196 struct sockaddr_in *sin;
197
198 sin = malloc(sizeof *sin, M_SONAME,
199 M_WAITOK | M_ZERO);
200 sin->sin_family = AF_INET;
201 sin->sin_len = sizeof(*sin);
202 sin->sin_addr = *addr_p;
203 sin->sin_port = port;
204
205 return (struct sockaddr *)sin;
206 }
207
208 static int
sdp_getsockaddr(struct socket * so,struct sockaddr ** nam)209 sdp_getsockaddr(struct socket *so, struct sockaddr **nam)
210 {
211 struct sdp_sock *ssk;
212 struct in_addr addr;
213 in_port_t port;
214
215 ssk = sdp_sk(so);
216 SDP_RLOCK(ssk);
217 port = ssk->lport;
218 addr.s_addr = ssk->laddr;
219 SDP_RUNLOCK(ssk);
220
221 *nam = sdp_sockaddr(port, &addr);
222 return 0;
223 }
224
225 static int
sdp_getpeeraddr(struct socket * so,struct sockaddr ** nam)226 sdp_getpeeraddr(struct socket *so, struct sockaddr **nam)
227 {
228 struct sdp_sock *ssk;
229 struct in_addr addr;
230 in_port_t port;
231
232 ssk = sdp_sk(so);
233 SDP_RLOCK(ssk);
234 port = ssk->fport;
235 addr.s_addr = ssk->faddr;
236 SDP_RUNLOCK(ssk);
237
238 *nam = sdp_sockaddr(port, &addr);
239 return 0;
240 }
241
242 static void
sdp_pcbnotifyall(struct in_addr faddr,int errno,struct sdp_sock * (* notify)(struct sdp_sock *,int))243 sdp_pcbnotifyall(struct in_addr faddr, int errno,
244 struct sdp_sock *(*notify)(struct sdp_sock *, int))
245 {
246 struct sdp_sock *ssk, *ssk_temp;
247
248 SDP_LIST_WLOCK();
249 LIST_FOREACH_SAFE(ssk, &sdp_list, list, ssk_temp) {
250 SDP_WLOCK(ssk);
251 if (ssk->faddr != faddr.s_addr || ssk->socket == NULL) {
252 SDP_WUNLOCK(ssk);
253 continue;
254 }
255 if ((ssk->flags & SDP_DESTROY) == 0)
256 if ((*notify)(ssk, errno))
257 SDP_WUNLOCK(ssk);
258 }
259 SDP_LIST_WUNLOCK();
260 }
261
262 #if 0
263 static void
264 sdp_apply_all(void (*func)(struct sdp_sock *, void *), void *arg)
265 {
266 struct sdp_sock *ssk;
267
268 SDP_LIST_RLOCK();
269 LIST_FOREACH(ssk, &sdp_list, list) {
270 SDP_WLOCK(ssk);
271 func(ssk, arg);
272 SDP_WUNLOCK(ssk);
273 }
274 SDP_LIST_RUNLOCK();
275 }
276 #endif
277
278 static void
sdp_output_reset(struct sdp_sock * ssk)279 sdp_output_reset(struct sdp_sock *ssk)
280 {
281 struct rdma_cm_id *id;
282
283 SDP_WLOCK_ASSERT(ssk);
284 if (ssk->id) {
285 id = ssk->id;
286 ssk->qp_active = 0;
287 SDP_WUNLOCK(ssk);
288 rdma_disconnect(id);
289 SDP_WLOCK(ssk);
290 }
291 ssk->state = TCPS_CLOSED;
292 }
293
294 /*
295 * Attempt to close a SDP socket, marking it as dropped, and freeing
296 * the socket if we hold the only reference.
297 */
298 static struct sdp_sock *
sdp_closed(struct sdp_sock * ssk)299 sdp_closed(struct sdp_sock *ssk)
300 {
301 struct socket *so;
302
303 SDP_WLOCK_ASSERT(ssk);
304
305 ssk->flags |= SDP_DROPPED;
306 so = ssk->socket;
307 soisdisconnected(so);
308 if (ssk->flags & SDP_SOCKREF) {
309 KASSERT(so->so_state & SS_PROTOREF,
310 ("sdp_closed: !SS_PROTOREF"));
311 ssk->flags &= ~SDP_SOCKREF;
312 SDP_WUNLOCK(ssk);
313 ACCEPT_LOCK();
314 SOCK_LOCK(so);
315 so->so_state &= ~SS_PROTOREF;
316 sofree(so);
317 return (NULL);
318 }
319 return (ssk);
320 }
321
322 /*
323 * Perform timer based shutdowns which can not operate in
324 * callout context.
325 */
326 static void
sdp_shutdown_task(void * data,int pending)327 sdp_shutdown_task(void *data, int pending)
328 {
329 struct sdp_sock *ssk;
330
331 ssk = data;
332 SDP_WLOCK(ssk);
333 /*
334 * I don't think this can race with another call to pcbfree()
335 * because SDP_TIMEWAIT protects it. SDP_DESTROY may be redundant.
336 */
337 if (ssk->flags & SDP_DESTROY)
338 panic("sdp_shutdown_task: Racing with pcbfree for ssk %p",
339 ssk);
340 if (ssk->flags & SDP_DISCON)
341 sdp_output_reset(ssk);
342 /* We have to clear this so sdp_detach() will call pcbfree(). */
343 ssk->flags &= ~(SDP_TIMEWAIT | SDP_DREQWAIT);
344 if ((ssk->flags & SDP_DROPPED) == 0 &&
345 sdp_closed(ssk) == NULL)
346 return;
347 if (ssk->socket == NULL) {
348 sdp_pcbfree(ssk);
349 return;
350 }
351 SDP_WUNLOCK(ssk);
352 }
353
354 /*
355 * 2msl has expired, schedule the shutdown task.
356 */
357 static void
sdp_2msl_timeout(void * data)358 sdp_2msl_timeout(void *data)
359 {
360 struct sdp_sock *ssk;
361
362 ssk = data;
363 /* Callout canceled. */
364 if (!callout_active(&ssk->keep2msl))
365 goto out;
366 callout_deactivate(&ssk->keep2msl);
367 /* Should be impossible, defensive programming. */
368 if ((ssk->flags & SDP_TIMEWAIT) == 0)
369 goto out;
370 taskqueue_enqueue(taskqueue_thread, &ssk->shutdown_task);
371 out:
372 SDP_WUNLOCK(ssk);
373 return;
374 }
375
376 /*
377 * Schedule the 2msl wait timer.
378 */
379 static void
sdp_2msl_wait(struct sdp_sock * ssk)380 sdp_2msl_wait(struct sdp_sock *ssk)
381 {
382
383 SDP_WLOCK_ASSERT(ssk);
384 ssk->flags |= SDP_TIMEWAIT;
385 ssk->state = TCPS_TIME_WAIT;
386 soisdisconnected(ssk->socket);
387 callout_reset(&ssk->keep2msl, TCPTV_MSL, sdp_2msl_timeout, ssk);
388 }
389
390 /*
391 * Timed out waiting for the final fin/ack from rdma_disconnect().
392 */
393 static void
sdp_dreq_timeout(void * data)394 sdp_dreq_timeout(void *data)
395 {
396 struct sdp_sock *ssk;
397
398 ssk = data;
399 /* Callout canceled. */
400 if (!callout_active(&ssk->keep2msl))
401 goto out;
402 /* Callout rescheduled, probably as a different timer. */
403 if (callout_pending(&ssk->keep2msl))
404 goto out;
405 callout_deactivate(&ssk->keep2msl);
406 if (ssk->state != TCPS_FIN_WAIT_1 && ssk->state != TCPS_LAST_ACK)
407 goto out;
408 if ((ssk->flags & SDP_DREQWAIT) == 0)
409 goto out;
410 ssk->flags &= ~SDP_DREQWAIT;
411 ssk->flags |= SDP_DISCON;
412 sdp_2msl_wait(ssk);
413 ssk->qp_active = 0;
414 out:
415 SDP_WUNLOCK(ssk);
416 }
417
418 /*
419 * Received the final fin/ack. Cancel the 2msl.
420 */
421 void
sdp_cancel_dreq_wait_timeout(struct sdp_sock * ssk)422 sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk)
423 {
424 sdp_dbg(ssk->socket, "cancelling dreq wait timeout\n");
425 ssk->flags &= ~SDP_DREQWAIT;
426 sdp_2msl_wait(ssk);
427 }
428
429 static int
sdp_init_sock(struct socket * sk)430 sdp_init_sock(struct socket *sk)
431 {
432 struct sdp_sock *ssk = sdp_sk(sk);
433
434 sdp_dbg(sk, "%s\n", __func__);
435
436 callout_init_rw(&ssk->keep2msl, &ssk->lock, CALLOUT_RETURNUNLOCKED);
437 TASK_INIT(&ssk->shutdown_task, 0, sdp_shutdown_task, ssk);
438 #ifdef SDP_ZCOPY
439 INIT_DELAYED_WORK(&ssk->srcavail_cancel_work, srcavail_cancel_timeout);
440 ssk->zcopy_thresh = -1; /* use global sdp_zcopy_thresh */
441 ssk->tx_ring.rdma_inflight = NULL;
442 #endif
443 atomic_set(&ssk->mseq_ack, 0);
444 sdp_rx_ring_init(ssk);
445 ssk->tx_ring.buffer = NULL;
446
447 return 0;
448 }
449
450 /*
451 * Allocate an sdp_sock for the socket and reserve socket buffer space.
452 */
453 static int
sdp_attach(struct socket * so,int proto,struct thread * td)454 sdp_attach(struct socket *so, int proto, struct thread *td)
455 {
456 struct sdp_sock *ssk;
457 int error;
458
459 ssk = sdp_sk(so);
460 KASSERT(ssk == NULL, ("sdp_attach: ssk already set on so %p", so));
461 if (so->so_snd.sb_hiwat == 0 || so->so_rcv.sb_hiwat == 0) {
462 error = soreserve(so, sdp_sendspace, sdp_recvspace);
463 if (error)
464 return (error);
465 }
466 so->so_rcv.sb_flags |= SB_AUTOSIZE;
467 so->so_snd.sb_flags |= SB_AUTOSIZE;
468 ssk = uma_zalloc(sdp_zone, M_NOWAIT | M_ZERO);
469 if (ssk == NULL)
470 return (ENOBUFS);
471 rw_init(&ssk->lock, "sdpsock");
472 ssk->socket = so;
473 ssk->cred = crhold(so->so_cred);
474 so->so_pcb = (caddr_t)ssk;
475 sdp_init_sock(so);
476 ssk->flags = 0;
477 ssk->qp_active = 0;
478 ssk->state = TCPS_CLOSED;
479 mbufq_init(&ssk->rxctlq, INT_MAX);
480 SDP_LIST_WLOCK();
481 LIST_INSERT_HEAD(&sdp_list, ssk, list);
482 sdp_count++;
483 SDP_LIST_WUNLOCK();
484 if ((so->so_options & SO_LINGER) && so->so_linger == 0)
485 so->so_linger = TCP_LINGERTIME;
486
487 return (0);
488 }
489
490 /*
491 * Detach SDP from the socket, potentially leaving it around for the
492 * timewait to expire.
493 */
494 static void
sdp_detach(struct socket * so)495 sdp_detach(struct socket *so)
496 {
497 struct sdp_sock *ssk;
498
499 ssk = sdp_sk(so);
500 SDP_WLOCK(ssk);
501 KASSERT(ssk->socket != NULL, ("sdp_detach: socket is NULL"));
502 ssk->socket->so_pcb = NULL;
503 ssk->socket = NULL;
504 if (ssk->flags & (SDP_TIMEWAIT | SDP_DREQWAIT))
505 SDP_WUNLOCK(ssk);
506 else if (ssk->flags & SDP_DROPPED || ssk->state < TCPS_SYN_SENT)
507 sdp_pcbfree(ssk);
508 else
509 panic("sdp_detach: Unexpected state, ssk %p.\n", ssk);
510 }
511
512 /*
513 * Allocate a local address for the socket.
514 */
515 static int
sdp_bind(struct socket * so,struct sockaddr * nam,struct thread * td)516 sdp_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
517 {
518 int error = 0;
519 struct sdp_sock *ssk;
520 struct sockaddr_in *sin;
521
522 sin = (struct sockaddr_in *)nam;
523 if (nam->sa_len != sizeof (*sin))
524 return (EINVAL);
525 if (sin->sin_family != AF_INET)
526 return (EINVAL);
527 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr)))
528 return (EAFNOSUPPORT);
529
530 ssk = sdp_sk(so);
531 SDP_WLOCK(ssk);
532 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
533 error = EINVAL;
534 goto out;
535 }
536 error = sdp_pcbbind(ssk, nam, td->td_ucred);
537 out:
538 SDP_WUNLOCK(ssk);
539
540 return (error);
541 }
542
543 /*
544 * Prepare to accept connections.
545 */
546 static int
sdp_listen(struct socket * so,int backlog,struct thread * td)547 sdp_listen(struct socket *so, int backlog, struct thread *td)
548 {
549 int error = 0;
550 struct sdp_sock *ssk;
551
552 ssk = sdp_sk(so);
553 SDP_WLOCK(ssk);
554 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
555 error = EINVAL;
556 goto out;
557 }
558 if (error == 0 && ssk->lport == 0)
559 error = sdp_pcbbind(ssk, (struct sockaddr *)0, td->td_ucred);
560 SOCK_LOCK(so);
561 if (error == 0)
562 error = solisten_proto_check(so);
563 if (error == 0) {
564 solisten_proto(so, backlog);
565 ssk->state = TCPS_LISTEN;
566 }
567 SOCK_UNLOCK(so);
568
569 out:
570 SDP_WUNLOCK(ssk);
571 if (error == 0)
572 error = -rdma_listen(ssk->id, backlog);
573 return (error);
574 }
575
576 /*
577 * Initiate a SDP connection to nam.
578 */
579 static int
sdp_start_connect(struct sdp_sock * ssk,struct sockaddr * nam,struct thread * td)580 sdp_start_connect(struct sdp_sock *ssk, struct sockaddr *nam, struct thread *td)
581 {
582 struct sockaddr_in src;
583 struct socket *so;
584 int error;
585
586 so = ssk->socket;
587
588 SDP_WLOCK_ASSERT(ssk);
589 if (ssk->lport == 0) {
590 error = sdp_pcbbind(ssk, (struct sockaddr *)0, td->td_ucred);
591 if (error)
592 return error;
593 }
594 src.sin_family = AF_INET;
595 src.sin_len = sizeof(src);
596 bzero(&src.sin_zero, sizeof(src.sin_zero));
597 src.sin_port = ssk->lport;
598 src.sin_addr.s_addr = ssk->laddr;
599 soisconnecting(so);
600 SDP_WUNLOCK(ssk);
601 error = -rdma_resolve_addr(ssk->id, (struct sockaddr *)&src, nam,
602 SDP_RESOLVE_TIMEOUT);
603 SDP_WLOCK(ssk);
604 if (error == 0)
605 ssk->state = TCPS_SYN_SENT;
606
607 return 0;
608 }
609
610 /*
611 * Initiate SDP connection.
612 */
613 static int
sdp_connect(struct socket * so,struct sockaddr * nam,struct thread * td)614 sdp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
615 {
616 int error = 0;
617 struct sdp_sock *ssk;
618 struct sockaddr_in *sin;
619
620 sin = (struct sockaddr_in *)nam;
621 if (nam->sa_len != sizeof (*sin))
622 return (EINVAL);
623 if (sin->sin_family != AF_INET)
624 return (EINVAL);
625 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr)))
626 return (EAFNOSUPPORT);
627 if ((error = prison_remote_ip4(td->td_ucred, &sin->sin_addr)) != 0)
628 return (error);
629 ssk = sdp_sk(so);
630 SDP_WLOCK(ssk);
631 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED))
632 error = EINVAL;
633 else
634 error = sdp_start_connect(ssk, nam, td);
635 SDP_WUNLOCK(ssk);
636 return (error);
637 }
638
639 /*
640 * Drop a SDP socket, reporting
641 * the specified error. If connection is synchronized,
642 * then send a RST to peer.
643 */
644 static struct sdp_sock *
sdp_drop(struct sdp_sock * ssk,int errno)645 sdp_drop(struct sdp_sock *ssk, int errno)
646 {
647 struct socket *so;
648
649 SDP_WLOCK_ASSERT(ssk);
650 so = ssk->socket;
651 if (TCPS_HAVERCVDSYN(ssk->state))
652 sdp_output_reset(ssk);
653 if (errno == ETIMEDOUT && ssk->softerror)
654 errno = ssk->softerror;
655 so->so_error = errno;
656 return (sdp_closed(ssk));
657 }
658
659 /*
660 * User issued close, and wish to trail through shutdown states:
661 * if never received SYN, just forget it. If got a SYN from peer,
662 * but haven't sent FIN, then go to FIN_WAIT_1 state to send peer a FIN.
663 * If already got a FIN from peer, then almost done; go to LAST_ACK
664 * state. In all other cases, have already sent FIN to peer (e.g.
665 * after PRU_SHUTDOWN), and just have to play tedious game waiting
666 * for peer to send FIN or not respond to keep-alives, etc.
667 * We can let the user exit from the close as soon as the FIN is acked.
668 */
669 static void
sdp_usrclosed(struct sdp_sock * ssk)670 sdp_usrclosed(struct sdp_sock *ssk)
671 {
672
673 SDP_WLOCK_ASSERT(ssk);
674
675 switch (ssk->state) {
676 case TCPS_LISTEN:
677 ssk->state = TCPS_CLOSED;
678 SDP_WUNLOCK(ssk);
679 sdp_destroy_cma(ssk);
680 SDP_WLOCK(ssk);
681 /* FALLTHROUGH */
682 case TCPS_CLOSED:
683 ssk = sdp_closed(ssk);
684 /*
685 * sdp_closed() should never return NULL here as the socket is
686 * still open.
687 */
688 KASSERT(ssk != NULL,
689 ("sdp_usrclosed: sdp_closed() returned NULL"));
690 break;
691
692 case TCPS_SYN_SENT:
693 /* FALLTHROUGH */
694 case TCPS_SYN_RECEIVED:
695 ssk->flags |= SDP_NEEDFIN;
696 break;
697
698 case TCPS_ESTABLISHED:
699 ssk->flags |= SDP_NEEDFIN;
700 ssk->state = TCPS_FIN_WAIT_1;
701 break;
702
703 case TCPS_CLOSE_WAIT:
704 ssk->state = TCPS_LAST_ACK;
705 break;
706 }
707 if (ssk->state >= TCPS_FIN_WAIT_2) {
708 /* Prevent the connection hanging in FIN_WAIT_2 forever. */
709 if (ssk->state == TCPS_FIN_WAIT_2)
710 sdp_2msl_wait(ssk);
711 else
712 soisdisconnected(ssk->socket);
713 }
714 }
715
716 static void
sdp_output_disconnect(struct sdp_sock * ssk)717 sdp_output_disconnect(struct sdp_sock *ssk)
718 {
719
720 SDP_WLOCK_ASSERT(ssk);
721 callout_reset(&ssk->keep2msl, SDP_FIN_WAIT_TIMEOUT,
722 sdp_dreq_timeout, ssk);
723 ssk->flags |= SDP_NEEDFIN | SDP_DREQWAIT;
724 sdp_post_sends(ssk, M_NOWAIT);
725 }
726
727 /*
728 * Initiate or continue a disconnect.
729 * If embryonic state, just send reset (once).
730 * If in ``let data drain'' option and linger null, just drop.
731 * Otherwise (hard), mark socket disconnecting and drop
732 * current input data; switch states based on user close, and
733 * send segment to peer (with FIN).
734 */
735 static void
sdp_start_disconnect(struct sdp_sock * ssk)736 sdp_start_disconnect(struct sdp_sock *ssk)
737 {
738 struct socket *so;
739 int unread;
740
741 so = ssk->socket;
742 SDP_WLOCK_ASSERT(ssk);
743 sdp_stop_keepalive_timer(so);
744 /*
745 * Neither sdp_closed() nor sdp_drop() should return NULL, as the
746 * socket is still open.
747 */
748 if (ssk->state < TCPS_ESTABLISHED) {
749 ssk = sdp_closed(ssk);
750 KASSERT(ssk != NULL,
751 ("sdp_start_disconnect: sdp_close() returned NULL"));
752 } else if ((so->so_options & SO_LINGER) && so->so_linger == 0) {
753 ssk = sdp_drop(ssk, 0);
754 KASSERT(ssk != NULL,
755 ("sdp_start_disconnect: sdp_drop() returned NULL"));
756 } else {
757 soisdisconnecting(so);
758 unread = sbused(&so->so_rcv);
759 sbflush(&so->so_rcv);
760 sdp_usrclosed(ssk);
761 if (!(ssk->flags & SDP_DROPPED)) {
762 if (unread)
763 sdp_output_reset(ssk);
764 else
765 sdp_output_disconnect(ssk);
766 }
767 }
768 }
769
770 /*
771 * User initiated disconnect.
772 */
773 static int
sdp_disconnect(struct socket * so)774 sdp_disconnect(struct socket *so)
775 {
776 struct sdp_sock *ssk;
777 int error = 0;
778
779 ssk = sdp_sk(so);
780 SDP_WLOCK(ssk);
781 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
782 error = ECONNRESET;
783 goto out;
784 }
785 sdp_start_disconnect(ssk);
786 out:
787 SDP_WUNLOCK(ssk);
788 return (error);
789 }
790
791 /*
792 * Accept a connection. Essentially all the work is done at higher levels;
793 * just return the address of the peer, storing through addr.
794 *
795 *
796 * XXX This is broken XXX
797 *
798 * The rationale for acquiring the sdp lock here is somewhat complicated,
799 * and is described in detail in the commit log entry for r175612. Acquiring
800 * it delays an accept(2) racing with sonewconn(), which inserts the socket
801 * before the address/port fields are initialized. A better fix would
802 * prevent the socket from being placed in the listen queue until all fields
803 * are fully initialized.
804 */
805 static int
sdp_accept(struct socket * so,struct sockaddr ** nam)806 sdp_accept(struct socket *so, struct sockaddr **nam)
807 {
808 struct sdp_sock *ssk = NULL;
809 struct in_addr addr;
810 in_port_t port;
811 int error;
812
813 if (so->so_state & SS_ISDISCONNECTED)
814 return (ECONNABORTED);
815
816 port = 0;
817 addr.s_addr = 0;
818 error = 0;
819 ssk = sdp_sk(so);
820 SDP_WLOCK(ssk);
821 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
822 error = ECONNABORTED;
823 goto out;
824 }
825 port = ssk->fport;
826 addr.s_addr = ssk->faddr;
827 out:
828 SDP_WUNLOCK(ssk);
829 if (error == 0)
830 *nam = sdp_sockaddr(port, &addr);
831 return error;
832 }
833
834 /*
835 * Mark the connection as being incapable of further output.
836 */
837 static int
sdp_shutdown(struct socket * so)838 sdp_shutdown(struct socket *so)
839 {
840 int error = 0;
841 struct sdp_sock *ssk;
842
843 ssk = sdp_sk(so);
844 SDP_WLOCK(ssk);
845 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
846 error = ECONNRESET;
847 goto out;
848 }
849 socantsendmore(so);
850 sdp_usrclosed(ssk);
851 if (!(ssk->flags & SDP_DROPPED))
852 sdp_output_disconnect(ssk);
853
854 out:
855 SDP_WUNLOCK(ssk);
856
857 return (error);
858 }
859
860 static void
sdp_append(struct sdp_sock * ssk,struct sockbuf * sb,struct mbuf * mb,int cnt)861 sdp_append(struct sdp_sock *ssk, struct sockbuf *sb, struct mbuf *mb, int cnt)
862 {
863 struct mbuf *n;
864 int ncnt;
865
866 SOCKBUF_LOCK_ASSERT(sb);
867 SBLASTRECORDCHK(sb);
868 KASSERT(mb->m_flags & M_PKTHDR,
869 ("sdp_append: %p Missing packet header.\n", mb));
870 n = sb->sb_lastrecord;
871 /*
872 * If the queue is empty just set all pointers and proceed.
873 */
874 if (n == NULL) {
875 sb->sb_lastrecord = sb->sb_mb = sb->sb_sndptr = mb;
876 for (; mb; mb = mb->m_next) {
877 sb->sb_mbtail = mb;
878 sballoc(sb, mb);
879 }
880 return;
881 }
882 /*
883 * Count the number of mbufs in the current tail.
884 */
885 for (ncnt = 0; n->m_next; n = n->m_next)
886 ncnt++;
887 n = sb->sb_lastrecord;
888 /*
889 * If the two chains can fit in a single sdp packet and
890 * the last record has not been sent yet (WRITABLE) coalesce
891 * them. The lastrecord remains the same but we must strip the
892 * packet header and then let sbcompress do the hard part.
893 */
894 if (M_WRITABLE(n) && ncnt + cnt < SDP_MAX_SEND_SGES &&
895 n->m_pkthdr.len + mb->m_pkthdr.len - SDP_HEAD_SIZE <
896 ssk->xmit_size_goal) {
897 m_adj(mb, SDP_HEAD_SIZE);
898 n->m_pkthdr.len += mb->m_pkthdr.len;
899 n->m_flags |= mb->m_flags & (M_PUSH | M_URG);
900 m_demote(mb, 1, 0);
901 sbcompress(sb, mb, sb->sb_mbtail);
902 return;
903 }
904 /*
905 * Not compressible, just append to the end and adjust counters.
906 */
907 sb->sb_lastrecord->m_flags |= M_PUSH;
908 sb->sb_lastrecord->m_nextpkt = mb;
909 sb->sb_lastrecord = mb;
910 if (sb->sb_sndptr == NULL)
911 sb->sb_sndptr = mb;
912 for (; mb; mb = mb->m_next) {
913 sb->sb_mbtail = mb;
914 sballoc(sb, mb);
915 }
916 }
917
918 /*
919 * Do a send by putting data in output queue and updating urgent
920 * marker if URG set. Possibly send more data. Unlike the other
921 * pru_*() routines, the mbuf chains are our responsibility. We
922 * must either enqueue them or free them. The other pru_* routines
923 * generally are caller-frees.
924 *
925 * This comes from sendfile, normal sends will come from sdp_sosend().
926 */
927 static int
sdp_send(struct socket * so,int flags,struct mbuf * m,struct sockaddr * nam,struct mbuf * control,struct thread * td)928 sdp_send(struct socket *so, int flags, struct mbuf *m,
929 struct sockaddr *nam, struct mbuf *control, struct thread *td)
930 {
931 struct sdp_sock *ssk;
932 struct mbuf *n;
933 int error;
934 int cnt;
935
936 error = 0;
937 ssk = sdp_sk(so);
938 KASSERT(m->m_flags & M_PKTHDR,
939 ("sdp_send: %p no packet header", m));
940 M_PREPEND(m, SDP_HEAD_SIZE, M_WAITOK);
941 mtod(m, struct sdp_bsdh *)->mid = SDP_MID_DATA;
942 for (n = m, cnt = 0; n->m_next; n = n->m_next)
943 cnt++;
944 if (cnt > SDP_MAX_SEND_SGES) {
945 n = m_collapse(m, M_WAITOK, SDP_MAX_SEND_SGES);
946 if (n == NULL) {
947 m_freem(m);
948 return (EMSGSIZE);
949 }
950 m = n;
951 for (cnt = 0; n->m_next; n = n->m_next)
952 cnt++;
953 }
954 SDP_WLOCK(ssk);
955 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
956 if (control)
957 m_freem(control);
958 if (m)
959 m_freem(m);
960 error = ECONNRESET;
961 goto out;
962 }
963 if (control) {
964 /* SDP doesn't support control messages. */
965 if (control->m_len) {
966 m_freem(control);
967 if (m)
968 m_freem(m);
969 error = EINVAL;
970 goto out;
971 }
972 m_freem(control); /* empty control, just free it */
973 }
974 if (!(flags & PRUS_OOB)) {
975 SOCKBUF_LOCK(&so->so_snd);
976 sdp_append(ssk, &so->so_snd, m, cnt);
977 SOCKBUF_UNLOCK(&so->so_snd);
978 if (nam && ssk->state < TCPS_SYN_SENT) {
979 /*
980 * Do implied connect if not yet connected.
981 */
982 error = sdp_start_connect(ssk, nam, td);
983 if (error)
984 goto out;
985 }
986 if (flags & PRUS_EOF) {
987 /*
988 * Close the send side of the connection after
989 * the data is sent.
990 */
991 socantsendmore(so);
992 sdp_usrclosed(ssk);
993 if (!(ssk->flags & SDP_DROPPED))
994 sdp_output_disconnect(ssk);
995 } else if (!(ssk->flags & SDP_DROPPED) &&
996 !(flags & PRUS_MORETOCOME))
997 sdp_post_sends(ssk, M_NOWAIT);
998 SDP_WUNLOCK(ssk);
999 return (0);
1000 } else {
1001 SOCKBUF_LOCK(&so->so_snd);
1002 if (sbspace(&so->so_snd) < -512) {
1003 SOCKBUF_UNLOCK(&so->so_snd);
1004 m_freem(m);
1005 error = ENOBUFS;
1006 goto out;
1007 }
1008 /*
1009 * According to RFC961 (Assigned Protocols),
1010 * the urgent pointer points to the last octet
1011 * of urgent data. We continue, however,
1012 * to consider it to indicate the first octet
1013 * of data past the urgent section.
1014 * Otherwise, snd_up should be one lower.
1015 */
1016 m->m_flags |= M_URG | M_PUSH;
1017 sdp_append(ssk, &so->so_snd, m, cnt);
1018 SOCKBUF_UNLOCK(&so->so_snd);
1019 if (nam && ssk->state < TCPS_SYN_SENT) {
1020 /*
1021 * Do implied connect if not yet connected.
1022 */
1023 error = sdp_start_connect(ssk, nam, td);
1024 if (error)
1025 goto out;
1026 }
1027 sdp_post_sends(ssk, M_NOWAIT);
1028 SDP_WUNLOCK(ssk);
1029 return (0);
1030 }
1031 out:
1032 SDP_WUNLOCK(ssk);
1033 return (error);
1034 }
1035
1036 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT)
1037
1038 /*
1039 * Send on a socket. If send must go all at once and message is larger than
1040 * send buffering, then hard error. Lock against other senders. If must go
1041 * all at once and not enough room now, then inform user that this would
1042 * block and do nothing. Otherwise, if nonblocking, send as much as
1043 * possible. The data to be sent is described by "uio" if nonzero, otherwise
1044 * by the mbuf chain "top" (which must be null if uio is not). Data provided
1045 * in mbuf chain must be small enough to send all at once.
1046 *
1047 * Returns nonzero on error, timeout or signal; callers must check for short
1048 * counts if EINTR/ERESTART are returned. Data and control buffers are freed
1049 * on return.
1050 */
1051 static int
sdp_sosend(struct socket * so,struct sockaddr * addr,struct uio * uio,struct mbuf * top,struct mbuf * control,int flags,struct thread * td)1052 sdp_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
1053 struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
1054 {
1055 struct sdp_sock *ssk;
1056 long space, resid;
1057 int atomic;
1058 int error;
1059 int copy;
1060
1061 if (uio != NULL)
1062 resid = uio->uio_resid;
1063 else
1064 resid = top->m_pkthdr.len;
1065 atomic = top != NULL;
1066 if (control != NULL) {
1067 if (control->m_len) {
1068 m_freem(control);
1069 if (top)
1070 m_freem(top);
1071 return (EINVAL);
1072 }
1073 m_freem(control);
1074 control = NULL;
1075 }
1076 /*
1077 * In theory resid should be unsigned. However, space must be
1078 * signed, as it might be less than 0 if we over-committed, and we
1079 * must use a signed comparison of space and resid. On the other
1080 * hand, a negative resid causes us to loop sending 0-length
1081 * segments to the protocol.
1082 *
1083 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM
1084 * type sockets since that's an error.
1085 */
1086 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) {
1087 error = EINVAL;
1088 goto out;
1089 }
1090 if (td != NULL)
1091 td->td_ru.ru_msgsnd++;
1092
1093 ssk = sdp_sk(so);
1094 error = sblock(&so->so_snd, SBLOCKWAIT(flags));
1095 if (error)
1096 goto out;
1097
1098 restart:
1099 do {
1100 SOCKBUF_LOCK(&so->so_snd);
1101 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
1102 SOCKBUF_UNLOCK(&so->so_snd);
1103 error = EPIPE;
1104 goto release;
1105 }
1106 if (so->so_error) {
1107 error = so->so_error;
1108 so->so_error = 0;
1109 SOCKBUF_UNLOCK(&so->so_snd);
1110 goto release;
1111 }
1112 if ((so->so_state & SS_ISCONNECTED) == 0 && addr == NULL) {
1113 SOCKBUF_UNLOCK(&so->so_snd);
1114 error = ENOTCONN;
1115 goto release;
1116 }
1117 space = sbspace(&so->so_snd);
1118 if (flags & MSG_OOB)
1119 space += 1024;
1120 if (atomic && resid > ssk->xmit_size_goal - SDP_HEAD_SIZE) {
1121 SOCKBUF_UNLOCK(&so->so_snd);
1122 error = EMSGSIZE;
1123 goto release;
1124 }
1125 if (space < resid &&
1126 (atomic || space < so->so_snd.sb_lowat)) {
1127 if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) {
1128 SOCKBUF_UNLOCK(&so->so_snd);
1129 error = EWOULDBLOCK;
1130 goto release;
1131 }
1132 error = sbwait(&so->so_snd);
1133 SOCKBUF_UNLOCK(&so->so_snd);
1134 if (error)
1135 goto release;
1136 goto restart;
1137 }
1138 SOCKBUF_UNLOCK(&so->so_snd);
1139 do {
1140 if (uio == NULL) {
1141 resid = 0;
1142 if (flags & MSG_EOR)
1143 top->m_flags |= M_EOR;
1144 } else {
1145 /*
1146 * Copy the data from userland into a mbuf
1147 * chain. If no data is to be copied in,
1148 * a single empty mbuf is returned.
1149 */
1150 copy = min(space,
1151 ssk->xmit_size_goal - SDP_HEAD_SIZE);
1152 top = m_uiotombuf(uio, M_WAITOK, copy,
1153 0, M_PKTHDR |
1154 ((flags & MSG_EOR) ? M_EOR : 0));
1155 if (top == NULL) {
1156 /* only possible error */
1157 error = EFAULT;
1158 goto release;
1159 }
1160 space -= resid - uio->uio_resid;
1161 resid = uio->uio_resid;
1162 }
1163 /*
1164 * XXX all the SBS_CANTSENDMORE checks previously
1165 * done could be out of date after dropping the
1166 * socket lock.
1167 */
1168 error = sdp_send(so, (flags & MSG_OOB) ? PRUS_OOB :
1169 /*
1170 * Set EOF on the last send if the user specified
1171 * MSG_EOF.
1172 */
1173 ((flags & MSG_EOF) && (resid <= 0)) ? PRUS_EOF :
1174 /* If there is more to send set PRUS_MORETOCOME. */
1175 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
1176 top, addr, NULL, td);
1177 top = NULL;
1178 if (error)
1179 goto release;
1180 } while (resid && space > 0);
1181 } while (resid);
1182
1183 release:
1184 sbunlock(&so->so_snd);
1185 out:
1186 if (top != NULL)
1187 m_freem(top);
1188 return (error);
1189 }
1190
1191 /*
1192 * The part of soreceive() that implements reading non-inline out-of-band
1193 * data from a socket. For more complete comments, see soreceive(), from
1194 * which this code originated.
1195 *
1196 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is
1197 * unable to return an mbuf chain to the caller.
1198 */
1199 static int
soreceive_rcvoob(struct socket * so,struct uio * uio,int flags)1200 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags)
1201 {
1202 struct protosw *pr = so->so_proto;
1203 struct mbuf *m;
1204 int error;
1205
1206 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0"));
1207
1208 m = m_get(M_WAITOK, MT_DATA);
1209 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
1210 if (error)
1211 goto bad;
1212 do {
1213 error = uiomove(mtod(m, void *),
1214 (int) min(uio->uio_resid, m->m_len), uio);
1215 m = m_free(m);
1216 } while (uio->uio_resid && error == 0 && m);
1217 bad:
1218 if (m != NULL)
1219 m_freem(m);
1220 return (error);
1221 }
1222
1223 /*
1224 * Optimized version of soreceive() for stream (TCP) sockets.
1225 */
1226 static int
sdp_sorecv(struct socket * so,struct sockaddr ** psa,struct uio * uio,struct mbuf ** mp0,struct mbuf ** controlp,int * flagsp)1227 sdp_sorecv(struct socket *so, struct sockaddr **psa, struct uio *uio,
1228 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
1229 {
1230 int len = 0, error = 0, flags, oresid;
1231 struct sockbuf *sb;
1232 struct mbuf *m, *n = NULL;
1233 struct sdp_sock *ssk;
1234
1235 /* We only do stream sockets. */
1236 if (so->so_type != SOCK_STREAM)
1237 return (EINVAL);
1238 if (psa != NULL)
1239 *psa = NULL;
1240 if (controlp != NULL)
1241 return (EINVAL);
1242 if (flagsp != NULL)
1243 flags = *flagsp &~ MSG_EOR;
1244 else
1245 flags = 0;
1246 if (flags & MSG_OOB)
1247 return (soreceive_rcvoob(so, uio, flags));
1248 if (mp0 != NULL)
1249 *mp0 = NULL;
1250
1251 sb = &so->so_rcv;
1252 ssk = sdp_sk(so);
1253
1254 /* Prevent other readers from entering the socket. */
1255 error = sblock(sb, SBLOCKWAIT(flags));
1256 if (error)
1257 goto out;
1258 SOCKBUF_LOCK(sb);
1259
1260 /* Easy one, no space to copyout anything. */
1261 if (uio->uio_resid == 0) {
1262 error = EINVAL;
1263 goto out;
1264 }
1265 oresid = uio->uio_resid;
1266
1267 /* We will never ever get anything unless we are connected. */
1268 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
1269 /* When disconnecting there may be still some data left. */
1270 if (sbavail(sb))
1271 goto deliver;
1272 if (!(so->so_state & SS_ISDISCONNECTED))
1273 error = ENOTCONN;
1274 goto out;
1275 }
1276
1277 /* Socket buffer is empty and we shall not block. */
1278 if (sbavail(sb) == 0 &&
1279 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) {
1280 error = EAGAIN;
1281 goto out;
1282 }
1283
1284 restart:
1285 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
1286
1287 /* Abort if socket has reported problems. */
1288 if (so->so_error) {
1289 if (sbavail(sb))
1290 goto deliver;
1291 if (oresid > uio->uio_resid)
1292 goto out;
1293 error = so->so_error;
1294 if (!(flags & MSG_PEEK))
1295 so->so_error = 0;
1296 goto out;
1297 }
1298
1299 /* Door is closed. Deliver what is left, if any. */
1300 if (sb->sb_state & SBS_CANTRCVMORE) {
1301 if (sbavail(sb))
1302 goto deliver;
1303 else
1304 goto out;
1305 }
1306
1307 /* Socket buffer got some data that we shall deliver now. */
1308 if (sbavail(sb) && !(flags & MSG_WAITALL) &&
1309 ((so->so_state & SS_NBIO) ||
1310 (flags & (MSG_DONTWAIT|MSG_NBIO)) ||
1311 sbavail(sb) >= sb->sb_lowat ||
1312 sbavail(sb) >= uio->uio_resid ||
1313 sbavail(sb) >= sb->sb_hiwat) ) {
1314 goto deliver;
1315 }
1316
1317 /* On MSG_WAITALL we must wait until all data or error arrives. */
1318 if ((flags & MSG_WAITALL) &&
1319 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_lowat))
1320 goto deliver;
1321
1322 /*
1323 * Wait and block until (more) data comes in.
1324 * NB: Drops the sockbuf lock during wait.
1325 */
1326 error = sbwait(sb);
1327 if (error)
1328 goto out;
1329 goto restart;
1330
1331 deliver:
1332 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
1333 KASSERT(sbavail(sb), ("%s: sockbuf empty", __func__));
1334 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__));
1335
1336 /* Statistics. */
1337 if (uio->uio_td)
1338 uio->uio_td->td_ru.ru_msgrcv++;
1339
1340 /* Fill uio until full or current end of socket buffer is reached. */
1341 len = min(uio->uio_resid, sbavail(sb));
1342 if (mp0 != NULL) {
1343 /* Dequeue as many mbufs as possible. */
1344 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) {
1345 for (*mp0 = m = sb->sb_mb;
1346 m != NULL && m->m_len <= len;
1347 m = m->m_next) {
1348 len -= m->m_len;
1349 uio->uio_resid -= m->m_len;
1350 sbfree(sb, m);
1351 n = m;
1352 }
1353 sb->sb_mb = m;
1354 if (sb->sb_mb == NULL)
1355 SB_EMPTY_FIXUP(sb);
1356 n->m_next = NULL;
1357 }
1358 /* Copy the remainder. */
1359 if (len > 0) {
1360 KASSERT(sb->sb_mb != NULL,
1361 ("%s: len > 0 && sb->sb_mb empty", __func__));
1362
1363 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT);
1364 if (m == NULL)
1365 len = 0; /* Don't flush data from sockbuf. */
1366 else
1367 uio->uio_resid -= m->m_len;
1368 if (*mp0 != NULL)
1369 n->m_next = m;
1370 else
1371 *mp0 = m;
1372 if (*mp0 == NULL) {
1373 error = ENOBUFS;
1374 goto out;
1375 }
1376 }
1377 } else {
1378 /* NB: Must unlock socket buffer as uiomove may sleep. */
1379 SOCKBUF_UNLOCK(sb);
1380 error = m_mbuftouio(uio, sb->sb_mb, len);
1381 SOCKBUF_LOCK(sb);
1382 if (error)
1383 goto out;
1384 }
1385 SBLASTRECORDCHK(sb);
1386 SBLASTMBUFCHK(sb);
1387
1388 /*
1389 * Remove the delivered data from the socket buffer unless we
1390 * were only peeking.
1391 */
1392 if (!(flags & MSG_PEEK)) {
1393 if (len > 0)
1394 sbdrop_locked(sb, len);
1395
1396 /* Notify protocol that we drained some data. */
1397 SOCKBUF_UNLOCK(sb);
1398 SDP_WLOCK(ssk);
1399 sdp_do_posts(ssk);
1400 SDP_WUNLOCK(ssk);
1401 SOCKBUF_LOCK(sb);
1402 }
1403
1404 /*
1405 * For MSG_WAITALL we may have to loop again and wait for
1406 * more data to come in.
1407 */
1408 if ((flags & MSG_WAITALL) && uio->uio_resid > 0)
1409 goto restart;
1410 out:
1411 SOCKBUF_LOCK_ASSERT(sb);
1412 SBLASTRECORDCHK(sb);
1413 SBLASTMBUFCHK(sb);
1414 SOCKBUF_UNLOCK(sb);
1415 sbunlock(sb);
1416 return (error);
1417 }
1418
1419 /*
1420 * Abort is used to teardown a connection typically while sitting in
1421 * the accept queue.
1422 */
1423 void
sdp_abort(struct socket * so)1424 sdp_abort(struct socket *so)
1425 {
1426 struct sdp_sock *ssk;
1427
1428 ssk = sdp_sk(so);
1429 SDP_WLOCK(ssk);
1430 /*
1431 * If we have not yet dropped, do it now.
1432 */
1433 if (!(ssk->flags & SDP_TIMEWAIT) &&
1434 !(ssk->flags & SDP_DROPPED))
1435 sdp_drop(ssk, ECONNABORTED);
1436 KASSERT(ssk->flags & SDP_DROPPED, ("sdp_abort: %p not dropped 0x%X",
1437 ssk, ssk->flags));
1438 SDP_WUNLOCK(ssk);
1439 }
1440
1441 /*
1442 * Close a SDP socket and initiate a friendly disconnect.
1443 */
1444 static void
sdp_close(struct socket * so)1445 sdp_close(struct socket *so)
1446 {
1447 struct sdp_sock *ssk;
1448
1449 ssk = sdp_sk(so);
1450 SDP_WLOCK(ssk);
1451 /*
1452 * If we have not yet dropped, do it now.
1453 */
1454 if (!(ssk->flags & SDP_TIMEWAIT) &&
1455 !(ssk->flags & SDP_DROPPED))
1456 sdp_start_disconnect(ssk);
1457
1458 /*
1459 * If we've still not dropped let the socket layer know we're
1460 * holding on to the socket and pcb for a while.
1461 */
1462 if (!(ssk->flags & SDP_DROPPED)) {
1463 SOCK_LOCK(so);
1464 so->so_state |= SS_PROTOREF;
1465 SOCK_UNLOCK(so);
1466 ssk->flags |= SDP_SOCKREF;
1467 }
1468 SDP_WUNLOCK(ssk);
1469 }
1470
1471 /*
1472 * User requests out-of-band data.
1473 */
1474 static int
sdp_rcvoob(struct socket * so,struct mbuf * m,int flags)1475 sdp_rcvoob(struct socket *so, struct mbuf *m, int flags)
1476 {
1477 int error = 0;
1478 struct sdp_sock *ssk;
1479
1480 ssk = sdp_sk(so);
1481 SDP_WLOCK(ssk);
1482 if (!rx_ring_trylock(&ssk->rx_ring)) {
1483 SDP_WUNLOCK(ssk);
1484 return (ECONNRESET);
1485 }
1486 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
1487 error = ECONNRESET;
1488 goto out;
1489 }
1490 if ((so->so_oobmark == 0 &&
1491 (so->so_rcv.sb_state & SBS_RCVATMARK) == 0) ||
1492 so->so_options & SO_OOBINLINE ||
1493 ssk->oobflags & SDP_HADOOB) {
1494 error = EINVAL;
1495 goto out;
1496 }
1497 if ((ssk->oobflags & SDP_HAVEOOB) == 0) {
1498 error = EWOULDBLOCK;
1499 goto out;
1500 }
1501 m->m_len = 1;
1502 *mtod(m, caddr_t) = ssk->iobc;
1503 if ((flags & MSG_PEEK) == 0)
1504 ssk->oobflags ^= (SDP_HAVEOOB | SDP_HADOOB);
1505 out:
1506 rx_ring_unlock(&ssk->rx_ring);
1507 SDP_WUNLOCK(ssk);
1508 return (error);
1509 }
1510
1511 void
sdp_urg(struct sdp_sock * ssk,struct mbuf * mb)1512 sdp_urg(struct sdp_sock *ssk, struct mbuf *mb)
1513 {
1514 struct mbuf *m;
1515 struct socket *so;
1516
1517 so = ssk->socket;
1518 if (so == NULL)
1519 return;
1520
1521 so->so_oobmark = sbused(&so->so_rcv) + mb->m_pkthdr.len - 1;
1522 sohasoutofband(so);
1523 ssk->oobflags &= ~(SDP_HAVEOOB | SDP_HADOOB);
1524 if (!(so->so_options & SO_OOBINLINE)) {
1525 for (m = mb; m->m_next != NULL; m = m->m_next);
1526 ssk->iobc = *(mtod(m, char *) + m->m_len - 1);
1527 ssk->oobflags |= SDP_HAVEOOB;
1528 m->m_len--;
1529 mb->m_pkthdr.len--;
1530 }
1531 }
1532
1533 /*
1534 * Notify a sdp socket of an asynchronous error.
1535 *
1536 * Do not wake up user since there currently is no mechanism for
1537 * reporting soft errors (yet - a kqueue filter may be added).
1538 */
1539 struct sdp_sock *
sdp_notify(struct sdp_sock * ssk,int error)1540 sdp_notify(struct sdp_sock *ssk, int error)
1541 {
1542
1543 SDP_WLOCK_ASSERT(ssk);
1544
1545 if ((ssk->flags & SDP_TIMEWAIT) ||
1546 (ssk->flags & SDP_DROPPED))
1547 return (ssk);
1548
1549 /*
1550 * Ignore some errors if we are hooked up.
1551 */
1552 if (ssk->state == TCPS_ESTABLISHED &&
1553 (error == EHOSTUNREACH || error == ENETUNREACH ||
1554 error == EHOSTDOWN))
1555 return (ssk);
1556 ssk->softerror = error;
1557 return sdp_drop(ssk, error);
1558 }
1559
1560 static void
sdp_ctlinput(int cmd,struct sockaddr * sa,void * vip)1561 sdp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
1562 {
1563 struct in_addr faddr;
1564
1565 faddr = ((struct sockaddr_in *)sa)->sin_addr;
1566 if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY)
1567 return;
1568
1569 sdp_pcbnotifyall(faddr, inetctlerrmap[cmd], sdp_notify);
1570 }
1571
1572 static int
sdp_control(struct socket * so,u_long cmd,caddr_t data,struct ifnet * ifp,struct thread * td)1573 sdp_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
1574 struct thread *td)
1575 {
1576 return (EOPNOTSUPP);
1577 }
1578
1579 static void
sdp_keepalive_timeout(void * data)1580 sdp_keepalive_timeout(void *data)
1581 {
1582 struct sdp_sock *ssk;
1583
1584 ssk = data;
1585 /* Callout canceled. */
1586 if (!callout_active(&ssk->keep2msl))
1587 return;
1588 /* Callout rescheduled as a different kind of timer. */
1589 if (callout_pending(&ssk->keep2msl))
1590 goto out;
1591 callout_deactivate(&ssk->keep2msl);
1592 if (ssk->flags & SDP_DROPPED ||
1593 (ssk->socket->so_options & SO_KEEPALIVE) == 0)
1594 goto out;
1595 sdp_post_keepalive(ssk);
1596 callout_reset(&ssk->keep2msl, SDP_KEEPALIVE_TIME,
1597 sdp_keepalive_timeout, ssk);
1598 out:
1599 SDP_WUNLOCK(ssk);
1600 }
1601
1602
1603 void
sdp_start_keepalive_timer(struct socket * so)1604 sdp_start_keepalive_timer(struct socket *so)
1605 {
1606 struct sdp_sock *ssk;
1607
1608 ssk = sdp_sk(so);
1609 if (!callout_pending(&ssk->keep2msl))
1610 callout_reset(&ssk->keep2msl, SDP_KEEPALIVE_TIME,
1611 sdp_keepalive_timeout, ssk);
1612 }
1613
1614 static void
sdp_stop_keepalive_timer(struct socket * so)1615 sdp_stop_keepalive_timer(struct socket *so)
1616 {
1617 struct sdp_sock *ssk;
1618
1619 ssk = sdp_sk(so);
1620 callout_stop(&ssk->keep2msl);
1621 }
1622
1623 /*
1624 * sdp_ctloutput() must drop the inpcb lock before performing copyin on
1625 * socket option arguments. When it re-acquires the lock after the copy, it
1626 * has to revalidate that the connection is still valid for the socket
1627 * option.
1628 */
1629 #define SDP_WLOCK_RECHECK(inp) do { \
1630 SDP_WLOCK(ssk); \
1631 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) { \
1632 SDP_WUNLOCK(ssk); \
1633 return (ECONNRESET); \
1634 } \
1635 } while(0)
1636
1637 static int
sdp_ctloutput(struct socket * so,struct sockopt * sopt)1638 sdp_ctloutput(struct socket *so, struct sockopt *sopt)
1639 {
1640 int error, opt, optval;
1641 struct sdp_sock *ssk;
1642
1643 error = 0;
1644 ssk = sdp_sk(so);
1645 if (sopt->sopt_level == SOL_SOCKET && sopt->sopt_name == SO_KEEPALIVE) {
1646 SDP_WLOCK(ssk);
1647 if (so->so_options & SO_KEEPALIVE)
1648 sdp_start_keepalive_timer(so);
1649 else
1650 sdp_stop_keepalive_timer(so);
1651 SDP_WUNLOCK(ssk);
1652 }
1653 if (sopt->sopt_level != IPPROTO_TCP)
1654 return (error);
1655
1656 SDP_WLOCK(ssk);
1657 if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
1658 SDP_WUNLOCK(ssk);
1659 return (ECONNRESET);
1660 }
1661
1662 switch (sopt->sopt_dir) {
1663 case SOPT_SET:
1664 switch (sopt->sopt_name) {
1665 case TCP_NODELAY:
1666 SDP_WUNLOCK(ssk);
1667 error = sooptcopyin(sopt, &optval, sizeof optval,
1668 sizeof optval);
1669 if (error)
1670 return (error);
1671
1672 SDP_WLOCK_RECHECK(ssk);
1673 opt = SDP_NODELAY;
1674 if (optval)
1675 ssk->flags |= opt;
1676 else
1677 ssk->flags &= ~opt;
1678 sdp_do_posts(ssk);
1679 SDP_WUNLOCK(ssk);
1680 break;
1681
1682 default:
1683 SDP_WUNLOCK(ssk);
1684 error = ENOPROTOOPT;
1685 break;
1686 }
1687 break;
1688
1689 case SOPT_GET:
1690 switch (sopt->sopt_name) {
1691 case TCP_NODELAY:
1692 optval = ssk->flags & SDP_NODELAY;
1693 SDP_WUNLOCK(ssk);
1694 error = sooptcopyout(sopt, &optval, sizeof optval);
1695 break;
1696 default:
1697 SDP_WUNLOCK(ssk);
1698 error = ENOPROTOOPT;
1699 break;
1700 }
1701 break;
1702 }
1703 return (error);
1704 }
1705 #undef SDP_WLOCK_RECHECK
1706
1707 int sdp_mod_count = 0;
1708 int sdp_mod_usec = 0;
1709
1710 void
sdp_set_default_moderation(struct sdp_sock * ssk)1711 sdp_set_default_moderation(struct sdp_sock *ssk)
1712 {
1713 if (sdp_mod_count <= 0 || sdp_mod_usec <= 0)
1714 return;
1715 ib_modify_cq(ssk->rx_ring.cq, sdp_mod_count, sdp_mod_usec);
1716 }
1717
1718 static void
sdp_dev_add(struct ib_device * device)1719 sdp_dev_add(struct ib_device *device)
1720 {
1721 struct ib_fmr_pool_param param;
1722 struct sdp_device *sdp_dev;
1723
1724 sdp_dev = malloc(sizeof(*sdp_dev), M_SDP, M_WAITOK | M_ZERO);
1725 sdp_dev->pd = ib_alloc_pd(device, 0);
1726 if (IS_ERR(sdp_dev->pd))
1727 goto out_pd;
1728 memset(¶m, 0, sizeof param);
1729 param.max_pages_per_fmr = SDP_FMR_SIZE;
1730 param.page_shift = PAGE_SHIFT;
1731 param.access = (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ);
1732 param.pool_size = SDP_FMR_POOL_SIZE;
1733 param.dirty_watermark = SDP_FMR_DIRTY_SIZE;
1734 param.cache = 1;
1735 sdp_dev->fmr_pool = ib_create_fmr_pool(sdp_dev->pd, ¶m);
1736 if (IS_ERR(sdp_dev->fmr_pool))
1737 goto out_fmr;
1738 ib_set_client_data(device, &sdp_client, sdp_dev);
1739 return;
1740
1741 out_fmr:
1742 ib_dealloc_pd(sdp_dev->pd);
1743 out_pd:
1744 free(sdp_dev, M_SDP);
1745 }
1746
1747 static void
sdp_dev_rem(struct ib_device * device,void * client_data)1748 sdp_dev_rem(struct ib_device *device, void *client_data)
1749 {
1750 struct sdp_device *sdp_dev;
1751 struct sdp_sock *ssk;
1752
1753 SDP_LIST_WLOCK();
1754 LIST_FOREACH(ssk, &sdp_list, list) {
1755 if (ssk->ib_device != device)
1756 continue;
1757 SDP_WLOCK(ssk);
1758 if ((ssk->flags & SDP_DESTROY) == 0)
1759 ssk = sdp_notify(ssk, ECONNRESET);
1760 if (ssk)
1761 SDP_WUNLOCK(ssk);
1762 }
1763 SDP_LIST_WUNLOCK();
1764 /*
1765 * XXX Do I need to wait between these two?
1766 */
1767 sdp_dev = ib_get_client_data(device, &sdp_client);
1768 if (!sdp_dev)
1769 return;
1770 ib_flush_fmr_pool(sdp_dev->fmr_pool);
1771 ib_destroy_fmr_pool(sdp_dev->fmr_pool);
1772 ib_dealloc_pd(sdp_dev->pd);
1773 free(sdp_dev, M_SDP);
1774 }
1775
1776 struct ib_client sdp_client =
1777 { .name = "sdp", .add = sdp_dev_add, .remove = sdp_dev_rem };
1778
1779
1780 static int
sdp_pcblist(SYSCTL_HANDLER_ARGS)1781 sdp_pcblist(SYSCTL_HANDLER_ARGS)
1782 {
1783 int error, n, i;
1784 struct sdp_sock *ssk;
1785 struct xinpgen xig;
1786
1787 /*
1788 * The process of preparing the TCB list is too time-consuming and
1789 * resource-intensive to repeat twice on every request.
1790 */
1791 if (req->oldptr == NULL) {
1792 n = sdp_count;
1793 n += imax(n / 8, 10);
1794 req->oldidx = 2 * (sizeof xig) + n * sizeof(struct xtcpcb);
1795 return (0);
1796 }
1797
1798 if (req->newptr != NULL)
1799 return (EPERM);
1800
1801 /*
1802 * OK, now we're committed to doing something.
1803 */
1804 SDP_LIST_RLOCK();
1805 n = sdp_count;
1806 SDP_LIST_RUNLOCK();
1807
1808 error = sysctl_wire_old_buffer(req, 2 * (sizeof xig)
1809 + n * sizeof(struct xtcpcb));
1810 if (error != 0)
1811 return (error);
1812
1813 bzero(&xig, sizeof(xig));
1814 xig.xig_len = sizeof xig;
1815 xig.xig_count = n;
1816 xig.xig_gen = 0;
1817 xig.xig_sogen = so_gencnt;
1818 error = SYSCTL_OUT(req, &xig, sizeof xig);
1819 if (error)
1820 return (error);
1821
1822 SDP_LIST_RLOCK();
1823 for (ssk = LIST_FIRST(&sdp_list), i = 0;
1824 ssk != NULL && i < n; ssk = LIST_NEXT(ssk, list)) {
1825 struct xtcpcb xt;
1826
1827 SDP_RLOCK(ssk);
1828 if (ssk->flags & SDP_TIMEWAIT) {
1829 if (ssk->cred != NULL)
1830 error = cr_cansee(req->td->td_ucred,
1831 ssk->cred);
1832 else
1833 error = EINVAL; /* Skip this inp. */
1834 } else if (ssk->socket)
1835 error = cr_canseesocket(req->td->td_ucred,
1836 ssk->socket);
1837 else
1838 error = EINVAL;
1839 if (error) {
1840 error = 0;
1841 goto next;
1842 }
1843
1844 bzero(&xt, sizeof(xt));
1845 xt.xt_len = sizeof xt;
1846 xt.xt_inp.inp_gencnt = 0;
1847 xt.xt_inp.inp_vflag = INP_IPV4;
1848 memcpy(&xt.xt_inp.inp_laddr, &ssk->laddr, sizeof(ssk->laddr));
1849 xt.xt_inp.inp_lport = ssk->lport;
1850 memcpy(&xt.xt_inp.inp_faddr, &ssk->faddr, sizeof(ssk->faddr));
1851 xt.xt_inp.inp_fport = ssk->fport;
1852 xt.xt_tp.t_state = ssk->state;
1853 if (ssk->socket != NULL)
1854 sotoxsocket(ssk->socket, &xt.xt_socket);
1855 xt.xt_socket.xso_protocol = IPPROTO_TCP;
1856 SDP_RUNLOCK(ssk);
1857 error = SYSCTL_OUT(req, &xt, sizeof xt);
1858 if (error)
1859 break;
1860 i++;
1861 continue;
1862 next:
1863 SDP_RUNLOCK(ssk);
1864 }
1865 if (!error) {
1866 /*
1867 * Give the user an updated idea of our state.
1868 * If the generation differs from what we told
1869 * her before, she knows that something happened
1870 * while we were processing this request, and it
1871 * might be necessary to retry.
1872 */
1873 xig.xig_gen = 0;
1874 xig.xig_sogen = so_gencnt;
1875 xig.xig_count = sdp_count;
1876 error = SYSCTL_OUT(req, &xig, sizeof xig);
1877 }
1878 SDP_LIST_RUNLOCK();
1879 return (error);
1880 }
1881
1882 static SYSCTL_NODE(_net_inet, -1, sdp, CTLFLAG_RW, 0, "SDP");
1883
1884 SYSCTL_PROC(_net_inet_sdp, TCPCTL_PCBLIST, pcblist,
1885 CTLFLAG_RD | CTLTYPE_STRUCT, 0, 0, sdp_pcblist, "S,xtcpcb",
1886 "List of active SDP connections");
1887
1888 static void
sdp_zone_change(void * tag)1889 sdp_zone_change(void *tag)
1890 {
1891
1892 uma_zone_set_max(sdp_zone, maxsockets);
1893 }
1894
1895 static void
sdp_init(void)1896 sdp_init(void)
1897 {
1898
1899 LIST_INIT(&sdp_list);
1900 sdp_zone = uma_zcreate("sdp_sock", sizeof(struct sdp_sock),
1901 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
1902 uma_zone_set_max(sdp_zone, maxsockets);
1903 EVENTHANDLER_REGISTER(maxsockets_change, sdp_zone_change, NULL,
1904 EVENTHANDLER_PRI_ANY);
1905 rx_comp_wq = create_singlethread_workqueue("rx_comp_wq");
1906 ib_register_client(&sdp_client);
1907 }
1908
1909 extern struct domain sdpdomain;
1910
1911 struct pr_usrreqs sdp_usrreqs = {
1912 .pru_abort = sdp_abort,
1913 .pru_accept = sdp_accept,
1914 .pru_attach = sdp_attach,
1915 .pru_bind = sdp_bind,
1916 .pru_connect = sdp_connect,
1917 .pru_control = sdp_control,
1918 .pru_detach = sdp_detach,
1919 .pru_disconnect = sdp_disconnect,
1920 .pru_listen = sdp_listen,
1921 .pru_peeraddr = sdp_getpeeraddr,
1922 .pru_rcvoob = sdp_rcvoob,
1923 .pru_send = sdp_send,
1924 .pru_sosend = sdp_sosend,
1925 .pru_soreceive = sdp_sorecv,
1926 .pru_shutdown = sdp_shutdown,
1927 .pru_sockaddr = sdp_getsockaddr,
1928 .pru_close = sdp_close,
1929 };
1930
1931 struct protosw sdpsw[] = {
1932 {
1933 .pr_type = SOCK_STREAM,
1934 .pr_domain = &sdpdomain,
1935 .pr_protocol = IPPROTO_IP,
1936 .pr_flags = PR_CONNREQUIRED|PR_IMPLOPCL|PR_WANTRCVD,
1937 .pr_ctlinput = sdp_ctlinput,
1938 .pr_ctloutput = sdp_ctloutput,
1939 .pr_usrreqs = &sdp_usrreqs
1940 },
1941 {
1942 .pr_type = SOCK_STREAM,
1943 .pr_domain = &sdpdomain,
1944 .pr_protocol = IPPROTO_TCP,
1945 .pr_flags = PR_CONNREQUIRED|PR_IMPLOPCL|PR_WANTRCVD,
1946 .pr_ctlinput = sdp_ctlinput,
1947 .pr_ctloutput = sdp_ctloutput,
1948 .pr_usrreqs = &sdp_usrreqs
1949 },
1950 };
1951
1952 struct domain sdpdomain = {
1953 .dom_family = AF_INET_SDP,
1954 .dom_name = "SDP",
1955 .dom_init = sdp_init,
1956 .dom_protosw = sdpsw,
1957 .dom_protoswNPROTOSW = &sdpsw[sizeof(sdpsw)/sizeof(sdpsw[0])],
1958 };
1959
1960 DOMAIN_SET(sdp);
1961
1962 int sdp_debug_level = 1;
1963 int sdp_data_debug_level = 0;
1964