1 /******************************************************************************
2 * evtchn.c
3 *
4 * Driver for receiving and demuxing event-channel signals.
5 *
6 * Copyright (c) 2004-2005, K A Fraser
7 * Multi-process extensions Copyright (c) 2004, Steven Smith
8 * FreeBSD port Copyright (c) 2014, Roger Pau Monné
9 * Fetched from git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
10 * File: drivers/xen/evtchn.c
11 * Git commit: 0dc0064add422bc0ef5165ebe9ece3052bbd457d
12 *
13 * This program is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License version 2
15 * as published by the Free Software Foundation; or, when distributed
16 * separately from the Linux kernel or incorporated into other
17 * software packages, subject to the following license:
18 *
19 * Permission is hereby granted, free of charge, to any person obtaining a copy
20 * of this source file (the "Software"), to deal in the Software without
21 * restriction, including without limitation the rights to use, copy, modify,
22 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
23 * and to permit persons to whom the Software is furnished to do so, subject to
24 * the following conditions:
25 *
26 * The above copyright notice and this permission notice shall be included in
27 * all copies or substantial portions of the Software.
28 *
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
32 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
33 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
34 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
35 * IN THE SOFTWARE.
36 */
37
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/uio.h>
44 #include <sys/bus.h>
45 #include <sys/malloc.h>
46 #include <sys/kernel.h>
47 #include <sys/lock.h>
48 #include <sys/mutex.h>
49 #include <sys/sx.h>
50 #include <sys/selinfo.h>
51 #include <sys/poll.h>
52 #include <sys/conf.h>
53 #include <sys/fcntl.h>
54 #include <sys/ioccom.h>
55 #include <sys/rman.h>
56 #include <sys/tree.h>
57 #include <sys/module.h>
58 #include <sys/filio.h>
59 #include <sys/vnode.h>
60
61 #include <machine/intr_machdep.h>
62 #include <machine/xen/synch_bitops.h>
63
64 #include <xen/xen-os.h>
65 #include <xen/evtchn.h>
66 #include <xen/xen_intr.h>
67
68 #include <xen/evtchn/evtchnvar.h>
69
70 MALLOC_DEFINE(M_EVTCHN, "evtchn_dev", "Xen event channel user-space device");
71
72 struct user_evtchn;
73
74 static int evtchn_cmp(struct user_evtchn *u1, struct user_evtchn *u2);
75
76 RB_HEAD(evtchn_tree, user_evtchn);
77
78 struct per_user_data {
79 struct mtx bind_mutex; /* serialize bind/unbind operations */
80 struct evtchn_tree evtchns;
81
82 /* Notification ring, accessed via /dev/xen/evtchn. */
83 #define EVTCHN_RING_SIZE (PAGE_SIZE / sizeof(evtchn_port_t))
84 #define EVTCHN_RING_MASK(_i) ((_i)&(EVTCHN_RING_SIZE-1))
85 evtchn_port_t *ring;
86 unsigned int ring_cons, ring_prod, ring_overflow;
87 struct sx ring_cons_mutex; /* protect against concurrent readers */
88 struct mtx ring_prod_mutex; /* product against concurrent interrupts */
89 struct selinfo ev_rsel;
90 };
91
92 struct user_evtchn {
93 RB_ENTRY(user_evtchn) node;
94 struct per_user_data *user;
95 evtchn_port_t port;
96 xen_intr_handle_t handle;
97 bool enabled;
98 };
99
100 RB_GENERATE_STATIC(evtchn_tree, user_evtchn, node, evtchn_cmp);
101
102 static device_t evtchn_dev;
103
104 static d_read_t evtchn_read;
105 static d_write_t evtchn_write;
106 static d_ioctl_t evtchn_ioctl;
107 static d_poll_t evtchn_poll;
108 static d_open_t evtchn_open;
109
110 static void evtchn_release(void *arg);
111
112 static struct cdevsw evtchn_devsw = {
113 .d_version = D_VERSION,
114 .d_open = evtchn_open,
115 .d_read = evtchn_read,
116 .d_write = evtchn_write,
117 .d_ioctl = evtchn_ioctl,
118 .d_poll = evtchn_poll,
119 .d_name = "evtchn",
120 };
121
122 /*------------------------- Red-black tree helpers ---------------------------*/
123 static int
evtchn_cmp(struct user_evtchn * u1,struct user_evtchn * u2)124 evtchn_cmp(struct user_evtchn *u1, struct user_evtchn *u2)
125 {
126
127 return (u1->port - u2->port);
128 }
129
130 static struct user_evtchn *
find_evtchn(struct per_user_data * u,evtchn_port_t port)131 find_evtchn(struct per_user_data *u, evtchn_port_t port)
132 {
133 struct user_evtchn tmp = {
134 .port = port,
135 };
136
137 return (RB_FIND(evtchn_tree, &u->evtchns, &tmp));
138 }
139
140 /*--------------------------- Interrupt handlers -----------------------------*/
141 static int
evtchn_filter(void * arg)142 evtchn_filter(void *arg)
143 {
144 struct user_evtchn *evtchn;
145
146 evtchn = arg;
147
148 if (!evtchn->enabled && bootverbose) {
149 device_printf(evtchn_dev,
150 "Received upcall for disabled event channel %d\n",
151 evtchn->port);
152 }
153
154 evtchn_mask_port(evtchn->port);
155 evtchn->enabled = false;
156
157 return (FILTER_SCHEDULE_THREAD);
158 }
159
160 static void
evtchn_interrupt(void * arg)161 evtchn_interrupt(void *arg)
162 {
163 struct user_evtchn *evtchn;
164 struct per_user_data *u;
165
166 evtchn = arg;
167 u = evtchn->user;
168
169 /*
170 * Protect against concurrent events using this handler
171 * on different CPUs.
172 */
173 mtx_lock(&u->ring_prod_mutex);
174 if ((u->ring_prod - u->ring_cons) < EVTCHN_RING_SIZE) {
175 u->ring[EVTCHN_RING_MASK(u->ring_prod)] = evtchn->port;
176 wmb(); /* Ensure ring contents visible */
177 if (u->ring_cons == u->ring_prod++) {
178 wakeup(u);
179 selwakeup(&u->ev_rsel);
180 }
181 } else
182 u->ring_overflow = 1;
183 mtx_unlock(&u->ring_prod_mutex);
184 }
185
186 /*------------------------- Character device methods -------------------------*/
187 static int
evtchn_open(struct cdev * dev,int flag,int otyp,struct thread * td)188 evtchn_open(struct cdev *dev, int flag, int otyp, struct thread *td)
189 {
190 struct per_user_data *u;
191 int error;
192
193 u = malloc(sizeof(*u), M_EVTCHN, M_WAITOK | M_ZERO);
194 u->ring = malloc(PAGE_SIZE, M_EVTCHN, M_WAITOK | M_ZERO);
195
196 /* Initialize locks */
197 mtx_init(&u->bind_mutex, "evtchn_bind_mutex", NULL, MTX_DEF);
198 sx_init(&u->ring_cons_mutex, "evtchn_ringc_sx");
199 mtx_init(&u->ring_prod_mutex, "evtchn_ringp_mutex", NULL, MTX_DEF);
200
201 /* Initialize red-black tree. */
202 RB_INIT(&u->evtchns);
203
204 /* Assign the allocated per_user_data to this open instance. */
205 error = devfs_set_cdevpriv(u, evtchn_release);
206 if (error != 0) {
207 mtx_destroy(&u->bind_mutex);
208 mtx_destroy(&u->ring_prod_mutex);
209 sx_destroy(&u->ring_cons_mutex);
210 free(u->ring, M_EVTCHN);
211 free(u, M_EVTCHN);
212 }
213
214 return (error);
215 }
216
217 static void
evtchn_release(void * arg)218 evtchn_release(void *arg)
219 {
220 struct per_user_data *u;
221 struct user_evtchn *evtchn, *tmp;
222
223 u = arg;
224
225 seldrain(&u->ev_rsel);
226
227 RB_FOREACH_SAFE(evtchn, evtchn_tree, &u->evtchns, tmp) {
228 xen_intr_unbind(&evtchn->handle);
229
230 RB_REMOVE(evtchn_tree, &u->evtchns, evtchn);
231 free(evtchn, M_EVTCHN);
232 }
233
234 mtx_destroy(&u->bind_mutex);
235 mtx_destroy(&u->ring_prod_mutex);
236 sx_destroy(&u->ring_cons_mutex);
237 free(u->ring, M_EVTCHN);
238 free(u, M_EVTCHN);
239 }
240
241 static int
evtchn_read(struct cdev * dev,struct uio * uio,int ioflag)242 evtchn_read(struct cdev *dev, struct uio *uio, int ioflag)
243 {
244 int error, count;
245 unsigned int c, p, bytes1 = 0, bytes2 = 0;
246 struct per_user_data *u;
247
248 error = devfs_get_cdevpriv((void **)&u);
249 if (error != 0)
250 return (EINVAL);
251
252 /* Whole number of ports. */
253 count = uio->uio_resid;
254 count &= ~(sizeof(evtchn_port_t)-1);
255
256 if (count == 0)
257 return (0);
258
259 if (count > PAGE_SIZE)
260 count = PAGE_SIZE;
261
262 sx_xlock(&u->ring_cons_mutex);
263 for (;;) {
264 error = EFBIG;
265 if (u->ring_overflow)
266 goto unlock_out;
267
268 c = u->ring_cons;
269 p = u->ring_prod;
270 if (c != p)
271 break;
272
273 if (ioflag & IO_NDELAY) {
274 sx_xunlock(&u->ring_cons_mutex);
275 return (EWOULDBLOCK);
276 }
277
278 error = sx_sleep(u, &u->ring_cons_mutex, PCATCH, "evtchw", 0);
279 if ((error != 0) && (error != EWOULDBLOCK))
280 return (error);
281 }
282
283 /* Byte lengths of two chunks. Chunk split (if any) is at ring wrap. */
284 if (((c ^ p) & EVTCHN_RING_SIZE) != 0) {
285 bytes1 = (EVTCHN_RING_SIZE - EVTCHN_RING_MASK(c)) *
286 sizeof(evtchn_port_t);
287 bytes2 = EVTCHN_RING_MASK(p) * sizeof(evtchn_port_t);
288 } else {
289 bytes1 = (p - c) * sizeof(evtchn_port_t);
290 bytes2 = 0;
291 }
292
293 /* Truncate chunks according to caller's maximum byte count. */
294 if (bytes1 > count) {
295 bytes1 = count;
296 bytes2 = 0;
297 } else if ((bytes1 + bytes2) > count) {
298 bytes2 = count - bytes1;
299 }
300
301 error = EFAULT;
302 rmb(); /* Ensure that we see the port before we copy it. */
303
304 if (uiomove(&u->ring[EVTCHN_RING_MASK(c)], bytes1, uio) ||
305 ((bytes2 != 0) && uiomove(&u->ring[0], bytes2, uio)))
306 goto unlock_out;
307
308 u->ring_cons += (bytes1 + bytes2) / sizeof(evtchn_port_t);
309 error = 0;
310
311 unlock_out:
312 sx_xunlock(&u->ring_cons_mutex);
313 return (error);
314 }
315
316 static int
evtchn_write(struct cdev * dev,struct uio * uio,int ioflag)317 evtchn_write(struct cdev *dev, struct uio *uio, int ioflag)
318 {
319 int error, i, count;
320 evtchn_port_t *kbuf;
321 struct per_user_data *u;
322
323 error = devfs_get_cdevpriv((void **)&u);
324 if (error != 0)
325 return (EINVAL);
326
327 kbuf = malloc(PAGE_SIZE, M_EVTCHN, M_WAITOK);
328 if (kbuf == NULL)
329 return (ENOMEM);
330
331 count = uio->uio_resid;
332 /* Whole number of ports. */
333 count &= ~(sizeof(evtchn_port_t)-1);
334
335 error = 0;
336 if (count == 0)
337 goto out;
338
339 if (count > PAGE_SIZE)
340 count = PAGE_SIZE;
341
342 error = uiomove(kbuf, count, uio);
343 if (error != 0)
344 goto out;
345
346 mtx_lock(&u->bind_mutex);
347
348 for (i = 0; i < (count/sizeof(evtchn_port_t)); i++) {
349 evtchn_port_t port = kbuf[i];
350 struct user_evtchn *evtchn;
351
352 evtchn = find_evtchn(u, port);
353 if (evtchn && !evtchn->enabled) {
354 evtchn->enabled = true;
355 evtchn_unmask_port(evtchn->port);
356 }
357 }
358
359 mtx_unlock(&u->bind_mutex);
360 error = 0;
361
362 out:
363 free(kbuf, M_EVTCHN);
364 return (error);
365 }
366
367 static inline int
evtchn_bind_user_port(struct per_user_data * u,struct user_evtchn * evtchn)368 evtchn_bind_user_port(struct per_user_data *u, struct user_evtchn *evtchn)
369 {
370 int error;
371
372 evtchn->port = xen_intr_port(evtchn->handle);
373 evtchn->user = u;
374 evtchn->enabled = true;
375 mtx_lock(&u->bind_mutex);
376 RB_INSERT(evtchn_tree, &u->evtchns, evtchn);
377 mtx_unlock(&u->bind_mutex);
378 error = xen_intr_add_handler(evtchn_dev, evtchn_filter,
379 evtchn_interrupt, evtchn, INTR_TYPE_MISC | INTR_MPSAFE,
380 evtchn->handle);
381 if (error != 0) {
382 xen_intr_unbind(&evtchn->handle);
383 mtx_lock(&u->bind_mutex);
384 RB_REMOVE(evtchn_tree, &u->evtchns, evtchn);
385 mtx_unlock(&u->bind_mutex);
386 free(evtchn, M_EVTCHN);
387 }
388 return (error);
389 }
390
391 static int
evtchn_ioctl(struct cdev * dev,unsigned long cmd,caddr_t arg,int mode,struct thread * td __unused)392 evtchn_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg,
393 int mode, struct thread *td __unused)
394 {
395 struct per_user_data *u;
396 int error;
397
398 error = devfs_get_cdevpriv((void **)&u);
399 if (error != 0)
400 return (EINVAL);
401
402 switch (cmd) {
403 case IOCTL_EVTCHN_BIND_VIRQ: {
404 struct ioctl_evtchn_bind_virq *bind;
405 struct user_evtchn *evtchn;
406
407 evtchn = malloc(sizeof(*evtchn), M_EVTCHN, M_WAITOK | M_ZERO);
408
409 bind = (struct ioctl_evtchn_bind_virq *)arg;
410
411 error = xen_intr_bind_virq(evtchn_dev, bind->virq, 0,
412 NULL, NULL, NULL, 0, &evtchn->handle);
413 if (error != 0) {
414 free(evtchn, M_EVTCHN);
415 break;
416 }
417 error = evtchn_bind_user_port(u, evtchn);
418 if (error != 0)
419 break;
420 bind->port = evtchn->port;
421 break;
422 }
423
424 case IOCTL_EVTCHN_BIND_INTERDOMAIN: {
425 struct ioctl_evtchn_bind_interdomain *bind;
426 struct user_evtchn *evtchn;
427
428 evtchn = malloc(sizeof(*evtchn), M_EVTCHN, M_WAITOK | M_ZERO);
429
430 bind = (struct ioctl_evtchn_bind_interdomain *)arg;
431
432 error = xen_intr_bind_remote_port(evtchn_dev,
433 bind->remote_domain, bind->remote_port, NULL,
434 NULL, NULL, 0, &evtchn->handle);
435 if (error != 0) {
436 free(evtchn, M_EVTCHN);
437 break;
438 }
439 error = evtchn_bind_user_port(u, evtchn);
440 if (error != 0)
441 break;
442 bind->port = evtchn->port;
443 break;
444 }
445
446 case IOCTL_EVTCHN_BIND_UNBOUND_PORT: {
447 struct ioctl_evtchn_bind_unbound_port *bind;
448 struct user_evtchn *evtchn;
449
450 evtchn = malloc(sizeof(*evtchn), M_EVTCHN, M_WAITOK | M_ZERO);
451
452 bind = (struct ioctl_evtchn_bind_unbound_port *)arg;
453
454 error = xen_intr_alloc_and_bind_local_port(evtchn_dev,
455 bind->remote_domain, NULL, NULL, NULL, 0, &evtchn->handle);
456 if (error != 0) {
457 free(evtchn, M_EVTCHN);
458 break;
459 }
460 error = evtchn_bind_user_port(u, evtchn);
461 if (error != 0)
462 break;
463 bind->port = evtchn->port;
464 break;
465 }
466
467 case IOCTL_EVTCHN_UNBIND: {
468 struct ioctl_evtchn_unbind *unbind;
469 struct user_evtchn *evtchn;
470
471 unbind = (struct ioctl_evtchn_unbind *)arg;
472
473 mtx_lock(&u->bind_mutex);
474 evtchn = find_evtchn(u, unbind->port);
475 if (evtchn == NULL) {
476 error = ENOTCONN;
477 break;
478 }
479
480 xen_intr_unbind(&evtchn->handle);
481 RB_REMOVE(evtchn_tree, &u->evtchns, evtchn);
482 mtx_unlock(&u->bind_mutex);
483 free(evtchn, M_EVTCHN);
484 error = 0;
485 break;
486 }
487
488 case IOCTL_EVTCHN_NOTIFY: {
489 struct ioctl_evtchn_notify *notify;
490 struct user_evtchn *evtchn;
491
492 notify = (struct ioctl_evtchn_notify *)arg;
493
494 mtx_lock(&u->bind_mutex);
495 evtchn = find_evtchn(u, notify->port);
496 if (evtchn == NULL) {
497 error = ENOTCONN;
498 break;
499 }
500
501 xen_intr_signal(evtchn->handle);
502 mtx_unlock(&u->bind_mutex);
503 error = 0;
504 break;
505 }
506
507 case IOCTL_EVTCHN_RESET: {
508 /* Initialise the ring to empty. Clear errors. */
509 sx_xlock(&u->ring_cons_mutex);
510 mtx_lock(&u->ring_prod_mutex);
511 u->ring_cons = u->ring_prod = u->ring_overflow = 0;
512 mtx_unlock(&u->ring_prod_mutex);
513 sx_xunlock(&u->ring_cons_mutex);
514 error = 0;
515 break;
516 }
517
518 case FIONBIO:
519 case FIOASYNC:
520 /* Handled in an upper layer */
521 error = 0;
522 break;
523
524 default:
525 error = ENOTTY;
526 break;
527 }
528
529 return (error);
530 }
531
532 static int
evtchn_poll(struct cdev * dev,int events,struct thread * td)533 evtchn_poll(struct cdev *dev, int events, struct thread *td)
534 {
535 struct per_user_data *u;
536 int error, mask;
537
538 error = devfs_get_cdevpriv((void **)&u);
539 if (error != 0)
540 return (POLLERR);
541
542 /* we can always write */
543 mask = events & (POLLOUT | POLLWRNORM);
544
545 mtx_lock(&u->ring_prod_mutex);
546 if (events & (POLLIN | POLLRDNORM)) {
547 if (u->ring_cons != u->ring_prod) {
548 mask |= events & (POLLIN | POLLRDNORM);
549 } else {
550 /* Record that someone is waiting */
551 selrecord(td, &u->ev_rsel);
552 }
553 }
554 mtx_unlock(&u->ring_prod_mutex);
555
556 return (mask);
557 }
558
559 /*------------------ Private Device Attachment Functions --------------------*/
560 static void
evtchn_identify(driver_t * driver,device_t parent)561 evtchn_identify(driver_t *driver, device_t parent)
562 {
563
564 KASSERT((xen_domain()),
565 ("Trying to attach evtchn device on non Xen domain"));
566
567 evtchn_dev = BUS_ADD_CHILD(parent, 0, "evtchn", 0);
568 if (evtchn_dev == NULL)
569 panic("unable to attach evtchn user-space device");
570 }
571
572 static int
evtchn_probe(device_t dev)573 evtchn_probe(device_t dev)
574 {
575
576 device_set_desc(dev, "Xen event channel user-space device");
577 return (BUS_PROBE_NOWILDCARD);
578 }
579
580 static int
evtchn_attach(device_t dev)581 evtchn_attach(device_t dev)
582 {
583
584 make_dev_credf(MAKEDEV_ETERNAL, &evtchn_devsw, 0, NULL, UID_ROOT,
585 GID_WHEEL, 0600, "xen/evtchn");
586 return (0);
587 }
588
589 /*-------------------- Private Device Attachment Data -----------------------*/
590 static device_method_t evtchn_methods[] = {
591 DEVMETHOD(device_identify, evtchn_identify),
592 DEVMETHOD(device_probe, evtchn_probe),
593 DEVMETHOD(device_attach, evtchn_attach),
594
595 DEVMETHOD_END
596 };
597
598 static driver_t evtchn_driver = {
599 "evtchn",
600 evtchn_methods,
601 0,
602 };
603
604 devclass_t evtchn_devclass;
605
606 DRIVER_MODULE(evtchn, xenpv, evtchn_driver, evtchn_devclass, 0, 0);
607 MODULE_DEPEND(evtchn, xenpv, 1, 1, 1);
608