1 /*-
2 * Copyright (c) 2018 VMware, Inc.
3 *
4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
5 */
6
7 /* This file implements the VMCI doorbell API. */
8
9 #include <sys/cdefs.h>
10 __FBSDID("$FreeBSD: stable/12/sys/dev/vmware/vmci/vmci_doorbell.c 332263 2018-04-08 01:32:56Z mp $");
11
12 #include <sys/types.h>
13
14 #include "vmci_doorbell.h"
15 #include "vmci_driver.h"
16 #include "vmci_kernel_api.h"
17 #include "vmci_kernel_defs.h"
18 #include "vmci_resource.h"
19 #include "vmci_utils.h"
20
21 #define LGPFX "vmci_doorbell: "
22
23 #define VMCI_DOORBELL_INDEX_TABLE_SIZE 64
24 #define VMCI_DOORBELL_HASH(_idx) \
25 vmci_hash_id((_idx), VMCI_DOORBELL_INDEX_TABLE_SIZE)
26
27 /* Describes a doorbell notification handle allocated by the host. */
28 struct vmci_doorbell_entry {
29 struct vmci_resource resource;
30 uint32_t idx;
31 vmci_list_item(vmci_doorbell_entry) idx_list_item;
32 vmci_privilege_flags priv_flags;
33 bool is_doorbell;
34 bool run_delayed;
35 vmci_callback notify_cb;
36 void *client_data;
37 vmci_event destroy_event;
38 volatile int active;
39 };
40
41 struct vmci_doorbell_index_table {
42 vmci_lock lock;
43 vmci_list(vmci_doorbell_entry) entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
44 };
45
46 /* The VMCI index table keeps track of currently registered doorbells. */
47 static struct vmci_doorbell_index_table vmci_doorbell_it;
48
49 /*
50 * The max_notify_idx is one larger than the currently known bitmap index in
51 * use, and is used to determine how much of the bitmap needs to be scanned.
52 */
53 static uint32_t max_notify_idx;
54
55 /*
56 * The notify_idx_count is used for determining whether there are free entries
57 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
58 */
59 static uint32_t notify_idx_count;
60
61 /*
62 * The last_notify_idx_reserved is used to track the last index handed out - in
63 * the case where multiple handles share a notification index, we hand out
64 * indexes round robin based on last_notify_idx_reserved.
65 */
66 static uint32_t last_notify_idx_reserved;
67
68 /* This is a one entry cache used to by the index allocation. */
69 static uint32_t last_notify_idx_released = PAGE_SIZE;
70
71 static void vmci_doorbell_free_cb(void *client_data);
72 static int vmci_doorbell_release_cb(void *client_data);
73 static void vmci_doorbell_delayed_dispatch_cb(void *data);
74
75 /*
76 *------------------------------------------------------------------------------
77 *
78 * vmci_doorbell_init --
79 *
80 * General init code.
81 *
82 * Result:
83 * VMCI_SUCCESS on success, lock allocation error otherwise.
84 *
85 * Side effects:
86 * None.
87 *
88 *------------------------------------------------------------------------------
89 */
90
91 int
vmci_doorbell_init(void)92 vmci_doorbell_init(void)
93 {
94 uint32_t bucket;
95
96 for (bucket = 0; bucket < ARRAYSIZE(vmci_doorbell_it.entries);
97 ++bucket)
98 vmci_list_init(&vmci_doorbell_it.entries[bucket]);
99
100 return (vmci_init_lock(&vmci_doorbell_it.lock,
101 "VMCI Doorbell index table lock"));
102 }
103
104 /*
105 *------------------------------------------------------------------------------
106 *
107 * vmci_doorbell_exit --
108 *
109 * General exit code.
110 *
111 * Result:
112 * None.
113 *
114 * Side effects:
115 * None.
116 *
117 *------------------------------------------------------------------------------
118 */
119
120 void
vmci_doorbell_exit(void)121 vmci_doorbell_exit(void)
122 {
123
124 vmci_cleanup_lock(&vmci_doorbell_it.lock);
125 }
126
127 /*
128 *------------------------------------------------------------------------------
129 *
130 * vmci_doorbell_free_cb --
131 *
132 * Callback to free doorbell entry structure when resource is no longer used,
133 * i.e. the reference count reached 0. The entry is freed in
134 * vmci_doorbell_destroy(), which is waiting on the signal that gets fired
135 * here.
136 *
137 * Result:
138 * None.
139 *
140 * Side effects:
141 * Signals VMCI event.
142 *
143 *------------------------------------------------------------------------------
144 */
145
146 static void
vmci_doorbell_free_cb(void * client_data)147 vmci_doorbell_free_cb(void *client_data)
148 {
149 struct vmci_doorbell_entry *entry;
150
151 entry = (struct vmci_doorbell_entry *)client_data;
152 ASSERT(entry);
153 vmci_signal_event(&entry->destroy_event);
154 }
155
156 /*
157 *------------------------------------------------------------------------------
158 *
159 * vmci_doorbell_release_cb --
160 *
161 * Callback to release the resource reference. It is called by the
162 * vmci_wait_on_event function before it blocks.
163 *
164 * Result:
165 * Always 0.
166 *
167 * Side effects:
168 * None.
169 *
170 *------------------------------------------------------------------------------
171 */
172
173 static int
vmci_doorbell_release_cb(void * client_data)174 vmci_doorbell_release_cb(void *client_data)
175 {
176 struct vmci_doorbell_entry *entry;
177
178 entry = (struct vmci_doorbell_entry *)client_data;
179 ASSERT(entry);
180 vmci_resource_release(&entry->resource);
181 return (0);
182 }
183
184 /*
185 *------------------------------------------------------------------------------
186 *
187 * vmci_doorbell_get_priv_flags --
188 *
189 * Utility function that retrieves the privilege flags associated with a
190 * given doorbell handle. For guest endpoints, the privileges are determined
191 * by the context ID, but for host endpoints privileges are associated with
192 * the complete handle. Hypervisor endpoints are not yet supported.
193 *
194 * Result:
195 * VMCI_SUCCESS on success,
196 * VMCI_ERROR_NOT_FOUND if handle isn't found,
197 * VMCI_ERROR_INVALID_ARGS if handle is invalid.
198 *
199 * Side effects:
200 * None.
201 *
202 *------------------------------------------------------------------------------
203 */
204
205 int
vmci_doorbell_get_priv_flags(struct vmci_handle handle,vmci_privilege_flags * priv_flags)206 vmci_doorbell_get_priv_flags(struct vmci_handle handle,
207 vmci_privilege_flags *priv_flags)
208 {
209
210 if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
211 return (VMCI_ERROR_INVALID_ARGS);
212
213 if (handle.context == VMCI_HOST_CONTEXT_ID) {
214 struct vmci_doorbell_entry *entry;
215 struct vmci_resource *resource;
216
217 resource = vmci_resource_get(handle,
218 VMCI_RESOURCE_TYPE_DOORBELL);
219 if (resource == NULL)
220 return (VMCI_ERROR_NOT_FOUND);
221 entry = RESOURCE_CONTAINER(
222 resource, struct vmci_doorbell_entry, resource);
223 *priv_flags = entry->priv_flags;
224 vmci_resource_release(resource);
225 } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
226 /* Hypervisor endpoints for notifications are not supported. */
227 return (VMCI_ERROR_INVALID_ARGS);
228 } else
229 *priv_flags = VMCI_NO_PRIVILEGE_FLAGS;
230
231 return (VMCI_SUCCESS);
232 }
233
234 /*
235 *------------------------------------------------------------------------------
236 *
237 * vmci_doorbell_index_table_find --
238 *
239 * Find doorbell entry by bitmap index.
240 *
241 * Results:
242 * Entry if found, NULL if not.
243 *
244 * Side effects:
245 * None.
246 *
247 *------------------------------------------------------------------------------
248 */
249
250 static struct vmci_doorbell_entry *
vmci_doorbell_index_table_find(uint32_t idx)251 vmci_doorbell_index_table_find(uint32_t idx)
252 {
253 struct vmci_doorbell_entry *iter;
254 uint32_t bucket;
255
256 bucket = VMCI_DOORBELL_HASH(idx);
257
258 vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
259 if (idx == iter->idx)
260 return (iter);
261 }
262
263 return (NULL);
264 }
265
266 /*
267 *------------------------------------------------------------------------------
268 *
269 * vmci_doorbell_index_table_add --
270 *
271 * Add the given entry to the index table. This will hold() the entry's
272 * resource so that the entry is not deleted before it is removed from the
273 * table.
274 *
275 * Results:
276 * None.
277 *
278 * Side effects:
279 * None.
280 *
281 *------------------------------------------------------------------------------
282 */
283
284 static void
vmci_doorbell_index_table_add(struct vmci_doorbell_entry * entry)285 vmci_doorbell_index_table_add(struct vmci_doorbell_entry *entry)
286 {
287 uint32_t bucket;
288 uint32_t new_notify_idx;
289
290 ASSERT(entry);
291
292 vmci_resource_hold(&entry->resource);
293
294 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
295
296 /*
297 * Below we try to allocate an index in the notification bitmap with
298 * "not too much" sharing between resources. If we use less that the
299 * full bitmap, we either add to the end if there are no unused flags
300 * within the currently used area, or we search for unused ones. If we
301 * use the full bitmap, we allocate the index round robin.
302 */
303
304 if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
305 if (last_notify_idx_released < max_notify_idx &&
306 !vmci_doorbell_index_table_find(last_notify_idx_released)) {
307 new_notify_idx = last_notify_idx_released;
308 last_notify_idx_released = PAGE_SIZE;
309 } else {
310 bool reused = false;
311 new_notify_idx = last_notify_idx_reserved;
312 if (notify_idx_count + 1 < max_notify_idx) {
313 do {
314 if (!vmci_doorbell_index_table_find(
315 new_notify_idx)) {
316 reused = true;
317 break;
318 }
319 new_notify_idx = (new_notify_idx + 1) %
320 max_notify_idx;
321 } while (new_notify_idx !=
322 last_notify_idx_released);
323 }
324 if (!reused) {
325 new_notify_idx = max_notify_idx;
326 max_notify_idx++;
327 }
328 }
329 } else {
330 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
331 }
332 last_notify_idx_reserved = new_notify_idx;
333 notify_idx_count++;
334
335 entry->idx = new_notify_idx;
336 bucket = VMCI_DOORBELL_HASH(entry->idx);
337 vmci_list_insert(&vmci_doorbell_it.entries[bucket], entry,
338 idx_list_item);
339
340 vmci_release_lock_bh(&vmci_doorbell_it.lock);
341 }
342
343 /*
344 *------------------------------------------------------------------------------
345 *
346 * vmci_doorbell_index_table_remove --
347 *
348 * Remove the given entry from the index table. This will release() the
349 * entry's resource.
350 *
351 * Results:
352 * None.
353 *
354 * Side effects:
355 * None.
356 *
357 *------------------------------------------------------------------------------
358 */
359
360 static void
vmci_doorbell_index_table_remove(struct vmci_doorbell_entry * entry)361 vmci_doorbell_index_table_remove(struct vmci_doorbell_entry *entry)
362 {
363 ASSERT(entry);
364
365 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
366
367 vmci_list_remove(entry, idx_list_item);
368
369 notify_idx_count--;
370 if (entry->idx == max_notify_idx - 1) {
371 /*
372 * If we delete an entry with the maximum known notification
373 * index, we take the opportunity to prune the current max. As
374 * there might be other unused indices immediately below, we
375 * lower the maximum until we hit an index in use
376 */
377
378 while (max_notify_idx > 0 &&
379 !vmci_doorbell_index_table_find(max_notify_idx - 1))
380 max_notify_idx--;
381 }
382 last_notify_idx_released = entry->idx;
383
384 vmci_release_lock_bh(&vmci_doorbell_it.lock);
385
386 vmci_resource_release(&entry->resource);
387 }
388
389 /*
390 *------------------------------------------------------------------------------
391 *
392 * vmci_doorbell_link --
393 *
394 * Creates a link between the given doorbell handle and the given index in
395 * the bitmap in the device backend.
396 *
397 * Results:
398 * VMCI_SUCCESS if success, appropriate error code otherwise.
399 *
400 * Side effects:
401 * Notification state is created in hypervisor.
402 *
403 *------------------------------------------------------------------------------
404 */
405
406 static int
vmci_doorbell_link(struct vmci_handle handle,bool is_doorbell,uint32_t notify_idx)407 vmci_doorbell_link(struct vmci_handle handle, bool is_doorbell,
408 uint32_t notify_idx)
409 {
410 struct vmci_doorbell_link_msg link_msg;
411 vmci_id resource_id;
412
413 ASSERT(!VMCI_HANDLE_INVALID(handle));
414
415 if (is_doorbell)
416 resource_id = VMCI_DOORBELL_LINK;
417 else {
418 ASSERT(false);
419 return (VMCI_ERROR_UNAVAILABLE);
420 }
421
422 link_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
423 resource_id);
424 link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
425 link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
426 link_msg.handle = handle;
427 link_msg.notify_idx = notify_idx;
428
429 return (vmci_send_datagram((struct vmci_datagram *)&link_msg));
430 }
431
432 /*
433 *------------------------------------------------------------------------------
434 *
435 * vmci_doorbell_unlink --
436 *
437 * Unlinks the given doorbell handle from an index in the bitmap in the
438 * device backend.
439 *
440 * Results:
441 * VMCI_SUCCESS if success, appropriate error code otherwise.
442 *
443 * Side effects:
444 * Notification state is destroyed in hypervisor.
445 *
446 *------------------------------------------------------------------------------
447 */
448
449 static int
vmci_doorbell_unlink(struct vmci_handle handle,bool is_doorbell)450 vmci_doorbell_unlink(struct vmci_handle handle, bool is_doorbell)
451 {
452 struct vmci_doorbell_unlink_msg unlink_msg;
453 vmci_id resource_id;
454
455 ASSERT(!VMCI_HANDLE_INVALID(handle));
456
457 if (is_doorbell)
458 resource_id = VMCI_DOORBELL_UNLINK;
459 else {
460 ASSERT(false);
461 return (VMCI_ERROR_UNAVAILABLE);
462 }
463
464 unlink_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
465 resource_id);
466 unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
467 unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
468 unlink_msg.handle = handle;
469
470 return (vmci_send_datagram((struct vmci_datagram *)&unlink_msg));
471 }
472
473 /*
474 *------------------------------------------------------------------------------
475 *
476 * vmci_doorbell_create --
477 *
478 * Creates a doorbell with the given callback. If the handle is
479 * VMCI_INVALID_HANDLE, a free handle will be assigned, if possible. The
480 * callback can be run immediately (potentially with locks held - the
481 * default) or delayed (in a kernel thread) by specifying the flag
482 * VMCI_FLAG_DELAYED_CB. If delayed execution is selected, a given callback
483 * may not be run if the kernel is unable to allocate memory for the delayed
484 * execution (highly unlikely).
485 *
486 * Results:
487 * VMCI_SUCCESS on success, appropriate error code otherwise.
488 *
489 * Side effects:
490 * None.
491 *
492 *------------------------------------------------------------------------------
493 */
494
495 int
vmci_doorbell_create(struct vmci_handle * handle,uint32_t flags,vmci_privilege_flags priv_flags,vmci_callback notify_cb,void * client_data)496 vmci_doorbell_create(struct vmci_handle *handle, uint32_t flags,
497 vmci_privilege_flags priv_flags, vmci_callback notify_cb, void *client_data)
498 {
499 struct vmci_doorbell_entry *entry;
500 struct vmci_handle new_handle;
501 int result;
502
503 if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
504 priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
505 return (VMCI_ERROR_INVALID_ARGS);
506
507 entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL);
508 if (entry == NULL) {
509 VMCI_LOG_WARNING(LGPFX"Failed allocating memory for datagram "
510 "entry.\n");
511 return (VMCI_ERROR_NO_MEM);
512 }
513
514 if (!vmci_can_schedule_delayed_work() &&
515 (flags & VMCI_FLAG_DELAYED_CB)) {
516 result = VMCI_ERROR_INVALID_ARGS;
517 goto free_mem;
518 }
519
520 if (VMCI_HANDLE_INVALID(*handle)) {
521 vmci_id context_id;
522
523 context_id = vmci_get_context_id();
524 vmci_id resource_id = vmci_resource_get_id(context_id);
525 if (resource_id == VMCI_INVALID_ID) {
526 result = VMCI_ERROR_NO_HANDLE;
527 goto free_mem;
528 }
529 new_handle = VMCI_MAKE_HANDLE(context_id, resource_id);
530 } else {
531 if (VMCI_INVALID_ID == handle->resource) {
532 VMCI_LOG_DEBUG(LGPFX"Invalid argument "
533 "(handle=0x%x:0x%x).\n", handle->context,
534 handle->resource);
535 result = VMCI_ERROR_INVALID_ARGS;
536 goto free_mem;
537 }
538 new_handle = *handle;
539 }
540
541 entry->idx = 0;
542 entry->priv_flags = priv_flags;
543 entry->is_doorbell = true;
544 entry->run_delayed = (flags & VMCI_FLAG_DELAYED_CB) ? true : false;
545 entry->notify_cb = notify_cb;
546 entry->client_data = client_data;
547 atomic_store_int(&entry->active, 0);
548 vmci_create_event(&entry->destroy_event);
549
550 result = vmci_resource_add(&entry->resource,
551 VMCI_RESOURCE_TYPE_DOORBELL, new_handle, vmci_doorbell_free_cb,
552 entry);
553 if (result != VMCI_SUCCESS) {
554 VMCI_LOG_WARNING(LGPFX"Failed to add new resource "
555 "(handle=0x%x:0x%x).\n", new_handle.context,
556 new_handle.resource);
557 if (result == VMCI_ERROR_DUPLICATE_ENTRY)
558 result = VMCI_ERROR_ALREADY_EXISTS;
559
560 goto destroy;
561 }
562
563 vmci_doorbell_index_table_add(entry);
564 result = vmci_doorbell_link(new_handle, entry->is_doorbell, entry->idx);
565 if (VMCI_SUCCESS != result)
566 goto destroy_resource;
567 atomic_store_int(&entry->active, 1);
568
569 if (VMCI_HANDLE_INVALID(*handle))
570 *handle = new_handle;
571
572 return (result);
573
574 destroy_resource:
575 vmci_doorbell_index_table_remove(entry);
576 vmci_resource_remove(new_handle, VMCI_RESOURCE_TYPE_DOORBELL);
577 destroy:
578 vmci_destroy_event(&entry->destroy_event);
579 free_mem:
580 vmci_free_kernel_mem(entry, sizeof(*entry));
581 return (result);
582 }
583
584 /*
585 *------------------------------------------------------------------------------
586 *
587 * vmci_doorbell_destroy --
588 *
589 * Destroys a doorbell previously created with vmci_doorbell_create. This
590 * operation may block waiting for a callback to finish.
591 *
592 * Results:
593 * VMCI_SUCCESS on success, appropriate error code otherwise.
594 *
595 * Side effects:
596 * May block.
597 *
598 *------------------------------------------------------------------------------
599 */
600
601 int
vmci_doorbell_destroy(struct vmci_handle handle)602 vmci_doorbell_destroy(struct vmci_handle handle)
603 {
604 struct vmci_doorbell_entry *entry;
605 struct vmci_resource *resource;
606 int result;
607
608 if (VMCI_HANDLE_INVALID(handle))
609 return (VMCI_ERROR_INVALID_ARGS);
610
611 resource = vmci_resource_get(handle, VMCI_RESOURCE_TYPE_DOORBELL);
612 if (resource == NULL) {
613 VMCI_LOG_DEBUG(LGPFX"Failed to destroy doorbell "
614 "(handle=0x%x:0x%x).\n", handle.context, handle.resource);
615 return (VMCI_ERROR_NOT_FOUND);
616 }
617 entry = RESOURCE_CONTAINER(resource, struct vmci_doorbell_entry,
618 resource);
619
620 vmci_doorbell_index_table_remove(entry);
621
622 result = vmci_doorbell_unlink(handle, entry->is_doorbell);
623 if (VMCI_SUCCESS != result) {
624
625 /*
626 * The only reason this should fail would be an inconsistency
627 * between guest and hypervisor state, where the guest believes
628 * it has an active registration whereas the hypervisor doesn't.
629 * One case where this may happen is if a doorbell is
630 * unregistered following a hibernation at a time where the
631 * doorbell state hasn't been restored on the hypervisor side
632 * yet. Since the handle has now been removed in the guest,
633 * we just print a warning and return success.
634 */
635
636 VMCI_LOG_DEBUG(LGPFX"Unlink of %s (handle=0x%x:0x%x) unknown "
637 "by hypervisor (error=%d).\n",
638 entry->is_doorbell ? "doorbell" : "queuepair",
639 handle.context, handle.resource, result);
640 }
641
642 /*
643 * Now remove the resource from the table. It might still be in use
644 * after this, in a callback or still on the delayed work queue.
645 */
646
647 vmci_resource_remove(handle, VMCI_RESOURCE_TYPE_DOORBELL);
648
649 /*
650 * We now wait on the destroyEvent and release the reference we got
651 * above.
652 */
653
654 vmci_wait_on_event(&entry->destroy_event, vmci_doorbell_release_cb,
655 entry);
656
657 /*
658 * We know that we are now the only reference to the above entry so
659 * can safely free it.
660 */
661
662 vmci_destroy_event(&entry->destroy_event);
663 vmci_free_kernel_mem(entry, sizeof(*entry));
664
665 return (VMCI_SUCCESS);
666 }
667
668 /*
669 *------------------------------------------------------------------------------
670 *
671 * vmci_doorbell_notify_as_guest --
672 *
673 * Notify another guest or the host. We send a datagram down to the host
674 * via the hypervisor with the notification info.
675 *
676 * Results:
677 * VMCI_SUCCESS on success, appropriate error code otherwise.
678 *
679 * Side effects:
680 * May do a hypercall.
681 *
682 *------------------------------------------------------------------------------
683 */
684
685 static int
vmci_doorbell_notify_as_guest(struct vmci_handle handle,vmci_privilege_flags priv_flags)686 vmci_doorbell_notify_as_guest(struct vmci_handle handle,
687 vmci_privilege_flags priv_flags)
688 {
689 struct vmci_doorbell_notify_msg notify_msg;
690
691 notify_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
692 VMCI_DOORBELL_NOTIFY);
693 notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
694 notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
695 notify_msg.handle = handle;
696
697 return (vmci_send_datagram((struct vmci_datagram *)¬ify_msg));
698 }
699
700 /*
701 *------------------------------------------------------------------------------
702 *
703 * vmci_doorbell_notify --
704 *
705 * Generates a notification on the doorbell identified by the handle. For
706 * host side generation of notifications, the caller can specify what the
707 * privilege of the calling side is.
708 *
709 * Results:
710 * VMCI_SUCCESS on success, appropriate error code otherwise.
711 *
712 * Side effects:
713 * May do a hypercall.
714 *
715 *------------------------------------------------------------------------------
716 */
717
718 int
vmci_doorbell_notify(struct vmci_handle dst,vmci_privilege_flags priv_flags)719 vmci_doorbell_notify(struct vmci_handle dst, vmci_privilege_flags priv_flags)
720 {
721 struct vmci_handle src;
722
723 if (VMCI_HANDLE_INVALID(dst) ||
724 (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
725 return (VMCI_ERROR_INVALID_ARGS);
726
727 src = VMCI_INVALID_HANDLE;
728
729 return (vmci_doorbell_notify_as_guest(dst, priv_flags));
730 }
731
732 /*
733 *------------------------------------------------------------------------------
734 *
735 * vmci_doorbell_delayed_dispatch_cb --
736 *
737 * Calls the specified callback in a delayed context.
738 *
739 * Results:
740 * None.
741 *
742 * Side effects:
743 * None.
744 *
745 *------------------------------------------------------------------------------
746 */
747
748 static void
vmci_doorbell_delayed_dispatch_cb(void * data)749 vmci_doorbell_delayed_dispatch_cb(void *data)
750 {
751 struct vmci_doorbell_entry *entry = (struct vmci_doorbell_entry *)data;
752
753 ASSERT(data);
754
755 entry->notify_cb(entry->client_data);
756
757 vmci_resource_release(&entry->resource);
758 }
759
760 /*
761 *------------------------------------------------------------------------------
762 *
763 * vmci_doorbell_sync --
764 *
765 * Use this as a synchronization point when setting globals, for example,
766 * during device shutdown.
767 *
768 * Results:
769 * None.
770 *
771 * Side effects:
772 * None.
773 *
774 *------------------------------------------------------------------------------
775 */
776
777 void
vmci_doorbell_sync(void)778 vmci_doorbell_sync(void)
779 {
780
781 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
782 vmci_release_lock_bh(&vmci_doorbell_it.lock);
783 vmci_resource_sync();
784 }
785
786 /*
787 *------------------------------------------------------------------------------
788 *
789 * vmci_register_notification_bitmap --
790 *
791 * Register the notification bitmap with the host.
792 *
793 * Results:
794 * true if the bitmap is registered successfully with the device, false
795 * otherwise.
796 *
797 * Side effects:
798 * None.
799 *
800 *------------------------------------------------------------------------------
801 */
802
803 bool
vmci_register_notification_bitmap(PPN bitmap_ppn)804 vmci_register_notification_bitmap(PPN bitmap_ppn)
805 {
806 struct vmci_notify_bitmap_set_msg bitmap_set_msg;
807 int result;
808
809 /*
810 * Do not ASSERT() on the guest device here. This function can get
811 * called during device initialization, so the ASSERT() will fail even
812 * though the device is (almost) up.
813 */
814
815 bitmap_set_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
816 VMCI_SET_NOTIFY_BITMAP);
817 bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
818 bitmap_set_msg.hdr.payload_size =
819 sizeof(bitmap_set_msg) - VMCI_DG_HEADERSIZE;
820 bitmap_set_msg.bitmap_ppn = bitmap_ppn;
821
822 result = vmci_send_datagram((struct vmci_datagram *)&bitmap_set_msg);
823 if (result != VMCI_SUCCESS) {
824 VMCI_LOG_DEBUG(LGPFX"Failed to register (PPN=%u) as "
825 "notification bitmap (error=%d).\n",
826 bitmap_ppn, result);
827 return (false);
828 }
829 return (true);
830 }
831
832 /*
833 *------------------------------------------------------------------------------
834 *
835 * vmci_doorbell_fire_entries --
836 *
837 * Executes or schedules the handlers for a given notify index.
838 *
839 * Result:
840 * Notification hash entry if found. NULL otherwise.
841 *
842 * Side effects:
843 * Whatever the side effects of the handlers are.
844 *
845 *------------------------------------------------------------------------------
846 */
847
848 static void
vmci_doorbell_fire_entries(uint32_t notify_idx)849 vmci_doorbell_fire_entries(uint32_t notify_idx)
850 {
851 struct vmci_doorbell_entry *iter;
852 uint32_t bucket = VMCI_DOORBELL_HASH(notify_idx);
853
854 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
855
856 vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
857 if (iter->idx == notify_idx &&
858 atomic_load_int(&iter->active) == 1) {
859 ASSERT(iter->notify_cb);
860 if (iter->run_delayed) {
861 int err;
862
863 vmci_resource_hold(&iter->resource);
864 err = vmci_schedule_delayed_work(
865 vmci_doorbell_delayed_dispatch_cb, iter);
866 if (err != VMCI_SUCCESS) {
867 vmci_resource_release(&iter->resource);
868 goto out;
869 }
870 } else
871 iter->notify_cb(iter->client_data);
872 }
873 }
874
875 out:
876 vmci_release_lock_bh(&vmci_doorbell_it.lock);
877 }
878
879 /*
880 *------------------------------------------------------------------------------
881 *
882 * vmci_scan_notification_bitmap --
883 *
884 * Scans the notification bitmap, collects pending notifications, resets
885 * the bitmap and invokes appropriate callbacks.
886 *
887 * Results:
888 * None.
889 *
890 * Side effects:
891 * May schedule tasks, allocate memory and run callbacks.
892 *
893 *------------------------------------------------------------------------------
894 */
895
896 void
vmci_scan_notification_bitmap(uint8_t * bitmap)897 vmci_scan_notification_bitmap(uint8_t *bitmap)
898 {
899 uint32_t idx;
900
901 ASSERT(bitmap);
902
903 for (idx = 0; idx < max_notify_idx; idx++) {
904 if (bitmap[idx] & 0x1) {
905 bitmap[idx] &= ~1;
906 vmci_doorbell_fire_entries(idx);
907 }
908 }
909 }
910