1 /*-
2 * Copyright (c) 2009-2012,2016 Microsoft Corp.
3 * Copyright (c) 2012 NetApp Inc.
4 * Copyright (c) 2012 Citrix Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include "opt_acpi.h"
30
31 #include <sys/param.h>
32 #include <sys/bus.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/module.h>
36 #include <sys/rman.h>
37 #include <sys/systimer.h>
38 #include <sys/thread.h>
39 #include <sys/thread2.h>
40
41 #include <machine/intr_machdep.h>
42 #include <machine/smp.h>
43
44 #include <dev/virtual/hyperv/hyperv_busdma.h>
45 #include <dev/virtual/hyperv/hyperv_machdep.h>
46 #include <dev/virtual/hyperv/hyperv_reg.h>
47 #include <dev/virtual/hyperv/hyperv_var.h>
48 #include <dev/virtual/hyperv/vmbus/vmbus_reg.h>
49 #include <dev/virtual/hyperv/vmbus/vmbus_var.h>
50
51 #include "acpi.h"
52 #include "acpi_if.h"
53 #include "pcib_if.h"
54
55 #define MSR_HV_STIMER0_CFG_SINT \
56 ((((uint64_t)VMBUS_SINT_TIMER) << MSR_HV_STIMER_CFG_SINT_SHIFT) & \
57 MSR_HV_STIMER_CFG_SINT_MASK)
58
59 /*
60 * Additionally required feature:
61 * - SynIC is needed for interrupt generation.
62 */
63 #define CPUID_HV_TIMER_MASK (CPUID_HV_MSR_SYNIC | \
64 CPUID_HV_MSR_SYNTIMER)
65
66 /*
67 * NOTE: DO NOT CHANGE THIS.
68 */
69 #define VMBUS_SINT_MESSAGE 2
70 /*
71 * NOTE:
72 * - DO NOT set it to the same value as VMBUS_SINT_MESSAGE.
73 * - DO NOT set it to 0.
74 */
75 #define VMBUS_SINT_TIMER 4
76
77 /*
78 * NOTE: DO NOT CHANGE THESE
79 */
80 #define VMBUS_CONNID_MESSAGE 1
81 #define VMBUS_CONNID_EVENT 2
82
83 struct vmbus_msghc {
84 struct hypercall_postmsg_in *mh_inprm;
85 struct hypercall_postmsg_in mh_inprm_save;
86 struct hyperv_dma mh_inprm_dma;
87
88 struct vmbus_message *mh_resp;
89 struct vmbus_message mh_resp0;
90 };
91
92 struct vmbus_msghc_ctx {
93 struct vmbus_msghc *mhc_free;
94 struct lwkt_token mhc_free_token;
95 uint32_t mhc_flags;
96
97 struct vmbus_msghc *mhc_active;
98 struct lwkt_token mhc_active_token;
99 };
100
101 #define VMBUS_MSGHC_CTXF_DESTROY 0x0001
102
103 static int vmbus_probe(device_t);
104 static int vmbus_attach(device_t);
105 static int vmbus_detach(device_t);
106 static void vmbus_intr(void *);
107 static void vmbus_timer_intr_reload(struct cputimer_intr *,
108 sysclock_t);
109 static void vmbus_timer_intr_pcpuhand(
110 struct cputimer_intr *);
111 static void vmbus_timer_intr_restart(
112 struct cputimer_intr *);
113
114 static int vmbus_dma_alloc(struct vmbus_softc *);
115 static void vmbus_dma_free(struct vmbus_softc *);
116 static int vmbus_intr_setup(struct vmbus_softc *);
117 static void vmbus_intr_teardown(struct vmbus_softc *);
118 static void vmbus_synic_setup(void *);
119 static void vmbus_synic_teardown(void *);
120 static void vmbus_timer_stop(void *);
121 static void vmbus_timer_config(void *);
122 static int vmbus_init(struct vmbus_softc *);
123 static int vmbus_init_contact(struct vmbus_softc *,
124 uint32_t);
125 static void vmbus_timer_restart(void *);
126 static void vmbus_timer_msgintr(struct vmbus_pcpu_data *);
127
128 static void vmbus_chan_msgproc(struct vmbus_softc *,
129 const struct vmbus_message *);
130
131 static struct vmbus_msghc_ctx *vmbus_msghc_ctx_create(bus_dma_tag_t);
132 static void vmbus_msghc_ctx_destroy(
133 struct vmbus_msghc_ctx *);
134 static void vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *);
135 static struct vmbus_msghc *vmbus_msghc_alloc(bus_dma_tag_t);
136 static void vmbus_msghc_free(struct vmbus_msghc *);
137 static struct vmbus_msghc *vmbus_msghc_get1(struct vmbus_msghc_ctx *,
138 uint32_t);
139
140 static device_method_t vmbus_methods[] = {
141 /* Device interface */
142 DEVMETHOD(device_probe, vmbus_probe),
143 DEVMETHOD(device_attach, vmbus_attach),
144 DEVMETHOD(device_detach, vmbus_detach),
145 DEVMETHOD(device_shutdown, bus_generic_shutdown),
146 DEVMETHOD(device_suspend, bus_generic_suspend),
147 DEVMETHOD(device_resume, bus_generic_resume),
148
149 DEVMETHOD_END
150 };
151
152 static driver_t vmbus_driver = {
153 "vmbus",
154 vmbus_methods,
155 sizeof(struct vmbus_softc)
156 };
157
158 static devclass_t vmbus_devclass;
159
160 DRIVER_MODULE(vmbus, acpi, vmbus_driver, vmbus_devclass, NULL, NULL);
161 MODULE_DEPEND(vmbus, acpi, 1, 1, 1);
162 MODULE_VERSION(vmbus, 1);
163
164 static struct cputimer_intr vmbus_cputimer_intr = {
165 .freq = HYPERV_TIMER_FREQ,
166 .reload = vmbus_timer_intr_reload,
167 .enable = cputimer_intr_default_enable,
168 .config = cputimer_intr_default_config,
169 .restart = vmbus_timer_intr_restart,
170 .pmfixup = cputimer_intr_default_pmfixup,
171 .initclock = cputimer_intr_default_initclock,
172 .pcpuhand = vmbus_timer_intr_pcpuhand,
173 .next = SLIST_ENTRY_INITIALIZER,
174 .name = "hyperv",
175 .type = CPUTIMER_INTR_VMM,
176 .prio = CPUTIMER_INTR_PRIO_VMM,
177 .caps = CPUTIMER_INTR_CAP_PS,
178 .priv = NULL
179 };
180
181 static const uint32_t vmbus_version[] = {
182 VMBUS_VERSION_WIN8_1,
183 VMBUS_VERSION_WIN8,
184 VMBUS_VERSION_WIN7,
185 VMBUS_VERSION_WS2008
186 };
187
188 static int vmbus_timer_intr_enable = 1;
189 TUNABLE_INT("hw.vmbus.timer_intr.enable", &vmbus_timer_intr_enable);
190
191 static int
vmbus_probe(device_t dev)192 vmbus_probe(device_t dev)
193 {
194 char *id[] = { "VMBUS", NULL };
195
196 if (ACPI_ID_PROBE(device_get_parent(dev), dev, id) == NULL ||
197 device_get_unit(dev) != 0 || vmm_guest != VMM_GUEST_HYPERV ||
198 (hyperv_features & CPUID_HV_MSR_SYNIC) == 0)
199 return (ENXIO);
200
201 device_set_desc(dev, "Hyper-V vmbus");
202
203 return (0);
204 }
205
206 static int
vmbus_attach(device_t dev)207 vmbus_attach(device_t dev)
208 {
209 struct vmbus_softc *sc = device_get_softc(dev);
210 int error, cpu, use_timer;
211
212 /*
213 * Basic setup.
214 */
215 sc->vmbus_dev = dev;
216 for (cpu = 0; cpu < ncpus; ++cpu) {
217 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);
218
219 psc->sc = sc;
220 psc->cpuid = cpu;
221 psc->timer_last = UINT64_MAX;
222 }
223
224 /*
225 * Should we use interrupt timer?
226 */
227 use_timer = 0;
228 if (device_get_unit(dev) == 0 &&
229 (hyperv_features & CPUID_HV_TIMER_MASK) == CPUID_HV_TIMER_MASK &&
230 hyperv_tc64 != NULL)
231 use_timer = 1;
232
233 /*
234 * Create context for "post message" Hypercalls
235 */
236 sc->vmbus_msg_hc = vmbus_msghc_ctx_create(
237 bus_get_dma_tag(sc->vmbus_dev));
238 if (sc->vmbus_msg_hc == NULL)
239 return ENXIO;
240
241 /*
242 * Allocate DMA stuffs.
243 */
244 error = vmbus_dma_alloc(sc);
245 if (error)
246 goto failed;
247
248 /*
249 * Setup interrupt.
250 */
251 error = vmbus_intr_setup(sc);
252 if (error)
253 goto failed;
254
255 if (use_timer) {
256 /*
257 * Make sure that interrupt timer is stopped.
258 */
259 lwkt_cpusync_simple(smp_active_mask, vmbus_timer_stop, sc);
260 }
261
262 /*
263 * Setup SynIC.
264 */
265 lwkt_cpusync_simple(smp_active_mask, vmbus_synic_setup, sc);
266 sc->vmbus_flags |= VMBUS_FLAG_SYNIC;
267
268 /*
269 * Initialize vmbus.
270 */
271 error = vmbus_init(sc);
272 if (error)
273 goto failed;
274
275 if (use_timer) {
276 /*
277 * Configure and register vmbus interrupt timer.
278 */
279 lwkt_cpusync_simple(smp_active_mask, vmbus_timer_config, sc);
280 vmbus_cputimer_intr.priv = sc;
281 cputimer_intr_register(&vmbus_cputimer_intr);
282 if (vmbus_timer_intr_enable)
283 cputimer_intr_select(&vmbus_cputimer_intr, 0);
284 }
285
286 return 0;
287 failed:
288 vmbus_detach(dev);
289 return error;
290 }
291
292 static int
vmbus_detach(device_t dev)293 vmbus_detach(device_t dev)
294 {
295 struct vmbus_softc *sc = device_get_softc(dev);
296
297 /* TODO: uninitialize vmbus. */
298 /* TODO: stop and deregister timer */
299
300 if (sc->vmbus_flags & VMBUS_FLAG_SYNIC)
301 lwkt_cpusync_simple(smp_active_mask, vmbus_synic_teardown, sc);
302 vmbus_intr_teardown(sc);
303 vmbus_dma_free(sc);
304
305 if (sc->vmbus_msg_hc != NULL) {
306 vmbus_msghc_ctx_destroy(sc->vmbus_msg_hc);
307 sc->vmbus_msg_hc = NULL;
308 }
309 return (0);
310 }
311
312 static __inline void
vmbus_msg_reset(volatile struct vmbus_message * msg)313 vmbus_msg_reset(volatile struct vmbus_message *msg)
314 {
315 msg->msg_type = HYPERV_MSGTYPE_NONE;
316 /*
317 * Make sure that the write to msg_type (i.e. set to
318 * HYPERV_MSGTYPE_NONE) happens before we read the
319 * msg_flags and send EOM to the hypervisor.
320 */
321 cpu_mfence();
322 if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) {
323 /*
324 * Ask the hypervisor to rescan message queue,
325 * and deliver new message if any.
326 */
327 wrmsr(MSR_HV_EOM, 0);
328 }
329 }
330
331 static void
vmbus_intr(void * xpsc)332 vmbus_intr(void *xpsc)
333 {
334 struct vmbus_pcpu_data *psc = xpsc;
335 volatile struct vmbus_message *msg;
336
337 msg = psc->message + VMBUS_SINT_MESSAGE;
338 while (__predict_false(msg->msg_type != HYPERV_MSGTYPE_NONE)) {
339 if (msg->msg_type == HYPERV_MSGTYPE_CHANNEL) {
340 /* Channel message */
341 vmbus_chan_msgproc(psc->sc,
342 __DEVOLATILE(const struct vmbus_message *, msg));
343 }
344 vmbus_msg_reset(msg);
345 }
346 }
347
348 static __inline void
vmbus_timer_oneshot(struct vmbus_pcpu_data * psc,uint64_t current)349 vmbus_timer_oneshot(struct vmbus_pcpu_data *psc, uint64_t current)
350 {
351 psc->timer_last = current;
352 wrmsr(MSR_HV_STIMER0_COUNT, current);
353 }
354
355 static void
vmbus_timer_intr_reload(struct cputimer_intr * cti,sysclock_t reload)356 vmbus_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload)
357 {
358 struct globaldata *gd = mycpu;
359 struct vmbus_softc *sc = cti->priv;
360 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, gd->gd_cpuid);
361 uint64_t current;
362
363 if ((ssysclock_t)reload < 0) /* neg value */
364 reload = 1;
365 reload = muldivu64(reload, cti->freq, sys_cputimer->freq);
366 current = hyperv_tc64() + reload;
367
368 if (gd->gd_timer_running) {
369 if (current < psc->timer_last)
370 vmbus_timer_oneshot(psc, current);
371 } else {
372 gd->gd_timer_running = 1;
373 vmbus_timer_oneshot(psc, current);
374 }
375 }
376
377 static void
vmbus_timer_intr_pcpuhand(struct cputimer_intr * cti)378 vmbus_timer_intr_pcpuhand(struct cputimer_intr *cti)
379 {
380 struct vmbus_softc *sc = cti->priv;
381 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid);
382
383 vmbus_timer_msgintr(psc);
384 }
385
386 static void
vmbus_timer_intr_restart(struct cputimer_intr * cti)387 vmbus_timer_intr_restart(struct cputimer_intr *cti)
388 {
389 lwkt_send_ipiq_mask(smp_active_mask, vmbus_timer_restart, cti->priv);
390 }
391
392 static struct vmbus_msghc *
vmbus_msghc_alloc(bus_dma_tag_t parent_dtag)393 vmbus_msghc_alloc(bus_dma_tag_t parent_dtag)
394 {
395 struct vmbus_msghc *mh;
396
397 mh = kmalloc(sizeof(*mh), M_DEVBUF, M_WAITOK | M_ZERO);
398
399 mh->mh_inprm = hyperv_dmamem_alloc(parent_dtag,
400 HYPERCALL_POSTMSGIN_ALIGN, 0, HYPERCALL_POSTMSGIN_SIZE,
401 &mh->mh_inprm_dma, BUS_DMA_WAITOK);
402 if (mh->mh_inprm == NULL) {
403 kfree(mh, M_DEVBUF);
404 return NULL;
405 }
406 return mh;
407 }
408
409 static void
vmbus_msghc_free(struct vmbus_msghc * mh)410 vmbus_msghc_free(struct vmbus_msghc *mh)
411 {
412 hyperv_dmamem_free(&mh->mh_inprm_dma, mh->mh_inprm);
413 kfree(mh, M_DEVBUF);
414 }
415
416 static void
vmbus_msghc_ctx_free(struct vmbus_msghc_ctx * mhc)417 vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *mhc)
418 {
419 KASSERT(mhc->mhc_active == NULL, ("still have active msg hypercall"));
420 KASSERT(mhc->mhc_free == NULL, ("still have hypercall msg"));
421
422 lwkt_token_uninit(&mhc->mhc_free_token);
423 lwkt_token_uninit(&mhc->mhc_active_token);
424 kfree(mhc, M_DEVBUF);
425 }
426
427 static struct vmbus_msghc_ctx *
vmbus_msghc_ctx_create(bus_dma_tag_t parent_dtag)428 vmbus_msghc_ctx_create(bus_dma_tag_t parent_dtag)
429 {
430 struct vmbus_msghc_ctx *mhc;
431
432 mhc = kmalloc(sizeof(*mhc), M_DEVBUF, M_WAITOK | M_ZERO);
433 lwkt_token_init(&mhc->mhc_free_token, "msghcf");
434 lwkt_token_init(&mhc->mhc_active_token, "msghca");
435
436 mhc->mhc_free = vmbus_msghc_alloc(parent_dtag);
437 if (mhc->mhc_free == NULL) {
438 vmbus_msghc_ctx_free(mhc);
439 return NULL;
440 }
441 return mhc;
442 }
443
444 static struct vmbus_msghc *
vmbus_msghc_get1(struct vmbus_msghc_ctx * mhc,uint32_t dtor_flag)445 vmbus_msghc_get1(struct vmbus_msghc_ctx *mhc, uint32_t dtor_flag)
446 {
447 struct vmbus_msghc *mh;
448
449 lwkt_gettoken(&mhc->mhc_free_token);
450
451 while ((mhc->mhc_flags & dtor_flag) == 0 && mhc->mhc_free == NULL)
452 tsleep(&mhc->mhc_free, 0, "gmsghc", 0);
453 if (mhc->mhc_flags & dtor_flag) {
454 /* Being destroyed */
455 mh = NULL;
456 } else {
457 mh = mhc->mhc_free;
458 KASSERT(mh != NULL, ("no free hypercall msg"));
459 KASSERT(mh->mh_resp == NULL,
460 ("hypercall msg has pending response"));
461 mhc->mhc_free = NULL;
462 }
463
464 lwkt_reltoken(&mhc->mhc_free_token);
465
466 return mh;
467 }
468
469 struct vmbus_msghc *
vmbus_msghc_get(struct vmbus_softc * sc,size_t dsize)470 vmbus_msghc_get(struct vmbus_softc *sc, size_t dsize)
471 {
472 struct hypercall_postmsg_in *inprm;
473 struct vmbus_msghc *mh;
474
475 if (dsize > HYPERCALL_POSTMSGIN_DSIZE_MAX)
476 return NULL;
477
478 mh = vmbus_msghc_get1(sc->vmbus_msg_hc, VMBUS_MSGHC_CTXF_DESTROY);
479 if (mh == NULL)
480 return NULL;
481
482 inprm = mh->mh_inprm;
483 memset(inprm, 0, HYPERCALL_POSTMSGIN_SIZE);
484 inprm->hc_connid = VMBUS_CONNID_MESSAGE;
485 inprm->hc_msgtype = HYPERV_MSGTYPE_CHANNEL;
486 inprm->hc_dsize = dsize;
487
488 return mh;
489 }
490
491 void
vmbus_msghc_put(struct vmbus_softc * sc,struct vmbus_msghc * mh)492 vmbus_msghc_put(struct vmbus_softc *sc, struct vmbus_msghc *mh)
493 {
494 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;
495
496 KASSERT(mhc->mhc_active == NULL, ("msg hypercall is active"));
497 mh->mh_resp = NULL;
498
499 lwkt_gettoken(&mhc->mhc_free_token);
500 KASSERT(mhc->mhc_free == NULL, ("has free hypercall msg"));
501 mhc->mhc_free = mh;
502 lwkt_reltoken(&mhc->mhc_free_token);
503 wakeup(&mhc->mhc_free);
504 }
505
506 void *
vmbus_msghc_dataptr(struct vmbus_msghc * mh)507 vmbus_msghc_dataptr(struct vmbus_msghc *mh)
508 {
509 return mh->mh_inprm->hc_data;
510 }
511
512 static void
vmbus_msghc_ctx_destroy(struct vmbus_msghc_ctx * mhc)513 vmbus_msghc_ctx_destroy(struct vmbus_msghc_ctx *mhc)
514 {
515 struct vmbus_msghc *mh;
516
517 lwkt_gettoken(&mhc->mhc_free_token);
518 mhc->mhc_flags |= VMBUS_MSGHC_CTXF_DESTROY;
519 lwkt_reltoken(&mhc->mhc_free_token);
520 wakeup(&mhc->mhc_free);
521
522 mh = vmbus_msghc_get1(mhc, 0);
523 if (mh == NULL)
524 panic("can't get msghc");
525
526 vmbus_msghc_free(mh);
527 vmbus_msghc_ctx_free(mhc);
528 }
529
530 int
vmbus_msghc_exec_noresult(struct vmbus_msghc * mh)531 vmbus_msghc_exec_noresult(struct vmbus_msghc *mh)
532 {
533 int i, wait_ticks = 1;
534
535 /*
536 * Save the input parameter so that we could restore the input
537 * parameter if the Hypercall failed.
538 *
539 * XXX
540 * Is this really necessary?! i.e. Will the Hypercall ever
541 * overwrite the input parameter?
542 */
543 memcpy(&mh->mh_inprm_save, mh->mh_inprm, HYPERCALL_POSTMSGIN_SIZE);
544
545 /*
546 * In order to cope with transient failures, e.g. insufficient
547 * resources on host side, we retry the post message Hypercall
548 * several times. 20 retries seem sufficient.
549 */
550 #define HC_RETRY_MAX 20
551
552 for (i = 0; i < HC_RETRY_MAX; ++i) {
553 uint64_t status;
554
555 status = hypercall_post_message(mh->mh_inprm_dma.hv_paddr);
556 if (status == HYPERCALL_STATUS_SUCCESS)
557 return 0;
558
559 tsleep(&status, 0, "hcpmsg", wait_ticks);
560 if (wait_ticks < hz)
561 wait_ticks *= 2;
562
563 /* Restore input parameter and try again */
564 memcpy(mh->mh_inprm, &mh->mh_inprm_save,
565 HYPERCALL_POSTMSGIN_SIZE);
566 }
567
568 #undef HC_RETRY_MAX
569
570 return EIO;
571 }
572
573 int
vmbus_msghc_exec(struct vmbus_softc * sc,struct vmbus_msghc * mh)574 vmbus_msghc_exec(struct vmbus_softc *sc, struct vmbus_msghc *mh)
575 {
576 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;
577 int error;
578
579 KASSERT(mh->mh_resp == NULL, ("hypercall msg has pending response"));
580
581 lwkt_gettoken(&mhc->mhc_active_token);
582 KASSERT(mhc->mhc_active == NULL, ("pending active msg hypercall"));
583 mhc->mhc_active = mh;
584 lwkt_reltoken(&mhc->mhc_active_token);
585
586 error = vmbus_msghc_exec_noresult(mh);
587 if (error) {
588 lwkt_gettoken(&mhc->mhc_active_token);
589 KASSERT(mhc->mhc_active == mh, ("msghc mismatch"));
590 mhc->mhc_active = NULL;
591 lwkt_reltoken(&mhc->mhc_active_token);
592 }
593 return error;
594 }
595
596 const struct vmbus_message *
vmbus_msghc_wait_result(struct vmbus_softc * sc,struct vmbus_msghc * mh)597 vmbus_msghc_wait_result(struct vmbus_softc *sc, struct vmbus_msghc *mh)
598 {
599 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;
600
601 lwkt_gettoken(&mhc->mhc_active_token);
602
603 KASSERT(mhc->mhc_active == mh, ("msghc mismatch"));
604 while (mh->mh_resp == NULL)
605 tsleep(&mhc->mhc_active, 0, "wmsghc", 0);
606 mhc->mhc_active = NULL;
607
608 lwkt_reltoken(&mhc->mhc_active_token);
609
610 return mh->mh_resp;
611 }
612
613 void
vmbus_msghc_wakeup(struct vmbus_softc * sc,const struct vmbus_message * msg)614 vmbus_msghc_wakeup(struct vmbus_softc *sc, const struct vmbus_message *msg)
615 {
616 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;
617 struct vmbus_msghc *mh;
618
619 lwkt_gettoken(&mhc->mhc_active_token);
620
621 mh = mhc->mhc_active;
622 KASSERT(mh != NULL, ("no pending msg hypercall"));
623 memcpy(&mh->mh_resp0, msg, sizeof(mh->mh_resp0));
624 mh->mh_resp = &mh->mh_resp0;
625
626 lwkt_reltoken(&mhc->mhc_active_token);
627 wakeup(&mhc->mhc_active);
628 }
629
630 static int
vmbus_dma_alloc(struct vmbus_softc * sc)631 vmbus_dma_alloc(struct vmbus_softc *sc)
632 {
633 bus_dma_tag_t parent_dtag;
634 uint8_t *evtflags;
635 int cpu;
636
637 parent_dtag = bus_get_dma_tag(sc->vmbus_dev);
638 for (cpu = 0; cpu < ncpus; ++cpu) {
639 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);
640
641 /*
642 * Per-cpu messages and event flags.
643 */
644 psc->message = hyperv_dmamem_alloc(parent_dtag,
645 PAGE_SIZE, 0, PAGE_SIZE, &psc->message_dma,
646 BUS_DMA_WAITOK | BUS_DMA_ZERO);
647 if (psc->message == NULL)
648 return ENOMEM;
649
650 psc->event_flags = hyperv_dmamem_alloc(parent_dtag,
651 PAGE_SIZE, 0, PAGE_SIZE, &psc->event_flags_dma,
652 BUS_DMA_WAITOK | BUS_DMA_ZERO);
653 if (psc->event_flags == NULL)
654 return ENOMEM;
655 }
656
657 evtflags = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
658 PAGE_SIZE, &sc->vmbus_evtflags_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
659 if (evtflags == NULL)
660 return ENOMEM;
661 sc->vmbus_rx_evtflags = (u_long *)evtflags;
662 sc->vmbus_tx_evtflags = (u_long *)(evtflags + (PAGE_SIZE / 2));
663 sc->vmbus_evtflags = evtflags;
664
665 sc->vmbus_mnf1 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
666 PAGE_SIZE, &sc->vmbus_mnf1_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
667 if (sc->vmbus_mnf1 == NULL)
668 return ENOMEM;
669
670 sc->vmbus_mnf2 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
671 PAGE_SIZE, &sc->vmbus_mnf2_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
672 if (sc->vmbus_mnf2 == NULL)
673 return ENOMEM;
674
675 return 0;
676 }
677
678 static void
vmbus_dma_free(struct vmbus_softc * sc)679 vmbus_dma_free(struct vmbus_softc *sc)
680 {
681 int cpu;
682
683 if (sc->vmbus_evtflags != NULL) {
684 hyperv_dmamem_free(&sc->vmbus_evtflags_dma, sc->vmbus_evtflags);
685 sc->vmbus_evtflags = NULL;
686 sc->vmbus_rx_evtflags = NULL;
687 sc->vmbus_tx_evtflags = NULL;
688 }
689 if (sc->vmbus_mnf1 != NULL) {
690 hyperv_dmamem_free(&sc->vmbus_mnf1_dma, sc->vmbus_mnf1);
691 sc->vmbus_mnf1 = NULL;
692 }
693 if (sc->vmbus_mnf2 != NULL) {
694 hyperv_dmamem_free(&sc->vmbus_mnf2_dma, sc->vmbus_mnf2);
695 sc->vmbus_mnf2 = NULL;
696 }
697
698 for (cpu = 0; cpu < ncpus; ++cpu) {
699 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);
700
701 if (psc->message != NULL) {
702 hyperv_dmamem_free(&psc->message_dma, psc->message);
703 psc->message = NULL;
704 }
705 if (psc->event_flags != NULL) {
706 hyperv_dmamem_free(&psc->event_flags_dma,
707 psc->event_flags);
708 psc->event_flags = NULL;
709 }
710 }
711 }
712
713 static int
vmbus_intr_setup(struct vmbus_softc * sc)714 vmbus_intr_setup(struct vmbus_softc *sc)
715 {
716 device_t dev = sc->vmbus_dev;
717 device_t bus = device_get_parent(device_get_parent(dev));
718 int rid, cpu;
719
720 rid = 0;
721 for (cpu = 0; cpu < ncpus; ++cpu) {
722 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);
723 uint64_t msi_addr;
724 uint32_t msi_data;
725 int error;
726
727 error = PCIB_ALLOC_MSIX(bus, dev, &psc->intr_irq, cpu);
728 if (error) {
729 device_printf(dev, "alloc vector on cpu%d failed: %d\n",
730 cpu, error);
731 return ENXIO;
732 }
733 psc->intr_rid = ++rid;
734
735 psc->intr_res = BUS_ALLOC_RESOURCE(bus, dev, SYS_RES_IRQ,
736 &psc->intr_rid, psc->intr_irq, psc->intr_irq, 1,
737 RF_ACTIVE, cpu);
738 if (psc->intr_res == NULL) {
739 device_printf(dev, "alloc irq on cpu%d failed: %d\n",
740 cpu, error);
741 return ENXIO;
742 }
743
744 error = PCIB_MAP_MSI(bus, dev, rman_get_start(psc->intr_res),
745 &msi_addr, &msi_data, cpu);
746 if (error) {
747 device_printf(dev, "map irq on cpu%d failed: %d\n",
748 cpu, error);
749 return ENXIO;
750 }
751 psc->intr_vec = hyperv_msi2vector(msi_addr, msi_data);
752
753 if (bootverbose) {
754 device_printf(dev, "vector %d irq %d on cpu%d\n",
755 psc->intr_vec, psc->intr_irq, cpu);
756 }
757
758 ksnprintf(psc->intr_desc, sizeof(psc->intr_desc), "%s cpu%d",
759 device_get_nameunit(dev), cpu);
760 error = bus_setup_intr_descr(dev, psc->intr_res, INTR_MPSAFE,
761 vmbus_intr, psc, &psc->intr_hand, NULL, psc->intr_desc);
762 if (error) {
763 device_printf(dev, "setup intr on cpu%d failed: %d\n",
764 cpu, error);
765 return ENXIO;
766 }
767 }
768 return 0;
769 }
770
771 static void
vmbus_intr_teardown(struct vmbus_softc * sc)772 vmbus_intr_teardown(struct vmbus_softc *sc)
773 {
774 device_t dev = sc->vmbus_dev;
775 device_t bus = device_get_parent(device_get_parent(dev));
776 int cpu;
777
778 for (cpu = 0; cpu < ncpus; ++cpu) {
779 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);
780
781 if (psc->intr_hand != NULL) {
782 bus_teardown_intr(dev, psc->intr_res, psc->intr_hand);
783 psc->intr_hand = NULL;
784 }
785
786 if (psc->intr_res != NULL) {
787 BUS_RELEASE_RESOURCE(bus, dev, SYS_RES_IRQ,
788 psc->intr_rid, psc->intr_res);
789 psc->intr_res = NULL;
790 }
791
792 if (psc->intr_rid != 0) {
793 PCIB_RELEASE_MSIX(bus, dev, psc->intr_irq, psc->cpuid);
794 psc->intr_rid = 0;
795 }
796 }
797 }
798
799 static void
vmbus_synic_setup(void * xsc)800 vmbus_synic_setup(void *xsc)
801 {
802 struct vmbus_softc *sc = xsc;
803 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid);
804 uint64_t val, orig;
805 uint32_t sint;
806
807 if (hyperv_features & CPUID_HV_MSR_VP_INDEX) {
808 /*
809 * Save virtual processor id.
810 */
811 psc->vcpuid = rdmsr(MSR_HV_VP_INDEX);
812 } else {
813 /*
814 * XXX
815 * Virtual processoor id is only used by a pretty broken
816 * channel selection code from storvsc. It's nothing
817 * critical even if CPUID_HV_MSR_VP_INDEX is not set; keep
818 * moving on.
819 */
820 psc->vcpuid = mycpuid;
821 }
822
823 /*
824 * Setup the SynIC message.
825 */
826 orig = rdmsr(MSR_HV_SIMP);
827 val = MSR_HV_SIMP_ENABLE | (orig & MSR_HV_SIMP_RSVD_MASK) |
828 ((psc->message_dma.hv_paddr >> PAGE_SHIFT) << MSR_HV_SIMP_PGSHIFT);
829 wrmsr(MSR_HV_SIMP, val);
830
831 /*
832 * Setup the SynIC event flags.
833 */
834 orig = rdmsr(MSR_HV_SIEFP);
835 val = MSR_HV_SIEFP_ENABLE | (orig & MSR_HV_SIEFP_RSVD_MASK) |
836 ((psc->event_flags_dma.hv_paddr >> PAGE_SHIFT) <<
837 MSR_HV_SIEFP_PGSHIFT);
838 wrmsr(MSR_HV_SIEFP, val);
839
840
841 /*
842 * Configure and unmask SINT for message and event flags.
843 */
844 sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE;
845 orig = rdmsr(sint);
846 val = psc->intr_vec | /* MSR_HV_SINT_AUTOEOI | notyet */
847 (orig & MSR_HV_SINT_RSVD_MASK);
848 wrmsr(sint, val);
849
850 /*
851 * Configure and unmask SINT for timer.
852 */
853 sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER;
854 orig = rdmsr(sint);
855 val = XTIMER_OFFSET | /* MSR_HV_SINT_AUTOEOI | notyet */
856 (orig & MSR_HV_SINT_RSVD_MASK);
857 wrmsr(sint, val);
858
859 /*
860 * All done; enable SynIC.
861 */
862 orig = rdmsr(MSR_HV_SCONTROL);
863 val = MSR_HV_SCTRL_ENABLE | (orig & MSR_HV_SCTRL_RSVD_MASK);
864 wrmsr(MSR_HV_SCONTROL, val);
865 }
866
867 static void
vmbus_timer_stop(void * arg __unused)868 vmbus_timer_stop(void *arg __unused)
869 {
870 for (;;) {
871 uint64_t val;
872
873 /* Stop counting, and this also implies disabling STIMER0 */
874 wrmsr(MSR_HV_STIMER0_COUNT, 0);
875
876 val = rdmsr(MSR_HV_STIMER0_CONFIG);
877 if ((val & MSR_HV_STIMER_CFG_ENABLE) == 0)
878 break;
879 cpu_pause();
880 }
881 }
882
883 static void
vmbus_timer_config(void * arg __unused)884 vmbus_timer_config(void *arg __unused)
885 {
886 /*
887 * Make sure that STIMER0 is really disabled before writing
888 * to STIMER0_CONFIG.
889 *
890 * "Writing to the configuration register of a timer that
891 * is already enabled may result in undefined behaviour."
892 */
893 vmbus_timer_stop(arg);
894 wrmsr(MSR_HV_STIMER0_CONFIG,
895 MSR_HV_STIMER_CFG_AUTOEN | MSR_HV_STIMER0_CFG_SINT);
896 }
897
898 static void
vmbus_timer_msgintr(struct vmbus_pcpu_data * psc)899 vmbus_timer_msgintr(struct vmbus_pcpu_data *psc)
900 {
901 volatile struct vmbus_message *msg;
902
903 msg = psc->message + VMBUS_SINT_TIMER;
904 if (msg->msg_type == HYPERV_MSGTYPE_TIMER_EXPIRED)
905 vmbus_msg_reset(msg);
906 }
907
908 static void
vmbus_timer_restart(void * xsc)909 vmbus_timer_restart(void *xsc)
910 {
911 struct vmbus_softc *sc = xsc;
912 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid);
913
914 crit_enter();
915 vmbus_timer_msgintr(psc);
916 vmbus_timer_oneshot(psc, hyperv_tc64() + 1);
917 crit_exit();
918 }
919
920 static void
vmbus_synic_teardown(void * arg __unused)921 vmbus_synic_teardown(void *arg __unused)
922 {
923 uint64_t orig;
924 uint32_t sint;
925
926 /*
927 * Disable SynIC.
928 */
929 orig = rdmsr(MSR_HV_SCONTROL);
930 wrmsr(MSR_HV_SCONTROL, (orig & MSR_HV_SCTRL_RSVD_MASK));
931
932 /*
933 * Mask message and event flags SINT.
934 */
935 sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE;
936 orig = rdmsr(sint);
937 wrmsr(sint, orig | MSR_HV_SINT_MASKED);
938
939 /*
940 * Mask timer SINT.
941 */
942 sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER;
943 orig = rdmsr(sint);
944 wrmsr(sint, orig | MSR_HV_SINT_MASKED);
945
946 /*
947 * Teardown SynIC message.
948 */
949 orig = rdmsr(MSR_HV_SIMP);
950 wrmsr(MSR_HV_SIMP, (orig & MSR_HV_SIMP_RSVD_MASK));
951
952 /*
953 * Teardown SynIC event flags.
954 */
955 orig = rdmsr(MSR_HV_SIEFP);
956 wrmsr(MSR_HV_SIEFP, (orig & MSR_HV_SIEFP_RSVD_MASK));
957 }
958
959 static int
vmbus_init_contact(struct vmbus_softc * sc,uint32_t version)960 vmbus_init_contact(struct vmbus_softc *sc, uint32_t version)
961 {
962 struct vmbus_chanmsg_init_contact *req;
963 const struct vmbus_chanmsg_version_resp *resp;
964 const struct vmbus_message *msg;
965 struct vmbus_msghc *mh;
966 int error, supp = 0;
967
968 mh = vmbus_msghc_get(sc, sizeof(*req));
969 if (mh == NULL)
970 return ENXIO;
971
972 req = vmbus_msghc_dataptr(mh);
973 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_INIT_CONTACT;
974 req->chm_ver = version;
975 req->chm_evtflags = sc->vmbus_evtflags_dma.hv_paddr;
976 req->chm_mnf1 = sc->vmbus_mnf1_dma.hv_paddr;
977 req->chm_mnf2 = sc->vmbus_mnf2_dma.hv_paddr;
978
979 error = vmbus_msghc_exec(sc, mh);
980 if (error) {
981 vmbus_msghc_put(sc, mh);
982 return error;
983 }
984
985 msg = vmbus_msghc_wait_result(sc, mh);
986 resp = (const struct vmbus_chanmsg_version_resp *)msg->msg_data;
987 supp = resp->chm_supp;
988
989 vmbus_msghc_put(sc, mh);
990
991 return (supp ? 0 : EOPNOTSUPP);
992 }
993
994 static int
vmbus_init(struct vmbus_softc * sc)995 vmbus_init(struct vmbus_softc *sc)
996 {
997 int i;
998
999 for (i = 0; i < nitems(vmbus_version); ++i) {
1000 int error;
1001
1002 error = vmbus_init_contact(sc, vmbus_version[i]);
1003 if (!error) {
1004 sc->vmbus_version = vmbus_version[i];
1005 device_printf(sc->vmbus_dev, "version %u.%u\n",
1006 (sc->vmbus_version >> 16),
1007 (sc->vmbus_version & 0xffff));
1008 return 0;
1009 }
1010 }
1011 return ENXIO;
1012 }
1013
1014 static void
vmbus_chan_msgproc(struct vmbus_softc * sc,const struct vmbus_message * msg)1015 vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg)
1016 {
1017 const struct vmbus_chanmsg_hdr *hdr;
1018
1019 hdr = (const struct vmbus_chanmsg_hdr *)msg->msg_data;
1020
1021 /* TODO */
1022 if (hdr->chm_type == VMBUS_CHANMSG_TYPE_VERSION_RESP)
1023 vmbus_msghc_wakeup(sc, msg);
1024 }
1025