1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 2005
5 * Bill Paul <wpaul@windriver.com>. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by Bill Paul.
18 * 4. Neither the name of the author nor the names of any co-contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD: stable/12/sys/compat/ndis/kern_windrv.c 345459 2019-03-23 22:44:11Z gonzo $");
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/unistd.h>
41 #include <sys/types.h>
42
43 #include <sys/kernel.h>
44 #include <sys/malloc.h>
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47 #include <sys/module.h>
48 #include <sys/conf.h>
49 #include <sys/mbuf.h>
50 #include <sys/bus.h>
51 #include <sys/proc.h>
52 #include <sys/sched.h>
53 #include <sys/smp.h>
54
55 #include <sys/queue.h>
56
57 #ifdef __i386__
58 #include <machine/segments.h>
59 #endif
60
61 #ifdef __amd64__
62 #include <machine/fpu.h>
63 #endif
64
65 #include <dev/usb/usb.h>
66
67 #include <compat/ndis/pe_var.h>
68 #include <compat/ndis/cfg_var.h>
69 #include <compat/ndis/resource_var.h>
70 #include <compat/ndis/ntoskrnl_var.h>
71 #include <compat/ndis/ndis_var.h>
72 #include <compat/ndis/hal_var.h>
73 #include <compat/ndis/usbd_var.h>
74
75 #ifdef __amd64__
76 struct fpu_cc_ent {
77 struct fpu_kern_ctx *ctx;
78 LIST_ENTRY(fpu_cc_ent) entries;
79 };
80 static LIST_HEAD(fpu_ctx_free, fpu_cc_ent) fpu_free_head =
81 LIST_HEAD_INITIALIZER(fpu_free_head);
82 static LIST_HEAD(fpu_ctx_busy, fpu_cc_ent) fpu_busy_head =
83 LIST_HEAD_INITIALIZER(fpu_busy_head);
84 static struct mtx fpu_free_mtx;
85 static struct mtx fpu_busy_mtx;
86 #endif
87
88 static struct mtx drvdb_mtx;
89 static STAILQ_HEAD(drvdb, drvdb_ent) drvdb_head;
90
91 static driver_object fake_pci_driver; /* serves both PCI and cardbus */
92 static driver_object fake_pccard_driver;
93
94 #ifdef __i386__
95 static void x86_oldldt(void *);
96 static void x86_newldt(void *);
97
98 struct tid {
99 void *tid_except_list; /* 0x00 */
100 uint32_t tid_oldfs; /* 0x04 */
101 uint32_t tid_selector; /* 0x08 */
102 struct tid *tid_self; /* 0x0C */
103 int tid_cpu; /* 0x10 */
104 };
105
106 static struct tid *my_tids;
107 #endif /* __i386__ */
108
109 #define DUMMY_REGISTRY_PATH "\\\\some\\bogus\\path"
110
111 int
windrv_libinit(void)112 windrv_libinit(void)
113 {
114 STAILQ_INIT(&drvdb_head);
115 mtx_init(&drvdb_mtx, "Windows driver DB lock",
116 "Windows internal lock", MTX_DEF);
117
118 #ifdef __amd64__
119 LIST_INIT(&fpu_free_head);
120 LIST_INIT(&fpu_busy_head);
121 mtx_init(&fpu_free_mtx, "free fpu context list lock", NULL, MTX_DEF);
122 mtx_init(&fpu_busy_mtx, "busy fpu context list lock", NULL, MTX_DEF);
123 #endif
124
125 /*
126 * PCI and pccard devices don't need to use IRPs to
127 * interact with their bus drivers (usually), so our
128 * emulated PCI and pccard drivers are just stubs.
129 * USB devices, on the other hand, do all their I/O
130 * by exchanging IRPs with the USB bus driver, so
131 * for that we need to provide emulator dispatcher
132 * routines, which are in a separate module.
133 */
134
135 windrv_bus_attach(&fake_pci_driver, "PCI Bus");
136 windrv_bus_attach(&fake_pccard_driver, "PCCARD Bus");
137
138 #ifdef __i386__
139
140 /*
141 * In order to properly support SMP machines, we have
142 * to modify the GDT on each CPU, since we never know
143 * on which one we'll end up running.
144 */
145
146 my_tids = ExAllocatePoolWithTag(NonPagedPool,
147 sizeof(struct tid) * mp_ncpus, 0);
148 if (my_tids == NULL)
149 panic("failed to allocate thread info blocks");
150 smp_rendezvous(NULL, x86_newldt, NULL, NULL);
151 #endif
152 return (0);
153 }
154
155 int
windrv_libfini(void)156 windrv_libfini(void)
157 {
158 struct drvdb_ent *d;
159 #ifdef __amd64__
160 struct fpu_cc_ent *ent;
161 #endif
162
163 mtx_lock(&drvdb_mtx);
164 while(STAILQ_FIRST(&drvdb_head) != NULL) {
165 d = STAILQ_FIRST(&drvdb_head);
166 STAILQ_REMOVE_HEAD(&drvdb_head, link);
167 free(d, M_DEVBUF);
168 }
169 mtx_unlock(&drvdb_mtx);
170
171 RtlFreeUnicodeString(&fake_pci_driver.dro_drivername);
172 RtlFreeUnicodeString(&fake_pccard_driver.dro_drivername);
173
174 mtx_destroy(&drvdb_mtx);
175
176 #ifdef __i386__
177 smp_rendezvous(NULL, x86_oldldt, NULL, NULL);
178 ExFreePool(my_tids);
179 #endif
180 #ifdef __amd64__
181 while ((ent = LIST_FIRST(&fpu_free_head)) != NULL) {
182 LIST_REMOVE(ent, entries);
183 fpu_kern_free_ctx(ent->ctx);
184 free(ent, M_DEVBUF);
185 }
186 mtx_destroy(&fpu_free_mtx);
187
188 ent = LIST_FIRST(&fpu_busy_head);
189 KASSERT(ent == NULL, ("busy fpu context list is not empty"));
190 mtx_destroy(&fpu_busy_mtx);
191 #endif
192 return (0);
193 }
194
195 /*
196 * Given the address of a driver image, find its corresponding
197 * driver_object.
198 */
199
200 driver_object *
windrv_lookup(img,name)201 windrv_lookup(img, name)
202 vm_offset_t img;
203 char *name;
204 {
205 struct drvdb_ent *d;
206 unicode_string us;
207 ansi_string as;
208
209 bzero((char *)&us, sizeof(us));
210
211 /* Damn unicode. */
212
213 if (name != NULL) {
214 RtlInitAnsiString(&as, name);
215 if (RtlAnsiStringToUnicodeString(&us, &as, TRUE))
216 return (NULL);
217 }
218
219 mtx_lock(&drvdb_mtx);
220 STAILQ_FOREACH(d, &drvdb_head, link) {
221 if (d->windrv_object->dro_driverstart == (void *)img ||
222 (bcmp((char *)d->windrv_object->dro_drivername.us_buf,
223 (char *)us.us_buf, us.us_len) == 0 && us.us_len)) {
224 mtx_unlock(&drvdb_mtx);
225 if (name != NULL)
226 ExFreePool(us.us_buf);
227 return (d->windrv_object);
228 }
229 }
230 mtx_unlock(&drvdb_mtx);
231
232 if (name != NULL)
233 RtlFreeUnicodeString(&us);
234
235 return (NULL);
236 }
237
238 struct drvdb_ent *
windrv_match(matchfunc,ctx)239 windrv_match(matchfunc, ctx)
240 matchfuncptr matchfunc;
241 void *ctx;
242 {
243 struct drvdb_ent *d;
244 int match;
245
246 mtx_lock(&drvdb_mtx);
247 STAILQ_FOREACH(d, &drvdb_head, link) {
248 if (d->windrv_devlist == NULL)
249 continue;
250 match = matchfunc(d->windrv_bustype, d->windrv_devlist, ctx);
251 if (match == TRUE) {
252 mtx_unlock(&drvdb_mtx);
253 return (d);
254 }
255 }
256 mtx_unlock(&drvdb_mtx);
257
258 return (NULL);
259 }
260
261 /*
262 * Remove a driver_object from our datatabase and destroy it. Throw
263 * away any custom driver extension info that may have been added.
264 */
265
266 int
windrv_unload(mod,img,len)267 windrv_unload(mod, img, len)
268 module_t mod;
269 vm_offset_t img;
270 int len;
271 {
272 struct drvdb_ent *db, *r = NULL;
273 driver_object *drv;
274 device_object *d, *pdo;
275 device_t dev;
276 list_entry *e;
277
278 drv = windrv_lookup(img, NULL);
279
280 /*
281 * When we unload a driver image, we need to force a
282 * detach of any devices that might be using it. We
283 * need the PDOs of all attached devices for this.
284 * Getting at them is a little hard. We basically
285 * have to walk the device lists of all our bus
286 * drivers.
287 */
288
289 mtx_lock(&drvdb_mtx);
290 STAILQ_FOREACH(db, &drvdb_head, link) {
291 /*
292 * Fake bus drivers have no devlist info.
293 * If this driver has devlist info, it's
294 * a loaded Windows driver and has no PDOs,
295 * so skip it.
296 */
297 if (db->windrv_devlist != NULL)
298 continue;
299 pdo = db->windrv_object->dro_devobj;
300 while (pdo != NULL) {
301 d = pdo->do_attacheddev;
302 if (d->do_drvobj != drv) {
303 pdo = pdo->do_nextdev;
304 continue;
305 }
306 dev = pdo->do_devext;
307 pdo = pdo->do_nextdev;
308 mtx_unlock(&drvdb_mtx);
309 device_detach(dev);
310 mtx_lock(&drvdb_mtx);
311 }
312 }
313
314 STAILQ_FOREACH(db, &drvdb_head, link) {
315 if (db->windrv_object->dro_driverstart == (void *)img) {
316 r = db;
317 STAILQ_REMOVE(&drvdb_head, db, drvdb_ent, link);
318 break;
319 }
320 }
321 mtx_unlock(&drvdb_mtx);
322
323 if (r == NULL)
324 return (ENOENT);
325
326 if (drv == NULL)
327 return (ENOENT);
328
329 /*
330 * Destroy any custom extensions that may have been added.
331 */
332 drv = r->windrv_object;
333 while (!IsListEmpty(&drv->dro_driverext->dre_usrext)) {
334 e = RemoveHeadList(&drv->dro_driverext->dre_usrext);
335 ExFreePool(e);
336 }
337
338 /* Free the driver extension */
339 free(drv->dro_driverext, M_DEVBUF);
340
341 /* Free the driver name */
342 RtlFreeUnicodeString(&drv->dro_drivername);
343
344 /* Free driver object */
345 free(drv, M_DEVBUF);
346
347 /* Free our DB handle */
348 free(r, M_DEVBUF);
349
350 return (0);
351 }
352
353 #define WINDRV_LOADED htonl(0x42534F44)
354
355 #ifdef __amd64__
356 static void
patch_user_shared_data_address(vm_offset_t img,size_t len)357 patch_user_shared_data_address(vm_offset_t img, size_t len)
358 {
359 unsigned long i, n, max_addr, *addr;
360
361 n = len - sizeof(unsigned long);
362 max_addr = KI_USER_SHARED_DATA + sizeof(kuser_shared_data);
363 for (i = 0; i < n; i++) {
364 addr = (unsigned long *)(img + i);
365 if (*addr >= KI_USER_SHARED_DATA && *addr < max_addr) {
366 *addr -= KI_USER_SHARED_DATA;
367 *addr += (unsigned long)&kuser_shared_data;
368 }
369 }
370 }
371 #endif
372
373 /*
374 * Loader routine for actual Windows driver modules, ultimately
375 * calls the driver's DriverEntry() routine.
376 */
377
378 int
windrv_load(mod,img,len,bustype,devlist,regvals)379 windrv_load(mod, img, len, bustype, devlist, regvals)
380 module_t mod;
381 vm_offset_t img;
382 int len;
383 interface_type bustype;
384 void *devlist;
385 ndis_cfg *regvals;
386 {
387 image_import_descriptor imp_desc;
388 image_optional_header opt_hdr;
389 driver_entry entry;
390 struct drvdb_ent *new;
391 struct driver_object *drv;
392 int status;
393 uint32_t *ptr;
394 ansi_string as;
395
396 /*
397 * First step: try to relocate and dynalink the executable
398 * driver image.
399 */
400
401 ptr = (uint32_t *)(img + 8);
402 if (*ptr == WINDRV_LOADED)
403 goto skipreloc;
404
405 /* Perform text relocation */
406 if (pe_relocate(img))
407 return (ENOEXEC);
408
409 /* Dynamically link the NDIS.SYS routines -- required. */
410 if (pe_patch_imports(img, "NDIS", ndis_functbl))
411 return (ENOEXEC);
412
413 /* Dynamically link the HAL.dll routines -- optional. */
414 if (pe_get_import_descriptor(img, &imp_desc, "HAL") == 0) {
415 if (pe_patch_imports(img, "HAL", hal_functbl))
416 return (ENOEXEC);
417 }
418
419 /* Dynamically link ntoskrnl.exe -- optional. */
420 if (pe_get_import_descriptor(img, &imp_desc, "ntoskrnl") == 0) {
421 if (pe_patch_imports(img, "ntoskrnl", ntoskrnl_functbl))
422 return (ENOEXEC);
423 }
424
425 #ifdef __amd64__
426 patch_user_shared_data_address(img, len);
427 #endif
428
429 /* Dynamically link USBD.SYS -- optional */
430 if (pe_get_import_descriptor(img, &imp_desc, "USBD") == 0) {
431 if (pe_patch_imports(img, "USBD", usbd_functbl))
432 return (ENOEXEC);
433 }
434
435 *ptr = WINDRV_LOADED;
436
437 skipreloc:
438
439 /* Next step: find the driver entry point. */
440
441 pe_get_optional_header(img, &opt_hdr);
442 entry = (driver_entry)pe_translate_addr(img, opt_hdr.ioh_entryaddr);
443
444 /* Next step: allocate and store a driver object. */
445
446 new = malloc(sizeof(struct drvdb_ent), M_DEVBUF, M_NOWAIT|M_ZERO);
447 if (new == NULL)
448 return (ENOMEM);
449
450 drv = malloc(sizeof(driver_object), M_DEVBUF, M_NOWAIT|M_ZERO);
451 if (drv == NULL) {
452 free (new, M_DEVBUF);
453 return (ENOMEM);
454 }
455
456 /* Allocate a driver extension structure too. */
457
458 drv->dro_driverext = malloc(sizeof(driver_extension),
459 M_DEVBUF, M_NOWAIT|M_ZERO);
460
461 if (drv->dro_driverext == NULL) {
462 free(new, M_DEVBUF);
463 free(drv, M_DEVBUF);
464 return (ENOMEM);
465 }
466
467 InitializeListHead((&drv->dro_driverext->dre_usrext));
468
469 drv->dro_driverstart = (void *)img;
470 drv->dro_driversize = len;
471
472 RtlInitAnsiString(&as, DUMMY_REGISTRY_PATH);
473 if (RtlAnsiStringToUnicodeString(&drv->dro_drivername, &as, TRUE)) {
474 free(new, M_DEVBUF);
475 free(drv, M_DEVBUF);
476 return (ENOMEM);
477 }
478
479 new->windrv_object = drv;
480 new->windrv_regvals = regvals;
481 new->windrv_devlist = devlist;
482 new->windrv_bustype = bustype;
483
484 /* Now call the DriverEntry() function. */
485
486 status = MSCALL2(entry, drv, &drv->dro_drivername);
487
488 if (status != STATUS_SUCCESS) {
489 RtlFreeUnicodeString(&drv->dro_drivername);
490 free(drv, M_DEVBUF);
491 free(new, M_DEVBUF);
492 return (ENODEV);
493 }
494
495 mtx_lock(&drvdb_mtx);
496 STAILQ_INSERT_HEAD(&drvdb_head, new, link);
497 mtx_unlock(&drvdb_mtx);
498
499 return (0);
500 }
501
502 /*
503 * Make a new Physical Device Object for a device that was
504 * detected/plugged in. For us, the PDO is just a way to
505 * get at the device_t.
506 */
507
508 int
windrv_create_pdo(drv,bsddev)509 windrv_create_pdo(drv, bsddev)
510 driver_object *drv;
511 device_t bsddev;
512 {
513 device_object *dev;
514
515 /*
516 * This is a new physical device object, which technically
517 * is the "top of the stack." Consequently, we don't do
518 * an IoAttachDeviceToDeviceStack() here.
519 */
520
521 mtx_lock(&drvdb_mtx);
522 IoCreateDevice(drv, 0, NULL, FILE_DEVICE_UNKNOWN, 0, FALSE, &dev);
523 mtx_unlock(&drvdb_mtx);
524
525 /* Stash pointer to our BSD device handle. */
526
527 dev->do_devext = bsddev;
528
529 return (STATUS_SUCCESS);
530 }
531
532 void
windrv_destroy_pdo(drv,bsddev)533 windrv_destroy_pdo(drv, bsddev)
534 driver_object *drv;
535 device_t bsddev;
536 {
537 device_object *pdo;
538
539 pdo = windrv_find_pdo(drv, bsddev);
540
541 /* Remove reference to device_t */
542
543 pdo->do_devext = NULL;
544
545 mtx_lock(&drvdb_mtx);
546 IoDeleteDevice(pdo);
547 mtx_unlock(&drvdb_mtx);
548 }
549
550 /*
551 * Given a device_t, find the corresponding PDO in a driver's
552 * device list.
553 */
554
555 device_object *
windrv_find_pdo(drv,bsddev)556 windrv_find_pdo(drv, bsddev)
557 driver_object *drv;
558 device_t bsddev;
559 {
560 device_object *pdo;
561
562 mtx_lock(&drvdb_mtx);
563 pdo = drv->dro_devobj;
564 while (pdo != NULL) {
565 if (pdo->do_devext == bsddev) {
566 mtx_unlock(&drvdb_mtx);
567 return (pdo);
568 }
569 pdo = pdo->do_nextdev;
570 }
571 mtx_unlock(&drvdb_mtx);
572
573 return (NULL);
574 }
575
576 /*
577 * Add an internally emulated driver to the database. We need this
578 * to set up an emulated bus driver so that it can receive IRPs.
579 */
580
581 int
windrv_bus_attach(drv,name)582 windrv_bus_attach(drv, name)
583 driver_object *drv;
584 char *name;
585 {
586 struct drvdb_ent *new;
587 ansi_string as;
588
589 new = malloc(sizeof(struct drvdb_ent), M_DEVBUF, M_NOWAIT|M_ZERO);
590 if (new == NULL)
591 return (ENOMEM);
592
593 RtlInitAnsiString(&as, name);
594 if (RtlAnsiStringToUnicodeString(&drv->dro_drivername, &as, TRUE))
595 {
596 free(new, M_DEVBUF);
597 return (ENOMEM);
598 }
599
600 /*
601 * Set up a fake image pointer to avoid false matches
602 * in windrv_lookup().
603 */
604 drv->dro_driverstart = (void *)0xFFFFFFFF;
605
606 new->windrv_object = drv;
607 new->windrv_devlist = NULL;
608 new->windrv_regvals = NULL;
609
610 mtx_lock(&drvdb_mtx);
611 STAILQ_INSERT_HEAD(&drvdb_head, new, link);
612 mtx_unlock(&drvdb_mtx);
613
614 return (0);
615 }
616
617 #ifdef __amd64__
618
619 extern void x86_64_wrap(void);
620 extern void x86_64_wrap_call(void);
621 extern void x86_64_wrap_end(void);
622
623 int
windrv_wrap(func,wrap,argcnt,ftype)624 windrv_wrap(func, wrap, argcnt, ftype)
625 funcptr func;
626 funcptr *wrap;
627 int argcnt;
628 int ftype;
629 {
630 funcptr p;
631 vm_offset_t *calladdr;
632 vm_offset_t wrapstart, wrapend, wrapcall;
633
634 wrapstart = (vm_offset_t)&x86_64_wrap;
635 wrapend = (vm_offset_t)&x86_64_wrap_end;
636 wrapcall = (vm_offset_t)&x86_64_wrap_call;
637
638 /* Allocate a new wrapper instance. */
639
640 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT);
641 if (p == NULL)
642 return (ENOMEM);
643
644 /* Copy over the code. */
645
646 bcopy((char *)wrapstart, p, (wrapend - wrapstart));
647
648 /* Insert the function address into the new wrapper instance. */
649
650 calladdr = (uint64_t *)((char *)p + (wrapcall - wrapstart) + 2);
651 *calladdr = (vm_offset_t)func;
652
653 *wrap = p;
654
655 return (0);
656 }
657
658 static struct fpu_cc_ent *
request_fpu_cc_ent(void)659 request_fpu_cc_ent(void)
660 {
661 struct fpu_cc_ent *ent;
662
663 mtx_lock(&fpu_free_mtx);
664 if ((ent = LIST_FIRST(&fpu_free_head)) != NULL) {
665 LIST_REMOVE(ent, entries);
666 mtx_unlock(&fpu_free_mtx);
667 mtx_lock(&fpu_busy_mtx);
668 LIST_INSERT_HEAD(&fpu_busy_head, ent, entries);
669 mtx_unlock(&fpu_busy_mtx);
670 return (ent);
671 }
672 mtx_unlock(&fpu_free_mtx);
673
674 if ((ent = malloc(sizeof(struct fpu_cc_ent), M_DEVBUF, M_NOWAIT |
675 M_ZERO)) != NULL) {
676 ent->ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL |
677 FPU_KERN_NOWAIT);
678 if (ent->ctx != NULL) {
679 mtx_lock(&fpu_busy_mtx);
680 LIST_INSERT_HEAD(&fpu_busy_head, ent, entries);
681 mtx_unlock(&fpu_busy_mtx);
682 } else {
683 free(ent, M_DEVBUF);
684 ent = NULL;
685 }
686 }
687
688 return (ent);
689 }
690
691 static void
release_fpu_cc_ent(struct fpu_cc_ent * ent)692 release_fpu_cc_ent(struct fpu_cc_ent *ent)
693 {
694 mtx_lock(&fpu_busy_mtx);
695 LIST_REMOVE(ent, entries);
696 mtx_unlock(&fpu_busy_mtx);
697 mtx_lock(&fpu_free_mtx);
698 LIST_INSERT_HEAD(&fpu_free_head, ent, entries);
699 mtx_unlock(&fpu_free_mtx);
700 }
701
702 uint64_t
_x86_64_call1(void * fn,uint64_t a)703 _x86_64_call1(void *fn, uint64_t a)
704 {
705 struct fpu_cc_ent *ent;
706 uint64_t ret;
707
708 if ((ent = request_fpu_cc_ent()) == NULL)
709 return (ENOMEM);
710 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL);
711 ret = x86_64_call1(fn, a);
712 fpu_kern_leave(curthread, ent->ctx);
713 release_fpu_cc_ent(ent);
714
715 return (ret);
716 }
717
718 uint64_t
_x86_64_call2(void * fn,uint64_t a,uint64_t b)719 _x86_64_call2(void *fn, uint64_t a, uint64_t b)
720 {
721 struct fpu_cc_ent *ent;
722 uint64_t ret;
723
724 if ((ent = request_fpu_cc_ent()) == NULL)
725 return (ENOMEM);
726 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL);
727 ret = x86_64_call2(fn, a, b);
728 fpu_kern_leave(curthread, ent->ctx);
729 release_fpu_cc_ent(ent);
730
731 return (ret);
732 }
733
734 uint64_t
_x86_64_call3(void * fn,uint64_t a,uint64_t b,uint64_t c)735 _x86_64_call3(void *fn, uint64_t a, uint64_t b, uint64_t c)
736 {
737 struct fpu_cc_ent *ent;
738 uint64_t ret;
739
740 if ((ent = request_fpu_cc_ent()) == NULL)
741 return (ENOMEM);
742 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL);
743 ret = x86_64_call3(fn, a, b, c);
744 fpu_kern_leave(curthread, ent->ctx);
745 release_fpu_cc_ent(ent);
746
747 return (ret);
748 }
749
750 uint64_t
_x86_64_call4(void * fn,uint64_t a,uint64_t b,uint64_t c,uint64_t d)751 _x86_64_call4(void *fn, uint64_t a, uint64_t b, uint64_t c, uint64_t d)
752 {
753 struct fpu_cc_ent *ent;
754 uint64_t ret;
755
756 if ((ent = request_fpu_cc_ent()) == NULL)
757 return (ENOMEM);
758 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL);
759 ret = x86_64_call4(fn, a, b, c, d);
760 fpu_kern_leave(curthread, ent->ctx);
761 release_fpu_cc_ent(ent);
762
763 return (ret);
764 }
765
766 uint64_t
_x86_64_call5(void * fn,uint64_t a,uint64_t b,uint64_t c,uint64_t d,uint64_t e)767 _x86_64_call5(void *fn, uint64_t a, uint64_t b, uint64_t c, uint64_t d,
768 uint64_t e)
769 {
770 struct fpu_cc_ent *ent;
771 uint64_t ret;
772
773 if ((ent = request_fpu_cc_ent()) == NULL)
774 return (ENOMEM);
775 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL);
776 ret = x86_64_call5(fn, a, b, c, d, e);
777 fpu_kern_leave(curthread, ent->ctx);
778 release_fpu_cc_ent(ent);
779
780 return (ret);
781 }
782
783 uint64_t
_x86_64_call6(void * fn,uint64_t a,uint64_t b,uint64_t c,uint64_t d,uint64_t e,uint64_t f)784 _x86_64_call6(void *fn, uint64_t a, uint64_t b, uint64_t c, uint64_t d,
785 uint64_t e, uint64_t f)
786 {
787 struct fpu_cc_ent *ent;
788 uint64_t ret;
789
790 if ((ent = request_fpu_cc_ent()) == NULL)
791 return (ENOMEM);
792 fpu_kern_enter(curthread, ent->ctx, FPU_KERN_NORMAL);
793 ret = x86_64_call6(fn, a, b, c, d, e, f);
794 fpu_kern_leave(curthread, ent->ctx);
795 release_fpu_cc_ent(ent);
796
797 return (ret);
798 }
799 #endif /* __amd64__ */
800
801
802 #ifdef __i386__
803
804 struct x86desc {
805 uint16_t x_lolimit;
806 uint16_t x_base0;
807 uint8_t x_base1;
808 uint8_t x_flags;
809 uint8_t x_hilimit;
810 uint8_t x_base2;
811 };
812
813 struct gdt {
814 uint16_t limit;
815 void *base;
816 } __attribute__((__packed__));
817
818 extern uint16_t x86_getfs(void);
819 extern void x86_setfs(uint16_t);
820 extern void *x86_gettid(void);
821 extern void x86_critical_enter(void);
822 extern void x86_critical_exit(void);
823 extern void x86_getldt(struct gdt *, uint16_t *);
824 extern void x86_setldt(struct gdt *, uint16_t);
825
826 #define SEL_LDT 4 /* local descriptor table */
827 #define SEL_TO_FS(x) (((x) << 3))
828
829 /*
830 * FreeBSD 6.0 and later has a special GDT segment reserved
831 * specifically for us, so if GNDIS_SEL is defined, use that.
832 * If not, use GTGATE_SEL, which is uninitialized and infrequently
833 * used.
834 */
835
836 #ifdef GNDIS_SEL
837 #define FREEBSD_EMPTYSEL GNDIS_SEL
838 #else
839 #define FREEBSD_EMPTYSEL GTGATE_SEL /* slot 7 */
840 #endif
841
842 /*
843 * The meanings of various bits in a descriptor vary a little
844 * depending on whether the descriptor will be used as a
845 * code, data or system descriptor. (And that in turn depends
846 * on which segment register selects the descriptor.)
847 * We're only trying to create a data segment, so the definitions
848 * below are the ones that apply to a data descriptor.
849 */
850
851 #define SEGFLAGLO_PRESENT 0x80 /* segment is present */
852 #define SEGFLAGLO_PRIVLVL 0x60 /* privlevel needed for this seg */
853 #define SEGFLAGLO_CD 0x10 /* 1 = code/data, 0 = system */
854 #define SEGFLAGLO_MBZ 0x08 /* must be zero */
855 #define SEGFLAGLO_EXPANDDOWN 0x04 /* limit expands down */
856 #define SEGFLAGLO_WRITEABLE 0x02 /* segment is writeable */
857 #define SEGGLAGLO_ACCESSED 0x01 /* segment has been accessed */
858
859 #define SEGFLAGHI_GRAN 0x80 /* granularity, 1 = byte, 0 = page */
860 #define SEGFLAGHI_BIG 0x40 /* 1 = 32 bit stack, 0 = 16 bit */
861
862 /*
863 * Context switch from UNIX to Windows. Save the existing value
864 * of %fs for this processor, then change it to point to our
865 * fake TID. Note that it is also possible to pin ourselves
866 * to our current CPU, though I'm not sure this is really
867 * necessary. It depends on whether or not an interrupt might
868 * preempt us while Windows code is running and we wind up
869 * scheduled onto another CPU as a result. So far, it doesn't
870 * seem like this is what happens.
871 */
872
873 void
ctxsw_utow(void)874 ctxsw_utow(void)
875 {
876 struct tid *t;
877
878 t = &my_tids[curthread->td_oncpu];
879
880 /*
881 * Ugly hack. During system bootstrap (cold == 1), only CPU 0
882 * is running. So if we were loaded at bootstrap, only CPU 0
883 * will have our special GDT entry. This is a problem for SMP
884 * systems, so to deal with this, we check here to make sure
885 * the TID for this processor has been initialized, and if it
886 * hasn't, we need to do it right now or else things will
887 * explode.
888 */
889
890 if (t->tid_self != t)
891 x86_newldt(NULL);
892
893 x86_critical_enter();
894 t->tid_oldfs = x86_getfs();
895 t->tid_cpu = curthread->td_oncpu;
896 sched_pin();
897 x86_setfs(SEL_TO_FS(t->tid_selector));
898 x86_critical_exit();
899
900 /* Now entering Windows land, population: you. */
901 }
902
903 /*
904 * Context switch from Windows back to UNIX. Restore %fs to
905 * its previous value. This always occurs after a call to
906 * ctxsw_utow().
907 */
908
909 void
ctxsw_wtou(void)910 ctxsw_wtou(void)
911 {
912 struct tid *t;
913
914 x86_critical_enter();
915 t = x86_gettid();
916 x86_setfs(t->tid_oldfs);
917 sched_unpin();
918 x86_critical_exit();
919
920 /* Welcome back to UNIX land, we missed you. */
921
922 #ifdef EXTRA_SANITY
923 if (t->tid_cpu != curthread->td_oncpu)
924 panic("ctxsw GOT MOVED TO OTHER CPU!");
925 #endif
926 }
927
928 static int windrv_wrap_stdcall(funcptr, funcptr *, int);
929 static int windrv_wrap_fastcall(funcptr, funcptr *, int);
930 static int windrv_wrap_regparm(funcptr, funcptr *);
931
932 extern void x86_fastcall_wrap(void);
933 extern void x86_fastcall_wrap_call(void);
934 extern void x86_fastcall_wrap_arg(void);
935 extern void x86_fastcall_wrap_end(void);
936
937 static int
windrv_wrap_fastcall(func,wrap,argcnt)938 windrv_wrap_fastcall(func, wrap, argcnt)
939 funcptr func;
940 funcptr *wrap;
941 int8_t argcnt;
942 {
943 funcptr p;
944 vm_offset_t *calladdr;
945 uint8_t *argaddr;
946 vm_offset_t wrapstart, wrapend, wrapcall, wraparg;
947
948 wrapstart = (vm_offset_t)&x86_fastcall_wrap;
949 wrapend = (vm_offset_t)&x86_fastcall_wrap_end;
950 wrapcall = (vm_offset_t)&x86_fastcall_wrap_call;
951 wraparg = (vm_offset_t)&x86_fastcall_wrap_arg;
952
953 /* Allocate a new wrapper instance. */
954
955 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT);
956 if (p == NULL)
957 return (ENOMEM);
958
959 /* Copy over the code. */
960
961 bcopy((char *)wrapstart, p, (wrapend - wrapstart));
962
963 /* Insert the function address into the new wrapper instance. */
964
965 calladdr = (vm_offset_t *)((char *)p + ((wrapcall - wrapstart) + 1));
966 *calladdr = (vm_offset_t)func;
967
968 argcnt -= 2;
969 if (argcnt < 1)
970 argcnt = 0;
971
972 argaddr = (u_int8_t *)((char *)p + ((wraparg - wrapstart) + 1));
973 *argaddr = argcnt * sizeof(uint32_t);
974
975 *wrap = p;
976
977 return (0);
978 }
979
980 extern void x86_stdcall_wrap(void);
981 extern void x86_stdcall_wrap_call(void);
982 extern void x86_stdcall_wrap_arg(void);
983 extern void x86_stdcall_wrap_end(void);
984
985 static int
windrv_wrap_stdcall(func,wrap,argcnt)986 windrv_wrap_stdcall(func, wrap, argcnt)
987 funcptr func;
988 funcptr *wrap;
989 uint8_t argcnt;
990 {
991 funcptr p;
992 vm_offset_t *calladdr;
993 uint8_t *argaddr;
994 vm_offset_t wrapstart, wrapend, wrapcall, wraparg;
995
996 wrapstart = (vm_offset_t)&x86_stdcall_wrap;
997 wrapend = (vm_offset_t)&x86_stdcall_wrap_end;
998 wrapcall = (vm_offset_t)&x86_stdcall_wrap_call;
999 wraparg = (vm_offset_t)&x86_stdcall_wrap_arg;
1000
1001 /* Allocate a new wrapper instance. */
1002
1003 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT);
1004 if (p == NULL)
1005 return (ENOMEM);
1006
1007 /* Copy over the code. */
1008
1009 bcopy((char *)wrapstart, p, (wrapend - wrapstart));
1010
1011 /* Insert the function address into the new wrapper instance. */
1012
1013 calladdr = (vm_offset_t *)((char *)p + ((wrapcall - wrapstart) + 1));
1014 *calladdr = (vm_offset_t)func;
1015
1016 argaddr = (u_int8_t *)((char *)p + ((wraparg - wrapstart) + 1));
1017 *argaddr = argcnt * sizeof(uint32_t);
1018
1019 *wrap = p;
1020
1021 return (0);
1022 }
1023
1024 extern void x86_regparm_wrap(void);
1025 extern void x86_regparm_wrap_call(void);
1026 extern void x86_regparm_wrap_end(void);
1027
1028 static int
windrv_wrap_regparm(func,wrap)1029 windrv_wrap_regparm(func, wrap)
1030 funcptr func;
1031 funcptr *wrap;
1032 {
1033 funcptr p;
1034 vm_offset_t *calladdr;
1035 vm_offset_t wrapstart, wrapend, wrapcall;
1036
1037 wrapstart = (vm_offset_t)&x86_regparm_wrap;
1038 wrapend = (vm_offset_t)&x86_regparm_wrap_end;
1039 wrapcall = (vm_offset_t)&x86_regparm_wrap_call;
1040
1041 /* Allocate a new wrapper instance. */
1042
1043 p = malloc((wrapend - wrapstart), M_DEVBUF, M_NOWAIT);
1044 if (p == NULL)
1045 return (ENOMEM);
1046
1047 /* Copy over the code. */
1048
1049 bcopy(x86_regparm_wrap, p, (wrapend - wrapstart));
1050
1051 /* Insert the function address into the new wrapper instance. */
1052
1053 calladdr = (vm_offset_t *)((char *)p + ((wrapcall - wrapstart) + 1));
1054 *calladdr = (vm_offset_t)func;
1055
1056 *wrap = p;
1057
1058 return (0);
1059 }
1060
1061 int
windrv_wrap(func,wrap,argcnt,ftype)1062 windrv_wrap(func, wrap, argcnt, ftype)
1063 funcptr func;
1064 funcptr *wrap;
1065 int argcnt;
1066 int ftype;
1067 {
1068 switch(ftype) {
1069 case WINDRV_WRAP_FASTCALL:
1070 return (windrv_wrap_fastcall(func, wrap, argcnt));
1071 case WINDRV_WRAP_STDCALL:
1072 return (windrv_wrap_stdcall(func, wrap, argcnt));
1073 case WINDRV_WRAP_REGPARM:
1074 return (windrv_wrap_regparm(func, wrap));
1075 case WINDRV_WRAP_CDECL:
1076 return (windrv_wrap_stdcall(func, wrap, 0));
1077 default:
1078 break;
1079 }
1080
1081 return (EINVAL);
1082 }
1083
1084 static void
x86_oldldt(dummy)1085 x86_oldldt(dummy)
1086 void *dummy;
1087 {
1088 struct x86desc *gdt;
1089 struct gdt gtable;
1090 uint16_t ltable;
1091
1092 mtx_lock_spin(&dt_lock);
1093
1094 /* Grab location of existing GDT. */
1095
1096 x86_getldt(>able, <able);
1097
1098 /* Find the slot we updated. */
1099
1100 gdt = gtable.base;
1101 gdt += FREEBSD_EMPTYSEL;
1102
1103 /* Empty it out. */
1104
1105 bzero((char *)gdt, sizeof(struct x86desc));
1106
1107 /* Restore GDT. */
1108
1109 x86_setldt(>able, ltable);
1110
1111 mtx_unlock_spin(&dt_lock);
1112 }
1113
1114 static void
x86_newldt(dummy)1115 x86_newldt(dummy)
1116 void *dummy;
1117 {
1118 struct gdt gtable;
1119 uint16_t ltable;
1120 struct x86desc *l;
1121 struct thread *t;
1122
1123 t = curthread;
1124
1125 mtx_lock_spin(&dt_lock);
1126
1127 /* Grab location of existing GDT. */
1128
1129 x86_getldt(>able, <able);
1130
1131 /* Get pointer to the GDT table. */
1132
1133 l = gtable.base;
1134
1135 /* Get pointer to empty slot */
1136
1137 l += FREEBSD_EMPTYSEL;
1138
1139 /* Initialize TID for this CPU. */
1140
1141 my_tids[t->td_oncpu].tid_selector = FREEBSD_EMPTYSEL;
1142 my_tids[t->td_oncpu].tid_self = &my_tids[t->td_oncpu];
1143
1144 /* Set up new GDT entry. */
1145
1146 l->x_lolimit = sizeof(struct tid);
1147 l->x_hilimit = SEGFLAGHI_GRAN|SEGFLAGHI_BIG;
1148 l->x_base0 = (vm_offset_t)(&my_tids[t->td_oncpu]) & 0xFFFF;
1149 l->x_base1 = ((vm_offset_t)(&my_tids[t->td_oncpu]) >> 16) & 0xFF;
1150 l->x_base2 = ((vm_offset_t)(&my_tids[t->td_oncpu]) >> 24) & 0xFF;
1151 l->x_flags = SEGFLAGLO_PRESENT|SEGFLAGLO_CD|SEGFLAGLO_WRITEABLE;
1152
1153 /* Update the GDT. */
1154
1155 x86_setldt(>able, ltable);
1156
1157 mtx_unlock_spin(&dt_lock);
1158
1159 /* Whew. */
1160 }
1161
1162 #endif /* __i386__ */
1163
1164 int
windrv_unwrap(func)1165 windrv_unwrap(func)
1166 funcptr func;
1167 {
1168 free(func, M_DEVBUF);
1169
1170 return (0);
1171 }
1172