1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 2003
5 * Bill Paul <wpaul@windriver.com>. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by Bill Paul.
18 * 4. Neither the name of the author nor the names of any co-contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include <sys/cdefs.h>
36 #include <sys/ctype.h>
37 #include <sys/unistd.h>
38 #include <sys/param.h>
39 #include <sys/types.h>
40 #include <sys/errno.h>
41 #include <sys/systm.h>
42 #include <sys/malloc.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45
46 #include <sys/callout.h>
47 #include <sys/kdb.h>
48 #include <sys/kernel.h>
49 #include <sys/proc.h>
50 #include <sys/condvar.h>
51 #include <sys/kthread.h>
52 #include <sys/module.h>
53 #include <sys/smp.h>
54 #include <sys/sched.h>
55 #include <sys/sysctl.h>
56
57 #include <machine/atomic.h>
58 #include <machine/bus.h>
59 #include <machine/stdarg.h>
60 #include <machine/resource.h>
61
62 #include <sys/bus.h>
63 #include <sys/rman.h>
64
65 #include <vm/vm.h>
66 #include <vm/vm_param.h>
67 #include <vm/pmap.h>
68 #include <vm/uma.h>
69 #include <vm/vm_kern.h>
70 #include <vm/vm_map.h>
71 #include <vm/vm_extern.h>
72
73 #include <compat/ndis/pe_var.h>
74 #include <compat/ndis/cfg_var.h>
75 #include <compat/ndis/resource_var.h>
76 #include <compat/ndis/ntoskrnl_var.h>
77 #include <compat/ndis/hal_var.h>
78 #include <compat/ndis/ndis_var.h>
79
80 #ifdef NTOSKRNL_DEBUG_TIMERS
81 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
82
83 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers,
84 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
85 sysctl_show_timers, "I",
86 "Show ntoskrnl timer stats");
87 #endif
88
89 struct kdpc_queue {
90 list_entry kq_disp;
91 struct thread *kq_td;
92 int kq_cpu;
93 int kq_exit;
94 int kq_running;
95 kspin_lock kq_lock;
96 nt_kevent kq_proc;
97 nt_kevent kq_done;
98 };
99
100 typedef struct kdpc_queue kdpc_queue;
101
102 struct wb_ext {
103 struct cv we_cv;
104 struct thread *we_td;
105 };
106
107 typedef struct wb_ext wb_ext;
108
109 #define NTOSKRNL_TIMEOUTS 256
110 #ifdef NTOSKRNL_DEBUG_TIMERS
111 static uint64_t ntoskrnl_timer_fires;
112 static uint64_t ntoskrnl_timer_sets;
113 static uint64_t ntoskrnl_timer_reloads;
114 static uint64_t ntoskrnl_timer_cancels;
115 #endif
116
117 struct callout_entry {
118 struct callout ce_callout;
119 list_entry ce_list;
120 };
121
122 typedef struct callout_entry callout_entry;
123
124 static struct list_entry ntoskrnl_calllist;
125 static struct mtx ntoskrnl_calllock;
126 struct kuser_shared_data kuser_shared_data;
127
128 static struct list_entry ntoskrnl_intlist;
129 static kspin_lock ntoskrnl_intlock;
130
131 static uint8_t RtlEqualUnicodeString(unicode_string *,
132 unicode_string *, uint8_t);
133 static void RtlCopyString(ansi_string *, const ansi_string *);
134 static void RtlCopyUnicodeString(unicode_string *,
135 unicode_string *);
136 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
137 void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
138 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
139 device_object *, void *, uint32_t, uint64_t *, io_status_block *);
140 static irp *IoBuildDeviceIoControlRequest(uint32_t,
141 device_object *, void *, uint32_t, void *, uint32_t,
142 uint8_t, nt_kevent *, io_status_block *);
143 static irp *IoAllocateIrp(uint8_t, uint8_t);
144 static void IoReuseIrp(irp *, uint32_t);
145 static void IoFreeIrp(irp *);
146 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
147 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
148 static uint32_t KeWaitForMultipleObjects(uint32_t,
149 nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
150 int64_t *, wait_block *);
151 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
152 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
153 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
154 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
155 static void ntoskrnl_insert_timer(ktimer *, int);
156 static void ntoskrnl_remove_timer(ktimer *);
157 #ifdef NTOSKRNL_DEBUG_TIMERS
158 static void ntoskrnl_show_timers(void);
159 #endif
160 static void ntoskrnl_timercall(void *);
161 static void ntoskrnl_dpc_thread(void *);
162 static void ntoskrnl_destroy_dpc_threads(void);
163 static void ntoskrnl_destroy_workitem_threads(void);
164 static void ntoskrnl_workitem_thread(void *);
165 static void ntoskrnl_workitem(device_object *, void *);
166 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
167 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
168 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
169 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
170 static uint16_t READ_REGISTER_USHORT(uint16_t *);
171 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
172 static uint32_t READ_REGISTER_ULONG(uint32_t *);
173 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
174 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
175 static int64_t _allmul(int64_t, int64_t);
176 static int64_t _alldiv(int64_t, int64_t);
177 static int64_t _allrem(int64_t, int64_t);
178 static int64_t _allshr(int64_t, uint8_t);
179 static int64_t _allshl(int64_t, uint8_t);
180 static uint64_t _aullmul(uint64_t, uint64_t);
181 static uint64_t _aulldiv(uint64_t, uint64_t);
182 static uint64_t _aullrem(uint64_t, uint64_t);
183 static uint64_t _aullshr(uint64_t, uint8_t);
184 static uint64_t _aullshl(uint64_t, uint8_t);
185 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
186 static void InitializeSListHead(slist_header *);
187 static slist_entry *ntoskrnl_popsl(slist_header *);
188 static void ExFreePoolWithTag(void *, uint32_t);
189 static void ExInitializePagedLookasideList(paged_lookaside_list *,
190 lookaside_alloc_func *, lookaside_free_func *,
191 uint32_t, size_t, uint32_t, uint16_t);
192 static void ExDeletePagedLookasideList(paged_lookaside_list *);
193 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
194 lookaside_alloc_func *, lookaside_free_func *,
195 uint32_t, size_t, uint32_t, uint16_t);
196 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
197 static slist_entry
198 *ExInterlockedPushEntrySList(slist_header *,
199 slist_entry *, kspin_lock *);
200 static slist_entry
201 *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
202 static uint32_t InterlockedIncrement(volatile uint32_t *);
203 static uint32_t InterlockedDecrement(volatile uint32_t *);
204 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
205 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
206 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
207 uint64_t, uint64_t, uint64_t, enum nt_caching_type);
208 static void MmFreeContiguousMemory(void *);
209 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
210 enum nt_caching_type);
211 static uint32_t MmSizeOfMdl(void *, size_t);
212 static void *MmMapLockedPages(mdl *, uint8_t);
213 static void *MmMapLockedPagesSpecifyCache(mdl *,
214 uint8_t, uint32_t, void *, uint32_t, uint32_t);
215 static void MmUnmapLockedPages(void *, mdl *);
216 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
217 static void RtlZeroMemory(void *, size_t);
218 static void RtlSecureZeroMemory(void *, size_t);
219 static void RtlFillMemory(void *, size_t, uint8_t);
220 static void RtlMoveMemory(void *, const void *, size_t);
221 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
222 static void RtlCopyMemory(void *, const void *, size_t);
223 static size_t RtlCompareMemory(const void *, const void *, size_t);
224 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
225 uint32_t, uint32_t *);
226 static int atoi (const char *);
227 static long atol (const char *);
228 static int rand(void);
229 static void srand(unsigned int);
230 static void KeQuerySystemTime(uint64_t *);
231 static uint32_t KeTickCount(void);
232 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
233 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
234 uint32_t, void **);
235 static void ntoskrnl_thrfunc(void *);
236 static ndis_status PsCreateSystemThread(ndis_handle *,
237 uint32_t, void *, ndis_handle, void *, void *, void *);
238 static ndis_status PsTerminateSystemThread(ndis_status);
239 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
240 uint32_t, void *, device_object *);
241 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
242 uint32_t, void *, uint32_t *);
243 static void KeInitializeMutex(kmutant *, uint32_t);
244 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
245 static uint32_t KeReadStateMutex(kmutant *);
246 static ndis_status ObReferenceObjectByHandle(ndis_handle,
247 uint32_t, void *, uint8_t, void **, void **);
248 static void ObfDereferenceObject(void *);
249 static uint32_t ZwClose(ndis_handle);
250 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
251 uint32_t, void *);
252 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
253 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
254 static void *ntoskrnl_memset(void *, int, size_t);
255 static void *ntoskrnl_memmove(void *, void *, size_t);
256 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
257 static char *ntoskrnl_strstr(char *, char *);
258 static char *ntoskrnl_strncat(char *, char *, size_t);
259 static int ntoskrnl_toupper(int);
260 static int ntoskrnl_tolower(int);
261 static funcptr ntoskrnl_findwrap(funcptr);
262 static uint32_t DbgPrint(char *, ...);
263 static void DbgBreakPoint(void);
264 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
265 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
266 static int32_t KeSetPriorityThread(struct thread *, int32_t);
267 static void dummy(void);
268
269 static struct mtx ntoskrnl_dispatchlock;
270 static struct mtx ntoskrnl_interlock;
271 static kspin_lock ntoskrnl_cancellock;
272 static int ntoskrnl_kth = 0;
273 static struct nt_objref_head ntoskrnl_reflist;
274 static uma_zone_t mdl_zone;
275 static uma_zone_t iw_zone;
276 static struct kdpc_queue *kq_queues;
277 static struct kdpc_queue *wq_queues;
278 static int wq_idx = 0;
279
280 int
ntoskrnl_libinit()281 ntoskrnl_libinit()
282 {
283 image_patch_table *patch;
284 int error;
285 struct proc *p;
286 kdpc_queue *kq;
287 callout_entry *e;
288 int i;
289
290 mtx_init(&ntoskrnl_dispatchlock,
291 "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
292 mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
293 KeInitializeSpinLock(&ntoskrnl_cancellock);
294 KeInitializeSpinLock(&ntoskrnl_intlock);
295 TAILQ_INIT(&ntoskrnl_reflist);
296
297 InitializeListHead(&ntoskrnl_calllist);
298 InitializeListHead(&ntoskrnl_intlist);
299 mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
300
301 kq_queues = ExAllocatePoolWithTag(NonPagedPool,
302 #ifdef NTOSKRNL_MULTIPLE_DPCS
303 sizeof(kdpc_queue) * mp_ncpus, 0);
304 #else
305 sizeof(kdpc_queue), 0);
306 #endif
307
308 if (kq_queues == NULL)
309 return (ENOMEM);
310
311 wq_queues = ExAllocatePoolWithTag(NonPagedPool,
312 sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
313
314 if (wq_queues == NULL)
315 return (ENOMEM);
316
317 #ifdef NTOSKRNL_MULTIPLE_DPCS
318 bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
319 #else
320 bzero((char *)kq_queues, sizeof(kdpc_queue));
321 #endif
322 bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
323
324 /*
325 * Launch the DPC threads.
326 */
327
328 #ifdef NTOSKRNL_MULTIPLE_DPCS
329 for (i = 0; i < mp_ncpus; i++) {
330 #else
331 for (i = 0; i < 1; i++) {
332 #endif
333 kq = kq_queues + i;
334 kq->kq_cpu = i;
335 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
336 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
337 if (error)
338 panic("failed to launch DPC thread");
339 }
340
341 /*
342 * Launch the workitem threads.
343 */
344
345 for (i = 0; i < WORKITEM_THREADS; i++) {
346 kq = wq_queues + i;
347 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
348 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
349 if (error)
350 panic("failed to launch workitem thread");
351 }
352
353 patch = ntoskrnl_functbl;
354 while (patch->ipt_func != NULL) {
355 windrv_wrap((funcptr)patch->ipt_func,
356 (funcptr *)&patch->ipt_wrap,
357 patch->ipt_argcnt, patch->ipt_ftype);
358 patch++;
359 }
360
361 for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
362 e = ExAllocatePoolWithTag(NonPagedPool,
363 sizeof(callout_entry), 0);
364 if (e == NULL)
365 panic("failed to allocate timeouts");
366 mtx_lock_spin(&ntoskrnl_calllock);
367 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
368 mtx_unlock_spin(&ntoskrnl_calllock);
369 }
370
371 /*
372 * MDLs are supposed to be variable size (they describe
373 * buffers containing some number of pages, but we don't
374 * know ahead of time how many pages that will be). But
375 * always allocating them off the heap is very slow. As
376 * a compromise, we create an MDL UMA zone big enough to
377 * handle any buffer requiring up to 16 pages, and we
378 * use those for any MDLs for buffers of 16 pages or less
379 * in size. For buffers larger than that (which we assume
380 * will be few and far between, we allocate the MDLs off
381 * the heap.
382 */
383
384 mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
385 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
386
387 iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
388 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
389
390 return (0);
391 }
392
393 int
394 ntoskrnl_libfini()
395 {
396 image_patch_table *patch;
397 callout_entry *e;
398 list_entry *l;
399
400 patch = ntoskrnl_functbl;
401 while (patch->ipt_func != NULL) {
402 windrv_unwrap(patch->ipt_wrap);
403 patch++;
404 }
405
406 /* Stop the workitem queues. */
407 ntoskrnl_destroy_workitem_threads();
408 /* Stop the DPC queues. */
409 ntoskrnl_destroy_dpc_threads();
410
411 ExFreePool(kq_queues);
412 ExFreePool(wq_queues);
413
414 uma_zdestroy(mdl_zone);
415 uma_zdestroy(iw_zone);
416
417 mtx_lock_spin(&ntoskrnl_calllock);
418 while(!IsListEmpty(&ntoskrnl_calllist)) {
419 l = RemoveHeadList(&ntoskrnl_calllist);
420 e = CONTAINING_RECORD(l, callout_entry, ce_list);
421 mtx_unlock_spin(&ntoskrnl_calllock);
422 ExFreePool(e);
423 mtx_lock_spin(&ntoskrnl_calllock);
424 }
425 mtx_unlock_spin(&ntoskrnl_calllock);
426
427 mtx_destroy(&ntoskrnl_dispatchlock);
428 mtx_destroy(&ntoskrnl_interlock);
429 mtx_destroy(&ntoskrnl_calllock);
430
431 return (0);
432 }
433
434 /*
435 * We need to be able to reference this externally from the wrapper;
436 * GCC only generates a local implementation of memset.
437 */
438 static void *
439 ntoskrnl_memset(buf, ch, size)
440 void *buf;
441 int ch;
442 size_t size;
443 {
444 return (memset(buf, ch, size));
445 }
446
447 static void *
448 ntoskrnl_memmove(dst, src, size)
449 void *src;
450 void *dst;
451 size_t size;
452 {
453 bcopy(src, dst, size);
454 return (dst);
455 }
456
457 static void *
458 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
459 {
460 if (len != 0) {
461 unsigned char *p = buf;
462
463 do {
464 if (*p++ == ch)
465 return (p - 1);
466 } while (--len != 0);
467 }
468 return (NULL);
469 }
470
471 static char *
472 ntoskrnl_strstr(s, find)
473 char *s, *find;
474 {
475 char c, sc;
476 size_t len;
477
478 if ((c = *find++) != 0) {
479 len = strlen(find);
480 do {
481 do {
482 if ((sc = *s++) == 0)
483 return (NULL);
484 } while (sc != c);
485 } while (strncmp(s, find, len) != 0);
486 s--;
487 }
488 return ((char *)s);
489 }
490
491 /* Taken from libc */
492 static char *
493 ntoskrnl_strncat(dst, src, n)
494 char *dst;
495 char *src;
496 size_t n;
497 {
498 if (n != 0) {
499 char *d = dst;
500 const char *s = src;
501
502 while (*d != 0)
503 d++;
504 do {
505 if ((*d = *s++) == 0)
506 break;
507 d++;
508 } while (--n != 0);
509 *d = 0;
510 }
511 return (dst);
512 }
513
514 static int
515 ntoskrnl_toupper(c)
516 int c;
517 {
518 return (toupper(c));
519 }
520
521 static int
522 ntoskrnl_tolower(c)
523 int c;
524 {
525 return (tolower(c));
526 }
527
528 static uint8_t
529 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
530 uint8_t caseinsensitive)
531 {
532 int i;
533
534 if (str1->us_len != str2->us_len)
535 return (FALSE);
536
537 for (i = 0; i < str1->us_len; i++) {
538 if (caseinsensitive == TRUE) {
539 if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
540 toupper((char)(str2->us_buf[i] & 0xFF)))
541 return (FALSE);
542 } else {
543 if (str1->us_buf[i] != str2->us_buf[i])
544 return (FALSE);
545 }
546 }
547
548 return (TRUE);
549 }
550
551 static void
552 RtlCopyString(dst, src)
553 ansi_string *dst;
554 const ansi_string *src;
555 {
556 if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
557 dst->as_len = min(src->as_len, dst->as_maxlen);
558 memcpy(dst->as_buf, src->as_buf, dst->as_len);
559 if (dst->as_len < dst->as_maxlen)
560 dst->as_buf[dst->as_len] = 0;
561 } else
562 dst->as_len = 0;
563 }
564
565 static void
566 RtlCopyUnicodeString(dest, src)
567 unicode_string *dest;
568 unicode_string *src;
569 {
570
571 if (dest->us_maxlen >= src->us_len)
572 dest->us_len = src->us_len;
573 else
574 dest->us_len = dest->us_maxlen;
575 memcpy(dest->us_buf, src->us_buf, dest->us_len);
576 }
577
578 static void
579 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
580 char *ascii;
581 uint16_t *unicode;
582 int len;
583 {
584 int i;
585 uint16_t *ustr;
586
587 ustr = unicode;
588 for (i = 0; i < len; i++) {
589 *ustr = (uint16_t)ascii[i];
590 ustr++;
591 }
592 }
593
594 static void
595 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
596 uint16_t *unicode;
597 char *ascii;
598 int len;
599 {
600 int i;
601 uint8_t *astr;
602
603 astr = ascii;
604 for (i = 0; i < len / 2; i++) {
605 *astr = (uint8_t)unicode[i];
606 astr++;
607 }
608 }
609
610 uint32_t
611 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
612 {
613 if (dest == NULL || src == NULL)
614 return (STATUS_INVALID_PARAMETER);
615
616 dest->as_len = src->us_len / 2;
617 if (dest->as_maxlen < dest->as_len)
618 dest->as_len = dest->as_maxlen;
619
620 if (allocate == TRUE) {
621 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
622 (src->us_len / 2) + 1, 0);
623 if (dest->as_buf == NULL)
624 return (STATUS_INSUFFICIENT_RESOURCES);
625 dest->as_len = dest->as_maxlen = src->us_len / 2;
626 } else {
627 dest->as_len = src->us_len / 2; /* XXX */
628 if (dest->as_maxlen < dest->as_len)
629 dest->as_len = dest->as_maxlen;
630 }
631
632 ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
633 dest->as_len * 2);
634
635 return (STATUS_SUCCESS);
636 }
637
638 uint32_t
639 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
640 uint8_t allocate)
641 {
642 if (dest == NULL || src == NULL)
643 return (STATUS_INVALID_PARAMETER);
644
645 if (allocate == TRUE) {
646 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
647 src->as_len * 2, 0);
648 if (dest->us_buf == NULL)
649 return (STATUS_INSUFFICIENT_RESOURCES);
650 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
651 } else {
652 dest->us_len = src->as_len * 2; /* XXX */
653 if (dest->us_maxlen < dest->us_len)
654 dest->us_len = dest->us_maxlen;
655 }
656
657 ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
658 dest->us_len / 2);
659
660 return (STATUS_SUCCESS);
661 }
662
663 void *
664 ExAllocatePoolWithTag(pooltype, len, tag)
665 uint32_t pooltype;
666 size_t len;
667 uint32_t tag;
668 {
669 void *buf;
670
671 buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
672 if (buf == NULL)
673 return (NULL);
674
675 return (buf);
676 }
677
678 static void
679 ExFreePoolWithTag(buf, tag)
680 void *buf;
681 uint32_t tag;
682 {
683 ExFreePool(buf);
684 }
685
686 void
687 ExFreePool(buf)
688 void *buf;
689 {
690 free(buf, M_DEVBUF);
691 }
692
693 uint32_t
694 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
695 driver_object *drv;
696 void *clid;
697 uint32_t extlen;
698 void **ext;
699 {
700 custom_extension *ce;
701
702 ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
703 + extlen, 0);
704
705 if (ce == NULL)
706 return (STATUS_INSUFFICIENT_RESOURCES);
707
708 ce->ce_clid = clid;
709 InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
710
711 *ext = (void *)(ce + 1);
712
713 return (STATUS_SUCCESS);
714 }
715
716 void *
717 IoGetDriverObjectExtension(drv, clid)
718 driver_object *drv;
719 void *clid;
720 {
721 list_entry *e;
722 custom_extension *ce;
723
724 /*
725 * Sanity check. Our dummy bus drivers don't have
726 * any driver extensions.
727 */
728
729 if (drv->dro_driverext == NULL)
730 return (NULL);
731
732 e = drv->dro_driverext->dre_usrext.nle_flink;
733 while (e != &drv->dro_driverext->dre_usrext) {
734 ce = (custom_extension *)e;
735 if (ce->ce_clid == clid)
736 return ((void *)(ce + 1));
737 e = e->nle_flink;
738 }
739
740 return (NULL);
741 }
742
743 uint32_t
744 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
745 uint32_t devtype, uint32_t devchars, uint8_t exclusive,
746 device_object **newdev)
747 {
748 device_object *dev;
749
750 dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
751 if (dev == NULL)
752 return (STATUS_INSUFFICIENT_RESOURCES);
753
754 dev->do_type = devtype;
755 dev->do_drvobj = drv;
756 dev->do_currirp = NULL;
757 dev->do_flags = 0;
758
759 if (devextlen) {
760 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
761 devextlen, 0);
762
763 if (dev->do_devext == NULL) {
764 ExFreePool(dev);
765 return (STATUS_INSUFFICIENT_RESOURCES);
766 }
767
768 bzero(dev->do_devext, devextlen);
769 } else
770 dev->do_devext = NULL;
771
772 dev->do_size = sizeof(device_object) + devextlen;
773 dev->do_refcnt = 1;
774 dev->do_attacheddev = NULL;
775 dev->do_nextdev = NULL;
776 dev->do_devtype = devtype;
777 dev->do_stacksize = 1;
778 dev->do_alignreq = 1;
779 dev->do_characteristics = devchars;
780 dev->do_iotimer = NULL;
781 KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
782
783 /*
784 * Vpd is used for disk/tape devices,
785 * but we don't support those. (Yet.)
786 */
787 dev->do_vpb = NULL;
788
789 dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
790 sizeof(devobj_extension), 0);
791
792 if (dev->do_devobj_ext == NULL) {
793 if (dev->do_devext != NULL)
794 ExFreePool(dev->do_devext);
795 ExFreePool(dev);
796 return (STATUS_INSUFFICIENT_RESOURCES);
797 }
798
799 dev->do_devobj_ext->dve_type = 0;
800 dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
801 dev->do_devobj_ext->dve_devobj = dev;
802
803 /*
804 * Attach this device to the driver object's list
805 * of devices. Note: this is not the same as attaching
806 * the device to the device stack. The driver's AddDevice
807 * routine must explicitly call IoAddDeviceToDeviceStack()
808 * to do that.
809 */
810
811 if (drv->dro_devobj == NULL) {
812 drv->dro_devobj = dev;
813 dev->do_nextdev = NULL;
814 } else {
815 dev->do_nextdev = drv->dro_devobj;
816 drv->dro_devobj = dev;
817 }
818
819 *newdev = dev;
820
821 return (STATUS_SUCCESS);
822 }
823
824 void
825 IoDeleteDevice(dev)
826 device_object *dev;
827 {
828 device_object *prev;
829
830 if (dev == NULL)
831 return;
832
833 if (dev->do_devobj_ext != NULL)
834 ExFreePool(dev->do_devobj_ext);
835
836 if (dev->do_devext != NULL)
837 ExFreePool(dev->do_devext);
838
839 /* Unlink the device from the driver's device list. */
840
841 prev = dev->do_drvobj->dro_devobj;
842 if (prev == dev)
843 dev->do_drvobj->dro_devobj = dev->do_nextdev;
844 else {
845 while (prev->do_nextdev != dev)
846 prev = prev->do_nextdev;
847 prev->do_nextdev = dev->do_nextdev;
848 }
849
850 ExFreePool(dev);
851 }
852
853 device_object *
854 IoGetAttachedDevice(dev)
855 device_object *dev;
856 {
857 device_object *d;
858
859 if (dev == NULL)
860 return (NULL);
861
862 d = dev;
863
864 while (d->do_attacheddev != NULL)
865 d = d->do_attacheddev;
866
867 return (d);
868 }
869
870 static irp *
871 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
872 uint32_t func;
873 device_object *dobj;
874 void *buf;
875 uint32_t len;
876 uint64_t *off;
877 nt_kevent *event;
878 io_status_block *status;
879 {
880 irp *ip;
881
882 ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
883 if (ip == NULL)
884 return (NULL);
885 ip->irp_usrevent = event;
886
887 return (ip);
888 }
889
890 static irp *
891 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
892 uint32_t func;
893 device_object *dobj;
894 void *buf;
895 uint32_t len;
896 uint64_t *off;
897 io_status_block *status;
898 {
899 irp *ip;
900 io_stack_location *sl;
901
902 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
903 if (ip == NULL)
904 return (NULL);
905
906 ip->irp_usriostat = status;
907 ip->irp_tail.irp_overlay.irp_thread = NULL;
908
909 sl = IoGetNextIrpStackLocation(ip);
910 sl->isl_major = func;
911 sl->isl_minor = 0;
912 sl->isl_flags = 0;
913 sl->isl_ctl = 0;
914 sl->isl_devobj = dobj;
915 sl->isl_fileobj = NULL;
916 sl->isl_completionfunc = NULL;
917
918 ip->irp_userbuf = buf;
919
920 if (dobj->do_flags & DO_BUFFERED_IO) {
921 ip->irp_assoc.irp_sysbuf =
922 ExAllocatePoolWithTag(NonPagedPool, len, 0);
923 if (ip->irp_assoc.irp_sysbuf == NULL) {
924 IoFreeIrp(ip);
925 return (NULL);
926 }
927 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
928 }
929
930 if (dobj->do_flags & DO_DIRECT_IO) {
931 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
932 if (ip->irp_mdl == NULL) {
933 if (ip->irp_assoc.irp_sysbuf != NULL)
934 ExFreePool(ip->irp_assoc.irp_sysbuf);
935 IoFreeIrp(ip);
936 return (NULL);
937 }
938 ip->irp_userbuf = NULL;
939 ip->irp_assoc.irp_sysbuf = NULL;
940 }
941
942 if (func == IRP_MJ_READ) {
943 sl->isl_parameters.isl_read.isl_len = len;
944 if (off != NULL)
945 sl->isl_parameters.isl_read.isl_byteoff = *off;
946 else
947 sl->isl_parameters.isl_read.isl_byteoff = 0;
948 }
949
950 if (func == IRP_MJ_WRITE) {
951 sl->isl_parameters.isl_write.isl_len = len;
952 if (off != NULL)
953 sl->isl_parameters.isl_write.isl_byteoff = *off;
954 else
955 sl->isl_parameters.isl_write.isl_byteoff = 0;
956 }
957
958 return (ip);
959 }
960
961 static irp *
962 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
963 uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
964 nt_kevent *event, io_status_block *status)
965 {
966 irp *ip;
967 io_stack_location *sl;
968 uint32_t buflen;
969
970 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
971 if (ip == NULL)
972 return (NULL);
973 ip->irp_usrevent = event;
974 ip->irp_usriostat = status;
975 ip->irp_tail.irp_overlay.irp_thread = NULL;
976
977 sl = IoGetNextIrpStackLocation(ip);
978 sl->isl_major = isinternal == TRUE ?
979 IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
980 sl->isl_minor = 0;
981 sl->isl_flags = 0;
982 sl->isl_ctl = 0;
983 sl->isl_devobj = dobj;
984 sl->isl_fileobj = NULL;
985 sl->isl_completionfunc = NULL;
986 sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
987 sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
988 sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
989
990 switch(IO_METHOD(iocode)) {
991 case METHOD_BUFFERED:
992 if (ilen > olen)
993 buflen = ilen;
994 else
995 buflen = olen;
996 if (buflen) {
997 ip->irp_assoc.irp_sysbuf =
998 ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
999 if (ip->irp_assoc.irp_sysbuf == NULL) {
1000 IoFreeIrp(ip);
1001 return (NULL);
1002 }
1003 }
1004 if (ilen && ibuf != NULL) {
1005 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1006 bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1007 buflen - ilen);
1008 } else
1009 bzero(ip->irp_assoc.irp_sysbuf, ilen);
1010 ip->irp_userbuf = obuf;
1011 break;
1012 case METHOD_IN_DIRECT:
1013 case METHOD_OUT_DIRECT:
1014 if (ilen && ibuf != NULL) {
1015 ip->irp_assoc.irp_sysbuf =
1016 ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1017 if (ip->irp_assoc.irp_sysbuf == NULL) {
1018 IoFreeIrp(ip);
1019 return (NULL);
1020 }
1021 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1022 }
1023 if (olen && obuf != NULL) {
1024 ip->irp_mdl = IoAllocateMdl(obuf, olen,
1025 FALSE, FALSE, ip);
1026 /*
1027 * Normally we would MmProbeAndLockPages()
1028 * here, but we don't have to in our
1029 * imlementation.
1030 */
1031 }
1032 break;
1033 case METHOD_NEITHER:
1034 ip->irp_userbuf = obuf;
1035 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1036 break;
1037 default:
1038 break;
1039 }
1040
1041 /*
1042 * Ideally, we should associate this IRP with the calling
1043 * thread here.
1044 */
1045
1046 return (ip);
1047 }
1048
1049 static irp *
1050 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1051 {
1052 irp *i;
1053
1054 i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1055 if (i == NULL)
1056 return (NULL);
1057
1058 IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1059
1060 return (i);
1061 }
1062
1063 static irp *
1064 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1065 {
1066 irp *associrp;
1067
1068 associrp = IoAllocateIrp(stsize, FALSE);
1069 if (associrp == NULL)
1070 return (NULL);
1071
1072 mtx_lock(&ntoskrnl_dispatchlock);
1073 associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1074 associrp->irp_tail.irp_overlay.irp_thread =
1075 ip->irp_tail.irp_overlay.irp_thread;
1076 associrp->irp_assoc.irp_master = ip;
1077 mtx_unlock(&ntoskrnl_dispatchlock);
1078
1079 return (associrp);
1080 }
1081
1082 static void
1083 IoFreeIrp(ip)
1084 irp *ip;
1085 {
1086 ExFreePool(ip);
1087 }
1088
1089 static void
1090 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1091 {
1092 bzero((char *)io, IoSizeOfIrp(ssize));
1093 io->irp_size = psize;
1094 io->irp_stackcnt = ssize;
1095 io->irp_currentstackloc = ssize;
1096 InitializeListHead(&io->irp_thlist);
1097 io->irp_tail.irp_overlay.irp_csl =
1098 (io_stack_location *)(io + 1) + ssize;
1099 }
1100
1101 static void
1102 IoReuseIrp(ip, status)
1103 irp *ip;
1104 uint32_t status;
1105 {
1106 uint8_t allocflags;
1107
1108 allocflags = ip->irp_allocflags;
1109 IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1110 ip->irp_iostat.isb_status = status;
1111 ip->irp_allocflags = allocflags;
1112 }
1113
1114 void
1115 IoAcquireCancelSpinLock(uint8_t *irql)
1116 {
1117 KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1118 }
1119
1120 void
1121 IoReleaseCancelSpinLock(uint8_t irql)
1122 {
1123 KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1124 }
1125
1126 uint8_t
1127 IoCancelIrp(irp *ip)
1128 {
1129 cancel_func cfunc;
1130 uint8_t cancelirql;
1131
1132 IoAcquireCancelSpinLock(&cancelirql);
1133 cfunc = IoSetCancelRoutine(ip, NULL);
1134 ip->irp_cancel = TRUE;
1135 if (cfunc == NULL) {
1136 IoReleaseCancelSpinLock(cancelirql);
1137 return (FALSE);
1138 }
1139 ip->irp_cancelirql = cancelirql;
1140 MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1141 return (uint8_t)IoSetCancelValue(ip, TRUE);
1142 }
1143
1144 uint32_t
1145 IofCallDriver(dobj, ip)
1146 device_object *dobj;
1147 irp *ip;
1148 {
1149 driver_object *drvobj;
1150 io_stack_location *sl;
1151 uint32_t status;
1152 driver_dispatch disp;
1153
1154 drvobj = dobj->do_drvobj;
1155
1156 if (ip->irp_currentstackloc <= 0)
1157 panic("IoCallDriver(): out of stack locations");
1158
1159 IoSetNextIrpStackLocation(ip);
1160 sl = IoGetCurrentIrpStackLocation(ip);
1161
1162 sl->isl_devobj = dobj;
1163
1164 disp = drvobj->dro_dispatch[sl->isl_major];
1165 status = MSCALL2(disp, dobj, ip);
1166
1167 return (status);
1168 }
1169
1170 void
1171 IofCompleteRequest(irp *ip, uint8_t prioboost)
1172 {
1173 uint32_t status;
1174 device_object *dobj;
1175 io_stack_location *sl;
1176 completion_func cf;
1177
1178 KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1179 ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1180
1181 sl = IoGetCurrentIrpStackLocation(ip);
1182 IoSkipCurrentIrpStackLocation(ip);
1183
1184 do {
1185 if (sl->isl_ctl & SL_PENDING_RETURNED)
1186 ip->irp_pendingreturned = TRUE;
1187
1188 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1189 dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1190 else
1191 dobj = NULL;
1192
1193 if (sl->isl_completionfunc != NULL &&
1194 ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1195 sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1196 (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1197 sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1198 (ip->irp_cancel == TRUE &&
1199 sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1200 cf = sl->isl_completionfunc;
1201 status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1202 if (status == STATUS_MORE_PROCESSING_REQUIRED)
1203 return;
1204 } else {
1205 if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1206 (ip->irp_pendingreturned == TRUE))
1207 IoMarkIrpPending(ip);
1208 }
1209
1210 /* move to the next. */
1211 IoSkipCurrentIrpStackLocation(ip);
1212 sl++;
1213 } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1214
1215 if (ip->irp_usriostat != NULL)
1216 *ip->irp_usriostat = ip->irp_iostat;
1217 if (ip->irp_usrevent != NULL)
1218 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1219
1220 /* Handle any associated IRPs. */
1221
1222 if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1223 uint32_t masterirpcnt;
1224 irp *masterirp;
1225 mdl *m;
1226
1227 masterirp = ip->irp_assoc.irp_master;
1228 masterirpcnt =
1229 InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1230
1231 while ((m = ip->irp_mdl) != NULL) {
1232 ip->irp_mdl = m->mdl_next;
1233 IoFreeMdl(m);
1234 }
1235 IoFreeIrp(ip);
1236 if (masterirpcnt == 0)
1237 IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1238 return;
1239 }
1240
1241 /* With any luck, these conditions will never arise. */
1242
1243 if (ip->irp_flags & IRP_PAGING_IO) {
1244 if (ip->irp_mdl != NULL)
1245 IoFreeMdl(ip->irp_mdl);
1246 IoFreeIrp(ip);
1247 }
1248 }
1249
1250 void
1251 ntoskrnl_intr(arg)
1252 void *arg;
1253 {
1254 kinterrupt *iobj;
1255 uint8_t irql;
1256 uint8_t claimed;
1257 list_entry *l;
1258
1259 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1260 l = ntoskrnl_intlist.nle_flink;
1261 while (l != &ntoskrnl_intlist) {
1262 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1263 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1264 if (claimed == TRUE)
1265 break;
1266 l = l->nle_flink;
1267 }
1268 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1269 }
1270
1271 uint8_t
1272 KeAcquireInterruptSpinLock(iobj)
1273 kinterrupt *iobj;
1274 {
1275 uint8_t irql;
1276 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1277 return (irql);
1278 }
1279
1280 void
1281 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1282 {
1283 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1284 }
1285
1286 uint8_t
1287 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1288 kinterrupt *iobj;
1289 void *syncfunc;
1290 void *syncctx;
1291 {
1292 uint8_t irql;
1293
1294 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1295 MSCALL1(syncfunc, syncctx);
1296 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1297
1298 return (TRUE);
1299 }
1300
1301 /*
1302 * IoConnectInterrupt() is passed only the interrupt vector and
1303 * irql that a device wants to use, but no device-specific tag
1304 * of any kind. This conflicts rather badly with FreeBSD's
1305 * bus_setup_intr(), which needs the device_t for the device
1306 * requesting interrupt delivery. In order to bypass this
1307 * inconsistency, we implement a second level of interrupt
1308 * dispatching on top of bus_setup_intr(). All devices use
1309 * ntoskrnl_intr() as their ISR, and any device requesting
1310 * interrupts will be registered with ntoskrnl_intr()'s interrupt
1311 * dispatch list. When an interrupt arrives, we walk the list
1312 * and invoke all the registered ISRs. This effectively makes all
1313 * interrupts shared, but it's the only way to duplicate the
1314 * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1315 */
1316
1317 uint32_t
1318 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1319 kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1320 uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1321 {
1322 uint8_t curirql;
1323
1324 *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1325 if (*iobj == NULL)
1326 return (STATUS_INSUFFICIENT_RESOURCES);
1327
1328 (*iobj)->ki_svcfunc = svcfunc;
1329 (*iobj)->ki_svcctx = svcctx;
1330
1331 if (lock == NULL) {
1332 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1333 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1334 } else
1335 (*iobj)->ki_lock = lock;
1336
1337 KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1338 InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1339 KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1340
1341 return (STATUS_SUCCESS);
1342 }
1343
1344 void
1345 IoDisconnectInterrupt(iobj)
1346 kinterrupt *iobj;
1347 {
1348 uint8_t irql;
1349
1350 if (iobj == NULL)
1351 return;
1352
1353 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1354 RemoveEntryList((&iobj->ki_list));
1355 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1356
1357 ExFreePool(iobj);
1358 }
1359
1360 device_object *
1361 IoAttachDeviceToDeviceStack(src, dst)
1362 device_object *src;
1363 device_object *dst;
1364 {
1365 device_object *attached;
1366
1367 mtx_lock(&ntoskrnl_dispatchlock);
1368 attached = IoGetAttachedDevice(dst);
1369 attached->do_attacheddev = src;
1370 src->do_attacheddev = NULL;
1371 src->do_stacksize = attached->do_stacksize + 1;
1372 mtx_unlock(&ntoskrnl_dispatchlock);
1373
1374 return (attached);
1375 }
1376
1377 void
1378 IoDetachDevice(topdev)
1379 device_object *topdev;
1380 {
1381 device_object *tail;
1382
1383 mtx_lock(&ntoskrnl_dispatchlock);
1384
1385 /* First, break the chain. */
1386 tail = topdev->do_attacheddev;
1387 if (tail == NULL) {
1388 mtx_unlock(&ntoskrnl_dispatchlock);
1389 return;
1390 }
1391 topdev->do_attacheddev = tail->do_attacheddev;
1392 topdev->do_refcnt--;
1393
1394 /* Now reduce the stacksize count for the takm_il objects. */
1395
1396 tail = topdev->do_attacheddev;
1397 while (tail != NULL) {
1398 tail->do_stacksize--;
1399 tail = tail->do_attacheddev;
1400 }
1401
1402 mtx_unlock(&ntoskrnl_dispatchlock);
1403 }
1404
1405 /*
1406 * For the most part, an object is considered signalled if
1407 * dh_sigstate == TRUE. The exception is for mutant objects
1408 * (mutexes), where the logic works like this:
1409 *
1410 * - If the thread already owns the object and sigstate is
1411 * less than or equal to 0, then the object is considered
1412 * signalled (recursive acquisition).
1413 * - If dh_sigstate == 1, the object is also considered
1414 * signalled.
1415 */
1416
1417 static int
1418 ntoskrnl_is_signalled(obj, td)
1419 nt_dispatch_header *obj;
1420 struct thread *td;
1421 {
1422 kmutant *km;
1423
1424 if (obj->dh_type == DISP_TYPE_MUTANT) {
1425 km = (kmutant *)obj;
1426 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1427 obj->dh_sigstate == 1)
1428 return (TRUE);
1429 return (FALSE);
1430 }
1431
1432 if (obj->dh_sigstate > 0)
1433 return (TRUE);
1434 return (FALSE);
1435 }
1436
1437 static void
1438 ntoskrnl_satisfy_wait(obj, td)
1439 nt_dispatch_header *obj;
1440 struct thread *td;
1441 {
1442 kmutant *km;
1443
1444 switch (obj->dh_type) {
1445 case DISP_TYPE_MUTANT:
1446 km = (struct kmutant *)obj;
1447 obj->dh_sigstate--;
1448 /*
1449 * If sigstate reaches 0, the mutex is now
1450 * non-signalled (the new thread owns it).
1451 */
1452 if (obj->dh_sigstate == 0) {
1453 km->km_ownerthread = td;
1454 if (km->km_abandoned == TRUE)
1455 km->km_abandoned = FALSE;
1456 }
1457 break;
1458 /* Synchronization objects get reset to unsignalled. */
1459 case DISP_TYPE_SYNCHRONIZATION_EVENT:
1460 case DISP_TYPE_SYNCHRONIZATION_TIMER:
1461 obj->dh_sigstate = 0;
1462 break;
1463 case DISP_TYPE_SEMAPHORE:
1464 obj->dh_sigstate--;
1465 break;
1466 default:
1467 break;
1468 }
1469 }
1470
1471 static void
1472 ntoskrnl_satisfy_multiple_waits(wb)
1473 wait_block *wb;
1474 {
1475 wait_block *cur;
1476 struct thread *td;
1477
1478 cur = wb;
1479 td = wb->wb_kthread;
1480
1481 do {
1482 ntoskrnl_satisfy_wait(wb->wb_object, td);
1483 cur->wb_awakened = TRUE;
1484 cur = cur->wb_next;
1485 } while (cur != wb);
1486 }
1487
1488 /* Always called with dispatcher lock held. */
1489 static void
1490 ntoskrnl_waittest(obj, increment)
1491 nt_dispatch_header *obj;
1492 uint32_t increment;
1493 {
1494 wait_block *w, *next;
1495 list_entry *e;
1496 struct thread *td;
1497 wb_ext *we;
1498 int satisfied;
1499
1500 /*
1501 * Once an object has been signalled, we walk its list of
1502 * wait blocks. If a wait block can be awakened, then satisfy
1503 * waits as necessary and wake the thread.
1504 *
1505 * The rules work like this:
1506 *
1507 * If a wait block is marked as WAITTYPE_ANY, then
1508 * we can satisfy the wait conditions on the current
1509 * object and wake the thread right away. Satisfying
1510 * the wait also has the effect of breaking us out
1511 * of the search loop.
1512 *
1513 * If the object is marked as WAITTYLE_ALL, then the
1514 * wait block will be part of a circularly linked
1515 * list of wait blocks belonging to a waiting thread
1516 * that's sleeping in KeWaitForMultipleObjects(). In
1517 * order to wake the thread, all the objects in the
1518 * wait list must be in the signalled state. If they
1519 * are, we then satisfy all of them and wake the
1520 * thread.
1521 *
1522 */
1523
1524 e = obj->dh_waitlisthead.nle_flink;
1525
1526 while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1527 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1528 we = w->wb_ext;
1529 td = we->we_td;
1530 satisfied = FALSE;
1531 if (w->wb_waittype == WAITTYPE_ANY) {
1532 /*
1533 * Thread can be awakened if
1534 * any wait is satisfied.
1535 */
1536 ntoskrnl_satisfy_wait(obj, td);
1537 satisfied = TRUE;
1538 w->wb_awakened = TRUE;
1539 } else {
1540 /*
1541 * Thread can only be woken up
1542 * if all waits are satisfied.
1543 * If the thread is waiting on multiple
1544 * objects, they should all be linked
1545 * through the wb_next pointers in the
1546 * wait blocks.
1547 */
1548 satisfied = TRUE;
1549 next = w->wb_next;
1550 while (next != w) {
1551 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1552 satisfied = FALSE;
1553 break;
1554 }
1555 next = next->wb_next;
1556 }
1557 ntoskrnl_satisfy_multiple_waits(w);
1558 }
1559
1560 if (satisfied == TRUE)
1561 cv_broadcastpri(&we->we_cv,
1562 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1563 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1564
1565 e = e->nle_flink;
1566 }
1567 }
1568
1569 /*
1570 * Return the number of 100 nanosecond intervals since
1571 * January 1, 1601. (?!?!)
1572 */
1573 void
1574 ntoskrnl_time(tval)
1575 uint64_t *tval;
1576 {
1577 struct timespec ts;
1578
1579 nanotime(&ts);
1580 *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1581 11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1582 }
1583
1584 static void
1585 KeQuerySystemTime(current_time)
1586 uint64_t *current_time;
1587 {
1588 ntoskrnl_time(current_time);
1589 }
1590
1591 static uint32_t
1592 KeTickCount(void)
1593 {
1594 struct timeval tv;
1595 getmicrouptime(&tv);
1596 return tvtohz(&tv);
1597 }
1598
1599 /*
1600 * KeWaitForSingleObject() is a tricky beast, because it can be used
1601 * with several different object types: semaphores, timers, events,
1602 * mutexes and threads. Semaphores don't appear very often, but the
1603 * other object types are quite common. KeWaitForSingleObject() is
1604 * what's normally used to acquire a mutex, and it can be used to
1605 * wait for a thread termination.
1606 *
1607 * The Windows NDIS API is implemented in terms of Windows kernel
1608 * primitives, and some of the object manipulation is duplicated in
1609 * NDIS. For example, NDIS has timers and events, which are actually
1610 * Windows kevents and ktimers. Now, you're supposed to only use the
1611 * NDIS variants of these objects within the confines of the NDIS API,
1612 * but there are some naughty developers out there who will use
1613 * KeWaitForSingleObject() on NDIS timer and event objects, so we
1614 * have to support that as well. Conseqently, our NDIS timer and event
1615 * code has to be closely tied into our ntoskrnl timer and event code,
1616 * just as it is in Windows.
1617 *
1618 * KeWaitForSingleObject() may do different things for different kinds
1619 * of objects:
1620 *
1621 * - For events, we check if the event has been signalled. If the
1622 * event is already in the signalled state, we just return immediately,
1623 * otherwise we wait for it to be set to the signalled state by someone
1624 * else calling KeSetEvent(). Events can be either synchronization or
1625 * notification events.
1626 *
1627 * - For timers, if the timer has already fired and the timer is in
1628 * the signalled state, we just return, otherwise we wait on the
1629 * timer. Unlike an event, timers get signalled automatically when
1630 * they expire rather than someone having to trip them manually.
1631 * Timers initialized with KeInitializeTimer() are always notification
1632 * events: KeInitializeTimerEx() lets you initialize a timer as
1633 * either a notification or synchronization event.
1634 *
1635 * - For mutexes, we try to acquire the mutex and if we can't, we wait
1636 * on the mutex until it's available and then grab it. When a mutex is
1637 * released, it enters the signalled state, which wakes up one of the
1638 * threads waiting to acquire it. Mutexes are always synchronization
1639 * events.
1640 *
1641 * - For threads, the only thing we do is wait until the thread object
1642 * enters a signalled state, which occurs when the thread terminates.
1643 * Threads are always notification events.
1644 *
1645 * A notification event wakes up all threads waiting on an object. A
1646 * synchronization event wakes up just one. Also, a synchronization event
1647 * is auto-clearing, which means we automatically set the event back to
1648 * the non-signalled state once the wakeup is done.
1649 */
1650
1651 uint32_t
1652 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1653 uint8_t alertable, int64_t *duetime)
1654 {
1655 wait_block w;
1656 struct thread *td = curthread;
1657 struct timeval tv;
1658 int error = 0;
1659 uint64_t curtime;
1660 wb_ext we;
1661 nt_dispatch_header *obj;
1662
1663 obj = arg;
1664
1665 if (obj == NULL)
1666 return (STATUS_INVALID_PARAMETER);
1667
1668 mtx_lock(&ntoskrnl_dispatchlock);
1669
1670 cv_init(&we.we_cv, "KeWFS");
1671 we.we_td = td;
1672
1673 /*
1674 * Check to see if this object is already signalled,
1675 * and just return without waiting if it is.
1676 */
1677 if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1678 /* Sanity check the signal state value. */
1679 if (obj->dh_sigstate != INT32_MIN) {
1680 ntoskrnl_satisfy_wait(obj, curthread);
1681 mtx_unlock(&ntoskrnl_dispatchlock);
1682 return (STATUS_SUCCESS);
1683 } else {
1684 /*
1685 * There's a limit to how many times we can
1686 * recursively acquire a mutant. If we hit
1687 * the limit, something is very wrong.
1688 */
1689 if (obj->dh_type == DISP_TYPE_MUTANT) {
1690 mtx_unlock(&ntoskrnl_dispatchlock);
1691 panic("mutant limit exceeded");
1692 }
1693 }
1694 }
1695
1696 bzero((char *)&w, sizeof(wait_block));
1697 w.wb_object = obj;
1698 w.wb_ext = &we;
1699 w.wb_waittype = WAITTYPE_ANY;
1700 w.wb_next = &w;
1701 w.wb_waitkey = 0;
1702 w.wb_awakened = FALSE;
1703 w.wb_oldpri = td->td_priority;
1704
1705 InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1706
1707 /*
1708 * The timeout value is specified in 100 nanosecond units
1709 * and can be a positive or negative number. If it's positive,
1710 * then the duetime is absolute, and we need to convert it
1711 * to an absolute offset relative to now in order to use it.
1712 * If it's negative, then the duetime is relative and we
1713 * just have to convert the units.
1714 */
1715
1716 if (duetime != NULL) {
1717 if (*duetime < 0) {
1718 tv.tv_sec = - (*duetime) / 10000000;
1719 tv.tv_usec = (- (*duetime) / 10) -
1720 (tv.tv_sec * 1000000);
1721 } else {
1722 ntoskrnl_time(&curtime);
1723 if (*duetime < curtime)
1724 tv.tv_sec = tv.tv_usec = 0;
1725 else {
1726 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1727 tv.tv_usec = ((*duetime) - curtime) / 10 -
1728 (tv.tv_sec * 1000000);
1729 }
1730 }
1731 }
1732
1733 if (duetime == NULL)
1734 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1735 else
1736 error = cv_timedwait(&we.we_cv,
1737 &ntoskrnl_dispatchlock, tvtohz(&tv));
1738
1739 RemoveEntryList(&w.wb_waitlist);
1740
1741 cv_destroy(&we.we_cv);
1742
1743 /* We timed out. Leave the object alone and return status. */
1744
1745 if (error == EWOULDBLOCK) {
1746 mtx_unlock(&ntoskrnl_dispatchlock);
1747 return (STATUS_TIMEOUT);
1748 }
1749
1750 mtx_unlock(&ntoskrnl_dispatchlock);
1751
1752 return (STATUS_SUCCESS);
1753 /*
1754 return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1755 mode, alertable, duetime, &w));
1756 */
1757 }
1758
1759 static uint32_t
1760 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1761 uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1762 wait_block *wb_array)
1763 {
1764 struct thread *td = curthread;
1765 wait_block *whead, *w;
1766 wait_block _wb_array[MAX_WAIT_OBJECTS];
1767 nt_dispatch_header *cur;
1768 struct timeval tv;
1769 int i, wcnt = 0, error = 0;
1770 uint64_t curtime;
1771 struct timespec t1, t2;
1772 uint32_t status = STATUS_SUCCESS;
1773 wb_ext we;
1774
1775 if (cnt > MAX_WAIT_OBJECTS)
1776 return (STATUS_INVALID_PARAMETER);
1777 if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1778 return (STATUS_INVALID_PARAMETER);
1779
1780 mtx_lock(&ntoskrnl_dispatchlock);
1781
1782 cv_init(&we.we_cv, "KeWFM");
1783 we.we_td = td;
1784
1785 if (wb_array == NULL)
1786 whead = _wb_array;
1787 else
1788 whead = wb_array;
1789
1790 bzero((char *)whead, sizeof(wait_block) * cnt);
1791
1792 /* First pass: see if we can satisfy any waits immediately. */
1793
1794 wcnt = 0;
1795 w = whead;
1796
1797 for (i = 0; i < cnt; i++) {
1798 InsertTailList((&obj[i]->dh_waitlisthead),
1799 (&w->wb_waitlist));
1800 w->wb_ext = &we;
1801 w->wb_object = obj[i];
1802 w->wb_waittype = wtype;
1803 w->wb_waitkey = i;
1804 w->wb_awakened = FALSE;
1805 w->wb_oldpri = td->td_priority;
1806 w->wb_next = w + 1;
1807 w++;
1808 wcnt++;
1809 if (ntoskrnl_is_signalled(obj[i], td)) {
1810 /*
1811 * There's a limit to how many times
1812 * we can recursively acquire a mutant.
1813 * If we hit the limit, something
1814 * is very wrong.
1815 */
1816 if (obj[i]->dh_sigstate == INT32_MIN &&
1817 obj[i]->dh_type == DISP_TYPE_MUTANT) {
1818 mtx_unlock(&ntoskrnl_dispatchlock);
1819 panic("mutant limit exceeded");
1820 }
1821
1822 /*
1823 * If this is a WAITTYPE_ANY wait, then
1824 * satisfy the waited object and exit
1825 * right now.
1826 */
1827
1828 if (wtype == WAITTYPE_ANY) {
1829 ntoskrnl_satisfy_wait(obj[i], td);
1830 status = STATUS_WAIT_0 + i;
1831 goto wait_done;
1832 } else {
1833 w--;
1834 wcnt--;
1835 w->wb_object = NULL;
1836 RemoveEntryList(&w->wb_waitlist);
1837 }
1838 }
1839 }
1840
1841 /*
1842 * If this is a WAITTYPE_ALL wait and all objects are
1843 * already signalled, satisfy the waits and exit now.
1844 */
1845
1846 if (wtype == WAITTYPE_ALL && wcnt == 0) {
1847 for (i = 0; i < cnt; i++)
1848 ntoskrnl_satisfy_wait(obj[i], td);
1849 status = STATUS_SUCCESS;
1850 goto wait_done;
1851 }
1852
1853 /*
1854 * Create a circular waitblock list. The waitcount
1855 * must always be non-zero when we get here.
1856 */
1857
1858 (w - 1)->wb_next = whead;
1859
1860 /* Wait on any objects that aren't yet signalled. */
1861
1862 /* Calculate timeout, if any. */
1863
1864 if (duetime != NULL) {
1865 if (*duetime < 0) {
1866 tv.tv_sec = - (*duetime) / 10000000;
1867 tv.tv_usec = (- (*duetime) / 10) -
1868 (tv.tv_sec * 1000000);
1869 } else {
1870 ntoskrnl_time(&curtime);
1871 if (*duetime < curtime)
1872 tv.tv_sec = tv.tv_usec = 0;
1873 else {
1874 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1875 tv.tv_usec = ((*duetime) - curtime) / 10 -
1876 (tv.tv_sec * 1000000);
1877 }
1878 }
1879 }
1880
1881 while (wcnt) {
1882 nanotime(&t1);
1883
1884 if (duetime == NULL)
1885 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1886 else
1887 error = cv_timedwait(&we.we_cv,
1888 &ntoskrnl_dispatchlock, tvtohz(&tv));
1889
1890 /* Wait with timeout expired. */
1891
1892 if (error) {
1893 status = STATUS_TIMEOUT;
1894 goto wait_done;
1895 }
1896
1897 nanotime(&t2);
1898
1899 /* See what's been signalled. */
1900
1901 w = whead;
1902 do {
1903 cur = w->wb_object;
1904 if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1905 w->wb_awakened == TRUE) {
1906 /* Sanity check the signal state value. */
1907 if (cur->dh_sigstate == INT32_MIN &&
1908 cur->dh_type == DISP_TYPE_MUTANT) {
1909 mtx_unlock(&ntoskrnl_dispatchlock);
1910 panic("mutant limit exceeded");
1911 }
1912 wcnt--;
1913 if (wtype == WAITTYPE_ANY) {
1914 status = w->wb_waitkey &
1915 STATUS_WAIT_0;
1916 goto wait_done;
1917 }
1918 }
1919 w = w->wb_next;
1920 } while (w != whead);
1921
1922 /*
1923 * If all objects have been signalled, or if this
1924 * is a WAITTYPE_ANY wait and we were woke up by
1925 * someone, we can bail.
1926 */
1927
1928 if (wcnt == 0) {
1929 status = STATUS_SUCCESS;
1930 goto wait_done;
1931 }
1932
1933 /*
1934 * If this is WAITTYPE_ALL wait, and there's still
1935 * objects that haven't been signalled, deduct the
1936 * time that's elapsed so far from the timeout and
1937 * wait again (or continue waiting indefinitely if
1938 * there's no timeout).
1939 */
1940
1941 if (duetime != NULL) {
1942 tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1943 tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1944 }
1945 }
1946
1947 wait_done:
1948
1949 cv_destroy(&we.we_cv);
1950
1951 for (i = 0; i < cnt; i++) {
1952 if (whead[i].wb_object != NULL)
1953 RemoveEntryList(&whead[i].wb_waitlist);
1954 }
1955 mtx_unlock(&ntoskrnl_dispatchlock);
1956
1957 return (status);
1958 }
1959
1960 static void
1961 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1962 {
1963 bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1964 }
1965
1966 static uint16_t
1967 READ_REGISTER_USHORT(reg)
1968 uint16_t *reg;
1969 {
1970 return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1971 }
1972
1973 static void
1974 WRITE_REGISTER_ULONG(reg, val)
1975 uint32_t *reg;
1976 uint32_t val;
1977 {
1978 bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1979 }
1980
1981 static uint32_t
1982 READ_REGISTER_ULONG(reg)
1983 uint32_t *reg;
1984 {
1985 return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1986 }
1987
1988 static uint8_t
1989 READ_REGISTER_UCHAR(uint8_t *reg)
1990 {
1991 return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1992 }
1993
1994 static void
1995 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
1996 {
1997 bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1998 }
1999
2000 static int64_t
2001 _allmul(a, b)
2002 int64_t a;
2003 int64_t b;
2004 {
2005 return (a * b);
2006 }
2007
2008 static int64_t
2009 _alldiv(a, b)
2010 int64_t a;
2011 int64_t b;
2012 {
2013 return (a / b);
2014 }
2015
2016 static int64_t
2017 _allrem(a, b)
2018 int64_t a;
2019 int64_t b;
2020 {
2021 return (a % b);
2022 }
2023
2024 static uint64_t
2025 _aullmul(a, b)
2026 uint64_t a;
2027 uint64_t b;
2028 {
2029 return (a * b);
2030 }
2031
2032 static uint64_t
2033 _aulldiv(a, b)
2034 uint64_t a;
2035 uint64_t b;
2036 {
2037 return (a / b);
2038 }
2039
2040 static uint64_t
2041 _aullrem(a, b)
2042 uint64_t a;
2043 uint64_t b;
2044 {
2045 return (a % b);
2046 }
2047
2048 static int64_t
2049 _allshl(int64_t a, uint8_t b)
2050 {
2051 return (a << b);
2052 }
2053
2054 static uint64_t
2055 _aullshl(uint64_t a, uint8_t b)
2056 {
2057 return (a << b);
2058 }
2059
2060 static int64_t
2061 _allshr(int64_t a, uint8_t b)
2062 {
2063 return (a >> b);
2064 }
2065
2066 static uint64_t
2067 _aullshr(uint64_t a, uint8_t b)
2068 {
2069 return (a >> b);
2070 }
2071
2072 static slist_entry *
2073 ntoskrnl_pushsl(head, entry)
2074 slist_header *head;
2075 slist_entry *entry;
2076 {
2077 slist_entry *oldhead;
2078
2079 oldhead = head->slh_list.slh_next;
2080 entry->sl_next = head->slh_list.slh_next;
2081 head->slh_list.slh_next = entry;
2082 head->slh_list.slh_depth++;
2083 head->slh_list.slh_seq++;
2084
2085 return (oldhead);
2086 }
2087
2088 static void
2089 InitializeSListHead(head)
2090 slist_header *head;
2091 {
2092 memset(head, 0, sizeof(*head));
2093 }
2094
2095 static slist_entry *
2096 ntoskrnl_popsl(head)
2097 slist_header *head;
2098 {
2099 slist_entry *first;
2100
2101 first = head->slh_list.slh_next;
2102 if (first != NULL) {
2103 head->slh_list.slh_next = first->sl_next;
2104 head->slh_list.slh_depth--;
2105 head->slh_list.slh_seq++;
2106 }
2107
2108 return (first);
2109 }
2110
2111 /*
2112 * We need this to make lookaside lists work for amd64.
2113 * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2114 * list structure. For amd64 to work right, this has to be a
2115 * pointer to the wrapped version of the routine, not the
2116 * original. Letting the Windows driver invoke the original
2117 * function directly will result in a convention calling
2118 * mismatch and a pretty crash. On x86, this effectively
2119 * becomes a no-op since ipt_func and ipt_wrap are the same.
2120 */
2121
2122 static funcptr
2123 ntoskrnl_findwrap(func)
2124 funcptr func;
2125 {
2126 image_patch_table *patch;
2127
2128 patch = ntoskrnl_functbl;
2129 while (patch->ipt_func != NULL) {
2130 if ((funcptr)patch->ipt_func == func)
2131 return ((funcptr)patch->ipt_wrap);
2132 patch++;
2133 }
2134
2135 return (NULL);
2136 }
2137
2138 static void
2139 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2140 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2141 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2142 {
2143 bzero((char *)lookaside, sizeof(paged_lookaside_list));
2144
2145 if (size < sizeof(slist_entry))
2146 lookaside->nll_l.gl_size = sizeof(slist_entry);
2147 else
2148 lookaside->nll_l.gl_size = size;
2149 lookaside->nll_l.gl_tag = tag;
2150 if (allocfunc == NULL)
2151 lookaside->nll_l.gl_allocfunc =
2152 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2153 else
2154 lookaside->nll_l.gl_allocfunc = allocfunc;
2155
2156 if (freefunc == NULL)
2157 lookaside->nll_l.gl_freefunc =
2158 ntoskrnl_findwrap((funcptr)ExFreePool);
2159 else
2160 lookaside->nll_l.gl_freefunc = freefunc;
2161
2162 #ifdef __i386__
2163 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2164 #endif
2165
2166 lookaside->nll_l.gl_type = NonPagedPool;
2167 lookaside->nll_l.gl_depth = depth;
2168 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2169 }
2170
2171 static void
2172 ExDeletePagedLookasideList(lookaside)
2173 paged_lookaside_list *lookaside;
2174 {
2175 void *buf;
2176 void (*freefunc)(void *);
2177
2178 freefunc = lookaside->nll_l.gl_freefunc;
2179 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2180 MSCALL1(freefunc, buf);
2181 }
2182
2183 static void
2184 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2185 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2186 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2187 {
2188 bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2189
2190 if (size < sizeof(slist_entry))
2191 lookaside->nll_l.gl_size = sizeof(slist_entry);
2192 else
2193 lookaside->nll_l.gl_size = size;
2194 lookaside->nll_l.gl_tag = tag;
2195 if (allocfunc == NULL)
2196 lookaside->nll_l.gl_allocfunc =
2197 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2198 else
2199 lookaside->nll_l.gl_allocfunc = allocfunc;
2200
2201 if (freefunc == NULL)
2202 lookaside->nll_l.gl_freefunc =
2203 ntoskrnl_findwrap((funcptr)ExFreePool);
2204 else
2205 lookaside->nll_l.gl_freefunc = freefunc;
2206
2207 #ifdef __i386__
2208 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2209 #endif
2210
2211 lookaside->nll_l.gl_type = NonPagedPool;
2212 lookaside->nll_l.gl_depth = depth;
2213 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2214 }
2215
2216 static void
2217 ExDeleteNPagedLookasideList(lookaside)
2218 npaged_lookaside_list *lookaside;
2219 {
2220 void *buf;
2221 void (*freefunc)(void *);
2222
2223 freefunc = lookaside->nll_l.gl_freefunc;
2224 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2225 MSCALL1(freefunc, buf);
2226 }
2227
2228 slist_entry *
2229 InterlockedPushEntrySList(head, entry)
2230 slist_header *head;
2231 slist_entry *entry;
2232 {
2233 slist_entry *oldhead;
2234
2235 mtx_lock_spin(&ntoskrnl_interlock);
2236 oldhead = ntoskrnl_pushsl(head, entry);
2237 mtx_unlock_spin(&ntoskrnl_interlock);
2238
2239 return (oldhead);
2240 }
2241
2242 slist_entry *
2243 InterlockedPopEntrySList(head)
2244 slist_header *head;
2245 {
2246 slist_entry *first;
2247
2248 mtx_lock_spin(&ntoskrnl_interlock);
2249 first = ntoskrnl_popsl(head);
2250 mtx_unlock_spin(&ntoskrnl_interlock);
2251
2252 return (first);
2253 }
2254
2255 static slist_entry *
2256 ExInterlockedPushEntrySList(head, entry, lock)
2257 slist_header *head;
2258 slist_entry *entry;
2259 kspin_lock *lock;
2260 {
2261 return (InterlockedPushEntrySList(head, entry));
2262 }
2263
2264 static slist_entry *
2265 ExInterlockedPopEntrySList(head, lock)
2266 slist_header *head;
2267 kspin_lock *lock;
2268 {
2269 return (InterlockedPopEntrySList(head));
2270 }
2271
2272 uint16_t
2273 ExQueryDepthSList(head)
2274 slist_header *head;
2275 {
2276 uint16_t depth;
2277
2278 mtx_lock_spin(&ntoskrnl_interlock);
2279 depth = head->slh_list.slh_depth;
2280 mtx_unlock_spin(&ntoskrnl_interlock);
2281
2282 return (depth);
2283 }
2284
2285 void
2286 KeInitializeSpinLock(lock)
2287 kspin_lock *lock;
2288 {
2289 *lock = 0;
2290 }
2291
2292 #ifdef __i386__
2293 void
2294 KefAcquireSpinLockAtDpcLevel(lock)
2295 kspin_lock *lock;
2296 {
2297 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2298 int i = 0;
2299 #endif
2300
2301 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2302 /* sit and spin */;
2303 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2304 i++;
2305 if (i > 200000000)
2306 panic("DEADLOCK!");
2307 #endif
2308 }
2309 }
2310
2311 void
2312 KefReleaseSpinLockFromDpcLevel(lock)
2313 kspin_lock *lock;
2314 {
2315 atomic_store_rel_int((volatile u_int *)lock, 0);
2316 }
2317
2318 uint8_t
2319 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2320 {
2321 uint8_t oldirql;
2322
2323 if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2324 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2325
2326 KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2327 KeAcquireSpinLockAtDpcLevel(lock);
2328
2329 return (oldirql);
2330 }
2331 #else
2332 void
2333 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2334 {
2335 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2336 /* sit and spin */;
2337 }
2338
2339 void
2340 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2341 {
2342 atomic_store_rel_int((volatile u_int *)lock, 0);
2343 }
2344 #endif /* __i386__ */
2345
2346 uintptr_t
2347 InterlockedExchange(dst, val)
2348 volatile uint32_t *dst;
2349 uintptr_t val;
2350 {
2351 uintptr_t r;
2352
2353 mtx_lock_spin(&ntoskrnl_interlock);
2354 r = *dst;
2355 *dst = val;
2356 mtx_unlock_spin(&ntoskrnl_interlock);
2357
2358 return (r);
2359 }
2360
2361 static uint32_t
2362 InterlockedIncrement(addend)
2363 volatile uint32_t *addend;
2364 {
2365 atomic_add_long((volatile u_long *)addend, 1);
2366 return (*addend);
2367 }
2368
2369 static uint32_t
2370 InterlockedDecrement(addend)
2371 volatile uint32_t *addend;
2372 {
2373 atomic_subtract_long((volatile u_long *)addend, 1);
2374 return (*addend);
2375 }
2376
2377 static void
2378 ExInterlockedAddLargeStatistic(addend, inc)
2379 uint64_t *addend;
2380 uint32_t inc;
2381 {
2382 mtx_lock_spin(&ntoskrnl_interlock);
2383 *addend += inc;
2384 mtx_unlock_spin(&ntoskrnl_interlock);
2385 };
2386
2387 mdl *
2388 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2389 uint8_t chargequota, irp *iopkt)
2390 {
2391 mdl *m;
2392 int zone = 0;
2393
2394 if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2395 m = ExAllocatePoolWithTag(NonPagedPool,
2396 MmSizeOfMdl(vaddr, len), 0);
2397 else {
2398 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2399 zone++;
2400 }
2401
2402 if (m == NULL)
2403 return (NULL);
2404
2405 MmInitializeMdl(m, vaddr, len);
2406
2407 /*
2408 * MmInitializMdl() clears the flags field, so we
2409 * have to set this here. If the MDL came from the
2410 * MDL UMA zone, tag it so we can release it to
2411 * the right place later.
2412 */
2413 if (zone)
2414 m->mdl_flags = MDL_ZONE_ALLOCED;
2415
2416 if (iopkt != NULL) {
2417 if (secondarybuf == TRUE) {
2418 mdl *last;
2419 last = iopkt->irp_mdl;
2420 while (last->mdl_next != NULL)
2421 last = last->mdl_next;
2422 last->mdl_next = m;
2423 } else {
2424 if (iopkt->irp_mdl != NULL)
2425 panic("leaking an MDL in IoAllocateMdl()");
2426 iopkt->irp_mdl = m;
2427 }
2428 }
2429
2430 return (m);
2431 }
2432
2433 void
2434 IoFreeMdl(m)
2435 mdl *m;
2436 {
2437 if (m == NULL)
2438 return;
2439
2440 if (m->mdl_flags & MDL_ZONE_ALLOCED)
2441 uma_zfree(mdl_zone, m);
2442 else
2443 ExFreePool(m);
2444 }
2445
2446 static void *
2447 MmAllocateContiguousMemory(size, highest)
2448 uint32_t size;
2449 uint64_t highest;
2450 {
2451 void *addr;
2452 size_t pagelength = roundup(size, PAGE_SIZE);
2453
2454 addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2455
2456 return (addr);
2457 }
2458
2459 static void *
2460 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2461 boundary, cachetype)
2462 uint32_t size;
2463 uint64_t lowest;
2464 uint64_t highest;
2465 uint64_t boundary;
2466 enum nt_caching_type cachetype;
2467 {
2468 vm_memattr_t memattr;
2469 void *ret;
2470
2471 switch (cachetype) {
2472 case MmNonCached:
2473 memattr = VM_MEMATTR_UNCACHEABLE;
2474 break;
2475 case MmWriteCombined:
2476 memattr = VM_MEMATTR_WRITE_COMBINING;
2477 break;
2478 case MmNonCachedUnordered:
2479 memattr = VM_MEMATTR_UNCACHEABLE;
2480 break;
2481 case MmCached:
2482 case MmHardwareCoherentCached:
2483 case MmUSWCCached:
2484 default:
2485 memattr = VM_MEMATTR_DEFAULT;
2486 break;
2487 }
2488
2489 ret = (void *)kmem_alloc_contig(size, M_ZERO | M_NOWAIT, lowest,
2490 highest, PAGE_SIZE, boundary, memattr);
2491 if (ret != NULL)
2492 malloc_type_allocated(M_DEVBUF, round_page(size));
2493 return (ret);
2494 }
2495
2496 static void
2497 MmFreeContiguousMemory(base)
2498 void *base;
2499 {
2500 ExFreePool(base);
2501 }
2502
2503 static void
2504 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2505 void *base;
2506 uint32_t size;
2507 enum nt_caching_type cachetype;
2508 {
2509 contigfree(base, size, M_DEVBUF);
2510 }
2511
2512 static uint32_t
2513 MmSizeOfMdl(vaddr, len)
2514 void *vaddr;
2515 size_t len;
2516 {
2517 uint32_t l;
2518
2519 l = sizeof(struct mdl) +
2520 (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2521
2522 return (l);
2523 }
2524
2525 /*
2526 * The Microsoft documentation says this routine fills in the
2527 * page array of an MDL with the _physical_ page addresses that
2528 * comprise the buffer, but we don't really want to do that here.
2529 * Instead, we just fill in the page array with the kernel virtual
2530 * addresses of the buffers.
2531 */
2532 void
2533 MmBuildMdlForNonPagedPool(m)
2534 mdl *m;
2535 {
2536 vm_offset_t *mdl_pages;
2537 int pagecnt, i;
2538
2539 pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2540
2541 if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2542 panic("not enough pages in MDL to describe buffer");
2543
2544 mdl_pages = MmGetMdlPfnArray(m);
2545
2546 for (i = 0; i < pagecnt; i++)
2547 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2548
2549 m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2550 m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2551 }
2552
2553 static void *
2554 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2555 {
2556 buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2557 return (MmGetMdlVirtualAddress(buf));
2558 }
2559
2560 static void *
2561 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2562 void *vaddr, uint32_t bugcheck, uint32_t prio)
2563 {
2564 return (MmMapLockedPages(buf, accessmode));
2565 }
2566
2567 static void
2568 MmUnmapLockedPages(vaddr, buf)
2569 void *vaddr;
2570 mdl *buf;
2571 {
2572 buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2573 }
2574
2575 /*
2576 * This function has a problem in that it will break if you
2577 * compile this module without PAE and try to use it on a PAE
2578 * kernel. Unfortunately, there's no way around this at the
2579 * moment. It's slightly less broken that using pmap_kextract().
2580 * You'd think the virtual memory subsystem would help us out
2581 * here, but it doesn't.
2582 */
2583
2584 static uint64_t
2585 MmGetPhysicalAddress(void *base)
2586 {
2587 return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2588 }
2589
2590 void *
2591 MmGetSystemRoutineAddress(ustr)
2592 unicode_string *ustr;
2593 {
2594 ansi_string astr;
2595
2596 if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2597 return (NULL);
2598 return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2599 }
2600
2601 uint8_t
2602 MmIsAddressValid(vaddr)
2603 void *vaddr;
2604 {
2605 if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2606 return (TRUE);
2607
2608 return (FALSE);
2609 }
2610
2611 void *
2612 MmMapIoSpace(paddr, len, cachetype)
2613 uint64_t paddr;
2614 uint32_t len;
2615 uint32_t cachetype;
2616 {
2617 devclass_t nexus_class;
2618 device_t *nexus_devs, devp;
2619 int nexus_count = 0;
2620 device_t matching_dev = NULL;
2621 struct resource *res;
2622 int i;
2623 vm_offset_t v;
2624
2625 /* There will always be at least one nexus. */
2626
2627 nexus_class = devclass_find("nexus");
2628 devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2629
2630 for (i = 0; i < nexus_count; i++) {
2631 devp = nexus_devs[i];
2632 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2633 if (matching_dev)
2634 break;
2635 }
2636
2637 free(nexus_devs, M_TEMP);
2638
2639 if (matching_dev == NULL)
2640 return (NULL);
2641
2642 v = (vm_offset_t)rman_get_virtual(res);
2643 if (paddr > rman_get_start(res))
2644 v += paddr - rman_get_start(res);
2645
2646 return ((void *)v);
2647 }
2648
2649 void
2650 MmUnmapIoSpace(vaddr, len)
2651 void *vaddr;
2652 size_t len;
2653 {
2654 }
2655
2656 static device_t
2657 ntoskrnl_finddev(dev, paddr, res)
2658 device_t dev;
2659 uint64_t paddr;
2660 struct resource **res;
2661 {
2662 device_t *children = NULL;
2663 device_t matching_dev;
2664 int childcnt;
2665 struct resource *r;
2666 struct resource_list *rl;
2667 struct resource_list_entry *rle;
2668 uint32_t flags;
2669 int i;
2670
2671 /* We only want devices that have been successfully probed. */
2672
2673 if (device_is_alive(dev) == FALSE)
2674 return (NULL);
2675
2676 rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2677 if (rl != NULL) {
2678 STAILQ_FOREACH(rle, rl, link) {
2679 r = rle->res;
2680
2681 if (r == NULL)
2682 continue;
2683
2684 flags = rman_get_flags(r);
2685
2686 if (rle->type == SYS_RES_MEMORY &&
2687 paddr >= rman_get_start(r) &&
2688 paddr <= rman_get_end(r)) {
2689 if (!(flags & RF_ACTIVE))
2690 bus_activate_resource(dev,
2691 SYS_RES_MEMORY, 0, r);
2692 *res = r;
2693 return (dev);
2694 }
2695 }
2696 }
2697
2698 /*
2699 * If this device has children, do another
2700 * level of recursion to inspect them.
2701 */
2702
2703 device_get_children(dev, &children, &childcnt);
2704
2705 for (i = 0; i < childcnt; i++) {
2706 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2707 if (matching_dev != NULL) {
2708 free(children, M_TEMP);
2709 return (matching_dev);
2710 }
2711 }
2712
2713 /* Won't somebody please think of the children! */
2714
2715 if (children != NULL)
2716 free(children, M_TEMP);
2717
2718 return (NULL);
2719 }
2720
2721 /*
2722 * Workitems are unlike DPCs, in that they run in a user-mode thread
2723 * context rather than at DISPATCH_LEVEL in kernel context. In our
2724 * case we run them in kernel context anyway.
2725 */
2726 static void
2727 ntoskrnl_workitem_thread(arg)
2728 void *arg;
2729 {
2730 kdpc_queue *kq;
2731 list_entry *l;
2732 io_workitem *iw;
2733 uint8_t irql;
2734
2735 kq = arg;
2736
2737 InitializeListHead(&kq->kq_disp);
2738 kq->kq_td = curthread;
2739 kq->kq_exit = 0;
2740 KeInitializeSpinLock(&kq->kq_lock);
2741 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2742
2743 while (1) {
2744 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2745
2746 KeAcquireSpinLock(&kq->kq_lock, &irql);
2747
2748 if (kq->kq_exit) {
2749 kq->kq_exit = 0;
2750 KeReleaseSpinLock(&kq->kq_lock, irql);
2751 break;
2752 }
2753
2754 while (!IsListEmpty(&kq->kq_disp)) {
2755 l = RemoveHeadList(&kq->kq_disp);
2756 iw = CONTAINING_RECORD(l,
2757 io_workitem, iw_listentry);
2758 InitializeListHead((&iw->iw_listentry));
2759 if (iw->iw_func == NULL)
2760 continue;
2761 KeReleaseSpinLock(&kq->kq_lock, irql);
2762 MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2763 KeAcquireSpinLock(&kq->kq_lock, &irql);
2764 }
2765
2766 KeReleaseSpinLock(&kq->kq_lock, irql);
2767 }
2768
2769 kproc_exit(0);
2770 return; /* notreached */
2771 }
2772
2773 static ndis_status
2774 RtlCharToInteger(src, base, val)
2775 const char *src;
2776 uint32_t base;
2777 uint32_t *val;
2778 {
2779 int negative = 0;
2780 uint32_t res;
2781
2782 if (!src || !val)
2783 return (STATUS_ACCESS_VIOLATION);
2784 while (*src != '\0' && *src <= ' ')
2785 src++;
2786 if (*src == '+')
2787 src++;
2788 else if (*src == '-') {
2789 src++;
2790 negative = 1;
2791 }
2792 if (base == 0) {
2793 base = 10;
2794 if (*src == '0') {
2795 src++;
2796 if (*src == 'b') {
2797 base = 2;
2798 src++;
2799 } else if (*src == 'o') {
2800 base = 8;
2801 src++;
2802 } else if (*src == 'x') {
2803 base = 16;
2804 src++;
2805 }
2806 }
2807 } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2808 return (STATUS_INVALID_PARAMETER);
2809
2810 for (res = 0; *src; src++) {
2811 int v;
2812 if (isdigit(*src))
2813 v = *src - '0';
2814 else if (isxdigit(*src))
2815 v = tolower(*src) - 'a' + 10;
2816 else
2817 v = base;
2818 if (v >= base)
2819 return (STATUS_INVALID_PARAMETER);
2820 res = res * base + v;
2821 }
2822 *val = negative ? -res : res;
2823 return (STATUS_SUCCESS);
2824 }
2825
2826 static void
2827 ntoskrnl_destroy_workitem_threads(void)
2828 {
2829 kdpc_queue *kq;
2830 int i;
2831
2832 for (i = 0; i < WORKITEM_THREADS; i++) {
2833 kq = wq_queues + i;
2834 kq->kq_exit = 1;
2835 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2836 while (kq->kq_exit)
2837 tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2838 }
2839 }
2840
2841 io_workitem *
2842 IoAllocateWorkItem(dobj)
2843 device_object *dobj;
2844 {
2845 io_workitem *iw;
2846
2847 iw = uma_zalloc(iw_zone, M_NOWAIT);
2848 if (iw == NULL)
2849 return (NULL);
2850
2851 InitializeListHead(&iw->iw_listentry);
2852 iw->iw_dobj = dobj;
2853
2854 mtx_lock(&ntoskrnl_dispatchlock);
2855 iw->iw_idx = wq_idx;
2856 WORKIDX_INC(wq_idx);
2857 mtx_unlock(&ntoskrnl_dispatchlock);
2858
2859 return (iw);
2860 }
2861
2862 void
2863 IoFreeWorkItem(iw)
2864 io_workitem *iw;
2865 {
2866 uma_zfree(iw_zone, iw);
2867 }
2868
2869 void
2870 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2871 io_workitem *iw;
2872 io_workitem_func iw_func;
2873 uint32_t qtype;
2874 void *ctx;
2875 {
2876 kdpc_queue *kq;
2877 list_entry *l;
2878 io_workitem *cur;
2879 uint8_t irql;
2880
2881 kq = wq_queues + iw->iw_idx;
2882
2883 KeAcquireSpinLock(&kq->kq_lock, &irql);
2884
2885 /*
2886 * Traverse the list and make sure this workitem hasn't
2887 * already been inserted. Queuing the same workitem
2888 * twice will hose the list but good.
2889 */
2890
2891 l = kq->kq_disp.nle_flink;
2892 while (l != &kq->kq_disp) {
2893 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2894 if (cur == iw) {
2895 /* Already queued -- do nothing. */
2896 KeReleaseSpinLock(&kq->kq_lock, irql);
2897 return;
2898 }
2899 l = l->nle_flink;
2900 }
2901
2902 iw->iw_func = iw_func;
2903 iw->iw_ctx = ctx;
2904
2905 InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2906 KeReleaseSpinLock(&kq->kq_lock, irql);
2907
2908 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2909 }
2910
2911 static void
2912 ntoskrnl_workitem(dobj, arg)
2913 device_object *dobj;
2914 void *arg;
2915 {
2916 io_workitem *iw;
2917 work_queue_item *w;
2918 work_item_func f;
2919
2920 iw = arg;
2921 w = (work_queue_item *)dobj;
2922 f = (work_item_func)w->wqi_func;
2923 uma_zfree(iw_zone, iw);
2924 MSCALL2(f, w, w->wqi_ctx);
2925 }
2926
2927 /*
2928 * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2929 * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2930 * problem with ExQueueWorkItem() is that it can't guard against
2931 * the condition where a driver submits a job to the work queue and
2932 * is then unloaded before the job is able to run. IoQueueWorkItem()
2933 * acquires a reference to the device's device_object via the
2934 * object manager and retains it until after the job has completed,
2935 * which prevents the driver from being unloaded before the job
2936 * runs. (We don't currently support this behavior, though hopefully
2937 * that will change once the object manager API is fleshed out a bit.)
2938 *
2939 * Having said all that, the ExQueueWorkItem() API remains, because
2940 * there are still other parts of Windows that use it, including
2941 * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2942 * We fake up the ExQueueWorkItem() API on top of our implementation
2943 * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2944 * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2945 * queue item (provided by the caller) in to IoAllocateWorkItem()
2946 * instead of the device_object. We need to save this pointer so
2947 * we can apply a sanity check: as with the DPC queue and other
2948 * workitem queues, we can't allow the same work queue item to
2949 * be queued twice. If it's already pending, we silently return
2950 */
2951
2952 void
2953 ExQueueWorkItem(w, qtype)
2954 work_queue_item *w;
2955 uint32_t qtype;
2956 {
2957 io_workitem *iw;
2958 io_workitem_func iwf;
2959 kdpc_queue *kq;
2960 list_entry *l;
2961 io_workitem *cur;
2962 uint8_t irql;
2963
2964 /*
2965 * We need to do a special sanity test to make sure
2966 * the ExQueueWorkItem() API isn't used to queue
2967 * the same workitem twice. Rather than checking the
2968 * io_workitem pointer itself, we test the attached
2969 * device object, which is really a pointer to the
2970 * legacy work queue item structure.
2971 */
2972
2973 kq = wq_queues + WORKITEM_LEGACY_THREAD;
2974 KeAcquireSpinLock(&kq->kq_lock, &irql);
2975 l = kq->kq_disp.nle_flink;
2976 while (l != &kq->kq_disp) {
2977 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2978 if (cur->iw_dobj == (device_object *)w) {
2979 /* Already queued -- do nothing. */
2980 KeReleaseSpinLock(&kq->kq_lock, irql);
2981 return;
2982 }
2983 l = l->nle_flink;
2984 }
2985 KeReleaseSpinLock(&kq->kq_lock, irql);
2986
2987 iw = IoAllocateWorkItem((device_object *)w);
2988 if (iw == NULL)
2989 return;
2990
2991 iw->iw_idx = WORKITEM_LEGACY_THREAD;
2992 iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
2993 IoQueueWorkItem(iw, iwf, qtype, iw);
2994 }
2995
2996 static void
2997 RtlZeroMemory(dst, len)
2998 void *dst;
2999 size_t len;
3000 {
3001 bzero(dst, len);
3002 }
3003
3004 static void
3005 RtlSecureZeroMemory(dst, len)
3006 void *dst;
3007 size_t len;
3008 {
3009 memset(dst, 0, len);
3010 }
3011
3012 static void
3013 RtlFillMemory(void *dst, size_t len, uint8_t c)
3014 {
3015 memset(dst, c, len);
3016 }
3017
3018 static void
3019 RtlMoveMemory(dst, src, len)
3020 void *dst;
3021 const void *src;
3022 size_t len;
3023 {
3024 memmove(dst, src, len);
3025 }
3026
3027 static void
3028 RtlCopyMemory(dst, src, len)
3029 void *dst;
3030 const void *src;
3031 size_t len;
3032 {
3033 bcopy(src, dst, len);
3034 }
3035
3036 static size_t
3037 RtlCompareMemory(s1, s2, len)
3038 const void *s1;
3039 const void *s2;
3040 size_t len;
3041 {
3042 size_t i;
3043 uint8_t *m1, *m2;
3044
3045 m1 = __DECONST(char *, s1);
3046 m2 = __DECONST(char *, s2);
3047
3048 for (i = 0; i < len && m1[i] == m2[i]; i++);
3049 return (i);
3050 }
3051
3052 void
3053 RtlInitAnsiString(dst, src)
3054 ansi_string *dst;
3055 char *src;
3056 {
3057 ansi_string *a;
3058
3059 a = dst;
3060 if (a == NULL)
3061 return;
3062 if (src == NULL) {
3063 a->as_len = a->as_maxlen = 0;
3064 a->as_buf = NULL;
3065 } else {
3066 a->as_buf = src;
3067 a->as_len = a->as_maxlen = strlen(src);
3068 }
3069 }
3070
3071 void
3072 RtlInitUnicodeString(dst, src)
3073 unicode_string *dst;
3074 uint16_t *src;
3075 {
3076 unicode_string *u;
3077 int i;
3078
3079 u = dst;
3080 if (u == NULL)
3081 return;
3082 if (src == NULL) {
3083 u->us_len = u->us_maxlen = 0;
3084 u->us_buf = NULL;
3085 } else {
3086 i = 0;
3087 while(src[i] != 0)
3088 i++;
3089 u->us_buf = src;
3090 u->us_len = u->us_maxlen = i * 2;
3091 }
3092 }
3093
3094 ndis_status
3095 RtlUnicodeStringToInteger(ustr, base, val)
3096 unicode_string *ustr;
3097 uint32_t base;
3098 uint32_t *val;
3099 {
3100 uint16_t *uchr;
3101 int len, neg = 0;
3102 char abuf[64];
3103 char *astr;
3104
3105 uchr = ustr->us_buf;
3106 len = ustr->us_len;
3107 bzero(abuf, sizeof(abuf));
3108
3109 if ((char)((*uchr) & 0xFF) == '-') {
3110 neg = 1;
3111 uchr++;
3112 len -= 2;
3113 } else if ((char)((*uchr) & 0xFF) == '+') {
3114 neg = 0;
3115 uchr++;
3116 len -= 2;
3117 }
3118
3119 if (base == 0) {
3120 if ((char)((*uchr) & 0xFF) == 'b') {
3121 base = 2;
3122 uchr++;
3123 len -= 2;
3124 } else if ((char)((*uchr) & 0xFF) == 'o') {
3125 base = 8;
3126 uchr++;
3127 len -= 2;
3128 } else if ((char)((*uchr) & 0xFF) == 'x') {
3129 base = 16;
3130 uchr++;
3131 len -= 2;
3132 } else
3133 base = 10;
3134 }
3135
3136 astr = abuf;
3137 if (neg) {
3138 strcpy(astr, "-");
3139 astr++;
3140 }
3141
3142 ntoskrnl_unicode_to_ascii(uchr, astr, len);
3143 *val = strtoul(abuf, NULL, base);
3144
3145 return (STATUS_SUCCESS);
3146 }
3147
3148 void
3149 RtlFreeUnicodeString(ustr)
3150 unicode_string *ustr;
3151 {
3152 if (ustr->us_buf == NULL)
3153 return;
3154 ExFreePool(ustr->us_buf);
3155 ustr->us_buf = NULL;
3156 }
3157
3158 void
3159 RtlFreeAnsiString(astr)
3160 ansi_string *astr;
3161 {
3162 if (astr->as_buf == NULL)
3163 return;
3164 ExFreePool(astr->as_buf);
3165 astr->as_buf = NULL;
3166 }
3167
3168 static int
3169 atoi(str)
3170 const char *str;
3171 {
3172 return (int)strtol(str, (char **)NULL, 10);
3173 }
3174
3175 static long
3176 atol(str)
3177 const char *str;
3178 {
3179 return strtol(str, (char **)NULL, 10);
3180 }
3181
3182 static int
3183 rand(void)
3184 {
3185
3186 return (random());
3187 }
3188
3189 static void
3190 srand(unsigned int seed __unused)
3191 {
3192 }
3193
3194 static uint8_t
3195 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3196 {
3197 if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3198 return (TRUE);
3199 return (FALSE);
3200 }
3201
3202 static int32_t
3203 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3204 uint32_t mask, void **key)
3205 {
3206 return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3207 }
3208
3209 static ndis_status
3210 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3211 unicode_string *name;
3212 uint32_t reqaccess;
3213 void *fileobj;
3214 device_object *devobj;
3215 {
3216 return (STATUS_SUCCESS);
3217 }
3218
3219 static ndis_status
3220 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3221 device_object *devobj;
3222 uint32_t regprop;
3223 uint32_t buflen;
3224 void *prop;
3225 uint32_t *reslen;
3226 {
3227 driver_object *drv;
3228 uint16_t **name;
3229
3230 drv = devobj->do_drvobj;
3231
3232 switch (regprop) {
3233 case DEVPROP_DRIVER_KEYNAME:
3234 name = prop;
3235 *name = drv->dro_drivername.us_buf;
3236 *reslen = drv->dro_drivername.us_len;
3237 break;
3238 default:
3239 return (STATUS_INVALID_PARAMETER_2);
3240 break;
3241 }
3242
3243 return (STATUS_SUCCESS);
3244 }
3245
3246 static void
3247 KeInitializeMutex(kmutex, level)
3248 kmutant *kmutex;
3249 uint32_t level;
3250 {
3251 InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3252 kmutex->km_abandoned = FALSE;
3253 kmutex->km_apcdisable = 1;
3254 kmutex->km_header.dh_sigstate = 1;
3255 kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3256 kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3257 kmutex->km_ownerthread = NULL;
3258 }
3259
3260 static uint32_t
3261 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3262 {
3263 uint32_t prevstate;
3264
3265 mtx_lock(&ntoskrnl_dispatchlock);
3266 prevstate = kmutex->km_header.dh_sigstate;
3267 if (kmutex->km_ownerthread != curthread) {
3268 mtx_unlock(&ntoskrnl_dispatchlock);
3269 return (STATUS_MUTANT_NOT_OWNED);
3270 }
3271
3272 kmutex->km_header.dh_sigstate++;
3273 kmutex->km_abandoned = FALSE;
3274
3275 if (kmutex->km_header.dh_sigstate == 1) {
3276 kmutex->km_ownerthread = NULL;
3277 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3278 }
3279
3280 mtx_unlock(&ntoskrnl_dispatchlock);
3281
3282 return (prevstate);
3283 }
3284
3285 static uint32_t
3286 KeReadStateMutex(kmutex)
3287 kmutant *kmutex;
3288 {
3289 return (kmutex->km_header.dh_sigstate);
3290 }
3291
3292 void
3293 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3294 {
3295 InitializeListHead((&kevent->k_header.dh_waitlisthead));
3296 kevent->k_header.dh_sigstate = state;
3297 if (type == EVENT_TYPE_NOTIFY)
3298 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3299 else
3300 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3301 kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3302 }
3303
3304 uint32_t
3305 KeResetEvent(kevent)
3306 nt_kevent *kevent;
3307 {
3308 uint32_t prevstate;
3309
3310 mtx_lock(&ntoskrnl_dispatchlock);
3311 prevstate = kevent->k_header.dh_sigstate;
3312 kevent->k_header.dh_sigstate = FALSE;
3313 mtx_unlock(&ntoskrnl_dispatchlock);
3314
3315 return (prevstate);
3316 }
3317
3318 uint32_t
3319 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3320 {
3321 uint32_t prevstate;
3322 wait_block *w;
3323 nt_dispatch_header *dh;
3324 struct thread *td;
3325 wb_ext *we;
3326
3327 mtx_lock(&ntoskrnl_dispatchlock);
3328 prevstate = kevent->k_header.dh_sigstate;
3329 dh = &kevent->k_header;
3330
3331 if (IsListEmpty(&dh->dh_waitlisthead))
3332 /*
3333 * If there's nobody in the waitlist, just set
3334 * the state to signalled.
3335 */
3336 dh->dh_sigstate = 1;
3337 else {
3338 /*
3339 * Get the first waiter. If this is a synchronization
3340 * event, just wake up that one thread (don't bother
3341 * setting the state to signalled since we're supposed
3342 * to automatically clear synchronization events anyway).
3343 *
3344 * If it's a notification event, or the first
3345 * waiter is doing a WAITTYPE_ALL wait, go through
3346 * the full wait satisfaction process.
3347 */
3348 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3349 wait_block, wb_waitlist);
3350 we = w->wb_ext;
3351 td = we->we_td;
3352 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3353 w->wb_waittype == WAITTYPE_ALL) {
3354 if (prevstate == 0) {
3355 dh->dh_sigstate = 1;
3356 ntoskrnl_waittest(dh, increment);
3357 }
3358 } else {
3359 w->wb_awakened |= TRUE;
3360 cv_broadcastpri(&we->we_cv,
3361 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3362 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3363 }
3364 }
3365
3366 mtx_unlock(&ntoskrnl_dispatchlock);
3367
3368 return (prevstate);
3369 }
3370
3371 void
3372 KeClearEvent(kevent)
3373 nt_kevent *kevent;
3374 {
3375 kevent->k_header.dh_sigstate = FALSE;
3376 }
3377
3378 uint32_t
3379 KeReadStateEvent(kevent)
3380 nt_kevent *kevent;
3381 {
3382 return (kevent->k_header.dh_sigstate);
3383 }
3384
3385 /*
3386 * The object manager in Windows is responsible for managing
3387 * references and access to various types of objects, including
3388 * device_objects, events, threads, timers and so on. However,
3389 * there's a difference in the way objects are handled in user
3390 * mode versus kernel mode.
3391 *
3392 * In user mode (i.e. Win32 applications), all objects are
3393 * managed by the object manager. For example, when you create
3394 * a timer or event object, you actually end up with an
3395 * object_header (for the object manager's bookkeeping
3396 * purposes) and an object body (which contains the actual object
3397 * structure, e.g. ktimer, kevent, etc...). This allows Windows
3398 * to manage resource quotas and to enforce access restrictions
3399 * on basically every kind of system object handled by the kernel.
3400 *
3401 * However, in kernel mode, you only end up using the object
3402 * manager some of the time. For example, in a driver, you create
3403 * a timer object by simply allocating the memory for a ktimer
3404 * structure and initializing it with KeInitializeTimer(). Hence,
3405 * the timer has no object_header and no reference counting or
3406 * security/resource checks are done on it. The assumption in
3407 * this case is that if you're running in kernel mode, you know
3408 * what you're doing, and you're already at an elevated privilege
3409 * anyway.
3410 *
3411 * There are some exceptions to this. The two most important ones
3412 * for our purposes are device_objects and threads. We need to use
3413 * the object manager to do reference counting on device_objects,
3414 * and for threads, you can only get a pointer to a thread's
3415 * dispatch header by using ObReferenceObjectByHandle() on the
3416 * handle returned by PsCreateSystemThread().
3417 */
3418
3419 static ndis_status
3420 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3421 uint8_t accessmode, void **object, void **handleinfo)
3422 {
3423 nt_objref *nr;
3424
3425 nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3426 if (nr == NULL)
3427 return (STATUS_INSUFFICIENT_RESOURCES);
3428
3429 InitializeListHead((&nr->no_dh.dh_waitlisthead));
3430 nr->no_obj = handle;
3431 nr->no_dh.dh_type = DISP_TYPE_THREAD;
3432 nr->no_dh.dh_sigstate = 0;
3433 nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3434 sizeof(uint32_t));
3435 TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3436 *object = nr;
3437
3438 return (STATUS_SUCCESS);
3439 }
3440
3441 static void
3442 ObfDereferenceObject(object)
3443 void *object;
3444 {
3445 nt_objref *nr;
3446
3447 nr = object;
3448 TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3449 free(nr, M_DEVBUF);
3450 }
3451
3452 static uint32_t
3453 ZwClose(handle)
3454 ndis_handle handle;
3455 {
3456 return (STATUS_SUCCESS);
3457 }
3458
3459 static uint32_t
3460 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3461 uint32_t traceclass;
3462 void *traceinfo;
3463 uint32_t infolen;
3464 uint32_t reqlen;
3465 void *buf;
3466 {
3467 return (STATUS_NOT_FOUND);
3468 }
3469
3470 static uint32_t
3471 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3472 void *guid, uint16_t messagenum, ...)
3473 {
3474 return (STATUS_SUCCESS);
3475 }
3476
3477 static uint32_t
3478 IoWMIRegistrationControl(dobj, action)
3479 device_object *dobj;
3480 uint32_t action;
3481 {
3482 return (STATUS_SUCCESS);
3483 }
3484
3485 /*
3486 * This is here just in case the thread returns without calling
3487 * PsTerminateSystemThread().
3488 */
3489 static void
3490 ntoskrnl_thrfunc(arg)
3491 void *arg;
3492 {
3493 thread_context *thrctx;
3494 uint32_t (*tfunc)(void *);
3495 void *tctx;
3496 uint32_t rval;
3497
3498 thrctx = arg;
3499 tfunc = thrctx->tc_thrfunc;
3500 tctx = thrctx->tc_thrctx;
3501 free(thrctx, M_TEMP);
3502
3503 rval = MSCALL1(tfunc, tctx);
3504
3505 PsTerminateSystemThread(rval);
3506 return; /* notreached */
3507 }
3508
3509 static ndis_status
3510 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3511 clientid, thrfunc, thrctx)
3512 ndis_handle *handle;
3513 uint32_t reqaccess;
3514 void *objattrs;
3515 ndis_handle phandle;
3516 void *clientid;
3517 void *thrfunc;
3518 void *thrctx;
3519 {
3520 int error;
3521 thread_context *tc;
3522 struct proc *p;
3523
3524 tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3525 if (tc == NULL)
3526 return (STATUS_INSUFFICIENT_RESOURCES);
3527
3528 tc->tc_thrctx = thrctx;
3529 tc->tc_thrfunc = thrfunc;
3530
3531 error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3532 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3533
3534 if (error) {
3535 free(tc, M_TEMP);
3536 return (STATUS_INSUFFICIENT_RESOURCES);
3537 }
3538
3539 *handle = p;
3540 ntoskrnl_kth++;
3541
3542 return (STATUS_SUCCESS);
3543 }
3544
3545 /*
3546 * In Windows, the exit of a thread is an event that you're allowed
3547 * to wait on, assuming you've obtained a reference to the thread using
3548 * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3549 * simulate this behavior is to register each thread we create in a
3550 * reference list, and if someone holds a reference to us, we poke
3551 * them.
3552 */
3553 static ndis_status
3554 PsTerminateSystemThread(status)
3555 ndis_status status;
3556 {
3557 struct nt_objref *nr;
3558
3559 mtx_lock(&ntoskrnl_dispatchlock);
3560 TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3561 if (nr->no_obj != curthread->td_proc)
3562 continue;
3563 nr->no_dh.dh_sigstate = 1;
3564 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3565 break;
3566 }
3567 mtx_unlock(&ntoskrnl_dispatchlock);
3568
3569 ntoskrnl_kth--;
3570
3571 kproc_exit(0);
3572 return (0); /* notreached */
3573 }
3574
3575 static uint32_t
3576 DbgPrint(char *fmt, ...)
3577 {
3578 va_list ap;
3579
3580 if (bootverbose) {
3581 va_start(ap, fmt);
3582 vprintf(fmt, ap);
3583 va_end(ap);
3584 }
3585
3586 return (STATUS_SUCCESS);
3587 }
3588
3589 static void
3590 DbgBreakPoint(void)
3591 {
3592
3593 kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3594 }
3595
3596 static void
3597 KeBugCheckEx(code, param1, param2, param3, param4)
3598 uint32_t code;
3599 u_long param1;
3600 u_long param2;
3601 u_long param3;
3602 u_long param4;
3603 {
3604 panic("KeBugCheckEx: STOP 0x%X", code);
3605 }
3606
3607 static void
3608 ntoskrnl_timercall(arg)
3609 void *arg;
3610 {
3611 ktimer *timer;
3612 struct timeval tv;
3613 kdpc *dpc;
3614
3615 mtx_lock(&ntoskrnl_dispatchlock);
3616
3617 timer = arg;
3618
3619 #ifdef NTOSKRNL_DEBUG_TIMERS
3620 ntoskrnl_timer_fires++;
3621 #endif
3622 ntoskrnl_remove_timer(timer);
3623
3624 /*
3625 * This should never happen, but complain
3626 * if it does.
3627 */
3628
3629 if (timer->k_header.dh_inserted == FALSE) {
3630 mtx_unlock(&ntoskrnl_dispatchlock);
3631 printf("NTOS: timer %p fired even though "
3632 "it was canceled\n", timer);
3633 return;
3634 }
3635
3636 /* Mark the timer as no longer being on the timer queue. */
3637
3638 timer->k_header.dh_inserted = FALSE;
3639
3640 /* Now signal the object and satisfy any waits on it. */
3641
3642 timer->k_header.dh_sigstate = 1;
3643 ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3644
3645 /*
3646 * If this is a periodic timer, re-arm it
3647 * so it will fire again. We do this before
3648 * calling any deferred procedure calls because
3649 * it's possible the DPC might cancel the timer,
3650 * in which case it would be wrong for us to
3651 * re-arm it again afterwards.
3652 */
3653
3654 if (timer->k_period) {
3655 tv.tv_sec = 0;
3656 tv.tv_usec = timer->k_period * 1000;
3657 timer->k_header.dh_inserted = TRUE;
3658 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3659 #ifdef NTOSKRNL_DEBUG_TIMERS
3660 ntoskrnl_timer_reloads++;
3661 #endif
3662 }
3663
3664 dpc = timer->k_dpc;
3665
3666 mtx_unlock(&ntoskrnl_dispatchlock);
3667
3668 /* If there's a DPC associated with the timer, queue it up. */
3669
3670 if (dpc != NULL)
3671 KeInsertQueueDpc(dpc, NULL, NULL);
3672 }
3673
3674 #ifdef NTOSKRNL_DEBUG_TIMERS
3675 static int
3676 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3677 {
3678 int ret;
3679
3680 ret = 0;
3681 ntoskrnl_show_timers();
3682 return (sysctl_handle_int(oidp, &ret, 0, req));
3683 }
3684
3685 static void
3686 ntoskrnl_show_timers()
3687 {
3688 int i = 0;
3689 list_entry *l;
3690
3691 mtx_lock_spin(&ntoskrnl_calllock);
3692 l = ntoskrnl_calllist.nle_flink;
3693 while(l != &ntoskrnl_calllist) {
3694 i++;
3695 l = l->nle_flink;
3696 }
3697 mtx_unlock_spin(&ntoskrnl_calllock);
3698
3699 printf("\n");
3700 printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3701 printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3702 printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3703 printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3704 printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3705 printf("\n");
3706 }
3707 #endif
3708
3709 /*
3710 * Must be called with dispatcher lock held.
3711 */
3712
3713 static void
3714 ntoskrnl_insert_timer(timer, ticks)
3715 ktimer *timer;
3716 int ticks;
3717 {
3718 callout_entry *e;
3719 list_entry *l;
3720 struct callout *c;
3721
3722 /*
3723 * Try and allocate a timer.
3724 */
3725 mtx_lock_spin(&ntoskrnl_calllock);
3726 if (IsListEmpty(&ntoskrnl_calllist)) {
3727 mtx_unlock_spin(&ntoskrnl_calllock);
3728 #ifdef NTOSKRNL_DEBUG_TIMERS
3729 ntoskrnl_show_timers();
3730 #endif
3731 panic("out of timers!");
3732 }
3733 l = RemoveHeadList(&ntoskrnl_calllist);
3734 mtx_unlock_spin(&ntoskrnl_calllock);
3735
3736 e = CONTAINING_RECORD(l, callout_entry, ce_list);
3737 c = &e->ce_callout;
3738
3739 timer->k_callout = c;
3740
3741 callout_init(c, 1);
3742 callout_reset(c, ticks, ntoskrnl_timercall, timer);
3743 }
3744
3745 static void
3746 ntoskrnl_remove_timer(timer)
3747 ktimer *timer;
3748 {
3749 callout_entry *e;
3750
3751 e = (callout_entry *)timer->k_callout;
3752 callout_stop(timer->k_callout);
3753
3754 mtx_lock_spin(&ntoskrnl_calllock);
3755 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3756 mtx_unlock_spin(&ntoskrnl_calllock);
3757 }
3758
3759 void
3760 KeInitializeTimer(timer)
3761 ktimer *timer;
3762 {
3763 if (timer == NULL)
3764 return;
3765
3766 KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY);
3767 }
3768
3769 void
3770 KeInitializeTimerEx(timer, type)
3771 ktimer *timer;
3772 uint32_t type;
3773 {
3774 if (timer == NULL)
3775 return;
3776
3777 bzero((char *)timer, sizeof(ktimer));
3778 InitializeListHead((&timer->k_header.dh_waitlisthead));
3779 timer->k_header.dh_sigstate = FALSE;
3780 timer->k_header.dh_inserted = FALSE;
3781 if (type == EVENT_TYPE_NOTIFY)
3782 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3783 else
3784 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3785 timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3786 }
3787
3788 /*
3789 * DPC subsystem. A Windows Defered Procedure Call has the following
3790 * properties:
3791 * - It runs at DISPATCH_LEVEL.
3792 * - It can have one of 3 importance values that control when it
3793 * runs relative to other DPCs in the queue.
3794 * - On SMP systems, it can be set to run on a specific processor.
3795 * In order to satisfy the last property, we create a DPC thread for
3796 * each CPU in the system and bind it to that CPU. Each thread
3797 * maintains three queues with different importance levels, which
3798 * will be processed in order from lowest to highest.
3799 *
3800 * In Windows, interrupt handlers run as DPCs. (Not to be confused
3801 * with ISRs, which run in interrupt context and can preempt DPCs.)
3802 * ISRs are given the highest importance so that they'll take
3803 * precedence over timers and other things.
3804 */
3805
3806 static void
3807 ntoskrnl_dpc_thread(arg)
3808 void *arg;
3809 {
3810 kdpc_queue *kq;
3811 kdpc *d;
3812 list_entry *l;
3813 uint8_t irql;
3814
3815 kq = arg;
3816
3817 InitializeListHead(&kq->kq_disp);
3818 kq->kq_td = curthread;
3819 kq->kq_exit = 0;
3820 kq->kq_running = FALSE;
3821 KeInitializeSpinLock(&kq->kq_lock);
3822 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3823 KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3824
3825 /*
3826 * Elevate our priority. DPCs are used to run interrupt
3827 * handlers, and they should trigger as soon as possible
3828 * once scheduled by an ISR.
3829 */
3830
3831 thread_lock(curthread);
3832 #ifdef NTOSKRNL_MULTIPLE_DPCS
3833 sched_bind(curthread, kq->kq_cpu);
3834 #endif
3835 sched_prio(curthread, PRI_MIN_KERN);
3836 thread_unlock(curthread);
3837
3838 while (1) {
3839 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3840
3841 KeAcquireSpinLock(&kq->kq_lock, &irql);
3842
3843 if (kq->kq_exit) {
3844 kq->kq_exit = 0;
3845 KeReleaseSpinLock(&kq->kq_lock, irql);
3846 break;
3847 }
3848
3849 kq->kq_running = TRUE;
3850
3851 while (!IsListEmpty(&kq->kq_disp)) {
3852 l = RemoveHeadList((&kq->kq_disp));
3853 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3854 InitializeListHead((&d->k_dpclistentry));
3855 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3856 MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3857 d->k_sysarg1, d->k_sysarg2);
3858 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3859 }
3860
3861 kq->kq_running = FALSE;
3862
3863 KeReleaseSpinLock(&kq->kq_lock, irql);
3864
3865 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3866 }
3867
3868 kproc_exit(0);
3869 return; /* notreached */
3870 }
3871
3872 static void
3873 ntoskrnl_destroy_dpc_threads(void)
3874 {
3875 kdpc_queue *kq;
3876 kdpc dpc;
3877 int i;
3878
3879 kq = kq_queues;
3880 #ifdef NTOSKRNL_MULTIPLE_DPCS
3881 for (i = 0; i < mp_ncpus; i++) {
3882 #else
3883 for (i = 0; i < 1; i++) {
3884 #endif
3885 kq += i;
3886
3887 kq->kq_exit = 1;
3888 KeInitializeDpc(&dpc, NULL, NULL);
3889 KeSetTargetProcessorDpc(&dpc, i);
3890 KeInsertQueueDpc(&dpc, NULL, NULL);
3891 while (kq->kq_exit)
3892 tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3893 }
3894 }
3895
3896 static uint8_t
3897 ntoskrnl_insert_dpc(head, dpc)
3898 list_entry *head;
3899 kdpc *dpc;
3900 {
3901 list_entry *l;
3902 kdpc *d;
3903
3904 l = head->nle_flink;
3905 while (l != head) {
3906 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3907 if (d == dpc)
3908 return (FALSE);
3909 l = l->nle_flink;
3910 }
3911
3912 if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3913 InsertTailList((head), (&dpc->k_dpclistentry));
3914 else
3915 InsertHeadList((head), (&dpc->k_dpclistentry));
3916
3917 return (TRUE);
3918 }
3919
3920 void
3921 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3922 kdpc *dpc;
3923 void *dpcfunc;
3924 void *dpcctx;
3925 {
3926
3927 if (dpc == NULL)
3928 return;
3929
3930 dpc->k_deferedfunc = dpcfunc;
3931 dpc->k_deferredctx = dpcctx;
3932 dpc->k_num = KDPC_CPU_DEFAULT;
3933 dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3934 InitializeListHead((&dpc->k_dpclistentry));
3935 }
3936
3937 uint8_t
3938 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3939 kdpc *dpc;
3940 void *sysarg1;
3941 void *sysarg2;
3942 {
3943 kdpc_queue *kq;
3944 uint8_t r;
3945 uint8_t irql;
3946
3947 if (dpc == NULL)
3948 return (FALSE);
3949
3950 kq = kq_queues;
3951
3952 #ifdef NTOSKRNL_MULTIPLE_DPCS
3953 KeRaiseIrql(DISPATCH_LEVEL, &irql);
3954
3955 /*
3956 * By default, the DPC is queued to run on the same CPU
3957 * that scheduled it.
3958 */
3959
3960 if (dpc->k_num == KDPC_CPU_DEFAULT)
3961 kq += curthread->td_oncpu;
3962 else
3963 kq += dpc->k_num;
3964 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3965 #else
3966 KeAcquireSpinLock(&kq->kq_lock, &irql);
3967 #endif
3968
3969 r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3970 if (r == TRUE) {
3971 dpc->k_sysarg1 = sysarg1;
3972 dpc->k_sysarg2 = sysarg2;
3973 }
3974 KeReleaseSpinLock(&kq->kq_lock, irql);
3975
3976 if (r == FALSE)
3977 return (r);
3978
3979 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3980
3981 return (r);
3982 }
3983
3984 uint8_t
3985 KeRemoveQueueDpc(dpc)
3986 kdpc *dpc;
3987 {
3988 kdpc_queue *kq;
3989 uint8_t irql;
3990
3991 if (dpc == NULL)
3992 return (FALSE);
3993
3994 #ifdef NTOSKRNL_MULTIPLE_DPCS
3995 KeRaiseIrql(DISPATCH_LEVEL, &irql);
3996
3997 kq = kq_queues + dpc->k_num;
3998
3999 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4000 #else
4001 kq = kq_queues;
4002 KeAcquireSpinLock(&kq->kq_lock, &irql);
4003 #endif
4004
4005 if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4006 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4007 KeLowerIrql(irql);
4008 return (FALSE);
4009 }
4010
4011 RemoveEntryList((&dpc->k_dpclistentry));
4012 InitializeListHead((&dpc->k_dpclistentry));
4013
4014 KeReleaseSpinLock(&kq->kq_lock, irql);
4015
4016 return (TRUE);
4017 }
4018
4019 void
4020 KeSetImportanceDpc(dpc, imp)
4021 kdpc *dpc;
4022 uint32_t imp;
4023 {
4024 if (imp != KDPC_IMPORTANCE_LOW &&
4025 imp != KDPC_IMPORTANCE_MEDIUM &&
4026 imp != KDPC_IMPORTANCE_HIGH)
4027 return;
4028
4029 dpc->k_importance = (uint8_t)imp;
4030 }
4031
4032 void
4033 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4034 {
4035 if (cpu > mp_ncpus)
4036 return;
4037
4038 dpc->k_num = cpu;
4039 }
4040
4041 void
4042 KeFlushQueuedDpcs(void)
4043 {
4044 kdpc_queue *kq;
4045 int i;
4046
4047 /*
4048 * Poke each DPC queue and wait
4049 * for them to drain.
4050 */
4051
4052 #ifdef NTOSKRNL_MULTIPLE_DPCS
4053 for (i = 0; i < mp_ncpus; i++) {
4054 #else
4055 for (i = 0; i < 1; i++) {
4056 #endif
4057 kq = kq_queues + i;
4058 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4059 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4060 }
4061 }
4062
4063 uint32_t
4064 KeGetCurrentProcessorNumber(void)
4065 {
4066 return ((uint32_t)curthread->td_oncpu);
4067 }
4068
4069 uint8_t
4070 KeSetTimerEx(timer, duetime, period, dpc)
4071 ktimer *timer;
4072 int64_t duetime;
4073 uint32_t period;
4074 kdpc *dpc;
4075 {
4076 struct timeval tv;
4077 uint64_t curtime;
4078 uint8_t pending;
4079
4080 if (timer == NULL)
4081 return (FALSE);
4082
4083 mtx_lock(&ntoskrnl_dispatchlock);
4084
4085 if (timer->k_header.dh_inserted == TRUE) {
4086 ntoskrnl_remove_timer(timer);
4087 #ifdef NTOSKRNL_DEBUG_TIMERS
4088 ntoskrnl_timer_cancels++;
4089 #endif
4090 timer->k_header.dh_inserted = FALSE;
4091 pending = TRUE;
4092 } else
4093 pending = FALSE;
4094
4095 timer->k_duetime = duetime;
4096 timer->k_period = period;
4097 timer->k_header.dh_sigstate = FALSE;
4098 timer->k_dpc = dpc;
4099
4100 if (duetime < 0) {
4101 tv.tv_sec = - (duetime) / 10000000;
4102 tv.tv_usec = (- (duetime) / 10) -
4103 (tv.tv_sec * 1000000);
4104 } else {
4105 ntoskrnl_time(&curtime);
4106 if (duetime < curtime)
4107 tv.tv_sec = tv.tv_usec = 0;
4108 else {
4109 tv.tv_sec = ((duetime) - curtime) / 10000000;
4110 tv.tv_usec = ((duetime) - curtime) / 10 -
4111 (tv.tv_sec * 1000000);
4112 }
4113 }
4114
4115 timer->k_header.dh_inserted = TRUE;
4116 ntoskrnl_insert_timer(timer, tvtohz(&tv));
4117 #ifdef NTOSKRNL_DEBUG_TIMERS
4118 ntoskrnl_timer_sets++;
4119 #endif
4120
4121 mtx_unlock(&ntoskrnl_dispatchlock);
4122
4123 return (pending);
4124 }
4125
4126 uint8_t
4127 KeSetTimer(timer, duetime, dpc)
4128 ktimer *timer;
4129 int64_t duetime;
4130 kdpc *dpc;
4131 {
4132 return (KeSetTimerEx(timer, duetime, 0, dpc));
4133 }
4134
4135 /*
4136 * The Windows DDK documentation seems to say that cancelling
4137 * a timer that has a DPC will result in the DPC also being
4138 * cancelled, but this isn't really the case.
4139 */
4140
4141 uint8_t
4142 KeCancelTimer(timer)
4143 ktimer *timer;
4144 {
4145 uint8_t pending;
4146
4147 if (timer == NULL)
4148 return (FALSE);
4149
4150 mtx_lock(&ntoskrnl_dispatchlock);
4151
4152 pending = timer->k_header.dh_inserted;
4153
4154 if (timer->k_header.dh_inserted == TRUE) {
4155 timer->k_header.dh_inserted = FALSE;
4156 ntoskrnl_remove_timer(timer);
4157 #ifdef NTOSKRNL_DEBUG_TIMERS
4158 ntoskrnl_timer_cancels++;
4159 #endif
4160 }
4161
4162 mtx_unlock(&ntoskrnl_dispatchlock);
4163
4164 return (pending);
4165 }
4166
4167 uint8_t
4168 KeReadStateTimer(timer)
4169 ktimer *timer;
4170 {
4171 return (timer->k_header.dh_sigstate);
4172 }
4173
4174 static int32_t
4175 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4176 {
4177 ktimer timer;
4178
4179 if (wait_mode != 0)
4180 panic("invalid wait_mode %d", wait_mode);
4181
4182 KeInitializeTimer(&timer);
4183 KeSetTimer(&timer, *interval, NULL);
4184 KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4185
4186 return STATUS_SUCCESS;
4187 }
4188
4189 static uint64_t
4190 KeQueryInterruptTime(void)
4191 {
4192 int ticks;
4193 struct timeval tv;
4194
4195 getmicrouptime(&tv);
4196
4197 ticks = tvtohz(&tv);
4198
4199 return ticks * howmany(10000000, hz);
4200 }
4201
4202 static struct thread *
4203 KeGetCurrentThread(void)
4204 {
4205
4206 return curthread;
4207 }
4208
4209 static int32_t
4210 KeSetPriorityThread(td, pri)
4211 struct thread *td;
4212 int32_t pri;
4213 {
4214 int32_t old;
4215
4216 if (td == NULL)
4217 return LOW_REALTIME_PRIORITY;
4218
4219 if (td->td_priority <= PRI_MIN_KERN)
4220 old = HIGH_PRIORITY;
4221 else if (td->td_priority >= PRI_MAX_KERN)
4222 old = LOW_PRIORITY;
4223 else
4224 old = LOW_REALTIME_PRIORITY;
4225
4226 thread_lock(td);
4227 if (pri == HIGH_PRIORITY)
4228 sched_prio(td, PRI_MIN_KERN);
4229 if (pri == LOW_REALTIME_PRIORITY)
4230 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4231 if (pri == LOW_PRIORITY)
4232 sched_prio(td, PRI_MAX_KERN);
4233 thread_unlock(td);
4234
4235 return old;
4236 }
4237
4238 static void
4239 dummy()
4240 {
4241 printf("ntoskrnl dummy called...\n");
4242 }
4243
4244 image_patch_table ntoskrnl_functbl[] = {
4245 IMPORT_SFUNC(RtlZeroMemory, 2),
4246 IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4247 IMPORT_SFUNC(RtlFillMemory, 3),
4248 IMPORT_SFUNC(RtlMoveMemory, 3),
4249 IMPORT_SFUNC(RtlCharToInteger, 3),
4250 IMPORT_SFUNC(RtlCopyMemory, 3),
4251 IMPORT_SFUNC(RtlCopyString, 2),
4252 IMPORT_SFUNC(RtlCompareMemory, 3),
4253 IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4254 IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4255 IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4256 IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4257 IMPORT_SFUNC(RtlInitAnsiString, 2),
4258 IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4259 IMPORT_SFUNC(RtlInitUnicodeString, 2),
4260 IMPORT_SFUNC(RtlFreeAnsiString, 1),
4261 IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4262 IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4263 IMPORT_CFUNC(sprintf, 0),
4264 IMPORT_CFUNC(vsprintf, 0),
4265 IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4266 IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4267 IMPORT_CFUNC(DbgPrint, 0),
4268 IMPORT_SFUNC(DbgBreakPoint, 0),
4269 IMPORT_SFUNC(KeBugCheckEx, 5),
4270 IMPORT_CFUNC(strncmp, 0),
4271 IMPORT_CFUNC(strcmp, 0),
4272 IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4273 IMPORT_CFUNC(strncpy, 0),
4274 IMPORT_CFUNC(strcpy, 0),
4275 IMPORT_CFUNC(strlen, 0),
4276 IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4277 IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4278 IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4279 IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4280 IMPORT_CFUNC_MAP(strchr, index, 0),
4281 IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4282 IMPORT_CFUNC(memcpy, 0),
4283 IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4284 IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4285 IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4286 IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4287 IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4288 IMPORT_FFUNC(IofCallDriver, 2),
4289 IMPORT_FFUNC(IofCompleteRequest, 2),
4290 IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4291 IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4292 IMPORT_SFUNC(IoCancelIrp, 1),
4293 IMPORT_SFUNC(IoConnectInterrupt, 11),
4294 IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4295 IMPORT_SFUNC(IoCreateDevice, 7),
4296 IMPORT_SFUNC(IoDeleteDevice, 1),
4297 IMPORT_SFUNC(IoGetAttachedDevice, 1),
4298 IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4299 IMPORT_SFUNC(IoDetachDevice, 1),
4300 IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4301 IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4302 IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4303 IMPORT_SFUNC(IoAllocateIrp, 2),
4304 IMPORT_SFUNC(IoReuseIrp, 2),
4305 IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4306 IMPORT_SFUNC(IoFreeIrp, 1),
4307 IMPORT_SFUNC(IoInitializeIrp, 3),
4308 IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4309 IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4310 IMPORT_SFUNC(KeSynchronizeExecution, 3),
4311 IMPORT_SFUNC(KeWaitForSingleObject, 5),
4312 IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4313 IMPORT_SFUNC(_allmul, 4),
4314 IMPORT_SFUNC(_alldiv, 4),
4315 IMPORT_SFUNC(_allrem, 4),
4316 IMPORT_RFUNC(_allshr, 0),
4317 IMPORT_RFUNC(_allshl, 0),
4318 IMPORT_SFUNC(_aullmul, 4),
4319 IMPORT_SFUNC(_aulldiv, 4),
4320 IMPORT_SFUNC(_aullrem, 4),
4321 IMPORT_RFUNC(_aullshr, 0),
4322 IMPORT_RFUNC(_aullshl, 0),
4323 IMPORT_CFUNC(atoi, 0),
4324 IMPORT_CFUNC(atol, 0),
4325 IMPORT_CFUNC(rand, 0),
4326 IMPORT_CFUNC(srand, 0),
4327 IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4328 IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4329 IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4330 IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4331 IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4332 IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4333 IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4334 IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4335 IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4336 IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4337 IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4338 IMPORT_FFUNC(InitializeSListHead, 1),
4339 IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4340 IMPORT_SFUNC(ExQueryDepthSList, 1),
4341 IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4342 InterlockedPopEntrySList, 1),
4343 IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4344 InterlockedPushEntrySList, 2),
4345 IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4346 IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4347 IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4348 IMPORT_SFUNC(ExFreePoolWithTag, 2),
4349 IMPORT_SFUNC(ExFreePool, 1),
4350 #ifdef __i386__
4351 IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4352 IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4353 IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4354 #else
4355 /*
4356 * For AMD64, we can get away with just mapping
4357 * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4358 * because the calling conventions end up being the same.
4359 * On i386, we have to be careful because KfAcquireSpinLock()
4360 * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4361 */
4362 IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4363 IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4364 IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4365 #endif
4366 IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4367 IMPORT_FFUNC(InterlockedIncrement, 1),
4368 IMPORT_FFUNC(InterlockedDecrement, 1),
4369 IMPORT_FFUNC(InterlockedExchange, 2),
4370 IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4371 IMPORT_SFUNC(IoAllocateMdl, 5),
4372 IMPORT_SFUNC(IoFreeMdl, 1),
4373 IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4374 IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4375 IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4376 IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4377 IMPORT_SFUNC(MmSizeOfMdl, 1),
4378 IMPORT_SFUNC(MmMapLockedPages, 2),
4379 IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4380 IMPORT_SFUNC(MmUnmapLockedPages, 2),
4381 IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4382 IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4383 IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4384 IMPORT_SFUNC(MmIsAddressValid, 1),
4385 IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4386 IMPORT_SFUNC(MmUnmapIoSpace, 2),
4387 IMPORT_SFUNC(KeInitializeSpinLock, 1),
4388 IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4389 IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4390 IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4391 IMPORT_SFUNC(IoGetDeviceProperty, 5),
4392 IMPORT_SFUNC(IoAllocateWorkItem, 1),
4393 IMPORT_SFUNC(IoFreeWorkItem, 1),
4394 IMPORT_SFUNC(IoQueueWorkItem, 4),
4395 IMPORT_SFUNC(ExQueueWorkItem, 2),
4396 IMPORT_SFUNC(ntoskrnl_workitem, 2),
4397 IMPORT_SFUNC(KeInitializeMutex, 2),
4398 IMPORT_SFUNC(KeReleaseMutex, 2),
4399 IMPORT_SFUNC(KeReadStateMutex, 1),
4400 IMPORT_SFUNC(KeInitializeEvent, 3),
4401 IMPORT_SFUNC(KeSetEvent, 3),
4402 IMPORT_SFUNC(KeResetEvent, 1),
4403 IMPORT_SFUNC(KeClearEvent, 1),
4404 IMPORT_SFUNC(KeReadStateEvent, 1),
4405 IMPORT_SFUNC(KeInitializeTimer, 1),
4406 IMPORT_SFUNC(KeInitializeTimerEx, 2),
4407 IMPORT_SFUNC(KeSetTimer, 3),
4408 IMPORT_SFUNC(KeSetTimerEx, 4),
4409 IMPORT_SFUNC(KeCancelTimer, 1),
4410 IMPORT_SFUNC(KeReadStateTimer, 1),
4411 IMPORT_SFUNC(KeInitializeDpc, 3),
4412 IMPORT_SFUNC(KeInsertQueueDpc, 3),
4413 IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4414 IMPORT_SFUNC(KeSetImportanceDpc, 2),
4415 IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4416 IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4417 IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4418 IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4419 IMPORT_FFUNC(ObfDereferenceObject, 1),
4420 IMPORT_SFUNC(ZwClose, 1),
4421 IMPORT_SFUNC(PsCreateSystemThread, 7),
4422 IMPORT_SFUNC(PsTerminateSystemThread, 1),
4423 IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4424 IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4425 IMPORT_CFUNC(WmiTraceMessage, 0),
4426 IMPORT_SFUNC(KeQuerySystemTime, 1),
4427 IMPORT_CFUNC(KeTickCount, 0),
4428 IMPORT_SFUNC(KeDelayExecutionThread, 3),
4429 IMPORT_SFUNC(KeQueryInterruptTime, 0),
4430 IMPORT_SFUNC(KeGetCurrentThread, 0),
4431 IMPORT_SFUNC(KeSetPriorityThread, 2),
4432
4433 /*
4434 * This last entry is a catch-all for any function we haven't
4435 * implemented yet. The PE import list patching routine will
4436 * use it for any function that doesn't have an explicit match
4437 * in this table.
4438 */
4439
4440 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4441
4442 /* End of list. */
4443 { NULL, NULL, NULL }
4444 };
4445