1 /*-
2 * Copyright (c) 2003
3 * Bill Paul <wpaul@windriver.com>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by Bill Paul.
16 * 4. Neither the name of the author nor the names of any co-contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
30 * THE POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35
36 #include <sys/param.h>
37 #include <sys/types.h>
38 #include <sys/errno.h>
39
40 #include <sys/callout.h>
41 #include <sys/kernel.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/proc.h>
45 #include <sys/sched.h>
46 #include <sys/module.h>
47
48 #include <sys/systm.h>
49 #include <machine/bus.h>
50
51 #include <sys/bus.h>
52 #include <sys/rman.h>
53
54 #include <compat/ndis/pe_var.h>
55 #include <compat/ndis/resource_var.h>
56 #include <compat/ndis/cfg_var.h>
57 #include <compat/ndis/ntoskrnl_var.h>
58 #include <compat/ndis/hal_var.h>
59
60 static void KeStallExecutionProcessor(uint32_t);
61 static void WRITE_PORT_BUFFER_ULONG(uint32_t *,
62 uint32_t *, uint32_t);
63 static void WRITE_PORT_BUFFER_USHORT(uint16_t *,
64 uint16_t *, uint32_t);
65 static void WRITE_PORT_BUFFER_UCHAR(uint8_t *,
66 uint8_t *, uint32_t);
67 static void WRITE_PORT_ULONG(uint32_t *, uint32_t);
68 static void WRITE_PORT_USHORT(uint16_t *, uint16_t);
69 static void WRITE_PORT_UCHAR(uint8_t *, uint8_t);
70 static uint32_t READ_PORT_ULONG(uint32_t *);
71 static uint16_t READ_PORT_USHORT(uint16_t *);
72 static uint8_t READ_PORT_UCHAR(uint8_t *);
73 static void READ_PORT_BUFFER_ULONG(uint32_t *,
74 uint32_t *, uint32_t);
75 static void READ_PORT_BUFFER_USHORT(uint16_t *,
76 uint16_t *, uint32_t);
77 static void READ_PORT_BUFFER_UCHAR(uint8_t *,
78 uint8_t *, uint32_t);
79 static uint64_t KeQueryPerformanceCounter(uint64_t *);
80 static void _KeLowerIrql(uint8_t);
81 static uint8_t KeRaiseIrqlToDpcLevel(void);
82 static void dummy (void);
83
84 #define NDIS_MAXCPUS 64
85 static struct mtx disp_lock[NDIS_MAXCPUS];
86
87 int
hal_libinit()88 hal_libinit()
89 {
90 image_patch_table *patch;
91 int i;
92
93 for (i = 0; i < NDIS_MAXCPUS; i++)
94 mtx_init(&disp_lock[i], "HAL preemption lock",
95 "HAL lock", MTX_RECURSE|MTX_DEF);
96
97 patch = hal_functbl;
98 while (patch->ipt_func != NULL) {
99 windrv_wrap((funcptr)patch->ipt_func,
100 (funcptr *)&patch->ipt_wrap,
101 patch->ipt_argcnt, patch->ipt_ftype);
102 patch++;
103 }
104
105 return (0);
106 }
107
108 int
hal_libfini()109 hal_libfini()
110 {
111 image_patch_table *patch;
112 int i;
113
114 for (i = 0; i < NDIS_MAXCPUS; i++)
115 mtx_destroy(&disp_lock[i]);
116
117 patch = hal_functbl;
118 while (patch->ipt_func != NULL) {
119 windrv_unwrap(patch->ipt_wrap);
120 patch++;
121 }
122
123 return (0);
124 }
125
126 static void
KeStallExecutionProcessor(usecs)127 KeStallExecutionProcessor(usecs)
128 uint32_t usecs;
129 {
130 DELAY(usecs);
131 }
132
133 static void
WRITE_PORT_ULONG(port,val)134 WRITE_PORT_ULONG(port, val)
135 uint32_t *port;
136 uint32_t val;
137 {
138 bus_space_write_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
139 }
140
141 static void
WRITE_PORT_USHORT(uint16_t * port,uint16_t val)142 WRITE_PORT_USHORT(uint16_t *port, uint16_t val)
143 {
144 bus_space_write_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
145 }
146
147 static void
WRITE_PORT_UCHAR(uint8_t * port,uint8_t val)148 WRITE_PORT_UCHAR(uint8_t *port, uint8_t val)
149 {
150 bus_space_write_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
151 }
152
153 static void
WRITE_PORT_BUFFER_ULONG(port,val,cnt)154 WRITE_PORT_BUFFER_ULONG(port, val, cnt)
155 uint32_t *port;
156 uint32_t *val;
157 uint32_t cnt;
158 {
159 bus_space_write_multi_4(NDIS_BUS_SPACE_IO, 0x0,
160 (bus_size_t)port, val, cnt);
161 }
162
163 static void
WRITE_PORT_BUFFER_USHORT(port,val,cnt)164 WRITE_PORT_BUFFER_USHORT(port, val, cnt)
165 uint16_t *port;
166 uint16_t *val;
167 uint32_t cnt;
168 {
169 bus_space_write_multi_2(NDIS_BUS_SPACE_IO, 0x0,
170 (bus_size_t)port, val, cnt);
171 }
172
173 static void
WRITE_PORT_BUFFER_UCHAR(port,val,cnt)174 WRITE_PORT_BUFFER_UCHAR(port, val, cnt)
175 uint8_t *port;
176 uint8_t *val;
177 uint32_t cnt;
178 {
179 bus_space_write_multi_1(NDIS_BUS_SPACE_IO, 0x0,
180 (bus_size_t)port, val, cnt);
181 }
182
183 static uint16_t
READ_PORT_USHORT(port)184 READ_PORT_USHORT(port)
185 uint16_t *port;
186 {
187 return (bus_space_read_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
188 }
189
190 static uint32_t
READ_PORT_ULONG(port)191 READ_PORT_ULONG(port)
192 uint32_t *port;
193 {
194 return (bus_space_read_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
195 }
196
197 static uint8_t
READ_PORT_UCHAR(port)198 READ_PORT_UCHAR(port)
199 uint8_t *port;
200 {
201 return (bus_space_read_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
202 }
203
204 static void
READ_PORT_BUFFER_ULONG(port,val,cnt)205 READ_PORT_BUFFER_ULONG(port, val, cnt)
206 uint32_t *port;
207 uint32_t *val;
208 uint32_t cnt;
209 {
210 bus_space_read_multi_4(NDIS_BUS_SPACE_IO, 0x0,
211 (bus_size_t)port, val, cnt);
212 }
213
214 static void
READ_PORT_BUFFER_USHORT(port,val,cnt)215 READ_PORT_BUFFER_USHORT(port, val, cnt)
216 uint16_t *port;
217 uint16_t *val;
218 uint32_t cnt;
219 {
220 bus_space_read_multi_2(NDIS_BUS_SPACE_IO, 0x0,
221 (bus_size_t)port, val, cnt);
222 }
223
224 static void
READ_PORT_BUFFER_UCHAR(port,val,cnt)225 READ_PORT_BUFFER_UCHAR(port, val, cnt)
226 uint8_t *port;
227 uint8_t *val;
228 uint32_t cnt;
229 {
230 bus_space_read_multi_1(NDIS_BUS_SPACE_IO, 0x0,
231 (bus_size_t)port, val, cnt);
232 }
233
234 /*
235 * The spinlock implementation in Windows differs from that of FreeBSD.
236 * The basic operation of spinlocks involves two steps: 1) spin in a
237 * tight loop while trying to acquire a lock, 2) after obtaining the
238 * lock, disable preemption. (Note that on uniprocessor systems, you're
239 * allowed to skip the first step and just lock out pre-emption, since
240 * it's not possible for you to be in contention with another running
241 * thread.) Later, you release the lock then re-enable preemption.
242 * The difference between Windows and FreeBSD lies in how preemption
243 * is disabled. In FreeBSD, it's done using critical_enter(), which on
244 * the x86 arch translates to a cli instruction. This masks off all
245 * interrupts, and effectively stops the scheduler from ever running
246 * so _nothing_ can execute except the current thread. In Windows,
247 * preemption is disabled by raising the processor IRQL to DISPATCH_LEVEL.
248 * This stops other threads from running, but does _not_ block device
249 * interrupts. This means ISRs can still run, and they can make other
250 * threads runable, but those other threads won't be able to execute
251 * until the current thread lowers the IRQL to something less than
252 * DISPATCH_LEVEL.
253 *
254 * There's another commonly used IRQL in Windows, which is APC_LEVEL.
255 * An APC is an Asynchronous Procedure Call, which differs from a DPC
256 * (Defered Procedure Call) in that a DPC is queued up to run in
257 * another thread, while an APC runs in the thread that scheduled
258 * it (similar to a signal handler in a UNIX process). We don't
259 * actually support the notion of APCs in FreeBSD, so for now, the
260 * only IRQLs we're interested in are DISPATCH_LEVEL and PASSIVE_LEVEL.
261 *
262 * To simulate DISPATCH_LEVEL, we raise the current thread's priority
263 * to PI_REALTIME, which is the highest we can give it. This should,
264 * if I understand things correctly, prevent anything except for an
265 * interrupt thread from preempting us. PASSIVE_LEVEL is basically
266 * everything else.
267 *
268 * Be aware that, at least on the x86 arch, the Windows spinlock
269 * functions are divided up in peculiar ways. The actual spinlock
270 * functions are KfAcquireSpinLock() and KfReleaseSpinLock(), and
271 * they live in HAL.dll. Meanwhile, KeInitializeSpinLock(),
272 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel()
273 * live in ntoskrnl.exe. Most Windows source code will call
274 * KeAcquireSpinLock() and KeReleaseSpinLock(), but these are just
275 * macros that call KfAcquireSpinLock() and KfReleaseSpinLock().
276 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel()
277 * perform the lock aquisition/release functions without doing the
278 * IRQL manipulation, and are used when one is already running at
279 * DISPATCH_LEVEL. Make sense? Good.
280 *
281 * According to the Microsoft documentation, any thread that calls
282 * KeAcquireSpinLock() must be running at IRQL <= DISPATCH_LEVEL. If
283 * we detect someone trying to acquire a spinlock from DEVICE_LEVEL
284 * or HIGH_LEVEL, we panic.
285 *
286 * Alternate sleep-lock-based spinlock implementation
287 * --------------------------------------------------
288 *
289 * The earlier spinlock implementation was arguably a bit of a hack
290 * and presented several problems. It was basically designed to provide
291 * the functionality of spinlocks without incurring the wrath of
292 * WITNESS. We could get away with using both our spinlock implementation
293 * and FreeBSD sleep locks at the same time, but if WITNESS knew what
294 * we were really up to, it would have spanked us rather severely.
295 *
296 * There's another method we can use based entirely on sleep locks.
297 * First, it's important to realize that everything we're locking
298 * resides inside Project Evil itself: any critical data being locked
299 * by drivers belongs to the drivers, and should not be referenced
300 * by any other OS code outside of the NDISulator. The priority-based
301 * locking scheme has system-wide effects, just like real spinlocks
302 * (blocking preemption affects the whole CPU), but since we keep all
303 * our critical data private, we can use a simpler mechanism that
304 * affects only code/threads directly related to Project Evil.
305 *
306 * The idea is to create a sleep lock mutex for each CPU in the system.
307 * When a CPU running in the NDISulator wants to acquire a spinlock, it
308 * does the following:
309 * - Pin ourselves to the current CPU
310 * - Acquire the mutex for the current CPU
311 * - Spin on the spinlock variable using atomic test and set, just like
312 * a real spinlock.
313 * - Once we have the lock, we execute our critical code
314 *
315 * To give up the lock, we do:
316 * - Clear the spinlock variable with an atomic op
317 * - Release the per-CPU mutex
318 * - Unpin ourselves from the current CPU.
319 *
320 * On a uniprocessor system, this means all threads that access protected
321 * data are serialized through the per-CPU mutex. After one thread
322 * acquires the 'spinlock,' any other thread that uses a spinlock on the
323 * current CPU will block on the per-CPU mutex, which has the same general
324 * effect of blocking pre-emption, but _only_ for those threads that are
325 * running NDISulator code.
326 *
327 * On a multiprocessor system, threads on different CPUs all block on
328 * their respective per-CPU mutex, and the atomic test/set operation
329 * on the spinlock variable provides inter-CPU synchronization, though
330 * only for threads running NDISulator code.
331 *
332 * This method solves an important problem. In Windows, you're allowed
333 * to do an ExAllocatePoolWithTag() with a spinlock held, provided you
334 * allocate from NonPagedPool. This implies an atomic heap allocation
335 * that will not cause the current thread to sleep. (You can't sleep
336 * while holding real spinlock: clowns will eat you.) But in FreeBSD,
337 * malloc(9) _always_ triggers the acquisition of a sleep lock, even
338 * when you use M_NOWAIT. This is not a problem for FreeBSD native
339 * code: you're allowed to sleep in things like interrupt threads. But
340 * it is a problem with the old priority-based spinlock implementation:
341 * even though we get away with it most of the time, we really can't
342 * do a malloc(9) after doing a KeAcquireSpinLock() or KeRaiseIrql().
343 * With the new implementation, it's not a problem: you're allowed to
344 * acquire more than one sleep lock (as long as you avoid lock order
345 * reversals).
346 *
347 * The one drawback to this approach is that now we have a lot of
348 * contention on one per-CPU mutex within the NDISulator code. Whether
349 * or not this is preferable to the expected Windows spinlock behavior
350 * of blocking pre-emption is debatable.
351 */
352
353 uint8_t
KfAcquireSpinLock(lock)354 KfAcquireSpinLock(lock)
355 kspin_lock *lock;
356 {
357 uint8_t oldirql;
358
359 KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
360 KeAcquireSpinLockAtDpcLevel(lock);
361
362 return (oldirql);
363 }
364
365 void
KfReleaseSpinLock(kspin_lock * lock,uint8_t newirql)366 KfReleaseSpinLock(kspin_lock *lock, uint8_t newirql)
367 {
368 KeReleaseSpinLockFromDpcLevel(lock);
369 KeLowerIrql(newirql);
370 }
371
372 uint8_t
KeGetCurrentIrql()373 KeGetCurrentIrql()
374 {
375 if (mtx_owned(&disp_lock[curthread->td_oncpu]))
376 return (DISPATCH_LEVEL);
377 return (PASSIVE_LEVEL);
378 }
379
380 static uint64_t
KeQueryPerformanceCounter(freq)381 KeQueryPerformanceCounter(freq)
382 uint64_t *freq;
383 {
384 if (freq != NULL)
385 *freq = hz;
386
387 return ((uint64_t)ticks);
388 }
389
390 uint8_t
KfRaiseIrql(uint8_t irql)391 KfRaiseIrql(uint8_t irql)
392 {
393 uint8_t oldirql;
394
395 sched_pin();
396 oldirql = KeGetCurrentIrql();
397
398 /* I am so going to hell for this. */
399 if (oldirql > irql)
400 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
401
402 if (oldirql != DISPATCH_LEVEL)
403 mtx_lock(&disp_lock[curthread->td_oncpu]);
404 else
405 sched_unpin();
406
407 /*printf("RAISE IRQL: %d %d\n", irql, oldirql);*/
408
409 return (oldirql);
410 }
411
412 void
KfLowerIrql(uint8_t oldirql)413 KfLowerIrql(uint8_t oldirql)
414 {
415 if (oldirql == DISPATCH_LEVEL)
416 return;
417
418 if (KeGetCurrentIrql() != DISPATCH_LEVEL)
419 panic("IRQL_NOT_GREATER_THAN");
420
421 mtx_unlock(&disp_lock[curthread->td_oncpu]);
422 sched_unpin();
423 }
424
425 static uint8_t
KeRaiseIrqlToDpcLevel(void)426 KeRaiseIrqlToDpcLevel(void)
427 {
428 uint8_t irql;
429
430 KeRaiseIrql(DISPATCH_LEVEL, &irql);
431 return (irql);
432 }
433
434 static void
_KeLowerIrql(uint8_t oldirql)435 _KeLowerIrql(uint8_t oldirql)
436 {
437 KeLowerIrql(oldirql);
438 }
439
dummy()440 static void dummy()
441 {
442 printf("hal dummy called...\n");
443 }
444
445 image_patch_table hal_functbl[] = {
446 IMPORT_SFUNC(KeStallExecutionProcessor, 1),
447 IMPORT_SFUNC(WRITE_PORT_ULONG, 2),
448 IMPORT_SFUNC(WRITE_PORT_USHORT, 2),
449 IMPORT_SFUNC(WRITE_PORT_UCHAR, 2),
450 IMPORT_SFUNC(WRITE_PORT_BUFFER_ULONG, 3),
451 IMPORT_SFUNC(WRITE_PORT_BUFFER_USHORT, 3),
452 IMPORT_SFUNC(WRITE_PORT_BUFFER_UCHAR, 3),
453 IMPORT_SFUNC(READ_PORT_ULONG, 1),
454 IMPORT_SFUNC(READ_PORT_USHORT, 1),
455 IMPORT_SFUNC(READ_PORT_UCHAR, 1),
456 IMPORT_SFUNC(READ_PORT_BUFFER_ULONG, 3),
457 IMPORT_SFUNC(READ_PORT_BUFFER_USHORT, 3),
458 IMPORT_SFUNC(READ_PORT_BUFFER_UCHAR, 3),
459 IMPORT_FFUNC(KfAcquireSpinLock, 1),
460 IMPORT_FFUNC(KfReleaseSpinLock, 1),
461 IMPORT_SFUNC(KeGetCurrentIrql, 0),
462 IMPORT_SFUNC(KeQueryPerformanceCounter, 1),
463 IMPORT_FFUNC(KfLowerIrql, 1),
464 IMPORT_FFUNC(KfRaiseIrql, 1),
465 IMPORT_SFUNC(KeRaiseIrqlToDpcLevel, 0),
466 #undef KeLowerIrql
467 IMPORT_SFUNC_MAP(KeLowerIrql, _KeLowerIrql, 1),
468
469 /*
470 * This last entry is a catch-all for any function we haven't
471 * implemented yet. The PE import list patching routine will
472 * use it for any function that doesn't have an explicit match
473 * in this table.
474 */
475
476 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
477
478 /* End of list. */
479
480 { NULL, NULL, NULL }
481 };
482