1 /*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1993 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * $FreeBSD$
31 */
32
33 /*
34 * Functions to provide access to special i386 instructions.
35 * This in included in sys/systm.h, and that file should be
36 * used in preference to this.
37 */
38
39 #ifndef _MACHINE_CPUFUNC_H_
40 #define _MACHINE_CPUFUNC_H_
41
42 #ifndef _SYS_CDEFS_H_
43 #error this file needs sys/cdefs.h as a prerequisite
44 #endif
45
46 struct region_descriptor;
47
48 #define readb(va) (*(volatile uint8_t *) (va))
49 #define readw(va) (*(volatile uint16_t *) (va))
50 #define readl(va) (*(volatile uint32_t *) (va))
51 #define readq(va) (*(volatile uint64_t *) (va))
52
53 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
54 #define writew(va, d) (*(volatile uint16_t *) (va) = (d))
55 #define writel(va, d) (*(volatile uint32_t *) (va) = (d))
56 #define writeq(va, d) (*(volatile uint64_t *) (va) = (d))
57
58 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
59
60 static __inline void
breakpoint(void)61 breakpoint(void)
62 {
63 __asm __volatile("int $3");
64 }
65
66 static __inline u_int
bsfl(u_int mask)67 bsfl(u_int mask)
68 {
69 u_int result;
70
71 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
72 return (result);
73 }
74
75 static __inline u_long
bsfq(u_long mask)76 bsfq(u_long mask)
77 {
78 u_long result;
79
80 __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask));
81 return (result);
82 }
83
84 static __inline u_int
bsrl(u_int mask)85 bsrl(u_int mask)
86 {
87 u_int result;
88
89 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
90 return (result);
91 }
92
93 static __inline u_long
bsrq(u_long mask)94 bsrq(u_long mask)
95 {
96 u_long result;
97
98 __asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask));
99 return (result);
100 }
101
102 static __inline void
clflush(u_long addr)103 clflush(u_long addr)
104 {
105
106 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
107 }
108
109 static __inline void
clflushopt(u_long addr)110 clflushopt(u_long addr)
111 {
112
113 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
114 }
115
116 static __inline void
clwb(u_long addr)117 clwb(u_long addr)
118 {
119
120 __asm __volatile("clwb %0" : : "m" (*(char *)addr));
121 }
122
123 static __inline void
clts(void)124 clts(void)
125 {
126
127 __asm __volatile("clts");
128 }
129
130 static __inline void
disable_intr(void)131 disable_intr(void)
132 {
133 __asm __volatile("cli" : : : "memory");
134 }
135
136 static __inline void
do_cpuid(u_int ax,u_int * p)137 do_cpuid(u_int ax, u_int *p)
138 {
139 __asm __volatile("cpuid"
140 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
141 : "0" (ax));
142 }
143
144 static __inline void
cpuid_count(u_int ax,u_int cx,u_int * p)145 cpuid_count(u_int ax, u_int cx, u_int *p)
146 {
147 __asm __volatile("cpuid"
148 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
149 : "0" (ax), "c" (cx));
150 }
151
152 static __inline void
enable_intr(void)153 enable_intr(void)
154 {
155 __asm __volatile("sti");
156 }
157
158 #ifdef _KERNEL
159
160 #define HAVE_INLINE_FFS
161 #define ffs(x) __builtin_ffs(x)
162
163 #define HAVE_INLINE_FFSL
164
165 static __inline int
ffsl(long mask)166 ffsl(long mask)
167 {
168 return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1);
169 }
170
171 #define HAVE_INLINE_FFSLL
172
173 static __inline int
ffsll(long long mask)174 ffsll(long long mask)
175 {
176 return (ffsl((long)mask));
177 }
178
179 #define HAVE_INLINE_FLS
180
181 static __inline int
fls(int mask)182 fls(int mask)
183 {
184 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
185 }
186
187 #define HAVE_INLINE_FLSL
188
189 static __inline int
flsl(long mask)190 flsl(long mask)
191 {
192 return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1);
193 }
194
195 #define HAVE_INLINE_FLSLL
196
197 static __inline int
flsll(long long mask)198 flsll(long long mask)
199 {
200 return (flsl((long)mask));
201 }
202
203 #endif /* _KERNEL */
204
205 static __inline void
halt(void)206 halt(void)
207 {
208 __asm __volatile("hlt");
209 }
210
211 static __inline u_char
inb(u_int port)212 inb(u_int port)
213 {
214 u_char data;
215
216 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
217 return (data);
218 }
219
220 static __inline u_int
inl(u_int port)221 inl(u_int port)
222 {
223 u_int data;
224
225 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
226 return (data);
227 }
228
229 static __inline void
insb(u_int port,void * addr,size_t count)230 insb(u_int port, void *addr, size_t count)
231 {
232 __asm __volatile("cld; rep; insb"
233 : "+D" (addr), "+c" (count)
234 : "d" (port)
235 : "memory");
236 }
237
238 static __inline void
insw(u_int port,void * addr,size_t count)239 insw(u_int port, void *addr, size_t count)
240 {
241 __asm __volatile("cld; rep; insw"
242 : "+D" (addr), "+c" (count)
243 : "d" (port)
244 : "memory");
245 }
246
247 static __inline void
insl(u_int port,void * addr,size_t count)248 insl(u_int port, void *addr, size_t count)
249 {
250 __asm __volatile("cld; rep; insl"
251 : "+D" (addr), "+c" (count)
252 : "d" (port)
253 : "memory");
254 }
255
256 static __inline void
invd(void)257 invd(void)
258 {
259 __asm __volatile("invd");
260 }
261
262 static __inline u_short
inw(u_int port)263 inw(u_int port)
264 {
265 u_short data;
266
267 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
268 return (data);
269 }
270
271 static __inline void
outb(u_int port,u_char data)272 outb(u_int port, u_char data)
273 {
274 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
275 }
276
277 static __inline void
outl(u_int port,u_int data)278 outl(u_int port, u_int data)
279 {
280 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
281 }
282
283 static __inline void
outsb(u_int port,const void * addr,size_t count)284 outsb(u_int port, const void *addr, size_t count)
285 {
286 __asm __volatile("cld; rep; outsb"
287 : "+S" (addr), "+c" (count)
288 : "d" (port));
289 }
290
291 static __inline void
outsw(u_int port,const void * addr,size_t count)292 outsw(u_int port, const void *addr, size_t count)
293 {
294 __asm __volatile("cld; rep; outsw"
295 : "+S" (addr), "+c" (count)
296 : "d" (port));
297 }
298
299 static __inline void
outsl(u_int port,const void * addr,size_t count)300 outsl(u_int port, const void *addr, size_t count)
301 {
302 __asm __volatile("cld; rep; outsl"
303 : "+S" (addr), "+c" (count)
304 : "d" (port));
305 }
306
307 static __inline void
outw(u_int port,u_short data)308 outw(u_int port, u_short data)
309 {
310 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
311 }
312
313 static __inline u_long
popcntq(u_long mask)314 popcntq(u_long mask)
315 {
316 u_long result;
317
318 __asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask));
319 return (result);
320 }
321
322 static __inline void
lfence(void)323 lfence(void)
324 {
325
326 __asm __volatile("lfence" : : : "memory");
327 }
328
329 static __inline void
mfence(void)330 mfence(void)
331 {
332
333 __asm __volatile("mfence" : : : "memory");
334 }
335
336 static __inline void
sfence(void)337 sfence(void)
338 {
339
340 __asm __volatile("sfence" : : : "memory");
341 }
342
343 static __inline void
ia32_pause(void)344 ia32_pause(void)
345 {
346 __asm __volatile("pause");
347 }
348
349 static __inline u_long
read_rflags(void)350 read_rflags(void)
351 {
352 u_long rf;
353
354 __asm __volatile("pushfq; popq %0" : "=r" (rf));
355 return (rf);
356 }
357
358 static __inline uint64_t
rdmsr(u_int msr)359 rdmsr(u_int msr)
360 {
361 uint32_t low, high;
362
363 __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr));
364 return (low | ((uint64_t)high << 32));
365 }
366
367 static __inline uint32_t
rdmsr32(u_int msr)368 rdmsr32(u_int msr)
369 {
370 uint32_t low;
371
372 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "rdx");
373 return (low);
374 }
375
376 static __inline uint64_t
rdpmc(u_int pmc)377 rdpmc(u_int pmc)
378 {
379 uint32_t low, high;
380
381 __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc));
382 return (low | ((uint64_t)high << 32));
383 }
384
385 static __inline uint64_t
rdtsc(void)386 rdtsc(void)
387 {
388 uint32_t low, high;
389
390 __asm __volatile("rdtsc" : "=a" (low), "=d" (high));
391 return (low | ((uint64_t)high << 32));
392 }
393
394 static __inline uint32_t
rdtsc32(void)395 rdtsc32(void)
396 {
397 uint32_t rv;
398
399 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
400 return (rv);
401 }
402
403 static __inline void
wbinvd(void)404 wbinvd(void)
405 {
406 __asm __volatile("wbinvd");
407 }
408
409 static __inline void
write_rflags(u_long rf)410 write_rflags(u_long rf)
411 {
412 __asm __volatile("pushq %0; popfq" : : "r" (rf));
413 }
414
415 static __inline void
wrmsr(u_int msr,uint64_t newval)416 wrmsr(u_int msr, uint64_t newval)
417 {
418 uint32_t low, high;
419
420 low = newval;
421 high = newval >> 32;
422 __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr));
423 }
424
425 static __inline void
load_cr0(u_long data)426 load_cr0(u_long data)
427 {
428
429 __asm __volatile("movq %0,%%cr0" : : "r" (data));
430 }
431
432 static __inline u_long
rcr0(void)433 rcr0(void)
434 {
435 u_long data;
436
437 __asm __volatile("movq %%cr0,%0" : "=r" (data));
438 return (data);
439 }
440
441 static __inline u_long
rcr2(void)442 rcr2(void)
443 {
444 u_long data;
445
446 __asm __volatile("movq %%cr2,%0" : "=r" (data));
447 return (data);
448 }
449
450 static __inline void
load_cr3(u_long data)451 load_cr3(u_long data)
452 {
453
454 __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory");
455 }
456
457 static __inline u_long
rcr3(void)458 rcr3(void)
459 {
460 u_long data;
461
462 __asm __volatile("movq %%cr3,%0" : "=r" (data));
463 return (data);
464 }
465
466 static __inline void
load_cr4(u_long data)467 load_cr4(u_long data)
468 {
469 __asm __volatile("movq %0,%%cr4" : : "r" (data));
470 }
471
472 static __inline u_long
rcr4(void)473 rcr4(void)
474 {
475 u_long data;
476
477 __asm __volatile("movq %%cr4,%0" : "=r" (data));
478 return (data);
479 }
480
481 static __inline u_long
rxcr(u_int reg)482 rxcr(u_int reg)
483 {
484 u_int low, high;
485
486 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
487 return (low | ((uint64_t)high << 32));
488 }
489
490 static __inline void
load_xcr(u_int reg,u_long val)491 load_xcr(u_int reg, u_long val)
492 {
493 u_int low, high;
494
495 low = val;
496 high = val >> 32;
497 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
498 }
499
500 /*
501 * Global TLB flush (except for thise for pages marked PG_G)
502 */
503 static __inline void
invltlb(void)504 invltlb(void)
505 {
506
507 load_cr3(rcr3());
508 }
509
510 #ifndef CR4_PGE
511 #define CR4_PGE 0x00000080 /* Page global enable */
512 #endif
513
514 /*
515 * Perform the guaranteed invalidation of all TLB entries. This
516 * includes the global entries, and entries in all PCIDs, not only the
517 * current context. The function works both on non-PCID CPUs and CPUs
518 * with the PCID turned off or on. See IA-32 SDM Vol. 3a 4.10.4.1
519 * Operations that Invalidate TLBs and Paging-Structure Caches.
520 */
521 static __inline void
invltlb_glob(void)522 invltlb_glob(void)
523 {
524 uint64_t cr4;
525
526 cr4 = rcr4();
527 load_cr4(cr4 & ~CR4_PGE);
528 /*
529 * Although preemption at this point could be detrimental to
530 * performance, it would not lead to an error. PG_G is simply
531 * ignored if CR4.PGE is clear. Moreover, in case this block
532 * is re-entered, the load_cr4() either above or below will
533 * modify CR4.PGE flushing the TLB.
534 */
535 load_cr4(cr4 | CR4_PGE);
536 }
537
538 /*
539 * TLB flush for an individual page (even if it has PG_G).
540 * Only works on 486+ CPUs (i386 does not have PG_G).
541 */
542 static __inline void
invlpg(u_long addr)543 invlpg(u_long addr)
544 {
545
546 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
547 }
548
549 #define INVPCID_ADDR 0
550 #define INVPCID_CTX 1
551 #define INVPCID_CTXGLOB 2
552 #define INVPCID_ALLCTX 3
553
554 struct invpcid_descr {
555 uint64_t pcid:12 __packed;
556 uint64_t pad:52 __packed;
557 uint64_t addr;
558 } __packed;
559
560 static __inline void
invpcid(struct invpcid_descr * d,int type)561 invpcid(struct invpcid_descr *d, int type)
562 {
563
564 __asm __volatile("invpcid (%0),%1"
565 : : "r" (d), "r" ((u_long)type) : "memory");
566 }
567
568 static __inline u_short
rfs(void)569 rfs(void)
570 {
571 u_short sel;
572 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
573 return (sel);
574 }
575
576 static __inline u_short
rgs(void)577 rgs(void)
578 {
579 u_short sel;
580 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
581 return (sel);
582 }
583
584 static __inline u_short
rss(void)585 rss(void)
586 {
587 u_short sel;
588 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
589 return (sel);
590 }
591
592 static __inline void
load_ds(u_short sel)593 load_ds(u_short sel)
594 {
595 __asm __volatile("movw %0,%%ds" : : "rm" (sel));
596 }
597
598 static __inline void
load_es(u_short sel)599 load_es(u_short sel)
600 {
601 __asm __volatile("movw %0,%%es" : : "rm" (sel));
602 }
603
604 static __inline void
cpu_monitor(const void * addr,u_long extensions,u_int hints)605 cpu_monitor(const void *addr, u_long extensions, u_int hints)
606 {
607
608 __asm __volatile("monitor"
609 : : "a" (addr), "c" (extensions), "d" (hints));
610 }
611
612 static __inline void
cpu_mwait(u_long extensions,u_int hints)613 cpu_mwait(u_long extensions, u_int hints)
614 {
615
616 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
617 }
618
619 #ifdef _KERNEL
620 /* This is defined in <machine/specialreg.h> but is too painful to get to */
621 #ifndef MSR_FSBASE
622 #define MSR_FSBASE 0xc0000100
623 #endif
624 static __inline void
load_fs(u_short sel)625 load_fs(u_short sel)
626 {
627 /* Preserve the fsbase value across the selector load */
628 __asm __volatile("rdmsr; movw %0,%%fs; wrmsr"
629 : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx");
630 }
631
632 #ifndef MSR_GSBASE
633 #define MSR_GSBASE 0xc0000101
634 #endif
635 static __inline void
load_gs(u_short sel)636 load_gs(u_short sel)
637 {
638 /*
639 * Preserve the gsbase value across the selector load.
640 * Note that we have to disable interrupts because the gsbase
641 * being trashed happens to be the kernel gsbase at the time.
642 */
643 __asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq"
644 : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx");
645 }
646 #else
647 /* Usable by userland */
648 static __inline void
load_fs(u_short sel)649 load_fs(u_short sel)
650 {
651 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
652 }
653
654 static __inline void
load_gs(u_short sel)655 load_gs(u_short sel)
656 {
657 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
658 }
659 #endif
660
661 static __inline uint64_t
rdfsbase(void)662 rdfsbase(void)
663 {
664 uint64_t x;
665
666 __asm __volatile("rdfsbase %0" : "=r" (x));
667 return (x);
668 }
669
670 static __inline void
wrfsbase(uint64_t x)671 wrfsbase(uint64_t x)
672 {
673
674 __asm __volatile("wrfsbase %0" : : "r" (x));
675 }
676
677 static __inline uint64_t
rdgsbase(void)678 rdgsbase(void)
679 {
680 uint64_t x;
681
682 __asm __volatile("rdgsbase %0" : "=r" (x));
683 return (x);
684 }
685
686 static __inline void
wrgsbase(uint64_t x)687 wrgsbase(uint64_t x)
688 {
689
690 __asm __volatile("wrgsbase %0" : : "r" (x));
691 }
692
693 static __inline void
bare_lgdt(struct region_descriptor * addr)694 bare_lgdt(struct region_descriptor *addr)
695 {
696 __asm __volatile("lgdt (%0)" : : "r" (addr));
697 }
698
699 static __inline void
sgdt(struct region_descriptor * addr)700 sgdt(struct region_descriptor *addr)
701 {
702 char *loc;
703
704 loc = (char *)addr;
705 __asm __volatile("sgdt %0" : "=m" (*loc) : : "memory");
706 }
707
708 static __inline void
lidt(struct region_descriptor * addr)709 lidt(struct region_descriptor *addr)
710 {
711 __asm __volatile("lidt (%0)" : : "r" (addr));
712 }
713
714 static __inline void
sidt(struct region_descriptor * addr)715 sidt(struct region_descriptor *addr)
716 {
717 char *loc;
718
719 loc = (char *)addr;
720 __asm __volatile("sidt %0" : "=m" (*loc) : : "memory");
721 }
722
723 static __inline void
lldt(u_short sel)724 lldt(u_short sel)
725 {
726 __asm __volatile("lldt %0" : : "r" (sel));
727 }
728
729 static __inline u_short
sldt(void)730 sldt(void)
731 {
732 u_short sel;
733
734 __asm __volatile("sldt %0" : "=r" (sel));
735 return (sel);
736 }
737
738 static __inline void
ltr(u_short sel)739 ltr(u_short sel)
740 {
741 __asm __volatile("ltr %0" : : "r" (sel));
742 }
743
744 static __inline uint32_t
read_tr(void)745 read_tr(void)
746 {
747 u_short sel;
748
749 __asm __volatile("str %0" : "=r" (sel));
750 return (sel);
751 }
752
753 static __inline uint64_t
rdr0(void)754 rdr0(void)
755 {
756 uint64_t data;
757 __asm __volatile("movq %%dr0,%0" : "=r" (data));
758 return (data);
759 }
760
761 static __inline void
load_dr0(uint64_t dr0)762 load_dr0(uint64_t dr0)
763 {
764 __asm __volatile("movq %0,%%dr0" : : "r" (dr0));
765 }
766
767 static __inline uint64_t
rdr1(void)768 rdr1(void)
769 {
770 uint64_t data;
771 __asm __volatile("movq %%dr1,%0" : "=r" (data));
772 return (data);
773 }
774
775 static __inline void
load_dr1(uint64_t dr1)776 load_dr1(uint64_t dr1)
777 {
778 __asm __volatile("movq %0,%%dr1" : : "r" (dr1));
779 }
780
781 static __inline uint64_t
rdr2(void)782 rdr2(void)
783 {
784 uint64_t data;
785 __asm __volatile("movq %%dr2,%0" : "=r" (data));
786 return (data);
787 }
788
789 static __inline void
load_dr2(uint64_t dr2)790 load_dr2(uint64_t dr2)
791 {
792 __asm __volatile("movq %0,%%dr2" : : "r" (dr2));
793 }
794
795 static __inline uint64_t
rdr3(void)796 rdr3(void)
797 {
798 uint64_t data;
799 __asm __volatile("movq %%dr3,%0" : "=r" (data));
800 return (data);
801 }
802
803 static __inline void
load_dr3(uint64_t dr3)804 load_dr3(uint64_t dr3)
805 {
806 __asm __volatile("movq %0,%%dr3" : : "r" (dr3));
807 }
808
809 static __inline uint64_t
rdr4(void)810 rdr4(void)
811 {
812 uint64_t data;
813 __asm __volatile("movq %%dr4,%0" : "=r" (data));
814 return (data);
815 }
816
817 static __inline void
load_dr4(uint64_t dr4)818 load_dr4(uint64_t dr4)
819 {
820 __asm __volatile("movq %0,%%dr4" : : "r" (dr4));
821 }
822
823 static __inline uint64_t
rdr5(void)824 rdr5(void)
825 {
826 uint64_t data;
827 __asm __volatile("movq %%dr5,%0" : "=r" (data));
828 return (data);
829 }
830
831 static __inline void
load_dr5(uint64_t dr5)832 load_dr5(uint64_t dr5)
833 {
834 __asm __volatile("movq %0,%%dr5" : : "r" (dr5));
835 }
836
837 static __inline uint64_t
rdr6(void)838 rdr6(void)
839 {
840 uint64_t data;
841 __asm __volatile("movq %%dr6,%0" : "=r" (data));
842 return (data);
843 }
844
845 static __inline void
load_dr6(uint64_t dr6)846 load_dr6(uint64_t dr6)
847 {
848 __asm __volatile("movq %0,%%dr6" : : "r" (dr6));
849 }
850
851 static __inline uint64_t
rdr7(void)852 rdr7(void)
853 {
854 uint64_t data;
855 __asm __volatile("movq %%dr7,%0" : "=r" (data));
856 return (data);
857 }
858
859 static __inline void
load_dr7(uint64_t dr7)860 load_dr7(uint64_t dr7)
861 {
862 __asm __volatile("movq %0,%%dr7" : : "r" (dr7));
863 }
864
865 static __inline register_t
intr_disable(void)866 intr_disable(void)
867 {
868 register_t rflags;
869
870 rflags = read_rflags();
871 disable_intr();
872 return (rflags);
873 }
874
875 static __inline void
intr_restore(register_t rflags)876 intr_restore(register_t rflags)
877 {
878 write_rflags(rflags);
879 }
880
881 static __inline void
stac(void)882 stac(void)
883 {
884
885 __asm __volatile("stac" : : : "cc");
886 }
887
888 static __inline void
clac(void)889 clac(void)
890 {
891
892 __asm __volatile("clac" : : : "cc");
893 }
894
895 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
896
897 int breakpoint(void);
898 u_int bsfl(u_int mask);
899 u_int bsrl(u_int mask);
900 void clflush(u_long addr);
901 void clts(void);
902 void cpuid_count(u_int ax, u_int cx, u_int *p);
903 void disable_intr(void);
904 void do_cpuid(u_int ax, u_int *p);
905 void enable_intr(void);
906 void halt(void);
907 void ia32_pause(void);
908 u_char inb(u_int port);
909 u_int inl(u_int port);
910 void insb(u_int port, void *addr, size_t count);
911 void insl(u_int port, void *addr, size_t count);
912 void insw(u_int port, void *addr, size_t count);
913 register_t intr_disable(void);
914 void intr_restore(register_t rf);
915 void invd(void);
916 void invlpg(u_int addr);
917 void invltlb(void);
918 u_short inw(u_int port);
919 void lidt(struct region_descriptor *addr);
920 void lldt(u_short sel);
921 void load_cr0(u_long cr0);
922 void load_cr3(u_long cr3);
923 void load_cr4(u_long cr4);
924 void load_dr0(uint64_t dr0);
925 void load_dr1(uint64_t dr1);
926 void load_dr2(uint64_t dr2);
927 void load_dr3(uint64_t dr3);
928 void load_dr4(uint64_t dr4);
929 void load_dr5(uint64_t dr5);
930 void load_dr6(uint64_t dr6);
931 void load_dr7(uint64_t dr7);
932 void load_fs(u_short sel);
933 void load_gs(u_short sel);
934 void ltr(u_short sel);
935 void outb(u_int port, u_char data);
936 void outl(u_int port, u_int data);
937 void outsb(u_int port, const void *addr, size_t count);
938 void outsl(u_int port, const void *addr, size_t count);
939 void outsw(u_int port, const void *addr, size_t count);
940 void outw(u_int port, u_short data);
941 u_long rcr0(void);
942 u_long rcr2(void);
943 u_long rcr3(void);
944 u_long rcr4(void);
945 uint64_t rdmsr(u_int msr);
946 uint32_t rdmsr32(u_int msr);
947 uint64_t rdpmc(u_int pmc);
948 uint64_t rdr0(void);
949 uint64_t rdr1(void);
950 uint64_t rdr2(void);
951 uint64_t rdr3(void);
952 uint64_t rdr4(void);
953 uint64_t rdr5(void);
954 uint64_t rdr6(void);
955 uint64_t rdr7(void);
956 uint64_t rdtsc(void);
957 u_long read_rflags(void);
958 u_int rfs(void);
959 u_int rgs(void);
960 void wbinvd(void);
961 void write_rflags(u_int rf);
962 void wrmsr(u_int msr, uint64_t newval);
963
964 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
965
966 void reset_dbregs(void);
967
968 #ifdef _KERNEL
969 int rdmsr_safe(u_int msr, uint64_t *val);
970 int wrmsr_safe(u_int msr, uint64_t newval);
971 #endif
972
973 #endif /* !_MACHINE_CPUFUNC_H_ */
974