1 /*
2 * Copyright (c) 2021 Maxime Villard, m00nbsd.net
3 * Copyright (c) 2021 The DragonFly Project.
4 * All rights reserved.
5 *
6 * This code is part of the NVMM hypervisor.
7 *
8 * This code is derived from software contributed to The DragonFly Project
9 * by Aaron LI <aly@aaronly.me>
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #ifndef _NVMM_OS_H_
34 #define _NVMM_OS_H_
35
36 #ifndef _KERNEL
37 #error "This file should not be included by userland programs."
38 #endif
39
40 #if defined(__NetBSD__)
41 #include <sys/cpu.h>
42 #include <uvm/uvm_object.h>
43 #include <uvm/uvm_extern.h>
44 #include <uvm/uvm_page.h>
45 #elif defined(__DragonFly__)
46 #include <sys/lock.h>
47 #include <sys/malloc.h> /* contigmalloc, contigfree */
48 #include <sys/proc.h> /* LWP_MP_URETMASK */
49 #include <vm/vm.h>
50 #include <vm/vm_extern.h>
51 #include <vm/vm_map.h>
52 #include <vm/vm_object.h>
53 #include <vm/vm_page.h>
54 #include <vm/vm_pager.h>
55 #include <vm/vm_param.h> /* KERN_SUCCESS, etc. */
56 #include <vm/pmap.h> /* pmap_ept_transform, pmap_npt_transform */
57 #include <machine/cpu.h> /* hvm_break_wanted */
58 #include <machine/cpufunc.h> /* ffsl, ffs, etc. */
59 #endif
60
61 /* Types. */
62 #if defined(__NetBSD__)
63 typedef struct vmspace os_vmspace_t;
64 typedef struct uvm_object os_vmobj_t;
65 typedef krwlock_t os_rwl_t;
66 typedef kmutex_t os_mtx_t;
67 #elif defined(__DragonFly__)
68 typedef struct vmspace os_vmspace_t;
69 typedef struct vm_object os_vmobj_t;
70 typedef struct lock os_rwl_t;
71 typedef struct lock os_mtx_t;
72 /* A few standard types. */
73 typedef vm_offset_t vaddr_t;
74 typedef vm_offset_t voff_t;
75 typedef vm_size_t vsize_t;
76 typedef vm_paddr_t paddr_t;
77 #endif
78
79 /* Attributes. */
80 #if defined(__DragonFly__)
81 #define __cacheline_aligned __cachealign
82 #define __diagused __debugvar
83 #endif
84
85 /* Macros. */
86 #if defined(__DragonFly__)
87 #define __arraycount(__x) (sizeof(__x) / sizeof(__x[0]))
88 #define __insn_barrier() __asm __volatile("":::"memory")
89 #endif
90
91 /* Bitops. */
92 #if defined(__NetBSD__)
93 #include <sys/bitops.h>
94 #elif defined(__DragonFly__)
95 #include <sys/bitops.h>
96 #ifdef __x86_64__
97 #undef __BIT
98 #define __BIT(__n) __BIT64(__n)
99 #undef __BITS
100 #define __BITS(__m, __n) __BITS64(__m, __n)
101 #endif /* __x86_64__ */
102 #endif
103
104 /* Maps. */
105 #if defined(__NetBSD__) || defined(__DragonFly__)
106 #define os_kernel_map kernel_map
107 #define os_curproc_map &curproc->p_vmspace->vm_map
108 #endif
109
110 /* R/W locks. */
111 #if defined(__NetBSD__)
112 #define os_rwl_init(lock) rw_init(lock)
113 #define os_rwl_destroy(lock) rw_destroy(lock)
114 #define os_rwl_rlock(lock) rw_enter(lock, RW_READER);
115 #define os_rwl_wlock(lock) rw_enter(lock, RW_WRITER);
116 #define os_rwl_unlock(lock) rw_exit(lock)
117 #define os_rwl_wheld(lock) rw_write_held(lock)
118 #elif defined(__DragonFly__)
119 #define os_rwl_init(lock) lockinit(lock, "nvmmrw", 0, 0)
120 #define os_rwl_destroy(lock) lockuninit(lock)
121 #define os_rwl_rlock(lock) lockmgr(lock, LK_SHARED);
122 #define os_rwl_wlock(lock) lockmgr(lock, LK_EXCLUSIVE);
123 #define os_rwl_unlock(lock) lockmgr(lock, LK_RELEASE)
124 #define os_rwl_wheld(lock) (lockstatus(lock, curthread) == LK_EXCLUSIVE)
125 #endif
126
127 /* Mutexes. */
128 #if defined(__NetBSD__)
129 #define os_mtx_init(lock) mutex_init(lock, MUTEX_DEFAULT, IPL_NONE)
130 #define os_mtx_destroy(lock) mutex_destroy(lock)
131 #define os_mtx_lock(lock) mutex_enter(lock)
132 #define os_mtx_unlock(lock) mutex_exit(lock)
133 #define os_mtx_owned(lock) mutex_owned(lock)
134 #elif defined(__DragonFly__)
135 #define os_mtx_init(lock) lockinit(lock, "nvmmmtx", 0, 0)
136 #define os_mtx_destroy(lock) lockuninit(lock)
137 #define os_mtx_lock(lock) lockmgr(lock, LK_EXCLUSIVE)
138 #define os_mtx_unlock(lock) lockmgr(lock, LK_RELEASE)
139 #define os_mtx_owned(lock) (lockstatus(lock, curthread) == LK_EXCLUSIVE)
140 #endif
141
142 /* Malloc. */
143 #if defined(__NetBSD__)
144 #include <sys/kmem.h>
145 #define os_mem_alloc(size) kmem_alloc(size, KM_SLEEP)
146 #define os_mem_zalloc(size) kmem_zalloc(size, KM_SLEEP)
147 #define os_mem_free(ptr, size) kmem_free(ptr, size)
148 #elif defined(__DragonFly__)
149 #include <sys/malloc.h>
150 MALLOC_DECLARE(M_NVMM);
151 #define os_mem_alloc(size) kmalloc(size, M_NVMM, M_WAITOK)
152 #define os_mem_zalloc(size) kmalloc(size, M_NVMM, M_WAITOK | M_ZERO)
153 #define os_mem_free(ptr, size) kfree(ptr, M_NVMM)
154 #endif
155
156 /* Printf. */
157 #if defined(__NetBSD__)
158 #define os_printf printf
159 #elif defined(__DragonFly__)
160 #define os_printf kprintf
161 #endif
162
163 /* Atomics. */
164 #if defined(__NetBSD__)
165 #include <sys/atomic.h>
166 #define os_atomic_inc_uint(x) atomic_inc_uint(x)
167 #define os_atomic_dec_uint(x) atomic_dec_uint(x)
168 #define os_atomic_load_uint(x) atomic_load_relaxed(x)
169 #define os_atomic_inc_64(x) atomic_inc_64(x)
170 #elif defined(__DragonFly__)
171 #include <machine/atomic.h>
172 #define os_atomic_inc_uint(x) atomic_add_int(x, 1)
173 #define os_atomic_dec_uint(x) atomic_subtract_int(x, 1)
174 #define os_atomic_load_uint(x) atomic_load_acq_int(x)
175 #define os_atomic_inc_64(x) atomic_add_64(x, 1)
176 #endif
177
178 /* Pmap. */
179 #if defined(__NetBSD__)
180 extern bool pmap_ept_has_ad;
181 #define os_vmspace_pmap(vm) ((vm)->vm_map.pmap)
182 #define os_vmspace_pdirpa(vm) ((vm)->vm_map.pmap->pm_pdirpa[0])
183 #define os_pmap_mach(pm) ((pm)->pm_data)
184 #elif defined(__DragonFly__)
185 #define os_vmspace_pmap(vm) vmspace_pmap(vm)
186 #define os_vmspace_pdirpa(vm) (vtophys(vmspace_pmap(vm)->pm_pml4))
187 #endif
188
189 /* CPU. */
190 #if defined(__NetBSD__)
191 #include <sys/cpu.h>
192 typedef struct cpu_info os_cpu_t;
193 #define OS_MAXCPUS MAXCPUS
194 #define OS_CPU_FOREACH(cpu) for (CPU_INFO_FOREACH(, cpu))
195 #define os_cpu_number(cpu) cpu_index(cpu)
196 #define os_curcpu() curcpu()
197 #define os_curcpu_number() cpu_number()
198 #define os_curcpu_tss_sel() curcpu()->ci_tss_sel
199 #define os_curcpu_tss() curcpu()->ci_tss
200 #define os_curcpu_gdt() curcpu()->ci_gdt
201 #define os_curcpu_idt() curcpu()->ci_idtvec.iv_idt
202 #elif defined(__DragonFly__)
203 #include <sys/globaldata.h>
204 #include <machine/segments.h>
205 typedef struct globaldata os_cpu_t;
206 #define OS_MAXCPUS SMP_MAXCPU
207 #define OS_CPU_FOREACH(cpu) \
208 for (int idx = 0; idx < ncpus && (cpu = globaldata_find(idx)); idx++)
209 #define os_cpu_number(cpu) (cpu)->gd_cpuid
210 #define os_curcpu() mycpu
211 #define os_curcpu_number() mycpuid
212 #define os_curcpu_tss_sel() GSEL(GPROC0_SEL, SEL_KPL)
213 #define os_curcpu_tss() &mycpu->gd_prvspace->common_tss
214 #define os_curcpu_gdt() mdcpu->gd_gdt
215 #define os_curcpu_idt() r_idt_arr[mycpuid].rd_base
216 #endif
217
218 /* Cpusets. */
219 #if defined(__NetBSD__)
220 #include <sys/kcpuset.h>
221 typedef kcpuset_t os_cpuset_t;
222 #define os_cpuset_init(s) kcpuset_create(s, true)
223 #define os_cpuset_destroy(s) kcpuset_destroy(s)
224 #define os_cpuset_isset(s, c) kcpuset_isset(s, c)
225 #define os_cpuset_clear(s, c) kcpuset_clear(s, c)
226 #define os_cpuset_setrunning(s) kcpuset_copy(s, kcpuset_running)
227 #elif defined(__DragonFly__)
228 #include <sys/cpumask.h>
229 #include <machine/smp.h> /* smp_active_mask */
230 typedef cpumask_t os_cpuset_t;
231 #define os_cpuset_init(s) \
232 ({ *(s) = kmalloc(sizeof(cpumask_t), M_NVMM, M_WAITOK | M_ZERO); })
233 #define os_cpuset_destroy(s) kfree((s), M_NVMM)
234 #define os_cpuset_isset(s, c) CPUMASK_TESTBIT(*(s), c)
235 #define os_cpuset_clear(s, c) ATOMIC_CPUMASK_NANDBIT(*(s), c)
236 #define os_cpuset_setrunning(s) ATOMIC_CPUMASK_ORMASK(*(s), smp_active_mask)
237 #endif
238
239 /* Preemption. */
240 #if defined(__NetBSD__)
241 #define os_preempt_disable() kpreempt_disable()
242 #define os_preempt_enable() kpreempt_enable()
243 #define os_preempt_disabled() kpreempt_disabled()
244 #elif defined(__DragonFly__)
245 /*
246 * In DragonFly, a thread in kernel mode will not be preemptively migrated
247 * to another CPU or preemptively switched to another normal kernel thread,
248 * but can be preemptively switched to an interrupt thread (which switches
249 * back to the kernel thread it preempted the instant it is done or blocks).
250 *
251 * However, we still need to use a critical section to prevent this nominal
252 * interrupt thread preemption to avoid exposing interrupt threads to
253 * guest DB and FP register state. We operate under the assumption that
254 * the hard interrupt code won't mess with this state.
255 */
256 #define os_preempt_disable() crit_enter()
257 #define os_preempt_enable() crit_exit()
258 #define os_preempt_disabled() (curthread->td_critcount != 0)
259 #endif
260
261 /* Asserts. */
262 #if defined(__NetBSD__)
263 #define OS_ASSERT KASSERT
264 #elif defined(__DragonFly__)
265 #define OS_ASSERT KKASSERT
266 #endif
267
268 /* Misc. */
269 #if defined(__DragonFly__)
270 #define uimin(a, b) ((u_int)a < (u_int)b ? (u_int)a : (u_int)b)
271 #endif
272
273 /* -------------------------------------------------------------------------- */
274
275 os_vmspace_t * os_vmspace_create(vaddr_t, vaddr_t);
276 void os_vmspace_destroy(os_vmspace_t *);
277 int os_vmspace_fault(os_vmspace_t *, vaddr_t, vm_prot_t);
278
279 os_vmobj_t * os_vmobj_create(voff_t);
280 void os_vmobj_ref(os_vmobj_t *);
281 void os_vmobj_rel(os_vmobj_t *);
282
283 int os_vmobj_map(struct vm_map *, vaddr_t *, vsize_t, os_vmobj_t *,
284 voff_t, bool, bool, bool, int, int);
285 void os_vmobj_unmap(struct vm_map *map, vaddr_t, vaddr_t, bool);
286
287 void * os_pagemem_zalloc(size_t);
288 void os_pagemem_free(void *, size_t);
289
290 paddr_t os_pa_zalloc(void);
291 void os_pa_free(paddr_t);
292
293 int os_contigpa_zalloc(paddr_t *, vaddr_t *, size_t);
294 void os_contigpa_free(paddr_t, vaddr_t, size_t);
295
296 static inline bool
os_return_needed(void)297 os_return_needed(void)
298 {
299 #if defined(__NetBSD__)
300 if (preempt_needed()) {
301 return true;
302 }
303 if (curlwp->l_flag & LW_USERRET) {
304 return true;
305 }
306 return false;
307 #elif defined(__DragonFly__)
308 if (__predict_false(hvm_break_wanted())) {
309 return true;
310 }
311 if (__predict_false(curthread->td_lwp->lwp_mpflags & LWP_MP_URETMASK)) {
312 return true;
313 }
314 return false;
315 #endif
316 }
317
318 /* -------------------------------------------------------------------------- */
319
320 /* IPIs. */
321
322 #if defined(__NetBSD__)
323
324 #include <sys/xcall.h>
325 #define OS_IPI_FUNC(func) void func(void *arg, void *unused)
326
327 static inline void
os_ipi_unicast(os_cpu_t * cpu,void (* func)(void *,void *),void * arg)328 os_ipi_unicast(os_cpu_t *cpu, void (*func)(void *, void *), void *arg)
329 {
330 xc_wait(xc_unicast(XC_HIGHPRI, func, arg, NULL, cpu));
331 }
332
333 static inline void
os_ipi_broadcast(void (* func)(void *,void *),void * arg)334 os_ipi_broadcast(void (*func)(void *, void *), void *arg)
335 {
336 xc_wait(xc_broadcast(0, func, arg, NULL));
337 }
338
339 static inline void
os_ipi_kickall(void)340 os_ipi_kickall(void)
341 {
342 /*
343 * XXX: this is probably too expensive. NetBSD should have a dummy
344 * interrupt handler that just IRETs without doing anything.
345 */
346 pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_NVMM);
347 }
348
349 #elif defined(__DragonFly__)
350
351 #include <sys/thread2.h>
352 #define OS_IPI_FUNC(func) void func(void *arg)
353
354 static inline void
os_ipi_unicast(os_cpu_t * cpu,void (* func)(void *),void * arg)355 os_ipi_unicast(os_cpu_t *cpu, void (*func)(void *), void *arg)
356 {
357 int seq;
358
359 seq = lwkt_send_ipiq(cpu, func, arg);
360 lwkt_wait_ipiq(cpu, seq);
361 }
362
363 static inline void
os_ipi_broadcast(void (* func)(void *),void * arg)364 os_ipi_broadcast(void (*func)(void *), void *arg)
365 {
366 cpumask_t mask;
367 int i;
368
369 for (i = 0; i < ncpus; i++) {
370 CPUMASK_ASSBIT(mask, i);
371 lwkt_cpusync_simple(mask, func, arg);
372 }
373 }
374
375 /*
376 * On DragonFly, no need to bind the thread, because any normal kernel
377 * thread will not migrate to another CPU or be preempted (except by an
378 * interrupt thread).
379 */
380 #define curlwp_bind() ((int)0)
381 #define curlwp_bindx(bound) /* nothing */
382
383 #endif /* __NetBSD__ */
384
385 #endif /* _NVMM_OS_H_ */
386