Lines Matching refs:va
101 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va);
191 vm_offset_t va, uint16_t ap) in radix_tlbie() argument
195 MPASS((va & PAGE_MASK) == 0); in radix_tlbie()
198 rb = va | is | ap; in radix_tlbie()
204 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap) in radix_tlbie_fixup() argument
209 TLBIEL_INVAL_PAGE, 0, 0, va, ap); in radix_tlbie_fixup()
212 TLBIEL_INVAL_PAGE, pid, 0, va, ap); in radix_tlbie_fixup()
216 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) in radix_tlbie_invlpg_user_4k() argument
220 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); in radix_tlbie_invlpg_user_4k()
221 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K); in radix_tlbie_invlpg_user_4k()
225 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) in radix_tlbie_invlpg_user_2m() argument
229 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); in radix_tlbie_invlpg_user_2m()
230 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M); in radix_tlbie_invlpg_user_2m()
250 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) in radix_tlbie_invlpg_kernel_4k() argument
254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); in radix_tlbie_invlpg_kernel_4k()
255 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K); in radix_tlbie_invlpg_kernel_4k()
259 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) in radix_tlbie_invlpg_kernel_2m() argument
263 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); in radix_tlbie_invlpg_kernel_2m()
264 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M); in radix_tlbie_invlpg_kernel_2m()
269 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) in radix_tlbie_invlpg_kernel_1g() argument
273 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); in radix_tlbie_invlpg_kernel_1g()
274 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G); in radix_tlbie_invlpg_kernel_1g()
294 pmap_l3e_pindex(vm_offset_t va) in pmap_l3e_pindex() argument
296 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); in pmap_l3e_pindex()
300 pmap_pml3e_index(vm_offset_t va) in pmap_pml3e_index() argument
303 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); in pmap_pml3e_index()
307 pmap_pml2e_index(vm_offset_t va) in pmap_pml2e_index() argument
309 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); in pmap_pml2e_index()
313 pmap_pml1e_index(vm_offset_t va) in pmap_pml1e_index() argument
315 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); in pmap_pml1e_index()
320 pmap_pte_index(vm_offset_t va) in pmap_pte_index() argument
323 return ((va >> PAGE_SHIFT) & RPTE_MASK); in pmap_pte_index()
328 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) in pmap_l3e_to_pte() argument
335 return (&pte[pmap_pte_index(va)]); in pmap_l3e_to_pte()
340 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) in pmap_l2e_to_l3e() argument
347 return (&l3e[pmap_pml3e_index(va)]); in pmap_l2e_to_l3e()
352 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) in pmap_l1e_to_l2e() argument
360 return (&l2e[pmap_pml2e_index(va)]); in pmap_l1e_to_l2e()
364 pmap_pml1e(pmap_t pmap, vm_offset_t va) in pmap_pml1e() argument
367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); in pmap_pml1e()
371 pmap_pml2e(pmap_t pmap, vm_offset_t va) in pmap_pml2e() argument
375 l1e = pmap_pml1e(pmap, va); in pmap_pml2e()
378 return (pmap_l1e_to_l2e(l1e, va)); in pmap_pml2e()
382 pmap_pml3e(pmap_t pmap, vm_offset_t va) in pmap_pml3e() argument
386 l2e = pmap_pml2e(pmap, va); in pmap_pml3e()
389 return (pmap_l2e_to_l3e(l2e, va)); in pmap_pml3e()
393 pmap_pte(pmap_t pmap, vm_offset_t va) in pmap_pte() argument
397 l3e = pmap_pml3e(pmap, va); in pmap_pte()
400 return (pmap_l3e_to_pte(l3e, va)); in pmap_pte()
437 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);
465 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz);
494 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va);
565 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
567 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);
573 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
574 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,
579 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,
582 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
585 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
587 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
590 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
592 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,
598 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,
600 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
602 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
604 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);
608 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush);
885 pagezero(vm_offset_t va) in pagezero() argument
887 va = trunc_page(va); in pagezero()
889 bzero((void *)va, PAGE_SIZE); in pagezero()
904 kvtopte(vm_offset_t va) in kvtopte() argument
908 l3e = pmap_pml3e(kernel_pmap, va); in kvtopte()
911 return (pmap_l3e_to_pte(l3e, va)); in kvtopte()
915 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) in mmu_radix_kenter() argument
919 pte = kvtopte(va); in mmu_radix_kenter()
932 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) in pmap_nofault_pte() argument
937 va &= PG_PS_FRAME; in pmap_nofault_pte()
938 l3e = pmap_pml3e(pmap, va); in pmap_nofault_pte()
947 va &= PG_FRAME; in pmap_nofault_pte()
948 pte = pmap_l3e_to_pte(l3e, va); in pmap_nofault_pte()
955 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) in pmap_nofault() argument
964 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) in pmap_nofault()
975 __func__, pmap, va, flags, origpte); in pmap_nofault()
982 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, in pmap_nofault()
986 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || in pmap_nofault()
1138 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) in pmap_invalidate_l3e_page() argument
1154 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); in pmap_invalidate_l3e_page()
1156 pmap_invalidate_page_2m(pmap, va); in pmap_invalidate_l3e_page()
1256 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) in pmap_pvh_remove() argument
1269 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { in pmap_pvh_remove()
1284 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, in pmap_pv_demote_l3e() argument
1305 va = trunc_2mpage(va); in pmap_pv_demote_l3e()
1306 pv = pmap_pvh_remove(pvh, pmap, va); in pmap_pv_demote_l3e()
1314 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; in pmap_pv_demote_l3e()
1324 va += PAGE_SIZE; in pmap_pv_demote_l3e()
1325 pv->pv_va = va; in pmap_pv_demote_l3e()
1332 if (va == va_last) in pmap_pv_demote_l3e()
1381 vm_offset_t va; in reclaim_pv_chunk() local
1454 va = pv->pv_va; in reclaim_pv_chunk()
1455 l3e = pmap_pml3e(pmap, va); in reclaim_pv_chunk()
1458 pte = pmap_l3e_to_pte(l3e, va); in reclaim_pv_chunk()
1480 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free); in reclaim_pv_chunk()
1673 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, in pmap_pv_promote_l3e() argument
1693 va = trunc_2mpage(va); in pmap_pv_promote_l3e()
1694 pv = pmap_pvh_remove(&m->md, pmap, va); in pmap_pv_promote_l3e()
1700 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; in pmap_pv_promote_l3e()
1703 va += PAGE_SIZE; in pmap_pv_promote_l3e()
1704 pmap_pvh_free(&m->md, pmap, va); in pmap_pv_promote_l3e()
1705 } while (va < va_last); in pmap_pv_promote_l3e()
1715 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) in pmap_pvh_free() argument
1719 pv = pmap_pvh_remove(pvh, pmap, va); in pmap_pvh_free()
1729 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, in pmap_try_insert_pv_entry() argument
1737 pv->pv_va = va; in pmap_try_insert_pv_entry()
2054 vm_offset_t va; in mmu_radix_late_bootstrap() local
2081 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; in mmu_radix_late_bootstrap()
2082 virtual_avail = va + kstack_pages * PAGE_SIZE; in mmu_radix_late_bootstrap()
2083 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); in mmu_radix_late_bootstrap()
2084 thread0.td_kstack = va; in mmu_radix_late_bootstrap()
2086 mmu_radix_kenter(va, pa); in mmu_radix_late_bootstrap()
2088 va += PAGE_SIZE; in mmu_radix_late_bootstrap()
2223 vm_offset_t va, va_next; in mmu_radix_advise() local
2287 va = va_next; in mmu_radix_advise()
2312 if (va != va_next) { in mmu_radix_advise()
2314 va = va_next; in mmu_radix_advise()
2317 if (va != va_next) in mmu_radix_advise()
2422 vm_offset_t va; in mmu_radix_clear_modify() local
2454 va = pv->pv_va; in mmu_radix_clear_modify()
2455 l3e = pmap_pml3e(pmap, va); in mmu_radix_clear_modify()
2458 if (pmap_demote_l3e_locked(pmap, l3e, va, &lock)) { in mmu_radix_clear_modify()
2465 va += VM_PAGE_TO_PHYS(m) - (oldl3e & in mmu_radix_clear_modify()
2467 pte = pmap_l3e_to_pte(l3e, va); in mmu_radix_clear_modify()
2475 pmap_invalidate_page(pmap, va); in mmu_radix_clear_modify()
2716 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, in pmap_promote_l3e() argument
2735 " in pmap %p", va, pmap); in pmap_promote_l3e()
2759 " in pmap %p", va, pmap); in pmap_promote_l3e()
2772 (va & ~L3_PAGE_MASK), pmap); in pmap_promote_l3e()
2776 " in pmap %p", va, pmap); in pmap_promote_l3e()
2791 KASSERT(mpte->pindex == pmap_l3e_pindex(va), in pmap_promote_l3e()
2795 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, in pmap_promote_l3e()
2804 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); in pmap_promote_l3e()
2810 " in pmap %p", va, pmap); in pmap_promote_l3e()
2819 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, in mmu_radix_enter() argument
2832 va = trunc_page(va); in mmu_radix_enter()
2835 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, in mmu_radix_enter()
2837 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); in mmu_radix_enter()
2838 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va), in mmu_radix_enter()
2862 if (va >= DMAP_MIN_ADDRESS) in mmu_radix_enter()
2880 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); in mmu_radix_enter()
2882 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); in mmu_radix_enter()
2892 l3e = pmap_pml3e(pmap, va); in mmu_radix_enter()
2894 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { in mmu_radix_enter()
2895 pte = pmap_l3e_to_pte(l3e, va); in mmu_radix_enter()
2896 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { in mmu_radix_enter()
2900 } else if (va < VM_MAXUSER_ADDRESS) { in mmu_radix_enter()
2906 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), in mmu_radix_enter()
2917 panic("pmap_enter: invalid page directory va=%#lx", va); in mmu_radix_enter()
2930 pmap, va, m, prot, flags, psind, pmap->pm_pid, in mmu_radix_enter()
2932 pmap_pte_walk(pmap->pm_pml1, va); in mmu_radix_enter()
2953 " va: 0x%lx", va)); in mmu_radix_enter()
2996 ("pmap_enter: unexpected pa update for %#lx", va)); in mmu_radix_enter()
3010 pv = pmap_pvh_remove(&om->md, pmap, va); in mmu_radix_enter()
3035 pmap, va, m, prot, flags, psind, in mmu_radix_enter()
3055 pv->pv_va = va; in mmu_radix_enter()
3076 ("pmap_enter: unexpected pa update for %#lx", va)); in mmu_radix_enter()
3112 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) in mmu_radix_enter()
3118 pmap_invalidate_page(pmap, va); in mmu_radix_enter()
3137 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, in pmap_enter_2mpage() argument
3151 if (va >= DMAP_MIN_ADDRESS) in pmap_enter_2mpage()
3153 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | in pmap_enter_2mpage()
3170 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, in pmap_enter_l3e() argument
3181 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? in pmap_enter_l3e()
3184 " in pmap %p", va, pmap); in pmap_enter_l3e()
3188 l3e = &l3e[pmap_pml3e_index(va)]; in pmap_enter_l3e()
3196 " in pmap %p", va, pmap); in pmap_enter_l3e()
3208 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); in pmap_enter_l3e()
3209 pmap_invalidate_l3e_page(pmap, va, oldl3e); in pmap_enter_l3e()
3211 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, in pmap_enter_l3e()
3216 if (va >= VM_MAXUSER_ADDRESS) { in pmap_enter_l3e()
3234 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { in pmap_enter_l3e()
3236 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { in pmap_enter_l3e()
3243 pmap_invalidate_page(pmap, va); in pmap_enter_l3e()
3247 " in pmap %p", va, pmap); in pmap_enter_l3e()
3272 " in pmap %p", va, pmap); in pmap_enter_l3e()
3282 vm_offset_t va; in mmu_radix_enter_object() local
3298 va = start + ptoa(diff); in mmu_radix_enter_object()
3299 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && in mmu_radix_enter_object()
3301 pmap_enter_2mpage(pmap, va, m, prot, &lock)) in mmu_radix_enter_object()
3304 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, in mmu_radix_enter_object()
3317 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, in mmu_radix_enter_quick_locked() argument
3324 KASSERT(!VA_IS_CLEANMAP(va) || in mmu_radix_enter_quick_locked()
3333 if (va < VM_MAXUSER_ADDRESS) { in mmu_radix_enter_quick_locked()
3340 ptepindex = pmap_l3e_pindex(va); in mmu_radix_enter_quick_locked()
3347 ptepa = pmap_pml3e(pmap, va); in mmu_radix_enter_quick_locked()
3371 pte = &pte[pmap_pte_index(va)]; in mmu_radix_enter_quick_locked()
3374 pte = pmap_pte(pmap, va); in mmu_radix_enter_quick_locked()
3388 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { in mmu_radix_enter_quick_locked()
3391 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { in mmu_radix_enter_quick_locked()
3424 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, in mmu_radix_enter_quick() argument
3433 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, in mmu_radix_enter_quick()
3444 mmu_radix_extract(pmap_t pmap, vm_offset_t va) in mmu_radix_extract() argument
3450 l3e = pmap_pml3e(pmap, va); in mmu_radix_extract()
3454 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); in mmu_radix_extract()
3455 pa |= (va & L3_PAGE_MASK); in mmu_radix_extract()
3465 pte = pmap_l3e_to_pte(l3e, va); in mmu_radix_extract()
3469 pa = (pa & PG_FRAME) | (va & PAGE_MASK); in mmu_radix_extract()
3470 pa |= (va & PAGE_MASK); in mmu_radix_extract()
3476 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) in mmu_radix_extract_and_hold() argument
3485 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); in mmu_radix_extract_and_hold()
3487 l3ep = pmap_pml3e(pmap, va); in mmu_radix_extract_and_hold()
3492 (va & L3_PAGE_MASK)); in mmu_radix_extract_and_hold()
3495 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); in mmu_radix_extract_and_hold()
4351 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) in pmap_allocl3e() argument
4358 pdpe = pmap_pml2e(pmap, va); in pmap_allocl3e()
4365 ptepindex = pmap_l3e_pindex(va); in pmap_allocl3e()
4375 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) in pmap_allocpte() argument
4384 ptepindex = pmap_l3e_pindex(va); in pmap_allocpte()
4389 pd = pmap_pml3e(pmap, va); in pmap_allocpte()
4396 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { in pmap_allocpte()
4446 vm_offset_t eva, va; in pmap_protect_l3e() local
4459 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); in pmap_protect_l3e()
4460 va < eva; va += PAGE_SIZE, m++) in pmap_protect_l3e()
4617 vm_offset_t va; in mmu_radix_qenter() local
4621 va = sva; in mmu_radix_qenter()
4622 pte = kvtopte(va); in mmu_radix_qenter()
4623 while (va < sva + PAGE_SIZE * count) { in mmu_radix_qenter()
4624 if (__predict_false((va & L3_PAGE_MASK) == 0)) in mmu_radix_qenter()
4625 pte = kvtopte(va); in mmu_radix_qenter()
4626 MPASS(pte == pmap_pte(kernel_pmap, va)); in mmu_radix_qenter()
4641 va += PAGE_SIZE; in mmu_radix_qenter()
4654 vm_offset_t va; in mmu_radix_qremove() local
4660 va = sva; in mmu_radix_qremove()
4661 pte = kvtopte(va); in mmu_radix_qremove()
4662 while (va < sva + PAGE_SIZE * count) { in mmu_radix_qremove()
4663 if (__predict_false((va & L3_PAGE_MASK) == 0)) in mmu_radix_qremove()
4664 pte = kvtopte(va); in mmu_radix_qremove()
4667 va += PAGE_SIZE; in mmu_radix_qremove()
4669 pmap_invalidate_range(kernel_pmap, sva, va); in mmu_radix_qremove()
4713 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) in pmap_remove_pt_page() argument
4717 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); in pmap_remove_pt_page()
4727 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) in pmap_unwire_ptp() argument
4732 _pmap_unwire_ptp(pmap, va, m, free); in pmap_unwire_ptp()
4739 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) in _pmap_unwire_ptp() argument
4749 pml1 = pmap_pml1e(pmap, va); in _pmap_unwire_ptp()
4754 l2e = pmap_pml2e(pmap, va); in _pmap_unwire_ptp()
4759 l3e = pmap_pml3e(pmap, va); in _pmap_unwire_ptp()
4767 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME); in _pmap_unwire_ptp()
4768 pmap_unwire_ptp(pmap, va, pdpg, free); in _pmap_unwire_ptp()
4774 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME); in _pmap_unwire_ptp()
4775 pmap_unwire_ptp(pmap, va, pdppg, free); in _pmap_unwire_ptp()
4790 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, in pmap_unuse_pt() argument
4795 if (va >= VM_MAXUSER_ADDRESS) in pmap_unuse_pt()
4799 return (pmap_unwire_ptp(pmap, va, mpte, free)); in pmap_unuse_pt()
4825 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, in pmap_pv_insert_l3e() argument
4837 pv->pv_va = va; in pmap_pv_insert_l3e()
4861 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) in pmap_demote_l3e() argument
4867 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); in pmap_demote_l3e()
4874 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, in pmap_demote_l3e_locked() argument
4889 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == in pmap_demote_l3e_locked()
4908 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ? in pmap_demote_l3e_locked()
4911 sva = trunc_2mpage(va); in pmap_demote_l3e_locked()
4916 " in pmap %p", va, pmap); in pmap_demote_l3e_locked()
4919 mpte->pindex = pmap_l3e_pindex(va); in pmap_demote_l3e_locked()
4920 if (va < VM_MAXUSER_ADDRESS) in pmap_demote_l3e_locked()
4968 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde); in pmap_demote_l3e_locked()
4973 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); in pmap_demote_l3e_locked()
4977 " in pmap %p", va, pmap); in pmap_demote_l3e_locked()
4985 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) in pmap_remove_kernel_l3e() argument
4992 mpte = pmap_remove_pt_page(pmap, va); in pmap_remove_kernel_l3e()
5019 vm_offset_t eva, va; in pmap_remove_l3e() local
5034 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); in pmap_remove_l3e()
5035 va < eva; va += PAGE_SIZE, m++) { in pmap_remove_l3e()
5064 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, in pmap_remove_pte() argument
5083 pmap_pvh_free(&m->md, pmap, va); in pmap_remove_pte()
5091 return (pmap_unuse_pt(pmap, va, ptepde, free)); in pmap_remove_pte()
5098 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, in pmap_remove_page() argument
5109 pte = pmap_l3e_to_pte(l3e, va); in pmap_remove_page()
5115 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); in pmap_remove_page()
5119 pmap_invalidate_page(pmap, va); in pmap_remove_page()
5131 vm_offset_t va; in pmap_remove_ptes() local
5136 va = eva; in pmap_remove_ptes()
5141 if (va != eva) { in pmap_remove_ptes()
5143 va = eva; in pmap_remove_ptes()
5147 if (va == eva) in pmap_remove_ptes()
5148 va = sva; in pmap_remove_ptes()
5157 else if (va != eva) in pmap_remove_ptes()
5158 pmap_invalidate_range(pmap, va, sva); in pmap_remove_ptes()
5288 vm_offset_t va; in mmu_radix_remove_all() local
5314 va = pv->pv_va; in mmu_radix_remove_all()
5315 l3e = pmap_pml3e(pmap, va); in mmu_radix_remove_all()
5316 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); in mmu_radix_remove_all()
5841 vm_offset_t va, tmpva, ppa, offset; in mmu_radix_mapdev_attr() local
5849 va = kva_alloc(size); in mmu_radix_mapdev_attr()
5854 if (!va) in mmu_radix_mapdev_attr()
5857 for (tmpva = va; size > 0;) { in mmu_radix_mapdev_attr()
5865 return ((void *)(va + offset)); in mmu_radix_mapdev_attr()
5896 mmu_radix_unmapdev(vm_offset_t va, vm_size_t size) in mmu_radix_unmapdev() argument
5900 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size); in mmu_radix_unmapdev()
5902 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) in mmu_radix_unmapdev()
5905 offset = va & PAGE_MASK; in mmu_radix_unmapdev()
5907 va = trunc_page(va); in mmu_radix_unmapdev()
5910 mmu_radix_qremove(va, atop(size)); in mmu_radix_unmapdev()
5911 kva_free(va, size); in mmu_radix_unmapdev()
5916 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) in mmu_radix_sync_icache() argument
5922 pa = pmap_extract(pm, va); in mmu_radix_sync_icache()
5923 sync_sz = PAGE_SIZE - (va & PAGE_MASK); in mmu_radix_sync_icache()
5926 pa += (va & PAGE_MASK); in mmu_radix_sync_icache()
5929 va += sync_sz; in mmu_radix_sync_icache()
5954 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) in pmap_demote_l2e() argument
5968 " in pmap %p", va, pmap); in pmap_demote_l2e()
5971 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT; in pmap_demote_l2e()
6000 " in pmap %p", va, pmap); in pmap_demote_l2e()
6005 mmu_radix_kextract(vm_offset_t va) in mmu_radix_kextract() argument
6010 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); in mmu_radix_kextract()
6011 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { in mmu_radix_kextract()
6012 pa = DMAP_TO_PHYS(va); in mmu_radix_kextract()
6015 l3e = *pmap_pml3e(kernel_pmap, va); in mmu_radix_kextract()
6017 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); in mmu_radix_kextract()
6018 pa |= (va & L3_PAGE_MASK); in mmu_radix_kextract()
6028 pa = be64toh(*pmap_l3e_to_pte(&l3e, va)); in mmu_radix_kextract()
6029 pa = (pa & PG_FRAME) | (va & PAGE_MASK); in mmu_radix_kextract()
6030 pa |= (va & PAGE_MASK); in mmu_radix_kextract()
6057 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) in mmu_radix_kenter_attr() argument
6062 pte = kvtopte(va); in mmu_radix_kenter_attr()
6070 mmu_radix_kremove(vm_offset_t va) in mmu_radix_kremove() argument
6074 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); in mmu_radix_kremove()
6076 pte = kvtopte(va); in mmu_radix_kremove()
6109 void **va) in mmu_radix_dumpsys_map() argument
6111 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); in mmu_radix_dumpsys_map()
6139 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, in mmu_radix_change_attr() argument
6144 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); in mmu_radix_change_attr()
6146 error = pmap_change_attr_locked(va, size, mode, true); in mmu_radix_change_attr()
6152 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) in pmap_change_attr_locked() argument
6163 base = trunc_page(va); in pmap_change_attr_locked()
6164 offset = va & PAGE_MASK; in pmap_change_attr_locked()
6370 vm_offset_t va; in mmu_radix_page_array_startup() local
6387 for (va = start; va < end; va += L3_PAGE_SIZE) { in mmu_radix_page_array_startup()
6388 pfn = first_page + (va - start) / sizeof(struct vm_page); in mmu_radix_page_array_startup()
6390 l2e = pmap_pml2e(kernel_pmap, va); in mmu_radix_page_array_startup()
6397 pde = pmap_l2e_to_l3e(l2e, va); in mmu_radix_page_array_startup()
6415 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) in pmap_pte_walk() argument
6422 l1e = &l1[pmap_pml1e_index(va)]; in pmap_pte_walk()
6423 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e)); in pmap_pte_walk()
6428 l2e = pmap_l1e_to_l2e(l1e, va); in pmap_pte_walk()
6434 l3e = pmap_l2e_to_l3e(l2e, va); in pmap_pte_walk()
6440 pte = pmap_l3e_to_pte(l3e, va); in pmap_pte_walk()
6466 vm_offset_t va; in DB_SHOW_COMMAND() local
6473 va = (vm_offset_t)addr; in DB_SHOW_COMMAND()
6475 if (va >= DMAP_MIN_ADDRESS) in DB_SHOW_COMMAND()
6482 pmap_pte_walk(pmap->pm_pml1, va); in DB_SHOW_COMMAND()