1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 Matthew Macy
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "opt_platform.h"
29
30 #include <sys/cdefs.h>
31 #include <sys/param.h>
32 #include <sys/kernel.h>
33 #include <sys/systm.h>
34 #include <sys/conf.h>
35 #include <sys/bitstring.h>
36 #include <sys/queue.h>
37 #include <sys/cpuset.h>
38 #include <sys/endian.h>
39 #include <sys/kerneldump.h>
40 #include <sys/ktr.h>
41 #include <sys/lock.h>
42 #include <sys/syslog.h>
43 #include <sys/msgbuf.h>
44 #include <sys/malloc.h>
45 #include <sys/mman.h>
46 #include <sys/mutex.h>
47 #include <sys/proc.h>
48 #include <sys/rwlock.h>
49 #include <sys/sched.h>
50 #include <sys/sysctl.h>
51 #include <sys/systm.h>
52 #include <sys/vmem.h>
53 #include <sys/vmmeter.h>
54 #include <sys/smp.h>
55
56 #include <sys/kdb.h>
57
58 #include <dev/ofw/openfirm.h>
59
60 #include <vm/vm.h>
61 #include <vm/pmap.h>
62 #include <vm/vm_param.h>
63 #include <vm/vm_kern.h>
64 #include <vm/vm_page.h>
65 #include <vm/vm_map.h>
66 #include <vm/vm_object.h>
67 #include <vm/vm_extern.h>
68 #include <vm/vm_pageout.h>
69 #include <vm/vm_phys.h>
70 #include <vm/vm_reserv.h>
71 #include <vm/vm_dumpset.h>
72 #include <vm/uma.h>
73
74 #include <machine/_inttypes.h>
75 #include <machine/cpu.h>
76 #include <machine/platform.h>
77 #include <machine/frame.h>
78 #include <machine/md_var.h>
79 #include <machine/psl.h>
80 #include <machine/bat.h>
81 #include <machine/hid.h>
82 #include <machine/pte.h>
83 #include <machine/sr.h>
84 #include <machine/trap.h>
85 #include <machine/mmuvar.h>
86
87 /* For pseries bit. */
88 #include <powerpc/pseries/phyp-hvcall.h>
89
90 #ifdef INVARIANTS
91 #include <vm/uma_dbg.h>
92 #endif
93
94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit))
95 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit))
96 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit))
97
98 #include "opt_ddb.h"
99
100 #ifdef DDB
101 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va);
102 #endif
103
104 #define PG_W RPTE_WIRED
105 #define PG_V RPTE_VALID
106 #define PG_MANAGED RPTE_MANAGED
107 #define PG_PROMOTED RPTE_PROMOTED
108 #define PG_M RPTE_C
109 #define PG_A RPTE_R
110 #define PG_X RPTE_EAA_X
111 #define PG_RW RPTE_EAA_W
112 #define PG_PTE_CACHE RPTE_ATTR_MASK
113
114 #define RPTE_SHIFT 9
115 #define NLS_MASK ((1UL<<5)-1)
116 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT)
117 #define RPTE_MASK (RPTE_ENTRIES-1)
118
119 #define NLB_SHIFT 0
120 #define NLB_MASK (((1UL<<52)-1) << 8)
121
122 extern int nkpt;
123 extern caddr_t crashdumpmap;
124
125 #define RIC_FLUSH_TLB 0
126 #define RIC_FLUSH_PWC 1
127 #define RIC_FLUSH_ALL 2
128
129 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
130
131 #define PPC_INST_TLBIE 0x7c000264
132 #define PPC_INST_TLBIEL 0x7c000224
133 #define PPC_INST_SLBIA 0x7c0003e4
134
135 #define ___PPC_RA(a) (((a) & 0x1f) << 16)
136 #define ___PPC_RB(b) (((b) & 0x1f) << 11)
137 #define ___PPC_RS(s) (((s) & 0x1f) << 21)
138 #define ___PPC_RT(t) ___PPC_RS(t)
139 #define ___PPC_R(r) (((r) & 0x1) << 16)
140 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17)
141 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18)
142
143 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \
144 ((IH & 0x7) << 21))
145 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \
146 __XSTRING(.long PPC_INST_TLBIE | \
147 ___PPC_RB(rb) | ___PPC_RS(rs) | \
148 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
149 ___PPC_R(r))
150
151 #define PPC_TLBIEL(rb,rs,ric,prs,r) \
152 __XSTRING(.long PPC_INST_TLBIEL | \
153 ___PPC_RB(rb) | ___PPC_RS(rs) | \
154 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
155 ___PPC_R(r))
156
157 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7)
158
159 static __inline void
ttusync(void)160 ttusync(void)
161 {
162 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
163 }
164
165 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */
166 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */
167 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */
168 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */
169 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */
170
171 #define TLBIE_ACTUAL_PAGE_MASK 0xe0
172 #define TLBIE_ACTUAL_PAGE_4K 0x00
173 #define TLBIE_ACTUAL_PAGE_64K 0xa0
174 #define TLBIE_ACTUAL_PAGE_2M 0x20
175 #define TLBIE_ACTUAL_PAGE_1G 0x40
176
177 #define TLBIE_PRS_PARTITION_SCOPE 0x0
178 #define TLBIE_PRS_PROCESS_SCOPE 0x1
179
180 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */
181 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */
182 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC,
183 * cached {proc, part}tab entries
184 */
185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only:
186 * Invalidate a range of translations
187 */
188
189 static __always_inline void
radix_tlbie(uint8_t ric,uint8_t prs,uint16_t is,uint32_t pid,uint32_t lpid,vm_offset_t va,uint16_t ap)190 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid,
191 vm_offset_t va, uint16_t ap)
192 {
193 uint64_t rb, rs;
194
195 MPASS((va & PAGE_MASK) == 0);
196
197 rs = ((uint64_t)pid << 32) | lpid;
198 rb = va | is | ap;
199 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : :
200 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory");
201 }
202
203 static __inline void
radix_tlbie_fixup(uint32_t pid,vm_offset_t va,int ap)204 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap)
205 {
206
207 __asm __volatile("ptesync" ::: "memory");
208 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
209 TLBIEL_INVAL_PAGE, 0, 0, va, ap);
210 __asm __volatile("ptesync" ::: "memory");
211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
212 TLBIEL_INVAL_PAGE, pid, 0, va, ap);
213 }
214
215 static __inline void
radix_tlbie_invlpg_user_4k(uint32_t pid,vm_offset_t va)216 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va)
217 {
218
219 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
220 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K);
221 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K);
222 }
223
224 static __inline void
radix_tlbie_invlpg_user_2m(uint32_t pid,vm_offset_t va)225 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va)
226 {
227
228 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
229 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M);
230 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M);
231 }
232
233 static __inline void
radix_tlbie_invlpwc_user(uint32_t pid)234 radix_tlbie_invlpwc_user(uint32_t pid)
235 {
236
237 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
238 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
239 }
240
241 static __inline void
radix_tlbie_flush_user(uint32_t pid)242 radix_tlbie_flush_user(uint32_t pid)
243 {
244
245 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
246 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
247 }
248
249 static __inline void
radix_tlbie_invlpg_kernel_4k(vm_offset_t va)250 radix_tlbie_invlpg_kernel_4k(vm_offset_t va)
251 {
252
253 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K);
255 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K);
256 }
257
258 static __inline void
radix_tlbie_invlpg_kernel_2m(vm_offset_t va)259 radix_tlbie_invlpg_kernel_2m(vm_offset_t va)
260 {
261
262 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
263 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M);
264 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M);
265 }
266
267 /* 1GB pages aren't currently supported. */
268 static __inline __unused void
radix_tlbie_invlpg_kernel_1g(vm_offset_t va)269 radix_tlbie_invlpg_kernel_1g(vm_offset_t va)
270 {
271
272 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
273 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G);
274 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G);
275 }
276
277 static __inline void
radix_tlbie_invlpwc_kernel(void)278 radix_tlbie_invlpwc_kernel(void)
279 {
280
281 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
282 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
283 }
284
285 static __inline void
radix_tlbie_flush_kernel(void)286 radix_tlbie_flush_kernel(void)
287 {
288
289 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
290 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
291 }
292
293 static __inline vm_pindex_t
pmap_l3e_pindex(vm_offset_t va)294 pmap_l3e_pindex(vm_offset_t va)
295 {
296 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT);
297 }
298
299 static __inline vm_pindex_t
pmap_pml3e_index(vm_offset_t va)300 pmap_pml3e_index(vm_offset_t va)
301 {
302
303 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK);
304 }
305
306 static __inline vm_pindex_t
pmap_pml2e_index(vm_offset_t va)307 pmap_pml2e_index(vm_offset_t va)
308 {
309 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK);
310 }
311
312 static __inline vm_pindex_t
pmap_pml1e_index(vm_offset_t va)313 pmap_pml1e_index(vm_offset_t va)
314 {
315 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT);
316 }
317
318 /* Return various clipped indexes for a given VA */
319 static __inline vm_pindex_t
pmap_pte_index(vm_offset_t va)320 pmap_pte_index(vm_offset_t va)
321 {
322
323 return ((va >> PAGE_SHIFT) & RPTE_MASK);
324 }
325
326 /* Return a pointer to the PT slot that corresponds to a VA */
327 static __inline pt_entry_t *
pmap_l3e_to_pte(pt_entry_t * l3e,vm_offset_t va)328 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va)
329 {
330 pt_entry_t *pte;
331 vm_paddr_t ptepa;
332
333 ptepa = (be64toh(*l3e) & NLB_MASK);
334 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa);
335 return (&pte[pmap_pte_index(va)]);
336 }
337
338 /* Return a pointer to the PD slot that corresponds to a VA */
339 static __inline pt_entry_t *
pmap_l2e_to_l3e(pt_entry_t * l2e,vm_offset_t va)340 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va)
341 {
342 pt_entry_t *l3e;
343 vm_paddr_t l3pa;
344
345 l3pa = (be64toh(*l2e) & NLB_MASK);
346 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa);
347 return (&l3e[pmap_pml3e_index(va)]);
348 }
349
350 /* Return a pointer to the PD slot that corresponds to a VA */
351 static __inline pt_entry_t *
pmap_l1e_to_l2e(pt_entry_t * l1e,vm_offset_t va)352 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va)
353 {
354 pt_entry_t *l2e;
355 vm_paddr_t l2pa;
356
357 l2pa = (be64toh(*l1e) & NLB_MASK);
358
359 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa);
360 return (&l2e[pmap_pml2e_index(va)]);
361 }
362
363 static __inline pml1_entry_t *
pmap_pml1e(pmap_t pmap,vm_offset_t va)364 pmap_pml1e(pmap_t pmap, vm_offset_t va)
365 {
366
367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
368 }
369
370 static pt_entry_t *
pmap_pml2e(pmap_t pmap,vm_offset_t va)371 pmap_pml2e(pmap_t pmap, vm_offset_t va)
372 {
373 pt_entry_t *l1e;
374
375 l1e = pmap_pml1e(pmap, va);
376 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0)
377 return (NULL);
378 return (pmap_l1e_to_l2e(l1e, va));
379 }
380
381 static __inline pt_entry_t *
pmap_pml3e(pmap_t pmap,vm_offset_t va)382 pmap_pml3e(pmap_t pmap, vm_offset_t va)
383 {
384 pt_entry_t *l2e;
385
386 l2e = pmap_pml2e(pmap, va);
387 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0)
388 return (NULL);
389 return (pmap_l2e_to_l3e(l2e, va));
390 }
391
392 static __inline pt_entry_t *
pmap_pte(pmap_t pmap,vm_offset_t va)393 pmap_pte(pmap_t pmap, vm_offset_t va)
394 {
395 pt_entry_t *l3e;
396
397 l3e = pmap_pml3e(pmap, va);
398 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
399 return (NULL);
400 return (pmap_l3e_to_pte(l3e, va));
401 }
402
403 int nkpt = 64;
404 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0,
405 "Number of kernel page table pages allocated on bootup");
406
407 vm_paddr_t dmaplimit;
408
409 SYSCTL_DECL(_vm_pmap);
410
411 #ifdef INVARIANTS
412 #define VERBOSE_PMAP 0
413 #define VERBOSE_PROTECT 0
414 static int pmap_logging;
415 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN,
416 &pmap_logging, 0, "verbose debug logging");
417 #endif
418
419 static u_int64_t KPTphys; /* phys addr of kernel level 1 */
420
421 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */
422
423 static vm_offset_t qframe = 0;
424 static struct mtx qframe_mtx;
425
426 void mmu_radix_activate(struct thread *);
427 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int);
428 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *,
429 vm_size_t);
430 void mmu_radix_clear_modify(vm_page_t);
431 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
432 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *);
433 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t);
434 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
435 vm_prot_t);
436 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
437 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);
438 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);
439 void mmu_radix_kenter(vm_offset_t, vm_paddr_t);
440 vm_paddr_t mmu_radix_kextract(vm_offset_t);
441 void mmu_radix_kremove(vm_offset_t);
442 boolean_t mmu_radix_is_modified(vm_page_t);
443 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t);
444 boolean_t mmu_radix_is_referenced(vm_page_t);
445 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t,
446 vm_pindex_t, vm_size_t);
447 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t);
448 void mmu_radix_page_init(vm_page_t);
449 boolean_t mmu_radix_page_is_mapped(vm_page_t m);
450 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t);
451 int mmu_radix_page_wired_mappings(vm_page_t);
452 int mmu_radix_pinit(pmap_t);
453 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
454 bool mmu_radix_ps_enabled(pmap_t);
455 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int);
456 void mmu_radix_qremove(vm_offset_t, int);
457 vm_offset_t mmu_radix_quick_enter_page(vm_page_t);
458 void mmu_radix_quick_remove_page(vm_offset_t);
459 int mmu_radix_ts_referenced(vm_page_t);
460 void mmu_radix_release(pmap_t);
461 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t);
462 void mmu_radix_remove_all(vm_page_t);
463 void mmu_radix_remove_pages(pmap_t);
464 void mmu_radix_remove_write(vm_page_t);
465 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz);
466 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t);
467 void mmu_radix_zero_page(vm_page_t);
468 void mmu_radix_zero_page_area(vm_page_t, int, int);
469 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t);
470 void mmu_radix_page_array_startup(long pages);
471
472 #include "mmu_oea64.h"
473
474 /*
475 * Kernel MMU interface
476 */
477
478 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t);
479
480 static void mmu_radix_copy_page(vm_page_t, vm_page_t);
481 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
482 vm_page_t *mb, vm_offset_t b_offset, int xfersize);
483 static void mmu_radix_growkernel(vm_offset_t);
484 static void mmu_radix_init(void);
485 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *);
486 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
487 static void mmu_radix_pinit0(pmap_t);
488
489 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t);
490 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
491 static void mmu_radix_unmapdev(vm_offset_t, vm_size_t);
492 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);
493 static int mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t);
494 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va);
495 static void mmu_radix_scan_init(void);
496 static void mmu_radix_cpu_bootstrap(int ap);
497 static void mmu_radix_tlbie_all(void);
498
499 static struct pmap_funcs mmu_radix_methods = {
500 .bootstrap = mmu_radix_bootstrap,
501 .copy_page = mmu_radix_copy_page,
502 .copy_pages = mmu_radix_copy_pages,
503 .cpu_bootstrap = mmu_radix_cpu_bootstrap,
504 .growkernel = mmu_radix_growkernel,
505 .init = mmu_radix_init,
506 .map = mmu_radix_map,
507 .mincore = mmu_radix_mincore,
508 .pinit = mmu_radix_pinit,
509 .pinit0 = mmu_radix_pinit0,
510
511 .mapdev = mmu_radix_mapdev,
512 .mapdev_attr = mmu_radix_mapdev_attr,
513 .unmapdev = mmu_radix_unmapdev,
514 .kenter_attr = mmu_radix_kenter_attr,
515 .dev_direct_mapped = mmu_radix_dev_direct_mapped,
516 .dumpsys_pa_init = mmu_radix_scan_init,
517 .dumpsys_map_chunk = mmu_radix_dumpsys_map,
518 .page_is_mapped = mmu_radix_page_is_mapped,
519 .ps_enabled = mmu_radix_ps_enabled,
520 .object_init_pt = mmu_radix_object_init_pt,
521 .protect = mmu_radix_protect,
522 /* pmap dispatcher interface */
523 .clear_modify = mmu_radix_clear_modify,
524 .copy = mmu_radix_copy,
525 .enter = mmu_radix_enter,
526 .enter_object = mmu_radix_enter_object,
527 .enter_quick = mmu_radix_enter_quick,
528 .extract = mmu_radix_extract,
529 .extract_and_hold = mmu_radix_extract_and_hold,
530 .is_modified = mmu_radix_is_modified,
531 .is_prefaultable = mmu_radix_is_prefaultable,
532 .is_referenced = mmu_radix_is_referenced,
533 .ts_referenced = mmu_radix_ts_referenced,
534 .page_exists_quick = mmu_radix_page_exists_quick,
535 .page_init = mmu_radix_page_init,
536 .page_wired_mappings = mmu_radix_page_wired_mappings,
537 .qenter = mmu_radix_qenter,
538 .qremove = mmu_radix_qremove,
539 .release = mmu_radix_release,
540 .remove = mmu_radix_remove,
541 .remove_all = mmu_radix_remove_all,
542 .remove_write = mmu_radix_remove_write,
543 .sync_icache = mmu_radix_sync_icache,
544 .unwire = mmu_radix_unwire,
545 .zero_page = mmu_radix_zero_page,
546 .zero_page_area = mmu_radix_zero_page_area,
547 .activate = mmu_radix_activate,
548 .quick_enter_page = mmu_radix_quick_enter_page,
549 .quick_remove_page = mmu_radix_quick_remove_page,
550 .page_set_memattr = mmu_radix_page_set_memattr,
551 .page_array_startup = mmu_radix_page_array_startup,
552
553 /* Internal interfaces */
554 .kenter = mmu_radix_kenter,
555 .kextract = mmu_radix_kextract,
556 .kremove = mmu_radix_kremove,
557 .change_attr = mmu_radix_change_attr,
558 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr,
559
560 .tlbie_all = mmu_radix_tlbie_all,
561 };
562
563 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods);
564
565 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
566 struct rwlock **lockp);
567 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);
568 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *);
569 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
570 struct spglist *free, struct rwlock **lockp);
571 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
572 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
573 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
574 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,
575 struct spglist *free);
576 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
577 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp);
578
579 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,
580 u_int flags, struct rwlock **lockp);
581 #if VM_NRESERVLEVEL > 0
582 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
583 struct rwlock **lockp);
584 #endif
585 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
586 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
587 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
588 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate);
589
590 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
591 vm_prot_t prot, struct rwlock **lockp);
592 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,
593 u_int flags, vm_page_t m, struct rwlock **lockp);
594
595 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
596 static void free_pv_chunk(struct pv_chunk *pc);
597 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);
598 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,
599 struct rwlock **lockp);
600 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
601 struct rwlock **lockp);
602 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
603 struct spglist *free);
604 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);
605
606 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);
607 static void pmap_invalidate_all(pmap_t pmap);
608 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush);
609
610 /*
611 * Internal flags for pmap_enter()'s helper functions.
612 */
613 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
614 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
615
616 #define UNIMPLEMENTED() panic("%s not implemented", __func__)
617 #define UNTESTED() panic("%s not yet tested", __func__)
618
619 /* Number of supported PID bits */
620 static unsigned int isa3_pid_bits;
621
622 /* PID to start allocating from */
623 static unsigned int isa3_base_pid;
624
625 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4)
626 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits)
627
628 /*
629 * Map of physical memory regions.
630 */
631 static struct mem_region *regions, *pregions;
632 static struct numa_mem_region *numa_pregions;
633 static u_int phys_avail_count;
634 static int regions_sz, pregions_sz, numa_pregions_sz;
635 static struct pate *isa3_parttab;
636 static struct prte *isa3_proctab;
637 static vmem_t *asid_arena;
638
639 extern void bs_remap_earlyboot(void);
640
641 #define RADIX_PGD_SIZE_SHIFT 16
642 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT)
643
644 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3)
645 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t))
646 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t))
647
648 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */
649 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */
650 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */
651
652 /* POWER9 only permits a 64k partition table size. */
653 #define PARTTAB_SIZE_SHIFT 16
654 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT)
655
656 #define PARTTAB_HR (1UL << 63) /* host uses radix */
657 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */
658
659 /* TLB flush actions. Used as argument to tlbiel_flush() */
660 enum {
661 TLB_INVAL_SCOPE_LPID = 2, /* invalidate TLBs for current LPID */
662 TLB_INVAL_SCOPE_GLOBAL = 3, /* invalidate all TLBs */
663 };
664
665 #define NPV_LIST_LOCKS MAXCPU
666 static int pmap_initialized;
667 static vm_paddr_t proctab0pa;
668 static vm_paddr_t parttab_phys;
669 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
670
671 /*
672 * Data for the pv entry allocation mechanism.
673 * Updates to pv_invl_gen are protected by the pv_list_locks[]
674 * elements, but reads are not.
675 */
676 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
677 static struct mtx __exclusive_cache_line pv_chunks_mutex;
678 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS];
679 static struct md_page *pv_table;
680 static struct md_page pv_dummy;
681
682 #ifdef PV_STATS
683 #define PV_STAT(x) do { x ; } while (0)
684 #else
685 #define PV_STAT(x) do { } while (0)
686 #endif
687
688 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT)
689 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)])
690
691 #define PHYS_TO_PV_LIST_LOCK(pa) \
692 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS])
693
694 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \
695 struct rwlock **_lockp = (lockp); \
696 struct rwlock *_new_lock; \
697 \
698 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \
699 if (_new_lock != *_lockp) { \
700 if (*_lockp != NULL) \
701 rw_wunlock(*_lockp); \
702 *_lockp = _new_lock; \
703 rw_wlock(*_lockp); \
704 } \
705 } while (0)
706
707 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \
708 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
709
710 #define RELEASE_PV_LIST_LOCK(lockp) do { \
711 struct rwlock **_lockp = (lockp); \
712 \
713 if (*_lockp != NULL) { \
714 rw_wunlock(*_lockp); \
715 *_lockp = NULL; \
716 } \
717 } while (0)
718
719 #define VM_PAGE_TO_PV_LIST_LOCK(m) \
720 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
721
722 /*
723 * We support 52 bits, hence:
724 * bits 52 - 31 = 21, 0b10101
725 * RTS encoding details
726 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
727 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
728 */
729 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5))
730
731 static int powernv_enabled = 1;
732
733 static __always_inline void
tlbiel_radix_set_isa300(uint32_t set,uint32_t is,uint32_t pid,uint32_t ric,uint32_t prs)734 tlbiel_radix_set_isa300(uint32_t set, uint32_t is,
735 uint32_t pid, uint32_t ric, uint32_t prs)
736 {
737 uint64_t rb;
738 uint64_t rs;
739
740 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53);
741 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31);
742
743 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
744 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
745 : "memory");
746 }
747
748 static void
tlbiel_flush_isa3(uint32_t num_sets,uint32_t is)749 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is)
750 {
751 uint32_t set;
752
753 __asm __volatile("ptesync": : :"memory");
754
755 /*
756 * Flush the first set of the TLB, and the entire Page Walk Cache
757 * and partition table entries. Then flush the remaining sets of the
758 * TLB.
759 */
760 if (is == TLB_INVAL_SCOPE_GLOBAL) {
761 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
762 for (set = 1; set < num_sets; set++)
763 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
764 }
765
766 /* Do the same for process scoped entries. */
767 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
768 for (set = 1; set < num_sets; set++)
769 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
770
771 __asm __volatile("ptesync": : :"memory");
772 }
773
774 static void
mmu_radix_tlbiel_flush(int scope)775 mmu_radix_tlbiel_flush(int scope)
776 {
777 MPASS(scope == TLB_INVAL_SCOPE_LPID ||
778 scope == TLB_INVAL_SCOPE_GLOBAL);
779
780 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope);
781 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
782 }
783
784 static void
mmu_radix_tlbie_all(void)785 mmu_radix_tlbie_all(void)
786 {
787 if (powernv_enabled)
788 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
789 else
790 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
791 }
792
793 static void
mmu_radix_init_amor(void)794 mmu_radix_init_amor(void)
795 {
796 /*
797 * In HV mode, we init AMOR (Authority Mask Override Register) so that
798 * the hypervisor and guest can setup IAMR (Instruction Authority Mask
799 * Register), enable key 0 and set it to 1.
800 *
801 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
802 */
803 mtspr(SPR_AMOR, (3ul << 62));
804 }
805
806 static void
mmu_radix_init_iamr(void)807 mmu_radix_init_iamr(void)
808 {
809 /*
810 * Radix always uses key0 of the IAMR to determine if an access is
811 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
812 * fetch.
813 */
814 mtspr(SPR_IAMR, (1ul << 62));
815 }
816
817 static void
mmu_radix_pid_set(pmap_t pmap)818 mmu_radix_pid_set(pmap_t pmap)
819 {
820
821 mtspr(SPR_PID, pmap->pm_pid);
822 isync();
823 }
824
825 /* Quick sort callout for comparing physical addresses. */
826 static int
pa_cmp(const void * a,const void * b)827 pa_cmp(const void *a, const void *b)
828 {
829 const vm_paddr_t *pa = a, *pb = b;
830
831 if (*pa < *pb)
832 return (-1);
833 else if (*pa > *pb)
834 return (1);
835 else
836 return (0);
837 }
838
839 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte)
840 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0)
841 #define pte_store(ptep, pte) do { \
842 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \
843 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \
844 } while (0)
845 /*
846 * NB: should only be used for adding directories - not for direct mappings
847 */
848 #define pde_store(ptep, pa) do { \
849 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \
850 } while (0)
851
852 #define pte_clear(ptep) do { \
853 *(u_long *)(ptep) = (u_long)(0); \
854 } while (0)
855
856 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */
857
858 /*
859 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
860 * (PTE) page mappings have identical settings for the following fields:
861 */
862 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \
863 PG_M | PG_A | RPTE_EAA_MASK | PG_V)
864
865 static __inline void
pmap_resident_count_inc(pmap_t pmap,int count)866 pmap_resident_count_inc(pmap_t pmap, int count)
867 {
868
869 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
870 pmap->pm_stats.resident_count += count;
871 }
872
873 static __inline void
pmap_resident_count_dec(pmap_t pmap,int count)874 pmap_resident_count_dec(pmap_t pmap, int count)
875 {
876
877 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
878 KASSERT(pmap->pm_stats.resident_count >= count,
879 ("pmap %p resident count underflow %ld %d", pmap,
880 pmap->pm_stats.resident_count, count));
881 pmap->pm_stats.resident_count -= count;
882 }
883
884 static void
pagezero(vm_offset_t va)885 pagezero(vm_offset_t va)
886 {
887 va = trunc_page(va);
888
889 bzero((void *)va, PAGE_SIZE);
890 }
891
892 static uint64_t
allocpages(int n)893 allocpages(int n)
894 {
895 u_int64_t ret;
896
897 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE);
898 for (int i = 0; i < n; i++)
899 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE));
900 return (ret);
901 }
902
903 static pt_entry_t *
kvtopte(vm_offset_t va)904 kvtopte(vm_offset_t va)
905 {
906 pt_entry_t *l3e;
907
908 l3e = pmap_pml3e(kernel_pmap, va);
909 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
910 return (NULL);
911 return (pmap_l3e_to_pte(l3e, va));
912 }
913
914 void
mmu_radix_kenter(vm_offset_t va,vm_paddr_t pa)915 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa)
916 {
917 pt_entry_t *pte;
918
919 pte = kvtopte(va);
920 MPASS(pte != NULL);
921 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \
922 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A);
923 }
924
925 bool
mmu_radix_ps_enabled(pmap_t pmap)926 mmu_radix_ps_enabled(pmap_t pmap)
927 {
928 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
929 }
930
931 static pt_entry_t *
pmap_nofault_pte(pmap_t pmap,vm_offset_t va,int * is_l3e)932 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e)
933 {
934 pml3_entry_t *l3e;
935 pt_entry_t *pte;
936
937 va &= PG_PS_FRAME;
938 l3e = pmap_pml3e(pmap, va);
939 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0)
940 return (NULL);
941
942 if (be64toh(*l3e) & RPTE_LEAF) {
943 *is_l3e = 1;
944 return (l3e);
945 }
946 *is_l3e = 0;
947 va &= PG_FRAME;
948 pte = pmap_l3e_to_pte(l3e, va);
949 if (pte == NULL || (be64toh(*pte) & PG_V) == 0)
950 return (NULL);
951 return (pte);
952 }
953
954 int
pmap_nofault(pmap_t pmap,vm_offset_t va,vm_prot_t flags)955 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags)
956 {
957 pt_entry_t *pte;
958 pt_entry_t startpte, origpte, newpte;
959 vm_page_t m;
960 int is_l3e;
961
962 startpte = 0;
963 retry:
964 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL)
965 return (KERN_INVALID_ADDRESS);
966 origpte = newpte = be64toh(*pte);
967 if (startpte == 0) {
968 startpte = origpte;
969 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) ||
970 ((flags & VM_PROT_READ) && (startpte & PG_A))) {
971 pmap_invalidate_all(pmap);
972 #ifdef INVARIANTS
973 if (VERBOSE_PMAP || pmap_logging)
974 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n",
975 __func__, pmap, va, flags, origpte);
976 #endif
977 return (KERN_FAILURE);
978 }
979 }
980 #ifdef INVARIANTS
981 if (VERBOSE_PMAP || pmap_logging)
982 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va,
983 flags, origpte);
984 #endif
985 PMAP_LOCK(pmap);
986 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL ||
987 be64toh(*pte) != origpte) {
988 PMAP_UNLOCK(pmap);
989 return (KERN_FAILURE);
990 }
991 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME);
992 MPASS(m != NULL);
993 switch (flags) {
994 case VM_PROT_READ:
995 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0)
996 goto protfail;
997 newpte |= PG_A;
998 vm_page_aflag_set(m, PGA_REFERENCED);
999 break;
1000 case VM_PROT_WRITE:
1001 if ((newpte & RPTE_EAA_W) == 0)
1002 goto protfail;
1003 if (is_l3e)
1004 goto protfail;
1005 newpte |= PG_M;
1006 vm_page_dirty(m);
1007 break;
1008 case VM_PROT_EXECUTE:
1009 if ((newpte & RPTE_EAA_X) == 0)
1010 goto protfail;
1011 newpte |= PG_A;
1012 vm_page_aflag_set(m, PGA_REFERENCED);
1013 break;
1014 }
1015
1016 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
1017 goto retry;
1018 ptesync();
1019 PMAP_UNLOCK(pmap);
1020 if (startpte == newpte)
1021 return (KERN_FAILURE);
1022 return (0);
1023 protfail:
1024 PMAP_UNLOCK(pmap);
1025 return (KERN_PROTECTION_FAILURE);
1026 }
1027
1028 /*
1029 * Returns TRUE if the given page is mapped individually or as part of
1030 * a 2mpage. Otherwise, returns FALSE.
1031 */
1032 boolean_t
mmu_radix_page_is_mapped(vm_page_t m)1033 mmu_radix_page_is_mapped(vm_page_t m)
1034 {
1035 struct rwlock *lock;
1036 boolean_t rv;
1037
1038 if ((m->oflags & VPO_UNMANAGED) != 0)
1039 return (FALSE);
1040 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
1041 rw_rlock(lock);
1042 rv = !TAILQ_EMPTY(&m->md.pv_list) ||
1043 ((m->flags & PG_FICTITIOUS) == 0 &&
1044 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
1045 rw_runlock(lock);
1046 return (rv);
1047 }
1048
1049 /*
1050 * Determine the appropriate bits to set in a PTE or PDE for a specified
1051 * caching mode.
1052 */
1053 static int
pmap_cache_bits(vm_memattr_t ma)1054 pmap_cache_bits(vm_memattr_t ma)
1055 {
1056 if (ma != VM_MEMATTR_DEFAULT) {
1057 switch (ma) {
1058 case VM_MEMATTR_UNCACHEABLE:
1059 return (RPTE_ATTR_GUARDEDIO);
1060 case VM_MEMATTR_CACHEABLE:
1061 return (RPTE_ATTR_MEM);
1062 case VM_MEMATTR_WRITE_BACK:
1063 case VM_MEMATTR_PREFETCHABLE:
1064 case VM_MEMATTR_WRITE_COMBINING:
1065 return (RPTE_ATTR_UNGUARDEDIO);
1066 }
1067 }
1068 return (0);
1069 }
1070
1071 static void
pmap_invalidate_page(pmap_t pmap,vm_offset_t start)1072 pmap_invalidate_page(pmap_t pmap, vm_offset_t start)
1073 {
1074 ptesync();
1075 if (pmap == kernel_pmap)
1076 radix_tlbie_invlpg_kernel_4k(start);
1077 else
1078 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1079 ttusync();
1080 }
1081
1082 static void
pmap_invalidate_page_2m(pmap_t pmap,vm_offset_t start)1083 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start)
1084 {
1085 ptesync();
1086 if (pmap == kernel_pmap)
1087 radix_tlbie_invlpg_kernel_2m(start);
1088 else
1089 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);
1090 ttusync();
1091 }
1092
1093 static void
pmap_invalidate_pwc(pmap_t pmap)1094 pmap_invalidate_pwc(pmap_t pmap)
1095 {
1096 ptesync();
1097 if (pmap == kernel_pmap)
1098 radix_tlbie_invlpwc_kernel();
1099 else
1100 radix_tlbie_invlpwc_user(pmap->pm_pid);
1101 ttusync();
1102 }
1103
1104 static void
pmap_invalidate_range(pmap_t pmap,vm_offset_t start,vm_offset_t end)1105 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
1106 {
1107 if (((start - end) >> PAGE_SHIFT) > 8) {
1108 pmap_invalidate_all(pmap);
1109 return;
1110 }
1111 ptesync();
1112 if (pmap == kernel_pmap) {
1113 while (start < end) {
1114 radix_tlbie_invlpg_kernel_4k(start);
1115 start += PAGE_SIZE;
1116 }
1117 } else {
1118 while (start < end) {
1119 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1120 start += PAGE_SIZE;
1121 }
1122 }
1123 ttusync();
1124 }
1125
1126 static void
pmap_invalidate_all(pmap_t pmap)1127 pmap_invalidate_all(pmap_t pmap)
1128 {
1129 ptesync();
1130 if (pmap == kernel_pmap)
1131 radix_tlbie_flush_kernel();
1132 else
1133 radix_tlbie_flush_user(pmap->pm_pid);
1134 ttusync();
1135 }
1136
1137 static void
pmap_invalidate_l3e_page(pmap_t pmap,vm_offset_t va,pml3_entry_t l3e)1138 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e)
1139 {
1140
1141 /*
1142 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created
1143 * by a promotion that did not invalidate the 512 4KB page mappings
1144 * that might exist in the TLB. Consequently, at this point, the TLB
1145 * may hold both 4KB and 2MB page mappings for the address range [va,
1146 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here.
1147 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any
1148 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a
1149 * single INVLPG suffices to invalidate the 2MB page mapping from the
1150 * TLB.
1151 */
1152 ptesync();
1153 if ((l3e & PG_PROMOTED) != 0)
1154 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);
1155 else
1156 pmap_invalidate_page_2m(pmap, va);
1157
1158 pmap_invalidate_pwc(pmap);
1159 }
1160
1161 static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)1162 pv_to_chunk(pv_entry_t pv)
1163 {
1164
1165 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
1166 }
1167
1168 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1169
1170 #define PC_FREE0 0xfffffffffffffffful
1171 #define PC_FREE1 0x3ffffffffffffffful
1172
1173 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 };
1174
1175 /*
1176 * Ensure that the number of spare PV entries in the specified pmap meets or
1177 * exceeds the given count, "needed".
1178 *
1179 * The given PV list lock may be released.
1180 */
1181 static void
reserve_pv_entries(pmap_t pmap,int needed,struct rwlock ** lockp)1182 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
1183 {
1184 struct pch new_tail;
1185 struct pv_chunk *pc;
1186 vm_page_t m;
1187 int avail, free;
1188 bool reclaimed;
1189
1190 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1191 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
1192
1193 /*
1194 * Newly allocated PV chunks must be stored in a private list until
1195 * the required number of PV chunks have been allocated. Otherwise,
1196 * reclaim_pv_chunk() could recycle one of these chunks. In
1197 * contrast, these chunks must be added to the pmap upon allocation.
1198 */
1199 TAILQ_INIT(&new_tail);
1200 retry:
1201 avail = 0;
1202 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
1203 // if ((cpu_feature2 & CPUID2_POPCNT) == 0)
1204 bit_count((bitstr_t *)pc->pc_map, 0,
1205 sizeof(pc->pc_map) * NBBY, &free);
1206 #if 0
1207 free = popcnt_pc_map_pq(pc->pc_map);
1208 #endif
1209 if (free == 0)
1210 break;
1211 avail += free;
1212 if (avail >= needed)
1213 break;
1214 }
1215 for (reclaimed = false; avail < needed; avail += _NPCPV) {
1216 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1217 if (m == NULL) {
1218 m = reclaim_pv_chunk(pmap, lockp);
1219 if (m == NULL)
1220 goto retry;
1221 reclaimed = true;
1222 }
1223 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1224 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1225 dump_add_page(m->phys_addr);
1226 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1227 pc->pc_pmap = pmap;
1228 pc->pc_map[0] = PC_FREE0;
1229 pc->pc_map[1] = PC_FREE1;
1230 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1231 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
1232 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
1233
1234 /*
1235 * The reclaim might have freed a chunk from the current pmap.
1236 * If that chunk contained available entries, we need to
1237 * re-count the number of available entries.
1238 */
1239 if (reclaimed)
1240 goto retry;
1241 }
1242 if (!TAILQ_EMPTY(&new_tail)) {
1243 mtx_lock(&pv_chunks_mutex);
1244 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
1245 mtx_unlock(&pv_chunks_mutex);
1246 }
1247 }
1248
1249 /*
1250 * First find and then remove the pv entry for the specified pmap and virtual
1251 * address from the specified pv list. Returns the pv entry if found and NULL
1252 * otherwise. This operation can be performed on pv lists for either 4KB or
1253 * 2MB page mappings.
1254 */
1255 static __inline pv_entry_t
pmap_pvh_remove(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1256 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1257 {
1258 pv_entry_t pv;
1259
1260 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
1261 #ifdef INVARIANTS
1262 if (PV_PMAP(pv) == NULL) {
1263 printf("corrupted pv_chunk/pv %p\n", pv);
1264 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":");
1265 }
1266 MPASS(PV_PMAP(pv) != NULL);
1267 MPASS(pv->pv_va != 0);
1268 #endif
1269 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
1270 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
1271 pvh->pv_gen++;
1272 break;
1273 }
1274 }
1275 return (pv);
1276 }
1277
1278 /*
1279 * After demotion from a 2MB page mapping to 512 4KB page mappings,
1280 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
1281 * entries for each of the 4KB page mappings.
1282 */
1283 static void
pmap_pv_demote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1284 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1285 struct rwlock **lockp)
1286 {
1287 struct md_page *pvh;
1288 struct pv_chunk *pc;
1289 pv_entry_t pv;
1290 vm_offset_t va_last;
1291 vm_page_t m;
1292 int bit, field;
1293
1294 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1295 KASSERT((pa & L3_PAGE_MASK) == 0,
1296 ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
1297 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1298
1299 /*
1300 * Transfer the 2mpage's pv entry for this mapping to the first
1301 * page's pv list. Once this transfer begins, the pv list lock
1302 * must not be released until the last pv entry is reinstantiated.
1303 */
1304 pvh = pa_to_pvh(pa);
1305 va = trunc_2mpage(va);
1306 pv = pmap_pvh_remove(pvh, pmap, va);
1307 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
1308 m = PHYS_TO_VM_PAGE(pa);
1309 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1310
1311 m->md.pv_gen++;
1312 /* Instantiate the remaining NPTEPG - 1 pv entries. */
1313 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
1314 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1315 for (;;) {
1316 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1317 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0
1318 , ("pmap_pv_demote_pde: missing spare"));
1319 for (field = 0; field < _NPCM; field++) {
1320 while (pc->pc_map[field]) {
1321 bit = cnttzd(pc->pc_map[field]);
1322 pc->pc_map[field] &= ~(1ul << bit);
1323 pv = &pc->pc_pventry[field * 64 + bit];
1324 va += PAGE_SIZE;
1325 pv->pv_va = va;
1326 m++;
1327 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1328 ("pmap_pv_demote_pde: page %p is not managed", m));
1329 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1330
1331 m->md.pv_gen++;
1332 if (va == va_last)
1333 goto out;
1334 }
1335 }
1336 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1337 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1338 }
1339 out:
1340 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1341 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1342 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1343 }
1344 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
1345 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
1346 }
1347
1348 static void
reclaim_pv_chunk_leave_pmap(pmap_t pmap,pmap_t locked_pmap)1349 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap)
1350 {
1351
1352 if (pmap == NULL)
1353 return;
1354 pmap_invalidate_all(pmap);
1355 if (pmap != locked_pmap)
1356 PMAP_UNLOCK(pmap);
1357 }
1358
1359 /*
1360 * We are in a serious low memory condition. Resort to
1361 * drastic measures to free some pages so we can allocate
1362 * another pv entry chunk.
1363 *
1364 * Returns NULL if PV entries were reclaimed from the specified pmap.
1365 *
1366 * We do not, however, unmap 2mpages because subsequent accesses will
1367 * allocate per-page pv entries until repromotion occurs, thereby
1368 * exacerbating the shortage of free pv entries.
1369 */
1370 static int active_reclaims = 0;
1371 static vm_page_t
reclaim_pv_chunk(pmap_t locked_pmap,struct rwlock ** lockp)1372 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
1373 {
1374 struct pv_chunk *pc, *pc_marker, *pc_marker_end;
1375 struct pv_chunk_header pc_marker_b, pc_marker_end_b;
1376 struct md_page *pvh;
1377 pml3_entry_t *l3e;
1378 pmap_t next_pmap, pmap;
1379 pt_entry_t *pte, tpte;
1380 pv_entry_t pv;
1381 vm_offset_t va;
1382 vm_page_t m, m_pc;
1383 struct spglist free;
1384 uint64_t inuse;
1385 int bit, field, freed;
1386
1387 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
1388 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
1389 pmap = NULL;
1390 m_pc = NULL;
1391 SLIST_INIT(&free);
1392 bzero(&pc_marker_b, sizeof(pc_marker_b));
1393 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
1394 pc_marker = (struct pv_chunk *)&pc_marker_b;
1395 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
1396
1397 mtx_lock(&pv_chunks_mutex);
1398 active_reclaims++;
1399 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
1400 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
1401 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
1402 SLIST_EMPTY(&free)) {
1403 next_pmap = pc->pc_pmap;
1404 if (next_pmap == NULL) {
1405 /*
1406 * The next chunk is a marker. However, it is
1407 * not our marker, so active_reclaims must be
1408 * > 1. Consequently, the next_chunk code
1409 * will not rotate the pv_chunks list.
1410 */
1411 goto next_chunk;
1412 }
1413 mtx_unlock(&pv_chunks_mutex);
1414
1415 /*
1416 * A pv_chunk can only be removed from the pc_lru list
1417 * when both pc_chunks_mutex is owned and the
1418 * corresponding pmap is locked.
1419 */
1420 if (pmap != next_pmap) {
1421 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1422 pmap = next_pmap;
1423 /* Avoid deadlock and lock recursion. */
1424 if (pmap > locked_pmap) {
1425 RELEASE_PV_LIST_LOCK(lockp);
1426 PMAP_LOCK(pmap);
1427 mtx_lock(&pv_chunks_mutex);
1428 continue;
1429 } else if (pmap != locked_pmap) {
1430 if (PMAP_TRYLOCK(pmap)) {
1431 mtx_lock(&pv_chunks_mutex);
1432 continue;
1433 } else {
1434 pmap = NULL; /* pmap is not locked */
1435 mtx_lock(&pv_chunks_mutex);
1436 pc = TAILQ_NEXT(pc_marker, pc_lru);
1437 if (pc == NULL ||
1438 pc->pc_pmap != next_pmap)
1439 continue;
1440 goto next_chunk;
1441 }
1442 }
1443 }
1444
1445 /*
1446 * Destroy every non-wired, 4 KB page mapping in the chunk.
1447 */
1448 freed = 0;
1449 for (field = 0; field < _NPCM; field++) {
1450 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1451 inuse != 0; inuse &= ~(1UL << bit)) {
1452 bit = cnttzd(inuse);
1453 pv = &pc->pc_pventry[field * 64 + bit];
1454 va = pv->pv_va;
1455 l3e = pmap_pml3e(pmap, va);
1456 if ((be64toh(*l3e) & RPTE_LEAF) != 0)
1457 continue;
1458 pte = pmap_l3e_to_pte(l3e, va);
1459 if ((be64toh(*pte) & PG_W) != 0)
1460 continue;
1461 tpte = be64toh(pte_load_clear(pte));
1462 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
1463 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
1464 vm_page_dirty(m);
1465 if ((tpte & PG_A) != 0)
1466 vm_page_aflag_set(m, PGA_REFERENCED);
1467 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1468 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
1469
1470 m->md.pv_gen++;
1471 if (TAILQ_EMPTY(&m->md.pv_list) &&
1472 (m->flags & PG_FICTITIOUS) == 0) {
1473 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
1474 if (TAILQ_EMPTY(&pvh->pv_list)) {
1475 vm_page_aflag_clear(m,
1476 PGA_WRITEABLE);
1477 }
1478 }
1479 pc->pc_map[field] |= 1UL << bit;
1480 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);
1481 freed++;
1482 }
1483 }
1484 if (freed == 0) {
1485 mtx_lock(&pv_chunks_mutex);
1486 goto next_chunk;
1487 }
1488 /* Every freed mapping is for a 4 KB page. */
1489 pmap_resident_count_dec(pmap, freed);
1490 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
1491 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
1492 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
1493 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1494 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) {
1495 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1496 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1497 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1498 /* Entire chunk is free; return it. */
1499 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
1500 dump_drop_page(m_pc->phys_addr);
1501 mtx_lock(&pv_chunks_mutex);
1502 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1503 break;
1504 }
1505 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1506 mtx_lock(&pv_chunks_mutex);
1507 /* One freed pv entry in locked_pmap is sufficient. */
1508 if (pmap == locked_pmap)
1509 break;
1510 next_chunk:
1511 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1512 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
1513 if (active_reclaims == 1 && pmap != NULL) {
1514 /*
1515 * Rotate the pv chunks list so that we do not
1516 * scan the same pv chunks that could not be
1517 * freed (because they contained a wired
1518 * and/or superpage mapping) on every
1519 * invocation of reclaim_pv_chunk().
1520 */
1521 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
1522 MPASS(pc->pc_pmap != NULL);
1523 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1524 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1525 }
1526 }
1527 }
1528 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1529 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
1530 active_reclaims--;
1531 mtx_unlock(&pv_chunks_mutex);
1532 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1533 if (m_pc == NULL && !SLIST_EMPTY(&free)) {
1534 m_pc = SLIST_FIRST(&free);
1535 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
1536 /* Recycle a freed page table page. */
1537 m_pc->ref_count = 1;
1538 }
1539 vm_page_free_pages_toq(&free, true);
1540 return (m_pc);
1541 }
1542
1543 /*
1544 * free the pv_entry back to the free list
1545 */
1546 static void
free_pv_entry(pmap_t pmap,pv_entry_t pv)1547 free_pv_entry(pmap_t pmap, pv_entry_t pv)
1548 {
1549 struct pv_chunk *pc;
1550 int idx, field, bit;
1551
1552 #ifdef VERBOSE_PV
1553 if (pmap != kernel_pmap)
1554 printf("%s(%p, %p)\n", __func__, pmap, pv);
1555 #endif
1556 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1557 PV_STAT(atomic_add_long(&pv_entry_frees, 1));
1558 PV_STAT(atomic_add_int(&pv_entry_spare, 1));
1559 PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
1560 pc = pv_to_chunk(pv);
1561 idx = pv - &pc->pc_pventry[0];
1562 field = idx / 64;
1563 bit = idx % 64;
1564 pc->pc_map[field] |= 1ul << bit;
1565 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) {
1566 /* 98% of the time, pc is already at the head of the list. */
1567 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
1568 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1569 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1570 }
1571 return;
1572 }
1573 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1574 free_pv_chunk(pc);
1575 }
1576
1577 static void
free_pv_chunk(struct pv_chunk * pc)1578 free_pv_chunk(struct pv_chunk *pc)
1579 {
1580 vm_page_t m;
1581
1582 mtx_lock(&pv_chunks_mutex);
1583 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1584 mtx_unlock(&pv_chunks_mutex);
1585 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1586 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1587 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1588 /* entire chunk is free, return it */
1589 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
1590 dump_drop_page(m->phys_addr);
1591 vm_page_unwire_noq(m);
1592 vm_page_free(m);
1593 }
1594
1595 /*
1596 * Returns a new PV entry, allocating a new PV chunk from the system when
1597 * needed. If this PV chunk allocation fails and a PV list lock pointer was
1598 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is
1599 * returned.
1600 *
1601 * The given PV list lock may be released.
1602 */
1603 static pv_entry_t
get_pv_entry(pmap_t pmap,struct rwlock ** lockp)1604 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
1605 {
1606 int bit, field;
1607 pv_entry_t pv;
1608 struct pv_chunk *pc;
1609 vm_page_t m;
1610
1611 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1612 PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
1613 retry:
1614 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1615 if (pc != NULL) {
1616 for (field = 0; field < _NPCM; field++) {
1617 if (pc->pc_map[field]) {
1618 bit = cnttzd(pc->pc_map[field]);
1619 break;
1620 }
1621 }
1622 if (field < _NPCM) {
1623 pv = &pc->pc_pventry[field * 64 + bit];
1624 pc->pc_map[field] &= ~(1ul << bit);
1625 /* If this was the last item, move it to tail */
1626 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1627 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1628 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
1629 pc_list);
1630 }
1631 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1632 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
1633 MPASS(PV_PMAP(pv) != NULL);
1634 return (pv);
1635 }
1636 }
1637 /* No free items, allocate another chunk */
1638 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1639 if (m == NULL) {
1640 if (lockp == NULL) {
1641 PV_STAT(pc_chunk_tryfail++);
1642 return (NULL);
1643 }
1644 m = reclaim_pv_chunk(pmap, lockp);
1645 if (m == NULL)
1646 goto retry;
1647 }
1648 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1649 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1650 dump_add_page(m->phys_addr);
1651 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1652 pc->pc_pmap = pmap;
1653 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
1654 pc->pc_map[1] = PC_FREE1;
1655 mtx_lock(&pv_chunks_mutex);
1656 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1657 mtx_unlock(&pv_chunks_mutex);
1658 pv = &pc->pc_pventry[0];
1659 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1660 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1661 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
1662 MPASS(PV_PMAP(pv) != NULL);
1663 return (pv);
1664 }
1665
1666 #if VM_NRESERVLEVEL > 0
1667 /*
1668 * After promotion from 512 4KB page mappings to a single 2MB page mapping,
1669 * replace the many pv entries for the 4KB page mappings by a single pv entry
1670 * for the 2MB page mapping.
1671 */
1672 static void
pmap_pv_promote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1673 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1674 struct rwlock **lockp)
1675 {
1676 struct md_page *pvh;
1677 pv_entry_t pv;
1678 vm_offset_t va_last;
1679 vm_page_t m;
1680
1681 KASSERT((pa & L3_PAGE_MASK) == 0,
1682 ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
1683 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1684
1685 /*
1686 * Transfer the first page's pv entry for this mapping to the 2mpage's
1687 * pv list. Aside from avoiding the cost of a call to get_pv_entry(),
1688 * a transfer avoids the possibility that get_pv_entry() calls
1689 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
1690 * mappings that is being promoted.
1691 */
1692 m = PHYS_TO_VM_PAGE(pa);
1693 va = trunc_2mpage(va);
1694 pv = pmap_pvh_remove(&m->md, pmap, va);
1695 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
1696 pvh = pa_to_pvh(pa);
1697 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
1698 pvh->pv_gen++;
1699 /* Free the remaining NPTEPG - 1 pv entries. */
1700 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1701 do {
1702 m++;
1703 va += PAGE_SIZE;
1704 pmap_pvh_free(&m->md, pmap, va);
1705 } while (va < va_last);
1706 }
1707 #endif /* VM_NRESERVLEVEL > 0 */
1708
1709 /*
1710 * First find and then destroy the pv entry for the specified pmap and virtual
1711 * address. This operation can be performed on pv lists for either 4KB or 2MB
1712 * page mappings.
1713 */
1714 static void
pmap_pvh_free(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1715 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1716 {
1717 pv_entry_t pv;
1718
1719 pv = pmap_pvh_remove(pvh, pmap, va);
1720 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
1721 free_pv_entry(pmap, pv);
1722 }
1723
1724 /*
1725 * Conditionally create the PV entry for a 4KB page mapping if the required
1726 * memory can be allocated without resorting to reclamation.
1727 */
1728 static boolean_t
pmap_try_insert_pv_entry(pmap_t pmap,vm_offset_t va,vm_page_t m,struct rwlock ** lockp)1729 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
1730 struct rwlock **lockp)
1731 {
1732 pv_entry_t pv;
1733
1734 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1735 /* Pass NULL instead of the lock pointer to disable reclamation. */
1736 if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
1737 pv->pv_va = va;
1738 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1739 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1740 m->md.pv_gen++;
1741 return (TRUE);
1742 } else
1743 return (FALSE);
1744 }
1745
1746 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX];
1747 #ifdef INVARIANTS
1748 static void
validate_addr(vm_paddr_t addr,vm_size_t size)1749 validate_addr(vm_paddr_t addr, vm_size_t size)
1750 {
1751 vm_paddr_t end = addr + size;
1752 bool found = false;
1753
1754 for (int i = 0; i < 2 * phys_avail_count; i += 2) {
1755 if (addr >= phys_avail_debug[i] &&
1756 end <= phys_avail_debug[i + 1]) {
1757 found = true;
1758 break;
1759 }
1760 }
1761 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array",
1762 addr, end));
1763 }
1764 #else
validate_addr(vm_paddr_t addr,vm_size_t size)1765 static void validate_addr(vm_paddr_t addr, vm_size_t size) {}
1766 #endif
1767 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A)
1768
1769 static vm_paddr_t
alloc_pt_page(void)1770 alloc_pt_page(void)
1771 {
1772 vm_paddr_t page;
1773
1774 page = allocpages(1);
1775 pagezero(PHYS_TO_DMAP(page));
1776 return (page);
1777 }
1778
1779 static void
mmu_radix_dmap_range(vm_paddr_t start,vm_paddr_t end)1780 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end)
1781 {
1782 pt_entry_t *pte, pteval;
1783 vm_paddr_t page;
1784
1785 if (bootverbose)
1786 printf("%s %lx -> %lx\n", __func__, start, end);
1787 while (start < end) {
1788 pteval = start | DMAP_PAGE_BITS;
1789 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start));
1790 if ((be64toh(*pte) & RPTE_VALID) == 0) {
1791 page = alloc_pt_page();
1792 pde_store(pte, page);
1793 }
1794 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start));
1795 if ((start & L2_PAGE_MASK) == 0 &&
1796 end - start >= L2_PAGE_SIZE) {
1797 start += L2_PAGE_SIZE;
1798 goto done;
1799 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1800 page = alloc_pt_page();
1801 pde_store(pte, page);
1802 }
1803
1804 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start));
1805 if ((start & L3_PAGE_MASK) == 0 &&
1806 end - start >= L3_PAGE_SIZE) {
1807 start += L3_PAGE_SIZE;
1808 goto done;
1809 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1810 page = alloc_pt_page();
1811 pde_store(pte, page);
1812 }
1813 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start));
1814 start += PAGE_SIZE;
1815 done:
1816 pte_store(pte, pteval);
1817 }
1818 }
1819
1820 static void
mmu_radix_dmap_populate(vm_size_t hwphyssz)1821 mmu_radix_dmap_populate(vm_size_t hwphyssz)
1822 {
1823 vm_paddr_t start, end;
1824
1825 for (int i = 0; i < pregions_sz; i++) {
1826 start = pregions[i].mr_start;
1827 end = start + pregions[i].mr_size;
1828 if (hwphyssz && start >= hwphyssz)
1829 break;
1830 if (hwphyssz && hwphyssz < end)
1831 end = hwphyssz;
1832 mmu_radix_dmap_range(start, end);
1833 }
1834 }
1835
1836 static void
mmu_radix_setup_pagetables(vm_size_t hwphyssz)1837 mmu_radix_setup_pagetables(vm_size_t hwphyssz)
1838 {
1839 vm_paddr_t ptpages, pages;
1840 pt_entry_t *pte;
1841 vm_paddr_t l1phys;
1842
1843 bzero(kernel_pmap, sizeof(struct pmap));
1844 PMAP_LOCK_INIT(kernel_pmap);
1845
1846 ptpages = allocpages(3);
1847 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE);
1848 validate_addr(l1phys, RADIX_PGD_SIZE);
1849 if (bootverbose)
1850 printf("l1phys=%lx\n", l1phys);
1851 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0);
1852 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++)
1853 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE));
1854 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys);
1855
1856 mmu_radix_dmap_populate(hwphyssz);
1857
1858 /*
1859 * Create page tables for first 128MB of KVA
1860 */
1861 pages = ptpages;
1862 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS);
1863 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1864 pages += PAGE_SIZE;
1865 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS);
1866 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1867 pages += PAGE_SIZE;
1868 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS);
1869 /*
1870 * the kernel page table pages need to be preserved in
1871 * phys_avail and not overlap with previous allocations
1872 */
1873 pages = allocpages(nkpt);
1874 if (bootverbose) {
1875 printf("phys_avail after dmap populate and nkpt allocation\n");
1876 for (int j = 0; j < 2 * phys_avail_count; j+=2)
1877 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
1878 j, phys_avail[j], j + 1, phys_avail[j + 1]);
1879 }
1880 KPTphys = pages;
1881 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE)
1882 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1883 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE;
1884 if (bootverbose)
1885 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1);
1886 /*
1887 * Add a physical memory segment (vm_phys_seg) corresponding to the
1888 * preallocated kernel page table pages so that vm_page structures
1889 * representing these pages will be created. The vm_page structures
1890 * are required for promotion of the corresponding kernel virtual
1891 * addresses to superpage mappings.
1892 */
1893 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt));
1894 }
1895
1896 static void
mmu_radix_early_bootstrap(vm_offset_t start,vm_offset_t end)1897 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end)
1898 {
1899 vm_paddr_t kpstart, kpend;
1900 vm_size_t physsz, hwphyssz;
1901 //uint64_t l2virt;
1902 int rm_pavail, proctab_size;
1903 int i, j;
1904
1905 kpstart = start & ~DMAP_BASE_ADDRESS;
1906 kpend = end & ~DMAP_BASE_ADDRESS;
1907
1908 /* Get physical memory regions from firmware */
1909 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz);
1910 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory");
1911
1912 if (2 * VM_PHYSSEG_MAX < regions_sz)
1913 panic("mmu_radix_early_bootstrap: phys_avail too small");
1914
1915 if (bootverbose)
1916 for (int i = 0; i < regions_sz; i++)
1917 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n",
1918 i, regions[i].mr_start, i, regions[i].mr_size);
1919 /*
1920 * XXX workaround a simulator bug
1921 */
1922 for (int i = 0; i < regions_sz; i++)
1923 if (regions[i].mr_start & PAGE_MASK) {
1924 regions[i].mr_start += PAGE_MASK;
1925 regions[i].mr_start &= ~PAGE_MASK;
1926 regions[i].mr_size &= ~PAGE_MASK;
1927 }
1928 if (bootverbose)
1929 for (int i = 0; i < pregions_sz; i++)
1930 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n",
1931 i, pregions[i].mr_start, i, pregions[i].mr_size);
1932
1933 phys_avail_count = 0;
1934 physsz = 0;
1935 hwphyssz = 0;
1936 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
1937 for (i = 0, j = 0; i < regions_sz; i++) {
1938 if (bootverbose)
1939 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n",
1940 i, regions[i].mr_start, i, regions[i].mr_size);
1941
1942 if (regions[i].mr_size < PAGE_SIZE)
1943 continue;
1944
1945 if (hwphyssz != 0 &&
1946 (physsz + regions[i].mr_size) >= hwphyssz) {
1947 if (physsz < hwphyssz) {
1948 phys_avail[j] = regions[i].mr_start;
1949 phys_avail[j + 1] = regions[i].mr_start +
1950 (hwphyssz - physsz);
1951 physsz = hwphyssz;
1952 phys_avail_count++;
1953 dump_avail[j] = phys_avail[j];
1954 dump_avail[j + 1] = phys_avail[j + 1];
1955 }
1956 break;
1957 }
1958 phys_avail[j] = regions[i].mr_start;
1959 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
1960 dump_avail[j] = phys_avail[j];
1961 dump_avail[j + 1] = phys_avail[j + 1];
1962
1963 phys_avail_count++;
1964 physsz += regions[i].mr_size;
1965 j += 2;
1966 }
1967
1968 /* Check for overlap with the kernel and exception vectors */
1969 rm_pavail = 0;
1970 for (j = 0; j < 2 * phys_avail_count; j+=2) {
1971 if (phys_avail[j] < EXC_LAST)
1972 phys_avail[j] += EXC_LAST;
1973
1974 if (phys_avail[j] >= kpstart &&
1975 phys_avail[j + 1] <= kpend) {
1976 phys_avail[j] = phys_avail[j + 1] = ~0;
1977 rm_pavail++;
1978 continue;
1979 }
1980
1981 if (kpstart >= phys_avail[j] &&
1982 kpstart < phys_avail[j + 1]) {
1983 if (kpend < phys_avail[j + 1]) {
1984 phys_avail[2 * phys_avail_count] =
1985 (kpend & ~PAGE_MASK) + PAGE_SIZE;
1986 phys_avail[2 * phys_avail_count + 1] =
1987 phys_avail[j + 1];
1988 phys_avail_count++;
1989 }
1990
1991 phys_avail[j + 1] = kpstart & ~PAGE_MASK;
1992 }
1993
1994 if (kpend >= phys_avail[j] &&
1995 kpend < phys_avail[j + 1]) {
1996 if (kpstart > phys_avail[j]) {
1997 phys_avail[2 * phys_avail_count] = phys_avail[j];
1998 phys_avail[2 * phys_avail_count + 1] =
1999 kpstart & ~PAGE_MASK;
2000 phys_avail_count++;
2001 }
2002
2003 phys_avail[j] = (kpend & ~PAGE_MASK) +
2004 PAGE_SIZE;
2005 }
2006 }
2007 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp);
2008 for (i = 0; i < 2 * phys_avail_count; i++)
2009 phys_avail_debug[i] = phys_avail[i];
2010
2011 /* Remove physical available regions marked for removal (~0) */
2012 if (rm_pavail) {
2013 phys_avail_count -= rm_pavail;
2014 for (i = 2 * phys_avail_count;
2015 i < 2*(phys_avail_count + rm_pavail); i+=2)
2016 phys_avail[i] = phys_avail[i + 1] = 0;
2017 }
2018 if (bootverbose) {
2019 printf("phys_avail ranges after filtering:\n");
2020 for (j = 0; j < 2 * phys_avail_count; j+=2)
2021 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
2022 j, phys_avail[j], j + 1, phys_avail[j + 1]);
2023 }
2024 physmem = btoc(physsz);
2025
2026 /* XXX assume we're running non-virtualized and
2027 * we don't support BHYVE
2028 */
2029 if (isa3_pid_bits == 0)
2030 isa3_pid_bits = 20;
2031 if (powernv_enabled) {
2032 parttab_phys =
2033 moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE);
2034 validate_addr(parttab_phys, PARTTAB_SIZE);
2035 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++)
2036 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE));
2037
2038 }
2039 proctab_size = 1UL << PROCTAB_SIZE_SHIFT;
2040 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size);
2041 validate_addr(proctab0pa, proctab_size);
2042 for (int i = 0; i < proctab_size/PAGE_SIZE; i++)
2043 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE));
2044
2045 mmu_radix_setup_pagetables(hwphyssz);
2046 }
2047
2048 static void
mmu_radix_late_bootstrap(vm_offset_t start,vm_offset_t end)2049 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end)
2050 {
2051 int i;
2052 vm_paddr_t pa;
2053 void *dpcpu;
2054 vm_offset_t va;
2055
2056 /*
2057 * Set up the Open Firmware pmap and add its mappings if not in real
2058 * mode.
2059 */
2060 if (bootverbose)
2061 printf("%s enter\n", __func__);
2062
2063 /*
2064 * Calculate the last available physical address, and reserve the
2065 * vm_page_array (upper bound).
2066 */
2067 Maxmem = 0;
2068 for (i = 0; phys_avail[i + 1] != 0; i += 2)
2069 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1]));
2070
2071 /*
2072 * Remap any early IO mappings (console framebuffer, etc.)
2073 */
2074 bs_remap_earlyboot();
2075
2076 /*
2077 * Allocate a kernel stack with a guard page for thread0 and map it
2078 * into the kernel page map.
2079 */
2080 pa = allocpages(kstack_pages);
2081 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
2082 virtual_avail = va + kstack_pages * PAGE_SIZE;
2083 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va);
2084 thread0.td_kstack = va;
2085 for (i = 0; i < kstack_pages; i++) {
2086 mmu_radix_kenter(va, pa);
2087 pa += PAGE_SIZE;
2088 va += PAGE_SIZE;
2089 }
2090 thread0.td_kstack_pages = kstack_pages;
2091
2092 /*
2093 * Allocate virtual address space for the message buffer.
2094 */
2095 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT);
2096 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa);
2097
2098 /*
2099 * Allocate virtual address space for the dynamic percpu area.
2100 */
2101 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT);
2102 dpcpu = (void *)PHYS_TO_DMAP(pa);
2103 dpcpu_init(dpcpu, curcpu);
2104
2105 crashdumpmap = (caddr_t)virtual_avail;
2106 virtual_avail += MAXDUMPPGS * PAGE_SIZE;
2107
2108 /*
2109 * Reserve some special page table entries/VA space for temporary
2110 * mapping of pages.
2111 */
2112 }
2113
2114 static void
mmu_parttab_init(void)2115 mmu_parttab_init(void)
2116 {
2117 uint64_t ptcr;
2118
2119 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys);
2120
2121 if (bootverbose)
2122 printf("%s parttab: %p\n", __func__, isa3_parttab);
2123 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2124 if (bootverbose)
2125 printf("setting ptcr %lx\n", ptcr);
2126 mtspr(SPR_PTCR, ptcr);
2127 }
2128
2129 static void
mmu_parttab_update(uint64_t lpid,uint64_t pagetab,uint64_t proctab)2130 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab)
2131 {
2132 uint64_t prev;
2133
2134 if (bootverbose)
2135 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab,
2136 lpid, pagetab, proctab);
2137 prev = be64toh(isa3_parttab[lpid].pagetab);
2138 isa3_parttab[lpid].pagetab = htobe64(pagetab);
2139 isa3_parttab[lpid].proctab = htobe64(proctab);
2140
2141 if (prev & PARTTAB_HR) {
2142 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
2143 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2144 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2145 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2146 } else {
2147 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
2148 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2149 }
2150 ttusync();
2151 }
2152
2153 static void
mmu_radix_parttab_init(void)2154 mmu_radix_parttab_init(void)
2155 {
2156 uint64_t pagetab;
2157
2158 mmu_parttab_init();
2159 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \
2160 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR;
2161 mmu_parttab_update(0, pagetab, 0);
2162 }
2163
2164 static void
mmu_radix_proctab_register(vm_paddr_t proctabpa,uint64_t table_size)2165 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size)
2166 {
2167 uint64_t pagetab, proctab;
2168
2169 pagetab = be64toh(isa3_parttab[0].pagetab);
2170 proctab = proctabpa | table_size | PARTTAB_GR;
2171 mmu_parttab_update(0, pagetab, proctab);
2172 }
2173
2174 static void
mmu_radix_proctab_init(void)2175 mmu_radix_proctab_init(void)
2176 {
2177
2178 isa3_base_pid = 1;
2179
2180 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa);
2181 isa3_proctab->proctab0 =
2182 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) |
2183 RADIX_PGD_INDEX_SHIFT);
2184
2185 if (powernv_enabled) {
2186 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12);
2187 __asm __volatile("ptesync" : : : "memory");
2188 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2189 "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
2190 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory");
2191 #ifdef PSERIES
2192 } else {
2193 int64_t rc;
2194
2195 rc = phyp_hcall(H_REGISTER_PROC_TBL,
2196 PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE,
2197 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12);
2198 if (rc != H_SUCCESS)
2199 panic("mmu_radix_proctab_init: "
2200 "failed to register process table: rc=%jd",
2201 (intmax_t)rc);
2202 #endif
2203 }
2204
2205 if (bootverbose)
2206 printf("process table %p and kernel radix PDE: %p\n",
2207 isa3_proctab, kernel_pmap->pm_pml1);
2208 mtmsr(mfmsr() | PSL_DR );
2209 mtmsr(mfmsr() & ~PSL_DR);
2210 kernel_pmap->pm_pid = isa3_base_pid;
2211 isa3_base_pid++;
2212 }
2213
2214 void
mmu_radix_advise(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,int advice)2215 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
2216 int advice)
2217 {
2218 struct rwlock *lock;
2219 pml1_entry_t *l1e;
2220 pml2_entry_t *l2e;
2221 pml3_entry_t oldl3e, *l3e;
2222 pt_entry_t *pte;
2223 vm_offset_t va, va_next;
2224 vm_page_t m;
2225 boolean_t anychanged;
2226
2227 if (advice != MADV_DONTNEED && advice != MADV_FREE)
2228 return;
2229 anychanged = FALSE;
2230 PMAP_LOCK(pmap);
2231 for (; sva < eva; sva = va_next) {
2232 l1e = pmap_pml1e(pmap, sva);
2233 if ((be64toh(*l1e) & PG_V) == 0) {
2234 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2235 if (va_next < sva)
2236 va_next = eva;
2237 continue;
2238 }
2239 l2e = pmap_l1e_to_l2e(l1e, sva);
2240 if ((be64toh(*l2e) & PG_V) == 0) {
2241 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2242 if (va_next < sva)
2243 va_next = eva;
2244 continue;
2245 }
2246 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2247 if (va_next < sva)
2248 va_next = eva;
2249 l3e = pmap_l2e_to_l3e(l2e, sva);
2250 oldl3e = be64toh(*l3e);
2251 if ((oldl3e & PG_V) == 0)
2252 continue;
2253 else if ((oldl3e & RPTE_LEAF) != 0) {
2254 if ((oldl3e & PG_MANAGED) == 0)
2255 continue;
2256 lock = NULL;
2257 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) {
2258 if (lock != NULL)
2259 rw_wunlock(lock);
2260
2261 /*
2262 * The large page mapping was destroyed.
2263 */
2264 continue;
2265 }
2266
2267 /*
2268 * Unless the page mappings are wired, remove the
2269 * mapping to a single page so that a subsequent
2270 * access may repromote. Since the underlying page
2271 * table page is fully populated, this removal never
2272 * frees a page table page.
2273 */
2274 if ((oldl3e & PG_W) == 0) {
2275 pte = pmap_l3e_to_pte(l3e, sva);
2276 KASSERT((be64toh(*pte) & PG_V) != 0,
2277 ("pmap_advise: invalid PTE"));
2278 pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), NULL,
2279 &lock);
2280 anychanged = TRUE;
2281 }
2282 if (lock != NULL)
2283 rw_wunlock(lock);
2284 }
2285 if (va_next > eva)
2286 va_next = eva;
2287 va = va_next;
2288 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next;
2289 pte++, sva += PAGE_SIZE) {
2290 MPASS(pte == pmap_pte(pmap, sva));
2291
2292 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
2293 goto maybe_invlrng;
2294 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2295 if (advice == MADV_DONTNEED) {
2296 /*
2297 * Future calls to pmap_is_modified()
2298 * can be avoided by making the page
2299 * dirty now.
2300 */
2301 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME);
2302 vm_page_dirty(m);
2303 }
2304 atomic_clear_long(pte, htobe64(PG_M | PG_A));
2305 } else if ((be64toh(*pte) & PG_A) != 0)
2306 atomic_clear_long(pte, htobe64(PG_A));
2307 else
2308 goto maybe_invlrng;
2309 anychanged = TRUE;
2310 continue;
2311 maybe_invlrng:
2312 if (va != va_next) {
2313 anychanged = true;
2314 va = va_next;
2315 }
2316 }
2317 if (va != va_next)
2318 anychanged = true;
2319 }
2320 if (anychanged)
2321 pmap_invalidate_all(pmap);
2322 PMAP_UNLOCK(pmap);
2323 }
2324
2325 /*
2326 * Routines used in machine-dependent code
2327 */
2328 static void
mmu_radix_bootstrap(vm_offset_t start,vm_offset_t end)2329 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end)
2330 {
2331 uint64_t lpcr;
2332
2333 if (bootverbose)
2334 printf("%s\n", __func__);
2335 hw_direct_map = 1;
2336 powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0;
2337 mmu_radix_early_bootstrap(start, end);
2338 if (bootverbose)
2339 printf("early bootstrap complete\n");
2340 if (powernv_enabled) {
2341 lpcr = mfspr(SPR_LPCR);
2342 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2343 mmu_radix_parttab_init();
2344 mmu_radix_init_amor();
2345 if (bootverbose)
2346 printf("powernv init complete\n");
2347 }
2348 mmu_radix_init_iamr();
2349 mmu_radix_proctab_init();
2350 mmu_radix_pid_set(kernel_pmap);
2351 if (powernv_enabled)
2352 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2353 else
2354 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2355
2356 mmu_radix_late_bootstrap(start, end);
2357 numa_mem_regions(&numa_pregions, &numa_pregions_sz);
2358 if (bootverbose)
2359 printf("%s done\n", __func__);
2360 pmap_bootstrapped = 1;
2361 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE);
2362 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS);
2363 }
2364
2365 static void
mmu_radix_cpu_bootstrap(int ap)2366 mmu_radix_cpu_bootstrap(int ap)
2367 {
2368 uint64_t lpcr;
2369 uint64_t ptcr;
2370
2371 if (powernv_enabled) {
2372 lpcr = mfspr(SPR_LPCR);
2373 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2374
2375 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2376 mtspr(SPR_PTCR, ptcr);
2377 mmu_radix_init_amor();
2378 }
2379 mmu_radix_init_iamr();
2380 mmu_radix_pid_set(kernel_pmap);
2381 if (powernv_enabled)
2382 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2383 else
2384 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2385 }
2386
2387 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0,
2388 "2MB page mapping counters");
2389
2390 static u_long pmap_l3e_demotions;
2391 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD,
2392 &pmap_l3e_demotions, 0, "2MB page demotions");
2393
2394 static u_long pmap_l3e_mappings;
2395 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD,
2396 &pmap_l3e_mappings, 0, "2MB page mappings");
2397
2398 static u_long pmap_l3e_p_failures;
2399 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD,
2400 &pmap_l3e_p_failures, 0, "2MB page promotion failures");
2401
2402 static u_long pmap_l3e_promotions;
2403 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD,
2404 &pmap_l3e_promotions, 0, "2MB page promotions");
2405
2406 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0,
2407 "1GB page mapping counters");
2408
2409 static u_long pmap_l2e_demotions;
2410 SYSCTL_ULONG(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD,
2411 &pmap_l2e_demotions, 0, "1GB page demotions");
2412
2413 void
mmu_radix_clear_modify(vm_page_t m)2414 mmu_radix_clear_modify(vm_page_t m)
2415 {
2416 struct md_page *pvh;
2417 pmap_t pmap;
2418 pv_entry_t next_pv, pv;
2419 pml3_entry_t oldl3e, *l3e;
2420 pt_entry_t oldpte, *pte;
2421 struct rwlock *lock;
2422 vm_offset_t va;
2423 int md_gen, pvh_gen;
2424
2425 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2426 ("pmap_clear_modify: page %p is not managed", m));
2427 vm_page_assert_busied(m);
2428 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
2429
2430 /*
2431 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
2432 * If the object containing the page is locked and the page is not
2433 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2434 */
2435 if ((m->a.flags & PGA_WRITEABLE) == 0)
2436 return;
2437 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2438 pa_to_pvh(VM_PAGE_TO_PHYS(m));
2439 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
2440 rw_wlock(lock);
2441 restart:
2442 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
2443 pmap = PV_PMAP(pv);
2444 if (!PMAP_TRYLOCK(pmap)) {
2445 pvh_gen = pvh->pv_gen;
2446 rw_wunlock(lock);
2447 PMAP_LOCK(pmap);
2448 rw_wlock(lock);
2449 if (pvh_gen != pvh->pv_gen) {
2450 PMAP_UNLOCK(pmap);
2451 goto restart;
2452 }
2453 }
2454 va = pv->pv_va;
2455 l3e = pmap_pml3e(pmap, va);
2456 oldl3e = be64toh(*l3e);
2457 if ((oldl3e & PG_RW) != 0) {
2458 if (pmap_demote_l3e_locked(pmap, l3e, va, &lock)) {
2459 if ((oldl3e & PG_W) == 0) {
2460 /*
2461 * Write protect the mapping to a
2462 * single page so that a subsequent
2463 * write access may repromote.
2464 */
2465 va += VM_PAGE_TO_PHYS(m) - (oldl3e &
2466 PG_PS_FRAME);
2467 pte = pmap_l3e_to_pte(l3e, va);
2468 oldpte = be64toh(*pte);
2469 if ((oldpte & PG_V) != 0) {
2470 while (!atomic_cmpset_long(pte,
2471 htobe64(oldpte),
2472 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW))))
2473 oldpte = be64toh(*pte);
2474 vm_page_dirty(m);
2475 pmap_invalidate_page(pmap, va);
2476 }
2477 }
2478 }
2479 }
2480 PMAP_UNLOCK(pmap);
2481 }
2482 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
2483 pmap = PV_PMAP(pv);
2484 if (!PMAP_TRYLOCK(pmap)) {
2485 md_gen = m->md.pv_gen;
2486 pvh_gen = pvh->pv_gen;
2487 rw_wunlock(lock);
2488 PMAP_LOCK(pmap);
2489 rw_wlock(lock);
2490 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2491 PMAP_UNLOCK(pmap);
2492 goto restart;
2493 }
2494 }
2495 l3e = pmap_pml3e(pmap, pv->pv_va);
2496 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found"
2497 " a 2mpage in page %p's pv list", m));
2498 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
2499 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2500 atomic_clear_long(pte, htobe64(PG_M));
2501 pmap_invalidate_page(pmap, pv->pv_va);
2502 }
2503 PMAP_UNLOCK(pmap);
2504 }
2505 rw_wunlock(lock);
2506 }
2507
2508 void
mmu_radix_copy(pmap_t dst_pmap,pmap_t src_pmap,vm_offset_t dst_addr,vm_size_t len,vm_offset_t src_addr)2509 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
2510 vm_size_t len, vm_offset_t src_addr)
2511 {
2512 struct rwlock *lock;
2513 struct spglist free;
2514 vm_offset_t addr;
2515 vm_offset_t end_addr = src_addr + len;
2516 vm_offset_t va_next;
2517 vm_page_t dst_pdpg, dstmpte, srcmpte;
2518 bool invalidate_all;
2519
2520 CTR6(KTR_PMAP,
2521 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n",
2522 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr);
2523
2524 if (dst_addr != src_addr)
2525 return;
2526 lock = NULL;
2527 invalidate_all = false;
2528 if (dst_pmap < src_pmap) {
2529 PMAP_LOCK(dst_pmap);
2530 PMAP_LOCK(src_pmap);
2531 } else {
2532 PMAP_LOCK(src_pmap);
2533 PMAP_LOCK(dst_pmap);
2534 }
2535
2536 for (addr = src_addr; addr < end_addr; addr = va_next) {
2537 pml1_entry_t *l1e;
2538 pml2_entry_t *l2e;
2539 pml3_entry_t srcptepaddr, *l3e;
2540 pt_entry_t *src_pte, *dst_pte;
2541
2542 l1e = pmap_pml1e(src_pmap, addr);
2543 if ((be64toh(*l1e) & PG_V) == 0) {
2544 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2545 if (va_next < addr)
2546 va_next = end_addr;
2547 continue;
2548 }
2549
2550 l2e = pmap_l1e_to_l2e(l1e, addr);
2551 if ((be64toh(*l2e) & PG_V) == 0) {
2552 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2553 if (va_next < addr)
2554 va_next = end_addr;
2555 continue;
2556 }
2557
2558 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2559 if (va_next < addr)
2560 va_next = end_addr;
2561
2562 l3e = pmap_l2e_to_l3e(l2e, addr);
2563 srcptepaddr = be64toh(*l3e);
2564 if (srcptepaddr == 0)
2565 continue;
2566
2567 if (srcptepaddr & RPTE_LEAF) {
2568 if ((addr & L3_PAGE_MASK) != 0 ||
2569 addr + L3_PAGE_SIZE > end_addr)
2570 continue;
2571 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL);
2572 if (dst_pdpg == NULL)
2573 break;
2574 l3e = (pml3_entry_t *)
2575 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg));
2576 l3e = &l3e[pmap_pml3e_index(addr)];
2577 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
2578 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr,
2579 PMAP_ENTER_NORECLAIM, &lock))) {
2580 *l3e = htobe64(srcptepaddr & ~PG_W);
2581 pmap_resident_count_inc(dst_pmap,
2582 L3_PAGE_SIZE / PAGE_SIZE);
2583 atomic_add_long(&pmap_l3e_mappings, 1);
2584 } else
2585 dst_pdpg->ref_count--;
2586 continue;
2587 }
2588
2589 srcptepaddr &= PG_FRAME;
2590 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
2591 KASSERT(srcmpte->ref_count > 0,
2592 ("pmap_copy: source page table page is unused"));
2593
2594 if (va_next > end_addr)
2595 va_next = end_addr;
2596
2597 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
2598 src_pte = &src_pte[pmap_pte_index(addr)];
2599 dstmpte = NULL;
2600 while (addr < va_next) {
2601 pt_entry_t ptetemp;
2602 ptetemp = be64toh(*src_pte);
2603 /*
2604 * we only virtual copy managed pages
2605 */
2606 if ((ptetemp & PG_MANAGED) != 0) {
2607 if (dstmpte != NULL &&
2608 dstmpte->pindex == pmap_l3e_pindex(addr))
2609 dstmpte->ref_count++;
2610 else if ((dstmpte = pmap_allocpte(dst_pmap,
2611 addr, NULL)) == NULL)
2612 goto out;
2613 dst_pte = (pt_entry_t *)
2614 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
2615 dst_pte = &dst_pte[pmap_pte_index(addr)];
2616 if (be64toh(*dst_pte) == 0 &&
2617 pmap_try_insert_pv_entry(dst_pmap, addr,
2618 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
2619 &lock)) {
2620 /*
2621 * Clear the wired, modified, and
2622 * accessed (referenced) bits
2623 * during the copy.
2624 */
2625 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M |
2626 PG_A));
2627 pmap_resident_count_inc(dst_pmap, 1);
2628 } else {
2629 SLIST_INIT(&free);
2630 if (pmap_unwire_ptp(dst_pmap, addr,
2631 dstmpte, &free)) {
2632 /*
2633 * Although "addr" is not
2634 * mapped, paging-structure
2635 * caches could nonetheless
2636 * have entries that refer to
2637 * the freed page table pages.
2638 * Invalidate those entries.
2639 */
2640 invalidate_all = true;
2641 vm_page_free_pages_toq(&free,
2642 true);
2643 }
2644 goto out;
2645 }
2646 if (dstmpte->ref_count >= srcmpte->ref_count)
2647 break;
2648 }
2649 addr += PAGE_SIZE;
2650 if (__predict_false((addr & L3_PAGE_MASK) == 0))
2651 src_pte = pmap_pte(src_pmap, addr);
2652 else
2653 src_pte++;
2654 }
2655 }
2656 out:
2657 if (invalidate_all)
2658 pmap_invalidate_all(dst_pmap);
2659 if (lock != NULL)
2660 rw_wunlock(lock);
2661 PMAP_UNLOCK(src_pmap);
2662 PMAP_UNLOCK(dst_pmap);
2663 }
2664
2665 static void
mmu_radix_copy_page(vm_page_t msrc,vm_page_t mdst)2666 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst)
2667 {
2668 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
2669 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
2670
2671 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst);
2672 /*
2673 * XXX slow
2674 */
2675 bcopy((void *)src, (void *)dst, PAGE_SIZE);
2676 }
2677
2678 static void
mmu_radix_copy_pages(vm_page_t ma[],vm_offset_t a_offset,vm_page_t mb[],vm_offset_t b_offset,int xfersize)2679 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
2680 vm_offset_t b_offset, int xfersize)
2681 {
2682 void *a_cp, *b_cp;
2683 vm_offset_t a_pg_offset, b_pg_offset;
2684 int cnt;
2685
2686 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma,
2687 a_offset, mb, b_offset, xfersize);
2688
2689 while (xfersize > 0) {
2690 a_pg_offset = a_offset & PAGE_MASK;
2691 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2692 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
2693 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) +
2694 a_pg_offset;
2695 b_pg_offset = b_offset & PAGE_MASK;
2696 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2697 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
2698 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) +
2699 b_pg_offset;
2700 bcopy(a_cp, b_cp, cnt);
2701 a_offset += cnt;
2702 b_offset += cnt;
2703 xfersize -= cnt;
2704 }
2705 }
2706
2707 #if VM_NRESERVLEVEL > 0
2708 /*
2709 * Tries to promote the 512, contiguous 4KB page mappings that are within a
2710 * single page table page (PTP) to a single 2MB page mapping. For promotion
2711 * to occur, two conditions must be met: (1) the 4KB page mappings must map
2712 * aligned, contiguous physical memory and (2) the 4KB page mappings must have
2713 * identical characteristics.
2714 */
2715 static int
pmap_promote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va,struct rwlock ** lockp)2716 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va,
2717 struct rwlock **lockp)
2718 {
2719 pml3_entry_t newpde;
2720 pt_entry_t *firstpte, oldpte, pa, *pte;
2721 vm_page_t mpte;
2722
2723 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2724
2725 /*
2726 * Examine the first PTE in the specified PTP. Abort if this PTE is
2727 * either invalid, unused, or does not map the first 4KB physical page
2728 * within a 2MB page.
2729 */
2730 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME);
2731 setpde:
2732 newpde = *firstpte;
2733 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
2734 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2735 " in pmap %p", va, pmap);
2736 goto fail;
2737 }
2738 if ((newpde & (PG_M | PG_RW)) == PG_RW) {
2739 /*
2740 * When PG_M is already clear, PG_RW can be cleared without
2741 * a TLB invalidation.
2742 */
2743 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W)))
2744 goto setpde;
2745 newpde &= ~RPTE_EAA_W;
2746 }
2747
2748 /*
2749 * Examine each of the other PTEs in the specified PTP. Abort if this
2750 * PTE maps an unexpected 4KB physical page or does not have identical
2751 * characteristics to the first PTE.
2752 */
2753 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE;
2754 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
2755 setpte:
2756 oldpte = be64toh(*pte);
2757 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
2758 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2759 " in pmap %p", va, pmap);
2760 goto fail;
2761 }
2762 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
2763 /*
2764 * When PG_M is already clear, PG_RW can be cleared
2765 * without a TLB invalidation.
2766 */
2767 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W)))
2768 goto setpte;
2769 oldpte &= ~RPTE_EAA_W;
2770 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx"
2771 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) |
2772 (va & ~L3_PAGE_MASK), pmap);
2773 }
2774 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
2775 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2776 " in pmap %p", va, pmap);
2777 goto fail;
2778 }
2779 pa -= PAGE_SIZE;
2780 }
2781
2782 /*
2783 * Save the page table page in its current state until the PDE
2784 * mapping the superpage is demoted by pmap_demote_pde() or
2785 * destroyed by pmap_remove_pde().
2786 */
2787 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME);
2788 KASSERT(mpte >= vm_page_array &&
2789 mpte < &vm_page_array[vm_page_array_size],
2790 ("pmap_promote_l3e: page table page is out of range"));
2791 KASSERT(mpte->pindex == pmap_l3e_pindex(va),
2792 ("pmap_promote_l3e: page table page's pindex is wrong"));
2793 if (pmap_insert_pt_page(pmap, mpte)) {
2794 CTR2(KTR_PMAP,
2795 "pmap_promote_l3e: failure for va %#lx in pmap %p", va,
2796 pmap);
2797 goto fail;
2798 }
2799
2800 /*
2801 * Promote the pv entries.
2802 */
2803 if ((newpde & PG_MANAGED) != 0)
2804 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp);
2805
2806 pte_store(pde, PG_PROMOTED | newpde);
2807 ptesync();
2808 atomic_add_long(&pmap_l3e_promotions, 1);
2809 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx"
2810 " in pmap %p", va, pmap);
2811 return (0);
2812 fail:
2813 atomic_add_long(&pmap_l3e_p_failures, 1);
2814 return (KERN_FAILURE);
2815 }
2816 #endif /* VM_NRESERVLEVEL > 0 */
2817
2818 int
mmu_radix_enter(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,u_int flags,int8_t psind)2819 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
2820 vm_prot_t prot, u_int flags, int8_t psind)
2821 {
2822 struct rwlock *lock;
2823 pml3_entry_t *l3e;
2824 pt_entry_t *pte;
2825 pt_entry_t newpte, origpte;
2826 pv_entry_t pv;
2827 vm_paddr_t opa, pa;
2828 vm_page_t mpte, om;
2829 int rv, retrycount;
2830 boolean_t nosleep, invalidate_all, invalidate_page;
2831
2832 va = trunc_page(va);
2833 retrycount = 0;
2834 invalidate_page = invalidate_all = false;
2835 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va,
2836 m, prot, flags, psind);
2837 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
2838 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va),
2839 ("pmap_enter: managed mapping within the clean submap"));
2840 if ((m->oflags & VPO_UNMANAGED) == 0)
2841 VM_PAGE_OBJECT_BUSY_ASSERT(m);
2842
2843 KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
2844 ("pmap_enter: flags %u has reserved bits set", flags));
2845 pa = VM_PAGE_TO_PHYS(m);
2846 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF);
2847 if ((flags & VM_PROT_WRITE) != 0)
2848 newpte |= PG_M;
2849 if ((flags & VM_PROT_READ) != 0)
2850 newpte |= PG_A;
2851 if (prot & VM_PROT_READ)
2852 newpte |= RPTE_EAA_R;
2853 if ((prot & VM_PROT_WRITE) != 0)
2854 newpte |= RPTE_EAA_W;
2855 KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
2856 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
2857
2858 if (prot & VM_PROT_EXECUTE)
2859 newpte |= PG_X;
2860 if ((flags & PMAP_ENTER_WIRED) != 0)
2861 newpte |= PG_W;
2862 if (va >= DMAP_MIN_ADDRESS)
2863 newpte |= RPTE_EAA_P;
2864 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs);
2865 /*
2866 * Set modified bit gratuitously for writeable mappings if
2867 * the page is unmanaged. We do not want to take a fault
2868 * to do the dirty bit accounting for these mappings.
2869 */
2870 if ((m->oflags & VPO_UNMANAGED) != 0) {
2871 if ((newpte & PG_RW) != 0)
2872 newpte |= PG_M;
2873 } else
2874 newpte |= PG_MANAGED;
2875
2876 lock = NULL;
2877 PMAP_LOCK(pmap);
2878 if (psind == 1) {
2879 /* Assert the required virtual and physical alignment. */
2880 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned"));
2881 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
2882 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock);
2883 goto out;
2884 }
2885 mpte = NULL;
2886
2887 /*
2888 * In the case that a page table page is not
2889 * resident, we are creating it here.
2890 */
2891 retry:
2892 l3e = pmap_pml3e(pmap, va);
2893 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 ||
2894 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) {
2895 pte = pmap_l3e_to_pte(l3e, va);
2896 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
2897 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
2898 mpte->ref_count++;
2899 }
2900 } else if (va < VM_MAXUSER_ADDRESS) {
2901 /*
2902 * Here if the pte page isn't mapped, or if it has been
2903 * deallocated.
2904 */
2905 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
2906 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),
2907 nosleep ? NULL : &lock);
2908 if (mpte == NULL && nosleep) {
2909 rv = KERN_RESOURCE_SHORTAGE;
2910 goto out;
2911 }
2912 if (__predict_false(retrycount++ == 6))
2913 panic("too many retries");
2914 invalidate_all = true;
2915 goto retry;
2916 } else
2917 panic("pmap_enter: invalid page directory va=%#lx", va);
2918
2919 origpte = be64toh(*pte);
2920 pv = NULL;
2921
2922 /*
2923 * Is the specified virtual address already mapped?
2924 */
2925 if ((origpte & PG_V) != 0) {
2926 #ifdef INVARIANTS
2927 if (VERBOSE_PMAP || pmap_logging) {
2928 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --"
2929 " asid=%lu curpid=%d name=%s origpte0x%lx\n",
2930 pmap, va, m, prot, flags, psind, pmap->pm_pid,
2931 curproc->p_pid, curproc->p_comm, origpte);
2932 pmap_pte_walk(pmap->pm_pml1, va);
2933 }
2934 #endif
2935 /*
2936 * Wiring change, just update stats. We don't worry about
2937 * wiring PT pages as they remain resident as long as there
2938 * are valid mappings in them. Hence, if a user page is wired,
2939 * the PT page will be also.
2940 */
2941 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
2942 pmap->pm_stats.wired_count++;
2943 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
2944 pmap->pm_stats.wired_count--;
2945
2946 /*
2947 * Remove the extra PT page reference.
2948 */
2949 if (mpte != NULL) {
2950 mpte->ref_count--;
2951 KASSERT(mpte->ref_count > 0,
2952 ("pmap_enter: missing reference to page table page,"
2953 " va: 0x%lx", va));
2954 }
2955
2956 /*
2957 * Has the physical page changed?
2958 */
2959 opa = origpte & PG_FRAME;
2960 if (opa == pa) {
2961 /*
2962 * No, might be a protection or wiring change.
2963 */
2964 if ((origpte & PG_MANAGED) != 0 &&
2965 (newpte & PG_RW) != 0)
2966 vm_page_aflag_set(m, PGA_WRITEABLE);
2967 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) {
2968 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) {
2969 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
2970 goto retry;
2971 if ((newpte & PG_M) != (origpte & PG_M))
2972 vm_page_dirty(m);
2973 if ((newpte & PG_A) != (origpte & PG_A))
2974 vm_page_aflag_set(m, PGA_REFERENCED);
2975 ptesync();
2976 } else
2977 invalidate_all = true;
2978 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
2979 goto unchanged;
2980 }
2981 goto validate;
2982 }
2983
2984 /*
2985 * The physical page has changed. Temporarily invalidate
2986 * the mapping. This ensures that all threads sharing the
2987 * pmap keep a consistent view of the mapping, which is
2988 * necessary for the correct handling of COW faults. It
2989 * also permits reuse of the old mapping's PV entry,
2990 * avoiding an allocation.
2991 *
2992 * For consistency, handle unmanaged mappings the same way.
2993 */
2994 origpte = be64toh(pte_load_clear(pte));
2995 KASSERT((origpte & PG_FRAME) == opa,
2996 ("pmap_enter: unexpected pa update for %#lx", va));
2997 if ((origpte & PG_MANAGED) != 0) {
2998 om = PHYS_TO_VM_PAGE(opa);
2999
3000 /*
3001 * The pmap lock is sufficient to synchronize with
3002 * concurrent calls to pmap_page_test_mappings() and
3003 * pmap_ts_referenced().
3004 */
3005 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3006 vm_page_dirty(om);
3007 if ((origpte & PG_A) != 0)
3008 vm_page_aflag_set(om, PGA_REFERENCED);
3009 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
3010 pv = pmap_pvh_remove(&om->md, pmap, va);
3011 if ((newpte & PG_MANAGED) == 0)
3012 free_pv_entry(pmap, pv);
3013 #ifdef INVARIANTS
3014 else if (origpte & PG_MANAGED) {
3015 if (pv == NULL) {
3016 pmap_page_print_mappings(om);
3017 MPASS(pv != NULL);
3018 }
3019 }
3020 #endif
3021 if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3022 TAILQ_EMPTY(&om->md.pv_list) &&
3023 ((om->flags & PG_FICTITIOUS) != 0 ||
3024 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3025 vm_page_aflag_clear(om, PGA_WRITEABLE);
3026 }
3027 if ((origpte & PG_A) != 0)
3028 invalidate_page = true;
3029 origpte = 0;
3030 } else {
3031 if (pmap != kernel_pmap) {
3032 #ifdef INVARIANTS
3033 if (VERBOSE_PMAP || pmap_logging)
3034 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n",
3035 pmap, va, m, prot, flags, psind,
3036 pmap->pm_pid, curproc->p_pid,
3037 curproc->p_comm);
3038 #endif
3039 }
3040
3041 /*
3042 * Increment the counters.
3043 */
3044 if ((newpte & PG_W) != 0)
3045 pmap->pm_stats.wired_count++;
3046 pmap_resident_count_inc(pmap, 1);
3047 }
3048
3049 /*
3050 * Enter on the PV list if part of our managed memory.
3051 */
3052 if ((newpte & PG_MANAGED) != 0) {
3053 if (pv == NULL) {
3054 pv = get_pv_entry(pmap, &lock);
3055 pv->pv_va = va;
3056 }
3057 #ifdef VERBOSE_PV
3058 else
3059 printf("reassigning pv: %p to pmap: %p\n",
3060 pv, pmap);
3061 #endif
3062 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
3063 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3064 m->md.pv_gen++;
3065 if ((newpte & PG_RW) != 0)
3066 vm_page_aflag_set(m, PGA_WRITEABLE);
3067 }
3068
3069 /*
3070 * Update the PTE.
3071 */
3072 if ((origpte & PG_V) != 0) {
3073 validate:
3074 origpte = be64toh(pte_load_store(pte, htobe64(newpte)));
3075 KASSERT((origpte & PG_FRAME) == pa,
3076 ("pmap_enter: unexpected pa update for %#lx", va));
3077 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
3078 (PG_M | PG_RW)) {
3079 if ((origpte & PG_MANAGED) != 0)
3080 vm_page_dirty(m);
3081 invalidate_page = true;
3082
3083 /*
3084 * Although the PTE may still have PG_RW set, TLB
3085 * invalidation may nonetheless be required because
3086 * the PTE no longer has PG_M set.
3087 */
3088 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) {
3089 /*
3090 * Removing capabilities requires invalidation on POWER
3091 */
3092 invalidate_page = true;
3093 goto unchanged;
3094 }
3095 if ((origpte & PG_A) != 0)
3096 invalidate_page = true;
3097 } else {
3098 pte_store(pte, newpte);
3099 ptesync();
3100 }
3101 unchanged:
3102
3103 #if VM_NRESERVLEVEL > 0
3104 /*
3105 * If both the page table page and the reservation are fully
3106 * populated, then attempt promotion.
3107 */
3108 if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3109 mmu_radix_ps_enabled(pmap) &&
3110 (m->flags & PG_FICTITIOUS) == 0 &&
3111 vm_reserv_level_iffullpop(m) == 0 &&
3112 pmap_promote_l3e(pmap, l3e, va, &lock) == 0)
3113 invalidate_all = true;
3114 #endif
3115 if (invalidate_all)
3116 pmap_invalidate_all(pmap);
3117 else if (invalidate_page)
3118 pmap_invalidate_page(pmap, va);
3119
3120 rv = KERN_SUCCESS;
3121 out:
3122 if (lock != NULL)
3123 rw_wunlock(lock);
3124 PMAP_UNLOCK(pmap);
3125
3126 return (rv);
3127 }
3128
3129 /*
3130 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true
3131 * if successful. Returns false if (1) a page table page cannot be allocated
3132 * without sleeping, (2) a mapping already exists at the specified virtual
3133 * address, or (3) a PV entry cannot be allocated without reclaiming another
3134 * PV entry.
3135 */
3136 static bool
pmap_enter_2mpage(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,struct rwlock ** lockp)3137 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3138 struct rwlock **lockp)
3139 {
3140 pml3_entry_t newpde;
3141
3142 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3143 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) |
3144 RPTE_LEAF | PG_V;
3145 if ((m->oflags & VPO_UNMANAGED) == 0)
3146 newpde |= PG_MANAGED;
3147 if (prot & VM_PROT_EXECUTE)
3148 newpde |= PG_X;
3149 if (prot & VM_PROT_READ)
3150 newpde |= RPTE_EAA_R;
3151 if (va >= DMAP_MIN_ADDRESS)
3152 newpde |= RPTE_EAA_P;
3153 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
3154 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
3155 KERN_SUCCESS);
3156 }
3157
3158 /*
3159 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
3160 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
3161 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
3162 * a mapping already exists at the specified virtual address. Returns
3163 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
3164 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if
3165 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
3166 *
3167 * The parameter "m" is only used when creating a managed, writeable mapping.
3168 */
3169 static int
pmap_enter_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t newpde,u_int flags,vm_page_t m,struct rwlock ** lockp)3170 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags,
3171 vm_page_t m, struct rwlock **lockp)
3172 {
3173 struct spglist free;
3174 pml3_entry_t oldl3e, *l3e;
3175 vm_page_t mt, pdpg;
3176
3177 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
3178 ("pmap_enter_pde: newpde is missing PG_M"));
3179 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3180
3181 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
3182 NULL : lockp)) == NULL) {
3183 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3184 " in pmap %p", va, pmap);
3185 return (KERN_RESOURCE_SHORTAGE);
3186 }
3187 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
3188 l3e = &l3e[pmap_pml3e_index(va)];
3189 oldl3e = be64toh(*l3e);
3190 if ((oldl3e & PG_V) != 0) {
3191 KASSERT(pdpg->ref_count > 1,
3192 ("pmap_enter_pde: pdpg's wire count is too low"));
3193 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
3194 pdpg->ref_count--;
3195 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3196 " in pmap %p", va, pmap);
3197 return (KERN_FAILURE);
3198 }
3199 /* Break the existing mapping(s). */
3200 SLIST_INIT(&free);
3201 if ((oldl3e & RPTE_LEAF) != 0) {
3202 /*
3203 * The reference to the PD page that was acquired by
3204 * pmap_allocl3e() ensures that it won't be freed.
3205 * However, if the PDE resulted from a promotion, then
3206 * a reserved PT page could be freed.
3207 */
3208 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp);
3209 pmap_invalidate_l3e_page(pmap, va, oldl3e);
3210 } else {
3211 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e,
3212 &free, lockp))
3213 pmap_invalidate_all(pmap);
3214 }
3215 vm_page_free_pages_toq(&free, true);
3216 if (va >= VM_MAXUSER_ADDRESS) {
3217 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
3218 if (pmap_insert_pt_page(pmap, mt)) {
3219 /*
3220 * XXX Currently, this can't happen because
3221 * we do not perform pmap_enter(psind == 1)
3222 * on the kernel pmap.
3223 */
3224 panic("pmap_enter_pde: trie insert failed");
3225 }
3226 } else
3227 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",
3228 l3e));
3229 }
3230 if ((newpde & PG_MANAGED) != 0) {
3231 /*
3232 * Abort this mapping if its PV entry could not be created.
3233 */
3234 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) {
3235 SLIST_INIT(&free);
3236 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
3237 /*
3238 * Although "va" is not mapped, paging-
3239 * structure caches could nonetheless have
3240 * entries that refer to the freed page table
3241 * pages. Invalidate those entries.
3242 */
3243 pmap_invalidate_page(pmap, va);
3244 vm_page_free_pages_toq(&free, true);
3245 }
3246 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3247 " in pmap %p", va, pmap);
3248 return (KERN_RESOURCE_SHORTAGE);
3249 }
3250 if ((newpde & PG_RW) != 0) {
3251 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
3252 vm_page_aflag_set(mt, PGA_WRITEABLE);
3253 }
3254 }
3255
3256 /*
3257 * Increment counters.
3258 */
3259 if ((newpde & PG_W) != 0)
3260 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;
3261 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
3262
3263 /*
3264 * Map the superpage. (This is not a promoted mapping; there will not
3265 * be any lingering 4KB page mappings in the TLB.)
3266 */
3267 pte_store(l3e, newpde);
3268 ptesync();
3269
3270 atomic_add_long(&pmap_l3e_mappings, 1);
3271 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
3272 " in pmap %p", va, pmap);
3273 return (KERN_SUCCESS);
3274 }
3275
3276 void
mmu_radix_enter_object(pmap_t pmap,vm_offset_t start,vm_offset_t end,vm_page_t m_start,vm_prot_t prot)3277 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start,
3278 vm_offset_t end, vm_page_t m_start, vm_prot_t prot)
3279 {
3280
3281 struct rwlock *lock;
3282 vm_offset_t va;
3283 vm_page_t m, mpte;
3284 vm_pindex_t diff, psize;
3285 bool invalidate;
3286 VM_OBJECT_ASSERT_LOCKED(m_start->object);
3287
3288 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,
3289 end, m_start, prot);
3290
3291 invalidate = false;
3292 psize = atop(end - start);
3293 mpte = NULL;
3294 m = m_start;
3295 lock = NULL;
3296 PMAP_LOCK(pmap);
3297 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
3298 va = start + ptoa(diff);
3299 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end &&
3300 m->psind == 1 && mmu_radix_ps_enabled(pmap) &&
3301 pmap_enter_2mpage(pmap, va, m, prot, &lock))
3302 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1];
3303 else
3304 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,
3305 mpte, &lock, &invalidate);
3306 m = TAILQ_NEXT(m, listq);
3307 }
3308 ptesync();
3309 if (lock != NULL)
3310 rw_wunlock(lock);
3311 if (invalidate)
3312 pmap_invalidate_all(pmap);
3313 PMAP_UNLOCK(pmap);
3314 }
3315
3316 static vm_page_t
mmu_radix_enter_quick_locked(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,vm_page_t mpte,struct rwlock ** lockp,bool * invalidate)3317 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
3318 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate)
3319 {
3320 struct spglist free;
3321 pt_entry_t *pte;
3322 vm_paddr_t pa;
3323
3324 KASSERT(!VA_IS_CLEANMAP(va) ||
3325 (m->oflags & VPO_UNMANAGED) != 0,
3326 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap"));
3327 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3328
3329 /*
3330 * In the case that a page table page is not
3331 * resident, we are creating it here.
3332 */
3333 if (va < VM_MAXUSER_ADDRESS) {
3334 vm_pindex_t ptepindex;
3335 pml3_entry_t *ptepa;
3336
3337 /*
3338 * Calculate pagetable page index
3339 */
3340 ptepindex = pmap_l3e_pindex(va);
3341 if (mpte && (mpte->pindex == ptepindex)) {
3342 mpte->ref_count++;
3343 } else {
3344 /*
3345 * Get the page directory entry
3346 */
3347 ptepa = pmap_pml3e(pmap, va);
3348
3349 /*
3350 * If the page table page is mapped, we just increment
3351 * the hold count, and activate it. Otherwise, we
3352 * attempt to allocate a page table page. If this
3353 * attempt fails, we don't retry. Instead, we give up.
3354 */
3355 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) {
3356 if (be64toh(*ptepa) & RPTE_LEAF)
3357 return (NULL);
3358 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME);
3359 mpte->ref_count++;
3360 } else {
3361 /*
3362 * Pass NULL instead of the PV list lock
3363 * pointer, because we don't intend to sleep.
3364 */
3365 mpte = _pmap_allocpte(pmap, ptepindex, NULL);
3366 if (mpte == NULL)
3367 return (mpte);
3368 }
3369 }
3370 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
3371 pte = &pte[pmap_pte_index(va)];
3372 } else {
3373 mpte = NULL;
3374 pte = pmap_pte(pmap, va);
3375 }
3376 if (be64toh(*pte)) {
3377 if (mpte != NULL) {
3378 mpte->ref_count--;
3379 mpte = NULL;
3380 }
3381 return (mpte);
3382 }
3383
3384 /*
3385 * Enter on the PV list if part of our managed memory.
3386 */
3387 if ((m->oflags & VPO_UNMANAGED) == 0 &&
3388 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
3389 if (mpte != NULL) {
3390 SLIST_INIT(&free);
3391 if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
3392 /*
3393 * Although "va" is not mapped, paging-
3394 * structure caches could nonetheless have
3395 * entries that refer to the freed page table
3396 * pages. Invalidate those entries.
3397 */
3398 *invalidate = true;
3399 vm_page_free_pages_toq(&free, true);
3400 }
3401 mpte = NULL;
3402 }
3403 return (mpte);
3404 }
3405
3406 /*
3407 * Increment counters
3408 */
3409 pmap_resident_count_inc(pmap, 1);
3410
3411 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs);
3412 if (prot & VM_PROT_EXECUTE)
3413 pa |= PG_X;
3414 else
3415 pa |= RPTE_EAA_R;
3416 if ((m->oflags & VPO_UNMANAGED) == 0)
3417 pa |= PG_MANAGED;
3418
3419 pte_store(pte, pa);
3420 return (mpte);
3421 }
3422
3423 void
mmu_radix_enter_quick(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot)3424 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
3425 vm_prot_t prot)
3426 {
3427 struct rwlock *lock;
3428 bool invalidate;
3429
3430 lock = NULL;
3431 invalidate = false;
3432 PMAP_LOCK(pmap);
3433 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock,
3434 &invalidate);
3435 ptesync();
3436 if (lock != NULL)
3437 rw_wunlock(lock);
3438 if (invalidate)
3439 pmap_invalidate_all(pmap);
3440 PMAP_UNLOCK(pmap);
3441 }
3442
3443 vm_paddr_t
mmu_radix_extract(pmap_t pmap,vm_offset_t va)3444 mmu_radix_extract(pmap_t pmap, vm_offset_t va)
3445 {
3446 pml3_entry_t *l3e;
3447 pt_entry_t *pte;
3448 vm_paddr_t pa;
3449
3450 l3e = pmap_pml3e(pmap, va);
3451 if (__predict_false(l3e == NULL))
3452 return (0);
3453 if (be64toh(*l3e) & RPTE_LEAF) {
3454 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
3455 pa |= (va & L3_PAGE_MASK);
3456 } else {
3457 /*
3458 * Beware of a concurrent promotion that changes the
3459 * PDE at this point! For example, vtopte() must not
3460 * be used to access the PTE because it would use the
3461 * new PDE. It is, however, safe to use the old PDE
3462 * because the page table page is preserved by the
3463 * promotion.
3464 */
3465 pte = pmap_l3e_to_pte(l3e, va);
3466 if (__predict_false(pte == NULL))
3467 return (0);
3468 pa = be64toh(*pte);
3469 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
3470 pa |= (va & PAGE_MASK);
3471 }
3472 return (pa);
3473 }
3474
3475 vm_page_t
mmu_radix_extract_and_hold(pmap_t pmap,vm_offset_t va,vm_prot_t prot)3476 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
3477 {
3478 pml3_entry_t l3e, *l3ep;
3479 pt_entry_t pte;
3480 vm_paddr_t pa;
3481 vm_page_t m;
3482
3483 pa = 0;
3484 m = NULL;
3485 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);
3486 PMAP_LOCK(pmap);
3487 l3ep = pmap_pml3e(pmap, va);
3488 if (l3ep != NULL && (l3e = be64toh(*l3ep))) {
3489 if (l3e & RPTE_LEAF) {
3490 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0)
3491 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) |
3492 (va & L3_PAGE_MASK));
3493 } else {
3494 /* Native endian PTE, do not pass to pmap functions */
3495 pte = be64toh(*pmap_l3e_to_pte(l3ep, va));
3496 if ((pte & PG_V) &&
3497 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0))
3498 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
3499 }
3500 if (m != NULL && !vm_page_wire_mapped(m))
3501 m = NULL;
3502 }
3503 PMAP_UNLOCK(pmap);
3504 return (m);
3505 }
3506
3507 static void
mmu_radix_growkernel(vm_offset_t addr)3508 mmu_radix_growkernel(vm_offset_t addr)
3509 {
3510 vm_paddr_t paddr;
3511 vm_page_t nkpg;
3512 pml3_entry_t *l3e;
3513 pml2_entry_t *l2e;
3514
3515 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
3516 if (VM_MIN_KERNEL_ADDRESS < addr &&
3517 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE))
3518 return;
3519
3520 addr = roundup2(addr, L3_PAGE_SIZE);
3521 if (addr - 1 >= vm_map_max(kernel_map))
3522 addr = vm_map_max(kernel_map);
3523 while (kernel_vm_end < addr) {
3524 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end);
3525 if ((be64toh(*l2e) & PG_V) == 0) {
3526 /* We need a new PDP entry */
3527 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3528 VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3529 if (nkpg == NULL)
3530 panic("pmap_growkernel: no memory to grow kernel");
3531 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT;
3532 paddr = VM_PAGE_TO_PHYS(nkpg);
3533 pde_store(l2e, paddr);
3534 continue; /* try again */
3535 }
3536 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end);
3537 if ((be64toh(*l3e) & PG_V) != 0) {
3538 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3539 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3540 kernel_vm_end = vm_map_max(kernel_map);
3541 break;
3542 }
3543 continue;
3544 }
3545
3546 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED |
3547 VM_ALLOC_ZERO);
3548 if (nkpg == NULL)
3549 panic("pmap_growkernel: no memory to grow kernel");
3550 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end);
3551 paddr = VM_PAGE_TO_PHYS(nkpg);
3552 pde_store(l3e, paddr);
3553
3554 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3555 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3556 kernel_vm_end = vm_map_max(kernel_map);
3557 break;
3558 }
3559 }
3560 ptesync();
3561 }
3562
3563 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory");
3564 static uma_zone_t zone_radix_pgd;
3565
3566 static int
radix_pgd_import(void * arg __unused,void ** store,int count,int domain __unused,int flags)3567 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused,
3568 int flags)
3569 {
3570 int req;
3571
3572 req = VM_ALLOC_WIRED | malloc2vm_flags(flags);
3573 for (int i = 0; i < count; i++) {
3574 vm_page_t m = vm_page_alloc_noobj_contig(req,
3575 RADIX_PGD_SIZE / PAGE_SIZE,
3576 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE,
3577 VM_MEMATTR_DEFAULT);
3578 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
3579 }
3580 return (count);
3581 }
3582
3583 static void
radix_pgd_release(void * arg __unused,void ** store,int count)3584 radix_pgd_release(void *arg __unused, void **store, int count)
3585 {
3586 vm_page_t m;
3587 struct spglist free;
3588 int page_count;
3589
3590 SLIST_INIT(&free);
3591 page_count = RADIX_PGD_SIZE/PAGE_SIZE;
3592
3593 for (int i = 0; i < count; i++) {
3594 /*
3595 * XXX selectively remove dmap and KVA entries so we don't
3596 * need to bzero
3597 */
3598 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i]));
3599 for (int j = page_count-1; j >= 0; j--) {
3600 vm_page_unwire_noq(&m[j]);
3601 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss);
3602 }
3603 vm_page_free_pages_toq(&free, false);
3604 }
3605 }
3606
3607 static void
mmu_radix_init(void)3608 mmu_radix_init(void)
3609 {
3610 vm_page_t mpte;
3611 vm_size_t s;
3612 int error, i, pv_npg;
3613
3614 /* XXX is this really needed for POWER? */
3615 /* L1TF, reserve page @0 unconditionally */
3616 vm_page_blacklist_add(0, bootverbose);
3617
3618 zone_radix_pgd = uma_zcache_create("radix_pgd_cache",
3619 RADIX_PGD_SIZE, NULL, NULL,
3620 #ifdef INVARIANTS
3621 trash_init, trash_fini,
3622 #else
3623 NULL, NULL,
3624 #endif
3625 radix_pgd_import, radix_pgd_release,
3626 NULL, UMA_ZONE_NOBUCKET);
3627
3628 /*
3629 * Initialize the vm page array entries for the kernel pmap's
3630 * page table pages.
3631 */
3632 PMAP_LOCK(kernel_pmap);
3633 for (i = 0; i < nkpt; i++) {
3634 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
3635 KASSERT(mpte >= vm_page_array &&
3636 mpte < &vm_page_array[vm_page_array_size],
3637 ("pmap_init: page table page is out of range size: %lu",
3638 vm_page_array_size));
3639 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;
3640 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
3641 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);
3642 //pmap_insert_pt_page(kernel_pmap, mpte);
3643 mpte->ref_count = 1;
3644 }
3645 PMAP_UNLOCK(kernel_pmap);
3646 vm_wire_add(nkpt);
3647
3648 CTR1(KTR_PMAP, "%s()", __func__);
3649 TAILQ_INIT(&pv_dummy.pv_list);
3650
3651 /*
3652 * Are large page mappings enabled?
3653 */
3654 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
3655 if (superpages_enabled) {
3656 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
3657 ("pmap_init: can't assign to pagesizes[1]"));
3658 pagesizes[1] = L3_PAGE_SIZE;
3659 }
3660
3661 /*
3662 * Initialize the pv chunk list mutex.
3663 */
3664 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
3665
3666 /*
3667 * Initialize the pool of pv list locks.
3668 */
3669 for (i = 0; i < NPV_LIST_LOCKS; i++)
3670 rw_init(&pv_list_locks[i], "pmap pv list");
3671
3672 /*
3673 * Calculate the size of the pv head table for superpages.
3674 */
3675 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE);
3676
3677 /*
3678 * Allocate memory for the pv head table for superpages.
3679 */
3680 s = (vm_size_t)(pv_npg * sizeof(struct md_page));
3681 s = round_page(s);
3682 pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO);
3683 for (i = 0; i < pv_npg; i++)
3684 TAILQ_INIT(&pv_table[i].pv_list);
3685 TAILQ_INIT(&pv_dummy.pv_list);
3686
3687 pmap_initialized = 1;
3688 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN);
3689 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK,
3690 (vmem_addr_t *)&qframe);
3691
3692 if (error != 0)
3693 panic("qframe allocation failed");
3694 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits),
3695 1, 1, M_WAITOK);
3696 }
3697
3698 static boolean_t
pmap_page_test_mappings(vm_page_t m,boolean_t accessed,boolean_t modified)3699 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified)
3700 {
3701 struct rwlock *lock;
3702 pv_entry_t pv;
3703 struct md_page *pvh;
3704 pt_entry_t *pte, mask;
3705 pmap_t pmap;
3706 int md_gen, pvh_gen;
3707 boolean_t rv;
3708
3709 rv = FALSE;
3710 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
3711 rw_rlock(lock);
3712 restart:
3713 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
3714 pmap = PV_PMAP(pv);
3715 if (!PMAP_TRYLOCK(pmap)) {
3716 md_gen = m->md.pv_gen;
3717 rw_runlock(lock);
3718 PMAP_LOCK(pmap);
3719 rw_rlock(lock);
3720 if (md_gen != m->md.pv_gen) {
3721 PMAP_UNLOCK(pmap);
3722 goto restart;
3723 }
3724 }
3725 pte = pmap_pte(pmap, pv->pv_va);
3726 mask = 0;
3727 if (modified)
3728 mask |= PG_RW | PG_M;
3729 if (accessed)
3730 mask |= PG_V | PG_A;
3731 rv = (be64toh(*pte) & mask) == mask;
3732 PMAP_UNLOCK(pmap);
3733 if (rv)
3734 goto out;
3735 }
3736 if ((m->flags & PG_FICTITIOUS) == 0) {
3737 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
3738 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
3739 pmap = PV_PMAP(pv);
3740 if (!PMAP_TRYLOCK(pmap)) {
3741 md_gen = m->md.pv_gen;
3742 pvh_gen = pvh->pv_gen;
3743 rw_runlock(lock);
3744 PMAP_LOCK(pmap);
3745 rw_rlock(lock);
3746 if (md_gen != m->md.pv_gen ||
3747 pvh_gen != pvh->pv_gen) {
3748 PMAP_UNLOCK(pmap);
3749 goto restart;
3750 }
3751 }
3752 pte = pmap_pml3e(pmap, pv->pv_va);
3753 mask = 0;
3754 if (modified)
3755 mask |= PG_RW | PG_M;
3756 if (accessed)
3757 mask |= PG_V | PG_A;
3758 rv = (be64toh(*pte) & mask) == mask;
3759 PMAP_UNLOCK(pmap);
3760 if (rv)
3761 goto out;
3762 }
3763 }
3764 out:
3765 rw_runlock(lock);
3766 return (rv);
3767 }
3768
3769 /*
3770 * pmap_is_modified:
3771 *
3772 * Return whether or not the specified physical page was modified
3773 * in any physical maps.
3774 */
3775 boolean_t
mmu_radix_is_modified(vm_page_t m)3776 mmu_radix_is_modified(vm_page_t m)
3777 {
3778
3779 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3780 ("pmap_is_modified: page %p is not managed", m));
3781
3782 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3783 /*
3784 * If the page is not busied then this check is racy.
3785 */
3786 if (!pmap_page_is_write_mapped(m))
3787 return (FALSE);
3788 return (pmap_page_test_mappings(m, FALSE, TRUE));
3789 }
3790
3791 boolean_t
mmu_radix_is_prefaultable(pmap_t pmap,vm_offset_t addr)3792 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr)
3793 {
3794 pml3_entry_t *l3e;
3795 pt_entry_t *pte;
3796 boolean_t rv;
3797
3798 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
3799 rv = FALSE;
3800 PMAP_LOCK(pmap);
3801 l3e = pmap_pml3e(pmap, addr);
3802 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) {
3803 pte = pmap_l3e_to_pte(l3e, addr);
3804 rv = (be64toh(*pte) & PG_V) == 0;
3805 }
3806 PMAP_UNLOCK(pmap);
3807 return (rv);
3808 }
3809
3810 boolean_t
mmu_radix_is_referenced(vm_page_t m)3811 mmu_radix_is_referenced(vm_page_t m)
3812 {
3813 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3814 ("pmap_is_referenced: page %p is not managed", m));
3815 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3816 return (pmap_page_test_mappings(m, TRUE, FALSE));
3817 }
3818
3819 /*
3820 * pmap_ts_referenced:
3821 *
3822 * Return a count of reference bits for a page, clearing those bits.
3823 * It is not necessary for every reference bit to be cleared, but it
3824 * is necessary that 0 only be returned when there are truly no
3825 * reference bits set.
3826 *
3827 * As an optimization, update the page's dirty field if a modified bit is
3828 * found while counting reference bits. This opportunistic update can be
3829 * performed at low cost and can eliminate the need for some future calls
3830 * to pmap_is_modified(). However, since this function stops after
3831 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
3832 * dirty pages. Those dirty pages will only be detected by a future call
3833 * to pmap_is_modified().
3834 *
3835 * A DI block is not needed within this function, because
3836 * invalidations are performed before the PV list lock is
3837 * released.
3838 */
3839 int
mmu_radix_ts_referenced(vm_page_t m)3840 mmu_radix_ts_referenced(vm_page_t m)
3841 {
3842 struct md_page *pvh;
3843 pv_entry_t pv, pvf;
3844 pmap_t pmap;
3845 struct rwlock *lock;
3846 pml3_entry_t oldl3e, *l3e;
3847 pt_entry_t *pte;
3848 vm_paddr_t pa;
3849 int cleared, md_gen, not_cleared, pvh_gen;
3850 struct spglist free;
3851
3852 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3853 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3854 ("pmap_ts_referenced: page %p is not managed", m));
3855 SLIST_INIT(&free);
3856 cleared = 0;
3857 pa = VM_PAGE_TO_PHYS(m);
3858 lock = PHYS_TO_PV_LIST_LOCK(pa);
3859 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
3860 rw_wlock(lock);
3861 retry:
3862 not_cleared = 0;
3863 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
3864 goto small_mappings;
3865 pv = pvf;
3866 do {
3867 if (pvf == NULL)
3868 pvf = pv;
3869 pmap = PV_PMAP(pv);
3870 if (!PMAP_TRYLOCK(pmap)) {
3871 pvh_gen = pvh->pv_gen;
3872 rw_wunlock(lock);
3873 PMAP_LOCK(pmap);
3874 rw_wlock(lock);
3875 if (pvh_gen != pvh->pv_gen) {
3876 PMAP_UNLOCK(pmap);
3877 goto retry;
3878 }
3879 }
3880 l3e = pmap_pml3e(pmap, pv->pv_va);
3881 oldl3e = be64toh(*l3e);
3882 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
3883 /*
3884 * Although "oldpde" is mapping a 2MB page, because
3885 * this function is called at a 4KB page granularity,
3886 * we only update the 4KB page under test.
3887 */
3888 vm_page_dirty(m);
3889 }
3890 if ((oldl3e & PG_A) != 0) {
3891 /*
3892 * Since this reference bit is shared by 512 4KB
3893 * pages, it should not be cleared every time it is
3894 * tested. Apply a simple "hash" function on the
3895 * physical page number, the virtual superpage number,
3896 * and the pmap address to select one 4KB page out of
3897 * the 512 on which testing the reference bit will
3898 * result in clearing that reference bit. This
3899 * function is designed to avoid the selection of the
3900 * same 4KB page for every 2MB page mapping.
3901 *
3902 * On demotion, a mapping that hasn't been referenced
3903 * is simply destroyed. To avoid the possibility of a
3904 * subsequent page fault on a demoted wired mapping,
3905 * always leave its reference bit set. Moreover,
3906 * since the superpage is wired, the current state of
3907 * its reference bit won't affect page replacement.
3908 */
3909 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^
3910 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
3911 (oldl3e & PG_W) == 0) {
3912 atomic_clear_long(l3e, htobe64(PG_A));
3913 pmap_invalidate_page(pmap, pv->pv_va);
3914 cleared++;
3915 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
3916 ("inconsistent pv lock %p %p for page %p",
3917 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
3918 } else
3919 not_cleared++;
3920 }
3921 PMAP_UNLOCK(pmap);
3922 /* Rotate the PV list if it has more than one entry. */
3923 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
3924 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
3925 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
3926 pvh->pv_gen++;
3927 }
3928 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX)
3929 goto out;
3930 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
3931 small_mappings:
3932 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
3933 goto out;
3934 pv = pvf;
3935 do {
3936 if (pvf == NULL)
3937 pvf = pv;
3938 pmap = PV_PMAP(pv);
3939 if (!PMAP_TRYLOCK(pmap)) {
3940 pvh_gen = pvh->pv_gen;
3941 md_gen = m->md.pv_gen;
3942 rw_wunlock(lock);
3943 PMAP_LOCK(pmap);
3944 rw_wlock(lock);
3945 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
3946 PMAP_UNLOCK(pmap);
3947 goto retry;
3948 }
3949 }
3950 l3e = pmap_pml3e(pmap, pv->pv_va);
3951 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
3952 ("pmap_ts_referenced: found a 2mpage in page %p's pv list",
3953 m));
3954 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
3955 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW))
3956 vm_page_dirty(m);
3957 if ((be64toh(*pte) & PG_A) != 0) {
3958 atomic_clear_long(pte, htobe64(PG_A));
3959 pmap_invalidate_page(pmap, pv->pv_va);
3960 cleared++;
3961 }
3962 PMAP_UNLOCK(pmap);
3963 /* Rotate the PV list if it has more than one entry. */
3964 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
3965 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
3966 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3967 m->md.pv_gen++;
3968 }
3969 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
3970 not_cleared < PMAP_TS_REFERENCED_MAX);
3971 out:
3972 rw_wunlock(lock);
3973 vm_page_free_pages_toq(&free, true);
3974 return (cleared + not_cleared);
3975 }
3976
3977 static vm_offset_t
mmu_radix_map(vm_offset_t * virt __unused,vm_paddr_t start,vm_paddr_t end,int prot __unused)3978 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start,
3979 vm_paddr_t end, int prot __unused)
3980 {
3981
3982 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end,
3983 prot);
3984 return (PHYS_TO_DMAP(start));
3985 }
3986
3987 void
mmu_radix_object_init_pt(pmap_t pmap,vm_offset_t addr,vm_object_t object,vm_pindex_t pindex,vm_size_t size)3988 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr,
3989 vm_object_t object, vm_pindex_t pindex, vm_size_t size)
3990 {
3991 pml3_entry_t *l3e;
3992 vm_paddr_t pa, ptepa;
3993 vm_page_t p, pdpg;
3994 vm_memattr_t ma;
3995
3996 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,
3997 object, pindex, size);
3998 VM_OBJECT_ASSERT_WLOCKED(object);
3999 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4000 ("pmap_object_init_pt: non-device object"));
4001 /* NB: size can be logically ored with addr here */
4002 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) {
4003 if (!mmu_radix_ps_enabled(pmap))
4004 return;
4005 if (!vm_object_populate(object, pindex, pindex + atop(size)))
4006 return;
4007 p = vm_page_lookup(object, pindex);
4008 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4009 ("pmap_object_init_pt: invalid page %p", p));
4010 ma = p->md.mdpg_cache_attrs;
4011
4012 /*
4013 * Abort the mapping if the first page is not physically
4014 * aligned to a 2MB page boundary.
4015 */
4016 ptepa = VM_PAGE_TO_PHYS(p);
4017 if (ptepa & L3_PAGE_MASK)
4018 return;
4019
4020 /*
4021 * Skip the first page. Abort the mapping if the rest of
4022 * the pages are not physically contiguous or have differing
4023 * memory attributes.
4024 */
4025 p = TAILQ_NEXT(p, listq);
4026 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
4027 pa += PAGE_SIZE) {
4028 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4029 ("pmap_object_init_pt: invalid page %p", p));
4030 if (pa != VM_PAGE_TO_PHYS(p) ||
4031 ma != p->md.mdpg_cache_attrs)
4032 return;
4033 p = TAILQ_NEXT(p, listq);
4034 }
4035
4036 PMAP_LOCK(pmap);
4037 for (pa = ptepa | pmap_cache_bits(ma);
4038 pa < ptepa + size; pa += L3_PAGE_SIZE) {
4039 pdpg = pmap_allocl3e(pmap, addr, NULL);
4040 if (pdpg == NULL) {
4041 /*
4042 * The creation of mappings below is only an
4043 * optimization. If a page directory page
4044 * cannot be allocated without blocking,
4045 * continue on to the next mapping rather than
4046 * blocking.
4047 */
4048 addr += L3_PAGE_SIZE;
4049 continue;
4050 }
4051 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
4052 l3e = &l3e[pmap_pml3e_index(addr)];
4053 if ((be64toh(*l3e) & PG_V) == 0) {
4054 pa |= PG_M | PG_A | PG_RW;
4055 pte_store(l3e, pa);
4056 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
4057 atomic_add_long(&pmap_l3e_mappings, 1);
4058 } else {
4059 /* Continue on if the PDE is already valid. */
4060 pdpg->ref_count--;
4061 KASSERT(pdpg->ref_count > 0,
4062 ("pmap_object_init_pt: missing reference "
4063 "to page directory page, va: 0x%lx", addr));
4064 }
4065 addr += L3_PAGE_SIZE;
4066 }
4067 ptesync();
4068 PMAP_UNLOCK(pmap);
4069 }
4070 }
4071
4072 boolean_t
mmu_radix_page_exists_quick(pmap_t pmap,vm_page_t m)4073 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m)
4074 {
4075 struct md_page *pvh;
4076 struct rwlock *lock;
4077 pv_entry_t pv;
4078 int loops = 0;
4079 boolean_t rv;
4080
4081 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4082 ("pmap_page_exists_quick: page %p is not managed", m));
4083 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);
4084 rv = FALSE;
4085 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4086 rw_rlock(lock);
4087 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4088 if (PV_PMAP(pv) == pmap) {
4089 rv = TRUE;
4090 break;
4091 }
4092 loops++;
4093 if (loops >= 16)
4094 break;
4095 }
4096 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4097 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4098 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4099 if (PV_PMAP(pv) == pmap) {
4100 rv = TRUE;
4101 break;
4102 }
4103 loops++;
4104 if (loops >= 16)
4105 break;
4106 }
4107 }
4108 rw_runlock(lock);
4109 return (rv);
4110 }
4111
4112 void
mmu_radix_page_init(vm_page_t m)4113 mmu_radix_page_init(vm_page_t m)
4114 {
4115
4116 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4117 TAILQ_INIT(&m->md.pv_list);
4118 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
4119 }
4120
4121 int
mmu_radix_page_wired_mappings(vm_page_t m)4122 mmu_radix_page_wired_mappings(vm_page_t m)
4123 {
4124 struct rwlock *lock;
4125 struct md_page *pvh;
4126 pmap_t pmap;
4127 pt_entry_t *pte;
4128 pv_entry_t pv;
4129 int count, md_gen, pvh_gen;
4130
4131 if ((m->oflags & VPO_UNMANAGED) != 0)
4132 return (0);
4133 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4134 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4135 rw_rlock(lock);
4136 restart:
4137 count = 0;
4138 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4139 pmap = PV_PMAP(pv);
4140 if (!PMAP_TRYLOCK(pmap)) {
4141 md_gen = m->md.pv_gen;
4142 rw_runlock(lock);
4143 PMAP_LOCK(pmap);
4144 rw_rlock(lock);
4145 if (md_gen != m->md.pv_gen) {
4146 PMAP_UNLOCK(pmap);
4147 goto restart;
4148 }
4149 }
4150 pte = pmap_pte(pmap, pv->pv_va);
4151 if ((be64toh(*pte) & PG_W) != 0)
4152 count++;
4153 PMAP_UNLOCK(pmap);
4154 }
4155 if ((m->flags & PG_FICTITIOUS) == 0) {
4156 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4157 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4158 pmap = PV_PMAP(pv);
4159 if (!PMAP_TRYLOCK(pmap)) {
4160 md_gen = m->md.pv_gen;
4161 pvh_gen = pvh->pv_gen;
4162 rw_runlock(lock);
4163 PMAP_LOCK(pmap);
4164 rw_rlock(lock);
4165 if (md_gen != m->md.pv_gen ||
4166 pvh_gen != pvh->pv_gen) {
4167 PMAP_UNLOCK(pmap);
4168 goto restart;
4169 }
4170 }
4171 pte = pmap_pml3e(pmap, pv->pv_va);
4172 if ((be64toh(*pte) & PG_W) != 0)
4173 count++;
4174 PMAP_UNLOCK(pmap);
4175 }
4176 }
4177 rw_runlock(lock);
4178 return (count);
4179 }
4180
4181 static void
mmu_radix_update_proctab(int pid,pml1_entry_t l1pa)4182 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa)
4183 {
4184 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT);
4185 }
4186
4187 int
mmu_radix_pinit(pmap_t pmap)4188 mmu_radix_pinit(pmap_t pmap)
4189 {
4190 vmem_addr_t pid;
4191 vm_paddr_t l1pa;
4192
4193 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4194
4195 /*
4196 * allocate the page directory page
4197 */
4198 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);
4199
4200 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++)
4201 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE);
4202 vm_radix_init(&pmap->pm_radix);
4203 TAILQ_INIT(&pmap->pm_pvchunk);
4204 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4205 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4206 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid);
4207
4208 pmap->pm_pid = pid;
4209 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1);
4210 mmu_radix_update_proctab(pid, l1pa);
4211 __asm __volatile("ptesync;isync" : : : "memory");
4212
4213 return (1);
4214 }
4215
4216 /*
4217 * This routine is called if the desired page table page does not exist.
4218 *
4219 * If page table page allocation fails, this routine may sleep before
4220 * returning NULL. It sleeps only if a lock pointer was given.
4221 *
4222 * Note: If a page allocation fails at page table level two or three,
4223 * one or two pages may be held during the wait, only to be released
4224 * afterwards. This conservative approach is easily argued to avoid
4225 * race conditions.
4226 */
4227 static vm_page_t
_pmap_allocpte(pmap_t pmap,vm_pindex_t ptepindex,struct rwlock ** lockp)4228 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
4229 {
4230 vm_page_t m, pdppg, pdpg;
4231
4232 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4233
4234 /*
4235 * Allocate a page table page.
4236 */
4237 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
4238 if (lockp != NULL) {
4239 RELEASE_PV_LIST_LOCK(lockp);
4240 PMAP_UNLOCK(pmap);
4241 vm_wait(NULL);
4242 PMAP_LOCK(pmap);
4243 }
4244 /*
4245 * Indicate the need to retry. While waiting, the page table
4246 * page may have been allocated.
4247 */
4248 return (NULL);
4249 }
4250 m->pindex = ptepindex;
4251
4252 /*
4253 * Map the pagetable page into the process address space, if
4254 * it isn't already there.
4255 */
4256
4257 if (ptepindex >= (NUPDE + NUPDPE)) {
4258 pml1_entry_t *l1e;
4259 vm_pindex_t pml1index;
4260
4261 /* Wire up a new PDPE page */
4262 pml1index = ptepindex - (NUPDE + NUPDPE);
4263 l1e = &pmap->pm_pml1[pml1index];
4264 KASSERT((be64toh(*l1e) & PG_V) == 0,
4265 ("%s: L1 entry %#lx is valid", __func__, *l1e));
4266 pde_store(l1e, VM_PAGE_TO_PHYS(m));
4267 } else if (ptepindex >= NUPDE) {
4268 vm_pindex_t pml1index;
4269 vm_pindex_t pdpindex;
4270 pml1_entry_t *l1e;
4271 pml2_entry_t *l2e;
4272
4273 /* Wire up a new l2e page */
4274 pdpindex = ptepindex - NUPDE;
4275 pml1index = pdpindex >> RPTE_SHIFT;
4276
4277 l1e = &pmap->pm_pml1[pml1index];
4278 if ((be64toh(*l1e) & PG_V) == 0) {
4279 /* Have to allocate a new pdp, recurse */
4280 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index,
4281 lockp) == NULL) {
4282 vm_page_unwire_noq(m);
4283 vm_page_free_zero(m);
4284 return (NULL);
4285 }
4286 } else {
4287 /* Add reference to l2e page */
4288 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME);
4289 pdppg->ref_count++;
4290 }
4291 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4292
4293 /* Now find the pdp page */
4294 l2e = &l2e[pdpindex & RPTE_MASK];
4295 KASSERT((be64toh(*l2e) & PG_V) == 0,
4296 ("%s: L2 entry %#lx is valid", __func__, *l2e));
4297 pde_store(l2e, VM_PAGE_TO_PHYS(m));
4298 } else {
4299 vm_pindex_t pml1index;
4300 vm_pindex_t pdpindex;
4301 pml1_entry_t *l1e;
4302 pml2_entry_t *l2e;
4303 pml3_entry_t *l3e;
4304
4305 /* Wire up a new PTE page */
4306 pdpindex = ptepindex >> RPTE_SHIFT;
4307 pml1index = pdpindex >> RPTE_SHIFT;
4308
4309 /* First, find the pdp and check that its valid. */
4310 l1e = &pmap->pm_pml1[pml1index];
4311 if ((be64toh(*l1e) & PG_V) == 0) {
4312 /* Have to allocate a new pd, recurse */
4313 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4314 lockp) == NULL) {
4315 vm_page_unwire_noq(m);
4316 vm_page_free_zero(m);
4317 return (NULL);
4318 }
4319 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4320 l2e = &l2e[pdpindex & RPTE_MASK];
4321 } else {
4322 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4323 l2e = &l2e[pdpindex & RPTE_MASK];
4324 if ((be64toh(*l2e) & PG_V) == 0) {
4325 /* Have to allocate a new pd, recurse */
4326 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4327 lockp) == NULL) {
4328 vm_page_unwire_noq(m);
4329 vm_page_free_zero(m);
4330 return (NULL);
4331 }
4332 } else {
4333 /* Add reference to the pd page */
4334 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME);
4335 pdpg->ref_count++;
4336 }
4337 }
4338 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME);
4339
4340 /* Now we know where the page directory page is */
4341 l3e = &l3e[ptepindex & RPTE_MASK];
4342 KASSERT((be64toh(*l3e) & PG_V) == 0,
4343 ("%s: L3 entry %#lx is valid", __func__, *l3e));
4344 pde_store(l3e, VM_PAGE_TO_PHYS(m));
4345 }
4346
4347 pmap_resident_count_inc(pmap, 1);
4348 return (m);
4349 }
4350 static vm_page_t
pmap_allocl3e(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4351 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4352 {
4353 vm_pindex_t pdpindex, ptepindex;
4354 pml2_entry_t *pdpe;
4355 vm_page_t pdpg;
4356
4357 retry:
4358 pdpe = pmap_pml2e(pmap, va);
4359 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) {
4360 /* Add a reference to the pd page. */
4361 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME);
4362 pdpg->ref_count++;
4363 } else {
4364 /* Allocate a pd page. */
4365 ptepindex = pmap_l3e_pindex(va);
4366 pdpindex = ptepindex >> RPTE_SHIFT;
4367 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
4368 if (pdpg == NULL && lockp != NULL)
4369 goto retry;
4370 }
4371 return (pdpg);
4372 }
4373
4374 static vm_page_t
pmap_allocpte(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4375 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4376 {
4377 vm_pindex_t ptepindex;
4378 pml3_entry_t *pd;
4379 vm_page_t m;
4380
4381 /*
4382 * Calculate pagetable page index
4383 */
4384 ptepindex = pmap_l3e_pindex(va);
4385 retry:
4386 /*
4387 * Get the page directory entry
4388 */
4389 pd = pmap_pml3e(pmap, va);
4390
4391 /*
4392 * This supports switching from a 2MB page to a
4393 * normal 4K page.
4394 */
4395 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) {
4396 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) {
4397 /*
4398 * Invalidation of the 2MB page mapping may have caused
4399 * the deallocation of the underlying PD page.
4400 */
4401 pd = NULL;
4402 }
4403 }
4404
4405 /*
4406 * If the page table page is mapped, we just increment the
4407 * hold count, and activate it.
4408 */
4409 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) {
4410 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME);
4411 m->ref_count++;
4412 } else {
4413 /*
4414 * Here if the pte page isn't mapped, or if it has been
4415 * deallocated.
4416 */
4417 m = _pmap_allocpte(pmap, ptepindex, lockp);
4418 if (m == NULL && lockp != NULL)
4419 goto retry;
4420 }
4421 return (m);
4422 }
4423
4424 static void
mmu_radix_pinit0(pmap_t pmap)4425 mmu_radix_pinit0(pmap_t pmap)
4426 {
4427
4428 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4429 PMAP_LOCK_INIT(pmap);
4430 pmap->pm_pml1 = kernel_pmap->pm_pml1;
4431 pmap->pm_pid = kernel_pmap->pm_pid;
4432
4433 vm_radix_init(&pmap->pm_radix);
4434 TAILQ_INIT(&pmap->pm_pvchunk);
4435 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4436 kernel_pmap->pm_flags =
4437 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4438 }
4439 /*
4440 * pmap_protect_l3e: do the things to protect a 2mpage in a process
4441 */
4442 static boolean_t
pmap_protect_l3e(pmap_t pmap,pt_entry_t * l3e,vm_offset_t sva,vm_prot_t prot)4443 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot)
4444 {
4445 pt_entry_t newpde, oldpde;
4446 vm_offset_t eva, va;
4447 vm_page_t m;
4448 boolean_t anychanged;
4449
4450 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4451 KASSERT((sva & L3_PAGE_MASK) == 0,
4452 ("pmap_protect_l3e: sva is not 2mpage aligned"));
4453 anychanged = FALSE;
4454 retry:
4455 oldpde = newpde = be64toh(*l3e);
4456 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
4457 (PG_MANAGED | PG_M | PG_RW)) {
4458 eva = sva + L3_PAGE_SIZE;
4459 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
4460 va < eva; va += PAGE_SIZE, m++)
4461 vm_page_dirty(m);
4462 }
4463 if ((prot & VM_PROT_WRITE) == 0) {
4464 newpde &= ~(PG_RW | PG_M);
4465 newpde |= RPTE_EAA_R;
4466 }
4467 if (prot & VM_PROT_EXECUTE)
4468 newpde |= PG_X;
4469 if (newpde != oldpde) {
4470 /*
4471 * As an optimization to future operations on this PDE, clear
4472 * PG_PROMOTED. The impending invalidation will remove any
4473 * lingering 4KB page mappings from the TLB.
4474 */
4475 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED)))
4476 goto retry;
4477 anychanged = TRUE;
4478 }
4479 return (anychanged);
4480 }
4481
4482 void
mmu_radix_protect(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,vm_prot_t prot)4483 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
4484 vm_prot_t prot)
4485 {
4486 vm_offset_t va_next;
4487 pml1_entry_t *l1e;
4488 pml2_entry_t *l2e;
4489 pml3_entry_t ptpaddr, *l3e;
4490 pt_entry_t *pte;
4491 boolean_t anychanged;
4492
4493 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva,
4494 prot);
4495
4496 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
4497 if (prot == VM_PROT_NONE) {
4498 mmu_radix_remove(pmap, sva, eva);
4499 return;
4500 }
4501
4502 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
4503 (VM_PROT_WRITE|VM_PROT_EXECUTE))
4504 return;
4505
4506 #ifdef INVARIANTS
4507 if (VERBOSE_PROTECT || pmap_logging)
4508 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n",
4509 pmap, sva, eva, prot, pmap->pm_pid);
4510 #endif
4511 anychanged = FALSE;
4512
4513 PMAP_LOCK(pmap);
4514 for (; sva < eva; sva = va_next) {
4515 l1e = pmap_pml1e(pmap, sva);
4516 if ((be64toh(*l1e) & PG_V) == 0) {
4517 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
4518 if (va_next < sva)
4519 va_next = eva;
4520 continue;
4521 }
4522
4523 l2e = pmap_l1e_to_l2e(l1e, sva);
4524 if ((be64toh(*l2e) & PG_V) == 0) {
4525 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
4526 if (va_next < sva)
4527 va_next = eva;
4528 continue;
4529 }
4530
4531 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
4532 if (va_next < sva)
4533 va_next = eva;
4534
4535 l3e = pmap_l2e_to_l3e(l2e, sva);
4536 ptpaddr = be64toh(*l3e);
4537
4538 /*
4539 * Weed out invalid mappings.
4540 */
4541 if (ptpaddr == 0)
4542 continue;
4543
4544 /*
4545 * Check for large page.
4546 */
4547 if ((ptpaddr & RPTE_LEAF) != 0) {
4548 /*
4549 * Are we protecting the entire large page? If not,
4550 * demote the mapping and fall through.
4551 */
4552 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
4553 if (pmap_protect_l3e(pmap, l3e, sva, prot))
4554 anychanged = TRUE;
4555 continue;
4556 } else if (!pmap_demote_l3e(pmap, l3e, sva)) {
4557 /*
4558 * The large page mapping was destroyed.
4559 */
4560 continue;
4561 }
4562 }
4563
4564 if (va_next > eva)
4565 va_next = eva;
4566
4567 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
4568 sva += PAGE_SIZE) {
4569 pt_entry_t obits, pbits;
4570 vm_page_t m;
4571
4572 retry:
4573 MPASS(pte == pmap_pte(pmap, sva));
4574 obits = pbits = be64toh(*pte);
4575 if ((pbits & PG_V) == 0)
4576 continue;
4577
4578 if ((prot & VM_PROT_WRITE) == 0) {
4579 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
4580 (PG_MANAGED | PG_M | PG_RW)) {
4581 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
4582 vm_page_dirty(m);
4583 }
4584 pbits &= ~(PG_RW | PG_M);
4585 pbits |= RPTE_EAA_R;
4586 }
4587 if (prot & VM_PROT_EXECUTE)
4588 pbits |= PG_X;
4589
4590 if (pbits != obits) {
4591 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits)))
4592 goto retry;
4593 if (obits & (PG_A|PG_M)) {
4594 anychanged = TRUE;
4595 #ifdef INVARIANTS
4596 if (VERBOSE_PROTECT || pmap_logging)
4597 printf("%#lx %#lx -> %#lx\n",
4598 sva, obits, pbits);
4599 #endif
4600 }
4601 }
4602 }
4603 }
4604 if (anychanged)
4605 pmap_invalidate_all(pmap);
4606 PMAP_UNLOCK(pmap);
4607 }
4608
4609 void
mmu_radix_qenter(vm_offset_t sva,vm_page_t * ma,int count)4610 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count)
4611 {
4612
4613 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count);
4614 pt_entry_t oldpte, pa, *pte;
4615 vm_page_t m;
4616 uint64_t cache_bits, attr_bits;
4617 vm_offset_t va;
4618
4619 oldpte = 0;
4620 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
4621 va = sva;
4622 pte = kvtopte(va);
4623 while (va < sva + PAGE_SIZE * count) {
4624 if (__predict_false((va & L3_PAGE_MASK) == 0))
4625 pte = kvtopte(va);
4626 MPASS(pte == pmap_pte(kernel_pmap, va));
4627
4628 /*
4629 * XXX there has to be a more efficient way than traversing
4630 * the page table every time - but go for correctness for
4631 * today
4632 */
4633
4634 m = *ma++;
4635 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs);
4636 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits;
4637 if (be64toh(*pte) != pa) {
4638 oldpte |= be64toh(*pte);
4639 pte_store(pte, pa);
4640 }
4641 va += PAGE_SIZE;
4642 pte++;
4643 }
4644 if (__predict_false((oldpte & RPTE_VALID) != 0))
4645 pmap_invalidate_range(kernel_pmap, sva, sva + count *
4646 PAGE_SIZE);
4647 else
4648 ptesync();
4649 }
4650
4651 void
mmu_radix_qremove(vm_offset_t sva,int count)4652 mmu_radix_qremove(vm_offset_t sva, int count)
4653 {
4654 vm_offset_t va;
4655 pt_entry_t *pte;
4656
4657 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count);
4658 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva));
4659
4660 va = sva;
4661 pte = kvtopte(va);
4662 while (va < sva + PAGE_SIZE * count) {
4663 if (__predict_false((va & L3_PAGE_MASK) == 0))
4664 pte = kvtopte(va);
4665 pte_clear(pte);
4666 pte++;
4667 va += PAGE_SIZE;
4668 }
4669 pmap_invalidate_range(kernel_pmap, sva, va);
4670 }
4671
4672 /***************************************************
4673 * Page table page management routines.....
4674 ***************************************************/
4675 /*
4676 * Schedule the specified unused page table page to be freed. Specifically,
4677 * add the page to the specified list of pages that will be released to the
4678 * physical memory manager after the TLB has been updated.
4679 */
4680 static __inline void
pmap_add_delayed_free_list(vm_page_t m,struct spglist * free,boolean_t set_PG_ZERO)4681 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
4682 boolean_t set_PG_ZERO)
4683 {
4684
4685 if (set_PG_ZERO)
4686 m->flags |= PG_ZERO;
4687 else
4688 m->flags &= ~PG_ZERO;
4689 SLIST_INSERT_HEAD(free, m, plinks.s.ss);
4690 }
4691
4692 /*
4693 * Inserts the specified page table page into the specified pmap's collection
4694 * of idle page table pages. Each of a pmap's page table pages is responsible
4695 * for mapping a distinct range of virtual addresses. The pmap's collection is
4696 * ordered by this virtual address range.
4697 */
4698 static __inline int
pmap_insert_pt_page(pmap_t pmap,vm_page_t mpte)4699 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
4700 {
4701
4702 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4703 return (vm_radix_insert(&pmap->pm_radix, mpte));
4704 }
4705
4706 /*
4707 * Removes the page table page mapping the specified virtual address from the
4708 * specified pmap's collection of idle page table pages, and returns it.
4709 * Otherwise, returns NULL if there is no page table page corresponding to the
4710 * specified virtual address.
4711 */
4712 static __inline vm_page_t
pmap_remove_pt_page(pmap_t pmap,vm_offset_t va)4713 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
4714 {
4715
4716 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4717 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));
4718 }
4719
4720 /*
4721 * Decrements a page table page's wire count, which is used to record the
4722 * number of valid page table entries within the page. If the wire count
4723 * drops to zero, then the page table page is unmapped. Returns TRUE if the
4724 * page table page was unmapped and FALSE otherwise.
4725 */
4726 static inline boolean_t
pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4727 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4728 {
4729
4730 --m->ref_count;
4731 if (m->ref_count == 0) {
4732 _pmap_unwire_ptp(pmap, va, m, free);
4733 return (TRUE);
4734 } else
4735 return (FALSE);
4736 }
4737
4738 static void
_pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4739 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4740 {
4741
4742 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4743 /*
4744 * unmap the page table page
4745 */
4746 if (m->pindex >= (NUPDE + NUPDPE)) {
4747 /* PDP page */
4748 pml1_entry_t *pml1;
4749 pml1 = pmap_pml1e(pmap, va);
4750 *pml1 = 0;
4751 } else if (m->pindex >= NUPDE) {
4752 /* PD page */
4753 pml2_entry_t *l2e;
4754 l2e = pmap_pml2e(pmap, va);
4755 *l2e = 0;
4756 } else {
4757 /* PTE page */
4758 pml3_entry_t *l3e;
4759 l3e = pmap_pml3e(pmap, va);
4760 *l3e = 0;
4761 }
4762 pmap_resident_count_dec(pmap, 1);
4763 if (m->pindex < NUPDE) {
4764 /* We just released a PT, unhold the matching PD */
4765 vm_page_t pdpg;
4766
4767 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);
4768 pmap_unwire_ptp(pmap, va, pdpg, free);
4769 }
4770 if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
4771 /* We just released a PD, unhold the matching PDP */
4772 vm_page_t pdppg;
4773
4774 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);
4775 pmap_unwire_ptp(pmap, va, pdppg, free);
4776 }
4777
4778 /*
4779 * Put page on a list so that it is released after
4780 * *ALL* TLB shootdown is done
4781 */
4782 pmap_add_delayed_free_list(m, free, TRUE);
4783 }
4784
4785 /*
4786 * After removing a page table entry, this routine is used to
4787 * conditionally free the page, and manage the hold/wire counts.
4788 */
4789 static int
pmap_unuse_pt(pmap_t pmap,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free)4790 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde,
4791 struct spglist *free)
4792 {
4793 vm_page_t mpte;
4794
4795 if (va >= VM_MAXUSER_ADDRESS)
4796 return (0);
4797 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
4798 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
4799 return (pmap_unwire_ptp(pmap, va, mpte, free));
4800 }
4801
4802 void
mmu_radix_release(pmap_t pmap)4803 mmu_radix_release(pmap_t pmap)
4804 {
4805
4806 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4807 KASSERT(pmap->pm_stats.resident_count == 0,
4808 ("pmap_release: pmap resident count %ld != 0",
4809 pmap->pm_stats.resident_count));
4810 KASSERT(vm_radix_is_empty(&pmap->pm_radix),
4811 ("pmap_release: pmap has reserved page table page(s)"));
4812
4813 pmap_invalidate_all(pmap);
4814 isa3_proctab[pmap->pm_pid].proctab0 = 0;
4815 uma_zfree(zone_radix_pgd, pmap->pm_pml1);
4816 vmem_free(asid_arena, pmap->pm_pid, 1);
4817 }
4818
4819 /*
4820 * Create the PV entry for a 2MB page mapping. Always returns true unless the
4821 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns
4822 * false if the PV entry cannot be allocated without resorting to reclamation.
4823 */
4824 static bool
pmap_pv_insert_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t pde,u_int flags,struct rwlock ** lockp)4825 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags,
4826 struct rwlock **lockp)
4827 {
4828 struct md_page *pvh;
4829 pv_entry_t pv;
4830 vm_paddr_t pa;
4831
4832 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4833 /* Pass NULL instead of the lock pointer to disable reclamation. */
4834 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
4835 NULL : lockp)) == NULL)
4836 return (false);
4837 pv->pv_va = va;
4838 pa = pde & PG_PS_FRAME;
4839 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
4840 pvh = pa_to_pvh(pa);
4841 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
4842 pvh->pv_gen++;
4843 return (true);
4844 }
4845
4846 /*
4847 * Fills a page table page with mappings to consecutive physical pages.
4848 */
4849 static void
pmap_fill_ptp(pt_entry_t * firstpte,pt_entry_t newpte)4850 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
4851 {
4852 pt_entry_t *pte;
4853
4854 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
4855 *pte = htobe64(newpte);
4856 newpte += PAGE_SIZE;
4857 }
4858 }
4859
4860 static boolean_t
pmap_demote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va)4861 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va)
4862 {
4863 struct rwlock *lock;
4864 boolean_t rv;
4865
4866 lock = NULL;
4867 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock);
4868 if (lock != NULL)
4869 rw_wunlock(lock);
4870 return (rv);
4871 }
4872
4873 static boolean_t
pmap_demote_l3e_locked(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va,struct rwlock ** lockp)4874 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
4875 struct rwlock **lockp)
4876 {
4877 pml3_entry_t oldpde;
4878 pt_entry_t *firstpte;
4879 vm_paddr_t mptepa;
4880 vm_page_t mpte;
4881 struct spglist free;
4882 vm_offset_t sva;
4883
4884 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4885 oldpde = be64toh(*l3e);
4886 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
4887 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx",
4888 oldpde));
4889 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
4890 NULL) {
4891 KASSERT((oldpde & PG_W) == 0,
4892 ("pmap_demote_l3e: page table page for a wired mapping"
4893 " is missing"));
4894
4895 /*
4896 * Invalidate the 2MB page mapping and return "failure" if the
4897 * mapping was never accessed or the allocation of the new
4898 * page table page fails. If the 2MB page mapping belongs to
4899 * the direct map region of the kernel's address space, then
4900 * the page allocation request specifies the highest possible
4901 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is
4902 * normal. Page table pages are preallocated for every other
4903 * part of the kernel address space, so the direct map region
4904 * is the only part of the kernel address space that must be
4905 * handled here.
4906 */
4907 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj(
4908 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ?
4909 VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) {
4910 SLIST_INIT(&free);
4911 sva = trunc_2mpage(va);
4912 pmap_remove_l3e(pmap, l3e, sva, &free, lockp);
4913 pmap_invalidate_l3e_page(pmap, sva, oldpde);
4914 vm_page_free_pages_toq(&free, true);
4915 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx"
4916 " in pmap %p", va, pmap);
4917 return (FALSE);
4918 }
4919 mpte->pindex = pmap_l3e_pindex(va);
4920 if (va < VM_MAXUSER_ADDRESS)
4921 pmap_resident_count_inc(pmap, 1);
4922 }
4923 mptepa = VM_PAGE_TO_PHYS(mpte);
4924 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
4925 KASSERT((oldpde & PG_A) != 0,
4926 ("pmap_demote_l3e: oldpde is missing PG_A"));
4927 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
4928 ("pmap_demote_l3e: oldpde is missing PG_M"));
4929
4930 /*
4931 * If the page table page is new, initialize it.
4932 */
4933 if (mpte->ref_count == 1) {
4934 mpte->ref_count = NPTEPG;
4935 pmap_fill_ptp(firstpte, oldpde);
4936 }
4937
4938 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME),
4939 ("pmap_demote_l3e: firstpte and newpte map different physical"
4940 " addresses"));
4941
4942 /*
4943 * If the mapping has changed attributes, update the page table
4944 * entries.
4945 */
4946 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE))
4947 pmap_fill_ptp(firstpte, oldpde);
4948
4949 /*
4950 * The spare PV entries must be reserved prior to demoting the
4951 * mapping, that is, prior to changing the PDE. Otherwise, the state
4952 * of the PDE and the PV lists will be inconsistent, which can result
4953 * in reclaim_pv_chunk() attempting to remove a PV entry from the
4954 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected
4955 * PV entry for the 2MB page mapping that is being demoted.
4956 */
4957 if ((oldpde & PG_MANAGED) != 0)
4958 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
4959
4960 /*
4961 * Demote the mapping. This pmap is locked. The old PDE has
4962 * PG_A set. If the old PDE has PG_RW set, it also has PG_M
4963 * set. Thus, there is no danger of a race with another
4964 * processor changing the setting of PG_A and/or PG_M between
4965 * the read above and the store below.
4966 */
4967 pde_store(l3e, mptepa);
4968 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde);
4969 /*
4970 * Demote the PV entry.
4971 */
4972 if ((oldpde & PG_MANAGED) != 0)
4973 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp);
4974
4975 atomic_add_long(&pmap_l3e_demotions, 1);
4976 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx"
4977 " in pmap %p", va, pmap);
4978 return (TRUE);
4979 }
4980
4981 /*
4982 * pmap_remove_kernel_pde: Remove a kernel superpage mapping.
4983 */
4984 static void
pmap_remove_kernel_l3e(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va)4985 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va)
4986 {
4987 vm_paddr_t mptepa;
4988 vm_page_t mpte;
4989
4990 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
4991 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4992 mpte = pmap_remove_pt_page(pmap, va);
4993 if (mpte == NULL)
4994 panic("pmap_remove_kernel_pde: Missing pt page.");
4995
4996 mptepa = VM_PAGE_TO_PHYS(mpte);
4997
4998 /*
4999 * Initialize the page table page.
5000 */
5001 pagezero(PHYS_TO_DMAP(mptepa));
5002
5003 /*
5004 * Demote the mapping.
5005 */
5006 pde_store(l3e, mptepa);
5007 ptesync();
5008 }
5009
5010 /*
5011 * pmap_remove_l3e: do the things to unmap a superpage in a process
5012 */
5013 static int
pmap_remove_l3e(pmap_t pmap,pml3_entry_t * pdq,vm_offset_t sva,struct spglist * free,struct rwlock ** lockp)5014 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
5015 struct spglist *free, struct rwlock **lockp)
5016 {
5017 struct md_page *pvh;
5018 pml3_entry_t oldpde;
5019 vm_offset_t eva, va;
5020 vm_page_t m, mpte;
5021
5022 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5023 KASSERT((sva & L3_PAGE_MASK) == 0,
5024 ("pmap_remove_l3e: sva is not 2mpage aligned"));
5025 oldpde = be64toh(pte_load_clear(pdq));
5026 if (oldpde & PG_W)
5027 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);
5028 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5029 if (oldpde & PG_MANAGED) {
5030 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
5031 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
5032 pmap_pvh_free(pvh, pmap, sva);
5033 eva = sva + L3_PAGE_SIZE;
5034 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
5035 va < eva; va += PAGE_SIZE, m++) {
5036 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
5037 vm_page_dirty(m);
5038 if (oldpde & PG_A)
5039 vm_page_aflag_set(m, PGA_REFERENCED);
5040 if (TAILQ_EMPTY(&m->md.pv_list) &&
5041 TAILQ_EMPTY(&pvh->pv_list))
5042 vm_page_aflag_clear(m, PGA_WRITEABLE);
5043 }
5044 }
5045 if (pmap == kernel_pmap) {
5046 pmap_remove_kernel_l3e(pmap, pdq, sva);
5047 } else {
5048 mpte = pmap_remove_pt_page(pmap, sva);
5049 if (mpte != NULL) {
5050 pmap_resident_count_dec(pmap, 1);
5051 KASSERT(mpte->ref_count == NPTEPG,
5052 ("pmap_remove_l3e: pte page wire count error"));
5053 mpte->ref_count = 0;
5054 pmap_add_delayed_free_list(mpte, free, FALSE);
5055 }
5056 }
5057 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));
5058 }
5059
5060 /*
5061 * pmap_remove_pte: do the things to unmap a page in a process
5062 */
5063 static int
pmap_remove_pte(pmap_t pmap,pt_entry_t * ptq,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free,struct rwlock ** lockp)5064 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
5065 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp)
5066 {
5067 struct md_page *pvh;
5068 pt_entry_t oldpte;
5069 vm_page_t m;
5070
5071 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5072 oldpte = be64toh(pte_load_clear(ptq));
5073 if (oldpte & RPTE_WIRED)
5074 pmap->pm_stats.wired_count -= 1;
5075 pmap_resident_count_dec(pmap, 1);
5076 if (oldpte & RPTE_MANAGED) {
5077 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
5078 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5079 vm_page_dirty(m);
5080 if (oldpte & PG_A)
5081 vm_page_aflag_set(m, PGA_REFERENCED);
5082 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
5083 pmap_pvh_free(&m->md, pmap, va);
5084 if (TAILQ_EMPTY(&m->md.pv_list) &&
5085 (m->flags & PG_FICTITIOUS) == 0) {
5086 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5087 if (TAILQ_EMPTY(&pvh->pv_list))
5088 vm_page_aflag_clear(m, PGA_WRITEABLE);
5089 }
5090 }
5091 return (pmap_unuse_pt(pmap, va, ptepde, free));
5092 }
5093
5094 /*
5095 * Remove a single page from a process address space
5096 */
5097 static bool
pmap_remove_page(pmap_t pmap,vm_offset_t va,pml3_entry_t * l3e,struct spglist * free)5098 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e,
5099 struct spglist *free)
5100 {
5101 struct rwlock *lock;
5102 pt_entry_t *pte;
5103 bool invalidate_all;
5104
5105 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5106 if ((be64toh(*l3e) & RPTE_VALID) == 0) {
5107 return (false);
5108 }
5109 pte = pmap_l3e_to_pte(l3e, va);
5110 if ((be64toh(*pte) & RPTE_VALID) == 0) {
5111 return (false);
5112 }
5113 lock = NULL;
5114
5115 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);
5116 if (lock != NULL)
5117 rw_wunlock(lock);
5118 if (!invalidate_all)
5119 pmap_invalidate_page(pmap, va);
5120 return (invalidate_all);
5121 }
5122
5123 /*
5124 * Removes the specified range of addresses from the page table page.
5125 */
5126 static bool
pmap_remove_ptes(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,pml3_entry_t * l3e,struct spglist * free,struct rwlock ** lockp)5127 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
5128 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp)
5129 {
5130 pt_entry_t *pte;
5131 vm_offset_t va;
5132 bool anyvalid;
5133
5134 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5135 anyvalid = false;
5136 va = eva;
5137 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++,
5138 sva += PAGE_SIZE) {
5139 MPASS(pte == pmap_pte(pmap, sva));
5140 if (*pte == 0) {
5141 if (va != eva) {
5142 anyvalid = true;
5143 va = eva;
5144 }
5145 continue;
5146 }
5147 if (va == eva)
5148 va = sva;
5149 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {
5150 anyvalid = true;
5151 sva += PAGE_SIZE;
5152 break;
5153 }
5154 }
5155 if (anyvalid)
5156 pmap_invalidate_all(pmap);
5157 else if (va != eva)
5158 pmap_invalidate_range(pmap, va, sva);
5159 return (anyvalid);
5160 }
5161
5162 void
mmu_radix_remove(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5163 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5164 {
5165 struct rwlock *lock;
5166 vm_offset_t va_next;
5167 pml1_entry_t *l1e;
5168 pml2_entry_t *l2e;
5169 pml3_entry_t ptpaddr, *l3e;
5170 struct spglist free;
5171 bool anyvalid;
5172
5173 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5174
5175 /*
5176 * Perform an unsynchronized read. This is, however, safe.
5177 */
5178 if (pmap->pm_stats.resident_count == 0)
5179 return;
5180
5181 anyvalid = false;
5182 SLIST_INIT(&free);
5183
5184 /* XXX something fishy here */
5185 sva = (sva + PAGE_MASK) & ~PAGE_MASK;
5186 eva = (eva + PAGE_MASK) & ~PAGE_MASK;
5187
5188 PMAP_LOCK(pmap);
5189
5190 /*
5191 * special handling of removing one page. a very
5192 * common operation and easy to short circuit some
5193 * code.
5194 */
5195 if (sva + PAGE_SIZE == eva) {
5196 l3e = pmap_pml3e(pmap, sva);
5197 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) {
5198 anyvalid = pmap_remove_page(pmap, sva, l3e, &free);
5199 goto out;
5200 }
5201 }
5202
5203 lock = NULL;
5204 for (; sva < eva; sva = va_next) {
5205 if (pmap->pm_stats.resident_count == 0)
5206 break;
5207 l1e = pmap_pml1e(pmap, sva);
5208 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) {
5209 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5210 if (va_next < sva)
5211 va_next = eva;
5212 continue;
5213 }
5214
5215 l2e = pmap_l1e_to_l2e(l1e, sva);
5216 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) {
5217 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5218 if (va_next < sva)
5219 va_next = eva;
5220 continue;
5221 }
5222
5223 /*
5224 * Calculate index for next page table.
5225 */
5226 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5227 if (va_next < sva)
5228 va_next = eva;
5229
5230 l3e = pmap_l2e_to_l3e(l2e, sva);
5231 ptpaddr = be64toh(*l3e);
5232
5233 /*
5234 * Weed out invalid mappings.
5235 */
5236 if (ptpaddr == 0)
5237 continue;
5238
5239 /*
5240 * Check for large page.
5241 */
5242 if ((ptpaddr & RPTE_LEAF) != 0) {
5243 /*
5244 * Are we removing the entire large page? If not,
5245 * demote the mapping and fall through.
5246 */
5247 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5248 pmap_remove_l3e(pmap, l3e, sva, &free, &lock);
5249 anyvalid = true;
5250 continue;
5251 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva,
5252 &lock)) {
5253 /* The large page mapping was destroyed. */
5254 continue;
5255 } else
5256 ptpaddr = be64toh(*l3e);
5257 }
5258
5259 /*
5260 * Limit our scan to either the end of the va represented
5261 * by the current page table page, or to the end of the
5262 * range being removed.
5263 */
5264 if (va_next > eva)
5265 va_next = eva;
5266
5267 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock))
5268 anyvalid = true;
5269 }
5270 if (lock != NULL)
5271 rw_wunlock(lock);
5272 out:
5273 if (anyvalid)
5274 pmap_invalidate_all(pmap);
5275 PMAP_UNLOCK(pmap);
5276 vm_page_free_pages_toq(&free, true);
5277 }
5278
5279 void
mmu_radix_remove_all(vm_page_t m)5280 mmu_radix_remove_all(vm_page_t m)
5281 {
5282 struct md_page *pvh;
5283 pv_entry_t pv;
5284 pmap_t pmap;
5285 struct rwlock *lock;
5286 pt_entry_t *pte, tpte;
5287 pml3_entry_t *l3e;
5288 vm_offset_t va;
5289 struct spglist free;
5290 int pvh_gen, md_gen;
5291
5292 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5293 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5294 ("pmap_remove_all: page %p is not managed", m));
5295 SLIST_INIT(&free);
5296 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5297 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5298 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5299 retry:
5300 rw_wlock(lock);
5301 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
5302 pmap = PV_PMAP(pv);
5303 if (!PMAP_TRYLOCK(pmap)) {
5304 pvh_gen = pvh->pv_gen;
5305 rw_wunlock(lock);
5306 PMAP_LOCK(pmap);
5307 rw_wlock(lock);
5308 if (pvh_gen != pvh->pv_gen) {
5309 rw_wunlock(lock);
5310 PMAP_UNLOCK(pmap);
5311 goto retry;
5312 }
5313 }
5314 va = pv->pv_va;
5315 l3e = pmap_pml3e(pmap, va);
5316 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock);
5317 PMAP_UNLOCK(pmap);
5318 }
5319 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
5320 pmap = PV_PMAP(pv);
5321 if (!PMAP_TRYLOCK(pmap)) {
5322 pvh_gen = pvh->pv_gen;
5323 md_gen = m->md.pv_gen;
5324 rw_wunlock(lock);
5325 PMAP_LOCK(pmap);
5326 rw_wlock(lock);
5327 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
5328 rw_wunlock(lock);
5329 PMAP_UNLOCK(pmap);
5330 goto retry;
5331 }
5332 }
5333 pmap_resident_count_dec(pmap, 1);
5334 l3e = pmap_pml3e(pmap, pv->pv_va);
5335 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found"
5336 " a 2mpage in page %p's pv list", m));
5337 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5338 tpte = be64toh(pte_load_clear(pte));
5339 if (tpte & PG_W)
5340 pmap->pm_stats.wired_count--;
5341 if (tpte & PG_A)
5342 vm_page_aflag_set(m, PGA_REFERENCED);
5343
5344 /*
5345 * Update the vm_page_t clean and reference bits.
5346 */
5347 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5348 vm_page_dirty(m);
5349 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);
5350 pmap_invalidate_page(pmap, pv->pv_va);
5351 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5352 m->md.pv_gen++;
5353 free_pv_entry(pmap, pv);
5354 PMAP_UNLOCK(pmap);
5355 }
5356 vm_page_aflag_clear(m, PGA_WRITEABLE);
5357 rw_wunlock(lock);
5358 vm_page_free_pages_toq(&free, true);
5359 }
5360
5361 /*
5362 * Destroy all managed, non-wired mappings in the given user-space
5363 * pmap. This pmap cannot be active on any processor besides the
5364 * caller.
5365 *
5366 * This function cannot be applied to the kernel pmap. Moreover, it
5367 * is not intended for general use. It is only to be used during
5368 * process termination. Consequently, it can be implemented in ways
5369 * that make it faster than pmap_remove(). First, it can more quickly
5370 * destroy mappings by iterating over the pmap's collection of PV
5371 * entries, rather than searching the page table. Second, it doesn't
5372 * have to test and clear the page table entries atomically, because
5373 * no processor is currently accessing the user address space. In
5374 * particular, a page table entry's dirty bit won't change state once
5375 * this function starts.
5376 *
5377 * Although this function destroys all of the pmap's managed,
5378 * non-wired mappings, it can delay and batch the invalidation of TLB
5379 * entries without calling pmap_delayed_invl_started() and
5380 * pmap_delayed_invl_finished(). Because the pmap is not active on
5381 * any other processor, none of these TLB entries will ever be used
5382 * before their eventual invalidation. Consequently, there is no need
5383 * for either pmap_remove_all() or pmap_remove_write() to wait for
5384 * that eventual TLB invalidation.
5385 */
5386
5387 void
mmu_radix_remove_pages(pmap_t pmap)5388 mmu_radix_remove_pages(pmap_t pmap)
5389 {
5390
5391 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
5392 pml3_entry_t ptel3e;
5393 pt_entry_t *pte, tpte;
5394 struct spglist free;
5395 vm_page_t m, mpte, mt;
5396 pv_entry_t pv;
5397 struct md_page *pvh;
5398 struct pv_chunk *pc, *npc;
5399 struct rwlock *lock;
5400 int64_t bit;
5401 uint64_t inuse, bitmask;
5402 int allfree, field, idx;
5403 #ifdef PV_STATS
5404 int freed;
5405 #endif
5406 boolean_t superpage;
5407 vm_paddr_t pa;
5408
5409 /*
5410 * Assert that the given pmap is only active on the current
5411 * CPU. Unfortunately, we cannot block another CPU from
5412 * activating the pmap while this function is executing.
5413 */
5414 KASSERT(pmap->pm_pid == mfspr(SPR_PID),
5415 ("non-current asid %lu - expected %lu", pmap->pm_pid,
5416 mfspr(SPR_PID)));
5417
5418 lock = NULL;
5419
5420 SLIST_INIT(&free);
5421 PMAP_LOCK(pmap);
5422 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
5423 allfree = 1;
5424 #ifdef PV_STATS
5425 freed = 0;
5426 #endif
5427 for (field = 0; field < _NPCM; field++) {
5428 inuse = ~pc->pc_map[field] & pc_freemask[field];
5429 while (inuse != 0) {
5430 bit = cnttzd(inuse);
5431 bitmask = 1UL << bit;
5432 idx = field * 64 + bit;
5433 pv = &pc->pc_pventry[idx];
5434 inuse &= ~bitmask;
5435
5436 pte = pmap_pml2e(pmap, pv->pv_va);
5437 ptel3e = be64toh(*pte);
5438 pte = pmap_l2e_to_l3e(pte, pv->pv_va);
5439 tpte = be64toh(*pte);
5440 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) {
5441 superpage = FALSE;
5442 ptel3e = tpte;
5443 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte &
5444 PG_FRAME);
5445 pte = &pte[pmap_pte_index(pv->pv_va)];
5446 tpte = be64toh(*pte);
5447 } else {
5448 /*
5449 * Keep track whether 'tpte' is a
5450 * superpage explicitly instead of
5451 * relying on RPTE_LEAF being set.
5452 *
5453 * This is because RPTE_LEAF is numerically
5454 * identical to PG_PTE_PAT and thus a
5455 * regular page could be mistaken for
5456 * a superpage.
5457 */
5458 superpage = TRUE;
5459 }
5460
5461 if ((tpte & PG_V) == 0) {
5462 panic("bad pte va %lx pte %lx",
5463 pv->pv_va, tpte);
5464 }
5465
5466 /*
5467 * We cannot remove wired pages from a process' mapping at this time
5468 */
5469 if (tpte & PG_W) {
5470 allfree = 0;
5471 continue;
5472 }
5473
5474 if (superpage)
5475 pa = tpte & PG_PS_FRAME;
5476 else
5477 pa = tpte & PG_FRAME;
5478
5479 m = PHYS_TO_VM_PAGE(pa);
5480 KASSERT(m->phys_addr == pa,
5481 ("vm_page_t %p phys_addr mismatch %016jx %016jx",
5482 m, (uintmax_t)m->phys_addr,
5483 (uintmax_t)tpte));
5484
5485 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
5486 m < &vm_page_array[vm_page_array_size],
5487 ("pmap_remove_pages: bad tpte %#jx",
5488 (uintmax_t)tpte));
5489
5490 pte_clear(pte);
5491
5492 /*
5493 * Update the vm_page_t clean/reference bits.
5494 */
5495 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5496 if (superpage) {
5497 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5498 vm_page_dirty(mt);
5499 } else
5500 vm_page_dirty(m);
5501 }
5502
5503 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
5504
5505 /* Mark free */
5506 pc->pc_map[field] |= bitmask;
5507 if (superpage) {
5508 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5509 pvh = pa_to_pvh(tpte & PG_PS_FRAME);
5510 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
5511 pvh->pv_gen++;
5512 if (TAILQ_EMPTY(&pvh->pv_list)) {
5513 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5514 if ((mt->a.flags & PGA_WRITEABLE) != 0 &&
5515 TAILQ_EMPTY(&mt->md.pv_list))
5516 vm_page_aflag_clear(mt, PGA_WRITEABLE);
5517 }
5518 mpte = pmap_remove_pt_page(pmap, pv->pv_va);
5519 if (mpte != NULL) {
5520 pmap_resident_count_dec(pmap, 1);
5521 KASSERT(mpte->ref_count == NPTEPG,
5522 ("pmap_remove_pages: pte page wire count error"));
5523 mpte->ref_count = 0;
5524 pmap_add_delayed_free_list(mpte, &free, FALSE);
5525 }
5526 } else {
5527 pmap_resident_count_dec(pmap, 1);
5528 #ifdef VERBOSE_PV
5529 printf("freeing pv (%p, %p)\n",
5530 pmap, pv);
5531 #endif
5532 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5533 m->md.pv_gen++;
5534 if ((m->a.flags & PGA_WRITEABLE) != 0 &&
5535 TAILQ_EMPTY(&m->md.pv_list) &&
5536 (m->flags & PG_FICTITIOUS) == 0) {
5537 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5538 if (TAILQ_EMPTY(&pvh->pv_list))
5539 vm_page_aflag_clear(m, PGA_WRITEABLE);
5540 }
5541 }
5542 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);
5543 #ifdef PV_STATS
5544 freed++;
5545 #endif
5546 }
5547 }
5548 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
5549 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
5550 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
5551 if (allfree) {
5552 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
5553 free_pv_chunk(pc);
5554 }
5555 }
5556 if (lock != NULL)
5557 rw_wunlock(lock);
5558 pmap_invalidate_all(pmap);
5559 PMAP_UNLOCK(pmap);
5560 vm_page_free_pages_toq(&free, true);
5561 }
5562
5563 void
mmu_radix_remove_write(vm_page_t m)5564 mmu_radix_remove_write(vm_page_t m)
5565 {
5566 struct md_page *pvh;
5567 pmap_t pmap;
5568 struct rwlock *lock;
5569 pv_entry_t next_pv, pv;
5570 pml3_entry_t *l3e;
5571 pt_entry_t oldpte, *pte;
5572 int pvh_gen, md_gen;
5573
5574 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5575 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5576 ("pmap_remove_write: page %p is not managed", m));
5577 vm_page_assert_busied(m);
5578
5579 if (!pmap_page_is_write_mapped(m))
5580 return;
5581 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5582 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5583 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5584 retry_pv_loop:
5585 rw_wlock(lock);
5586 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
5587 pmap = PV_PMAP(pv);
5588 if (!PMAP_TRYLOCK(pmap)) {
5589 pvh_gen = pvh->pv_gen;
5590 rw_wunlock(lock);
5591 PMAP_LOCK(pmap);
5592 rw_wlock(lock);
5593 if (pvh_gen != pvh->pv_gen) {
5594 PMAP_UNLOCK(pmap);
5595 rw_wunlock(lock);
5596 goto retry_pv_loop;
5597 }
5598 }
5599 l3e = pmap_pml3e(pmap, pv->pv_va);
5600 if ((be64toh(*l3e) & PG_RW) != 0)
5601 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);
5602 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
5603 ("inconsistent pv lock %p %p for page %p",
5604 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
5605 PMAP_UNLOCK(pmap);
5606 }
5607 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
5608 pmap = PV_PMAP(pv);
5609 if (!PMAP_TRYLOCK(pmap)) {
5610 pvh_gen = pvh->pv_gen;
5611 md_gen = m->md.pv_gen;
5612 rw_wunlock(lock);
5613 PMAP_LOCK(pmap);
5614 rw_wlock(lock);
5615 if (pvh_gen != pvh->pv_gen ||
5616 md_gen != m->md.pv_gen) {
5617 PMAP_UNLOCK(pmap);
5618 rw_wunlock(lock);
5619 goto retry_pv_loop;
5620 }
5621 }
5622 l3e = pmap_pml3e(pmap, pv->pv_va);
5623 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
5624 ("pmap_remove_write: found a 2mpage in page %p's pv list",
5625 m));
5626 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5627 retry:
5628 oldpte = be64toh(*pte);
5629 if (oldpte & PG_RW) {
5630 if (!atomic_cmpset_long(pte, htobe64(oldpte),
5631 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M))))
5632 goto retry;
5633 if ((oldpte & PG_M) != 0)
5634 vm_page_dirty(m);
5635 pmap_invalidate_page(pmap, pv->pv_va);
5636 }
5637 PMAP_UNLOCK(pmap);
5638 }
5639 rw_wunlock(lock);
5640 vm_page_aflag_clear(m, PGA_WRITEABLE);
5641 }
5642
5643 /*
5644 * Clear the wired attribute from the mappings for the specified range of
5645 * addresses in the given pmap. Every valid mapping within that range
5646 * must have the wired attribute set. In contrast, invalid mappings
5647 * cannot have the wired attribute set, so they are ignored.
5648 *
5649 * The wired attribute of the page table entry is not a hardware
5650 * feature, so there is no need to invalidate any TLB entries.
5651 * Since pmap_demote_l3e() for the wired entry must never fail,
5652 * pmap_delayed_invl_started()/finished() calls around the
5653 * function are not needed.
5654 */
5655 void
mmu_radix_unwire(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5656 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5657 {
5658 vm_offset_t va_next;
5659 pml1_entry_t *l1e;
5660 pml2_entry_t *l2e;
5661 pml3_entry_t *l3e;
5662 pt_entry_t *pte;
5663
5664 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5665 PMAP_LOCK(pmap);
5666 for (; sva < eva; sva = va_next) {
5667 l1e = pmap_pml1e(pmap, sva);
5668 if ((be64toh(*l1e) & PG_V) == 0) {
5669 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5670 if (va_next < sva)
5671 va_next = eva;
5672 continue;
5673 }
5674 l2e = pmap_l1e_to_l2e(l1e, sva);
5675 if ((be64toh(*l2e) & PG_V) == 0) {
5676 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5677 if (va_next < sva)
5678 va_next = eva;
5679 continue;
5680 }
5681 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5682 if (va_next < sva)
5683 va_next = eva;
5684 l3e = pmap_l2e_to_l3e(l2e, sva);
5685 if ((be64toh(*l3e) & PG_V) == 0)
5686 continue;
5687 if ((be64toh(*l3e) & RPTE_LEAF) != 0) {
5688 if ((be64toh(*l3e) & PG_W) == 0)
5689 panic("pmap_unwire: pde %#jx is missing PG_W",
5690 (uintmax_t)(be64toh(*l3e)));
5691
5692 /*
5693 * Are we unwiring the entire large page? If not,
5694 * demote the mapping and fall through.
5695 */
5696 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5697 atomic_clear_long(l3e, htobe64(PG_W));
5698 pmap->pm_stats.wired_count -= L3_PAGE_SIZE /
5699 PAGE_SIZE;
5700 continue;
5701 } else if (!pmap_demote_l3e(pmap, l3e, sva))
5702 panic("pmap_unwire: demotion failed");
5703 }
5704 if (va_next > eva)
5705 va_next = eva;
5706 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
5707 sva += PAGE_SIZE) {
5708 MPASS(pte == pmap_pte(pmap, sva));
5709 if ((be64toh(*pte) & PG_V) == 0)
5710 continue;
5711 if ((be64toh(*pte) & PG_W) == 0)
5712 panic("pmap_unwire: pte %#jx is missing PG_W",
5713 (uintmax_t)(be64toh(*pte)));
5714
5715 /*
5716 * PG_W must be cleared atomically. Although the pmap
5717 * lock synchronizes access to PG_W, another processor
5718 * could be setting PG_M and/or PG_A concurrently.
5719 */
5720 atomic_clear_long(pte, htobe64(PG_W));
5721 pmap->pm_stats.wired_count--;
5722 }
5723 }
5724 PMAP_UNLOCK(pmap);
5725 }
5726
5727 void
mmu_radix_zero_page(vm_page_t m)5728 mmu_radix_zero_page(vm_page_t m)
5729 {
5730 vm_offset_t addr;
5731
5732 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5733 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
5734 pagezero(addr);
5735 }
5736
5737 void
mmu_radix_zero_page_area(vm_page_t m,int off,int size)5738 mmu_radix_zero_page_area(vm_page_t m, int off, int size)
5739 {
5740 caddr_t addr;
5741
5742 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size);
5743 MPASS(off + size <= PAGE_SIZE);
5744 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
5745 memset(addr + off, 0, size);
5746 }
5747
5748 static int
mmu_radix_mincore(pmap_t pmap,vm_offset_t addr,vm_paddr_t * locked_pa)5749 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
5750 {
5751 pml3_entry_t *l3ep;
5752 pt_entry_t pte;
5753 vm_paddr_t pa;
5754 int val;
5755
5756 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
5757 PMAP_LOCK(pmap);
5758
5759 l3ep = pmap_pml3e(pmap, addr);
5760 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) {
5761 if (be64toh(*l3ep) & RPTE_LEAF) {
5762 pte = be64toh(*l3ep);
5763 /* Compute the physical address of the 4KB page. */
5764 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) &
5765 PG_FRAME;
5766 val = MINCORE_PSIND(1);
5767 } else {
5768 /* Native endian PTE, do not pass to functions */
5769 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr));
5770 pa = pte & PG_FRAME;
5771 val = 0;
5772 }
5773 } else {
5774 pte = 0;
5775 pa = 0;
5776 val = 0;
5777 }
5778 if ((pte & PG_V) != 0) {
5779 val |= MINCORE_INCORE;
5780 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5781 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
5782 if ((pte & PG_A) != 0)
5783 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
5784 }
5785 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
5786 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
5787 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
5788 *locked_pa = pa;
5789 }
5790 PMAP_UNLOCK(pmap);
5791 return (val);
5792 }
5793
5794 void
mmu_radix_activate(struct thread * td)5795 mmu_radix_activate(struct thread *td)
5796 {
5797 pmap_t pmap;
5798 uint32_t curpid;
5799
5800 CTR2(KTR_PMAP, "%s(%p)", __func__, td);
5801 critical_enter();
5802 pmap = vmspace_pmap(td->td_proc->p_vmspace);
5803 curpid = mfspr(SPR_PID);
5804 if (pmap->pm_pid > isa3_base_pid &&
5805 curpid != pmap->pm_pid) {
5806 mmu_radix_pid_set(pmap);
5807 }
5808 critical_exit();
5809 }
5810
5811 /*
5812 * Increase the starting virtual address of the given mapping if a
5813 * different alignment might result in more superpage mappings.
5814 */
5815 void
mmu_radix_align_superpage(vm_object_t object,vm_ooffset_t offset,vm_offset_t * addr,vm_size_t size)5816 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset,
5817 vm_offset_t *addr, vm_size_t size)
5818 {
5819
5820 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr,
5821 size);
5822 vm_offset_t superpage_offset;
5823
5824 if (size < L3_PAGE_SIZE)
5825 return;
5826 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5827 offset += ptoa(object->pg_color);
5828 superpage_offset = offset & L3_PAGE_MASK;
5829 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE ||
5830 (*addr & L3_PAGE_MASK) == superpage_offset)
5831 return;
5832 if ((*addr & L3_PAGE_MASK) < superpage_offset)
5833 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset;
5834 else
5835 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset;
5836 }
5837
5838 static void *
mmu_radix_mapdev_attr(vm_paddr_t pa,vm_size_t size,vm_memattr_t attr)5839 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr)
5840 {
5841 vm_offset_t va, tmpva, ppa, offset;
5842
5843 ppa = trunc_page(pa);
5844 offset = pa & PAGE_MASK;
5845 size = roundup2(offset + size, PAGE_SIZE);
5846 if (pa < powerpc_ptob(Maxmem))
5847 panic("bad pa: %#lx less than Maxmem %#lx\n",
5848 pa, powerpc_ptob(Maxmem));
5849 va = kva_alloc(size);
5850 if (bootverbose)
5851 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr);
5852 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr));
5853
5854 if (!va)
5855 panic("%s: Couldn't alloc kernel virtual memory", __func__);
5856
5857 for (tmpva = va; size > 0;) {
5858 mmu_radix_kenter_attr(tmpva, ppa, attr);
5859 size -= PAGE_SIZE;
5860 tmpva += PAGE_SIZE;
5861 ppa += PAGE_SIZE;
5862 }
5863 ptesync();
5864
5865 return ((void *)(va + offset));
5866 }
5867
5868 static void *
mmu_radix_mapdev(vm_paddr_t pa,vm_size_t size)5869 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size)
5870 {
5871
5872 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
5873
5874 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT));
5875 }
5876
5877 void
mmu_radix_page_set_memattr(vm_page_t m,vm_memattr_t ma)5878 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma)
5879 {
5880
5881 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma);
5882 m->md.mdpg_cache_attrs = ma;
5883
5884 /*
5885 * If "m" is a normal page, update its direct mapping. This update
5886 * can be relied upon to perform any cache operations that are
5887 * required for data coherence.
5888 */
5889 if ((m->flags & PG_FICTITIOUS) == 0 &&
5890 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)),
5891 PAGE_SIZE, m->md.mdpg_cache_attrs))
5892 panic("memory attribute change on the direct map failed");
5893 }
5894
5895 static void
mmu_radix_unmapdev(vm_offset_t va,vm_size_t size)5896 mmu_radix_unmapdev(vm_offset_t va, vm_size_t size)
5897 {
5898 vm_offset_t offset;
5899
5900 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size);
5901 /* If we gave a direct map region in pmap_mapdev, do nothing */
5902 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
5903 return;
5904
5905 offset = va & PAGE_MASK;
5906 size = round_page(offset + size);
5907 va = trunc_page(va);
5908
5909 if (pmap_initialized) {
5910 mmu_radix_qremove(va, atop(size));
5911 kva_free(va, size);
5912 }
5913 }
5914
5915 void
mmu_radix_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)5916 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
5917 {
5918 vm_paddr_t pa = 0;
5919 int sync_sz;
5920
5921 while (sz > 0) {
5922 pa = pmap_extract(pm, va);
5923 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
5924 sync_sz = min(sync_sz, sz);
5925 if (pa != 0) {
5926 pa += (va & PAGE_MASK);
5927 __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz);
5928 }
5929 va += sync_sz;
5930 sz -= sync_sz;
5931 }
5932 }
5933
5934 static __inline void
pmap_pte_attr(pt_entry_t * pte,uint64_t cache_bits,uint64_t mask)5935 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask)
5936 {
5937 uint64_t opte, npte;
5938
5939 /*
5940 * The cache mode bits are all in the low 32-bits of the
5941 * PTE, so we can just spin on updating the low 32-bits.
5942 */
5943 do {
5944 opte = be64toh(*pte);
5945 npte = opte & ~mask;
5946 npte |= cache_bits;
5947 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte)));
5948 }
5949
5950 /*
5951 * Tries to demote a 1GB page mapping.
5952 */
5953 static boolean_t
pmap_demote_l2e(pmap_t pmap,pml2_entry_t * l2e,vm_offset_t va)5954 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va)
5955 {
5956 pml2_entry_t oldpdpe;
5957 pml3_entry_t *firstpde, newpde, *pde;
5958 vm_paddr_t pdpgpa;
5959 vm_page_t pdpg;
5960
5961 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5962 oldpdpe = be64toh(*l2e);
5963 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
5964 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
5965 pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED);
5966 if (pdpg == NULL) {
5967 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
5968 " in pmap %p", va, pmap);
5969 return (FALSE);
5970 }
5971 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT;
5972 pdpgpa = VM_PAGE_TO_PHYS(pdpg);
5973 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa);
5974 KASSERT((oldpdpe & PG_A) != 0,
5975 ("pmap_demote_pdpe: oldpdpe is missing PG_A"));
5976 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW,
5977 ("pmap_demote_pdpe: oldpdpe is missing PG_M"));
5978 newpde = oldpdpe;
5979
5980 /*
5981 * Initialize the page directory page.
5982 */
5983 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) {
5984 *pde = htobe64(newpde);
5985 newpde += L3_PAGE_SIZE;
5986 }
5987
5988 /*
5989 * Demote the mapping.
5990 */
5991 pde_store(l2e, pdpgpa);
5992
5993 /*
5994 * Flush PWC --- XXX revisit
5995 */
5996 pmap_invalidate_all(pmap);
5997
5998 pmap_l2e_demotions++;
5999 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
6000 " in pmap %p", va, pmap);
6001 return (TRUE);
6002 }
6003
6004 vm_paddr_t
mmu_radix_kextract(vm_offset_t va)6005 mmu_radix_kextract(vm_offset_t va)
6006 {
6007 pml3_entry_t l3e;
6008 vm_paddr_t pa;
6009
6010 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6011 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
6012 pa = DMAP_TO_PHYS(va);
6013 } else {
6014 /* Big-endian PTE on stack */
6015 l3e = *pmap_pml3e(kernel_pmap, va);
6016 if (be64toh(l3e) & RPTE_LEAF) {
6017 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
6018 pa |= (va & L3_PAGE_MASK);
6019 } else {
6020 /*
6021 * Beware of a concurrent promotion that changes the
6022 * PDE at this point! For example, vtopte() must not
6023 * be used to access the PTE because it would use the
6024 * new PDE. It is, however, safe to use the old PDE
6025 * because the page table page is preserved by the
6026 * promotion.
6027 */
6028 pa = be64toh(*pmap_l3e_to_pte(&l3e, va));
6029 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
6030 pa |= (va & PAGE_MASK);
6031 }
6032 }
6033 return (pa);
6034 }
6035
6036 static pt_entry_t
mmu_radix_calc_wimg(vm_paddr_t pa,vm_memattr_t ma)6037 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
6038 {
6039
6040 if (ma != VM_MEMATTR_DEFAULT) {
6041 return pmap_cache_bits(ma);
6042 }
6043
6044 /*
6045 * Assume the page is cache inhibited and access is guarded unless
6046 * it's in our available memory array.
6047 */
6048 for (int i = 0; i < pregions_sz; i++) {
6049 if ((pa >= pregions[i].mr_start) &&
6050 (pa < (pregions[i].mr_start + pregions[i].mr_size)))
6051 return (RPTE_ATTR_MEM);
6052 }
6053 return (RPTE_ATTR_GUARDEDIO);
6054 }
6055
6056 static void
mmu_radix_kenter_attr(vm_offset_t va,vm_paddr_t pa,vm_memattr_t ma)6057 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
6058 {
6059 pt_entry_t *pte, pteval;
6060 uint64_t cache_bits;
6061
6062 pte = kvtopte(va);
6063 MPASS(pte != NULL);
6064 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
6065 cache_bits = mmu_radix_calc_wimg(pa, ma);
6066 pte_store(pte, pteval | cache_bits);
6067 }
6068
6069 void
mmu_radix_kremove(vm_offset_t va)6070 mmu_radix_kremove(vm_offset_t va)
6071 {
6072 pt_entry_t *pte;
6073
6074 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6075
6076 pte = kvtopte(va);
6077 pte_clear(pte);
6078 }
6079
6080 int
mmu_radix_decode_kernel_ptr(vm_offset_t addr,int * is_user,vm_offset_t * decoded)6081 mmu_radix_decode_kernel_ptr(vm_offset_t addr,
6082 int *is_user, vm_offset_t *decoded)
6083 {
6084
6085 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr);
6086 *decoded = addr;
6087 *is_user = (addr < VM_MAXUSER_ADDRESS);
6088 return (0);
6089 }
6090
6091 static int
mmu_radix_dev_direct_mapped(vm_paddr_t pa,vm_size_t size)6092 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
6093 {
6094
6095 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
6096 return (mem_valid(pa, size));
6097 }
6098
6099 static void
mmu_radix_scan_init(void)6100 mmu_radix_scan_init(void)
6101 {
6102
6103 CTR1(KTR_PMAP, "%s()", __func__);
6104 UNIMPLEMENTED();
6105 }
6106
6107 static void
mmu_radix_dumpsys_map(vm_paddr_t pa,size_t sz,void ** va)6108 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz,
6109 void **va)
6110 {
6111 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va);
6112 UNIMPLEMENTED();
6113 }
6114
6115 vm_offset_t
mmu_radix_quick_enter_page(vm_page_t m)6116 mmu_radix_quick_enter_page(vm_page_t m)
6117 {
6118 vm_paddr_t paddr;
6119
6120 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
6121 paddr = VM_PAGE_TO_PHYS(m);
6122 return (PHYS_TO_DMAP(paddr));
6123 }
6124
6125 void
mmu_radix_quick_remove_page(vm_offset_t addr __unused)6126 mmu_radix_quick_remove_page(vm_offset_t addr __unused)
6127 {
6128 /* no work to do here */
6129 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
6130 }
6131
6132 static void
pmap_invalidate_cache_range(vm_offset_t sva,vm_offset_t eva)6133 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
6134 {
6135 cpu_flush_dcache((void *)sva, eva - sva);
6136 }
6137
6138 int
mmu_radix_change_attr(vm_offset_t va,vm_size_t size,vm_memattr_t mode)6139 mmu_radix_change_attr(vm_offset_t va, vm_size_t size,
6140 vm_memattr_t mode)
6141 {
6142 int error;
6143
6144 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode);
6145 PMAP_LOCK(kernel_pmap);
6146 error = pmap_change_attr_locked(va, size, mode, true);
6147 PMAP_UNLOCK(kernel_pmap);
6148 return (error);
6149 }
6150
6151 static int
pmap_change_attr_locked(vm_offset_t va,vm_size_t size,int mode,bool flush)6152 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush)
6153 {
6154 vm_offset_t base, offset, tmpva;
6155 vm_paddr_t pa_start, pa_end, pa_end1;
6156 pml2_entry_t *l2e;
6157 pml3_entry_t *l3e;
6158 pt_entry_t *pte;
6159 int cache_bits, error;
6160 boolean_t changed;
6161
6162 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
6163 base = trunc_page(va);
6164 offset = va & PAGE_MASK;
6165 size = round_page(offset + size);
6166
6167 /*
6168 * Only supported on kernel virtual addresses, including the direct
6169 * map but excluding the recursive map.
6170 */
6171 if (base < DMAP_MIN_ADDRESS)
6172 return (EINVAL);
6173
6174 cache_bits = pmap_cache_bits(mode);
6175 changed = FALSE;
6176
6177 /*
6178 * Pages that aren't mapped aren't supported. Also break down 2MB pages
6179 * into 4KB pages if required.
6180 */
6181 for (tmpva = base; tmpva < base + size; ) {
6182 l2e = pmap_pml2e(kernel_pmap, tmpva);
6183 if (l2e == NULL || *l2e == 0)
6184 return (EINVAL);
6185 if (be64toh(*l2e) & RPTE_LEAF) {
6186 /*
6187 * If the current 1GB page already has the required
6188 * memory type, then we need not demote this page. Just
6189 * increment tmpva to the next 1GB page frame.
6190 */
6191 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) {
6192 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6193 continue;
6194 }
6195
6196 /*
6197 * If the current offset aligns with a 1GB page frame
6198 * and there is at least 1GB left within the range, then
6199 * we need not break down this page into 2MB pages.
6200 */
6201 if ((tmpva & L2_PAGE_MASK) == 0 &&
6202 tmpva + L2_PAGE_MASK < base + size) {
6203 tmpva += L2_PAGE_MASK;
6204 continue;
6205 }
6206 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva))
6207 return (ENOMEM);
6208 }
6209 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6210 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n",
6211 tmpva, l2e));
6212 if (*l3e == 0)
6213 return (EINVAL);
6214 if (be64toh(*l3e) & RPTE_LEAF) {
6215 /*
6216 * If the current 2MB page already has the required
6217 * memory type, then we need not demote this page. Just
6218 * increment tmpva to the next 2MB page frame.
6219 */
6220 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) {
6221 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6222 continue;
6223 }
6224
6225 /*
6226 * If the current offset aligns with a 2MB page frame
6227 * and there is at least 2MB left within the range, then
6228 * we need not break down this page into 4KB pages.
6229 */
6230 if ((tmpva & L3_PAGE_MASK) == 0 &&
6231 tmpva + L3_PAGE_MASK < base + size) {
6232 tmpva += L3_PAGE_SIZE;
6233 continue;
6234 }
6235 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva))
6236 return (ENOMEM);
6237 }
6238 pte = pmap_l3e_to_pte(l3e, tmpva);
6239 if (*pte == 0)
6240 return (EINVAL);
6241 tmpva += PAGE_SIZE;
6242 }
6243 error = 0;
6244
6245 /*
6246 * Ok, all the pages exist, so run through them updating their
6247 * cache mode if required.
6248 */
6249 pa_start = pa_end = 0;
6250 for (tmpva = base; tmpva < base + size; ) {
6251 l2e = pmap_pml2e(kernel_pmap, tmpva);
6252 if (be64toh(*l2e) & RPTE_LEAF) {
6253 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) {
6254 pmap_pte_attr(l2e, cache_bits,
6255 RPTE_ATTR_MASK);
6256 changed = TRUE;
6257 }
6258 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6259 (*l2e & PG_PS_FRAME) < dmaplimit) {
6260 if (pa_start == pa_end) {
6261 /* Start physical address run. */
6262 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6263 pa_end = pa_start + L2_PAGE_SIZE;
6264 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME))
6265 pa_end += L2_PAGE_SIZE;
6266 else {
6267 /* Run ended, update direct map. */
6268 error = pmap_change_attr_locked(
6269 PHYS_TO_DMAP(pa_start),
6270 pa_end - pa_start, mode, flush);
6271 if (error != 0)
6272 break;
6273 /* Start physical address run. */
6274 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6275 pa_end = pa_start + L2_PAGE_SIZE;
6276 }
6277 }
6278 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6279 continue;
6280 }
6281 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6282 if (be64toh(*l3e) & RPTE_LEAF) {
6283 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) {
6284 pmap_pte_attr(l3e, cache_bits,
6285 RPTE_ATTR_MASK);
6286 changed = TRUE;
6287 }
6288 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6289 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) {
6290 if (pa_start == pa_end) {
6291 /* Start physical address run. */
6292 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6293 pa_end = pa_start + L3_PAGE_SIZE;
6294 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME))
6295 pa_end += L3_PAGE_SIZE;
6296 else {
6297 /* Run ended, update direct map. */
6298 error = pmap_change_attr_locked(
6299 PHYS_TO_DMAP(pa_start),
6300 pa_end - pa_start, mode, flush);
6301 if (error != 0)
6302 break;
6303 /* Start physical address run. */
6304 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6305 pa_end = pa_start + L3_PAGE_SIZE;
6306 }
6307 }
6308 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6309 } else {
6310 pte = pmap_l3e_to_pte(l3e, tmpva);
6311 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) {
6312 pmap_pte_attr(pte, cache_bits,
6313 RPTE_ATTR_MASK);
6314 changed = TRUE;
6315 }
6316 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6317 (be64toh(*pte) & PG_FRAME) < dmaplimit) {
6318 if (pa_start == pa_end) {
6319 /* Start physical address run. */
6320 pa_start = be64toh(*pte) & PG_FRAME;
6321 pa_end = pa_start + PAGE_SIZE;
6322 } else if (pa_end == (be64toh(*pte) & PG_FRAME))
6323 pa_end += PAGE_SIZE;
6324 else {
6325 /* Run ended, update direct map. */
6326 error = pmap_change_attr_locked(
6327 PHYS_TO_DMAP(pa_start),
6328 pa_end - pa_start, mode, flush);
6329 if (error != 0)
6330 break;
6331 /* Start physical address run. */
6332 pa_start = be64toh(*pte) & PG_FRAME;
6333 pa_end = pa_start + PAGE_SIZE;
6334 }
6335 }
6336 tmpva += PAGE_SIZE;
6337 }
6338 }
6339 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) {
6340 pa_end1 = MIN(pa_end, dmaplimit);
6341 if (pa_start != pa_end1)
6342 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start),
6343 pa_end1 - pa_start, mode, flush);
6344 }
6345
6346 /*
6347 * Flush CPU caches if required to make sure any data isn't cached that
6348 * shouldn't be, etc.
6349 */
6350 if (changed) {
6351 pmap_invalidate_all(kernel_pmap);
6352
6353 if (flush)
6354 pmap_invalidate_cache_range(base, tmpva);
6355 }
6356 return (error);
6357 }
6358
6359 /*
6360 * Allocate physical memory for the vm_page array and map it into KVA,
6361 * attempting to back the vm_pages with domain-local memory.
6362 */
6363 void
mmu_radix_page_array_startup(long pages)6364 mmu_radix_page_array_startup(long pages)
6365 {
6366 #ifdef notyet
6367 pml2_entry_t *l2e;
6368 pml3_entry_t *pde;
6369 pml3_entry_t newl3;
6370 vm_offset_t va;
6371 long pfn;
6372 int domain, i;
6373 #endif
6374 vm_paddr_t pa;
6375 vm_offset_t start, end;
6376
6377 vm_page_array_size = pages;
6378
6379 start = VM_MIN_KERNEL_ADDRESS;
6380 end = start + pages * sizeof(struct vm_page);
6381
6382 pa = vm_phys_early_alloc(0, end - start);
6383
6384 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT);
6385 #ifdef notyet
6386 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */
6387 for (va = start; va < end; va += L3_PAGE_SIZE) {
6388 pfn = first_page + (va - start) / sizeof(struct vm_page);
6389 domain = vm_phys_domain(ptoa(pfn));
6390 l2e = pmap_pml2e(kernel_pmap, va);
6391 if ((be64toh(*l2e) & PG_V) == 0) {
6392 pa = vm_phys_early_alloc(domain, PAGE_SIZE);
6393 dump_add_page(pa);
6394 pagezero(PHYS_TO_DMAP(pa));
6395 pde_store(l2e, (pml2_entry_t)pa);
6396 }
6397 pde = pmap_l2e_to_l3e(l2e, va);
6398 if ((be64toh(*pde) & PG_V) != 0)
6399 panic("Unexpected pde %p", pde);
6400 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE);
6401 for (i = 0; i < NPDEPG; i++)
6402 dump_add_page(pa + i * PAGE_SIZE);
6403 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W);
6404 pte_store(pde, newl3);
6405 }
6406 #endif
6407 vm_page_array = (vm_page_t)start;
6408 }
6409
6410 #ifdef DDB
6411 #include <sys/kdb.h>
6412 #include <ddb/ddb.h>
6413
6414 static void
pmap_pte_walk(pml1_entry_t * l1,vm_offset_t va)6415 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va)
6416 {
6417 pml1_entry_t *l1e;
6418 pml2_entry_t *l2e;
6419 pml3_entry_t *l3e;
6420 pt_entry_t *pte;
6421
6422 l1e = &l1[pmap_pml1e_index(va)];
6423 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e));
6424 if ((be64toh(*l1e) & PG_V) == 0) {
6425 db_printf("\n");
6426 return;
6427 }
6428 l2e = pmap_l1e_to_l2e(l1e, va);
6429 db_printf(" l2e %#016lx", be64toh(*l2e));
6430 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) {
6431 db_printf("\n");
6432 return;
6433 }
6434 l3e = pmap_l2e_to_l3e(l2e, va);
6435 db_printf(" l3e %#016lx", be64toh(*l3e));
6436 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) {
6437 db_printf("\n");
6438 return;
6439 }
6440 pte = pmap_l3e_to_pte(l3e, va);
6441 db_printf(" pte %#016lx\n", be64toh(*pte));
6442 }
6443
6444 void
pmap_page_print_mappings(vm_page_t m)6445 pmap_page_print_mappings(vm_page_t m)
6446 {
6447 pmap_t pmap;
6448 pv_entry_t pv;
6449
6450 db_printf("page %p(%lx)\n", m, m->phys_addr);
6451 /* need to elide locks if running in ddb */
6452 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
6453 db_printf("pv: %p ", pv);
6454 db_printf("va: %#016lx ", pv->pv_va);
6455 pmap = PV_PMAP(pv);
6456 db_printf("pmap %p ", pmap);
6457 if (pmap != NULL) {
6458 db_printf("asid: %lu\n", pmap->pm_pid);
6459 pmap_pte_walk(pmap->pm_pml1, pv->pv_va);
6460 }
6461 }
6462 }
6463
DB_SHOW_COMMAND(pte,pmap_print_pte)6464 DB_SHOW_COMMAND(pte, pmap_print_pte)
6465 {
6466 vm_offset_t va;
6467 pmap_t pmap;
6468
6469 if (!have_addr) {
6470 db_printf("show pte addr\n");
6471 return;
6472 }
6473 va = (vm_offset_t)addr;
6474
6475 if (va >= DMAP_MIN_ADDRESS)
6476 pmap = kernel_pmap;
6477 else if (kdb_thread != NULL)
6478 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);
6479 else
6480 pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
6481
6482 pmap_pte_walk(pmap->pm_pml1, va);
6483 }
6484
6485 #endif
6486