1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2020 Justin Hibbits
5 * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
6 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * Some hw specific parts of this pmap were derived or influenced
30 * by NetBSD's ibm4xx pmap module. More generic code is shared with
31 * a few other pmap modules from the FreeBSD tree.
32 */
33
34 /*
35 * VM layout notes:
36 *
37 * Kernel and user threads run within one common virtual address space
38 * defined by AS=0.
39 *
40 * 32-bit pmap:
41 * Virtual address space layout:
42 * -----------------------------
43 * 0x0000_0000 - 0x7fff_ffff : user process
44 * 0x8000_0000 - 0xbfff_ffff : pmap_mapdev()-ed area (PCI/PCIE etc.)
45 * 0xc000_0000 - 0xffff_efff : KVA
46 */
47
48 #include <sys/cdefs.h>
49 #include "opt_ddb.h"
50 #include "opt_kstack_pages.h"
51
52 #include <sys/param.h>
53 #include <sys/conf.h>
54 #include <sys/malloc.h>
55 #include <sys/ktr.h>
56 #include <sys/proc.h>
57 #include <sys/user.h>
58 #include <sys/queue.h>
59 #include <sys/systm.h>
60 #include <sys/kernel.h>
61 #include <sys/kerneldump.h>
62 #include <sys/linker.h>
63 #include <sys/msgbuf.h>
64 #include <sys/lock.h>
65 #include <sys/mutex.h>
66 #include <sys/rwlock.h>
67 #include <sys/sched.h>
68 #include <sys/smp.h>
69 #include <sys/vmmeter.h>
70
71 #include <vm/vm.h>
72 #include <vm/vm_page.h>
73 #include <vm/vm_kern.h>
74 #include <vm/vm_pageout.h>
75 #include <vm/vm_extern.h>
76 #include <vm/vm_object.h>
77 #include <vm/vm_param.h>
78 #include <vm/vm_map.h>
79 #include <vm/vm_pager.h>
80 #include <vm/vm_phys.h>
81 #include <vm/vm_pagequeue.h>
82 #include <vm/uma.h>
83
84 #include <machine/_inttypes.h>
85 #include <machine/cpu.h>
86 #include <machine/pcb.h>
87 #include <machine/platform.h>
88
89 #include <machine/tlb.h>
90 #include <machine/spr.h>
91 #include <machine/md_var.h>
92 #include <machine/mmuvar.h>
93 #include <machine/pmap.h>
94 #include <machine/pte.h>
95
96 #include <ddb/ddb.h>
97
98 #define PRI0ptrX "08x"
99
100 /* Reserved KVA space and mutex for mmu_booke_zero_page. */
101 static vm_offset_t zero_page_va;
102 static struct mtx zero_page_mutex;
103
104 /* Reserved KVA space and mutex for mmu_booke_copy_page. */
105 static vm_offset_t copy_page_src_va;
106 static vm_offset_t copy_page_dst_va;
107 static struct mtx copy_page_mutex;
108
109 static vm_offset_t kernel_ptbl_root;
110 static unsigned int kernel_ptbls; /* Number of KVA ptbls. */
111
112 /**************************************************************************/
113 /* PMAP */
114 /**************************************************************************/
115
116 #define VM_MAPDEV_BASE ((vm_offset_t)VM_MAXUSER_ADDRESS + PAGE_SIZE)
117
118 static void tid_flush(tlbtid_t tid);
119 static unsigned long ilog2(unsigned long);
120
121 /**************************************************************************/
122 /* Page table management */
123 /**************************************************************************/
124
125 #define PMAP_ROOT_SIZE (sizeof(pte_t**) * PDIR_NENTRIES)
126 static void ptbl_init(void);
127 static struct ptbl_buf *ptbl_buf_alloc(void);
128 static void ptbl_buf_free(struct ptbl_buf *);
129 static void ptbl_free_pmap_ptbl(pmap_t, pte_t *);
130
131 static pte_t *ptbl_alloc(pmap_t, unsigned int, boolean_t);
132 static void ptbl_free(pmap_t, unsigned int);
133 static void ptbl_hold(pmap_t, unsigned int);
134 static int ptbl_unhold(pmap_t, unsigned int);
135
136 static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
137 static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t);
138 static int pte_remove(pmap_t, vm_offset_t, uint8_t);
139 static pte_t *pte_find(pmap_t, vm_offset_t);
140
141 struct ptbl_buf {
142 TAILQ_ENTRY(ptbl_buf) link; /* list link */
143 vm_offset_t kva; /* va of mapping */
144 };
145
146 /* Number of kva ptbl buffers, each covering one ptbl (PTBL_PAGES). */
147 #define PTBL_BUFS (128 * 16)
148
149 /* ptbl free list and a lock used for access synchronization. */
150 static TAILQ_HEAD(, ptbl_buf) ptbl_buf_freelist;
151 static struct mtx ptbl_buf_freelist_lock;
152
153 /* Base address of kva space allocated fot ptbl bufs. */
154 static vm_offset_t ptbl_buf_pool_vabase;
155
156 /* Pointer to ptbl_buf structures. */
157 static struct ptbl_buf *ptbl_bufs;
158
159 /**************************************************************************/
160 /* Page table related */
161 /**************************************************************************/
162
163 /* Initialize pool of kva ptbl buffers. */
164 static void
ptbl_init(void)165 ptbl_init(void)
166 {
167 int i;
168
169 CTR3(KTR_PMAP, "%s: s (ptbl_bufs = 0x%08x size 0x%08x)", __func__,
170 (uint32_t)ptbl_bufs, sizeof(struct ptbl_buf) * PTBL_BUFS);
171 CTR3(KTR_PMAP, "%s: s (ptbl_buf_pool_vabase = 0x%08x size = 0x%08x)",
172 __func__, ptbl_buf_pool_vabase, PTBL_BUFS * PTBL_PAGES * PAGE_SIZE);
173
174 mtx_init(&ptbl_buf_freelist_lock, "ptbl bufs lock", NULL, MTX_DEF);
175 TAILQ_INIT(&ptbl_buf_freelist);
176
177 for (i = 0; i < PTBL_BUFS; i++) {
178 ptbl_bufs[i].kva =
179 ptbl_buf_pool_vabase + i * PTBL_PAGES * PAGE_SIZE;
180 TAILQ_INSERT_TAIL(&ptbl_buf_freelist, &ptbl_bufs[i], link);
181 }
182 }
183
184 /* Get a ptbl_buf from the freelist. */
185 static struct ptbl_buf *
ptbl_buf_alloc(void)186 ptbl_buf_alloc(void)
187 {
188 struct ptbl_buf *buf;
189
190 mtx_lock(&ptbl_buf_freelist_lock);
191 buf = TAILQ_FIRST(&ptbl_buf_freelist);
192 if (buf != NULL)
193 TAILQ_REMOVE(&ptbl_buf_freelist, buf, link);
194 mtx_unlock(&ptbl_buf_freelist_lock);
195
196 CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf);
197
198 return (buf);
199 }
200
201 /* Return ptbl buff to free pool. */
202 static void
ptbl_buf_free(struct ptbl_buf * buf)203 ptbl_buf_free(struct ptbl_buf *buf)
204 {
205
206 CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf);
207
208 mtx_lock(&ptbl_buf_freelist_lock);
209 TAILQ_INSERT_TAIL(&ptbl_buf_freelist, buf, link);
210 mtx_unlock(&ptbl_buf_freelist_lock);
211 }
212
213 /*
214 * Search the list of allocated ptbl bufs and find on list of allocated ptbls
215 */
216 static void
ptbl_free_pmap_ptbl(pmap_t pmap,pte_t * ptbl)217 ptbl_free_pmap_ptbl(pmap_t pmap, pte_t *ptbl)
218 {
219 struct ptbl_buf *pbuf;
220
221 CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl);
222
223 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
224
225 TAILQ_FOREACH(pbuf, &pmap->pm_ptbl_list, link)
226 if (pbuf->kva == (vm_offset_t)ptbl) {
227 /* Remove from pmap ptbl buf list. */
228 TAILQ_REMOVE(&pmap->pm_ptbl_list, pbuf, link);
229
230 /* Free corresponding ptbl buf. */
231 ptbl_buf_free(pbuf);
232 break;
233 }
234 }
235
236 /* Allocate page table. */
237 static pte_t *
ptbl_alloc(pmap_t pmap,unsigned int pdir_idx,boolean_t nosleep)238 ptbl_alloc(pmap_t pmap, unsigned int pdir_idx, boolean_t nosleep)
239 {
240 vm_page_t mtbl[PTBL_PAGES];
241 vm_page_t m;
242 struct ptbl_buf *pbuf;
243 unsigned int pidx;
244 pte_t *ptbl;
245 int i, j;
246
247 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
248 (pmap == kernel_pmap), pdir_idx);
249
250 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
251 ("ptbl_alloc: invalid pdir_idx"));
252 KASSERT((pmap->pm_pdir[pdir_idx] == NULL),
253 ("pte_alloc: valid ptbl entry exists!"));
254
255 pbuf = ptbl_buf_alloc();
256 if (pbuf == NULL)
257 panic("pte_alloc: couldn't alloc kernel virtual memory");
258
259 ptbl = (pte_t *)pbuf->kva;
260
261 CTR2(KTR_PMAP, "%s: ptbl kva = %p", __func__, ptbl);
262
263 for (i = 0; i < PTBL_PAGES; i++) {
264 pidx = (PTBL_PAGES * pdir_idx) + i;
265 while ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED)) == NULL) {
266 if (nosleep) {
267 ptbl_free_pmap_ptbl(pmap, ptbl);
268 for (j = 0; j < i; j++)
269 vm_page_free(mtbl[j]);
270 vm_wire_sub(i);
271 return (NULL);
272 }
273 PMAP_UNLOCK(pmap);
274 rw_wunlock(&pvh_global_lock);
275 vm_wait(NULL);
276 rw_wlock(&pvh_global_lock);
277 PMAP_LOCK(pmap);
278 }
279 m->pindex = pidx;
280 mtbl[i] = m;
281 }
282
283 /* Map allocated pages into kernel_pmap. */
284 mmu_booke_qenter((vm_offset_t)ptbl, mtbl, PTBL_PAGES);
285
286 /* Zero whole ptbl. */
287 bzero((caddr_t)ptbl, PTBL_PAGES * PAGE_SIZE);
288
289 /* Add pbuf to the pmap ptbl bufs list. */
290 TAILQ_INSERT_TAIL(&pmap->pm_ptbl_list, pbuf, link);
291
292 return (ptbl);
293 }
294
295 /* Free ptbl pages and invalidate pdir entry. */
296 static void
ptbl_free(pmap_t pmap,unsigned int pdir_idx)297 ptbl_free(pmap_t pmap, unsigned int pdir_idx)
298 {
299 pte_t *ptbl;
300 vm_paddr_t pa;
301 vm_offset_t va;
302 vm_page_t m;
303 int i;
304
305 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
306 (pmap == kernel_pmap), pdir_idx);
307
308 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
309 ("ptbl_free: invalid pdir_idx"));
310
311 ptbl = pmap->pm_pdir[pdir_idx];
312
313 CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl);
314
315 KASSERT((ptbl != NULL), ("ptbl_free: null ptbl"));
316
317 /*
318 * Invalidate the pdir entry as soon as possible, so that other CPUs
319 * don't attempt to look up the page tables we are releasing.
320 */
321 mtx_lock_spin(&tlbivax_mutex);
322 tlb_miss_lock();
323
324 pmap->pm_pdir[pdir_idx] = NULL;
325
326 tlb_miss_unlock();
327 mtx_unlock_spin(&tlbivax_mutex);
328
329 for (i = 0; i < PTBL_PAGES; i++) {
330 va = ((vm_offset_t)ptbl + (i * PAGE_SIZE));
331 pa = pte_vatopa(kernel_pmap, va);
332 m = PHYS_TO_VM_PAGE(pa);
333 vm_page_free_zero(m);
334 vm_wire_sub(1);
335 mmu_booke_kremove(va);
336 }
337
338 ptbl_free_pmap_ptbl(pmap, ptbl);
339 }
340
341 /*
342 * Decrement ptbl pages hold count and attempt to free ptbl pages.
343 * Called when removing pte entry from ptbl.
344 *
345 * Return 1 if ptbl pages were freed.
346 */
347 static int
ptbl_unhold(pmap_t pmap,unsigned int pdir_idx)348 ptbl_unhold(pmap_t pmap, unsigned int pdir_idx)
349 {
350 pte_t *ptbl;
351 vm_paddr_t pa;
352 vm_page_t m;
353 int i;
354
355 CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
356 (pmap == kernel_pmap), pdir_idx);
357
358 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
359 ("ptbl_unhold: invalid pdir_idx"));
360 KASSERT((pmap != kernel_pmap),
361 ("ptbl_unhold: unholding kernel ptbl!"));
362
363 ptbl = pmap->pm_pdir[pdir_idx];
364
365 //debugf("ptbl_unhold: ptbl = 0x%08x\n", (u_int32_t)ptbl);
366 KASSERT(((vm_offset_t)ptbl >= VM_MIN_KERNEL_ADDRESS),
367 ("ptbl_unhold: non kva ptbl"));
368
369 /* decrement hold count */
370 for (i = 0; i < PTBL_PAGES; i++) {
371 pa = pte_vatopa(kernel_pmap,
372 (vm_offset_t)ptbl + (i * PAGE_SIZE));
373 m = PHYS_TO_VM_PAGE(pa);
374 m->ref_count--;
375 }
376
377 /*
378 * Free ptbl pages if there are no pte etries in this ptbl.
379 * ref_count has the same value for all ptbl pages, so check the last
380 * page.
381 */
382 if (m->ref_count == 0) {
383 ptbl_free(pmap, pdir_idx);
384
385 //debugf("ptbl_unhold: e (freed ptbl)\n");
386 return (1);
387 }
388
389 return (0);
390 }
391
392 /*
393 * Increment hold count for ptbl pages. This routine is used when a new pte
394 * entry is being inserted into the ptbl.
395 */
396 static void
ptbl_hold(pmap_t pmap,unsigned int pdir_idx)397 ptbl_hold(pmap_t pmap, unsigned int pdir_idx)
398 {
399 vm_paddr_t pa;
400 pte_t *ptbl;
401 vm_page_t m;
402 int i;
403
404 CTR3(KTR_PMAP, "%s: pmap = %p pdir_idx = %d", __func__, pmap,
405 pdir_idx);
406
407 KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
408 ("ptbl_hold: invalid pdir_idx"));
409 KASSERT((pmap != kernel_pmap),
410 ("ptbl_hold: holding kernel ptbl!"));
411
412 ptbl = pmap->pm_pdir[pdir_idx];
413
414 KASSERT((ptbl != NULL), ("ptbl_hold: null ptbl"));
415
416 for (i = 0; i < PTBL_PAGES; i++) {
417 pa = pte_vatopa(kernel_pmap,
418 (vm_offset_t)ptbl + (i * PAGE_SIZE));
419 m = PHYS_TO_VM_PAGE(pa);
420 m->ref_count++;
421 }
422 }
423
424 /*
425 * Clean pte entry, try to free page table page if requested.
426 *
427 * Return 1 if ptbl pages were freed, otherwise return 0.
428 */
429 static int
pte_remove(pmap_t pmap,vm_offset_t va,uint8_t flags)430 pte_remove(pmap_t pmap, vm_offset_t va, uint8_t flags)
431 {
432 unsigned int pdir_idx = PDIR_IDX(va);
433 unsigned int ptbl_idx = PTBL_IDX(va);
434 vm_page_t m;
435 pte_t *ptbl;
436 pte_t *pte;
437
438 //int su = (pmap == kernel_pmap);
439 //debugf("pte_remove: s (su = %d pmap = 0x%08x va = 0x%08x flags = %d)\n",
440 // su, (u_int32_t)pmap, va, flags);
441
442 ptbl = pmap->pm_pdir[pdir_idx];
443 KASSERT(ptbl, ("pte_remove: null ptbl"));
444
445 pte = &ptbl[ptbl_idx];
446
447 if (pte == NULL || !PTE_ISVALID(pte))
448 return (0);
449
450 if (PTE_ISWIRED(pte))
451 pmap->pm_stats.wired_count--;
452
453 /* Get vm_page_t for mapped pte. */
454 m = PHYS_TO_VM_PAGE(PTE_PA(pte));
455
456 /* Handle managed entry. */
457 if (PTE_ISMANAGED(pte)) {
458 if (PTE_ISMODIFIED(pte))
459 vm_page_dirty(m);
460
461 if (PTE_ISREFERENCED(pte))
462 vm_page_aflag_set(m, PGA_REFERENCED);
463
464 pv_remove(pmap, va, m);
465 } else if (pmap == kernel_pmap && m && m->md.pv_tracked) {
466 /*
467 * Always pv_insert()/pv_remove() on MPC85XX, in case DPAA is
468 * used. This is needed by the NCSW support code for fast
469 * VA<->PA translation.
470 */
471 pv_remove(pmap, va, m);
472 if (TAILQ_EMPTY(&m->md.pv_list))
473 m->md.pv_tracked = false;
474 }
475
476 mtx_lock_spin(&tlbivax_mutex);
477 tlb_miss_lock();
478
479 tlb0_flush_entry(va);
480 *pte = 0;
481
482 tlb_miss_unlock();
483 mtx_unlock_spin(&tlbivax_mutex);
484
485 pmap->pm_stats.resident_count--;
486
487 if (flags & PTBL_UNHOLD) {
488 //debugf("pte_remove: e (unhold)\n");
489 return (ptbl_unhold(pmap, pdir_idx));
490 }
491
492 //debugf("pte_remove: e\n");
493 return (0);
494 }
495
496 /*
497 * Insert PTE for a given page and virtual address.
498 */
499 static int
pte_enter(pmap_t pmap,vm_page_t m,vm_offset_t va,uint32_t flags,boolean_t nosleep)500 pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,
501 boolean_t nosleep)
502 {
503 unsigned int pdir_idx = PDIR_IDX(va);
504 unsigned int ptbl_idx = PTBL_IDX(va);
505 pte_t *ptbl, *pte, pte_tmp;
506
507 CTR4(KTR_PMAP, "%s: su = %d pmap = %p va = %p", __func__,
508 pmap == kernel_pmap, pmap, va);
509
510 /* Get the page table pointer. */
511 ptbl = pmap->pm_pdir[pdir_idx];
512
513 if (ptbl == NULL) {
514 /* Allocate page table pages. */
515 ptbl = ptbl_alloc(pmap, pdir_idx, nosleep);
516 if (ptbl == NULL) {
517 KASSERT(nosleep, ("nosleep and NULL ptbl"));
518 return (ENOMEM);
519 }
520 pmap->pm_pdir[pdir_idx] = ptbl;
521 pte = &ptbl[ptbl_idx];
522 } else {
523 /*
524 * Check if there is valid mapping for requested
525 * va, if there is, remove it.
526 */
527 pte = &pmap->pm_pdir[pdir_idx][ptbl_idx];
528 if (PTE_ISVALID(pte)) {
529 pte_remove(pmap, va, PTBL_HOLD);
530 } else {
531 /*
532 * pte is not used, increment hold count
533 * for ptbl pages.
534 */
535 if (pmap != kernel_pmap)
536 ptbl_hold(pmap, pdir_idx);
537 }
538 }
539
540 /*
541 * Insert pv_entry into pv_list for mapped page if part of managed
542 * memory.
543 */
544 if ((m->oflags & VPO_UNMANAGED) == 0) {
545 flags |= PTE_MANAGED;
546
547 /* Create and insert pv entry. */
548 pv_insert(pmap, va, m);
549 }
550
551 pmap->pm_stats.resident_count++;
552
553 pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
554 pte_tmp |= (PTE_VALID | flags | PTE_PS_4KB); /* 4KB pages only */
555
556 mtx_lock_spin(&tlbivax_mutex);
557 tlb_miss_lock();
558
559 tlb0_flush_entry(va);
560 *pte = pte_tmp;
561
562 tlb_miss_unlock();
563 mtx_unlock_spin(&tlbivax_mutex);
564 return (0);
565 }
566
567 /* Return the pa for the given pmap/va. */
568 static vm_paddr_t
pte_vatopa(pmap_t pmap,vm_offset_t va)569 pte_vatopa(pmap_t pmap, vm_offset_t va)
570 {
571 vm_paddr_t pa = 0;
572 pte_t *pte;
573
574 pte = pte_find(pmap, va);
575 if ((pte != NULL) && PTE_ISVALID(pte))
576 pa = (PTE_PA(pte) | (va & PTE_PA_MASK));
577 return (pa);
578 }
579
580 /* Get a pointer to a PTE in a page table. */
581 static pte_t *
pte_find(pmap_t pmap,vm_offset_t va)582 pte_find(pmap_t pmap, vm_offset_t va)
583 {
584 unsigned int pdir_idx = PDIR_IDX(va);
585 unsigned int ptbl_idx = PTBL_IDX(va);
586
587 KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
588
589 if (pmap->pm_pdir[pdir_idx])
590 return (&(pmap->pm_pdir[pdir_idx][ptbl_idx]));
591
592 return (NULL);
593 }
594
595 /* Get a pointer to a PTE in a page table, or the next closest (greater) one. */
596 static __inline pte_t *
pte_find_next(pmap_t pmap,vm_offset_t * pva)597 pte_find_next(pmap_t pmap, vm_offset_t *pva)
598 {
599 vm_offset_t va;
600 pte_t **pdir;
601 pte_t *pte;
602 unsigned long i, j;
603
604 KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
605
606 va = *pva;
607 i = PDIR_IDX(va);
608 j = PTBL_IDX(va);
609 pdir = pmap->pm_pdir;
610 for (; i < PDIR_NENTRIES; i++, j = 0) {
611 if (pdir[i] == NULL)
612 continue;
613 for (; j < PTBL_NENTRIES; j++) {
614 pte = &pdir[i][j];
615 if (!PTE_ISVALID(pte))
616 continue;
617 *pva = PDIR_SIZE * i + PAGE_SIZE * j;
618 return (pte);
619 }
620 }
621 return (NULL);
622 }
623
624 /* Set up kernel page tables. */
625 static void
kernel_pte_alloc(vm_offset_t data_end,vm_offset_t addr)626 kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
627 {
628 pte_t *pte;
629 vm_offset_t va;
630 vm_offset_t pdir_start;
631 int i;
632
633 kptbl_min = VM_MIN_KERNEL_ADDRESS / PDIR_SIZE;
634 kernel_pmap->pm_pdir = (pte_t **)kernel_ptbl_root;
635
636 pdir_start = kernel_ptbl_root + PDIR_NENTRIES * sizeof(pte_t);
637
638 /* Initialize kernel pdir */
639 for (i = 0; i < kernel_ptbls; i++) {
640 kernel_pmap->pm_pdir[kptbl_min + i] =
641 (pte_t *)(pdir_start + (i * PAGE_SIZE * PTBL_PAGES));
642 }
643
644 /*
645 * Fill in PTEs covering kernel code and data. They are not required
646 * for address translation, as this area is covered by static TLB1
647 * entries, but for pte_vatopa() to work correctly with kernel area
648 * addresses.
649 */
650 for (va = addr; va < data_end; va += PAGE_SIZE) {
651 pte = &(kernel_pmap->pm_pdir[PDIR_IDX(va)][PTBL_IDX(va)]);
652 powerpc_sync();
653 *pte = PTE_RPN_FROM_PA(kernload + (va - kernstart));
654 *pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED |
655 PTE_VALID | PTE_PS_4KB;
656 }
657 }
658
659 static vm_offset_t
mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)660 mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)
661 {
662 /* Allocate space for ptbl_bufs. */
663 ptbl_bufs = (struct ptbl_buf *)data_end;
664 data_end += sizeof(struct ptbl_buf) * PTBL_BUFS;
665 debugf(" ptbl_bufs at 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n",
666 (uintptr_t)ptbl_bufs, data_end);
667
668 data_end = round_page(data_end);
669
670 kernel_ptbl_root = data_end;
671 data_end += PDIR_NENTRIES * sizeof(pte_t*);
672
673 /* Allocate PTE tables for kernel KVA. */
674 kernel_ptbls = howmany(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
675 PDIR_SIZE);
676 data_end += kernel_ptbls * PTBL_PAGES * PAGE_SIZE;
677 debugf(" kernel ptbls: %d\n", kernel_ptbls);
678 debugf(" kernel pdir at %#jx end = %#jx\n",
679 (uintmax_t)kernel_ptbl_root, (uintmax_t)data_end);
680
681 return (data_end);
682 }
683
684 /*
685 * Initialize a preallocated and zeroed pmap structure,
686 * such as one in a vmspace structure.
687 */
688 static int
mmu_booke_pinit(pmap_t pmap)689 mmu_booke_pinit(pmap_t pmap)
690 {
691 int i;
692
693 CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,
694 curthread->td_proc->p_pid, curthread->td_proc->p_comm);
695
696 KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));
697
698 for (i = 0; i < MAXCPU; i++)
699 pmap->pm_tid[i] = TID_NONE;
700 CPU_ZERO(&kernel_pmap->pm_active);
701 bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
702 pmap->pm_pdir = uma_zalloc(ptbl_root_zone, M_WAITOK);
703 bzero(pmap->pm_pdir, sizeof(pte_t *) * PDIR_NENTRIES);
704 TAILQ_INIT(&pmap->pm_ptbl_list);
705
706 return (1);
707 }
708
709 /*
710 * Release any resources held by the given physical map.
711 * Called when a pmap initialized by mmu_booke_pinit is being released.
712 * Should only be called if the map contains no valid mappings.
713 */
714 static void
mmu_booke_release(pmap_t pmap)715 mmu_booke_release(pmap_t pmap)
716 {
717
718 KASSERT(pmap->pm_stats.resident_count == 0,
719 ("pmap_release: pmap resident count %ld != 0",
720 pmap->pm_stats.resident_count));
721 uma_zfree(ptbl_root_zone, pmap->pm_pdir);
722 }
723
724 static void
mmu_booke_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)725 mmu_booke_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
726 {
727 pte_t *pte;
728 vm_paddr_t pa = 0;
729 int sync_sz, valid;
730 pmap_t pmap;
731 vm_page_t m;
732 vm_offset_t addr;
733 int active;
734
735 rw_wlock(&pvh_global_lock);
736 pmap = PCPU_GET(curpmap);
737 active = (pm == kernel_pmap || pm == pmap) ? 1 : 0;
738 while (sz > 0) {
739 PMAP_LOCK(pm);
740 pte = pte_find(pm, va);
741 valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0;
742 if (valid)
743 pa = PTE_PA(pte);
744 PMAP_UNLOCK(pm);
745 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
746 sync_sz = min(sync_sz, sz);
747 if (valid) {
748 if (!active) {
749 /* Create a mapping in the active pmap. */
750 addr = 0;
751 m = PHYS_TO_VM_PAGE(pa);
752 PMAP_LOCK(pmap);
753 pte_enter(pmap, m, addr,
754 PTE_SR | PTE_VALID, FALSE);
755 addr += (va & PAGE_MASK);
756 __syncicache((void *)addr, sync_sz);
757 pte_remove(pmap, addr, PTBL_UNHOLD);
758 PMAP_UNLOCK(pmap);
759 } else
760 __syncicache((void *)va, sync_sz);
761 }
762 va += sync_sz;
763 sz -= sync_sz;
764 }
765 rw_wunlock(&pvh_global_lock);
766 }
767
768 /*
769 * mmu_booke_zero_page_area zeros the specified hardware page by
770 * mapping it into virtual memory and using bzero to clear
771 * its contents.
772 *
773 * off and size must reside within a single page.
774 */
775 static void
mmu_booke_zero_page_area(vm_page_t m,int off,int size)776 mmu_booke_zero_page_area(vm_page_t m, int off, int size)
777 {
778 vm_offset_t va;
779
780 /* XXX KASSERT off and size are within a single page? */
781
782 mtx_lock(&zero_page_mutex);
783 va = zero_page_va;
784
785 mmu_booke_kenter(va, VM_PAGE_TO_PHYS(m));
786 bzero((caddr_t)va + off, size);
787 mmu_booke_kremove(va);
788
789 mtx_unlock(&zero_page_mutex);
790 }
791
792 /*
793 * mmu_booke_zero_page zeros the specified hardware page.
794 */
795 static void
mmu_booke_zero_page(vm_page_t m)796 mmu_booke_zero_page(vm_page_t m)
797 {
798 vm_offset_t off, va;
799
800 va = zero_page_va;
801 mtx_lock(&zero_page_mutex);
802
803 mmu_booke_kenter(va, VM_PAGE_TO_PHYS(m));
804
805 for (off = 0; off < PAGE_SIZE; off += cacheline_size)
806 __asm __volatile("dcbz 0,%0" :: "r"(va + off));
807
808 mmu_booke_kremove(va);
809
810 mtx_unlock(&zero_page_mutex);
811 }
812
813 /*
814 * mmu_booke_copy_page copies the specified (machine independent) page by
815 * mapping the page into virtual memory and using memcopy to copy the page,
816 * one machine dependent page at a time.
817 */
818 static void
mmu_booke_copy_page(vm_page_t sm,vm_page_t dm)819 mmu_booke_copy_page(vm_page_t sm, vm_page_t dm)
820 {
821 vm_offset_t sva, dva;
822
823 sva = copy_page_src_va;
824 dva = copy_page_dst_va;
825
826 mtx_lock(©_page_mutex);
827 mmu_booke_kenter(sva, VM_PAGE_TO_PHYS(sm));
828 mmu_booke_kenter(dva, VM_PAGE_TO_PHYS(dm));
829
830 memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE);
831
832 mmu_booke_kremove(dva);
833 mmu_booke_kremove(sva);
834 mtx_unlock(©_page_mutex);
835 }
836
837 static inline void
mmu_booke_copy_pages(vm_page_t * ma,vm_offset_t a_offset,vm_page_t * mb,vm_offset_t b_offset,int xfersize)838 mmu_booke_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
839 vm_page_t *mb, vm_offset_t b_offset, int xfersize)
840 {
841 void *a_cp, *b_cp;
842 vm_offset_t a_pg_offset, b_pg_offset;
843 int cnt;
844
845 mtx_lock(©_page_mutex);
846 while (xfersize > 0) {
847 a_pg_offset = a_offset & PAGE_MASK;
848 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
849 mmu_booke_kenter(copy_page_src_va,
850 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]));
851 a_cp = (char *)copy_page_src_va + a_pg_offset;
852 b_pg_offset = b_offset & PAGE_MASK;
853 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
854 mmu_booke_kenter(copy_page_dst_va,
855 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]));
856 b_cp = (char *)copy_page_dst_va + b_pg_offset;
857 bcopy(a_cp, b_cp, cnt);
858 mmu_booke_kremove(copy_page_dst_va);
859 mmu_booke_kremove(copy_page_src_va);
860 a_offset += cnt;
861 b_offset += cnt;
862 xfersize -= cnt;
863 }
864 mtx_unlock(©_page_mutex);
865 }
866
867 static vm_offset_t
mmu_booke_quick_enter_page(vm_page_t m)868 mmu_booke_quick_enter_page(vm_page_t m)
869 {
870 vm_paddr_t paddr;
871 vm_offset_t qaddr;
872 uint32_t flags;
873 pte_t *pte;
874
875 paddr = VM_PAGE_TO_PHYS(m);
876
877 flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID;
878 flags |= tlb_calc_wimg(paddr, pmap_page_get_memattr(m)) << PTE_MAS2_SHIFT;
879 flags |= PTE_PS_4KB;
880
881 critical_enter();
882 qaddr = PCPU_GET(qmap_addr);
883
884 pte = pte_find(kernel_pmap, qaddr);
885
886 KASSERT(*pte == 0, ("mmu_booke_quick_enter_page: PTE busy"));
887
888 /*
889 * XXX: tlbivax is broadcast to other cores, but qaddr should
890 * not be present in other TLBs. Is there a better instruction
891 * sequence to use? Or just forget it & use mmu_booke_kenter()...
892 */
893 __asm __volatile("tlbivax 0, %0" :: "r"(qaddr & MAS2_EPN_MASK));
894 __asm __volatile("isync; msync");
895
896 *pte = PTE_RPN_FROM_PA(paddr) | flags;
897
898 /* Flush the real memory from the instruction cache. */
899 if ((flags & (PTE_I | PTE_G)) == 0)
900 __syncicache((void *)qaddr, PAGE_SIZE);
901
902 return (qaddr);
903 }
904
905 static void
mmu_booke_quick_remove_page(vm_offset_t addr)906 mmu_booke_quick_remove_page(vm_offset_t addr)
907 {
908 pte_t *pte;
909
910 pte = pte_find(kernel_pmap, addr);
911
912 KASSERT(PCPU_GET(qmap_addr) == addr,
913 ("mmu_booke_quick_remove_page: invalid address"));
914 KASSERT(*pte != 0,
915 ("mmu_booke_quick_remove_page: PTE not in use"));
916
917 *pte = 0;
918 critical_exit();
919 }
920
921 /**************************************************************************/
922 /* TID handling */
923 /**************************************************************************/
924
925 /*
926 * Return the largest uint value log such that 2^log <= num.
927 */
928 static unsigned long
ilog2(unsigned long num)929 ilog2(unsigned long num)
930 {
931 long lz;
932
933 __asm ("cntlzw %0, %1" : "=r" (lz) : "r" (num));
934 return (31 - lz);
935 }
936
937 /*
938 * Invalidate all TLB0 entries which match the given TID. Note this is
939 * dedicated for cases when invalidations should NOT be propagated to other
940 * CPUs.
941 */
942 static void
tid_flush(tlbtid_t tid)943 tid_flush(tlbtid_t tid)
944 {
945 register_t msr;
946 uint32_t mas0, mas1, mas2;
947 int entry, way;
948
949 /* Don't evict kernel translations */
950 if (tid == TID_KERNEL)
951 return;
952
953 msr = mfmsr();
954 __asm __volatile("wrteei 0");
955
956 /*
957 * Newer (e500mc and later) have tlbilx, which doesn't broadcast, so use
958 * it for PID invalidation.
959 */
960 switch ((mfpvr() >> 16) & 0xffff) {
961 case FSL_E500mc:
962 case FSL_E5500:
963 case FSL_E6500:
964 mtspr(SPR_MAS6, tid << MAS6_SPID0_SHIFT);
965 /* tlbilxpid */
966 __asm __volatile("isync; .long 0x7c200024; isync; msync");
967 __asm __volatile("wrtee %0" :: "r"(msr));
968 return;
969 }
970
971 for (way = 0; way < TLB0_WAYS; way++)
972 for (entry = 0; entry < TLB0_ENTRIES_PER_WAY; entry++) {
973 mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way);
974 mtspr(SPR_MAS0, mas0);
975
976 mas2 = entry << MAS2_TLB0_ENTRY_IDX_SHIFT;
977 mtspr(SPR_MAS2, mas2);
978
979 __asm __volatile("isync; tlbre");
980
981 mas1 = mfspr(SPR_MAS1);
982
983 if (!(mas1 & MAS1_VALID))
984 continue;
985 if (((mas1 & MAS1_TID_MASK) >> MAS1_TID_SHIFT) != tid)
986 continue;
987 mas1 &= ~MAS1_VALID;
988 mtspr(SPR_MAS1, mas1);
989 __asm __volatile("isync; tlbwe; isync; msync");
990 }
991 __asm __volatile("wrtee %0" :: "r"(msr));
992 }
993