1 /* $NetBSD: pmap.c,v 1.378 2024/02/10 09:30:06 andvar Exp $ */
2
3 /*
4 * Copyright (c) 1996
5 * The President and Fellows of Harvard College. All rights reserved.
6 * Copyright (c) 1992, 1993
7 * The Regents of the University of California. All rights reserved.
8 *
9 * This software was developed by the Computer Systems Engineering group
10 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
11 * contributed to Berkeley.
12 *
13 * All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by Harvard University.
16 * This product includes software developed by the University of
17 * California, Lawrence Berkeley Laboratory.
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
21 * are met:
22 *
23 * 1. Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * 2. Redistributions in binary form must reproduce the above copyright
26 * notice, this list of conditions and the following disclaimer in the
27 * documentation and/or other materials provided with the distribution.
28 * 3. All advertising materials mentioning features or use of this software
29 * must display the following acknowledgement:
30 * This product includes software developed by Aaron Brown and
31 * Harvard University.
32 * This product includes software developed by the University of
33 * California, Berkeley and its contributors.
34 * 4. Neither the name of the University nor the names of its contributors
35 * may be used to endorse or promote products derived from this software
36 * without specific prior written permission.
37 *
38 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
39 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
41 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
42 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
43 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
44 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
45 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
46 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
47 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
48 * SUCH DAMAGE.
49 *
50 * @(#)pmap.c 8.4 (Berkeley) 2/5/94
51 *
52 */
53
54 /*
55 * SPARC physical map management code.
56 */
57
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.378 2024/02/10 09:30:06 andvar Exp $");
60
61 #include "opt_ddb.h"
62 #include "opt_kgdb.h"
63 #include "opt_sparc_arch.h"
64
65 #include <sys/param.h>
66 #include <sys/systm.h>
67 #include <sys/device.h>
68 #include <sys/proc.h>
69 #include <sys/queue.h>
70 #include <sys/pool.h>
71 #include <sys/exec.h>
72 #include <sys/core.h>
73 #include <sys/kcore.h>
74 #include <sys/kernel.h>
75 #include <sys/atomic.h>
76
77 #include <sys/exec_aout.h> /* for MID_* */
78
79 #include <uvm/uvm.h>
80
81 #include <machine/autoconf.h>
82 #include <machine/bsd_openprom.h>
83 #include <machine/oldmon.h>
84 #include <machine/cpu.h>
85 #include <machine/ctlreg.h>
86 #include <machine/kcore.h>
87 #include <machine/locore.h>
88
89 #include <sparc/sparc/asm.h>
90 #include <sparc/sparc/cache.h>
91 #include <sparc/sparc/vaddrs.h>
92 #include <sparc/sparc/cpuvar.h>
93
94 /*
95 * The SPARCstation offers us the following challenges:
96 *
97 * 1. A virtual address cache. This is, strictly speaking, not
98 * part of the architecture, but the code below assumes one.
99 * This is a write-through cache on the 4c and a write-back cache
100 * on others.
101 *
102 * 2. (4/4c only) An MMU that acts like a cache. There is not enough
103 * space in the MMU to map everything all the time. Instead, we need
104 * to load MMU with the `working set' of translations for each
105 * process. The sun4m does not act like a cache; tables are maintained
106 * in physical memory.
107 *
108 * 3. Segmented virtual and physical spaces. The upper 12 bits of
109 * a virtual address (the virtual segment) index a segment table,
110 * giving a physical segment. The physical segment selects a
111 * `Page Map Entry Group' (PMEG) and the virtual page number---the
112 * next 5 or 6 bits of the virtual address---select the particular
113 * `Page Map Entry' for the page. We call the latter a PTE and
114 * call each Page Map Entry Group a pmeg (for want of a better name).
115 * Note that the sun4m has an unsegmented 36-bit physical space.
116 *
117 * Since there are no valid bits in the segment table, the only way
118 * to have an invalid segment is to make one full pmeg of invalid PTEs.
119 * We use the last one (since the ROM does as well) (sun4/4c only)
120 *
121 * 4. Discontiguous physical pages. The Mach VM expects physical pages
122 * to be in one sequential lump.
123 *
124 * 5. The MMU is always on: it is not possible to disable it. This is
125 * mainly a startup hassle.
126 */
127
128 struct pmap_stats {
129 int ps_unlink_pvfirst; /* # of pv_unlinks on head */
130 int ps_unlink_pvsearch; /* # of pv_unlink searches */
131 int ps_changeprots; /* # of calls to changeprot */
132 int ps_enter_firstpv; /* pv heads entered */
133 int ps_enter_secondpv; /* pv nonheads entered */
134 int ps_useless_changewire; /* useless wiring changes */
135 int ps_npg_prot_all; /* # of active pages protected */
136 int ps_npg_prot_actual; /* # pages actually affected */
137 int ps_npmeg_free; /* # of free pmegs */
138 int ps_npmeg_locked; /* # of pmegs on locked list */
139 int ps_npmeg_lru; /* # of pmegs on lru list */
140 } pmap_stats;
141
142 #if defined(SUN4) || defined(SUN4C)
143 struct evcnt mmu_stolenpmegs_evcnt =
144 EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","stln pmgs");
145 EVCNT_ATTACH_STATIC(mmu_stolenpmegs_evcnt);
146
147 struct evcnt mmu_pagein_evcnt =
148 EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","pagein");
149 EVCNT_ATTACH_STATIC(mmu_pagein_evcnt);
150 #endif /* SUN4 || SUN4C */
151
152 #ifdef DEBUG
153 #define PDB_CREATE 0x0001
154 #define PDB_DESTROY 0x0002
155 #define PDB_REMOVE 0x0004
156 #define PDB_CHANGEPROT 0x0008
157 #define PDB_ENTER 0x0010
158 #define PDB_FOLLOW 0x0020
159 #define PDB_INITLOUD 0x0040
160
161 #define PDB_MMU_ALLOC 0x0100
162 #define PDB_MMU_STEAL 0x0200
163 #define PDB_CTX_ALLOC 0x0400
164 #define PDB_CTX_STEAL 0x0800
165 #define PDB_MMUREG_ALLOC 0x1000
166 #define PDB_MMUREG_STEAL 0x2000
167 #define PDB_CACHESTUFF 0x4000
168 #define PDB_SWITCHMAP 0x8000
169 #define PDB_SANITYCHK 0x10000
170 int pmapdebug = 0;
171 #define DPRINTF(level, fmt, ...) do { \
172 if (pmapdebug & (level)) \
173 printf("%s:%d: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__); \
174 } while (0)
175 #else
176 #define DPRINTF(level, fmt, ...) /* nothing */
177 #endif
178
179 /*
180 * Bounds on managed physical addresses. Used by (MD) users
181 * of uvm_pglistalloc() to provide search hints.
182 */
183 paddr_t vm_first_phys = (paddr_t)-1;
184 paddr_t vm_last_phys = 0;
185 psize_t vm_num_phys;
186
187 #define PMAP_LOCK() mutex_enter(&pmap_lock)
188 #define PMAP_UNLOCK() mutex_exit(&pmap_lock)
189
190 /*
191 * Flags in pvlist.pv_flags. Note that PV_MOD must be 1 and PV_REF must be 2
192 * since they must line up with the bits in the hardware PTEs (see pte.h).
193 * SUN4M bits are at a slightly different location in the PTE.
194 *
195 * Note: the REF, MOD and ANC flag bits occur only in the head of a pvlist.
196 * The NC bit is meaningful in each individual pv entry and reflects the
197 * requested non-cacheability at the time the entry was made through
198 * pv_link() or when subsequently altered by kvm_uncache() (but the latter
199 * does not happen in kernels as of the time of this writing (March 2001)).
200 */
201 #define PV_MOD 1 /* page modified */
202 #define PV_REF 2 /* page referenced */
203 #define PV_NC 4 /* page cannot be cached */
204 #define PV_REF4M 1 /* page referenced (SRMMU) */
205 #define PV_MOD4M 2 /* page modified (SRMMU) */
206 #define PV_ANC 0x10 /* page has incongruent aliases */
207
208 static struct pool pv_pool;
209
210 /*
211 * pvhead(pte): find a VM page given a PTE entry.
212 */
213 #if defined(SUN4) || defined(SUN4C)
214 static struct vm_page *
pvhead4_4c(u_int pte)215 pvhead4_4c(u_int pte)
216 {
217 paddr_t pa = (pte & PG_PFNUM) << PGSHIFT;
218
219 return (PHYS_TO_VM_PAGE(pa));
220 }
221 #endif
222
223 #if defined(SUN4M) || defined(SUN4D)
224 static struct vm_page *
pvhead4m(u_int pte)225 pvhead4m(u_int pte)
226 {
227 paddr_t pa = (pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT;
228
229 return (PHYS_TO_VM_PAGE(pa));
230 }
231 #endif
232
233 /*
234 * Each virtual segment within each pmap is either valid or invalid.
235 * It is valid if pm_npte[VA_VSEG(va)] is not 0. This does not mean
236 * it is in the MMU, however; that is true iff pm_segmap[VA_VSEG(va)]
237 * does not point to the invalid PMEG.
238 *
239 * In the older SPARC architectures (sun4/sun4c), page tables are cached in
240 * the MMU. The following discussion applies to these architectures:
241 *
242 * If a virtual segment is valid and loaded, the correct PTEs appear
243 * in the MMU only. If it is valid and unloaded, the correct PTEs appear
244 * in the pm_pte[VA_VSEG(va)] only. However, some effort is made to keep
245 * the software copies consistent enough with the MMU so that libkvm can
246 * do user address translations. In particular, pv_changepte() and
247 * pmap_enu() maintain consistency, while less critical changes are
248 * not maintained. pm_pte[VA_VSEG(va)] always points to space for those
249 * PTEs.
250 *
251 * Each PMEG in the MMU is either free or contains PTEs corresponding to
252 * some pmap and virtual segment. If it contains some PTEs, it also contains
253 * reference and modify bits that belong in the pv_table. If we need
254 * to steal a PMEG from some process (if we need one and none are free)
255 * we must copy the ref and mod bits, and update pm_segmap in the other
256 * pmap to show that its virtual segment is no longer in the MMU.
257 *
258 * There are 128 PMEGs in a small Sun-4, of which only a few dozen are
259 * tied down permanently, leaving `about' 100 to be spread among
260 * running processes. These are managed as an LRU cache. Before
261 * calling the VM paging code for a user page fault, the fault handler
262 * calls mmu_load(pmap, va) to try to get a set of PTEs put into the
263 * MMU. mmu_load will check the validity of the segment and tell whether
264 * it did something.
265 *
266 * Since I hate the name PMEG I call this data structure an `mmu entry'.
267 * Each mmuentry is on exactly one of three `usage' lists: free, LRU,
268 * or locked. The locked list is only used for kernel mappings that need
269 * to be wired down.
270 *
271 *
272 * In the sun4m architecture using the SPARC Reference MMU (SRMMU), three
273 * levels of page tables are maintained in physical memory. We use the same
274 * structures as with the 3-level old-style MMU (pm_regmap, pm_segmap,
275 * rg_segmap, sg_pte, etc) to maintain kernel-edible page tables; we also
276 * build a parallel set of physical tables that can be used by the MMU.
277 * (XXX: This seems redundant, but is it necessary for the unified kernel?)
278 *
279 * If a virtual segment is valid, its entries will be in both parallel lists.
280 * If it is not valid, then its entry in the kernel tables will be zero, and
281 * its entry in the MMU tables will either be nonexistent or zero as well.
282 *
283 * The Reference MMU generally uses a Translation Look-aside Buffer (TLB)
284 * to cache the result of recently executed page table walks. When
285 * manipulating page tables, we need to ensure consistency of the
286 * in-memory and TLB copies of the page table entries. This is handled
287 * by flushing (and invalidating) a TLB entry when appropriate before
288 * altering an in-memory page table entry.
289 */
290 struct mmuentry {
291 struct {
292 struct mmuentry *prev, *next;
293 } me_list; /* usage list link */
294 TAILQ_ENTRY(mmuentry) me_pmchain; /* pmap owner link */
295 struct pmap *me_pmap; /* pmap, if in use */
296 u_short me_vreg; /* associated virtual region/segment */
297 u_short me_vseg; /* associated virtual region/segment */
298 u_short me_cookie; /* hardware SMEG/PMEG number */
299 #ifdef DIAGNOSTIC
300 int *me_statp;/*XXX*/
301 #endif
302 };
303 struct mmuentry *mmusegments; /* allocated in pmap_bootstrap */
304 struct mmuentry *mmuregions; /* allocated in pmap_bootstrap */
305
306 #if defined(SUN4) || defined(SUN4C)
307 struct mmuentry segm_freelist, segm_lru, segm_locked;
308 #if defined(SUN4_MMU3L)
309 struct mmuentry region_freelist, region_lru, region_locked;
310 #endif
311 /*
312 * We use a double linked list looping through its static head (which
313 * always remains on the list), so we can remove any other member from
314 * a list without knowing which list it is on.
315 */
316 static void inline
mmuq_remove(struct mmuentry * e)317 mmuq_remove(struct mmuentry *e)
318 {
319 e->me_list.next->me_list.prev = e->me_list.prev;
320 e->me_list.prev->me_list.next = e->me_list.next;
321 }
322
323 static void inline
mmuq_init(struct mmuentry * e)324 mmuq_init(struct mmuentry *e)
325 {
326 memset(e, 0, sizeof(*e));
327 e->me_list.next = e;
328 e->me_list.prev = e;
329 }
330
331 static inline struct mmuentry *
mmuq_first(struct mmuentry * head)332 mmuq_first(struct mmuentry *head)
333 {
334 KASSERT(head->me_list.next != head);
335 return head->me_list.next;
336 }
337
338 static inline bool
mmuq_empty(struct mmuentry * head)339 mmuq_empty(struct mmuentry *head)
340 {
341 return head->me_list.next == head;
342 }
343
344 static inline void
mmuq_insert_tail(struct mmuentry * head,struct mmuentry * e)345 mmuq_insert_tail(struct mmuentry *head, struct mmuentry *e)
346 {
347 e->me_list.prev = head->me_list.prev;
348 e->me_list.next = head;
349 head->me_list.prev->me_list.next = e;
350 head->me_list.prev = e;
351 }
352 #endif
353
354
355 int seginval; /* [4/4c] the invalid segment number */
356 int reginval; /* [4/3mmu] the invalid region number */
357
358 static kmutex_t pmap_lock;
359 static kmutex_t demap_lock;
360 static bool lock_available = false; /* demap_lock has been initialized */
361
362 /*
363 * (sun4/4c)
364 * A context is simply a small number that dictates which set of 4096
365 * segment map entries the MMU uses. The Sun 4c has eight (SS1,IPC) or
366 * sixteen (SS2,IPX) such sets. These are allotted in an `almost MRU' fashion.
367 * (sun4m)
368 * A context is simply a small number that indexes the context table, the
369 * root-level page table mapping 4G areas. Each entry in this table points
370 * to a 1st-level region table. A SPARC reference MMU will usually use 16
371 * such contexts, but some offer as many as 64k contexts; the theoretical
372 * maximum is 2^32 - 1, but this would create overlarge context tables.
373 *
374 * Each context is either free or attached to a pmap.
375 *
376 * Since the virtual address cache is tagged by context, when we steal
377 * a context we have to flush (that part of) the cache.
378 */
379 union ctxinfo {
380 union ctxinfo *c_nextfree; /* free list (if free) */
381 struct pmap *c_pmap; /* pmap (if busy) */
382 };
383
384 static kmutex_t ctx_lock; /* lock for below, and {,de}activate */
385 union ctxinfo *ctxinfo; /* allocated at in pmap_bootstrap */
386 union ctxinfo *ctx_freelist; /* context free list */
387 int ctx_kick; /* allocation rover when none free */
388 int ctx_kickdir; /* ctx_kick roves both directions */
389 int ncontext; /* sizeof ctx_freelist */
390
391 static void ctx_alloc(struct pmap *);
392 static void ctx_free(struct pmap *);
393
394 /*void * vdumppages; -* 32KB worth of reserved dump pages */
395
396 smeg_t tregion; /* [4/3mmu] Region for temporary mappings */
397
398 static struct pmap kernel_pmap_store; /* the kernel's pmap */
399 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; /* pmap_kernel() */
400 struct regmap kernel_regmap_store[NKREG]; /* the kernel's regmap */
401 struct segmap kernel_segmap_store[NKREG*NSEGRG];/* the kernel's segmaps */
402
403 #if defined(SUN4M) || defined(SUN4D)
404 u_int *kernel_regtable_store; /* 1k of storage to map the kernel */
405 u_int *kernel_segtable_store; /* 2k of storage to map the kernel */
406 u_int *kernel_pagtable_store; /* 128k of storage to map the kernel */
407
408 /*
409 * Memory pools and back-end supplier for SRMMU page tables.
410 * Share a pool between the level 2 and level 3 page tables,
411 * since these are equal in size.
412 */
413 static struct pool L1_pool;
414 static struct pool L23_pool;
415
416 static void *pgt_page_alloc(struct pool *, int);
417 static void pgt_page_free(struct pool *, void *);
418
419 static struct pool_allocator pgt_page_allocator = {
420 pgt_page_alloc, pgt_page_free, 0,
421 };
422
423 #endif /* SUN4M || SUN4D */
424
425 #if defined(SUN4) || defined(SUN4C)
426 /*
427 * Memory pool for user and kernel PTE tables.
428 */
429 static struct pool pte_pool;
430 #endif
431
432 struct memarr *pmemarr; /* physical memory regions */
433 int npmemarr; /* number of entries in pmemarr */
434
435 static paddr_t avail_start; /* first available physical page, other
436 than the `etext gap' defined below */
437 static vaddr_t etext_gap_start;/* start of gap between text & data */
438 static vaddr_t etext_gap_end; /* end of gap between text & data */
439 static vaddr_t virtual_avail; /* first free kernel virtual address */
440 static vaddr_t virtual_end; /* last free kernel virtual address */
441
442 static void pmap_page_upload(void);
443
444 int mmu_has_hole;
445
446 vaddr_t prom_vstart; /* For /dev/kmem */
447 vaddr_t prom_vend;
448
449 /*
450 * Memory pool for pmap structures.
451 */
452 static struct pool_cache pmap_cache;
453 static int pmap_pmap_pool_ctor(void *, void *, int);
454 static void pmap_pmap_pool_dtor(void *, void *);
455 static struct pool segmap_pool;
456
457 #if defined(SUN4)
458 /*
459 * [sun4]: segfixmask: on some systems (4/110) "getsegmap()" returns a
460 * partly invalid value. getsegmap returns a 16 bit value on the sun4,
461 * but only the first 8 or so bits are valid (the rest are *supposed* to
462 * be zero. On the 4/110 the bits that are supposed to be zero are
463 * all one instead. e.g. KERNBASE is usually mapped by pmeg number zero.
464 * On a 4/300 getsegmap(KERNBASE) == 0x0000, but
465 * on a 4/100 getsegmap(KERNBASE) == 0xff00
466 *
467 * This confuses mmu_reservemon() and causes it to not reserve the PROM's
468 * pmegs. Then the PROM's pmegs get used during autoconfig and everything
469 * falls apart! (not very fun to debug, BTW.)
470 *
471 * solution: mask the invalid bits in the getsetmap macro.
472 */
473
474 static u_int segfixmask = 0xffffffff; /* all bits valid to start */
475 #else
476 #define segfixmask 0xffffffff /* It's in getsegmap's scope */
477 #endif
478
479 /*
480 * pseudo-functions for mnemonic value
481 */
482 #define getsegmap(va) (CPU_ISSUN4C \
483 ? lduba(va, ASI_SEGMAP) \
484 : (lduha(va, ASI_SEGMAP) & segfixmask))
485 #define setsegmap(va, pmeg) (CPU_ISSUN4C \
486 ? stba(va, ASI_SEGMAP, pmeg) \
487 : stha(va, ASI_SEGMAP, pmeg))
488
489 /* 3-level sun4 MMU only: */
490 #define getregmap(va) ((unsigned)lduha((va)+2, ASI_REGMAP) >> 8)
491 #define setregmap(va, smeg) stha((va)+2, ASI_REGMAP, (smeg << 8))
492
493
494 #if defined(SUN4M) || defined(SUN4D)
495 #if 0
496 #if VM_PROT_READ != 1 || VM_PROT_WRITE != 2 || VM_PROT_EXECUTE != 4
497 #error fix protection code translation table
498 #endif
499 #endif
500 /*
501 * Translation table for kernel vs. PTE protection bits.
502 */
503 const u_int protection_codes[2][8] = {
504 /* kernel */
505 {
506 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE */
507 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_READ */
508 PPROT_N_RWX, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE */
509 PPROT_N_RWX, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_READ */
510 PPROT_N_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_NONE */
511 PPROT_N_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_READ */
512 PPROT_N_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
513 PPROT_N_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
514 },
515
516 /* user */
517 {
518 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE */
519 PPROT_R_R, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_READ */
520 PPROT_RW_RW, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE */
521 PPROT_RW_RW, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_READ */
522 PPROT_X_X, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_NONE */
523 PPROT_RX_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_READ */
524 PPROT_RWX_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
525 PPROT_RWX_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
526 }
527 };
528 #define pte_kprot4m(prot) (protection_codes[0][(prot)])
529 #define pte_uprot4m(prot) (protection_codes[1][(prot)])
530 #define pte_prot4m(pm, prot) \
531 (protection_codes[(pm) == pmap_kernel() ? 0 : 1][(prot)])
532
533 void setpte4m(vaddr_t va, int pte);
534 void setpgt4m(int *ptep, int pte);
535 void setpgt4m_va(vaddr_t, int *, int, int, int, u_int);
536 int updatepte4m(vaddr_t, int *, int, int, int, u_int);
537 #endif /* SUN4M || SUN4D */
538
539 #if defined(MULTIPROCESSOR)
540 #define PMAP_SET_CPUSET(pmap, cpi) \
541 (pmap->pm_cpuset |= (1 << (cpi)->ci_cpuid))
542 #define PMAP_CLR_CPUSET(pmap, cpi) \
543 (pmap->pm_cpuset &= ~(1 << (cpi)->ci_cpuid))
544 #define PMAP_CPUSET(pmap) (pmap->pm_cpuset)
545 #else
546 #define PMAP_SET_CPUSET(pmap, cpi) /* nothing */
547 #define PMAP_CLR_CPUSET(pmap, cpi) /* nothing */
548 #define PMAP_CPUSET(pmap) 1 /* XXX: 1 or 0? */
549 #endif /* MULTIPROCESSOR */
550
551
552 /* Function pointer messiness for supporting multiple sparc architectures
553 * within a single kernel: notice that there are two versions of many of the
554 * functions within this file/module, one for the sun4/sun4c and the other
555 * for the sun4m. For performance reasons (since things like pte bits don't
556 * map nicely between the two architectures), there are separate functions
557 * rather than unified functions which test the cputyp variable. If only
558 * one architecture is being used, then the non-suffixed function calls
559 * are macro-translated into the appropriate xxx4_4c or xxx4m call. If
560 * multiple architectures are defined, the calls translate to (*xxx_p),
561 * i.e. they indirect through function pointers initialized as appropriate
562 * to the run-time architecture in pmap_bootstrap. See also pmap.h.
563 */
564
565 #if defined(SUN4M) || defined(SUN4D)
566 static void mmu_setup4m_L1(int, struct pmap *);
567 static void mmu_setup4m_L2(int, struct regmap *);
568 static void mmu_setup4m_L3(int, struct segmap *);
569 /*static*/ void mmu_reservemon4m(struct pmap *);
570
571 /*static*/ void pmap_changeprot4m(pmap_t, vaddr_t, vm_prot_t, int);
572 /*static*/ void pmap_rmk4m(struct pmap *, vaddr_t, vaddr_t, int, int);
573 /*static*/ void pmap_rmu4m(struct pmap *, vaddr_t, vaddr_t, int, int);
574 /*static*/ int pmap_enk4m(struct pmap *, vaddr_t, vm_prot_t,
575 int, struct vm_page *, int);
576 /*static*/ int pmap_enu4m(struct pmap *, vaddr_t, vm_prot_t,
577 int, struct vm_page *, int);
578 /*static*/ void pv_changepte4m(struct vm_page *, int, int);
579 /*static*/ int pv_syncflags4m(struct vm_page *);
580 /*static*/ int pv_link4m(struct vm_page *, struct pmap *, vaddr_t, u_int *);
581 /*static*/ void pv_unlink4m(struct vm_page *, struct pmap *, vaddr_t);
582 #endif
583
584 #if defined(SUN4) || defined(SUN4C)
585 /*static*/ void mmu_reservemon4_4c(int *, int *);
586 /*static*/ void pmap_changeprot4_4c(pmap_t, vaddr_t, vm_prot_t, int);
587 /*static*/ void pmap_rmk4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
588 /*static*/ void pmap_rmu4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
589 /*static*/ int pmap_enk4_4c(struct pmap *, vaddr_t, vm_prot_t,
590 int, struct vm_page *, int);
591 /*static*/ int pmap_enu4_4c(struct pmap *, vaddr_t, vm_prot_t,
592 int, struct vm_page *, int);
593 /*static*/ void pv_changepte4_4c(struct vm_page *, int, int);
594 /*static*/ int pv_syncflags4_4c(struct vm_page *);
595 /*static*/ int pv_link4_4c(struct vm_page *, struct pmap *, vaddr_t, u_int *);
596 /*static*/ void pv_unlink4_4c(struct vm_page *, struct pmap *, vaddr_t);
597 #endif
598
599 #if !(defined(SUN4M) || defined(SUN4D)) && (defined(SUN4) || defined(SUN4C))
600 #define pmap_rmk pmap_rmk4_4c
601 #define pmap_rmu pmap_rmu4_4c
602
603 #elif (defined(SUN4M) || defined(SUN4D)) && !(defined(SUN4) || defined(SUN4C))
604 #define pmap_rmk pmap_rmk4m
605 #define pmap_rmu pmap_rmu4m
606
607 #else /* must use function pointers */
608
609 /* function pointer declarations */
610 /* from pmap.h: */
611 bool (*pmap_clear_modify_p)(struct vm_page *);
612 bool (*pmap_clear_reference_p)(struct vm_page *);
613 int (*pmap_enter_p)(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int);
614 bool (*pmap_extract_p)(pmap_t, vaddr_t, paddr_t *);
615 bool (*pmap_is_modified_p)(struct vm_page *);
616 bool (*pmap_is_referenced_p)(struct vm_page *);
617 void (*pmap_kenter_pa_p)(vaddr_t, paddr_t, vm_prot_t, u_int);
618 void (*pmap_kremove_p)(vaddr_t, vsize_t);
619 void (*pmap_kprotect_p)(vaddr_t, vsize_t, vm_prot_t);
620 void (*pmap_page_protect_p)(struct vm_page *, vm_prot_t);
621 void (*pmap_protect_p)(pmap_t, vaddr_t, vaddr_t, vm_prot_t);
622 /* local: */
623 void (*pmap_rmk_p)(struct pmap *, vaddr_t, vaddr_t, int, int);
624 void (*pmap_rmu_p)(struct pmap *, vaddr_t, vaddr_t, int, int);
625
626 #define pmap_rmk (*pmap_rmk_p)
627 #define pmap_rmu (*pmap_rmu_p)
628
629 #endif
630
631 /* --------------------------------------------------------------*/
632
633 /*
634 * Next we have some sun4m/4d-specific routines which have no 4/4c
635 * counterparts, or which are 4/4c macros.
636 */
637
638 #if defined(SUN4M) || defined(SUN4D)
639 /*
640 * SP versions of the tlb flush operations.
641 *
642 * Turn off traps to prevent register window overflows from writing
643 * user windows to the wrong stack. Cf. tlb_flush_page_real() &c.
644 */
645 static void
sp_tlb_flush(int va,int ctx,int lvl)646 sp_tlb_flush(int va, int ctx, int lvl)
647 {
648 int opsr, octx;
649
650 va &= ~0xfff;
651 va |= lvl;
652
653 /*
654 * Turn off traps.
655 *
656 * Like setpsr((opsr = getpsr()) & ~PSR_ET); but we can shave
657 * off one instruction b/c we never disable traps recursively,
658 * so we can use the xor done by wrpsr itself to clear the
659 * bit.
660 *
661 * XXX: Add to asm.h? We can use this in cache.c too.
662 */
663 opsr = getpsr(); /* KDASSERT(opsr & PSR_ET); */
664 __asm volatile ("wr %0, %1, %%psr"
665 :: "r"(opsr), "n"(PSR_ET) : "memory");
666 __asm volatile ("nop; nop; nop");
667
668 octx = getcontext4m(); /* save context */
669
670 /* Do the TLB flush in "ctx" */
671 setcontext4m(ctx);
672 __asm volatile ("sta %%g0, [%0]%1" :: "r"(va), "n"(ASI_SRMMUFP));
673
674 setcontext4m(octx); /* restore context */
675 setpsr(opsr); /* turn traps on again */
676 }
677
678 static inline void
sp_tlb_flush_all(void)679 sp_tlb_flush_all(void)
680 {
681
682 sta(ASI_SRMMUFP_LN, ASI_SRMMUFP, 0);
683 }
684
685 #if defined(MULTIPROCESSOR)
686 /*
687 * The SMP versions of the tlb flush routines. We only need to
688 * do a cross call for these on sun4m (Mbus) systems. sun4d systems
689 * have an Xbus which broadcasts TLB demaps in hardware.
690 */
691
692 static inline void smp_tlb_flush_page (int va, int ctx, u_int cpuset);
693 static inline void smp_tlb_flush_segment (int va, int ctx, u_int cpuset);
694 static inline void smp_tlb_flush_region (int va, int ctx, u_int cpuset);
695 static inline void smp_tlb_flush_context (int ctx, u_int cpuset);
696 static inline void smp_tlb_flush_all (void);
697
698 static inline void
smp_tlb_flush_page(int va,int ctx,u_int cpuset)699 smp_tlb_flush_page(int va, int ctx, u_int cpuset)
700 {
701
702 if (CPU_ISSUN4D) {
703 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L3);
704 } else
705 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L3, cpuset);
706 }
707
708 static inline void
smp_tlb_flush_segment(int va,int ctx,u_int cpuset)709 smp_tlb_flush_segment(int va, int ctx, u_int cpuset)
710 {
711
712 if (CPU_ISSUN4D) {
713 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L2);
714 } else
715 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L2, cpuset);
716 }
717
718 static inline void
smp_tlb_flush_region(int va,int ctx,u_int cpuset)719 smp_tlb_flush_region(int va, int ctx, u_int cpuset)
720 {
721
722 if (CPU_ISSUN4D) {
723 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L1);
724 } else
725 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L1, cpuset);
726 }
727
728 static inline void
smp_tlb_flush_context(int ctx,u_int cpuset)729 smp_tlb_flush_context(int ctx, u_int cpuset)
730 {
731
732 if (CPU_ISSUN4D) {
733 sp_tlb_flush(0, ctx, ASI_SRMMUFP_L0);
734 } else
735 FXCALL3(sp_tlb_flush, ft_tlb_flush, 0, ctx, ASI_SRMMUFP_L0, cpuset);
736 }
737
738 static inline void
smp_tlb_flush_all(void)739 smp_tlb_flush_all(void)
740 {
741
742 if (CPU_ISSUN4D) {
743 sp_tlb_flush_all();
744 } else
745 XCALL0(sp_tlb_flush_all, CPUSET_ALL);
746 }
747 #endif /* MULTIPROCESSOR */
748
749 #if defined(MULTIPROCESSOR)
750 #define tlb_flush_page(va,ctx,s) smp_tlb_flush_page(va,ctx,s)
751 #define tlb_flush_segment(va,ctx,s) smp_tlb_flush_segment(va,ctx,s)
752 #define tlb_flush_region(va,ctx,s) smp_tlb_flush_region(va,ctx,s)
753 #define tlb_flush_context(ctx,s) smp_tlb_flush_context(ctx,s)
754 #define tlb_flush_all() smp_tlb_flush_all()
755 #else
756 #define tlb_flush_page(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L3)
757 #define tlb_flush_segment(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L2)
758 #define tlb_flush_region(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L1)
759 #define tlb_flush_context(ctx,s) sp_tlb_flush(0,ctx,ASI_SRMMUFP_L0)
760 #define tlb_flush_all() sp_tlb_flush_all()
761 #endif /* MULTIPROCESSOR */
762
763 static u_int VA2PA(void *);
764 static u_long srmmu_bypass_read(u_long);
765
766 /*
767 * VA2PA(addr) -- converts a virtual address to a physical address using
768 * the MMU's currently-installed page tables. As a side effect, the address
769 * translation used may cause the associated pte to be encached. The correct
770 * context for VA must be set before this is called.
771 *
772 * This routine should work with any level of mapping, as it is used
773 * during bootup to interact with the ROM's initial L1 mapping of the kernel.
774 */
775 static u_int
VA2PA(void * addr)776 VA2PA(void *addr)
777 {
778 u_int pte;
779
780 /*
781 * We'll use that handy SRMMU flush/probe.
782 * Try each level in turn until we find a valid pte. Otherwise panic.
783 */
784
785 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L3, ASI_SRMMUFP);
786 /* Unlock fault status; required on Hypersparc modules */
787 (void)lda(SRMMU_SFSR, ASI_SRMMU);
788 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
789 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
790 ((u_int)addr & 0xfff));
791
792 /* A `TLB Flush Entire' is required before any L0, L1 or L2 probe */
793 tlb_flush_all_real();
794
795 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L2, ASI_SRMMUFP);
796 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
797 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
798 ((u_int)addr & 0x3ffff));
799 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L1, ASI_SRMMUFP);
800 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
801 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
802 ((u_int)addr & 0xffffff));
803 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L0, ASI_SRMMUFP);
804 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
805 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
806 ((u_int)addr & 0xffffffff));
807
808 #ifdef DIAGNOSTIC
809 panic("VA2PA: Asked to translate unmapped VA %p", addr);
810 #else
811 return (0);
812 #endif
813 }
814
815 /*
816 * Atomically update a PTE entry, coping with hardware updating the
817 * PTE at the same time we are. This is the procedure that is
818 * recommended in the SuperSPARC user's manual.
819 */
820 int
updatepte4m(vaddr_t va,int * pte,int bic,int bis,int ctx,u_int cpuset)821 updatepte4m(vaddr_t va, int *pte, int bic, int bis, int ctx, u_int cpuset)
822 {
823 int oldval, swapval;
824 volatile int *vpte = (volatile int *)pte;
825 bool can_lock = lock_available;
826
827 /*
828 * Can only be one of these happening in the system
829 * at any one time.
830 */
831 if (__predict_true(can_lock))
832 mutex_spin_enter(&demap_lock);
833
834 /*
835 * The idea is to loop swapping zero into the pte, flushing
836 * it, and repeating until it stays zero. At this point,
837 * there should be no more hardware accesses to this PTE
838 * so we can modify it without losing any mod/ref info.
839 */
840 oldval = 0;
841 do {
842 swapval = 0;
843 swap(vpte, swapval);
844 tlb_flush_page(va, ctx, cpuset);
845 oldval |= swapval;
846 } while (__predict_false(*vpte != 0));
847
848 swapval = (oldval & ~bic) | bis;
849 swap(vpte, swapval);
850
851 if (__predict_true(can_lock))
852 mutex_spin_exit(&demap_lock);
853
854 return (oldval);
855 }
856
857 inline void
setpgt4m(int * ptep,int pte)858 setpgt4m(int *ptep, int pte)
859 {
860
861 kpreempt_disable();
862 swap(ptep, pte);
863 kpreempt_enable();
864 }
865
866 inline void
setpgt4m_va(vaddr_t va,int * ptep,int pte,int pageflush,int ctx,u_int cpuset)867 setpgt4m_va(vaddr_t va, int *ptep, int pte, int pageflush, int ctx,
868 u_int cpuset)
869 {
870
871 #if defined(MULTIPROCESSOR)
872 updatepte4m(va, ptep, 0xffffffff, pte, pageflush ? ctx : 0, cpuset);
873 #else
874 kpreempt_disable();
875 if (__predict_true(pageflush))
876 tlb_flush_page(va, ctx, 0);
877 setpgt4m(ptep, pte);
878 kpreempt_enable();
879 #endif /* MULTIPROCESSOR */
880 }
881
882 /* Set the page table entry for va to pte. */
883 void
setpte4m(vaddr_t va,int pte)884 setpte4m(vaddr_t va, int pte)
885 {
886 struct pmap *pm;
887 struct regmap *rp;
888 struct segmap *sp;
889
890 #ifdef DEBUG
891 if (getcontext4m() != 0)
892 panic("setpte4m: user context");
893 #endif
894
895 pm = pmap_kernel();
896 rp = &pm->pm_regmap[VA_VREG(va)];
897 sp = &rp->rg_segmap[VA_VSEG(va)];
898
899 tlb_flush_page(va, 0, CPUSET_ALL);
900 setpgt4m(sp->sg_pte + VA_SUN4M_VPG(va), pte);
901 }
902
903 /*
904 * Page table pool back-end.
905 */
906 void *
pgt_page_alloc(struct pool * pp,int flags)907 pgt_page_alloc(struct pool *pp, int flags)
908 {
909 int cacheit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0;
910 struct vm_page *pg;
911 vaddr_t va;
912 paddr_t pa;
913
914 /* Allocate a page of physical memory */
915 while ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL &&
916 (flags & PR_WAITOK) != 0) {
917 uvm_wait("pgtpg");
918 }
919 if (pg == NULL) {
920 KASSERT((flags & PR_WAITOK) == 0);
921 return NULL;
922 }
923
924 /* Allocate virtual memory */
925 va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY |
926 ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK));
927 if (va == 0) {
928 KASSERT((flags & PR_WAITOK) == 0);
929 uvm_pagefree(pg);
930 return (NULL);
931 }
932
933 /*
934 * On systems with a physical data cache we need to flush this page
935 * from the cache if the pagetables cannot be cached.
936 * On systems with a virtually indexed data cache, we only need
937 * to map it non-cacheable, since the page is not currently mapped.
938 */
939 pa = VM_PAGE_TO_PHYS(pg);
940 if (cacheit == 0)
941 pcache_flush_page(pa, 1);
942
943 /* Map the page */
944 pmap_kenter_pa(va, pa | (cacheit ? 0 : PMAP_NC),
945 VM_PROT_READ | VM_PROT_WRITE, 0);
946 pmap_update(pmap_kernel());
947
948 return ((void *)va);
949 }
950
951 void
pgt_page_free(struct pool * pp,void * v)952 pgt_page_free(struct pool *pp, void *v)
953 {
954 vaddr_t va;
955 paddr_t pa;
956 bool rv __diagused;
957
958 va = (vaddr_t)v;
959 rv = pmap_extract(pmap_kernel(), va, &pa);
960 KASSERT(rv);
961 uvm_pagefree(PHYS_TO_VM_PAGE(pa));
962 pmap_kremove(va, PAGE_SIZE);
963 uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY);
964 }
965 #endif /* SUN4M || SUN4D */
966
967 /*----------------------------------------------------------------*/
968
969 /*
970 * The following three macros are to be used in sun4/sun4c code only.
971 */
972 #if defined(SUN4_MMU3L)
973 #define CTX_USABLE(pm,rp) ( \
974 ((pm)->pm_ctx != NULL && \
975 (!HASSUN4_MMU3L || (rp)->rg_smeg != reginval)) \
976 )
977 #else
978 #define CTX_USABLE(pm,rp) ((pm)->pm_ctx != NULL )
979 #endif
980
981 #define GAP_WIDEN(pm,vr) do if (CPU_HAS_SUNMMU) { \
982 if (vr + 1 == pm->pm_gap_start) \
983 pm->pm_gap_start = vr; \
984 if (vr == pm->pm_gap_end) \
985 pm->pm_gap_end = vr + 1; \
986 } while (0)
987
988 #define GAP_SHRINK(pm,vr) do if (CPU_HAS_SUNMMU) { \
989 int x; \
990 x = pm->pm_gap_start + (pm->pm_gap_end - pm->pm_gap_start) / 2; \
991 if (vr > x) { \
992 if (vr < pm->pm_gap_end) \
993 pm->pm_gap_end = vr; \
994 } else { \
995 if (vr >= pm->pm_gap_start && x != pm->pm_gap_start) \
996 pm->pm_gap_start = vr + 1; \
997 } \
998 } while (0)
999
1000
1001 static void get_phys_mem(void **);
1002 #if 0 /* not used */
1003 void kvm_iocache(char *, int);
1004 #endif
1005
1006 #ifdef DEBUG
1007 void pm_check(char *, struct pmap *);
1008 void pm_check_k(char *, struct pmap *);
1009 void pm_check_u(char *, struct pmap *);
1010 #endif
1011
1012 /*
1013 * During the PMAP bootstrap, we can use a simple translation to map a
1014 * kernel virtual address to a psysical memory address (this is arranged
1015 * in locore). Usually, KERNBASE maps to physical address 0. This is always
1016 * the case on sun4 and sun4c machines. On sun4m machines -- if no memory is
1017 * installed in the bank corresponding to physical address 0 -- the PROM may
1018 * elect to load us at some other address, presumably at the start of
1019 * the first memory bank that is available. We set the up the variable
1020 * `va2pa_offset' to hold the physical address corresponding to KERNBASE.
1021 */
1022
1023 static u_long va2pa_offset;
1024 #define PMAP_BOOTSTRAP_VA2PA(v) ((paddr_t)((u_long)(v) - va2pa_offset))
1025 #define PMAP_BOOTSTRAP_PA2VA(p) ((vaddr_t)((u_long)(p) + va2pa_offset))
1026
1027 /*
1028 * Grab physical memory list.
1029 * While here, compute `physmem'.
1030 */
1031 void
get_phys_mem(void ** top)1032 get_phys_mem(void **top)
1033 {
1034 struct memarr *mp;
1035 char *p;
1036 int i;
1037
1038 /* Load the memory descriptor array at the current kernel top */
1039 p = (void *)ALIGN(*top);
1040 pmemarr = (struct memarr *)p;
1041 npmemarr = prom_makememarr(pmemarr, 1000, MEMARR_AVAILPHYS);
1042
1043 /* Update kernel top */
1044 p += npmemarr * sizeof(struct memarr);
1045 *top = p;
1046
1047 for (physmem = 0, mp = pmemarr, i = npmemarr; --i >= 0; mp++)
1048 physmem += btoc(mp->len);
1049 }
1050
1051
1052 /*
1053 * Support functions for vm_page_bootstrap().
1054 */
1055
1056 /*
1057 * How much virtual space does this kernel have?
1058 * (After mapping kernel text, data, etc.)
1059 */
1060 void
pmap_virtual_space(vaddr_t * v_start,vaddr_t * v_end)1061 pmap_virtual_space(vaddr_t *v_start, vaddr_t *v_end)
1062 {
1063
1064 *v_start = virtual_avail;
1065 *v_end = virtual_end;
1066 }
1067
1068 #ifdef PMAP_GROWKERNEL
1069 vaddr_t
pmap_growkernel(vaddr_t eva)1070 pmap_growkernel(vaddr_t eva)
1071 {
1072 struct regmap *rp;
1073 struct segmap *sp;
1074 int vr, evr, M, N, i;
1075 struct vm_page *pg;
1076 vaddr_t va;
1077
1078 if (eva <= virtual_end)
1079 return (virtual_end);
1080
1081 /* For now, only implemented for sun4/sun4c */
1082 KASSERT(CPU_HAS_SUNMMU);
1083
1084 /*
1085 * Map in the next region(s)
1086 */
1087
1088 /* Get current end-of-kernel */
1089 vr = virtual_end >> RGSHIFT;
1090 evr = (eva + NBPRG - 1) >> RGSHIFT;
1091 eva = evr << RGSHIFT;
1092
1093 if (eva > VM_MAX_KERNEL_ADDRESS)
1094 panic("growkernel: grown too large: %lx", eva);
1095
1096 /*
1097 * Divide a region in N blocks of M segments, where each segment
1098 * block can have its PTEs mapped by one page.
1099 * N should come out to 1 for 8K pages and to 4 for 4K pages.
1100 */
1101 M = NBPG / (NPTESG * sizeof(int));
1102 N = (NBPRG/NBPSG) / M;
1103
1104 while (vr < evr) {
1105 rp = &pmap_kernel()->pm_regmap[vr];
1106 for (i = 0; i < N; i++) {
1107 sp = &rp->rg_segmap[i * M];
1108 va = (vaddr_t)sp->sg_pte;
1109 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
1110 if (pg == NULL)
1111 panic("growkernel: out of memory");
1112 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
1113 VM_PROT_READ | VM_PROT_WRITE, 0);
1114 }
1115 }
1116
1117 virtual_end = eva;
1118 return (eva);
1119 }
1120 #endif
1121
1122 /*
1123 * Helper routine that hands off available physical pages to the VM system.
1124 */
1125 static void
pmap_page_upload(void)1126 pmap_page_upload(void)
1127 {
1128 int n;
1129 paddr_t pstart, pend;
1130
1131 /* First, the `etext gap' */
1132 pstart = PMAP_BOOTSTRAP_VA2PA(etext_gap_start);
1133 pend = PMAP_BOOTSTRAP_VA2PA(etext_gap_end);
1134
1135 #ifdef DIAGNOSTIC
1136 if (avail_start <= pstart)
1137 panic("pmap_page_upload: etext gap overlap: %lx < %lx",
1138 (u_long)avail_start, (u_long)pstart);
1139 #endif
1140 if (etext_gap_start < etext_gap_end) {
1141 vm_first_phys = pstart;
1142 uvm_page_physload(
1143 atop(pstart),
1144 atop(pend),
1145 atop(pstart),
1146 atop(pend), VM_FREELIST_DEFAULT);
1147 }
1148
1149 for (n = 0; n < npmemarr; n++) {
1150
1151 pstart = pmemarr[n].addr;
1152 pend = pstart + pmemarr[n].len;
1153
1154 /* Update vm_{first_last}_phys */
1155 if (vm_first_phys > pstart)
1156 vm_first_phys = pstart;
1157 if (vm_last_phys < pend)
1158 vm_last_phys = pend;
1159
1160 /*
1161 * Exclude any memory allocated for the kernel as computed
1162 * by pmap_bootstrap(), i.e. the range
1163 * [KERNBASE_PA, avail_start>.
1164 * Note that this will also exclude the `etext gap' range
1165 * already uploaded above.
1166 */
1167 if (pstart < PMAP_BOOTSTRAP_VA2PA(KERNBASE)) {
1168 /*
1169 * This segment starts below the kernel load address.
1170 * Chop it off at the pstart of the kernel.
1171 */
1172 paddr_t chop = PMAP_BOOTSTRAP_VA2PA(KERNBASE);
1173
1174 if (pend < chop)
1175 chop = pend;
1176 #ifdef DEBUG
1177 prom_printf("bootstrap gap: pstart %lx, chop %lx, pend %lx\n",
1178 pstart, chop, pend);
1179 #endif
1180 uvm_page_physload(
1181 atop(pstart),
1182 atop(chop),
1183 atop(pstart),
1184 atop(chop),
1185 VM_FREELIST_DEFAULT);
1186
1187 /*
1188 * Adjust the start address to reflect the
1189 * uploaded portion of this segment.
1190 */
1191 pstart = chop;
1192 }
1193
1194 /* Skip the current kernel address range */
1195 if (pstart <= avail_start && avail_start < pend)
1196 pstart = avail_start;
1197
1198 if (pstart == pend)
1199 continue;
1200
1201 /* Upload (the rest of) this segment */
1202 uvm_page_physload(
1203 atop(pstart),
1204 atop(pend),
1205 atop(pstart),
1206 atop(pend), VM_FREELIST_DEFAULT);
1207 }
1208
1209 #if defined(MULTIPROCESSOR)
1210 {
1211 CPU_INFO_ITERATOR cpunum;
1212 struct cpu_info *cpi;
1213
1214 for (CPU_INFO_FOREACH(cpunum, cpi)) {
1215 if (cpi->ci_free_sva1)
1216 uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
1217 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
1218 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
1219 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
1220 VM_FREELIST_DEFAULT);
1221 if (cpi->ci_free_sva2)
1222 uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
1223 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
1224 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
1225 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
1226 VM_FREELIST_DEFAULT);
1227 }
1228 }
1229 #endif
1230 }
1231
1232 /*
1233 * This routine is used by mmrw() to validate access to `/dev/mem'.
1234 */
1235 int
pmap_pa_exists(paddr_t pa)1236 pmap_pa_exists(paddr_t pa)
1237 {
1238 int nmem;
1239 struct memarr *mp;
1240
1241 for (mp = pmemarr, nmem = npmemarr; --nmem >= 0; mp++) {
1242 if (pa >= mp->addr && pa < mp->addr + mp->len)
1243 return 1;
1244 }
1245
1246 return 0;
1247 }
1248
1249 /* update pv_flags given a valid pte */
1250 #define MR4_4C(pte) (((pte) >> PG_M_SHIFT) & (PV_MOD | PV_REF))
1251 #define MR4M(pte) (((pte) >> PG_M_SHIFT4M) & (PV_MOD4M | PV_REF4M))
1252
1253 /*----------------------------------------------------------------*/
1254
1255 /*
1256 * Agree with the monitor ROM as to how many MMU entries are
1257 * to be reserved, and map all of its segments into all contexts.
1258 *
1259 * Unfortunately, while the Version 0 PROM had a nice linked list of
1260 * taken virtual memory, the Version 2 PROM provides instead a convoluted
1261 * description of *free* virtual memory. Rather than invert this, we
1262 * resort to two magic constants from the PROM vector description file.
1263 */
1264 #if defined(SUN4) || defined(SUN4C)
1265 void
mmu_reservemon4_4c(int * nrp,int * nsp)1266 mmu_reservemon4_4c(int *nrp, int *nsp)
1267 {
1268 u_int va = 0, eva = 0;
1269 int mmuseg, i, nr, ns, vr;
1270 int *pte;
1271 #if defined(SUN4_MMU3L)
1272 int mmureg, lastvr = 0;
1273 #endif
1274 struct regmap *rp;
1275
1276 #if defined(SUN4)
1277 if (CPU_ISSUN4) {
1278 prom_vstart = va = OLDMON_STARTVADDR;
1279 prom_vend = eva = OLDMON_ENDVADDR;
1280 }
1281 #endif
1282 #if defined(SUN4C)
1283 if (CPU_ISSUN4C) {
1284 prom_vstart = va = OPENPROM_STARTVADDR;
1285 prom_vend = eva = OPENPROM_ENDVADDR;
1286 }
1287 #endif
1288 ns = *nsp;
1289 nr = *nrp;
1290 while (va < eva) {
1291 vr = VA_VREG(va);
1292 rp = &pmap_kernel()->pm_regmap[vr];
1293
1294 #if defined(SUN4_MMU3L)
1295 if (HASSUN4_MMU3L && vr != lastvr) {
1296 lastvr = vr;
1297 mmureg = getregmap(va);
1298 if (mmureg < nr)
1299 rp->rg_smeg = nr = mmureg;
1300 /*
1301 * On 3-level MMU machines, we distribute regions,
1302 * rather than segments, amongst the contexts.
1303 */
1304 for (i = ncontext; --i > 0;)
1305 prom_setcontext(i, (void *)va, mmureg);
1306 }
1307 #endif
1308 mmuseg = getsegmap(va);
1309 if (mmuseg < ns)
1310 ns = mmuseg;
1311
1312 if (!HASSUN4_MMU3L)
1313 for (i = ncontext; --i > 0;)
1314 prom_setcontext(i, (void *)va, mmuseg);
1315
1316 if (mmuseg == seginval) {
1317 va += NBPSG;
1318 continue;
1319 }
1320 /*
1321 * Another PROM segment. Enter into region map.
1322 * Assume the entire segment is valid.
1323 */
1324 rp->rg_nsegmap += 1;
1325 rp->rg_segmap[VA_VSEG(va)].sg_pmeg = mmuseg;
1326 rp->rg_segmap[VA_VSEG(va)].sg_npte = NPTESG;
1327 pte = rp->rg_segmap[VA_VSEG(va)].sg_pte;
1328
1329 /* PROM maps its memory user-accessible: fix it. */
1330 for (i = NPTESG; --i >= 0; va += NBPG, pte++) {
1331 *pte = getpte4(va) | PG_S;
1332 setpte4(va, *pte);
1333 }
1334 }
1335 *nsp = ns;
1336 *nrp = nr;
1337 return;
1338 }
1339 #endif
1340
1341 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */
1342
1343 u_long
srmmu_bypass_read(u_long paddr)1344 srmmu_bypass_read(u_long paddr)
1345 {
1346 unsigned long v;
1347
1348 if (cpuinfo.mxcc) {
1349 /*
1350 * We're going to have to use MMU passthrough. If we're on
1351 * a Viking SuperSPARC with a MultiCache Controller, we
1352 * need to set the AC (Alternate Cacheable) bit in the MMU's
1353 * control register in order to not by-pass the cache.
1354 */
1355
1356 unsigned long s = lda(SRMMU_PCR, ASI_SRMMU);
1357
1358 /* set MMU AC bit */
1359 sta(SRMMU_PCR, ASI_SRMMU, s | VIKING_PCR_AC);
1360 v = lda(paddr, ASI_BYPASS);
1361 sta(SRMMU_PCR, ASI_SRMMU, s);
1362 } else
1363 v = lda(paddr, ASI_BYPASS);
1364
1365 return (v);
1366 }
1367
1368
1369 /*
1370 * Take the monitor's initial page table layout, convert it to 3rd-level pte's
1371 * (it starts out as a L1 mapping), and install it along with a set of kernel
1372 * mapping tables as the kernel's initial page table setup. Also create and
1373 * enable a context table. I suppose we also want to block user-mode access
1374 * to the new kernel/ROM mappings.
1375 */
1376
1377 /*
1378 * mmu_reservemon4m(): Copies the existing (ROM) page tables to kernel space,
1379 * converting any L1/L2 PTEs to L3 PTEs. Does *not* copy the L1 entry mapping
1380 * the kernel at KERNBASE since we don't want to map 16M of physical
1381 * memory for the kernel. Thus the kernel must be installed later!
1382 * Also installs ROM mappings into the kernel pmap.
1383 * NOTE: This also revokes all user-mode access to the mapped regions.
1384 */
1385 void
mmu_reservemon4m(struct pmap * kpmap)1386 mmu_reservemon4m(struct pmap *kpmap)
1387 {
1388 unsigned int rom_ctxtbl;
1389 int te;
1390
1391 #if !(defined(PROM_AT_F0) || defined(MSIIEP))
1392 prom_vstart = OPENPROM_STARTVADDR;
1393 prom_vend = OPENPROM_ENDVADDR;
1394 #else /* OBP3/OFW in JavaStations */
1395 prom_vstart = 0xf0000000;
1396 #if defined(MSIIEP)
1397 prom_vend = 0xf0800000;
1398 #else
1399 prom_vend = 0xf0080000;
1400 #endif
1401 #endif
1402
1403 /*
1404 * XXX: although the sun4m can handle 36 bits of physical
1405 * address space, we assume that all these page tables, etc
1406 * are in the lower 4G (32-bits) of address space, i.e. out of I/O
1407 * space. Eventually this should be changed to support the 36 bit
1408 * physical addressing, in case some crazed ROM designer decides to
1409 * stick the pagetables up there. In that case, we should use MMU
1410 * transparent mode, (i.e. ASI 0x20 to 0x2f) to access
1411 * physical memory.
1412 */
1413
1414 rom_ctxtbl = (lda(SRMMU_CXTPTR,ASI_SRMMU) << SRMMU_PPNPASHIFT);
1415
1416 te = srmmu_bypass_read(rom_ctxtbl); /* i.e. context 0 */
1417
1418 switch (te & SRMMU_TETYPE) {
1419 case SRMMU_TEINVALID:
1420 cpuinfo.ctx_tbl[0] = SRMMU_TEINVALID;
1421 panic("mmu_reservemon4m: no existing L0 mapping! "
1422 "(How are we running?");
1423 break;
1424 case SRMMU_TEPTE:
1425 panic("mmu_reservemon4m: can't handle ROM 4G page size");
1426 /* XXX: Should make this work, however stupid it is */
1427 break;
1428 case SRMMU_TEPTD:
1429 mmu_setup4m_L1(te, kpmap);
1430 break;
1431 default:
1432 panic("mmu_reservemon4m: unknown pagetable entry type");
1433 }
1434 }
1435
1436 /* regtblptd - PTD for region table to be remapped */
1437 void
mmu_setup4m_L1(int regtblptd,struct pmap * kpmap)1438 mmu_setup4m_L1(int regtblptd, struct pmap *kpmap)
1439 {
1440 unsigned int regtblrover;
1441 int i;
1442 unsigned int te;
1443 struct regmap *rp;
1444 int j, k;
1445
1446 /*
1447 * Here we scan the region table to copy any entries which appear.
1448 * We are only concerned with regions in kernel space and above
1449 * (i.e. regions VA_VREG(KERNBASE)+1 to 0xff). We ignore the first
1450 * region (at VA_VREG(KERNBASE)), since that is the 16MB L1 mapping
1451 * that the ROM used to map the kernel in initially. Later, we will
1452 * rebuild a new L3 mapping for the kernel and install it before
1453 * switching to the new pagetables.
1454 */
1455 regtblrover =
1456 ((regtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT) +
1457 (VA_VREG(KERNBASE)+1) * sizeof(long); /* kernel only */
1458
1459 for (i = VA_VREG(KERNBASE) + 1; i < SRMMU_L1SIZE;
1460 i++, regtblrover += sizeof(long)) {
1461
1462 /* The region we're dealing with */
1463 rp = &kpmap->pm_regmap[i];
1464
1465 te = srmmu_bypass_read(regtblrover);
1466 switch(te & SRMMU_TETYPE) {
1467 case SRMMU_TEINVALID:
1468 break;
1469
1470 case SRMMU_TEPTE:
1471 #ifdef DEBUG
1472 prom_printf("mmu_setup4m_L1: "
1473 "converting region 0x%x from L1->L3\n", i);
1474 #endif
1475 /*
1476 * This region entry covers 64MB of memory -- or
1477 * (NSEGRG * NPTESG) pages -- which we must convert
1478 * into a 3-level description.
1479 */
1480
1481 for (j = 0; j < SRMMU_L2SIZE; j++) {
1482 struct segmap *sp = &rp->rg_segmap[j];
1483
1484 for (k = 0; k < SRMMU_L3SIZE; k++) {
1485 setpgt4m(&sp->sg_pte[k],
1486 (te & SRMMU_L1PPNMASK) |
1487 (j << SRMMU_L2PPNSHFT) |
1488 (k << SRMMU_L3PPNSHFT) |
1489 (te & SRMMU_PGBITSMSK) |
1490 ((te & SRMMU_PROT_MASK) |
1491 PPROT_U2S_OMASK) |
1492 SRMMU_TEPTE);
1493 }
1494 }
1495 break;
1496
1497 case SRMMU_TEPTD:
1498 mmu_setup4m_L2(te, rp);
1499 break;
1500
1501 default:
1502 panic("mmu_setup4m_L1: unknown pagetable entry type");
1503 }
1504 }
1505 }
1506
1507 void
mmu_setup4m_L2(int segtblptd,struct regmap * rp)1508 mmu_setup4m_L2(int segtblptd, struct regmap *rp)
1509 {
1510 unsigned int segtblrover;
1511 int i, k;
1512 unsigned int te;
1513 struct segmap *sp;
1514
1515 segtblrover = (segtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
1516 for (i = 0; i < SRMMU_L2SIZE; i++, segtblrover += sizeof(long)) {
1517
1518 sp = &rp->rg_segmap[i];
1519
1520 te = srmmu_bypass_read(segtblrover);
1521 switch(te & SRMMU_TETYPE) {
1522 case SRMMU_TEINVALID:
1523 break;
1524
1525 case SRMMU_TEPTE:
1526 #ifdef DEBUG
1527 prom_printf("mmu_setup4m_L2: converting L2 entry at segment 0x%x to L3\n",i);
1528 #endif
1529 /*
1530 * This segment entry covers 256KB of memory -- or
1531 * (NPTESG) pages -- which we must convert
1532 * into a 3-level description.
1533 */
1534 for (k = 0; k < SRMMU_L3SIZE; k++) {
1535 setpgt4m(&sp->sg_pte[k],
1536 (te & SRMMU_L1PPNMASK) |
1537 (te & SRMMU_L2PPNMASK) |
1538 (k << SRMMU_L3PPNSHFT) |
1539 (te & SRMMU_PGBITSMSK) |
1540 ((te & SRMMU_PROT_MASK) |
1541 PPROT_U2S_OMASK) |
1542 SRMMU_TEPTE);
1543 }
1544 break;
1545
1546 case SRMMU_TEPTD:
1547 mmu_setup4m_L3(te, sp);
1548 break;
1549
1550 default:
1551 panic("mmu_setup4m_L2: unknown pagetable entry type");
1552 }
1553 }
1554 }
1555
1556 void
mmu_setup4m_L3(int pagtblptd,struct segmap * sp)1557 mmu_setup4m_L3(int pagtblptd, struct segmap *sp)
1558 {
1559 unsigned int pagtblrover;
1560 int i;
1561 unsigned int te;
1562
1563 pagtblrover = (pagtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
1564 for (i = 0; i < SRMMU_L3SIZE; i++, pagtblrover += sizeof(long)) {
1565 te = srmmu_bypass_read(pagtblrover);
1566 switch(te & SRMMU_TETYPE) {
1567 case SRMMU_TEINVALID:
1568 break;
1569 case SRMMU_TEPTE:
1570 setpgt4m(&sp->sg_pte[i], te | PPROT_U2S_OMASK);
1571 pmap_kernel()->pm_stats.resident_count++;
1572 break;
1573 case SRMMU_TEPTD:
1574 panic("mmu_setup4m_L3: PTD found in L3 page table");
1575 default:
1576 panic("mmu_setup4m_L3: unknown pagetable entry type");
1577 }
1578 }
1579 }
1580 #endif /* defined SUN4M || defined SUN4D */
1581
1582 /*----------------------------------------------------------------*/
1583
1584 #if defined(SUN4) || defined(SUN4C)
1585 /*
1586 * MMU management.
1587 */
1588 static int me_alloc(struct mmuentry *, struct pmap *, int, int);
1589 static void me_free(struct pmap *, u_int);
1590 #if defined(SUN4_MMU3L)
1591 static int region_alloc(struct mmuentry *, struct pmap *, int);
1592 static void region_free(struct pmap *, u_int);
1593 #endif
1594
1595
1596 /*
1597 * Allocate an MMU entry (i.e., a PMEG).
1598 * If necessary, steal one from someone else.
1599 * Put it on the tail of the given queue
1600 * (which is either the LRU list or the locked list).
1601 * The locked list is not actually ordered, but this is easiest.
1602 * Also put it on the given (new) pmap's chain,
1603 * enter its pmeg number into that pmap's segmap,
1604 * and store the pmeg's new virtual segment number (me->me_vseg).
1605 *
1606 * This routine is large and complicated, but it must be fast
1607 * since it implements the dynamic allocation of MMU entries.
1608 */
1609
1610 static inline int
me_alloc(struct mmuentry * mh,struct pmap * newpm,int newvreg,int newvseg)1611 me_alloc(struct mmuentry *mh, struct pmap *newpm, int newvreg, int newvseg)
1612 {
1613 struct mmuentry *me;
1614 struct pmap *pm;
1615 int i, va, *ptep, pte;
1616 int ctx;
1617 struct regmap *rp;
1618 struct segmap *sp;
1619
1620 /* try free list first */
1621 if (!mmuq_empty(&segm_freelist)) {
1622 me = mmuq_first(&segm_freelist);
1623 mmuq_remove(me);
1624 #ifdef DEBUG
1625 if (me->me_pmap != NULL)
1626 panic("me_alloc: freelist entry has pmap");
1627 DPRINTF(PDB_MMU_ALLOC,
1628 "me_alloc: got pmeg %d", me->me_cookie);
1629 #endif
1630 mmuq_insert_tail(mh, me);
1631
1632 /* onto on pmap chain; pmap is already locked, if needed */
1633 TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);
1634 #ifdef DIAGNOSTIC
1635 pmap_stats.ps_npmeg_free--;
1636 if (mh == &segm_locked) {
1637 pmap_stats.ps_npmeg_locked++;
1638 me->me_statp = &pmap_stats.ps_npmeg_locked;
1639 } else {
1640 pmap_stats.ps_npmeg_lru++;
1641 me->me_statp = &pmap_stats.ps_npmeg_lru;
1642 }
1643 #endif
1644
1645 /* into pmap segment table, with backpointers */
1646 me->me_pmap = newpm;
1647 me->me_vseg = newvseg;
1648 me->me_vreg = newvreg;
1649
1650 return (me->me_cookie);
1651 }
1652
1653 /* no luck, take head of LRU list */
1654 if (mmuq_empty(&segm_lru))
1655 panic("me_alloc: all pmegs gone");
1656
1657 me = mmuq_first(&segm_lru);
1658 pm = me->me_pmap;
1659 DPRINTF(PDB_MMU_ALLOC | PDB_MMU_STEAL,
1660 "me_alloc: stealing pmeg 0x%x from pmap %p", me->me_cookie, pm);
1661
1662 mmu_stolenpmegs_evcnt.ev_count++;
1663
1664 /*
1665 * Remove from LRU list, and insert at end of new list
1666 * (probably the LRU list again, but so what?).
1667 */
1668 mmuq_remove(me);
1669 mmuq_insert_tail(mh, me);
1670
1671 #ifdef DIAGNOSTIC
1672 if (mh == &segm_locked) {
1673 pmap_stats.ps_npmeg_lru--;
1674 pmap_stats.ps_npmeg_locked++;
1675 me->me_statp = &pmap_stats.ps_npmeg_locked;
1676 } else {
1677 me->me_statp = &pmap_stats.ps_npmeg_lru;
1678 }
1679 #endif
1680
1681 rp = &pm->pm_regmap[me->me_vreg];
1682 sp = &rp->rg_segmap[me->me_vseg];
1683 ptep = sp->sg_pte;
1684
1685 #ifdef DEBUG
1686 if (sp->sg_pmeg != me->me_cookie)
1687 panic("me_alloc: wrong sg_pmeg (%d != %d)",
1688 sp->sg_pmeg, me->me_cookie);
1689 #endif
1690
1691 /*
1692 * The PMEG must be mapped into some context so that we can
1693 * read its PTEs. Use its current context if it has one;
1694 * if not, and since context 0 is reserved for the kernel,
1695 * the simplest method is to switch to 0 and map the PMEG
1696 * to virtual address 0---which, being a user space address,
1697 * is by definition not in use.
1698 *
1699 * XXX do not have to flush cache immediately
1700 */
1701 ctx = getcontext4();
1702
1703 /*
1704 * Even if we're stealing a PMEG from ourselves (i.e. if pm==newpm),
1705 * we must make sure there are no user register windows in the CPU
1706 * for the following reasons:
1707 * (1) if we have a write-allocate cache and the segment we are
1708 * stealing contains stack pages, an interrupt during the
1709 * interval that starts at cache_flush_segment() below and ends
1710 * when the segment is finally removed from the MMU, may cause
1711 * dirty cache lines to reappear.
1712 * (2) when re-wiring this PMEG for use by another segment (e.g.
1713 * in mmu_pagein()) a window exists where the PTEs in this PMEG
1714 * point at arbitrary pages allocated to this address space.
1715 * Again, a register window flush at this point is likely to
1716 * cause data corruption in case the segment being rewired
1717 * contains stack virtual addresses.
1718 */
1719 write_user_windows();
1720 if (CTX_USABLE(pm,rp)) {
1721 setcontext4(pm->pm_ctxnum);
1722 va = VSTOVA(me->me_vreg, me->me_vseg);
1723 #ifdef DEBUG
1724 if (getsegmap(va) != me->me_cookie)
1725 panic("me_alloc: wrong pmeg in MMU (%d != %d)",
1726 getsegmap(va), me->me_cookie);
1727 #endif
1728 cache_flush_segment(me->me_vreg, me->me_vseg, pm->pm_ctxnum);
1729 } else {
1730 va = 0;
1731 setcontext4(0);
1732 if (HASSUN4_MMU3L)
1733 setregmap(va, tregion);
1734 setsegmap(va, me->me_cookie);
1735 /*
1736 * No cache flush needed: it happened earlier when
1737 * the old context was taken.
1738 */
1739 }
1740
1741 /*
1742 * Record reference and modify bits for each page,
1743 * and copy PTEs into kernel memory so that they can
1744 * be reloaded later.
1745 */
1746 i = NPTESG;
1747 do {
1748 int swbits = *ptep & PG_MBZ;
1749 pte = getpte4(va);
1750 if ((pte & (PG_V | PG_TYPE)) == (PG_V | PG_OBMEM)) {
1751 struct vm_page *pg;
1752 if ((pg = pvhead4_4c(pte)) != NULL)
1753 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
1754 }
1755 *ptep++ = swbits | (pte & ~(PG_U|PG_M));
1756 va += NBPG;
1757 } while (--i > 0);
1758
1759 /* update segment tables */
1760 if (CTX_USABLE(pm,rp)) {
1761 va = VSTOVA(me->me_vreg,me->me_vseg);
1762 if (pm != pmap_kernel() || HASSUN4_MMU3L)
1763 setsegmap(va, seginval);
1764 else {
1765 /* Unmap segment from all contexts */
1766 for (i = ncontext; --i >= 0;) {
1767 setcontext4(i);
1768 setsegmap(va, seginval);
1769 }
1770 }
1771 }
1772 sp->sg_pmeg = seginval;
1773
1774 /* off old pmap chain */
1775 TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);
1776 setcontext4(ctx);
1777
1778 /* onto new pmap chain; new pmap is already locked, if needed */
1779 TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);
1780
1781 /* into new segment table, with backpointers */
1782 me->me_pmap = newpm;
1783 me->me_vseg = newvseg;
1784 me->me_vreg = newvreg;
1785
1786 return (me->me_cookie);
1787 }
1788
1789 /*
1790 * Free an MMU entry.
1791 *
1792 * Assumes the corresponding pmap is already locked.
1793 * Caller must update hardware.
1794 */
1795 static inline void
me_free(struct pmap * pm,u_int pmeg)1796 me_free(struct pmap *pm, u_int pmeg)
1797 {
1798 struct mmuentry *me = &mmusegments[pmeg];
1799 #ifdef DEBUG
1800 struct regmap *rp;
1801 int i, va, tpte, ctx;
1802 #endif
1803
1804 #ifdef DEBUG
1805 rp = &pm->pm_regmap[me->me_vreg];
1806 DPRINTF(PDB_MMU_ALLOC,
1807 "me_free: freeing pmeg %d from pmap %p", me->me_cookie, pm);
1808 if (me->me_cookie != pmeg)
1809 panic("me_free: wrong mmuentry");
1810 if (pm != me->me_pmap)
1811 panic("me_free: pm != me_pmap");
1812 if (rp->rg_segmap[me->me_vseg].sg_pmeg != pmeg &&
1813 rp->rg_segmap[me->me_vseg].sg_pmeg != seginval)
1814 panic("me_free: wrong sg_pmeg (%d != %d)",
1815 rp->rg_segmap[me->me_vseg].sg_pmeg, pmeg);
1816
1817 /* check for spurious mappings (using temp. mapping in context 0) */
1818 ctx = getcontext4();
1819 setcontext4(0);
1820 if (HASSUN4_MMU3L)
1821 setregmap(0, tregion);
1822 setsegmap(0, me->me_cookie);
1823 va = 0;
1824 i = NPTESG;
1825 do {
1826 tpte = getpte4(va);
1827 if ((tpte & PG_V) == PG_V)
1828 panic("me_free: segment not clean (pte=%x)", tpte);
1829 va += NBPG;
1830 } while (--i > 0);
1831 setcontext4(ctx);
1832 #endif /* DEBUG */
1833
1834 /* take mmu entry off pmap chain */
1835 TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);
1836
1837 /* off LRU or lock chain */
1838 mmuq_remove(me);
1839 #ifdef DIAGNOSTIC
1840 if (me->me_statp == NULL)
1841 panic("me_statp");
1842 (*me->me_statp)--;
1843 me->me_statp = NULL;
1844 #endif
1845
1846 /* no associated pmap; on free list */
1847 me->me_pmap = NULL;
1848 mmuq_insert_tail(&segm_freelist, me);
1849 #ifdef DIAGNOSTIC
1850 pmap_stats.ps_npmeg_free++;
1851 #endif
1852 }
1853
1854 #if defined(SUN4_MMU3L)
1855
1856 /* XXX - Merge with segm_alloc/segm_free ? */
1857
1858 int
region_alloc(struct mmuentry * mh,struct pmap * newpm,int newvr)1859 region_alloc(struct mmuentry *mh, struct pmap *newpm, int newvr)
1860 {
1861 struct mmuentry *me;
1862 struct pmap *pm;
1863 int ctx;
1864 struct regmap *rp;
1865
1866 /* try free list first */
1867 if (!mmuq_empty(®ion_freelist)) {
1868 me = mmuq_first(®ion_freelist);
1869 mmuq_remove(me);
1870 #ifdef DEBUG
1871 if (me->me_pmap != NULL)
1872 panic("region_alloc: freelist entry has pmap");
1873 DPRINTF(PDB_MMUREG_ALLOC,
1874 "region_alloc: got smeg 0x%x", me->me_cookie);
1875 #endif
1876 mmuq_insert_tail(mh, me);
1877
1878 /* onto on pmap chain; pmap is already locked, if needed */
1879 TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);
1880
1881 /* into pmap segment table, with backpointers */
1882 me->me_pmap = newpm;
1883 me->me_vreg = newvr;
1884
1885 return (me->me_cookie);
1886 }
1887
1888 /* no luck, take head of LRU list */
1889 if (mmuq_empty(®ion_lru))
1890 panic("region_alloc: all smegs gone");
1891
1892 me = mmuq_first(®ion_lru);
1893
1894 pm = me->me_pmap;
1895 if (pm == NULL)
1896 panic("region_alloc: LRU entry has no pmap");
1897 if (pm == pmap_kernel())
1898 panic("region_alloc: stealing from kernel");
1899 DPRINTF(PDB_MMUREG_ALLOC | PDB_MMUREG_STEAL,
1900 "region_alloc: stealing smeg 0x%x from pmap %p",
1901 me->me_cookie, pm);
1902
1903 /*
1904 * Remove from LRU list, and insert at end of new list
1905 * (probably the LRU list again, but so what?).
1906 */
1907 mmuq_remove(me);
1908 mmuq_insert_tail(mh, me);
1909
1910 rp = &pm->pm_regmap[me->me_vreg];
1911 ctx = getcontext4();
1912
1913 /* Flush register windows; see comment in me_alloc() */
1914 write_user_windows();
1915 if (pm->pm_ctx) {
1916 setcontext4(pm->pm_ctxnum);
1917 cache_flush_region(me->me_vreg, pm->pm_ctxnum);
1918 }
1919
1920 /* update region tables */
1921 if (pm->pm_ctx)
1922 setregmap(VRTOVA(me->me_vreg), reginval);
1923 rp->rg_smeg = reginval;
1924
1925 /* off old pmap chain */
1926 TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);
1927 setcontext4(ctx); /* done with old context */
1928
1929 /* onto new pmap chain; new pmap is already locked, if needed */
1930 TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);
1931
1932 /* into new segment table, with backpointers */
1933 me->me_pmap = newpm;
1934 me->me_vreg = newvr;
1935
1936 return (me->me_cookie);
1937 }
1938
1939 /*
1940 * Free an MMU entry.
1941 * Assumes the corresponding pmap is already locked.
1942 * Caller must update hardware.
1943 */
1944 void
region_free(struct pmap * pm,u_int smeg)1945 region_free(struct pmap *pm, u_int smeg)
1946 {
1947 struct mmuentry *me = &mmuregions[smeg];
1948
1949 DPRINTF(PDB_MMUREG_ALLOC,
1950 "region_free: freeing smeg 0x%x from pmap %p", me->me_cookie, pm);
1951 #ifdef DEBUG
1952 if (me->me_cookie != smeg)
1953 panic("region_free: wrong mmuentry");
1954 if (pm != me->me_pmap)
1955 panic("region_free: pm != me_pmap");
1956 #endif
1957
1958 /* take mmu entry off pmap chain */
1959 TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);
1960
1961 /* off LRU or lock chain */
1962 mmuq_remove(me);
1963
1964 /* no associated pmap; on free list */
1965 me->me_pmap = NULL;
1966 mmuq_insert_tail(®ion_freelist, me);
1967 }
1968
1969 static void
mmu_pagein_reg(struct pmap * pm,struct regmap * rp,vaddr_t va,int vr,struct mmuentry * mh)1970 mmu_pagein_reg(struct pmap *pm, struct regmap *rp, vaddr_t va,
1971 int vr, struct mmuentry *mh)
1972 {
1973 int i, s, smeg;
1974
1975 va = VA_ROUNDDOWNTOREG(va);
1976 rp->rg_smeg = smeg = region_alloc(mh, pm, vr);
1977
1978 s = splvm();
1979 if (pm == pmap_kernel()) {
1980 /* Map region into all contexts */
1981 int ctx = getcontext4();
1982 i = ncontext - 1;
1983 do {
1984 setcontext4(i);
1985 setregmap(va, smeg);
1986 } while (--i >= 0);
1987 setcontext4(ctx);
1988 } else
1989 setregmap(va, smeg);
1990
1991 /* Load PMEGs into this region */
1992 for (i = 0; i < NSEGRG; i++) {
1993 setsegmap(va, rp->rg_segmap[i].sg_pmeg);
1994 va += NBPSG;
1995 }
1996 splx(s);
1997 }
1998 #endif /* SUN4_MMU3L */
1999
2000 static void
mmu_pmeg_lock(int pmeg)2001 mmu_pmeg_lock(int pmeg)
2002 {
2003 struct mmuentry *me = &mmusegments[pmeg];
2004
2005 mmuq_remove(me);
2006 mmuq_insert_tail(&segm_locked, me);
2007 #ifdef DIAGNOSTIC
2008 (*me->me_statp)--;
2009 pmap_stats.ps_npmeg_locked++;
2010 me->me_statp = &pmap_stats.ps_npmeg_locked;
2011 #endif
2012 }
2013
2014 static void
mmu_pmeg_unlock(int pmeg)2015 mmu_pmeg_unlock(int pmeg)
2016 {
2017 struct mmuentry *me = &mmusegments[pmeg];
2018
2019 mmuq_remove(me);
2020 mmuq_insert_tail(&segm_lru, me);
2021 #ifdef DIAGNOSTIC
2022 (*me->me_statp)--;
2023 pmap_stats.ps_npmeg_lru++;
2024 me->me_statp = &pmap_stats.ps_npmeg_lru;
2025 #endif
2026 }
2027
2028 static void
mmu_pagein_seg(struct pmap * pm,struct segmap * sp,vaddr_t va,int vr,int vs,struct mmuentry * mh)2029 mmu_pagein_seg(struct pmap *pm, struct segmap *sp, vaddr_t va,
2030 int vr, int vs, struct mmuentry *mh)
2031 {
2032 int s, i, pmeg, *pte;
2033
2034 mmu_pagein_evcnt.ev_count++;
2035
2036 va = VA_ROUNDDOWNTOSEG(va);
2037 s = splvm(); /* paranoid */
2038 sp->sg_pmeg = pmeg = me_alloc(mh, pm, vr, vs);
2039 if (pm != pmap_kernel() || HASSUN4_MMU3L)
2040 setsegmap(va, pmeg);
2041 else {
2042 /* Map kernel address into all contexts */
2043 int ctx = getcontext4();
2044 i = ncontext - 1;
2045 do {
2046 setcontext4(i);
2047 setsegmap(va, pmeg);
2048 } while (--i >= 0);
2049 setcontext4(ctx);
2050 }
2051
2052 /* reload segment: write PTEs into a the MMU */
2053 pte = sp->sg_pte;
2054 i = NPTESG;
2055 do {
2056 setpte4(va, *pte++ & ~PG_MBZ);
2057 va += NBPG;
2058 } while (--i > 0);
2059 splx(s);
2060 }
2061
2062 /*
2063 * `Page in' (load or inspect) an MMU entry; called on page faults.
2064 * Returns 1 if we reloaded the segment, -1 if the segment was
2065 * already loaded and the page was marked valid (in which case the
2066 * fault must be a bus error or something), or 0 (segment loaded but
2067 * PTE not valid, or segment not loaded at all).
2068 */
2069 int
mmu_pagein(struct pmap * pm,vaddr_t va,int prot)2070 mmu_pagein(struct pmap *pm, vaddr_t va, int prot)
2071 {
2072 int vr, vs, bits;
2073 struct regmap *rp;
2074 struct segmap *sp;
2075
2076 PMAP_LOCK();
2077
2078 if (prot != VM_PROT_NONE)
2079 bits = PG_V | ((prot & VM_PROT_WRITE) ? PG_W : 0);
2080 else
2081 bits = 0;
2082
2083 vr = VA_VREG(va);
2084 vs = VA_VSEG(va);
2085 rp = &pm->pm_regmap[vr];
2086
2087 /* return 0 if we have no PMEGs to load */
2088 if (rp->rg_nsegmap == 0) {
2089 PMAP_UNLOCK();
2090 return (0);
2091 }
2092
2093 #ifdef DIAGNOSTIC
2094 if (rp->rg_segmap == NULL)
2095 panic("pagein: no segmap");
2096 #endif
2097
2098 #if defined(SUN4_MMU3L)
2099 if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
2100 mmu_pagein_reg(pm, rp, va, vr, ®ion_lru);
2101 #endif
2102 sp = &rp->rg_segmap[vs];
2103
2104 /* return 0 if we have no PTEs to load */
2105 if (sp->sg_npte == 0) {
2106 PMAP_UNLOCK();
2107 return (0);
2108 }
2109
2110 /* return -1 if the fault is `hard', 0 if not */
2111 if (sp->sg_pmeg != seginval) {
2112 PMAP_UNLOCK();
2113 return (bits && (getpte4(va) & bits) == bits ? -1 : 0);
2114 }
2115
2116 mmu_pagein_seg(pm, sp, va, vr, vs, &segm_lru);
2117 PMAP_UNLOCK();
2118 return (1);
2119 }
2120 #endif /* SUN4 or SUN4C */
2121
2122 /*
2123 * Allocate a context. If necessary, steal one from someone else.
2124 * Changes hardware context number and loads segment map.
2125 *
2126 * This routine is only ever called from locore.s just after it has
2127 * saved away the previous process, so there are no active user windows.
2128 */
2129 static void
ctx_alloc(struct pmap * pm)2130 ctx_alloc(struct pmap *pm)
2131 {
2132 union ctxinfo *c;
2133 int cnum, i = 0, doflush;
2134 struct regmap *rp;
2135 int gap_start, gap_end;
2136 vaddr_t va;
2137 #if defined(SUN4M) || defined(SUN4D)
2138 struct cpu_info *cpi;
2139 #endif
2140
2141 KASSERT(mutex_owned(&ctx_lock));
2142
2143 /*XXX-GCC!*/gap_start=gap_end=0;
2144 #ifdef DEBUG
2145 if (pm->pm_ctx)
2146 panic("ctx_alloc pm_ctx");
2147 #endif
2148 DPRINTF(PDB_CTX_ALLOC,
2149 "ctx_alloc[%d](%p)", cpu_number(), pm);
2150
2151 if (CPU_HAS_SUNMMU) {
2152 gap_start = pm->pm_gap_start;
2153 gap_end = pm->pm_gap_end;
2154 }
2155
2156 if ((c = ctx_freelist) != NULL) {
2157 ctx_freelist = c->c_nextfree;
2158 cnum = c - ctxinfo;
2159 doflush = 0;
2160 } else {
2161 if ((ctx_kick += ctx_kickdir) >= ncontext) {
2162 ctx_kick = ncontext - 1;
2163 ctx_kickdir = -1;
2164 } else if (ctx_kick < 1) {
2165 ctx_kick = 1;
2166 ctx_kickdir = 1;
2167 }
2168 c = &ctxinfo[cnum = ctx_kick];
2169 #ifdef DEBUG
2170 if (c->c_pmap == NULL)
2171 panic("ctx_alloc cu_pmap");
2172 #endif
2173 DPRINTF(PDB_CTX_ALLOC | PDB_CTX_STEAL,
2174 "ctx_alloc[%d]: steal context %d from %p",
2175 cpu_number(), cnum, c->c_pmap);
2176
2177 c->c_pmap->pm_ctx = NULL;
2178 c->c_pmap->pm_ctxnum = 0;
2179 doflush = (CACHEINFO.c_vactype != VAC_NONE);
2180 if (CPU_HAS_SUNMMU) {
2181 if (gap_start < c->c_pmap->pm_gap_start)
2182 gap_start = c->c_pmap->pm_gap_start;
2183 if (gap_end > c->c_pmap->pm_gap_end)
2184 gap_end = c->c_pmap->pm_gap_end;
2185 }
2186 }
2187
2188 c->c_pmap = pm;
2189 pm->pm_ctx = c;
2190 pm->pm_ctxnum = cnum;
2191
2192 if (CPU_HAS_SUNMMU) {
2193
2194 /*
2195 * Write pmap's region (3-level MMU) or segment table into
2196 * the MMU.
2197 *
2198 * Only write those entries that actually map something in
2199 * this context by maintaining a pair of region numbers in
2200 * between which the pmap has no valid mappings.
2201 *
2202 * If a context was just allocated from the free list, trust
2203 * that all its pmeg numbers are `seginval'. We make sure this
2204 * is the case initially in pmap_bootstrap(). Otherwise, the
2205 * context was freed by calling ctx_free() in pmap_release(),
2206 * which in turn is supposedly called only when all mappings
2207 * have been removed.
2208 *
2209 * On the other hand, if the context had to be stolen from
2210 * another pmap, we possibly shrink the gap to be the
2211 * disjuction of the new and the previous map.
2212 */
2213
2214 setcontext4(cnum);
2215 if (doflush)
2216 cache_flush_context(cnum);
2217
2218 rp = pm->pm_regmap;
2219 for (va = 0, i = NUREG; --i >= 0; ) {
2220 if (VA_VREG(va) >= gap_start) {
2221 va = VRTOVA(gap_end);
2222 i -= gap_end - gap_start;
2223 rp += gap_end - gap_start;
2224 if (i < 0)
2225 break;
2226 /* mustn't re-enter this branch */
2227 gap_start = NUREG;
2228 }
2229 if (HASSUN4_MMU3L) {
2230 setregmap(va, rp++->rg_smeg);
2231 va += NBPRG;
2232 } else {
2233 int j;
2234 struct segmap *sp = rp->rg_segmap;
2235 for (j = NSEGRG; --j >= 0; va += NBPSG)
2236 setsegmap(va,
2237 sp?sp++->sg_pmeg:seginval);
2238 rp++;
2239 }
2240 }
2241
2242 } else if (CPU_HAS_SRMMU) {
2243
2244 #if defined(SUN4M) || defined(SUN4D)
2245 /*
2246 * Reload page and context tables to activate the page tables
2247 * for this context.
2248 *
2249 * The gap stuff isn't really needed in the sun4m architecture,
2250 * since we don't have to worry about excessive mappings (all
2251 * mappings exist since the page tables must be complete for
2252 * the mmu to be happy).
2253 *
2254 * If a context was just allocated from the free list, trust
2255 * that all of its mmu-edible page tables are zeroed out
2256 * (except for those associated with the kernel). We make
2257 * sure this is the case initially in pmap_bootstrap() and
2258 * pmap_init() (?).
2259 * Otherwise, the context was freed by calling ctx_free() in
2260 * pmap_release(), which in turn is supposedly called only
2261 * when all mappings have been removed.
2262 *
2263 * XXX: Do we have to flush cache after reloading ctx tbl?
2264 */
2265
2266 /*
2267 * We need to flush the cache only when stealing a context
2268 * from another pmap. In that case it's Ok to switch the
2269 * context and leave it set, since the context table
2270 * will have a valid region table entry for this context
2271 * number.
2272 *
2273 * Otherwise, we switch to the new context after loading
2274 * the context table entry with the new pmap's region.
2275 */
2276 if (doflush) {
2277 cache_flush_context(cnum);
2278 }
2279
2280 /*
2281 * The context allocated to a process is the same on all CPUs.
2282 * Here we install the per-CPU region table in each CPU's
2283 * context table slot.
2284 *
2285 * Note on multi-threaded processes: a context must remain
2286 * valid as long as any thread is still running on a CPU.
2287 */
2288 for (CPU_INFO_FOREACH(i, cpi)) {
2289 setpgt4m(&cpi->ctx_tbl[cnum],
2290 (pm->pm_reg_ptps_pa[i] >> SRMMU_PPNPASHIFT) |
2291 SRMMU_TEPTD);
2292 }
2293
2294 /* And finally switch to the new context */
2295 (*cpuinfo.pure_vcache_flush)();
2296 setcontext4m(cnum);
2297 #endif /* SUN4M || SUN4D */
2298 }
2299 }
2300
2301 /*
2302 * Give away a context.
2303 */
2304 static void
ctx_free(struct pmap * pm)2305 ctx_free(struct pmap *pm)
2306 {
2307 union ctxinfo *c;
2308 int ctx;
2309 #if defined(SUN4M) || defined(SUN4D)
2310 struct cpu_info *cpi;
2311 #endif
2312
2313 KASSERT(mutex_owned(&ctx_lock));
2314
2315 c = pm->pm_ctx;
2316 ctx = pm->pm_ctxnum;
2317 pm->pm_ctx = NULL;
2318 pm->pm_ctxnum = 0;
2319 #if defined(SUN4) || defined(SUN4C)
2320 if (CPU_HAS_SUNMMU) {
2321 int octx = getcontext4();
2322 setcontext4(ctx);
2323 cache_flush_context(ctx);
2324 setcontext4(octx);
2325 }
2326 #endif /* SUN4 || SUN4C */
2327
2328 #if defined(SUN4M) || defined(SUN4D)
2329 if (CPU_HAS_SRMMU) {
2330 CPU_INFO_ITERATOR i;
2331
2332 __USE(i);
2333
2334 cache_flush_context(ctx);
2335 tlb_flush_context(ctx, PMAP_CPUSET(pm));
2336 for (CPU_INFO_FOREACH(i, cpi)) {
2337 setpgt4m(&cpi->ctx_tbl[ctx], SRMMU_TEINVALID);
2338 }
2339 }
2340 #endif
2341
2342 c->c_nextfree = ctx_freelist;
2343 ctx_freelist = c;
2344 }
2345
2346
2347 /*----------------------------------------------------------------*/
2348
2349 /*
2350 * pvlist functions.
2351 */
2352
2353 /*
2354 * Walk the given pv list, and for each PTE, set or clear some bits
2355 * (e.g., PG_W or PG_NC).
2356 *
2357 * This routine flushes the cache for any page whose PTE changes,
2358 * as long as the process has a context; this is overly conservative.
2359 * It also copies ref and mod bits to the pvlist, on the theory that
2360 * this might save work later. (XXX should test this theory)
2361 */
2362
2363 #if defined(SUN4) || defined(SUN4C)
2364
2365 void
pv_changepte4_4c(struct vm_page * pg,int bis,int bic)2366 pv_changepte4_4c(struct vm_page *pg, int bis, int bic)
2367 {
2368 int pte, *ptep;
2369 struct pvlist *pv;
2370 struct pmap *pm;
2371 int va, vr, vs;
2372 int ctx, s;
2373 struct regmap *rp;
2374 struct segmap *sp;
2375
2376 pv = VM_MDPAGE_PVHEAD(pg);
2377
2378 write_user_windows(); /* paranoid? */
2379 s = splvm(); /* paranoid? */
2380 if (pv->pv_pmap == NULL) {
2381 splx(s);
2382 return;
2383 }
2384 ctx = getcontext4();
2385 for (; pv != NULL; pv = pv->pv_next) {
2386 pm = pv->pv_pmap;
2387 va = pv->pv_va;
2388 vr = VA_VREG(va);
2389 vs = VA_VSEG(va);
2390 rp = &pm->pm_regmap[vr];
2391 sp = &rp->rg_segmap[vs];
2392 ptep = &sp->sg_pte[VA_VPG(va)];
2393
2394 if (sp->sg_pmeg == seginval) {
2395 /* not in hardware: just fix software copy */
2396 *ptep = (*ptep | bis) & ~bic;
2397 } else {
2398 /* in hardware: fix hardware copy */
2399 if (CTX_USABLE(pm,rp)) {
2400 setcontext4(pm->pm_ctxnum);
2401 /* XXX should flush only when necessary */
2402 pte = getpte4(va);
2403 /*
2404 * XXX: always flush cache; conservative, but
2405 * needed to invalidate cache tag protection
2406 * bits and when disabling caching.
2407 */
2408 cache_flush_page(va, pm->pm_ctxnum);
2409 } else {
2410 /* Make temp map in ctx 0 to access the PTE */
2411 setcontext4(0);
2412 if (HASSUN4_MMU3L)
2413 setregmap(0, tregion);
2414 setsegmap(0, sp->sg_pmeg);
2415 va = VA_VPG(va) << PGSHIFT;
2416 pte = getpte4(va);
2417 }
2418 if (pte & PG_V)
2419 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
2420 pte = (pte | bis) & ~bic;
2421 setpte4(va, pte);
2422 *ptep = (*ptep & PG_MBZ) | pte;
2423 }
2424 }
2425 setcontext4(ctx);
2426 splx(s);
2427 }
2428
2429 /*
2430 * Sync ref and mod bits in pvlist (turns off same in hardware PTEs).
2431 * Returns the new flags.
2432 *
2433 * This is just like pv_changepte, but we never add or remove bits,
2434 * hence never need to adjust software copies.
2435 */
2436 int
pv_syncflags4_4c(struct vm_page * pg)2437 pv_syncflags4_4c(struct vm_page *pg)
2438 {
2439 struct pvlist *pv;
2440 struct pmap *pm;
2441 int pte, va, vr, vs, pmeg, flags;
2442 int ctx, s;
2443 struct regmap *rp;
2444 struct segmap *sp;
2445
2446 pv = VM_MDPAGE_PVHEAD(pg);
2447
2448 s = splvm(); /* paranoid? */
2449 if (pv->pv_pmap == NULL) {
2450 /* Page not mapped; pv_flags is already up to date */
2451 splx(s);
2452 return (0);
2453 }
2454 ctx = getcontext4();
2455 flags = pv->pv_flags;
2456 for (; pv != NULL; pv = pv->pv_next) {
2457 pm = pv->pv_pmap;
2458 va = pv->pv_va;
2459 vr = VA_VREG(va);
2460 vs = VA_VSEG(va);
2461 rp = &pm->pm_regmap[vr];
2462 sp = &rp->rg_segmap[vs];
2463 if ((pmeg = sp->sg_pmeg) == seginval)
2464 continue;
2465 if (CTX_USABLE(pm,rp)) {
2466 setcontext4(pm->pm_ctxnum);
2467 /* XXX should flush only when necessary */
2468 pte = getpte4(va);
2469 if (pte & PG_M)
2470 cache_flush_page(va, pm->pm_ctxnum);
2471 } else {
2472 /* Make temp map in ctx 0 to access the PTE */
2473 setcontext4(0);
2474 if (HASSUN4_MMU3L)
2475 setregmap(0, tregion);
2476 setsegmap(0, pmeg);
2477 va = VA_VPG(va) << PGSHIFT;
2478 pte = getpte4(va);
2479 }
2480 if (pte & (PG_M|PG_U) && pte & PG_V) {
2481 flags |= MR4_4C(pte);
2482 pte &= ~(PG_M|PG_U);
2483 setpte4(va, pte);
2484 }
2485 }
2486
2487 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
2488 setcontext4(ctx);
2489 splx(s);
2490 return (flags);
2491 }
2492
2493 /*
2494 * pv_unlink is a helper function for pmap_remove.
2495 * It takes a pointer to the pv_table head for some physical address
2496 * and removes the appropriate (pmap, va) entry.
2497 *
2498 * Once the entry is removed, if the pv_table head has the cache
2499 * inhibit bit set, see if we can turn that off; if so, walk the
2500 * pvlist and turn off PG_NC in each PTE. (The pvlist is by
2501 * definition nonempty, since it must have at least two elements
2502 * in it to have PV_NC set, and we only remove one here.)
2503 */
2504 /*static*/ void
pv_unlink4_4c(struct vm_page * pg,struct pmap * pm,vaddr_t va)2505 pv_unlink4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va)
2506 {
2507 struct pvlist *pv0, *npv;
2508
2509 pv0 = VM_MDPAGE_PVHEAD(pg);
2510 npv = pv0->pv_next;
2511
2512 /*
2513 * First entry is special (sigh).
2514 */
2515 if (pv0->pv_pmap == pm && pv0->pv_va == va) {
2516 pmap_stats.ps_unlink_pvfirst++;
2517 if (npv != NULL) {
2518 /*
2519 * Shift next entry into the head.
2520 * Make sure to retain the REF, MOD and ANC flags.
2521 */
2522 pv0->pv_next = npv->pv_next;
2523 pv0->pv_pmap = npv->pv_pmap;
2524 pv0->pv_va = npv->pv_va;
2525 pv0->pv_flags &= ~PV_NC;
2526 pv0->pv_flags |= (npv->pv_flags & PV_NC);
2527 pool_put(&pv_pool, npv);
2528 } else {
2529 /*
2530 * No mappings left; we still need to maintain
2531 * the REF and MOD flags. since pmap_is_modified()
2532 * can still be called for this page.
2533 */
2534 pv0->pv_pmap = NULL;
2535 pv0->pv_flags &= ~(PV_NC|PV_ANC);
2536 return;
2537 }
2538 } else {
2539 struct pvlist *prev;
2540
2541 pmap_stats.ps_unlink_pvsearch++;
2542 for (prev = pv0;; prev = npv, npv = npv->pv_next) {
2543 if (npv == NULL) {
2544 panic("pv_unlink: pm %p is missing on pg %p",
2545 pm, pg);
2546 }
2547 if (npv->pv_pmap == pm && npv->pv_va == va)
2548 break;
2549 }
2550 prev->pv_next = npv->pv_next;
2551 pool_put(&pv_pool, npv);
2552 }
2553 if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {
2554 /*
2555 * Not cached: check whether we can fix that now.
2556 */
2557 va = pv0->pv_va;
2558 for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
2559 if (BADALIAS(va, npv->pv_va) ||
2560 (npv->pv_flags & PV_NC) != 0)
2561 return;
2562 pv0->pv_flags &= ~PV_ANC;
2563 pv_changepte4_4c(pg, 0, PG_NC);
2564 }
2565 }
2566
2567 /*
2568 * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
2569 * It returns PG_NC if the (new) pvlist says that the address cannot
2570 * be cached.
2571 */
2572 /*static*/ int
pv_link4_4c(struct vm_page * pg,struct pmap * pm,vaddr_t va,unsigned int * pteprotop)2573 pv_link4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va,
2574 unsigned int *pteprotop)
2575 {
2576 struct pvlist *pv0, *pv, *npv;
2577 int nc = (*pteprotop & PG_NC) != 0 ? PV_NC : 0;
2578
2579 pv0 = VM_MDPAGE_PVHEAD(pg);
2580
2581 if (pv0->pv_pmap == NULL) {
2582 /* no pvlist entries yet */
2583 pmap_stats.ps_enter_firstpv++;
2584 pv0->pv_next = NULL;
2585 pv0->pv_pmap = pm;
2586 pv0->pv_va = va;
2587 pv0->pv_flags |= nc;
2588 return (0);
2589 }
2590
2591 /*
2592 * Allocate the new PV entry now, and, if that fails, bail out
2593 * before changing the cacheable state of the existing mappings.
2594 */
2595 npv = pool_get(&pv_pool, PR_NOWAIT);
2596 if (npv == NULL)
2597 return (ENOMEM);
2598
2599 pmap_stats.ps_enter_secondpv++;
2600
2601 /*
2602 * Before entering the new mapping, see if
2603 * it will cause old mappings to become aliased
2604 * and thus need to be `discached'.
2605 */
2606 if (pv0->pv_flags & PV_ANC) {
2607 /* already uncached, just stay that way */
2608 *pteprotop |= PG_NC;
2609 goto link_npv;
2610 }
2611
2612 for (pv = pv0; pv != NULL; pv = pv->pv_next) {
2613 if ((pv->pv_flags & PV_NC) != 0) {
2614 *pteprotop |= PG_NC;
2615 #ifdef DEBUG
2616 /* Check currently illegal condition */
2617 if (nc == 0)
2618 printf("pv_link: proc %s, va=0x%lx: "
2619 "unexpected uncached mapping at 0x%lx\n",
2620 curproc ? curproc->p_comm : "--",
2621 va, pv->pv_va);
2622 #endif
2623 }
2624 if (BADALIAS(va, pv->pv_va)) {
2625 DPRINTF(PDB_CACHESTUFF,
2626 "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p",
2627 curproc ? curproc->p_comm : "--",
2628 va, pv->pv_va, pg);
2629 /* Mark list head `uncached due to aliases' */
2630 pv0->pv_flags |= PV_ANC;
2631 pv_changepte4_4c(pg, PG_NC, 0);
2632 *pteprotop |= PG_NC;
2633 break;
2634 }
2635 }
2636
2637 link_npv:
2638 npv->pv_next = pv0->pv_next;
2639 npv->pv_pmap = pm;
2640 npv->pv_va = va;
2641 npv->pv_flags = nc;
2642 pv0->pv_next = npv;
2643 return (0);
2644 }
2645
2646 #endif /* SUN4 || SUN4C */
2647
2648 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */
2649 /*
2650 * Walk the given pv list, and for each PTE, set or clear some bits
2651 * (e.g., PG_W or PG_NC).
2652 *
2653 * This routine flushes the cache for any page whose PTE changes,
2654 * as long as the process has a context; this is overly conservative.
2655 * It also copies ref and mod bits to the pvlist, on the theory that
2656 * this might save work later. (XXX should test this theory)
2657 *
2658 * Called with PV lock and pmap main lock held.
2659 */
2660 void
pv_changepte4m(struct vm_page * pg,int bis,int bic)2661 pv_changepte4m(struct vm_page *pg, int bis, int bic)
2662 {
2663 struct pvlist *pv;
2664 struct pmap *pm;
2665 vaddr_t va;
2666 struct regmap *rp;
2667 struct segmap *sp;
2668
2669 pv = VM_MDPAGE_PVHEAD(pg);
2670 if (pv->pv_pmap == NULL)
2671 return;
2672
2673 for (; pv != NULL; pv = pv->pv_next) {
2674 pm = pv->pv_pmap;
2675 /* XXXSMP: should lock pm */
2676 va = pv->pv_va;
2677 rp = &pm->pm_regmap[VA_VREG(va)];
2678 sp = &rp->rg_segmap[VA_VSEG(va)];
2679
2680 if (pm->pm_ctx) {
2681 /*
2682 * XXX: always flush cache; conservative, but
2683 * needed to invalidate cache tag protection
2684 * bits and when disabling caching.
2685 */
2686 cache_flush_page(va, pm->pm_ctxnum);
2687 }
2688
2689 KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) ==
2690 SRMMU_TEPTE);
2691 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(updatepte4m(va,
2692 &sp->sg_pte[VA_SUN4M_VPG(va)], bic, bis, pm->pm_ctxnum,
2693 PMAP_CPUSET(pm)));
2694 }
2695 }
2696
2697 /*
2698 * Sync ref and mod bits in pvlist. If page has been ref'd or modified,
2699 * update ref/mod bits in pvlist, and clear the hardware bits.
2700 *
2701 * Return the new flags.
2702 */
2703 int
pv_syncflags4m(struct vm_page * pg)2704 pv_syncflags4m(struct vm_page *pg)
2705 {
2706 struct pvlist *pv;
2707 struct pmap *pm;
2708 int va, flags;
2709 int s;
2710 struct regmap *rp;
2711 struct segmap *sp;
2712 int tpte;
2713
2714 s = splvm();
2715 PMAP_LOCK();
2716 pv = VM_MDPAGE_PVHEAD(pg);
2717 if (pv->pv_pmap == NULL) {
2718 /* Page not mapped; pv_flags is already up to date */
2719 flags = 0;
2720 goto out;
2721 }
2722
2723 flags = pv->pv_flags;
2724 for (; pv != NULL; pv = pv->pv_next) {
2725 pm = pv->pv_pmap;
2726 va = pv->pv_va;
2727 rp = &pm->pm_regmap[VA_VREG(va)];
2728 sp = &rp->rg_segmap[VA_VSEG(va)];
2729
2730 tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
2731 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE &&
2732 (tpte & (SRMMU_PG_R|SRMMU_PG_M)) != 0) {
2733 /*
2734 * Flush cache if modified to make sure the PTE
2735 * M bit will be set again on the next write access.
2736 */
2737 if (pm->pm_ctx && (tpte & SRMMU_PG_M) == SRMMU_PG_M)
2738 cache_flush_page(va, pm->pm_ctxnum);
2739
2740 flags |= MR4M(updatepte4m(va,
2741 &sp->sg_pte[VA_SUN4M_VPG(va)],
2742 SRMMU_PG_M | SRMMU_PG_R,
2743 0, pm->pm_ctxnum, PMAP_CPUSET(pm)));
2744 }
2745 }
2746
2747 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
2748 out:
2749 PMAP_UNLOCK();
2750 splx(s);
2751 return (flags);
2752 }
2753
2754 /*
2755 * Should be called with pmap already locked.
2756 */
2757 void
pv_unlink4m(struct vm_page * pg,struct pmap * pm,vaddr_t va)2758 pv_unlink4m(struct vm_page *pg, struct pmap *pm, vaddr_t va)
2759 {
2760 struct pvlist *pv0, *npv;
2761
2762 pv0 = VM_MDPAGE_PVHEAD(pg);
2763
2764 npv = pv0->pv_next;
2765 /*
2766 * First entry is special (sigh).
2767 */
2768 if (pv0->pv_pmap == pm && pv0->pv_va == va) {
2769 pmap_stats.ps_unlink_pvfirst++;
2770 if (npv != NULL) {
2771 /*
2772 * Shift next entry into the head.
2773 * Make sure to retain the REF, MOD and ANC flags
2774 * on the list head.
2775 */
2776 pv0->pv_next = npv->pv_next;
2777 pv0->pv_pmap = npv->pv_pmap;
2778 pv0->pv_va = npv->pv_va;
2779 pv0->pv_flags &= ~PV_NC;
2780 pv0->pv_flags |= (npv->pv_flags & PV_NC);
2781 pool_put(&pv_pool, npv);
2782 } else {
2783 /*
2784 * No mappings left; we need to maintain
2785 * the REF and MOD flags, since pmap_is_modified()
2786 * can still be called for this page.
2787 */
2788 pv0->pv_pmap = NULL;
2789 pv0->pv_flags &= ~(PV_NC|PV_ANC);
2790 return;
2791 }
2792 } else {
2793 struct pvlist *prev;
2794
2795 pmap_stats.ps_unlink_pvsearch++;
2796 for (prev = pv0;; prev = npv, npv = npv->pv_next) {
2797 if (npv == NULL) {
2798 panic("pv_unlink: pm %p is missing on pg %p",
2799 pm, pg);
2800 return;
2801 }
2802 if (npv->pv_pmap == pm && npv->pv_va == va)
2803 break;
2804 }
2805 prev->pv_next = npv->pv_next;
2806 pool_put(&pv_pool, npv);
2807 }
2808
2809 if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {
2810
2811 /*
2812 * Not cached: check whether we can fix that now.
2813 */
2814 va = pv0->pv_va;
2815 for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
2816 if (BADALIAS(va, npv->pv_va) ||
2817 (npv->pv_flags & PV_NC) != 0)
2818 return;
2819 DPRINTF(PDB_CACHESTUFF,
2820 "pv_unlink: alias ok: proc %s, va 0x%lx, pg %p",
2821 curproc ? curproc->p_comm : "--", va, pg);
2822 pv0->pv_flags &= ~PV_ANC;
2823 pv_changepte4m(pg, SRMMU_PG_C, 0);
2824 }
2825 }
2826
2827 /*
2828 * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
2829 * May turn off the cacheable bit in the pte prototype for the new mapping.
2830 * Called with pm locked.
2831 */
2832 /*static*/ int
pv_link4m(struct vm_page * pg,struct pmap * pm,vaddr_t va,unsigned int * pteprotop)2833 pv_link4m(struct vm_page *pg, struct pmap *pm, vaddr_t va,
2834 unsigned int *pteprotop)
2835 {
2836 struct pvlist *pv0, *pv, *npv;
2837 int nc = (*pteprotop & SRMMU_PG_C) == 0 ? PV_NC : 0;
2838 int error = 0;
2839
2840 pv0 = VM_MDPAGE_PVHEAD(pg);
2841
2842 if (pv0->pv_pmap == NULL) {
2843 /* no pvlist entries yet */
2844 pmap_stats.ps_enter_firstpv++;
2845 pv0->pv_next = NULL;
2846 pv0->pv_pmap = pm;
2847 pv0->pv_va = va;
2848 pv0->pv_flags |= nc;
2849 goto out;
2850 }
2851
2852 /*
2853 * Allocate the new PV entry now, and, if that fails, bail out
2854 * before changing the cacheable state of the existing mappings.
2855 */
2856 npv = pool_get(&pv_pool, PR_NOWAIT);
2857 if (npv == NULL) {
2858 error = ENOMEM;
2859 goto out;
2860 }
2861
2862 pmap_stats.ps_enter_secondpv++;
2863
2864 /*
2865 * See if the new mapping will cause old mappings to
2866 * become aliased and thus need to be `discached'.
2867 */
2868 if ((pv0->pv_flags & PV_ANC) != 0) {
2869 /* already uncached, just stay that way */
2870 *pteprotop &= ~SRMMU_PG_C;
2871 goto link_npv;
2872 }
2873
2874 for (pv = pv0; pv != NULL; pv = pv->pv_next) {
2875 if ((pv->pv_flags & PV_NC) != 0) {
2876 *pteprotop &= ~SRMMU_PG_C;
2877 #ifdef DEBUG
2878 /* Check currently illegal condition */
2879 if (nc == 0)
2880 printf("pv_link: proc %s, va=0x%lx: "
2881 "unexpected uncached mapping at 0x%lx\n",
2882 curproc ? curproc->p_comm : "--",
2883 va, pv->pv_va);
2884 #endif
2885 }
2886 if (BADALIAS(va, pv->pv_va)) {
2887 DPRINTF(PDB_CACHESTUFF,
2888 "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p",
2889 curproc ? curproc->p_comm : "--",
2890 va, pv->pv_va, pg);
2891 /* Mark list head `uncached due to aliases' */
2892 pv0->pv_flags |= PV_ANC;
2893 pv_changepte4m(pg, 0, SRMMU_PG_C);
2894 *pteprotop &= ~SRMMU_PG_C;
2895 break;
2896 }
2897 }
2898
2899 link_npv:
2900 /* Now link in the new PV entry */
2901 npv->pv_next = pv0->pv_next;
2902 npv->pv_pmap = pm;
2903 npv->pv_va = va;
2904 npv->pv_flags = nc;
2905 pv0->pv_next = npv;
2906
2907 out:
2908 return (error);
2909 }
2910 #endif
2911
2912 /*
2913 * Uncache all entries on behalf of kvm_uncache(). In addition to
2914 * removing the cache bit from the PTE, we are also setting PV_NC
2915 * in each entry to stop pv_unlink() from re-caching (i.e. when a
2916 * a bad alias is going away).
2917 */
2918 static void
pv_uncache(struct vm_page * pg)2919 pv_uncache(struct vm_page *pg)
2920 {
2921 struct pvlist *pv;
2922 int s;
2923
2924 s = splvm();
2925 PMAP_LOCK();
2926
2927 for (pv = VM_MDPAGE_PVHEAD(pg); pv != NULL; pv = pv->pv_next)
2928 pv->pv_flags |= PV_NC;
2929
2930 #if defined(SUN4M) || defined(SUN4D)
2931 if (CPU_HAS_SRMMU)
2932 pv_changepte4m(pg, 0, SRMMU_PG_C);
2933 #endif
2934 #if defined(SUN4) || defined(SUN4C)
2935 if (CPU_HAS_SUNMMU)
2936 pv_changepte4_4c(pg, PG_NC, 0);
2937 #endif
2938 PMAP_UNLOCK();
2939 splx(s);
2940 }
2941
2942 /*
2943 * Walk the given list and flush the cache for each (MI) page that is
2944 * potentially in the cache. Called only if vactype != VAC_NONE.
2945 */
2946 #if defined(SUN4) || defined(SUN4C)
2947 static void
pv_flushcache4_4c(struct vm_page * pg)2948 pv_flushcache4_4c(struct vm_page *pg)
2949 {
2950 struct pvlist *pv;
2951 struct pmap *pm;
2952 int s, ctx;
2953
2954 pv = VM_MDPAGE_PVHEAD(pg);
2955
2956 write_user_windows(); /* paranoia? */
2957 s = splvm(); /* XXX extreme paranoia */
2958 if ((pm = pv->pv_pmap) != NULL) {
2959 ctx = getcontext4();
2960 for (;;) {
2961 if (pm->pm_ctx) {
2962 setcontext4(pm->pm_ctxnum);
2963 cache_flush_page(pv->pv_va, pm->pm_ctxnum);
2964 }
2965 pv = pv->pv_next;
2966 if (pv == NULL)
2967 break;
2968 pm = pv->pv_pmap;
2969 }
2970 setcontext4(ctx);
2971 }
2972 splx(s);
2973 }
2974 #endif /* SUN4 || SUN4C */
2975
2976 #if defined(SUN4M) || defined(SUN4D)
2977 static void
pv_flushcache4m(struct vm_page * pg)2978 pv_flushcache4m(struct vm_page *pg)
2979 {
2980 struct pvlist *pv;
2981 struct pmap *pm;
2982 int s;
2983
2984 pv = VM_MDPAGE_PVHEAD(pg);
2985
2986 s = splvm(); /* XXX extreme paranoia */
2987 if ((pm = pv->pv_pmap) != NULL) {
2988 for (;;) {
2989 if (pm->pm_ctx) {
2990 cache_flush_page(pv->pv_va, pm->pm_ctxnum);
2991 }
2992 pv = pv->pv_next;
2993 if (pv == NULL)
2994 break;
2995 pm = pv->pv_pmap;
2996 }
2997 }
2998 splx(s);
2999 }
3000 #endif /* SUN4M || SUN4D */
3001
3002 /*----------------------------------------------------------------*/
3003
3004 /*
3005 * At last, pmap code.
3006 */
3007
3008 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
3009 int nptesg;
3010 #endif
3011
3012 #if defined(SUN4M) || defined(SUN4D)
3013 static void pmap_bootstrap4m(void *);
3014 #endif
3015 #if defined(SUN4) || defined(SUN4C)
3016 static void pmap_bootstrap4_4c(void *, int, int, int);
3017 #endif
3018
3019 /*
3020 * Bootstrap the system enough to run with VM enabled.
3021 *
3022 * nsegment is the number of mmu segment entries (``PMEGs'');
3023 * nregion is the number of mmu region entries (``SMEGs'');
3024 * nctx is the number of contexts.
3025 */
3026 void
pmap_bootstrap(int nctx,int nregion,int nsegment)3027 pmap_bootstrap(int nctx, int nregion, int nsegment)
3028 {
3029 void *p;
3030
3031 uvmexp.pagesize = NBPG;
3032 uvm_md_init();
3033
3034 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
3035 /* In this case NPTESG is a variable */
3036 nptesg = (NBPSG >> pgshift);
3037 #endif
3038
3039 /*
3040 * Grab physical memory list.
3041 */
3042 p = kernel_top;
3043 get_phys_mem(&p);
3044
3045 /*
3046 * The data segment in sparc ELF images is aligned to a 64KB
3047 * (the maximum page size defined by the ELF/sparc ABI) boundary.
3048 * This results in a unused portion of physical memory in between
3049 * the text/rodata and the data segment. We pick up that gap
3050 * here to remove it from the kernel map and give it to the
3051 * VM manager later.
3052 */
3053 etext_gap_start = (vaddr_t)(etext + NBPG - 1) & ~PGOFSET;
3054 etext_gap_end = (vaddr_t)kernel_data_start & ~PGOFSET;
3055
3056 if (CPU_HAS_SRMMU) {
3057 #if defined(SUN4M) || defined(SUN4D)
3058 pmap_bootstrap4m(p);
3059 #endif
3060 } else if (CPU_HAS_SUNMMU) {
3061 #if defined(SUN4) || defined(SUN4C)
3062 pmap_bootstrap4_4c(p, nctx, nregion, nsegment);
3063 #endif
3064 }
3065
3066 pmap_page_upload();
3067 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_NONE);
3068 mutex_init(&demap_lock, MUTEX_DEFAULT, IPL_VM);
3069 mutex_init(&ctx_lock, MUTEX_DEFAULT, IPL_SCHED);
3070 lock_available = true;
3071 }
3072
3073 #if defined(SUN4) || defined(SUN4C)
3074 void
pmap_bootstrap4_4c(void * top,int nctx,int nregion,int nsegment)3075 pmap_bootstrap4_4c(void *top, int nctx, int nregion, int nsegment)
3076 {
3077 union ctxinfo *ci;
3078 struct mmuentry *mmuseg;
3079 #if defined(SUN4_MMU3L)
3080 struct mmuentry *mmureg;
3081 #endif
3082 struct regmap *rp;
3083 struct segmap *sp;
3084 int i, j;
3085 int npte, zseg, vr, vs;
3086 int startscookie, scookie;
3087 #if defined(SUN4_MMU3L)
3088 int startrcookie = 0, rcookie = 0;
3089 #endif
3090 int *kptes;
3091 int lastpage;
3092 vaddr_t va;
3093 vaddr_t p;
3094
3095 /*
3096 * Compute `va2pa_offset'.
3097 * Use `kernel_text' to probe the MMU translation since
3098 * the pages at KERNBASE might not be mapped.
3099 */
3100 va2pa_offset = (vaddr_t)kernel_text -
3101 ((getpte4(kernel_text) & PG_PFNUM) << PGSHIFT);
3102
3103 ncontext = nctx;
3104
3105 switch (cputyp) {
3106 case CPU_SUN4C:
3107 mmu_has_hole = 1;
3108 break;
3109 case CPU_SUN4:
3110 if (cpuinfo.cpu_type != CPUTYP_4_400) {
3111 mmu_has_hole = 1;
3112 break;
3113 }
3114 }
3115
3116 #if defined(SUN4)
3117 /*
3118 * set up the segfixmask to mask off invalid bits
3119 */
3120 segfixmask = nsegment - 1; /* assume nsegment is a power of 2 */
3121 #ifdef DIAGNOSTIC
3122 if (((nsegment & segfixmask) | (nsegment & ~segfixmask)) != nsegment) {
3123 printf("pmap_bootstrap: unsuitable number of segments (%d)\n",
3124 nsegment);
3125 callrom();
3126 }
3127 #endif
3128 #endif
3129
3130 #if defined(SUN4M) || defined(SUN4D) /* We're in a dual-arch kernel.
3131 Setup 4/4c fn. ptrs */
3132 pmap_clear_modify_p = pmap_clear_modify4_4c;
3133 pmap_clear_reference_p = pmap_clear_reference4_4c;
3134 pmap_enter_p = pmap_enter4_4c;
3135 pmap_extract_p = pmap_extract4_4c;
3136 pmap_is_modified_p = pmap_is_modified4_4c;
3137 pmap_is_referenced_p = pmap_is_referenced4_4c;
3138 pmap_kenter_pa_p = pmap_kenter_pa4_4c;
3139 pmap_kremove_p = pmap_kremove4_4c;
3140 pmap_kprotect_p = pmap_kprotect4_4c;
3141 pmap_page_protect_p = pmap_page_protect4_4c;
3142 pmap_protect_p = pmap_protect4_4c;
3143 pmap_rmk_p = pmap_rmk4_4c;
3144 pmap_rmu_p = pmap_rmu4_4c;
3145 #endif /* defined SUN4M || defined SUN4D */
3146
3147 p = (vaddr_t)top;
3148
3149 /*
3150 * Last segment is the `invalid' one (one PMEG of pte's with !pg_v).
3151 * It will never be used for anything else.
3152 */
3153 seginval = --nsegment;
3154
3155 #if defined(SUN4_MMU3L)
3156 if (HASSUN4_MMU3L)
3157 reginval = --nregion;
3158 #endif
3159
3160 /*
3161 * Allocate and initialise mmu entries and context structures.
3162 */
3163 #if defined(SUN4_MMU3L)
3164 mmuregions = mmureg = (struct mmuentry *)p;
3165 p += nregion * sizeof(struct mmuentry);
3166 memset(mmuregions, 0, nregion * sizeof(struct mmuentry));
3167 #endif
3168 mmusegments = mmuseg = (struct mmuentry *)p;
3169 p += nsegment * sizeof(struct mmuentry);
3170 memset(mmusegments, 0, nsegment * sizeof(struct mmuentry));
3171
3172 pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
3173 p += nctx * sizeof *ci;
3174
3175 /* Initialize MMU resource queues */
3176 #if defined(SUN4_MMU3L)
3177 mmuq_init(®ion_freelist);
3178 mmuq_init(®ion_lru);
3179 mmuq_init(®ion_locked);
3180 #endif
3181 mmuq_init(&segm_freelist);
3182 mmuq_init(&segm_lru);
3183 mmuq_init(&segm_locked);
3184
3185
3186 /*
3187 * Initialize the kernel pmap.
3188 */
3189 /* kernel_pmap_store.pm_ctxnum = 0; */
3190 kernel_pmap_store.pm_refcount = 1;
3191 #if defined(SUN4_MMU3L)
3192 TAILQ_INIT(&kernel_pmap_store.pm_reglist);
3193 #endif
3194 TAILQ_INIT(&kernel_pmap_store.pm_seglist);
3195
3196 /*
3197 * Allocate memory for kernel PTEs
3198 * XXX Consider allocating memory for only a few regions
3199 * and use growkernel() to allocate more as needed.
3200 */
3201 kptes = (int *)p;
3202 p += NKREG * NSEGRG * NPTESG * sizeof(int);
3203 memset(kptes, 0, NKREG * NSEGRG * NPTESG * sizeof(int));
3204
3205 /*
3206 * Set up pm_regmap for kernel to point NUREG *below* the beginning
3207 * of kernel regmap storage. Since the kernel only uses regions
3208 * above NUREG, we save storage space and can index kernel and
3209 * user regions in the same way.
3210 */
3211 #pragma GCC diagnostic push
3212 #pragma GCC diagnostic ignored "-Warray-bounds"
3213 kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG;
3214 #pragma GCC diagnostic pop
3215 for (i = NKREG; --i >= 0;) {
3216 #if defined(SUN4_MMU3L)
3217 kernel_regmap_store[i].rg_smeg = reginval;
3218 #endif
3219 kernel_regmap_store[i].rg_segmap =
3220 &kernel_segmap_store[i * NSEGRG];
3221 for (j = NSEGRG; --j >= 0;) {
3222 sp = &kernel_segmap_store[i * NSEGRG + j];
3223 sp->sg_pmeg = seginval;
3224 sp->sg_pte = &kptes[(i * NSEGRG + j) * NPTESG];
3225 }
3226 }
3227
3228 /*
3229 * Preserve the monitor ROM's reserved VM region, so that
3230 * we can use L1-A or the monitor's debugger. As a side
3231 * effect we map the ROM's reserved VM into all contexts
3232 * (otherwise L1-A crashes the machine!).
3233 */
3234
3235 mmu_reservemon4_4c(&nregion, &nsegment);
3236
3237 #if defined(SUN4_MMU3L)
3238 /* Reserve one region for temporary mappings */
3239 if (HASSUN4_MMU3L)
3240 tregion = --nregion;
3241 #endif
3242
3243 /*
3244 * Set up the `constants' for the call to vm_init()
3245 * in main(). All pages beginning at p (rounded up to
3246 * the next whole page) and continuing through the number
3247 * of available pages are free, but they start at a higher
3248 * virtual address. This gives us two mappable MD pages
3249 * for pmap_zero_page and pmap_copy_page, and one MI page
3250 * for /dev/mem, all with no associated physical memory.
3251 */
3252 p = (p + NBPG - 1) & ~PGOFSET;
3253
3254 avail_start = PMAP_BOOTSTRAP_VA2PA(p);
3255
3256 i = p;
3257 cpuinfo.vpage[0] = (void *)p, p += NBPG;
3258 cpuinfo.vpage[1] = (void *)p, p += NBPG;
3259 p = (vaddr_t)reserve_dumppages((void *)p);
3260
3261 virtual_avail = p;
3262 virtual_end = VM_MAX_KERNEL_ADDRESS;
3263
3264 p = i; /* retract to first free phys */
3265
3266
3267 /*
3268 * All contexts are free except the kernel's.
3269 *
3270 * XXX sun4c could use context 0 for users?
3271 */
3272 ci->c_pmap = pmap_kernel();
3273 ctx_freelist = ci + 1;
3274 for (i = 1; i < ncontext; i++) {
3275 ci++;
3276 ci->c_nextfree = ci + 1;
3277 }
3278 ci->c_nextfree = NULL;
3279 ctx_kick = 0;
3280 ctx_kickdir = -1;
3281
3282 /*
3283 * Init mmu entries that map the kernel physical addresses.
3284 *
3285 * All the other MMU entries are free.
3286 *
3287 * THIS ASSUMES THE KERNEL IS MAPPED BY A CONTIGUOUS RANGE OF
3288 * MMU SEGMENTS/REGIONS DURING THE BOOT PROCESS
3289 */
3290
3291 /* Compute the number of segments used by the kernel */
3292 zseg = (((p + NBPSG - 1) & ~SGOFSET) - KERNBASE) >> SGSHIFT;
3293 lastpage = VA_VPG(p);
3294 if (lastpage == 0)
3295 /*
3296 * If the page bits in p are 0, we filled the last segment
3297 * exactly; if not, it is the last page filled in the
3298 * last segment.
3299 */
3300 lastpage = NPTESG;
3301
3302 p = KERNBASE; /* first va */
3303 vs = VA_VSEG(KERNBASE); /* first virtual segment */
3304 vr = VA_VREG(KERNBASE); /* first virtual region */
3305 rp = &pmap_kernel()->pm_regmap[vr];
3306
3307 /* Get region/segment where kernel addresses start */
3308 #if defined(SUN4_MMU3L)
3309 if (HASSUN4_MMU3L)
3310 startrcookie = rcookie = getregmap(p);
3311 mmureg = &mmuregions[rcookie];
3312 #endif
3313
3314 startscookie = scookie = getsegmap(p);
3315 mmuseg = &mmusegments[scookie];
3316 zseg += scookie; /* First free segment */
3317
3318 for (;;) {
3319
3320 /*
3321 * Distribute each kernel region/segment into all contexts.
3322 * This is done through the monitor ROM, rather than
3323 * directly here: if we do a setcontext we will fault,
3324 * as we are not (yet) mapped in any other context.
3325 */
3326
3327 if ((vs % NSEGRG) == 0) {
3328 /* Entering a new region */
3329 if (VA_VREG(p) > vr) {
3330 #ifdef DEBUG
3331 printf("note: giant kernel!\n");
3332 #endif
3333 vr++, rp++;
3334 }
3335 #if defined(SUN4_MMU3L)
3336 if (HASSUN4_MMU3L) {
3337 for (i = 1; i < nctx; i++)
3338 prom_setcontext(i, (void *)p, rcookie);
3339
3340 mmuq_insert_tail(®ion_locked,
3341 mmureg);
3342 TAILQ_INSERT_TAIL(&pmap_kernel()->pm_reglist,
3343 mmureg, me_pmchain);
3344 #ifdef DIAGNOSTIC
3345 mmuseg->me_statp = NULL;
3346 #endif
3347 mmureg->me_cookie = rcookie;
3348 mmureg->me_pmap = pmap_kernel();
3349 mmureg->me_vreg = vr;
3350 rp->rg_smeg = rcookie;
3351 mmureg++;
3352 rcookie++;
3353 }
3354 #endif /* SUN4_MMU3L */
3355 }
3356
3357 #if defined(SUN4_MMU3L)
3358 if (!HASSUN4_MMU3L)
3359 #endif
3360 for (i = 1; i < nctx; i++)
3361 prom_setcontext(i, (void *)p, scookie);
3362
3363 /* set up the mmu entry */
3364 mmuq_insert_tail(&segm_locked, mmuseg);
3365 #ifdef DIAGNOSTIC
3366 mmuseg->me_statp = &pmap_stats.ps_npmeg_locked;
3367 #endif
3368 TAILQ_INSERT_TAIL(&pmap_kernel()->pm_seglist, mmuseg, me_pmchain);
3369 pmap_stats.ps_npmeg_locked++;
3370 mmuseg->me_cookie = scookie;
3371 mmuseg->me_pmap = pmap_kernel();
3372 mmuseg->me_vreg = vr;
3373 mmuseg->me_vseg = vs % NSEGRG;
3374 sp = &rp->rg_segmap[vs % NSEGRG];
3375 sp->sg_pmeg = scookie;
3376 npte = ++scookie < zseg ? NPTESG : lastpage;
3377 sp->sg_npte = npte;
3378 sp->sg_nwired = npte;
3379 pmap_kernel()->pm_stats.resident_count += npte;
3380 rp->rg_nsegmap += 1;
3381 for (i = 0; i < npte; i++)
3382 sp->sg_pte[i] = getpte4(p + i * NBPG) | PG_WIRED;
3383 mmuseg++;
3384 vs++;
3385 if (scookie < zseg) {
3386 p += NBPSG;
3387 continue;
3388 }
3389
3390 /*
3391 * Unmap the pages, if any, that are not part of
3392 * the final segment.
3393 */
3394 for (p += npte << PGSHIFT; npte < NPTESG; npte++, p += NBPG)
3395 setpte4(p, 0);
3396
3397 #if defined(SUN4_MMU3L)
3398 if (HASSUN4_MMU3L) {
3399 /*
3400 * Unmap the segments, if any, that are not part of
3401 * the final region.
3402 */
3403 for (i = rp->rg_nsegmap; i < NSEGRG; i++, p += NBPSG)
3404 setsegmap(p, seginval);
3405
3406 /*
3407 * Unmap any kernel regions that we aren't using.
3408 */
3409 for (i = 0; i < nctx; i++) {
3410 setcontext4(i);
3411 for (va = p;
3412 va < (OPENPROM_STARTVADDR & ~(NBPRG - 1));
3413 va += NBPRG)
3414 setregmap(va, reginval);
3415 }
3416
3417 } else
3418 #endif
3419 {
3420 /*
3421 * Unmap any kernel segments that we aren't using.
3422 */
3423 for (i = 0; i < nctx; i++) {
3424 setcontext4(i);
3425 for (va = p;
3426 va < (OPENPROM_STARTVADDR & ~(NBPSG - 1));
3427 va += NBPSG)
3428 setsegmap(va, seginval);
3429 }
3430 }
3431 break;
3432 }
3433
3434 #if defined(SUN4_MMU3L)
3435 if (HASSUN4_MMU3L)
3436 for (rcookie = 0; rcookie < nregion; rcookie++) {
3437 if (rcookie == startrcookie)
3438 /* Kernel must fit in one region! */
3439 rcookie++;
3440 mmureg = &mmuregions[rcookie];
3441 mmureg->me_cookie = rcookie;
3442 mmuq_insert_tail(®ion_freelist, mmureg);
3443 #ifdef DIAGNOSTIC
3444 mmuseg->me_statp = NULL;
3445 #endif
3446 }
3447 #endif /* SUN4_MMU3L */
3448
3449 for (scookie = 0; scookie < nsegment; scookie++) {
3450 if (scookie == startscookie)
3451 /* Skip static kernel image */
3452 scookie = zseg;
3453 mmuseg = &mmusegments[scookie];
3454 mmuseg->me_cookie = scookie;
3455 mmuq_insert_tail(&segm_freelist, mmuseg);
3456 pmap_stats.ps_npmeg_free++;
3457 #ifdef DIAGNOSTIC
3458 mmuseg->me_statp = NULL;
3459 #endif
3460 }
3461
3462 /* Erase all spurious user-space segmaps */
3463 for (i = 1; i < ncontext; i++) {
3464 setcontext4(i);
3465 if (HASSUN4_MMU3L)
3466 for (p = 0, j = NUREG; --j >= 0; p += NBPRG)
3467 setregmap(p, reginval);
3468 else
3469 for (p = 0, vr = 0; vr < NUREG; vr++) {
3470 if (VA_INHOLE(p)) {
3471 p = MMU_HOLE_END;
3472 vr = VA_VREG(p);
3473 }
3474 for (j = NSEGRG; --j >= 0; p += NBPSG)
3475 setsegmap(p, seginval);
3476 }
3477 }
3478 setcontext4(0);
3479
3480 /*
3481 * write protect & encache kernel text;
3482 * set red zone at kernel base;
3483 * enable cache on message buffer and cpuinfo.
3484 */
3485
3486 /* Enable cache on message buffer and cpuinfo */
3487 for (p = KERNBASE; p < (vaddr_t)trapbase; p += NBPG)
3488 setpte4(p, getpte4(p) & ~PG_NC);
3489
3490 /* Enable cache and write protext kernel text */
3491 for (p = (vaddr_t)trapbase; p < (vaddr_t)etext; p += NBPG)
3492 setpte4(p, getpte4(p) & ~(PG_NC|PG_W));
3493
3494 /*
3495 * Unmap the `etext gap'; it'll be made available
3496 * to the VM manager.
3497 */
3498 for (p = etext_gap_start; p < etext_gap_end; p += NBPG) {
3499 rp = &pmap_kernel()->pm_regmap[VA_VREG(p)];
3500 sp = &rp->rg_segmap[VA_VSEG(p)];
3501 sp->sg_nwired--;
3502 sp->sg_npte--;
3503 pmap_kernel()->pm_stats.resident_count--;
3504 sp->sg_pte[VA_VPG(p)] = 0;
3505 setpte4(p, 0);
3506 }
3507
3508 /* Enable cache on data & bss */
3509 for (p = etext_gap_end; p < virtual_avail; p += NBPG)
3510 setpte4(p, getpte4(p) & ~PG_NC);
3511
3512 cpus[0] = (struct cpu_info *)CPUINFO_VA;
3513 }
3514 #endif
3515
3516 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_bootstrap */
3517 /*
3518 * Bootstrap the system enough to run with VM enabled on a sun4m machine.
3519 *
3520 * Switches from ROM to kernel page tables, and sets up initial mappings.
3521 */
3522 static void
pmap_bootstrap4m(void * top)3523 pmap_bootstrap4m(void *top)
3524 {
3525 int i, j;
3526 vaddr_t p, q;
3527 union ctxinfo *ci;
3528 int reg, seg;
3529 unsigned int ctxtblsize;
3530 vaddr_t pagetables_start, pagetables_end;
3531 paddr_t pagetables_start_pa;
3532 vaddr_t va;
3533 #if defined(MULTIPROCESSOR)
3534 vsize_t off;
3535 size_t cpuinfo_len = sizeof(struct cpu_info);
3536 uint8_t *cpuinfo_data;
3537 int align = PAGE_SIZE;
3538 vaddr_t sva, cpuinfo_va;
3539 vsize_t sz;
3540 #endif
3541
3542 /*
3543 * Compute `va2pa_offset'.
3544 * Use `kernel_text' to probe the MMU translation since
3545 * the pages at KERNBASE might not be mapped.
3546 */
3547 va2pa_offset = (vaddr_t)kernel_text - VA2PA(kernel_text);
3548
3549 ncontext = cpuinfo.mmu_ncontext;
3550
3551 #if defined(SUN4) || defined(SUN4C) /* setup SRMMU fn. ptrs for dual-arch
3552 kernel */
3553 pmap_clear_modify_p = pmap_clear_modify4m;
3554 pmap_clear_reference_p = pmap_clear_reference4m;
3555 pmap_enter_p = pmap_enter4m;
3556 pmap_extract_p = pmap_extract4m;
3557 pmap_is_modified_p = pmap_is_modified4m;
3558 pmap_is_referenced_p = pmap_is_referenced4m;
3559 pmap_kenter_pa_p = pmap_kenter_pa4m;
3560 pmap_kremove_p = pmap_kremove4m;
3561 pmap_kprotect_p = pmap_kprotect4m;
3562 pmap_page_protect_p = pmap_page_protect4m;
3563 pmap_protect_p = pmap_protect4m;
3564 pmap_rmk_p = pmap_rmk4m;
3565 pmap_rmu_p = pmap_rmu4m;
3566 #endif /* defined SUN4/SUN4C */
3567
3568 /*
3569 * p points to top of kernel mem
3570 */
3571 p = (vaddr_t)top;
3572
3573 p = (p + NBPG - 1) & ~PGOFSET;
3574 DPRINTF(PDB_INITLOUD, "initial p=%lx", p);
3575
3576 /*
3577 * Initialize the kernel pmap.
3578 */
3579 /* kernel_pmap_store.pm_ctxnum = 0; */
3580 kernel_pmap_store.pm_refcount = 1;
3581
3582 /*
3583 * Set up pm_regmap for kernel to point NUREG *below* the beginning
3584 * of kernel regmap storage. Since the kernel only uses regions
3585 * above NUREG, we save storage space and can index kernel and
3586 * user regions in the same way.
3587 */
3588 #pragma GCC diagnostic push
3589 #pragma GCC diagnostic ignored "-Warray-bounds"
3590 kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG;
3591 #pragma GCC diagnostic pop
3592 memset(kernel_regmap_store, 0, sizeof kernel_regmap_store);
3593 memset(kernel_segmap_store, 0, sizeof kernel_segmap_store);
3594 for (i = NKREG; --i >= 0;) {
3595 kernel_regmap_store[i].rg_segmap =
3596 &kernel_segmap_store[i * NSEGRG];
3597 kernel_regmap_store[i].rg_seg_ptps = NULL;
3598 for (j = NSEGRG; --j >= 0;)
3599 kernel_segmap_store[i * NSEGRG + j].sg_pte = NULL;
3600 }
3601
3602 /* Allocate kernel region pointer tables */
3603 pmap_kernel()->pm_reg_ptps = (int **)(q = p);
3604 DPRINTF(PDB_INITLOUD, "kernel region pointer tables p=%lx", p);
3605 p += sparc_ncpus * sizeof(int **);
3606 memset((void *)q, 0, (u_int)p - (u_int)q);
3607
3608 pmap_kernel()->pm_reg_ptps_pa = (int *)(q = p);
3609 DPRINTF(PDB_INITLOUD, "kernel region pointer tables pa p=%lx", p);
3610 p += sparc_ncpus * sizeof(int *);
3611 memset((void *)q, 0, (u_int)p - (u_int)q);
3612
3613 /* Allocate context administration */
3614 pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
3615 DPRINTF(PDB_INITLOUD, "context administration p=%lx", p);
3616 p += ncontext * sizeof *ci;
3617 memset((void *)ci, 0, (u_int)p - (u_int)ci);
3618
3619 /*
3620 * Set up the `constants' for the call to vm_init()
3621 * in main(). All pages beginning at p (rounded up to
3622 * the next whole page) and continuing through the number
3623 * of available pages are free.
3624 */
3625 p = (p + NBPG - 1) & ~PGOFSET;
3626 DPRINTF(PDB_INITLOUD, "align p=%lx", p);
3627
3628 /*
3629 * Reserve memory for MMU pagetables. Some of these have severe
3630 * alignment restrictions. We allocate in a sequence that
3631 * minimizes alignment gaps.
3632 */
3633
3634 pagetables_start = p;
3635 pagetables_start_pa = PMAP_BOOTSTRAP_VA2PA(p);
3636
3637 /*
3638 * Allocate context table.
3639 * To keep supersparc happy, minimum alignment is on a 4K boundary.
3640 */
3641 ctxtblsize = uimax(ncontext,1024) * sizeof(int);
3642 cpuinfo.ctx_tbl = (int *)roundup((u_int)p, ctxtblsize);
3643 cpuinfo.ctx_tbl_pa = PMAP_BOOTSTRAP_VA2PA(cpuinfo.ctx_tbl);
3644 p = (u_int)cpuinfo.ctx_tbl + ctxtblsize;
3645 DPRINTF(PDB_INITLOUD, "post ctx table p=%lx", p);
3646
3647 #if defined(MULTIPROCESSOR)
3648 /*
3649 * Make sure all smp_tlb_flush*() routines for kernel pmap are
3650 * broadcast to all CPU's.
3651 */
3652 pmap_kernel()->pm_cpuset = CPUSET_ALL;
3653 #endif
3654
3655 /*
3656 * Reserve memory for segment and page tables needed to map the entire
3657 * kernel. This takes (2K + NKREG * 16K) of space, but unfortunately
3658 * is necessary since pmap_enter() *must* be able to enter a kernel
3659 * mapping without delay.
3660 */
3661 p = (vaddr_t) roundup(p, SRMMU_L1SIZE * sizeof(u_int));
3662 DPRINTF(PDB_INITLOUD, "roundup kernel_regtable_store p=%lx", p);
3663 qzero((void *)p, SRMMU_L1SIZE * sizeof(u_int));
3664 kernel_regtable_store = (u_int *)p;
3665 p += SRMMU_L1SIZE * sizeof(u_int);
3666 DPRINTF(PDB_INITLOUD, "L1 pages p=%lx", p);
3667
3668 p = (vaddr_t) roundup(p, SRMMU_L2SIZE * sizeof(u_int));
3669 DPRINTF(PDB_INITLOUD, "roundup kernel_segtable_store p=%lx", p);
3670 qzero((void *)p, (SRMMU_L2SIZE * sizeof(u_int)) * NKREG);
3671 kernel_segtable_store = (u_int *)p;
3672 p += (SRMMU_L2SIZE * sizeof(u_int)) * NKREG;
3673 DPRINTF(PDB_INITLOUD, "L2 pages p=%lx", p);
3674
3675 p = (vaddr_t) roundup(p, SRMMU_L3SIZE * sizeof(u_int));
3676 DPRINTF(PDB_INITLOUD, "roundup kernel_pagtable_store p=%lx", p);
3677 /* zero it: all will be SRMMU_TEINVALID */
3678 qzero((void *)p, ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG);
3679 kernel_pagtable_store = (u_int *)p;
3680 p += ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG;
3681 DPRINTF(PDB_INITLOUD, "L3 pages p=%lx", p);
3682
3683 /* Round to next page and mark end of pre-wired kernel space */
3684 p = (p + NBPG - 1) & ~PGOFSET;
3685 DPRINTF(PDB_INITLOUD, "align p=%lx", p);
3686 pagetables_end = p;
3687
3688 #if defined(MULTIPROCESSOR)
3689 /*
3690 * Allocate aligned KVA. `cpuinfo' resides at a fixed virtual
3691 * address. Since we need to access an other CPU's cpuinfo
3692 * structure occasionally, this must be done at a virtual address
3693 * that's cache congruent to the fixed address CPUINFO_VA.
3694 *
3695 * NOTE: we're using the cache properties of the boot CPU to
3696 * determine the alignment (XXX).
3697 */
3698 sz = sizeof(struct cpu_info);
3699 if (sparc_ncpus > 1) {
3700 if (CACHEINFO.c_totalsize > align) {
3701 /* Need a power of two */
3702 while (align <= CACHEINFO.c_totalsize)
3703 align <<= 1;
3704 align >>= 1;
3705 }
3706
3707 sz = (sz + PAGE_SIZE - 1) & -PAGE_SIZE;
3708 cpuinfo_len = sz + align - PAGE_SIZE;
3709
3710 /* Grab as much space as we need */
3711 DPRINTF(PDB_INITLOUD, "cpuinfo=%lx", p);
3712 cpuinfo_data = (uint8_t *)p;
3713 p += (cpuinfo_len * sparc_ncpus);
3714 } else
3715 cpuinfo_data = (uint8_t *)CPUINFO_VA;
3716 #endif
3717
3718 DPRINTF(PDB_INITLOUD, "avail_start=%lx", p);
3719 avail_start = PMAP_BOOTSTRAP_VA2PA(p);
3720
3721 /*
3722 * Now wire the region and segment tables of the kernel map.
3723 */
3724 pmap_kernel()->pm_reg_ptps[0] = (int *) kernel_regtable_store;
3725 pmap_kernel()->pm_reg_ptps_pa[0] =
3726 PMAP_BOOTSTRAP_VA2PA(kernel_regtable_store);
3727
3728 /* Install L1 table in context 0 */
3729 setpgt4m(&cpuinfo.ctx_tbl[0],
3730 (pmap_kernel()->pm_reg_ptps_pa[0] >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
3731
3732 for (reg = 0; reg < NKREG; reg++) {
3733 struct regmap *rp;
3734 void *kphyssegtbl;
3735
3736 /*
3737 * Entering new region; install & build segtbl
3738 */
3739
3740 rp = &pmap_kernel()->pm_regmap[reg + VA_VREG(KERNBASE)];
3741
3742 kphyssegtbl = (void *)
3743 &kernel_segtable_store[reg * SRMMU_L2SIZE];
3744
3745 setpgt4m(&pmap_kernel()->pm_reg_ptps[0][reg + VA_VREG(KERNBASE)],
3746 (PMAP_BOOTSTRAP_VA2PA(kphyssegtbl) >> SRMMU_PPNPASHIFT) |
3747 SRMMU_TEPTD);
3748
3749 rp->rg_seg_ptps = (int *)kphyssegtbl;
3750
3751 for (seg = 0; seg < NSEGRG; seg++) {
3752 struct segmap *sp;
3753 void *kphyspagtbl;
3754
3755 rp->rg_nsegmap++;
3756
3757 sp = &rp->rg_segmap[seg];
3758 kphyspagtbl = (void *)
3759 &kernel_pagtable_store
3760 [((reg * NSEGRG) + seg) * SRMMU_L3SIZE];
3761
3762 setpgt4m(&rp->rg_seg_ptps[seg],
3763 (PMAP_BOOTSTRAP_VA2PA(kphyspagtbl) >> SRMMU_PPNPASHIFT) |
3764 SRMMU_TEPTD);
3765 sp->sg_pte = (int *) kphyspagtbl;
3766 }
3767 }
3768
3769 /*
3770 * Preserve the monitor ROM's reserved VM region, so that
3771 * we can use L1-A or the monitor's debugger.
3772 */
3773 mmu_reservemon4m(&kernel_pmap_store);
3774
3775 /*
3776 * Reserve virtual address space for two mappable MD pages
3777 * for pmap_zero_page and pmap_copy_page, one MI page
3778 * for /dev/mem, and some more for dumpsys().
3779 */
3780 q = p;
3781 cpuinfo.vpage[0] = (void *)p, p += NBPG;
3782 cpuinfo.vpage[1] = (void *)p, p += NBPG;
3783 p = (vaddr_t)reserve_dumppages((void *)p);
3784
3785 /* Find PTE locations of vpage[] to optimize zero_fill() et.al. */
3786 for (i = 0; i < 2; i++) {
3787 struct regmap *rp;
3788 struct segmap *sp;
3789 rp = &pmap_kernel()->pm_regmap[VA_VREG(cpuinfo.vpage[i])];
3790 sp = &rp->rg_segmap[VA_VSEG(cpuinfo.vpage[i])];
3791 cpuinfo.vpage_pte[i] =
3792 &sp->sg_pte[VA_SUN4M_VPG(cpuinfo.vpage[i])];
3793 }
3794
3795 #if !(defined(PROM_AT_F0) || defined(MSIIEP))
3796 virtual_avail = p;
3797 #elif defined(MSIIEP)
3798 virtual_avail = (vaddr_t)0xf0800000; /* Krups */
3799 #else
3800 virtual_avail = (vaddr_t)0xf0080000; /* Mr.Coffee/OFW */
3801 #endif
3802 virtual_end = VM_MAX_KERNEL_ADDRESS;
3803
3804 p = q; /* retract to first free phys */
3805
3806 /*
3807 * Set up the ctxinfo structures (freelist of contexts)
3808 */
3809 ci->c_pmap = pmap_kernel();
3810 ctx_freelist = ci + 1;
3811 for (i = 1; i < ncontext; i++) {
3812 ci++;
3813 ci->c_nextfree = ci + 1;
3814 }
3815 ci->c_nextfree = NULL;
3816 ctx_kick = 0;
3817 ctx_kickdir = -1;
3818
3819 /*
3820 * Now map the kernel into our new set of page tables, then
3821 * (finally) switch over to our running page tables.
3822 * We map from KERNBASE to p into context 0's page tables (and
3823 * the kernel pmap).
3824 */
3825 #ifdef DEBUG /* Sanity checks */
3826 if (p % NBPG != 0)
3827 panic("pmap_bootstrap4m: p misaligned?!?");
3828 if (KERNBASE % NBPRG != 0)
3829 panic("pmap_bootstrap4m: KERNBASE not region-aligned");
3830 #endif
3831
3832 for (q = KERNBASE; q < p; q += NBPG) {
3833 struct regmap *rp;
3834 struct segmap *sp;
3835 int pte, *ptep;
3836
3837 /*
3838 * Now install entry for current page.
3839 */
3840 rp = &pmap_kernel()->pm_regmap[VA_VREG(q)];
3841 sp = &rp->rg_segmap[VA_VSEG(q)];
3842 ptep = &sp->sg_pte[VA_VPG(q)];
3843
3844 /*
3845 * Unmap the `etext gap'; it'll be made available
3846 * to the VM manager.
3847 */
3848 if (q >= etext_gap_start && q < etext_gap_end) {
3849 setpgt4m(ptep, 0);
3850 continue;
3851 }
3852
3853 pte = PMAP_BOOTSTRAP_VA2PA(q) >> SRMMU_PPNPASHIFT;
3854 pte |= PPROT_N_RX | SRMMU_TEPTE;
3855
3856 /* Deal with the cacheable bit for pagetable memory */
3857 if ((CACHEINFO.c_flags & CACHE_PAGETABLES) != 0 ||
3858 q < pagetables_start || q >= pagetables_end)
3859 pte |= SRMMU_PG_C;
3860
3861 /* write-protect kernel text */
3862 if (q < (vaddr_t)trapbase || q >= (vaddr_t)etext)
3863 pte |= PPROT_WRITE;
3864
3865 setpgt4m(ptep, pte);
3866 pmap_kernel()->pm_stats.resident_count++;
3867 }
3868
3869 if ((CACHEINFO.c_flags & CACHE_PAGETABLES) == 0) {
3870 /*
3871 * The page tables have been setup. Since we're still
3872 * running on the PROM's memory map, the memory we
3873 * allocated for our page tables might still be cached.
3874 * Flush it now, and don't touch it again until we
3875 * switch to our own tables (will be done immediately below).
3876 */
3877 int size = pagetables_end - pagetables_start;
3878 if (CACHEINFO.c_vactype != VAC_NONE) {
3879 va = (vaddr_t)pagetables_start;
3880 while (size > 0) {
3881 cache_flush_page(va, 0);
3882 va += NBPG;
3883 size -= NBPG;
3884 }
3885 } else if (cpuinfo.pcache_flush_page != NULL) {
3886 paddr_t pa = pagetables_start_pa;
3887 while (size > 0) {
3888 pcache_flush_page(pa, 0);
3889 pa += NBPG;
3890 size -= NBPG;
3891 }
3892 }
3893 }
3894
3895 /*
3896 * Now switch to kernel pagetables (finally!)
3897 */
3898 mmu_install_tables(&cpuinfo);
3899
3900 #if defined(MULTIPROCESSOR)
3901 /*
3902 * Initialise any cpu-specific data now.
3903 */
3904 cpu_init_system();
3905
3906 /*
3907 * Setup the cpus[] array and the ci_self links.
3908 */
3909 for (i = 0; i < sparc_ncpus; i++) {
3910 sva = (vaddr_t) (cpuinfo_data + (cpuinfo_len * i));
3911 cpuinfo_va = sva +
3912 (((CPUINFO_VA & (align - 1)) + align - sva) & (align - 1));
3913
3914 /*
3915 * Either remap from CPUINFO_VA to the new correct value
3916 * or clear out this cpuinfo.
3917 */
3918 if (i == 0) {
3919 for (off = 0, va = cpuinfo_va;
3920 sparc_ncpus > 1 && off < sizeof(struct cpu_info);
3921 va += NBPG, off += NBPG) {
3922 paddr_t pa =
3923 PMAP_BOOTSTRAP_VA2PA(CPUINFO_VA + off);
3924
3925 pmap_kremove(va, NBPG);
3926 pmap_kenter_pa(va, pa,
3927 VM_PROT_READ | VM_PROT_WRITE, 0);
3928 }
3929
3930 } else
3931 memset((void *)cpuinfo_va, 0, sizeof(struct cpu_info));
3932
3933 cpus[i] = (struct cpu_info *)cpuinfo_va;
3934 cpus[i]->ci_self = cpus[i];
3935
3936 /* Unmap and prepare to return unused pages */
3937 if (cpuinfo_va != sva) {
3938 cpus[i]->ci_free_sva1 = sva;
3939 cpus[i]->ci_free_eva1 = cpuinfo_va;
3940 for (va = cpus[i]->ci_free_sva1;
3941 va < cpus[i]->ci_free_eva1;
3942 va += NBPG)
3943 setpte4m(va, 0);
3944 }
3945 if (cpuinfo_va + sz != sva + cpuinfo_len) {
3946 cpus[i]->ci_free_sva2 = cpuinfo_va + sz;
3947 cpus[i]->ci_free_eva2 = sva + cpuinfo_len;
3948 for (va = cpus[i]->ci_free_sva2;
3949 va < cpus[i]->ci_free_eva2;
3950 va += NBPG)
3951 setpte4m(va, 0);
3952 }
3953 }
3954 #else
3955 cpus[0] = (struct cpu_info *)CPUINFO_VA;
3956 #endif
3957
3958 pmap_update(pmap_kernel());
3959
3960 #ifdef DIAGNOSTIC
3961 if (curcpu()->ci_self != cpus[0]) {
3962 prom_printf("curcpu()->ci_self %p != cpus[0] %p\n", curcpu()->ci_self, cpus[0]);
3963 panic("cpuinfo inconsistent");
3964 }
3965 #endif
3966 }
3967
3968 static u_long prom_ctxreg;
3969
3970 void
mmu_install_tables(struct cpu_info * sc)3971 mmu_install_tables(struct cpu_info *sc)
3972 {
3973
3974 #ifdef DEBUG
3975 prom_printf("pmap_bootstrap: installing kernel page tables...");
3976 #endif
3977 setcontext4m(0); /* paranoia? %%%: Make 0x3 a define! below */
3978
3979 /* Enable MMU tablewalk caching, flush TLB */
3980 if (sc->mmu_enable != 0)
3981 sc->mmu_enable();
3982
3983 tlb_flush_all_real();
3984 prom_ctxreg = lda(SRMMU_CXTPTR, ASI_SRMMU);
3985
3986 sta(SRMMU_CXTPTR, ASI_SRMMU,
3987 (sc->ctx_tbl_pa >> SRMMU_PPNPASHIFT) & ~0x3);
3988
3989 tlb_flush_all_real();
3990
3991 #ifdef DEBUG
3992 prom_printf("done.\n");
3993 #endif
3994 }
3995
3996 void srmmu_restore_prom_ctx(void);
3997
3998 void
srmmu_restore_prom_ctx(void)3999 srmmu_restore_prom_ctx(void)
4000 {
4001
4002 tlb_flush_all();
4003 sta(SRMMU_CXTPTR, ASI_SRMMU, prom_ctxreg);
4004 tlb_flush_all();
4005 }
4006 #endif /* SUN4M || SUN4D */
4007
4008 #if defined(MULTIPROCESSOR)
4009 /*
4010 * Allocate per-CPU page tables. One region, segment and page table
4011 * is needed to map CPUINFO_VA to different physical addresses on
4012 * each CPU. Since the kernel region and segment tables are all
4013 * pre-wired (in bootstrap() above) and we also assume that the
4014 * first segment (256K) of kernel space is fully populated with
4015 * pages from the start, these per-CPU tables will never need
4016 * to be updated when mapping kernel virtual memory.
4017 *
4018 * Note: this routine is called in the context of the boot CPU
4019 * during autoconfig.
4020 */
4021 void
pmap_alloc_cpu(struct cpu_info * sc)4022 pmap_alloc_cpu(struct cpu_info *sc)
4023 {
4024 #if defined(SUN4M) || defined(SUN4D) /* Only implemented for SUN4M/D */
4025 vaddr_t va;
4026 paddr_t pa;
4027 paddr_t alignment;
4028 u_int *ctxtable, *regtable, *segtable, *pagtable;
4029 u_int *ctxtable_pa, *regtable_pa, *segtable_pa, *pagtable_pa;
4030 psize_t ctxsize, size;
4031 int vr, vs, vpg;
4032 struct regmap *rp;
4033 struct segmap *sp;
4034 struct pglist mlist;
4035 int cachebit;
4036 int pagesz = NBPG;
4037 int i;
4038
4039 cachebit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0;
4040
4041 /*
4042 * Allocate properly aligned and contiguous physically memory
4043 * for the PTE tables.
4044 */
4045 ctxsize = (sc->mmu_ncontext * sizeof(int) + pagesz - 1) & -pagesz;
4046 alignment = ctxsize;
4047
4048 /* The region, segment and page table we need fit in one page */
4049 size = ctxsize + pagesz;
4050
4051 if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys,
4052 alignment, 0, &mlist, 1, 0) != 0)
4053 panic("pmap_alloc_cpu: no memory");
4054
4055 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&mlist));
4056
4057 /* Allocate virtual memory */
4058 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
4059 if (va == 0)
4060 panic("pmap_alloc_cpu: no memory");
4061
4062 /*
4063 * Layout the page tables in our chunk of memory
4064 */
4065 ctxtable = (u_int *)va;
4066 regtable = (u_int *)(va + ctxsize);
4067 segtable = regtable + SRMMU_L1SIZE;
4068 pagtable = segtable + SRMMU_L2SIZE;
4069
4070 ctxtable_pa = (u_int *)pa;
4071 regtable_pa = (u_int *)(pa + ctxsize);
4072 segtable_pa = regtable_pa + SRMMU_L1SIZE;
4073 pagtable_pa = segtable_pa + SRMMU_L2SIZE;
4074
4075 /* Map the pages */
4076 while (size != 0) {
4077 pmap_kenter_pa(va, pa | (cachebit ? 0 : PMAP_NC),
4078 VM_PROT_READ | VM_PROT_WRITE, 0);
4079 va += pagesz;
4080 pa += pagesz;
4081 size -= pagesz;
4082 }
4083 pmap_update(pmap_kernel());
4084
4085 /*
4086 * Store the region table pointer (and its corresponding physical
4087 * address) in the CPU's slot in the kernel pmap region table
4088 * pointer table.
4089 */
4090 pmap_kernel()->pm_reg_ptps[sc->ci_cpuid] = regtable;
4091 pmap_kernel()->pm_reg_ptps_pa[sc->ci_cpuid] = (paddr_t)regtable_pa;
4092
4093 vr = VA_VREG(CPUINFO_VA);
4094 vs = VA_VSEG(CPUINFO_VA);
4095 vpg = VA_VPG(CPUINFO_VA);
4096 rp = &pmap_kernel()->pm_regmap[vr];
4097 sp = &rp->rg_segmap[vs];
4098
4099 /*
4100 * Copy page tables from CPU #0, then modify entry for CPUINFO_VA
4101 * so that it points at the per-CPU pages.
4102 */
4103 qcopy(pmap_kernel()->pm_reg_ptps[0], regtable,
4104 SRMMU_L1SIZE * sizeof(int));
4105 qcopy(rp->rg_seg_ptps, segtable, SRMMU_L2SIZE * sizeof(int));
4106 qcopy(sp->sg_pte, pagtable, SRMMU_L3SIZE * sizeof(int));
4107
4108 setpgt4m(&ctxtable[0],
4109 ((u_long)regtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
4110 setpgt4m(®table[vr],
4111 ((u_long)segtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
4112 setpgt4m(&segtable[vs],
4113 ((u_long)pagtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
4114 setpgt4m(&pagtable[vpg],
4115 (VA2PA((void *)sc) >> SRMMU_PPNPASHIFT) |
4116 (SRMMU_TEPTE | PPROT_N_RWX | SRMMU_PG_C));
4117
4118 /* Install this CPU's context table */
4119 sc->ctx_tbl = ctxtable;
4120 sc->ctx_tbl_pa = (paddr_t)ctxtable_pa;
4121
4122 /* Pre-compute this CPU's vpage[] PTEs */
4123 for (i = 0; i < 2; i++) {
4124 rp = &pmap_kernel()->pm_regmap[VA_VREG(sc->vpage[i])];
4125 sp = &rp->rg_segmap[VA_VSEG(sc->vpage[i])];
4126 sc->vpage_pte[i] = &sp->sg_pte[VA_SUN4M_VPG(sc->vpage[i])];
4127 }
4128 #endif /* SUN4M || SUN4D */
4129 }
4130 #endif /* MULTIPROCESSOR */
4131
4132
4133 void
pmap_init(void)4134 pmap_init(void)
4135 {
4136 u_int sz;
4137
4138 if (PAGE_SIZE != NBPG)
4139 panic("pmap_init: PAGE_SIZE!=NBPG");
4140
4141 vm_num_phys = vm_last_phys - vm_first_phys;
4142
4143 /* Setup a pool for additional pvlist structures */
4144 pool_init(&pv_pool, sizeof(struct pvlist), 0, 0, 0, "pvtable", NULL,
4145 IPL_NONE);
4146
4147 /*
4148 * Setup a pool for pmap structures.
4149 * The pool size includes space for an array of per-CPU
4150 * region table pointers & physical addresses
4151 */
4152 sz = ALIGN(sizeof(struct pmap)) +
4153 ALIGN(NUREG * sizeof(struct regmap)) +
4154 sparc_ncpus * sizeof(int *) + /* pm_reg_ptps */
4155 sparc_ncpus * sizeof(int); /* pm_reg_ptps_pa */
4156 pool_cache_bootstrap(&pmap_cache, sz, 0, 0, 0, "pmappl", NULL,
4157 IPL_NONE, pmap_pmap_pool_ctor, pmap_pmap_pool_dtor, NULL);
4158
4159 sz = NSEGRG * sizeof (struct segmap);
4160 pool_init(&segmap_pool, sz, 0, 0, 0, "segmap", NULL, IPL_NONE);
4161
4162 #if defined(SUN4M) || defined(SUN4D)
4163 if (CPU_HAS_SRMMU) {
4164 /*
4165 * The SRMMU only ever needs chunks in one of two sizes:
4166 * 1024 (for region level tables) and 256 (for segment
4167 * and page level tables).
4168 */
4169 sz = SRMMU_L1SIZE * sizeof(int);
4170 pool_init(&L1_pool, sz, sz, 0, 0, "L1 pagetable",
4171 &pgt_page_allocator, IPL_NONE);
4172
4173 sz = SRMMU_L2SIZE * sizeof(int);
4174 pool_init(&L23_pool, sz, sz, 0, 0, "L2/L3 pagetable",
4175 &pgt_page_allocator, IPL_NONE);
4176 }
4177 #endif /* SUN4M || SUN4D */
4178 #if defined(SUN4) || defined(SUN4C)
4179 if (CPU_HAS_SUNMMU) {
4180 sz = NPTESG * sizeof(int);
4181 pool_init(&pte_pool, sz, 0, 0, 0, "ptemap", NULL,
4182 IPL_NONE);
4183 }
4184 #endif /* SUN4 || SUN4C */
4185 }
4186
4187
4188 /*
4189 * Map physical addresses into kernel VM.
4190 */
4191 vaddr_t
pmap_map(vaddr_t va,paddr_t pa,paddr_t endpa,int prot)4192 pmap_map(vaddr_t va, paddr_t pa, paddr_t endpa, int prot)
4193 {
4194 int pgsize = PAGE_SIZE;
4195
4196 while (pa < endpa) {
4197 pmap_kenter_pa(va, pa, prot, 0);
4198 va += pgsize;
4199 pa += pgsize;
4200 }
4201 pmap_update(pmap_kernel());
4202 return (va);
4203 }
4204
4205 /*
4206 * Check a pmap for spuriously lingering mappings
4207 */
4208 static inline void
pmap_quiet_check(struct pmap * pm)4209 pmap_quiet_check(struct pmap *pm)
4210 {
4211 #ifdef DEBUG
4212 int vs, vr;
4213
4214 if (CPU_HAS_SUNMMU) {
4215 #if defined(SUN4_MMU3L)
4216 if (TAILQ_FIRST(&pm->pm_reglist))
4217 panic("pmap_destroy: region list not empty");
4218 #endif
4219 if (TAILQ_FIRST(&pm->pm_seglist))
4220 panic("pmap_destroy: segment list not empty");
4221 }
4222
4223 for (vr = 0; vr < NUREG; vr++) {
4224 struct regmap *rp = &pm->pm_regmap[vr];
4225
4226 if (HASSUN4_MMU3L) {
4227 if (rp->rg_smeg != reginval)
4228 printf("pmap_chk: spurious smeg in "
4229 "user region %d\n", vr);
4230 }
4231 if (CPU_HAS_SRMMU) {
4232 int n;
4233 #if defined(MULTIPROCESSOR)
4234 for (n = 0; n < sparc_ncpus; n++)
4235 #else
4236 n = 0;
4237 #endif
4238 {
4239 /* Did this cpu attach? */
4240 if (pmap_kernel()->pm_reg_ptps[n] == 0)
4241 continue;
4242
4243 if (pm->pm_reg_ptps[n][vr] != SRMMU_TEINVALID)
4244 printf("pmap_chk: spurious PTP in user "
4245 "region %d on CPU %d\n", vr, n);
4246 }
4247 }
4248 if (rp->rg_nsegmap != 0)
4249 printf("pmap_chk: %d segments remain in "
4250 "region %d\n", rp->rg_nsegmap, vr);
4251 if (rp->rg_segmap != NULL) {
4252 printf("pmap_chk: segments still "
4253 "allocated in region %d\n", vr);
4254 for (vs = 0; vs < NSEGRG; vs++) {
4255 struct segmap *sp = &rp->rg_segmap[vs];
4256 if (sp->sg_npte != 0)
4257 printf("pmap_chk: %d ptes "
4258 "remain in segment %d\n",
4259 sp->sg_npte, vs);
4260 if (sp->sg_pte != NULL) {
4261 printf("pmap_chk: ptes still "
4262 "allocated in segment %d\n", vs);
4263 }
4264 if (CPU_HAS_SUNMMU) {
4265 if (sp->sg_pmeg != seginval)
4266 printf("pmap_chk: pm %p(%d,%d) "
4267 "spurious soft pmeg %d\n",
4268 pm, vr, vs, sp->sg_pmeg);
4269 }
4270 }
4271 }
4272
4273 /* Check for spurious pmeg entries in the MMU */
4274 if (pm->pm_ctx == NULL)
4275 continue;
4276 if (CPU_HAS_SUNMMU) {
4277 int ctx;
4278 if (mmu_has_hole && (vr >= 32 && vr < (256 - 32)))
4279 continue;
4280 ctx = getcontext4();
4281 setcontext4(pm->pm_ctxnum);
4282 for (vs = 0; vs < NSEGRG; vs++) {
4283 vaddr_t va = VSTOVA(vr,vs);
4284 int pmeg = getsegmap(va);
4285 if (pmeg != seginval)
4286 printf("pmap_chk: pm %p(%d,%d:%x): "
4287 "spurious pmeg %d\n",
4288 pm, vr, vs, (u_int)va, pmeg);
4289 }
4290 setcontext4(ctx);
4291 }
4292 }
4293 if (pm->pm_stats.resident_count) {
4294 printf("pmap_chk: res count %ld\n",
4295 pm->pm_stats.resident_count);
4296 }
4297 if (pm->pm_stats.wired_count) {
4298 printf("pmap_chk: wired count %ld\n",
4299 pm->pm_stats.wired_count);
4300 }
4301 #endif /* DEBUG */
4302 }
4303
4304 int
pmap_pmap_pool_ctor(void * arg,void * object,int flags)4305 pmap_pmap_pool_ctor(void *arg, void *object, int flags)
4306 {
4307 struct pmap *pm = object;
4308 u_long addr;
4309
4310 memset(pm, 0, sizeof *pm);
4311
4312 /*
4313 * `pmap_pool' entries include space for the per-CPU
4314 * region table pointer arrays.
4315 */
4316 addr = (u_long)pm + ALIGN(sizeof(struct pmap));
4317 pm->pm_regmap = (void *)addr;
4318 addr += ALIGN(NUREG * sizeof(struct regmap));
4319 pm->pm_reg_ptps = (int **)addr;
4320 addr += sparc_ncpus * sizeof(int *);
4321 pm->pm_reg_ptps_pa = (int *)addr;
4322
4323 qzero((void *)pm->pm_regmap, NUREG * sizeof(struct regmap));
4324
4325 /* pm->pm_ctx = NULL; // already done */
4326
4327 if (CPU_HAS_SUNMMU) {
4328 TAILQ_INIT(&pm->pm_seglist);
4329 #if defined(SUN4_MMU3L)
4330 TAILQ_INIT(&pm->pm_reglist);
4331 if (HASSUN4_MMU3L) {
4332 int i;
4333 for (i = NUREG; --i >= 0;)
4334 pm->pm_regmap[i].rg_smeg = reginval;
4335 }
4336 #endif
4337 }
4338 #if defined(SUN4M) || defined(SUN4D)
4339 else {
4340 int i, n;
4341
4342 /*
4343 * We must allocate and initialize hardware-readable (MMU)
4344 * pagetables. We must also map the kernel regions into this
4345 * pmap's pagetables, so that we can access the kernel from
4346 * this user context.
4347 */
4348 #if defined(MULTIPROCESSOR)
4349 for (n = 0; n < sparc_ncpus; n++)
4350 #else
4351 n = 0;
4352 #endif
4353 {
4354 int *upt, *kpt;
4355
4356 #if defined(MULTIPROCESSOR)
4357 /* Did this cpu attach? */
4358 if (pmap_kernel()->pm_reg_ptps[n] == 0)
4359 continue;
4360 #endif
4361
4362 upt = pool_get(&L1_pool, flags);
4363 pm->pm_reg_ptps[n] = upt;
4364 pm->pm_reg_ptps_pa[n] = VA2PA((char *)upt);
4365
4366 /* Invalidate user space regions */
4367 for (i = 0; i < NUREG; i++)
4368 setpgt4m(upt++, SRMMU_TEINVALID);
4369
4370 /* Copy kernel regions */
4371 kpt = &pmap_kernel()->pm_reg_ptps[n][VA_VREG(KERNBASE)];
4372 for (i = 0; i < NKREG; i++)
4373 setpgt4m(upt++, kpt[i]);
4374 }
4375 }
4376 #endif /* SUN4M || SUN4D */
4377
4378 /* XXX - a peculiar place to do this, but we can't do it in pmap_init
4379 * and here at least it's off the beaten code track.
4380 */
4381 {static int x; if (x == 0) pool_setlowat(&pv_pool, 512), x = 1; }
4382
4383 return (0);
4384 }
4385
4386 void
pmap_pmap_pool_dtor(void * arg,void * object)4387 pmap_pmap_pool_dtor(void *arg, void *object)
4388 {
4389 struct pmap *pm = object;
4390 union ctxinfo *c;
4391 int s = splvm(); /* paranoia */
4392
4393 DPRINTF(PDB_DESTROY, "pmap_pmap_pool_dtor(%p)", pm);
4394
4395 if ((c = pm->pm_ctx) != NULL) {
4396 mutex_spin_enter(&ctx_lock);
4397 ctx_free(pm);
4398 mutex_spin_exit(&ctx_lock);
4399 }
4400
4401 #if defined(SUN4M) || defined(SUN4D)
4402 if (CPU_HAS_SRMMU) {
4403 int n;
4404
4405 #if defined(MULTIPROCESSOR)
4406 for (n = 0; n < sparc_ncpus; n++)
4407 #else
4408 n = 0;
4409 #endif
4410 {
4411 int *pt;
4412
4413 #if defined(MULTIPROCESSOR)
4414 /* Did this cpu attach? */
4415 if (pmap_kernel()->pm_reg_ptps[n] == 0)
4416 continue;
4417 #endif
4418
4419 pt = pm->pm_reg_ptps[n];
4420 pm->pm_reg_ptps[n] = NULL;
4421 pm->pm_reg_ptps_pa[n] = 0;
4422 pool_put(&L1_pool, pt);
4423 }
4424 }
4425 #endif /* SUN4M || SUN4D */
4426 splx(s);
4427 }
4428
4429 /*
4430 * Create and return a physical map.
4431 */
4432 struct pmap *
pmap_create(void)4433 pmap_create(void)
4434 {
4435 struct pmap *pm;
4436
4437 pm = pool_cache_get(&pmap_cache, PR_WAITOK);
4438
4439 /*
4440 * Reset fields that are not preserved in the pmap cache pool.
4441 */
4442 pm->pm_refcount = 1;
4443 #if defined(MULTIPROCESSOR)
4444 /* reset active CPU set */
4445 pm->pm_cpuset = 0;
4446 #endif
4447 if (CPU_HAS_SUNMMU) {
4448 /* reset the region gap */
4449 pm->pm_gap_start = 0;
4450 pm->pm_gap_end = VA_VREG(VM_MAXUSER_ADDRESS);
4451 }
4452
4453 DPRINTF(PDB_CREATE, "pmap_create[%d]: created %p", cpu_number(), pm);
4454 pmap_quiet_check(pm);
4455
4456 return (pm);
4457 }
4458
4459 /*
4460 * Retire the given pmap from service.
4461 * Should only be called if the map contains no valid mappings.
4462 */
4463 void
pmap_destroy(struct pmap * pm)4464 pmap_destroy(struct pmap *pm)
4465 {
4466
4467 DPRINTF(PDB_DESTROY, "pmap_destroy[%d](%p)", cpu_number(), pm);
4468 membar_release();
4469 if (atomic_dec_uint_nv(&pm->pm_refcount) == 0) {
4470 membar_acquire();
4471 pmap_quiet_check(pm);
4472 pool_cache_put(&pmap_cache, pm);
4473 }
4474 }
4475
4476 /*
4477 * Add a reference to the given pmap.
4478 */
4479 void
pmap_reference(struct pmap * pm)4480 pmap_reference(struct pmap *pm)
4481 {
4482
4483 atomic_inc_uint(&pm->pm_refcount);
4484 }
4485
4486 #if defined(SUN4) || defined(SUN4C)
4487 /*
4488 * helper to deallocate level 2 & 3 page tables.
4489 */
4490 static void
pgt_lvl23_remove4_4c(struct pmap * pm,struct regmap * rp,struct segmap * sp,int vr,int vs)4491 pgt_lvl23_remove4_4c(struct pmap *pm, struct regmap *rp, struct segmap *sp,
4492 int vr, int vs)
4493 {
4494 vaddr_t va, tva;
4495 int i, pmeg;
4496
4497 va = VSTOVA(vr,vs);
4498 if ((pmeg = sp->sg_pmeg) != seginval) {
4499 if (CTX_USABLE(pm,rp)) {
4500 setcontext4(pm->pm_ctxnum);
4501 setsegmap(va, seginval);
4502 } else {
4503 /* no context, use context 0 */
4504 setcontext4(0);
4505 if (HASSUN4_MMU3L && rp->rg_smeg != reginval) {
4506 setregmap(0, rp->rg_smeg);
4507 tva = vs << SGSHIFT;
4508 setsegmap(tva, seginval);
4509 }
4510 }
4511 if (!HASSUN4_MMU3L) {
4512 if (pm == pmap_kernel()) {
4513 /* Unmap segment from all contexts */
4514 for (i = ncontext; --i >= 0;) {
4515 setcontext4(i);
4516 setsegmap(va, seginval);
4517 }
4518 }
4519 }
4520 me_free(pm, pmeg);
4521 sp->sg_pmeg = seginval;
4522 }
4523 /* Free software tables for non-kernel maps */
4524 if (pm != pmap_kernel()) {
4525 pool_put(&pte_pool, sp->sg_pte);
4526 sp->sg_pte = NULL;
4527 }
4528
4529 if (rp->rg_nsegmap <= 0)
4530 panic("pgt_rm: pm %p: nsegmap = %d\n", pm, rp->rg_nsegmap);
4531
4532 if (--rp->rg_nsegmap == 0) {
4533 #if defined(SUN4_MMU3L)
4534 if (HASSUN4_MMU3L) {
4535 if (rp->rg_smeg != reginval) {
4536 if (pm == pmap_kernel()) {
4537 /* Unmap from all contexts */
4538 for (i = ncontext; --i >= 0;) {
4539 setcontext4(i);
4540 setregmap(va, reginval);
4541 }
4542 } else if (pm->pm_ctx) {
4543 setcontext4(pm->pm_ctxnum);
4544 setregmap(va, reginval);
4545 }
4546
4547 /* Release MMU resource */
4548 region_free(pm, rp->rg_smeg);
4549 rp->rg_smeg = reginval;
4550 }
4551 }
4552 #endif /* SUN4_MMU3L */
4553 /* Free software tables for non-kernel maps */
4554 if (pm != pmap_kernel()) {
4555 GAP_WIDEN(pm,vr);
4556 pool_put(&segmap_pool, rp->rg_segmap);
4557 rp->rg_segmap = NULL;
4558 }
4559 }
4560 }
4561 #endif /* SUN4 || SUN4C */
4562
4563 #if defined(SUN4M) || defined(SUN4D)
4564 /*
4565 * SRMMU helper to deallocate level 2 & 3 page tables.
4566 */
4567 static void
pgt_lvl23_remove4m(struct pmap * pm,struct regmap * rp,struct segmap * sp,int vr,int vs)4568 pgt_lvl23_remove4m(struct pmap *pm, struct regmap *rp, struct segmap *sp,
4569 int vr, int vs)
4570 {
4571
4572 /* Invalidate level 2 PTP entry */
4573 if (pm->pm_ctx)
4574 tlb_flush_segment(VSTOVA(vr,vs), pm->pm_ctxnum,
4575 PMAP_CPUSET(pm));
4576 setpgt4m(&rp->rg_seg_ptps[vs], SRMMU_TEINVALID);
4577 pool_put(&L23_pool, sp->sg_pte);
4578 sp->sg_pte = NULL;
4579
4580 /* If region is now empty, remove level 2 pagetable as well */
4581 if (--rp->rg_nsegmap == 0) {
4582 int n = 0;
4583 if (pm->pm_ctx)
4584 tlb_flush_region(VRTOVA(vr), pm->pm_ctxnum,
4585 PMAP_CPUSET(pm));
4586 #if defined(MULTIPROCESSOR)
4587 /* Invalidate level 1 PTP entries on all CPUs */
4588 for (; n < sparc_ncpus; n++) {
4589 if ((cpus[n]->flags & CPUFLG_HATCHED) == 0)
4590 continue;
4591 #endif
4592 setpgt4m(&pm->pm_reg_ptps[n][vr], SRMMU_TEINVALID);
4593 #if defined(MULTIPROCESSOR)
4594 }
4595 #endif
4596
4597 pool_put(&segmap_pool, rp->rg_segmap);
4598 rp->rg_segmap = NULL;
4599 pool_put(&L23_pool, rp->rg_seg_ptps);
4600 }
4601 }
4602 #endif /* SUN4M || SUN4D */
4603
4604 bool
pmap_remove_all(struct pmap * pm)4605 pmap_remove_all(struct pmap *pm)
4606 {
4607 if (pm->pm_ctx == NULL)
4608 return false;
4609
4610 #if defined(SUN4) || defined(SUN4C)
4611 if (CPU_HAS_SUNMMU) {
4612 int ctx = getcontext4();
4613 setcontext4(pm->pm_ctxnum);
4614 cache_flush_context(pm->pm_ctxnum);
4615 setcontext4(ctx);
4616 }
4617 #endif
4618
4619 #if defined(SUN4M) || defined(SUN4D)
4620 if (CPU_HAS_SRMMU) {
4621 cache_flush_context(pm->pm_ctxnum);
4622 }
4623 #endif
4624
4625 pm->pm_flags |= PMAP_USERCACHECLEAN;
4626 return false;
4627 }
4628
4629 /*
4630 * Remove the given range of mapping entries.
4631 * The starting and ending addresses are already rounded to pages.
4632 * Sheer lunacy: pmap_remove is often asked to remove nonexistent
4633 * mappings.
4634 */
4635 void
pmap_remove(struct pmap * pm,vaddr_t va,vaddr_t endva)4636 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva)
4637 {
4638 vaddr_t nva;
4639 int vr, vs, s, ctx;
4640 void (*rm)(struct pmap *, vaddr_t, vaddr_t, int, int);
4641
4642 DPRINTF(PDB_REMOVE, "pmap_remove[%d](%p, 0x%lx, 0x%lx)",
4643 cpu_number(), pm, va, endva);
4644
4645 if (!CPU_HAS_SRMMU)
4646 write_user_windows();
4647
4648 if (pm == pmap_kernel()) {
4649 /*
4650 * Removing from kernel address space.
4651 */
4652 rm = pmap_rmk;
4653 } else {
4654 /*
4655 * Removing from user address space.
4656 */
4657 rm = pmap_rmu;
4658 }
4659
4660 ctx = getcontext();
4661 s = splvm();
4662 PMAP_LOCK();
4663 for (; va < endva; va = nva) {
4664 /* do one virtual segment at a time */
4665 vr = VA_VREG(va);
4666 vs = VA_VSEG(va);
4667 nva = VSTOVA(vr, vs + 1);
4668 if (nva == 0 || nva > endva)
4669 nva = endva;
4670 if (pm->pm_regmap[vr].rg_nsegmap != 0)
4671 (*rm)(pm, va, nva, vr, vs);
4672 }
4673 PMAP_UNLOCK();
4674 splx(s);
4675 setcontext(ctx);
4676 }
4677
4678 /*
4679 * It is the same amount of work to cache_flush_page 16 pages
4680 * as to cache_flush_segment 1 segment, assuming a 64K cache size
4681 * and a 4K page size or a 128K cache size and 8K page size.
4682 */
4683 #define PMAP_SFL_THRESHOLD 16 /* if > magic, use cache_flush_segment */
4684
4685 /*
4686 * Remove a range contained within a single segment.
4687 * These are egregiously complicated routines.
4688 */
4689
4690 #if defined(SUN4) || defined(SUN4C)
4691
4692 /* remove from kernel */
4693 /*static*/ void
pmap_rmk4_4c(struct pmap * pm,vaddr_t va,vaddr_t endva,int vr,int vs)4694 pmap_rmk4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4695 {
4696 int pte, mmupte, *ptep, perpage, npg;
4697 struct vm_page *pg;
4698 int nleft, pmeg, inmmu;
4699 struct regmap *rp;
4700 struct segmap *sp;
4701
4702 rp = &pm->pm_regmap[vr];
4703 sp = &rp->rg_segmap[vs];
4704
4705 if (rp->rg_nsegmap == 0)
4706 return;
4707 if ((nleft = sp->sg_npte) == 0)
4708 return;
4709 pmeg = sp->sg_pmeg;
4710 inmmu = pmeg != seginval;
4711 ptep = &sp->sg_pte[VA_VPG(va)];
4712
4713 /* decide how to flush cache */
4714 npg = (endva - va) >> PGSHIFT;
4715 if (!inmmu) {
4716 perpage = 0;
4717 } else if (npg > PMAP_SFL_THRESHOLD) {
4718 /* flush the whole segment */
4719 perpage = 0;
4720 cache_flush_segment(vr, vs, 0);
4721 } else {
4722 /* flush each page individually; some never need flushing */
4723 perpage = (CACHEINFO.c_vactype != VAC_NONE);
4724 }
4725
4726 for (; va < endva; va += NBPG, ptep++) {
4727 pte = *ptep;
4728 mmupte = inmmu ? getpte4(va) : 0;
4729 if ((pte & PG_V) == 0) {
4730 #ifdef DIAGNOSTIC
4731 if (inmmu && (mmupte & PG_V) != 0)
4732 printf("rmk: inconsistent ptes va=%lx\n", va);
4733 #endif
4734 continue;
4735 }
4736 if ((pte & PG_TYPE) == PG_OBMEM) {
4737 /* if cacheable, flush page as needed */
4738 if (perpage && (mmupte & PG_NC) == 0)
4739 cache_flush_page(va, 0);
4740 if ((pg = pvhead4_4c(pte)) != NULL) {
4741 if (inmmu)
4742 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
4743 pv_unlink4_4c(pg, pm, va);
4744 }
4745 }
4746 nleft--;
4747 #ifdef DIAGNOSTIC
4748 if (nleft < 0)
4749 panic("pmap_rmk: too many PTEs in segment; "
4750 "va 0x%lx; endva 0x%lx", va, endva);
4751 #endif
4752 if (pte & PG_WIRED) {
4753 sp->sg_nwired--;
4754 pm->pm_stats.wired_count--;
4755 }
4756
4757 if (inmmu)
4758 setpte4(va, 0);
4759 *ptep = 0;
4760 pm->pm_stats.resident_count--;
4761 }
4762
4763 #ifdef DIAGNOSTIC
4764 if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
4765 panic("pmap_rmk: pm %p, va %lx: nleft=%d, nwired=%d",
4766 pm, va, nleft, sp->sg_nwired);
4767 #endif
4768 if ((sp->sg_npte = nleft) == 0)
4769 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
4770 else if (sp->sg_nwired == 0) {
4771 if (sp->sg_pmeg != seginval)
4772 mmu_pmeg_unlock(sp->sg_pmeg);
4773 }
4774 }
4775
4776 #endif /* SUN4 || SUN4C */
4777
4778 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_rmk */
4779
4780 /* remove from kernel (4m)*/
4781 /* pm is already locked */
4782 /*static*/ void
pmap_rmk4m(struct pmap * pm,vaddr_t va,vaddr_t endva,int vr,int vs)4783 pmap_rmk4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4784 {
4785 int tpte, perpage, npg;
4786 struct vm_page *pg;
4787 struct regmap *rp;
4788 struct segmap *sp;
4789
4790 rp = &pm->pm_regmap[vr];
4791 sp = &rp->rg_segmap[vs];
4792 if (rp->rg_nsegmap == 0)
4793 return;
4794
4795 /* decide how to flush cache */
4796 npg = (endva - va) >> PGSHIFT;
4797 if (npg > PMAP_SFL_THRESHOLD) {
4798 /* flush the whole segment */
4799 perpage = 0;
4800 if (CACHEINFO.c_vactype != VAC_NONE)
4801 cache_flush_segment(vr, vs, 0);
4802 } else {
4803 /* flush each page individually; some never need flushing */
4804 perpage = (CACHEINFO.c_vactype != VAC_NONE);
4805 }
4806 while (va < endva) {
4807 tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
4808 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
4809 #ifdef DEBUG
4810 if ((pmapdebug & PDB_SANITYCHK) &&
4811 (getpte4m(va) & SRMMU_TETYPE) == SRMMU_TEPTE)
4812 panic("pmap_rmk: Spurious kTLB entry for 0x%lx",
4813 va);
4814 #endif
4815 va += NBPG;
4816 continue;
4817 }
4818 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
4819 /* if cacheable, flush page as needed */
4820 if (perpage && (tpte & SRMMU_PG_C))
4821 cache_flush_page(va, 0);
4822 if ((pg = pvhead4m(tpte)) != NULL) {
4823 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
4824 pv_unlink4m(pg, pm, va);
4825 }
4826 }
4827 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
4828 SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
4829 pm->pm_stats.resident_count--;
4830 va += NBPG;
4831 }
4832 }
4833 #endif /* SUN4M || SUN4D */
4834
4835 #if defined(SUN4) || defined(SUN4C)
4836
4837 /* remove from user */
4838 /*static*/ void
pmap_rmu4_4c(struct pmap * pm,vaddr_t va,vaddr_t endva,int vr,int vs)4839 pmap_rmu4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4840 {
4841 int *ptep, pteva, pte, perpage, npg;
4842 struct vm_page *pg;
4843 int nleft, pmeg, inmmu;
4844 struct regmap *rp;
4845 struct segmap *sp;
4846
4847 rp = &pm->pm_regmap[vr];
4848 if (rp->rg_nsegmap == 0)
4849 return;
4850 sp = &rp->rg_segmap[vs];
4851 if ((nleft = sp->sg_npte) == 0)
4852 return;
4853 pmeg = sp->sg_pmeg;
4854 inmmu = pmeg != seginval;
4855
4856 /*
4857 * PTEs are in MMU. Invalidate in hardware, update ref &
4858 * mod bits, and flush cache if required.
4859 */
4860 if (!inmmu) {
4861 perpage = 0;
4862 pteva = 0;
4863 } else if (CTX_USABLE(pm,rp)) {
4864 /* process has a context, must flush cache */
4865 npg = (endva - va) >> PGSHIFT;
4866 setcontext4(pm->pm_ctxnum);
4867 if ((pm->pm_flags & PMAP_USERCACHECLEAN) != 0)
4868 perpage = 0;
4869 else if (npg > PMAP_SFL_THRESHOLD) {
4870 perpage = 0; /* flush the whole segment */
4871 cache_flush_segment(vr, vs, pm->pm_ctxnum);
4872 } else
4873 perpage = (CACHEINFO.c_vactype != VAC_NONE);
4874 pteva = va;
4875 } else {
4876 /* no context, use context 0; cache flush unnecessary */
4877 setcontext4(0);
4878 if (HASSUN4_MMU3L)
4879 setregmap(0, tregion);
4880 /* XXX use per-CPU pteva? */
4881 setsegmap(0, pmeg);
4882 pteva = VA_VPG(va) << PGSHIFT;
4883 perpage = 0;
4884 }
4885
4886 ptep = sp->sg_pte + VA_VPG(va);
4887 for (; va < endva; ptep++, pteva += NBPG, va += NBPG) {
4888 int mmupte;
4889 pte = *ptep;
4890 mmupte = inmmu ? getpte4(pteva) : 0;
4891
4892 if ((pte & PG_V) == 0) {
4893 #ifdef DIAGNOSTIC
4894 if (inmmu && (mmupte & PG_V) != 0)
4895 printf("pmap_rmu: pte=%x, mmupte=%x\n",
4896 pte, getpte4(pteva));
4897 #endif
4898 continue;
4899 }
4900 if ((pte & PG_TYPE) == PG_OBMEM) {
4901 /* if cacheable, flush page as needed */
4902 if (perpage && (mmupte & PG_NC) == 0)
4903 cache_flush_page(va, pm->pm_ctxnum);
4904 if ((pg = pvhead4_4c(pte)) != NULL) {
4905 if (inmmu)
4906 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
4907 pv_unlink4_4c(pg, pm, va);
4908 }
4909 }
4910 nleft--;
4911 #ifdef DIAGNOSTIC
4912 if (nleft < 0)
4913 panic("pmap_rmu: too many PTEs in segment; "
4914 "va 0x%lx; endva 0x%lx", va, endva);
4915 #endif
4916 if (inmmu)
4917 setpte4(pteva, 0);
4918
4919 if (pte & PG_WIRED) {
4920 sp->sg_nwired--;
4921 pm->pm_stats.wired_count--;
4922 }
4923 *ptep = 0;
4924 pm->pm_stats.resident_count--;
4925 }
4926
4927 #ifdef DIAGNOSTIC
4928 if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
4929 panic("pmap_rmu: pm %p, va %lx: nleft=%d, nwired=%d",
4930 pm, va, nleft, sp->sg_nwired);
4931 #endif
4932 if ((sp->sg_npte = nleft) == 0)
4933 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
4934 else if (sp->sg_nwired == 0) {
4935 if (sp->sg_pmeg != seginval)
4936 mmu_pmeg_unlock(sp->sg_pmeg);
4937 }
4938 }
4939
4940 #endif /* SUN4 || SUN4C */
4941
4942 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_rmu */
4943 /* remove from user */
4944 /* Note: pm is already locked */
4945 /*static*/ void
pmap_rmu4m(struct pmap * pm,vaddr_t va,vaddr_t endva,int vr,int vs)4946 pmap_rmu4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
4947 {
4948 int *pte0, perpage, npg;
4949 struct vm_page *pg;
4950 int nleft;
4951 struct regmap *rp;
4952 struct segmap *sp;
4953
4954 rp = &pm->pm_regmap[vr];
4955 if (rp->rg_nsegmap == 0)
4956 return;
4957 sp = &rp->rg_segmap[vs];
4958 if ((nleft = sp->sg_npte) == 0)
4959 return;
4960 pte0 = sp->sg_pte;
4961
4962 /*
4963 * Invalidate PTE in MMU pagetables. Flush cache if necessary.
4964 */
4965 if (pm->pm_ctx && (pm->pm_flags & PMAP_USERCACHECLEAN) == 0) {
4966 /* process has a context, must flush cache */
4967 if (CACHEINFO.c_vactype != VAC_NONE) {
4968 npg = (endva - va) >> PGSHIFT;
4969 if (npg > PMAP_SFL_THRESHOLD) {
4970 perpage = 0; /* flush the whole segment */
4971 cache_flush_segment(vr, vs, pm->pm_ctxnum);
4972 } else
4973 perpage = 1;
4974 } else
4975 perpage = 0;
4976 } else {
4977 /* no context; cache flush unnecessary */
4978 perpage = 0;
4979 }
4980 for (; va < endva; va += NBPG) {
4981 int tpte;
4982
4983 tpte = pte0[VA_SUN4M_VPG(va)];
4984
4985 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
4986 #ifdef DEBUG
4987 if ((pmapdebug & PDB_SANITYCHK) &&
4988 pm->pm_ctx &&
4989 (getpte4m(va) & SRMMU_TEPTE) == SRMMU_TEPTE)
4990 panic("pmap_rmu: Spurious uTLB entry for 0x%lx",
4991 va);
4992 #endif
4993 continue;
4994 }
4995
4996 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
4997 /* if cacheable, flush page as needed */
4998 if (perpage && (tpte & SRMMU_PG_C))
4999 cache_flush_page(va, pm->pm_ctxnum);
5000 if ((pg = pvhead4m(tpte)) != NULL) {
5001 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
5002 pv_unlink4m(pg, pm, va);
5003 }
5004 }
5005 nleft--;
5006 #ifdef DIAGNOSTIC
5007 if (nleft < 0)
5008 panic("pmap_rmu: too many PTEs in segment; "
5009 "va 0x%lx; endva 0x%lx", va, endva);
5010 #endif
5011 setpgt4m_va(va, &pte0[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
5012 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
5013 pm->pm_stats.resident_count--;
5014 if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
5015 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
5016 pm->pm_stats.wired_count--;
5017 }
5018 }
5019
5020 /*
5021 * If the segment is all gone, and the context is loaded, give
5022 * the segment back.
5023 */
5024 if ((sp->sg_npte = nleft) == 0)
5025 pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
5026 }
5027 #endif /* SUN4M || SUN4D */
5028
5029 /*
5030 * Lower (make more strict) the protection on the specified
5031 * physical page.
5032 *
5033 * There are only two cases: either the protection is going to 0
5034 * (in which case we do the dirty work here), or it is going from
5035 * to read-only (in which case pv_changepte does the trick).
5036 */
5037
5038 #if defined(SUN4) || defined(SUN4C)
5039 void
pmap_page_protect4_4c(struct vm_page * pg,vm_prot_t prot)5040 pmap_page_protect4_4c(struct vm_page *pg, vm_prot_t prot)
5041 {
5042 struct pvlist *pv, *npv;
5043 struct pmap *pm;
5044 vaddr_t va;
5045 int vr, vs, pteva, pte, *ptep;
5046 int flags, nleft, s, ctx;
5047 struct regmap *rp;
5048 struct segmap *sp;
5049
5050 #ifdef DEBUG
5051 if ((pmapdebug & PDB_CHANGEPROT) ||
5052 (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
5053 printf("pmap_page_protect(0x%lx, 0x%x)\n",
5054 VM_PAGE_TO_PHYS(pg), prot);
5055 #endif
5056
5057 /*
5058 * Skip unmanaged pages, or operations that do not take
5059 * away write permission.
5060 */
5061 if (prot & VM_PROT_WRITE)
5062 return;
5063
5064 write_user_windows(); /* paranoia */
5065 if (prot & VM_PROT_READ) {
5066 pv_changepte4_4c(pg, 0, PG_W);
5067 return;
5068 }
5069
5070 /*
5071 * Remove all access to all people talking to this page.
5072 * Walk down PV list, removing all mappings.
5073 * The logic is much like that for pmap_remove,
5074 * but we know we are removing exactly one page.
5075 */
5076 s = splvm();
5077 pv = VM_MDPAGE_PVHEAD(pg);
5078 if (pv->pv_pmap == NULL) {
5079 splx(s);
5080 return;
5081 }
5082 ctx = getcontext4();
5083
5084 /* This pv head will become empty, so clear caching state flags */
5085 flags = pv->pv_flags & ~(PV_NC|PV_ANC);
5086
5087 while (pv != NULL) {
5088 pm = pv->pv_pmap;
5089 va = pv->pv_va;
5090 vr = VA_VREG(va);
5091 vs = VA_VSEG(va);
5092 rp = &pm->pm_regmap[vr];
5093 sp = &rp->rg_segmap[vs];
5094 if ((nleft = sp->sg_npte) <= 0)
5095 panic("pmap_page_protect: empty vseg");
5096 sp->sg_npte = --nleft;
5097 ptep = &sp->sg_pte[VA_VPG(va)];
5098
5099 if (*ptep & PG_WIRED) {
5100 sp->sg_nwired--;
5101 pm->pm_stats.wired_count--;
5102 }
5103
5104 if (sp->sg_pmeg != seginval) {
5105 /* Update PV flags */
5106 if (CTX_USABLE(pm,rp)) {
5107 setcontext4(pm->pm_ctxnum);
5108 pteva = va;
5109 cache_flush_page(va, pm->pm_ctxnum);
5110 } else {
5111 setcontext4(0);
5112 /* XXX use per-CPU pteva? */
5113 if (HASSUN4_MMU3L)
5114 setregmap(0, tregion);
5115 setsegmap(0, sp->sg_pmeg);
5116 pteva = VA_VPG(va) << PGSHIFT;
5117 }
5118
5119 pte = getpte4(pteva);
5120 #ifdef DIAGNOSTIC
5121 if ((pte & PG_V) == 0)
5122 panic("pmap_page_protect !PG_V: pg %p "
5123 "ctx %d, va 0x%lx, pte 0x%x",
5124 pg, pm->pm_ctxnum, va, pte);
5125 #endif
5126 flags |= MR4_4C(pte);
5127
5128 setpte4(pteva, 0);
5129 #ifdef DIAGNOSTIC
5130 if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
5131 panic("pmap_page_protect: pm %p, va %lx: nleft=%d, nwired=%d",
5132 pm, va, nleft, sp->sg_nwired);
5133 #endif
5134 if (sp->sg_nwired == 0)
5135 mmu_pmeg_unlock(sp->sg_pmeg);
5136 }
5137
5138 *ptep = 0;
5139 pm->pm_stats.resident_count--;
5140 if (nleft == 0)
5141 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
5142 npv = pv->pv_next;
5143 if (pv != VM_MDPAGE_PVHEAD(pg))
5144 pool_put(&pv_pool, pv);
5145 pv = npv;
5146 }
5147
5148 /* Finally, update pv head */
5149 VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
5150 VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
5151 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
5152 setcontext4(ctx);
5153 splx(s);
5154 }
5155
5156 /*
5157 * Lower (make more strict) the protection on the specified
5158 * range of this pmap.
5159 *
5160 * There are only two cases: either the protection is going to 0
5161 * (in which case we call pmap_remove to do the dirty work), or
5162 * it is going from read/write to read-only. The latter is
5163 * fairly easy.
5164 */
5165 void
pmap_protect4_4c(struct pmap * pm,vaddr_t sva,vaddr_t eva,vm_prot_t prot)5166 pmap_protect4_4c(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
5167 {
5168 int va, nva, vr, vs;
5169 int s, ctx;
5170 struct regmap *rp;
5171 struct segmap *sp;
5172
5173 if ((prot & VM_PROT_READ) == 0) {
5174 pmap_remove(pm, sva, eva);
5175 return;
5176 }
5177
5178 write_user_windows();
5179 ctx = getcontext4();
5180 s = splvm();
5181 PMAP_LOCK();
5182 for (va = sva; va < eva;) {
5183 vr = VA_VREG(va);
5184 vs = VA_VSEG(va);
5185 rp = &pm->pm_regmap[vr];
5186 nva = VSTOVA(vr,vs + 1);
5187 if (nva > eva)
5188 nva = eva;
5189 if (rp->rg_nsegmap == 0) {
5190 va = nva;
5191 continue;
5192 }
5193 #ifdef DEBUG
5194 if (rp->rg_segmap == NULL)
5195 panic("pmap_protect: no segments");
5196 #endif
5197 sp = &rp->rg_segmap[vs];
5198 if (sp->sg_npte == 0) {
5199 va = nva;
5200 continue;
5201 }
5202 #ifdef DEBUG
5203 if (sp->sg_pte == NULL)
5204 panic("pmap_protect: no pages");
5205 #endif
5206 if (sp->sg_pmeg == seginval) {
5207 int *ptep = &sp->sg_pte[VA_VPG(va)];
5208
5209 /* not in MMU; just clear PG_W from core copies */
5210 for (; va < nva; va += NBPG)
5211 *ptep++ &= ~PG_W;
5212 } else {
5213 /* in MMU: take away write bits from MMU PTEs */
5214 if (CTX_USABLE(pm,rp)) {
5215 int pte;
5216
5217 /*
5218 * Flush cache so that any existing cache
5219 * tags are updated. This is really only
5220 * needed for PTEs that lose PG_W.
5221 */
5222 pmap_stats.ps_npg_prot_all +=
5223 (nva - va) >> PGSHIFT;
5224 setcontext4(pm->pm_ctxnum);
5225 for (; va < nva; va += NBPG) {
5226 pte = getpte4(va);
5227 if ((pte & (PG_W|PG_TYPE)) ==
5228 (PG_W|PG_OBMEM)) {
5229 pmap_stats.ps_npg_prot_actual++;
5230 cache_flush_page(va, pm->pm_ctxnum);
5231 setpte4(va, pte & ~PG_W);
5232 }
5233 }
5234 } else {
5235 int pteva;
5236
5237 /*
5238 * No context, hence not cached;
5239 * just update PTEs.
5240 */
5241 setcontext4(0);
5242 /* XXX use per-CPU pteva? */
5243 if (HASSUN4_MMU3L)
5244 setregmap(0, tregion);
5245 setsegmap(0, sp->sg_pmeg);
5246 pteva = VA_VPG(va) << PGSHIFT;
5247 for (; va < nva; pteva += NBPG, va += NBPG)
5248 setpte4(pteva, getpte4(pteva) & ~PG_W);
5249 }
5250 }
5251 }
5252 PMAP_UNLOCK();
5253 splx(s);
5254 setcontext4(ctx);
5255 }
5256
5257 /*
5258 * Change the protection and/or wired status of the given (MI) virtual page.
5259 * XXX: should have separate function (or flag) telling whether only wiring
5260 * is changing.
5261 */
5262 void
pmap_changeprot4_4c(struct pmap * pm,vaddr_t va,vm_prot_t prot,int flags)5263 pmap_changeprot4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
5264 {
5265 int vr, vs, newprot, ctx, pte, *ptep;
5266 int pmeg;
5267 struct regmap *rp;
5268 struct segmap *sp;
5269
5270 DPRINTF(PDB_CHANGEPROT, "pmap_changeprot(%p, 0x%lx, 0x%x, 0x%x)",
5271 pm, va, prot, flags);
5272
5273 if (pm == pmap_kernel())
5274 newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;
5275 else
5276 newprot = prot & VM_PROT_WRITE ? PG_W : 0;
5277 vr = VA_VREG(va);
5278 vs = VA_VSEG(va);
5279 rp = &pm->pm_regmap[vr];
5280 sp = &rp->rg_segmap[vs];
5281 ptep = &sp->sg_pte[VA_VPG(va)];
5282
5283 pmap_stats.ps_changeprots++;
5284
5285 pte = *ptep;
5286 if (pte & PG_WIRED && (flags & PMAP_WIRED) == 0) {
5287 pte &= ~PG_WIRED;
5288 sp->sg_nwired--;
5289 pm->pm_stats.wired_count--;
5290 } else if ((pte & PG_WIRED) == 0 && flags & PMAP_WIRED) {
5291 pte |= PG_WIRED;
5292 sp->sg_nwired++;
5293 pm->pm_stats.wired_count++;
5294 }
5295 pte = (pte & ~PG_PROT) | newprot;
5296 /* Update S/W pte entry */
5297 *ptep = pte;
5298
5299 /* update PTEs in software or hardware */
5300 if ((pmeg = sp->sg_pmeg) != seginval) {
5301 /* update in hardware */
5302 ctx = getcontext4();
5303 if (CTX_USABLE(pm,rp)) {
5304 /*
5305 * Use current context.
5306 * Flush cache if page has been referenced to
5307 * avoid stale protection bits in the cache tags.
5308 */
5309 setcontext4(pm->pm_ctxnum);
5310 pte = getpte4(va);
5311 if ((pte & (PG_U|PG_NC|PG_TYPE)) == (PG_U|PG_OBMEM))
5312 cache_flush_page(va, pm->pm_ctxnum);
5313 } else {
5314 setcontext4(0);
5315 /* XXX use per-CPU va? */
5316 if (HASSUN4_MMU3L)
5317 setregmap(0, tregion);
5318 setsegmap(0, pmeg);
5319 va = VA_VPG(va) << PGSHIFT;
5320 pte = getpte4(va);
5321 }
5322 pte = (pte & ~PG_PROT) | newprot;
5323 setpte4(va, pte);
5324 setcontext4(ctx);
5325 #ifdef DIAGNOSTIC
5326 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5327 panic("pmap_protect: pm %p, va %lx: nleft=%d, nwired=%d",
5328 pm, va, sp->sg_npte, sp->sg_nwired);
5329 #endif
5330 if (sp->sg_nwired == 0)
5331 mmu_pmeg_unlock(pmeg);
5332 else
5333 mmu_pmeg_lock(pmeg);
5334 }
5335 }
5336
5337 #endif /* SUN4 || SUN4C */
5338
5339 #if defined(SUN4M) || defined(SUN4D)
5340 /*
5341 * Lower (make more strict) the protection on the specified
5342 * physical page.
5343 *
5344 * There are only two cases: either the protection is going to 0
5345 * (in which case we do the dirty work here), or it is going
5346 * to read-only (in which case pv_changepte does the trick).
5347 */
5348 void
pmap_page_protect4m(struct vm_page * pg,vm_prot_t prot)5349 pmap_page_protect4m(struct vm_page *pg, vm_prot_t prot)
5350 {
5351 struct pvlist *pv, *npv;
5352 struct pmap *pm;
5353 vaddr_t va;
5354 int vr, vs, tpte;
5355 int flags, nleft, s;
5356 struct regmap *rp;
5357 struct segmap *sp;
5358
5359 #ifdef DEBUG
5360 if ((pmapdebug & PDB_CHANGEPROT) ||
5361 (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
5362 printf("pmap_page_protect[%d](0x%lx, 0x%x)\n",
5363 cpu_number(), VM_PAGE_TO_PHYS(pg), prot);
5364 #endif
5365 s = splvm();
5366 PMAP_LOCK();
5367
5368 if (prot & VM_PROT_READ) {
5369 pv_changepte4m(pg, 0, PPROT_WRITE);
5370 goto out;
5371 }
5372
5373 /*
5374 * Remove all access to all people talking to this page.
5375 * Walk down PV list, removing all mappings. The logic is much
5376 * like that for pmap_remove, but we know we are removing exactly
5377 * one page.
5378 */
5379 pv = VM_MDPAGE_PVHEAD(pg);
5380 if (pv->pv_pmap == NULL)
5381 goto out;
5382
5383 /* This pv head will become empty, so clear caching state flags */
5384 flags = pv->pv_flags & ~(PV_NC|PV_ANC);
5385 while (pv != NULL) {
5386 pm = pv->pv_pmap;
5387 va = pv->pv_va;
5388 vr = VA_VREG(va);
5389 vs = VA_VSEG(va);
5390 rp = &pm->pm_regmap[vr];
5391 if (rp->rg_nsegmap == 0)
5392 panic("pmap_remove_all: empty vreg");
5393 sp = &rp->rg_segmap[vs];
5394 nleft = sp->sg_npte;
5395 if (pm != pmap_kernel()) {
5396 if (nleft <= 0)
5397 panic("pmap_page_protect: empty vseg");
5398 sp->sg_npte = --nleft;
5399 }
5400
5401 /*
5402 * Invalidate PTE in MMU pagetables.
5403 * Flush cache if necessary.
5404 */
5405 if (pm->pm_ctx) {
5406 cache_flush_page(va, pm->pm_ctxnum);
5407 }
5408
5409 tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
5410 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
5411 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
5412
5413 pm->pm_stats.resident_count--;
5414 if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
5415 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
5416 pm->pm_stats.wired_count--;
5417 }
5418
5419 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
5420 panic("pmap_page_protect !PG_V: pg %p va %lx", pg, va);
5421
5422 flags |= MR4M(tpte);
5423
5424 if (pm != pmap_kernel() && nleft == 0)
5425 /*
5426 * Entire user mode segment is gone
5427 */
5428 pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
5429
5430 npv = pv->pv_next;
5431 if (pv != VM_MDPAGE_PVHEAD(pg))
5432 pool_put(&pv_pool, pv);
5433 pv = npv;
5434 }
5435
5436 /* Finally, update pv head */
5437 VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
5438 VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
5439 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
5440
5441 out:
5442 PMAP_UNLOCK();
5443 splx(s);
5444 }
5445
5446 /*
5447 * Lower (make more strict) the protection on the specified
5448 * range of this pmap.
5449 */
5450 void
pmap_protect4m(struct pmap * pm,vaddr_t sva,vaddr_t eva,vm_prot_t prot)5451 pmap_protect4m(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
5452 {
5453 vaddr_t va, nva;
5454 int s, vr, vs;
5455 struct regmap *rp;
5456 struct segmap *sp;
5457 int newprot;
5458
5459 if ((prot & VM_PROT_READ) == 0) {
5460 pmap_remove(pm, sva, eva);
5461 return;
5462 }
5463
5464 DPRINTF(PDB_CHANGEPROT,
5465 "pmap_protect[%d][curpid %d, ctx %d,%d](%lx, %lx, %x)",
5466 cpu_number(), curproc->p_pid, getcontext4m(),
5467 pm->pm_ctx ? pm->pm_ctxnum : -1, sva, eva, prot);
5468
5469 newprot = pte_prot4m(pm, prot);
5470
5471 write_user_windows();
5472 s = splvm();
5473 PMAP_LOCK();
5474
5475 for (va = sva; va < eva;) {
5476 vr = VA_VREG(va);
5477 vs = VA_VSEG(va);
5478 rp = &pm->pm_regmap[vr];
5479 nva = VSTOVA(vr,vs + 1);
5480 if (nva > eva)
5481 nva = eva;
5482 if (rp->rg_nsegmap == 0) {
5483 va = nva;
5484 continue;
5485 }
5486 sp = &rp->rg_segmap[vs];
5487 if (pm != pmap_kernel() && sp->sg_npte == 0) {
5488 va = nva;
5489 continue;
5490 }
5491
5492 /*
5493 * pages loaded: take away write bits from MMU PTEs
5494 */
5495 pmap_stats.ps_npg_prot_all += (nva - va) >> PGSHIFT;
5496 for (; va < nva; va += NBPG) {
5497 int tpte, npte;
5498
5499 tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
5500 if ((tpte & SRMMU_PGTYPE) != PG_SUN4M_OBMEM)
5501 continue;
5502 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
5503 continue;
5504 npte = (tpte & ~SRMMU_PROT_MASK) | newprot;
5505 if (npte == tpte)
5506 continue;
5507
5508 /*
5509 * Flush cache so that any existing cache
5510 * tags are updated.
5511 */
5512
5513 pmap_stats.ps_npg_prot_actual++;
5514 if (pm->pm_ctx) {
5515 cache_flush_page(va, pm->pm_ctxnum);
5516 }
5517 updatepte4m(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
5518 SRMMU_PROT_MASK, newprot, pm->pm_ctxnum,
5519 PMAP_CPUSET(pm));
5520 }
5521 }
5522 PMAP_UNLOCK();
5523 splx(s);
5524 }
5525
5526 /*
5527 * Change the protection and/or wired status of the given (MI) virtual page.
5528 * XXX: should have separate function (or flag) telling whether only wiring
5529 * is changing.
5530 */
5531 void
pmap_changeprot4m(struct pmap * pm,vaddr_t va,vm_prot_t prot,int flags)5532 pmap_changeprot4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
5533 {
5534 int pte, newprot;
5535 struct regmap *rp;
5536 struct segmap *sp;
5537 bool owired;
5538
5539 DPRINTF(PDB_CHANGEPROT, "pmap_changeprot[%d](%p, 0x%lx, 0x%x, 0x%x)",
5540 cpu_number(), pm, va, prot, flags);
5541
5542 newprot = pte_prot4m(pm, prot);
5543
5544 pmap_stats.ps_changeprots++;
5545
5546 rp = &pm->pm_regmap[VA_VREG(va)];
5547 sp = &rp->rg_segmap[VA_VSEG(va)];
5548
5549 pte = sp->sg_pte[VA_SUN4M_VPG(va)];
5550 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
5551
5552 if (owired) {
5553 pm->pm_stats.wired_count--;
5554 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
5555 }
5556 if (flags & PMAP_WIRED) {
5557 pm->pm_stats.wired_count++;
5558 sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
5559 }
5560
5561 if (pm->pm_ctx) {
5562 /*
5563 * Use current context.
5564 * Flush cache if page has been referenced to
5565 * avoid stale protection bits in the cache tags.
5566 */
5567
5568 if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
5569 (SRMMU_PG_C|PG_SUN4M_OBMEM))
5570 cache_flush_page(va, pm->pm_ctxnum);
5571 }
5572
5573 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
5574 (pte & ~SRMMU_PROT_MASK) | newprot,
5575 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
5576
5577 }
5578 #endif /* SUN4M || SUN4D */
5579
5580 /*
5581 * Insert (MI) physical page pa at virtual address va in the given pmap.
5582 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
5583 *
5584 * If pa is not in the `managed' range it will not be `bank mapped'.
5585 * This works during bootstrap only because the first 4MB happens to
5586 * map one-to-one.
5587 *
5588 * There may already be something else there, or we might just be
5589 * changing protections and/or wiring on an existing mapping.
5590 * XXX should have different entry points for changing!
5591 */
5592
5593 #if defined(SUN4) || defined(SUN4C)
5594
5595 int
pmap_enter4_4c(struct pmap * pm,vaddr_t va,paddr_t pa,vm_prot_t prot,u_int flags)5596 pmap_enter4_4c(struct pmap *pm, vaddr_t va, paddr_t pa,
5597 vm_prot_t prot, u_int flags)
5598 {
5599 struct vm_page *pg;
5600 int pteproto, ctx;
5601 int error;
5602
5603 if (VA_INHOLE(va)) {
5604 #ifdef DEBUG
5605 printf("pmap_enter: pm %p, va 0x%lx, pa 0x%lx: in MMU hole\n",
5606 pm, va, pa);
5607 #endif
5608 return 0;
5609 }
5610
5611 DPRINTF(PDB_ENTER, "pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)",
5612 pm, va, pa, prot, flags);
5613
5614 pg = PHYS_TO_VM_PAGE(pa);
5615 pteproto = PG_V | PMAP_T2PTE_4(pa);
5616 pa &= ~PMAP_TNC_4;
5617
5618 /*
5619 * Set up prototype for new PTE. Cannot set PG_NC from PV_NC yet
5620 * since the pvlist no-cache bit might change as a result of the
5621 * new mapping.
5622 */
5623 pteproto |= atop(pa) & PG_PFNUM;
5624 if (prot & VM_PROT_WRITE)
5625 pteproto |= PG_W;
5626 if ((flags & PMAP_WIRED) != 0)
5627 pteproto |= PG_WIRED;
5628 if (flags & VM_PROT_ALL) {
5629 pteproto |= PG_U;
5630 if (flags & VM_PROT_WRITE) {
5631 pteproto |= PG_M;
5632 }
5633 }
5634
5635 write_user_windows();
5636 ctx = getcontext4();
5637 if (pm == pmap_kernel())
5638 error = pmap_enk4_4c(pm, va, prot, flags, pg, pteproto | PG_S);
5639 else
5640 error = pmap_enu4_4c(pm, va, prot, flags, pg, pteproto);
5641 setcontext4(ctx);
5642 return (error);
5643 }
5644
5645 /* enter new (or change existing) kernel mapping */
5646 int
pmap_enk4_4c(struct pmap * pm,vaddr_t va,vm_prot_t prot,int flags,struct vm_page * pg,int pteproto)5647 pmap_enk4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
5648 struct vm_page *pg, int pteproto)
5649 {
5650 int vr, vs, pte, s, inmmu;
5651 int *ptep;
5652 struct regmap *rp;
5653 struct segmap *sp;
5654 int error = 0;
5655
5656 vr = VA_VREG(va);
5657 vs = VA_VSEG(va);
5658 rp = &pm->pm_regmap[vr];
5659 sp = &rp->rg_segmap[vs];
5660 ptep = &sp->sg_pte[VA_VPG(va)];
5661 s = splvm(); /* XXX way too conservative */
5662
5663 #if defined(SUN4_MMU3L)
5664 if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
5665 mmu_pagein_reg(pm, rp, va, vr, ®ion_locked);
5666 #endif
5667
5668 inmmu = sp->sg_pmeg != seginval;
5669 if ((pte = *ptep) & PG_V) {
5670
5671 /* old mapping exists, and is of the same pa type */
5672 if ((pte & (PG_PFNUM|PG_TYPE)) ==
5673 (pteproto & (PG_PFNUM|PG_TYPE))) {
5674 /* just changing protection and/or wiring */
5675 pmap_changeprot4_4c(pm, va, prot, flags);
5676 splx(s);
5677 return (0);
5678 }
5679
5680 if ((pte & PG_TYPE) == PG_OBMEM) {
5681 struct vm_page *opg;
5682
5683 /*
5684 * Switcheroo: changing pa for this va.
5685 * If old pa was managed, remove from pvlist.
5686 * If old page was cached, flush cache.
5687 */
5688 if ((opg = pvhead4_4c(pte)) != NULL)
5689 pv_unlink4_4c(opg, pm, va);
5690 if (inmmu && (pte & PG_NC) == 0) {
5691 setcontext4(0); /* ??? */
5692 cache_flush_page(va, 0);
5693 }
5694 }
5695 *ptep = 0;
5696 if (inmmu)
5697 setpte4(va, 0);
5698 if (pte & PG_WIRED) {
5699 sp->sg_nwired--;
5700 pm->pm_stats.wired_count--;
5701 }
5702 pm->pm_stats.resident_count--;
5703 } else {
5704 /* adding new entry */
5705 if (sp->sg_npte++ == 0) {
5706 #ifdef DIAGNOSTIC
5707 int i; for (i = 0; i < NPTESG; i++) {
5708 if (sp->sg_pte[i] == 0)
5709 continue;
5710 panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
5711 pm, va, i);
5712 }
5713 #endif
5714 rp->rg_nsegmap++;
5715 }
5716 }
5717
5718 /*
5719 * If the new mapping is for a managed PA, enter into pvlist.
5720 */
5721 if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
5722 if (--sp->sg_npte == 0)
5723 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
5724 if ((flags & PMAP_CANFAIL) != 0)
5725 goto out;
5726 panic("pmap_enter: cannot allocate PV entry");
5727 }
5728
5729 /* Update S/W page table */
5730 *ptep = pteproto;
5731 if (pteproto & PG_WIRED) {
5732 sp->sg_nwired++;
5733 pm->pm_stats.wired_count++;
5734 }
5735 pm->pm_stats.resident_count++;
5736
5737 #ifdef DIAGNOSTIC
5738 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5739 panic("pmap_enk: pm %p, va %lx: nleft=%d, nwired=%d",
5740 pm, va, sp->sg_npte, sp->sg_nwired);
5741 #endif
5742 if (sp->sg_pmeg == seginval)
5743 mmu_pagein_seg(pm, sp, va, vr, vs,
5744 (pteproto & PG_WIRED) != 0 ? &segm_locked : &segm_lru);
5745 else if ((pteproto & PG_WIRED) != 0)
5746 mmu_pmeg_lock(sp->sg_pmeg);
5747
5748 /* Update H/W page table */
5749 setpte4(va, pteproto & ~PG_MBZ);
5750 out:
5751 splx(s);
5752 return (error);
5753 }
5754
5755 /* enter new (or change existing) user mapping */
5756 int
pmap_enu4_4c(struct pmap * pm,vaddr_t va,vm_prot_t prot,int flags,struct vm_page * pg,int pteproto)5757 pmap_enu4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
5758 struct vm_page *pg, int pteproto)
5759 {
5760 int vr, vs, *ptep, pte, pmeg, s;
5761 int error = 0;
5762 struct regmap *rp;
5763 struct segmap *sp;
5764
5765 pm->pm_flags &= ~PMAP_USERCACHECLEAN;
5766
5767 vr = VA_VREG(va);
5768 vs = VA_VSEG(va);
5769 rp = &pm->pm_regmap[vr];
5770 s = splvm(); /* XXX conservative */
5771
5772 /*
5773 * If there is no space in which the PTEs can be written
5774 * while they are not in the hardware, this must be a new
5775 * virtual segment. Get PTE space and count the segment.
5776 *
5777 * TO SPEED UP CTX ALLOC, PUT SEGMENT BOUNDS STUFF HERE
5778 * AND IN pmap_rmu()
5779 */
5780
5781 GAP_SHRINK(pm,vr);
5782
5783 #ifdef DEBUG
5784 if (pm->pm_gap_end < pm->pm_gap_start) {
5785 printf("pmap_enu: gap_start 0x%x, gap_end 0x%x",
5786 pm->pm_gap_start, pm->pm_gap_end);
5787 panic("pmap_enu: gap botch");
5788 }
5789 #endif
5790
5791 if (rp->rg_segmap == NULL) {
5792 /* definitely a new mapping */
5793 int i;
5794 int mflag = PR_NOWAIT;
5795
5796 rretry:
5797 sp = (struct segmap *)pool_get(&segmap_pool, mflag);
5798 if (sp == NULL) {
5799 if ((flags & PMAP_CANFAIL) != 0) {
5800 error = ENOMEM;
5801 goto out;
5802 }
5803 mflag = PR_WAITOK;
5804 goto rretry;
5805 }
5806 #ifdef DEBUG
5807 if (rp->rg_segmap != NULL)
5808 panic("pmap_enter: segment filled during sleep");
5809 #endif
5810 qzero((void *)sp, NSEGRG * sizeof (struct segmap));
5811 rp->rg_segmap = sp;
5812 rp->rg_nsegmap = 0;
5813 for (i = NSEGRG; --i >= 0;)
5814 sp++->sg_pmeg = seginval;
5815 }
5816
5817 sp = &rp->rg_segmap[vs];
5818
5819 if ((ptep = sp->sg_pte) == NULL) {
5820 /* definitely a new mapping */
5821 int size = NPTESG * sizeof *ptep;
5822 int mflag = PR_NOWAIT;
5823
5824 sretry:
5825 ptep = (int *)pool_get(&pte_pool, mflag);
5826 if (ptep == NULL) {
5827 if ((flags & PMAP_CANFAIL) != 0) {
5828 error = ENOMEM;
5829 goto out;
5830 }
5831 mflag = PR_WAITOK;
5832 goto sretry;
5833 }
5834 #ifdef DEBUG
5835 if (sp->sg_pte != NULL)
5836 panic("pmap_enter: pte filled during sleep");
5837 if (sp->sg_pmeg != seginval)
5838 panic("pmap_enter: new ptes, but not seginval");
5839 #endif
5840 qzero((void *)ptep, size);
5841 sp->sg_pte = ptep;
5842 sp->sg_npte = 1;
5843 rp->rg_nsegmap++;
5844 } else {
5845 /* might be a change: fetch old pte */
5846 pte = ptep[VA_VPG(va)];
5847 if (pte & PG_V) {
5848 /* old mapping exists, and is of the same pa type */
5849 if ((pte & (PG_PFNUM|PG_TYPE)) ==
5850 (pteproto & (PG_PFNUM|PG_TYPE))) {
5851 /* just changing prot and/or wiring */
5852 pmap_changeprot4_4c(pm, va, prot, flags);
5853 splx(s);
5854 return (0);
5855 }
5856 /*
5857 * Switcheroo: changing pa for this va.
5858 * If old pa was managed, remove from pvlist.
5859 * If old page was cached, flush cache.
5860 */
5861 #if 0
5862 printf("%s[%d]: pmap_enu: changing existing "
5863 "va(0x%lx)=>pa entry\n",
5864 curproc->p_comm, curproc->p_pid, va);
5865 #endif
5866 if ((pte & PG_TYPE) == PG_OBMEM) {
5867 struct vm_page *opg;
5868 if ((opg = pvhead4_4c(pte)) != NULL)
5869 pv_unlink4_4c(opg, pm, va);
5870 if (CACHEINFO.c_vactype != VAC_NONE &&
5871 (pmeg = sp->sg_pmeg) != seginval) {
5872 /* hardware pte */
5873 if (CTX_USABLE(pm,rp)) {
5874 setcontext4(pm->pm_ctxnum);
5875 } else {
5876 setcontext4(0);
5877 /* XXX use per-CPU pteva? */
5878 if (HASSUN4_MMU3L)
5879 setregmap(0, tregion);
5880 setsegmap(0, pmeg);
5881 }
5882 cache_flush_page(va, pm->pm_ctxnum);
5883 }
5884 }
5885 if (pte & PG_WIRED) {
5886 sp->sg_nwired--;
5887 pm->pm_stats.wired_count--;
5888 }
5889 pm->pm_stats.resident_count--;
5890 ptep[VA_VPG(va)] = 0;
5891 if (sp->sg_pmeg != seginval)
5892 setpte4(va, 0);
5893 } else {
5894 /* adding new entry */
5895 sp->sg_npte++;
5896 }
5897 }
5898
5899 if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
5900 if (--sp->sg_npte == 0)
5901 /* Sigh, undo pgt allocations */
5902 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
5903
5904 if ((flags & PMAP_CANFAIL) != 0)
5905 goto out;
5906 panic("pmap_enter: cannot allocate PV entry");
5907 }
5908
5909 /* Update S/W page table */
5910 ptep += VA_VPG(va);
5911 *ptep = pteproto;
5912 if (pteproto & PG_WIRED) {
5913 sp->sg_nwired++;
5914 pm->pm_stats.wired_count++;
5915 }
5916 pm->pm_stats.resident_count++;
5917
5918 #ifdef DIAGNOSTIC
5919 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5920 panic("pmap_enu: pm %p, va %lx: nleft=%d, nwired=%d",
5921 pm, va, sp->sg_npte, sp->sg_nwired);
5922 #endif
5923
5924 if ((pmeg = sp->sg_pmeg) != seginval) {
5925 /* Update H/W page table */
5926 if (CTX_USABLE(pm,rp))
5927 setcontext4(pm->pm_ctxnum);
5928 else {
5929 setcontext4(0);
5930 if (HASSUN4_MMU3L)
5931 setregmap(0, tregion);
5932 setsegmap(0, pmeg);
5933 va = VA_VPG(va) << PGSHIFT;
5934 }
5935 setpte4(va, pteproto & ~PG_MBZ);
5936 }
5937
5938 out:
5939 splx(s);
5940 return (error);
5941 }
5942
5943 void
pmap_kenter_pa4_4c(vaddr_t va,paddr_t pa,vm_prot_t prot,u_int flags)5944 pmap_kenter_pa4_4c(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
5945 {
5946 struct pmap *pm = pmap_kernel();
5947 struct regmap *rp;
5948 struct segmap *sp;
5949 int vr, vs, s;
5950 int *ptep, pteproto;
5951 int lockit = 1;
5952
5953 pteproto = PG_S | PG_V | PMAP_T2PTE_4(pa);
5954 pa &= ~PMAP_TNC_4;
5955 pteproto |= atop(pa) & PG_PFNUM;
5956 if (prot & VM_PROT_WRITE)
5957 pteproto |= PG_W;
5958
5959 vr = VA_VREG(va);
5960 vs = VA_VSEG(va);
5961 rp = &pm->pm_regmap[vr];
5962 sp = &rp->rg_segmap[vs];
5963 ptep = &sp->sg_pte[VA_VPG(va)];
5964
5965 if (lockit) {
5966 pteproto |= PG_WIRED;
5967 sp->sg_nwired++;
5968 }
5969
5970 KASSERT((*ptep & PG_V) == 0);
5971
5972 s = splvm();
5973 #if defined(SUN4_MMU3L)
5974 if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
5975 mmu_pagein_reg(pm, rp, va, vr, ®ion_locked);
5976 #endif
5977
5978 if (sp->sg_npte++ == 0) {
5979 #ifdef DIAGNOSTIC
5980 int i; for (i = 0; i < NPTESG; i++) {
5981 if (sp->sg_pte[i] == 0)
5982 continue;
5983 panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
5984 pm, va, i);
5985 }
5986 #endif
5987 rp->rg_nsegmap++;
5988 }
5989
5990 /* Update S/W page table */
5991 *ptep = pteproto;
5992
5993 #ifdef DIAGNOSTIC
5994 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
5995 panic("pmap_kenter: pm %p, va %lx: nleft=%d, nwired=%d",
5996 pm, va, sp->sg_npte, sp->sg_nwired);
5997 #endif
5998
5999 if (sp->sg_pmeg == seginval) {
6000 mmu_pagein_seg(pm, sp, va, vr, vs,
6001 lockit ? &segm_locked : &segm_lru);
6002 } else if (lockit)
6003 mmu_pmeg_lock(sp->sg_pmeg);
6004
6005 /* Update H/W page table */
6006 setpte4(va, pteproto & ~PG_MBZ);
6007 splx(s);
6008 }
6009
6010 #if notyet /* XXXMRG delete */
6011 void pmap_lockmmu(vaddr_t sva, size_t sz);
6012
6013 void
pmap_lockmmu(vaddr_t sva,size_t sz)6014 pmap_lockmmu(vaddr_t sva, size_t sz)
6015 {
6016 struct pmap *pm = pmap_kernel();
6017 vaddr_t va, eva;
6018 struct regmap *rp;
6019 struct segmap *sp;
6020 int vr, vs;
6021
6022 if (CPU_HAS_SRMMU)
6023 return;
6024
6025 eva = sva + sz;
6026 va = VA_ROUNDDOWNTOSEG(sva);
6027
6028 for (; va < eva; va += NBPSG) {
6029 vr = VA_VREG(va);
6030 vs = VA_VSEG(va);
6031 rp = &pm->pm_regmap[vr];
6032 sp = &rp->rg_segmap[vs];
6033
6034 KASSERT(sp->sg_npte != 0);
6035
6036 if (sp->sg_pmeg == seginval)
6037 mmu_pagein_seg(pm, sp, va, vr, vs, &segm_locked);
6038 else
6039 mmu_pmeg_lock(sp->sg_pmeg);
6040 }
6041 }
6042 #endif
6043
6044 void
pmap_kremove4_4c(vaddr_t va,vsize_t len)6045 pmap_kremove4_4c(vaddr_t va, vsize_t len)
6046 {
6047 struct pmap *pm = pmap_kernel();
6048 struct regmap *rp;
6049 struct segmap *sp;
6050 vaddr_t nva, endva;
6051 int pte, mmupte, *ptep, perpage, npg, inmmu;
6052 int nleft, pmeg;
6053 int vr, vs, s, ctx;
6054
6055 endva = va + len;
6056 DPRINTF(PDB_REMOVE, "pmap_kremove(0x%lx, 0x%lx)", va, endva);
6057
6058 write_user_windows();
6059
6060 s = splvm();
6061 ctx = getcontext();
6062 PMAP_LOCK();
6063 setcontext4(0);
6064 for (; va < endva; va = nva) {
6065 /* do one virtual segment at a time */
6066 vr = VA_VREG(va);
6067 vs = VA_VSEG(va);
6068 nva = VSTOVA(vr, vs + 1);
6069 if (nva == 0 || nva > endva)
6070 nva = endva;
6071
6072 rp = &pm->pm_regmap[vr];
6073 sp = &rp->rg_segmap[vs];
6074
6075 if (rp->rg_nsegmap == 0)
6076 continue;
6077 nleft = sp->sg_npte;
6078 if (nleft == 0)
6079 continue;
6080 pmeg = sp->sg_pmeg;
6081 inmmu = (pmeg != seginval);
6082 ptep = &sp->sg_pte[VA_VPG(va)];
6083
6084 /* decide how to flush cache */
6085 npg = (nva - va) >> PGSHIFT;
6086 if (!inmmu) {
6087 perpage = 0;
6088 } else if (npg > PMAP_SFL_THRESHOLD) {
6089 /* flush the whole segment */
6090 perpage = 0;
6091 cache_flush_segment(vr, vs, 0);
6092 } else {
6093 /*
6094 * flush each page individually;
6095 * some never need flushing
6096 */
6097 perpage = (CACHEINFO.c_vactype != VAC_NONE);
6098 }
6099
6100 for (; va < nva; va += NBPG, ptep++) {
6101 pte = *ptep;
6102 mmupte = inmmu ? getpte4(va) : 0;
6103 if ((pte & PG_V) == 0) {
6104 #ifdef DIAGNOSTIC
6105 if (inmmu && (mmupte & PG_V) != 0)
6106 printf("rmk: inconsistent ptes va=%lx\n", va);
6107 #endif
6108 continue;
6109 }
6110 if ((pte & PG_TYPE) == PG_OBMEM) {
6111 /* if cacheable, flush page as needed */
6112 if (perpage && (mmupte & PG_NC) == 0)
6113 cache_flush_page(va, 0);
6114 }
6115 nleft--;
6116 #ifdef DIAGNOSTIC
6117 if (nleft < 0)
6118 panic("pmap_kremove: too many PTEs in segment; "
6119 "va 0x%lx; endva 0x%lx", va, endva);
6120 #endif
6121 if (pte & PG_WIRED)
6122 sp->sg_nwired--;
6123
6124 if (inmmu)
6125 setpte4(va, 0);
6126 *ptep = 0;
6127 }
6128
6129 #ifdef DIAGNOSTIC
6130 if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
6131 panic("pmap_kremove: pm %p, va %lx: nleft=%d, nwired=%d",
6132 pm, va, nleft, sp->sg_nwired);
6133 #endif
6134
6135 if ((sp->sg_npte = nleft) == 0)
6136 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
6137 else if (sp->sg_nwired == 0) {
6138 if (sp->sg_pmeg != seginval)
6139 mmu_pmeg_unlock(sp->sg_pmeg);
6140 }
6141 }
6142 PMAP_UNLOCK();
6143 setcontext4(ctx);
6144 splx(s);
6145 }
6146
6147 /*
6148 * Change protection on a range of kernel addresses.
6149 */
6150 void
pmap_kprotect4_4c(vaddr_t va,vsize_t size,vm_prot_t prot)6151 pmap_kprotect4_4c(vaddr_t va, vsize_t size, vm_prot_t prot)
6152 {
6153 int pte, newprot, ctx;
6154
6155 size = roundup(size,NBPG);
6156 newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;
6157
6158 ctx = getcontext4();
6159 setcontext4(0);
6160 while (size > 0) {
6161 pte = getpte4(va);
6162
6163 /*
6164 * Flush cache if page has been referenced to
6165 * avoid stale protection bits in the cache tags.
6166 */
6167 if ((pte & (PG_NC|PG_TYPE)) == PG_OBMEM)
6168 cache_flush_page(va, 0);
6169
6170 pte = (pte & ~PG_PROT) | newprot;
6171 setpte4(va, pte);
6172
6173 va += NBPG;
6174 size -= NBPG;
6175 }
6176 setcontext4(ctx);
6177 }
6178 #endif /* SUN4 || SUN4C */
6179
6180 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of enter routines */
6181 /*
6182 * Insert (MI) physical page pa at virtual address va in the given pmap.
6183 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
6184 *
6185 * If pa is not in the `managed' range it will not be `bank mapped'.
6186 * This works during bootstrap only because the first 4MB happens to
6187 * map one-to-one.
6188 *
6189 * There may already be something else there, or we might just be
6190 * changing protections and/or wiring on an existing mapping.
6191 * XXX should have different entry points for changing!
6192 */
6193
6194 int
pmap_enter4m(struct pmap * pm,vaddr_t va,paddr_t pa,vm_prot_t prot,u_int flags)6195 pmap_enter4m(struct pmap *pm, vaddr_t va, paddr_t pa,
6196 vm_prot_t prot, u_int flags)
6197 {
6198 struct vm_page *pg;
6199 int pteproto;
6200 int error;
6201
6202 DPRINTF(PDB_ENTER, "pmap_enter[curcpu %d, curpid %d, ctx %d,%d]"
6203 "(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)",
6204 cpu_number(), curproc == NULL ? -1 : curproc->p_pid,
6205 getcontext4m(), pm->pm_ctx == NULL ? -1 : pm->pm_ctxnum,
6206 pm, va, pa, prot, flags);
6207
6208 pg = PHYS_TO_VM_PAGE(pa);
6209
6210 /* Initialise pteproto with cache bit */
6211 pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;
6212
6213 #ifdef DEBUG
6214 if (pa & PMAP_TYPE_SRMMU) { /* this page goes in an iospace */
6215 if (cpuinfo.cpu_type == CPUTYP_MS1)
6216 panic("pmap_enter4m: attempt to use 36-bit iospace on"
6217 " MicroSPARC");
6218 }
6219 #endif
6220 pteproto |= SRMMU_TEPTE;
6221 pteproto |= PMAP_T2PTE_SRMMU(pa);
6222 pa &= ~PMAP_TNC_SRMMU;
6223
6224 /*
6225 * Set up prototype for new PTE. Cannot set PG_NC from PV_NC yet
6226 * since the pvlist no-cache bit might change as a result of the
6227 * new mapping.
6228 */
6229 pteproto |= (atop(pa) << SRMMU_PPNSHIFT);
6230
6231 /* Make sure we get a pte with appropriate perms! */
6232 pteproto |= pte_prot4m(pm, prot);
6233 if (flags & VM_PROT_ALL) {
6234 pteproto |= SRMMU_PG_R;
6235 if (flags & VM_PROT_WRITE) {
6236 pteproto |= SRMMU_PG_M;
6237 }
6238 }
6239
6240 if (pm == pmap_kernel())
6241 error = pmap_enk4m(pm, va, prot, flags, pg, pteproto | PPROT_S);
6242 else
6243 error = pmap_enu4m(pm, va, prot, flags, pg, pteproto);
6244
6245 return (error);
6246 }
6247
6248 /* enter new (or change existing) kernel mapping */
6249 int
pmap_enk4m(struct pmap * pm,vaddr_t va,vm_prot_t prot,int flags,struct vm_page * pg,int pteproto)6250 pmap_enk4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
6251 struct vm_page *pg, int pteproto)
6252 {
6253 int vr, vs, tpte, s;
6254 struct regmap *rp;
6255 struct segmap *sp;
6256 int error = 0;
6257
6258 #ifdef DEBUG
6259 if (va < KERNBASE)
6260 panic("pmap_enk4m: can't enter va 0x%lx below KERNBASE", va);
6261 #endif
6262 vr = VA_VREG(va);
6263 vs = VA_VSEG(va);
6264 rp = &pm->pm_regmap[vr];
6265 sp = &rp->rg_segmap[vs];
6266
6267 kpreempt_disable();
6268 s = splvm();
6269 PMAP_LOCK();
6270
6271 if (rp->rg_seg_ptps == NULL) /* enter new region */
6272 panic("pmap_enk4m: missing kernel region table for va 0x%lx",va);
6273
6274 tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
6275 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
6276
6277 /* old mapping exists, and is of the same pa type */
6278
6279 if ((tpte & SRMMU_PPNMASK) == (pteproto & SRMMU_PPNMASK)) {
6280 /* just changing protection and/or wiring */
6281 pmap_changeprot4m(pm, va, prot, flags);
6282 error = 0;
6283 goto out;
6284 }
6285
6286 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
6287 struct vm_page *opg;
6288 #ifdef DEBUG
6289 printf("pmap_enk4m: changing existing va=>pa entry: va 0x%lx, pteproto 0x%x, "
6290 "oldpte 0x%x\n", va, pteproto, tpte);
6291 #endif
6292 /*
6293 * Switcheroo: changing pa for this va.
6294 * If old pa was managed, remove from pvlist.
6295 * If old page was cached, flush cache.
6296 */
6297 if ((opg = pvhead4m(tpte)) != NULL)
6298 pv_unlink4m(opg, pm, va);
6299 if (tpte & SRMMU_PG_C) {
6300 cache_flush_page(va, 0);
6301 }
6302 }
6303
6304 /*
6305 * Invalidate the mapping now, so we can avoid the
6306 * de-map and update protocol when setting the new
6307 * PTE below.
6308 */
6309 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6310 SRMMU_TEINVALID, pm->pm_ctx != NULL,
6311 pm->pm_ctxnum, PMAP_CPUSET(pm));
6312 pm->pm_stats.resident_count--;
6313 }
6314
6315 /*
6316 * If the new mapping is for a managed PA, enter into pvlist.
6317 */
6318 if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
6319 if ((flags & PMAP_CANFAIL) != 0)
6320 goto out;
6321 panic("pmap_enter: cannot allocate PV entry");
6322 }
6323
6324 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
6325 pm->pm_stats.resident_count++;
6326 out:
6327 PMAP_UNLOCK();
6328 splx(s);
6329 kpreempt_enable();
6330 return (error);
6331 }
6332
6333 /* enter new (or change existing) user mapping */
6334 int
pmap_enu4m(struct pmap * pm,vaddr_t va,vm_prot_t prot,int flags,struct vm_page * pg,int pteproto)6335 pmap_enu4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
6336 struct vm_page *pg, int pteproto)
6337 {
6338 int vr, vs, *pte, tpte, s;
6339 int error = 0;
6340 struct regmap *rp;
6341 struct segmap *sp;
6342 bool owired;
6343
6344 #ifdef DEBUG
6345 if (KERNBASE < va)
6346 panic("pmap_enu4m: can't enter va 0x%lx above KERNBASE", va);
6347 #endif
6348
6349 pm->pm_flags &= ~PMAP_USERCACHECLEAN;
6350
6351 vr = VA_VREG(va);
6352 vs = VA_VSEG(va);
6353 rp = &pm->pm_regmap[vr];
6354 s = splvm();
6355 PMAP_LOCK();
6356
6357 if (rp->rg_segmap == NULL) {
6358 /* definitely a new mapping */
6359 int mflag = PR_NOWAIT;
6360
6361 rretry:
6362 sp = (struct segmap *)pool_get(&segmap_pool, mflag);
6363 if (sp == NULL) {
6364 if ((flags & PMAP_CANFAIL) != 0) {
6365 error = ENOMEM;
6366 goto out;
6367 }
6368 mflag = PR_WAITOK;
6369 goto rretry;
6370 }
6371 #ifdef DEBUG
6372 if (rp->rg_segmap != NULL)
6373 panic("pmap_enu4m: segment filled during sleep");
6374 #endif
6375 qzero((void *)sp, NSEGRG * sizeof (struct segmap));
6376 rp->rg_segmap = sp;
6377 rp->rg_nsegmap = 0;
6378 rp->rg_seg_ptps = NULL;
6379 }
6380 if (rp->rg_seg_ptps == NULL) {
6381 /* Need a segment table */
6382 int i, *ptd;
6383 int mflag = PR_NOWAIT;
6384
6385 sretry:
6386 ptd = pool_get(&L23_pool, mflag);
6387 if (ptd == NULL) {
6388 if ((flags & PMAP_CANFAIL) != 0) {
6389 error = ENOMEM;
6390 goto out;
6391 }
6392 mflag = PR_WAITOK;
6393 goto sretry;
6394 }
6395
6396 rp->rg_seg_ptps = ptd;
6397 for (i = 0; i < SRMMU_L2SIZE; i++)
6398 setpgt4m(&ptd[i], SRMMU_TEINVALID);
6399
6400 /* Replicate segment allocation in each CPU's region table */
6401 #if defined(MULTIPROCESSOR)
6402 for (i = 0; i < sparc_ncpus; i++)
6403 #else
6404 i = 0;
6405 #endif
6406 {
6407 #if defined(MULTIPROCESSOR)
6408 if ((cpus[i]->flags & CPUFLG_HATCHED) == 0)
6409 continue;
6410 #endif
6411 setpgt4m(&pm->pm_reg_ptps[i][vr],
6412 (VA2PA((void *)ptd) >> SRMMU_PPNPASHIFT) |
6413 SRMMU_TEPTD);
6414 }
6415 }
6416
6417 sp = &rp->rg_segmap[vs];
6418
6419 owired = false;
6420 if ((pte = sp->sg_pte) == NULL) {
6421 /* definitely a new mapping */
6422 int i;
6423 int mflag = PR_NOWAIT;
6424
6425 pte = pool_get(&L23_pool, mflag);
6426 if (pte == NULL) {
6427 if ((flags & PMAP_CANFAIL) != 0) {
6428 error = ENOMEM;
6429 goto out;
6430 }
6431 panic("pmap_enter: cannot allocate PTE table");
6432 }
6433
6434 sp->sg_pte = pte;
6435 sp->sg_npte = 1;
6436 rp->rg_nsegmap++;
6437 for (i = 0; i < SRMMU_L3SIZE; i++)
6438 setpgt4m(&pte[i], SRMMU_TEINVALID);
6439 setpgt4m(&rp->rg_seg_ptps[vs],
6440 (VA2PA((void *)pte) >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
6441 } else {
6442 #ifdef DIAGNOSTIC
6443 if (sp->sg_npte <= 0)
6444 panic("pm %p: npte %d", pm, sp->sg_npte);
6445 #endif
6446 /*
6447 * Might be a change: fetch old pte
6448 */
6449 tpte = pte[VA_SUN4M_VPG(va)];
6450
6451 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
6452
6453 /* old mapping exists, and is of the same pa type */
6454 if ((tpte & SRMMU_PPNMASK) ==
6455 (pteproto & SRMMU_PPNMASK)) {
6456 /* just changing prot and/or wiring */
6457 /* caller should call this directly: */
6458 pmap_changeprot4m(pm, va, prot, flags);
6459 error = 0;
6460 goto out;
6461 }
6462 /*
6463 * Switcheroo: changing pa for this va.
6464 * If old pa was managed, remove from pvlist.
6465 * If old page was cached, flush cache.
6466 */
6467 DPRINTF(PDB_SWITCHMAP,
6468 "%s[%d]: pmap_enu: changing existing "
6469 "va 0x%x: pte 0x%x=>0x%x",
6470 curproc->p_comm, curproc->p_pid,
6471 (int)va, tpte, pteproto);
6472
6473 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
6474 struct vm_page *opg;
6475 if ((opg = pvhead4m(tpte)) != NULL) {
6476 VM_MDPAGE_PVHEAD(opg)->pv_flags |=
6477 MR4M(tpte);
6478 pv_unlink4m(opg, pm, va);
6479 }
6480 if (pm->pm_ctx && (tpte & SRMMU_PG_C))
6481 cache_flush_page(va, pm->pm_ctxnum);
6482 }
6483 /*
6484 * We end up in this `change map' branch relatively
6485 * infrequently.
6486 * Invalidate the mapping now, so we can avoid the
6487 * de-map and update protocol when setting the new
6488 * PTE below.
6489 */
6490 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6491 SRMMU_TEINVALID, pm->pm_ctx != NULL,
6492 pm->pm_ctxnum, PMAP_CPUSET(pm));
6493 pm->pm_stats.resident_count--;
6494 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
6495 } else {
6496 /* adding new entry */
6497 sp->sg_npte++;
6498 }
6499 }
6500
6501 if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
6502 if (--sp->sg_npte == 0)
6503 /* Sigh, undo pgt allocations */
6504 pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
6505
6506 if ((flags & PMAP_CANFAIL) != 0)
6507 goto out;
6508 panic("pmap_enter: cannot allocate PV entry");
6509 }
6510
6511 /*
6512 * Update PTEs, flush TLB as necessary.
6513 */
6514 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
6515 pm->pm_stats.resident_count++;
6516 if (owired) {
6517 pm->pm_stats.wired_count--;
6518 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
6519 }
6520 if (flags & PMAP_WIRED) {
6521 pm->pm_stats.wired_count++;
6522 sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
6523 }
6524
6525 out:
6526 PMAP_UNLOCK();
6527 splx(s);
6528 return (error);
6529 }
6530
6531 void
pmap_kenter_pa4m(vaddr_t va,paddr_t pa,vm_prot_t prot,u_int flags)6532 pmap_kenter_pa4m(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
6533 {
6534 struct pmap *pm = pmap_kernel();
6535 struct regmap *rp;
6536 struct segmap *sp;
6537 int pteproto, vr, vs;
6538
6539 /* Initialise pteproto with cache bit */
6540 pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;
6541 pteproto |= SRMMU_TEPTE | PPROT_S;
6542 pteproto |= PMAP_T2PTE_SRMMU(pa);
6543 pteproto |= (atop(pa & ~PMAP_TNC_SRMMU) << SRMMU_PPNSHIFT);
6544 pteproto |= pte_kprot4m(prot);
6545
6546 vr = VA_VREG(va);
6547 vs = VA_VSEG(va);
6548 rp = &pm->pm_regmap[vr];
6549 sp = &rp->rg_segmap[vs];
6550
6551 KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) != SRMMU_TEPTE);
6552
6553 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
6554 }
6555
6556 void
pmap_kremove4m(vaddr_t va,vsize_t len)6557 pmap_kremove4m(vaddr_t va, vsize_t len)
6558 {
6559 struct pmap *pm = pmap_kernel();
6560 struct regmap *rp;
6561 struct segmap *sp;
6562 vaddr_t endva, nva;
6563 int vr, vs;
6564 int tpte, perpage, npg, s;
6565
6566 /*
6567 * The kernel pmap doesn't need to be locked, but the demap lock
6568 * in updatepte() requires interrupt protection.
6569 */
6570 kpreempt_disable();
6571 s = splvm();
6572
6573 endva = va + len;
6574 for (; va < endva; va = nva) {
6575 /* do one virtual segment at a time */
6576 vr = VA_VREG(va);
6577 vs = VA_VSEG(va);
6578 nva = VSTOVA(vr, vs + 1);
6579 if (nva == 0 || nva > endva) {
6580 nva = endva;
6581 }
6582
6583 rp = &pm->pm_regmap[vr];
6584 sp = &rp->rg_segmap[vs];
6585
6586 /* decide how to flush the cache */
6587 npg = (nva - va) >> PGSHIFT;
6588 if (npg > PMAP_SFL_THRESHOLD) {
6589 /* flush the whole segment */
6590 perpage = 0;
6591 if (CACHEINFO.c_vactype != VAC_NONE) {
6592 cache_flush_segment(vr, vs, 0);
6593 }
6594 } else {
6595 /*
6596 * flush each page individually;
6597 * some never need flushing
6598 */
6599 perpage = (CACHEINFO.c_vactype != VAC_NONE);
6600 }
6601 for (; va < nva; va += NBPG) {
6602 tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
6603 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
6604 continue;
6605
6606 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
6607 /* if cacheable, flush page as needed */
6608 if (perpage && (tpte & SRMMU_PG_C))
6609 cache_flush_page(va, 0);
6610 }
6611 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6612 SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
6613 }
6614 }
6615 splx(s);
6616 kpreempt_enable();
6617 }
6618
6619 /*
6620 * Change protection on a range of kernel addresses.
6621 */
6622 void
pmap_kprotect4m(vaddr_t va,vsize_t size,vm_prot_t prot)6623 pmap_kprotect4m(vaddr_t va, vsize_t size, vm_prot_t prot)
6624 {
6625 struct pmap *pm = pmap_kernel();
6626 int pte, newprot, s;
6627 struct regmap *rp;
6628 struct segmap *sp;
6629
6630 size = roundup(size,NBPG);
6631 newprot = pte_kprot4m(prot);
6632
6633 /*
6634 * The kernel pmap doesn't need to be locked, but the demap lock
6635 * in updatepte() requires interrupt protection.
6636 */
6637 kpreempt_disable();
6638 s = splvm();
6639
6640 while (size > 0) {
6641 rp = &pm->pm_regmap[VA_VREG(va)];
6642 sp = &rp->rg_segmap[VA_VSEG(va)];
6643 pte = sp->sg_pte[VA_SUN4M_VPG(va)];
6644
6645 /*
6646 * Flush cache if page has been referenced to
6647 * avoid stale protection bits in the cache tags.
6648 */
6649 if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
6650 (SRMMU_PG_C|PG_SUN4M_OBMEM))
6651 cache_flush_page(va, 0);
6652
6653 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
6654 (pte & ~SRMMU_PROT_MASK) | newprot,
6655 1, pm->pm_ctxnum, PMAP_CPUSET(pm));
6656
6657 va += NBPG;
6658 size -= NBPG;
6659 }
6660 splx(s);
6661 kpreempt_enable();
6662 }
6663 #endif /* SUN4M || SUN4D */
6664
6665 /*
6666 * Clear the wiring attribute for a map/virtual-address pair.
6667 */
6668 /* ARGSUSED */
6669 void
pmap_unwire(struct pmap * pm,vaddr_t va)6670 pmap_unwire(struct pmap *pm, vaddr_t va)
6671 {
6672 int vr, vs, *ptep;
6673 struct regmap *rp;
6674 struct segmap *sp;
6675 bool owired;
6676
6677 kpreempt_disable();
6678 vr = VA_VREG(va);
6679 vs = VA_VSEG(va);
6680 rp = &pm->pm_regmap[vr];
6681 sp = &rp->rg_segmap[vs];
6682
6683 owired = false;
6684 if (CPU_HAS_SUNMMU) {
6685 ptep = &sp->sg_pte[VA_VPG(va)];
6686 owired = *ptep & PG_WIRED;
6687 *ptep &= ~PG_WIRED;
6688 }
6689 if (CPU_HAS_SRMMU) {
6690 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
6691 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
6692 }
6693 if (!owired) {
6694 pmap_stats.ps_useless_changewire++;
6695 kpreempt_enable();
6696 return;
6697 }
6698
6699 pm->pm_stats.wired_count--;
6700 #if defined(SUN4) || defined(SUN4C)
6701 if (CPU_HAS_SUNMMU && --sp->sg_nwired <= 0) {
6702 #ifdef DIAGNOSTIC
6703 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
6704 panic("pmap_unwire: pm %p, va %lx: nleft=%d, nwired=%d",
6705 pm, va, sp->sg_npte, sp->sg_nwired);
6706 #endif
6707 if (sp->sg_pmeg != seginval)
6708 mmu_pmeg_unlock(sp->sg_pmeg);
6709 }
6710 #endif /* SUN4 || SUN4C */
6711 kpreempt_enable();
6712 }
6713
6714 /*
6715 * Extract the physical page address associated
6716 * with the given map/virtual_address pair.
6717 * GRR, the vm code knows; we should not have to do this!
6718 */
6719
6720 #if defined(SUN4) || defined(SUN4C)
6721 bool
pmap_extract4_4c(struct pmap * pm,vaddr_t va,paddr_t * pap)6722 pmap_extract4_4c(struct pmap *pm, vaddr_t va, paddr_t *pap)
6723 {
6724 int vr, vs;
6725 struct regmap *rp;
6726 struct segmap *sp;
6727 int pte, *ptep;
6728
6729 vr = VA_VREG(va);
6730 vs = VA_VSEG(va);
6731 rp = &pm->pm_regmap[vr];
6732 if (rp->rg_segmap == NULL) {
6733 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment (%d)", vr);
6734 return (false);
6735 }
6736 sp = &rp->rg_segmap[vs];
6737 ptep = sp->sg_pte;
6738 if (ptep == NULL) {
6739 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment");
6740 return (false);
6741 }
6742 pte = ptep[VA_VPG(va)];
6743
6744 if ((pte & PG_V) == 0) {
6745 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte");
6746 return (false);
6747 }
6748 pte &= PG_PFNUM;
6749 if (pap != NULL)
6750 *pap = (pte << PGSHIFT) | (va & PGOFSET);
6751 return (true);
6752 }
6753 #endif /* SUN4 || SUN4C */
6754
6755 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_extract */
6756 /*
6757 * Extract the physical page address associated
6758 * with the given map/virtual_address pair.
6759 * GRR, the vm code knows; we should not have to do this!
6760 */
6761 bool
pmap_extract4m(struct pmap * pm,vaddr_t va,paddr_t * pap)6762 pmap_extract4m(struct pmap *pm, vaddr_t va, paddr_t *pap)
6763 {
6764 struct regmap *rp;
6765 struct segmap *sp;
6766 int pte;
6767 int vr, vs, s, v = false;
6768 bool can_lock = lock_available;
6769
6770 vr = VA_VREG(va);
6771 vs = VA_VSEG(va);
6772
6773 /*
6774 * The kernel pmap doesn't need to be locked, but the demap lock
6775 * requires interrupt protection.
6776 */
6777 s = splvm();
6778 if (pm != pmap_kernel()) {
6779 PMAP_LOCK();
6780 }
6781
6782 rp = &pm->pm_regmap[vr];
6783 if (rp->rg_segmap == NULL) {
6784 DPRINTF(PDB_FOLLOW, "pmap_extract: no segmap");
6785 goto out;
6786 }
6787
6788 sp = &rp->rg_segmap[vs];
6789 if (sp->sg_pte == NULL) {
6790 DPRINTF(PDB_FOLLOW, "pmap_extract: no ptes");
6791 goto out;
6792 }
6793
6794 pte = sp->sg_pte[VA_SUN4M_VPG(va)];
6795 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) {
6796 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte of type %d",
6797 pte & SRMMU_TETYPE);
6798 /*
6799 * We can read a spurious invalid pte if the system is in
6800 * the middle of the PTE update protocol. So, acquire the
6801 * demap lock and retry.
6802 */
6803 if (__predict_true(can_lock))
6804 mutex_spin_enter(&demap_lock);
6805 pte = sp->sg_pte[VA_SUN4M_VPG(va)];
6806 if (__predict_true(can_lock))
6807 mutex_spin_exit(&demap_lock);
6808 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
6809 goto out;
6810 }
6811 #ifdef DIAGNOSTIC
6812 if (pm != pmap_kernel() && sp->sg_npte <= 0)
6813 panic("pmap_extract: pm %p: npte = %d\n", pm, sp->sg_npte);
6814 #endif
6815
6816 if (pap != NULL)
6817 *pap = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT) |
6818 VA_OFF(va);
6819
6820 v = true;
6821 out:
6822 if (pm != pmap_kernel()) {
6823 PMAP_UNLOCK();
6824 }
6825 splx(s);
6826 return (v);
6827 }
6828 #endif /* sun4m */
6829
6830 int pmap_copy_disabled=0;
6831
6832 /*
6833 * Copy the range specified by src_addr/len
6834 * from the source map to the range dst_addr/len
6835 * in the destination map.
6836 *
6837 * This routine is only advisory and need not do anything.
6838 */
6839 /* ARGSUSED */
6840 void
pmap_copy(struct pmap * dst_pmap,struct pmap * src_pmap,vaddr_t dst_addr,vsize_t len,vaddr_t src_addr)6841 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap,
6842 vaddr_t dst_addr, vsize_t len, vaddr_t src_addr)
6843 {
6844 #if notyet
6845 struct regmap *rp;
6846 struct segmap *sp;
6847
6848 if (pmap_copy_disabled)
6849 return;
6850 #ifdef DIAGNOSTIC
6851 if (VA_OFF(src_addr) != 0)
6852 printf("pmap_copy: addr not page aligned: 0x%lx\n", src_addr);
6853 if ((len & (NBPG-1)) != 0)
6854 printf("pmap_copy: length not page aligned: 0x%lx\n", len);
6855 #endif
6856
6857 if (src_pmap == NULL)
6858 return;
6859
6860 if (CPU_HAS_SRMMU) {
6861 int i, npg, pte;
6862 paddr_t pa;
6863
6864 npg = len >> PGSHIFT;
6865 for (i = 0; i < npg; i++) {
6866 if ((rp = src_pmap->pm_regmap) == NULL)
6867 continue;
6868 rp += VA_VREG(src_addr);
6869
6870 if ((sp = rp->rg_segmap) == NULL)
6871 continue;
6872 sp += VA_VSEG(src_addr);
6873 if (sp->sg_npte == 0)
6874 continue;
6875
6876 pte = sp->sg_pte[VA_SUN4M_VPG(src_addr)];
6877 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
6878 continue;
6879
6880 pa = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT);
6881 pmap_enter(dst_pmap, dst_addr,
6882 pa,
6883 /* XXX - need to copy VM_PROT_EXEC too */
6884 (pte & PPROT_WRITE)
6885 ? (VM_PROT_WRITE | VM_PROT_READ)
6886 : VM_PROT_READ,
6887 0);
6888 src_addr += NBPG;
6889 dst_addr += NBPG;
6890 }
6891 pmap_update(dst_pmap);
6892 }
6893 #endif
6894 }
6895
6896 #if defined(SUN4) || defined(SUN4C)
6897 /*
6898 * Clear the modify bit for the given physical page.
6899 */
6900 bool
pmap_clear_modify4_4c(struct vm_page * pg)6901 pmap_clear_modify4_4c(struct vm_page *pg)
6902 {
6903 bool rv;
6904
6905 (void) pv_syncflags4_4c(pg);
6906 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD;
6907 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD;
6908 return (rv);
6909 }
6910
6911 /*
6912 * Tell whether the given physical page has been modified.
6913 */
6914 bool
pmap_is_modified4_4c(struct vm_page * pg)6915 pmap_is_modified4_4c(struct vm_page *pg)
6916 {
6917
6918 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD ||
6919 pv_syncflags4_4c(pg) & PV_MOD);
6920 }
6921
6922 /*
6923 * Clear the reference bit for the given physical page.
6924 */
6925 bool
pmap_clear_reference4_4c(struct vm_page * pg)6926 pmap_clear_reference4_4c(struct vm_page *pg)
6927 {
6928 bool rv;
6929
6930 (void) pv_syncflags4_4c(pg);
6931 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF;
6932 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF;
6933 return (rv);
6934 }
6935
6936 /*
6937 * Tell whether the given physical page has been referenced.
6938 */
6939 bool
pmap_is_referenced4_4c(struct vm_page * pg)6940 pmap_is_referenced4_4c(struct vm_page *pg)
6941 {
6942
6943 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF ||
6944 pv_syncflags4_4c(pg) & PV_REF);
6945 }
6946 #endif /* SUN4 || SUN4C */
6947
6948 #if defined(SUN4M) || defined(SUN4D)
6949
6950 /*
6951 * SRMMU versions of bit test/set routines
6952 *
6953 * Note that the 4m-specific routines should eventually service these
6954 * requests from their page tables, and the whole pvlist bit mess should
6955 * be dropped for the 4m (unless this causes a performance hit from
6956 * tracing down pagetables/regmap/segmaps).
6957 */
6958
6959 /*
6960 * Clear the modify bit for the given physical page.
6961 */
6962 bool
pmap_clear_modify4m(struct vm_page * pg)6963 pmap_clear_modify4m(struct vm_page *pg)
6964 {
6965 bool rv;
6966
6967 (void) pv_syncflags4m(pg);
6968 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M;
6969 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD4M;
6970 return (rv);
6971 }
6972
6973 /*
6974 * Tell whether the given physical page has been modified.
6975 */
6976 bool
pmap_is_modified4m(struct vm_page * pg)6977 pmap_is_modified4m(struct vm_page *pg)
6978 {
6979
6980 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M ||
6981 pv_syncflags4m(pg) & PV_MOD4M);
6982 }
6983
6984 /*
6985 * Clear the reference bit for the given physical page.
6986 */
6987 bool
pmap_clear_reference4m(struct vm_page * pg)6988 pmap_clear_reference4m(struct vm_page *pg)
6989 {
6990 bool rv;
6991
6992 (void) pv_syncflags4m(pg);
6993 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M;
6994 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF4M;
6995 return (rv);
6996 }
6997
6998 /*
6999 * Tell whether the given physical page has been referenced.
7000 */
7001 bool
pmap_is_referenced4m(struct vm_page * pg)7002 pmap_is_referenced4m(struct vm_page *pg)
7003 {
7004
7005 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M ||
7006 pv_syncflags4m(pg) & PV_REF4M);
7007 }
7008 #endif /* SUN4M || SUN4D */
7009
7010 /*
7011 * Fill the given MI physical page with zero bytes.
7012 *
7013 * We avoid stomping on the cache.
7014 * XXX might be faster to use destination's context and allow cache to fill?
7015 */
7016
7017 #if defined(SUN4) || defined(SUN4C)
7018
7019 void
pmap_zero_page4_4c(paddr_t pa)7020 pmap_zero_page4_4c(paddr_t pa)
7021 {
7022 struct vm_page *pg;
7023 void *va;
7024 int pte;
7025
7026 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
7027 /*
7028 * The following might not be necessary since the page
7029 * is being cleared because it is about to be allocated,
7030 * i.e., is in use by no one.
7031 */
7032 pv_flushcache4_4c(pg);
7033 }
7034 pte = PG_V | PG_S | PG_W | PG_NC | (atop(pa) & PG_PFNUM);
7035
7036 va = cpuinfo.vpage[0];
7037 setpte4(va, pte);
7038 qzero(va, NBPG);
7039 setpte4(va, 0);
7040 }
7041
7042 /*
7043 * Copy the given MI physical source page to its destination.
7044 *
7045 * We avoid stomping on the cache as above (with same `XXX' note).
7046 * We must first flush any write-back cache for the source page.
7047 * We go ahead and stomp on the kernel's virtual cache for the
7048 * source page, since the cache can read memory MUCH faster than
7049 * the processor.
7050 */
7051 void
pmap_copy_page4_4c(paddr_t src,paddr_t dst)7052 pmap_copy_page4_4c(paddr_t src, paddr_t dst)
7053 {
7054 struct vm_page *pg;
7055 char *sva, *dva;
7056 int spte, dpte;
7057
7058 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
7059 if (CACHEINFO.c_vactype == VAC_WRITEBACK)
7060 pv_flushcache4_4c(pg);
7061 }
7062 spte = PG_V | PG_S | (atop(src) & PG_PFNUM);
7063
7064 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
7065 /* similar `might not be necessary' comment applies */
7066 if (CACHEINFO.c_vactype != VAC_NONE)
7067 pv_flushcache4_4c(pg);
7068 }
7069 dpte = PG_V | PG_S | PG_W | PG_NC | (atop(dst) & PG_PFNUM);
7070
7071 sva = cpuinfo.vpage[0];
7072 dva = cpuinfo.vpage[1];
7073 setpte4(sva, spte);
7074 setpte4(dva, dpte);
7075 qcopy(sva, dva, NBPG); /* loads cache, so we must ... */
7076 cache_flush_page((vaddr_t)sva, getcontext4());
7077 setpte4(sva, 0);
7078 setpte4(dva, 0);
7079 }
7080 #endif /* SUN4 || SUN4C */
7081
7082 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of copy/zero routines */
7083 /*
7084 * Fill the given MI physical page with zero bytes.
7085 *
7086 * We avoid stomping on the cache.
7087 * XXX might be faster to use destination's context and allow cache to fill?
7088 */
7089 void
pmap_zero_page4m(paddr_t pa)7090 pmap_zero_page4m(paddr_t pa)
7091 {
7092 struct vm_page *pg;
7093 void *va;
7094 int pte;
7095
7096 kpreempt_disable();
7097 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
7098 /*
7099 * The following VAC flush might not be necessary since the
7100 * page is being cleared because it is about to be allocated,
7101 * i.e., is in use by no one.
7102 * In the case of a physical cache, a flush (or just an
7103 * invalidate, if possible) is usually necessary when using
7104 * uncached access to clear it.
7105 */
7106 if (CACHEINFO.c_vactype != VAC_NONE)
7107 pv_flushcache4m(pg);
7108 else
7109 pcache_flush_page(pa, 1);
7110 }
7111 pte = SRMMU_TEPTE | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);
7112 if (CACHEINFO.c_flags & CACHE_MANDATORY)
7113 pte |= SRMMU_PG_C;
7114
7115 va = cpuinfo.vpage[0];
7116 setpgt4m(cpuinfo.vpage_pte[0], pte);
7117 qzero(va, NBPG);
7118 /*
7119 * Remove temporary mapping (which is kernel-only, so the
7120 * context used for TLB flushing does not matter)
7121 */
7122 sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
7123 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7124 kpreempt_enable();
7125 }
7126
7127 /*
7128 * Viking/MXCC specific version of pmap_zero_page
7129 */
7130 void
pmap_zero_page_viking_mxcc(paddr_t pa)7131 pmap_zero_page_viking_mxcc(paddr_t pa)
7132 {
7133 u_int offset;
7134 u_int stream_data_addr = MXCC_STREAM_DATA;
7135 uint64_t v = (uint64_t)pa;
7136
7137 kpreempt_disable();
7138 /* Load MXCC stream data register with 0 (bottom 32 bytes only) */
7139 stda(stream_data_addr+0, ASI_CONTROL, 0);
7140 stda(stream_data_addr+8, ASI_CONTROL, 0);
7141 stda(stream_data_addr+16, ASI_CONTROL, 0);
7142 stda(stream_data_addr+24, ASI_CONTROL, 0);
7143
7144 /* Then write the stream data register to each block in the page */
7145 v |= MXCC_STREAM_C;
7146 for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
7147 stda(MXCC_STREAM_DST, ASI_CONTROL, v | offset);
7148 }
7149 kpreempt_enable();
7150 }
7151
7152 /*
7153 * HyperSPARC/RT625 specific version of pmap_zero_page
7154 */
7155 void
pmap_zero_page_hypersparc(paddr_t pa)7156 pmap_zero_page_hypersparc(paddr_t pa)
7157 {
7158 struct vm_page *pg;
7159 void *va;
7160 int pte;
7161 int offset;
7162
7163 kpreempt_disable();
7164 /*
7165 * We still have to map the page, since ASI_BLOCKFILL
7166 * takes virtual addresses. This also means we have to
7167 * consider cache aliasing; therefore we still need
7168 * to flush the cache here. All we gain is the speed-up
7169 * in zero-fill loop itself..
7170 */
7171 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
7172 /*
7173 * The following might not be necessary since the page
7174 * is being cleared because it is about to be allocated,
7175 * i.e., is in use by no one.
7176 */
7177 if (CACHEINFO.c_vactype != VAC_NONE)
7178 pv_flushcache4m(pg);
7179 }
7180 pte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);
7181
7182 va = cpuinfo.vpage[0];
7183 setpgt4m(cpuinfo.vpage_pte[0], pte);
7184 for (offset = 0; offset < NBPG; offset += 32) {
7185 sta((char *)va + offset, ASI_BLOCKFILL, 0);
7186 }
7187 /* Remove temporary mapping */
7188 sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
7189 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7190 kpreempt_enable();
7191 }
7192
7193 /*
7194 * Copy the given MI physical source page to its destination.
7195 *
7196 * We avoid stomping on the cache as above (with same `XXX' note).
7197 * We must first flush any write-back cache for the source page.
7198 * We go ahead and stomp on the kernel's virtual cache for the
7199 * source page, since the cache can read memory MUCH faster than
7200 * the processor.
7201 */
7202 void
pmap_copy_page4m(paddr_t src,paddr_t dst)7203 pmap_copy_page4m(paddr_t src, paddr_t dst)
7204 {
7205 struct vm_page *pg;
7206 void *sva, *dva;
7207 int spte, dpte;
7208
7209 kpreempt_disable();
7210 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
7211 if (CACHEINFO.c_vactype == VAC_WRITEBACK)
7212 pv_flushcache4m(pg);
7213 }
7214
7215 spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
7216 (src >> SRMMU_PPNPASHIFT);
7217
7218 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
7219 /* similar `might not be necessary' comment applies */
7220 if (CACHEINFO.c_vactype != VAC_NONE)
7221 pv_flushcache4m(pg);
7222 else
7223 pcache_flush_page(dst, 1);
7224 }
7225
7226 dpte = SRMMU_TEPTE | PPROT_N_RWX | (dst >> SRMMU_PPNPASHIFT);
7227 if (CACHEINFO.c_flags & CACHE_MANDATORY)
7228 dpte |= SRMMU_PG_C;
7229
7230 sva = cpuinfo.vpage[0];
7231 dva = cpuinfo.vpage[1];
7232 setpgt4m(cpuinfo.vpage_pte[0], spte);
7233 setpgt4m(cpuinfo.vpage_pte[1], dpte);
7234 qcopy(sva, dva, NBPG); /* loads cache, so we must ... */
7235 cpuinfo.sp_vcache_flush_page((vaddr_t)sva, getcontext4m());
7236 sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
7237 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7238 sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
7239 setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
7240 kpreempt_enable();
7241 }
7242
7243 /*
7244 * Viking/MXCC specific version of pmap_copy_page
7245 */
7246 void
pmap_copy_page_viking_mxcc(paddr_t src,paddr_t dst)7247 pmap_copy_page_viking_mxcc(paddr_t src, paddr_t dst)
7248 {
7249 u_int offset;
7250 uint64_t v1 = (uint64_t)src;
7251 uint64_t v2 = (uint64_t)dst;
7252
7253 kpreempt_disable();
7254 /* Enable cache-coherency */
7255 v1 |= MXCC_STREAM_C;
7256 v2 |= MXCC_STREAM_C;
7257
7258 /* Copy through stream data register */
7259 for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
7260 stda(MXCC_STREAM_SRC, ASI_CONTROL, v1 | offset);
7261 stda(MXCC_STREAM_DST, ASI_CONTROL, v2 | offset);
7262 }
7263 kpreempt_enable();
7264 }
7265
7266 /*
7267 * HyperSPARC/RT625 specific version of pmap_copy_page
7268 */
7269 void
pmap_copy_page_hypersparc(paddr_t src,paddr_t dst)7270 pmap_copy_page_hypersparc(paddr_t src, paddr_t dst)
7271 {
7272 struct vm_page *pg;
7273 void *sva, *dva;
7274 int spte, dpte;
7275 int offset;
7276
7277 kpreempt_disable();
7278 /*
7279 * We still have to map the pages, since ASI_BLOCKCOPY
7280 * takes virtual addresses. This also means we have to
7281 * consider cache aliasing; therefore we still need
7282 * to flush the cache here. All we gain is the speed-up
7283 * in copy loop itself..
7284 */
7285
7286 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
7287 if (CACHEINFO.c_vactype == VAC_WRITEBACK)
7288 pv_flushcache4m(pg);
7289 }
7290
7291 spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
7292 (src >> SRMMU_PPNPASHIFT);
7293
7294 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
7295 /* similar `might not be necessary' comment applies */
7296 if (CACHEINFO.c_vactype != VAC_NONE)
7297 pv_flushcache4m(pg);
7298 }
7299
7300 dpte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX |
7301 (dst >> SRMMU_PPNPASHIFT);
7302
7303 sva = cpuinfo.vpage[0];
7304 dva = cpuinfo.vpage[1];
7305 setpgt4m(cpuinfo.vpage_pte[0], spte);
7306 setpgt4m(cpuinfo.vpage_pte[1], dpte);
7307
7308 for (offset = 0; offset < NBPG; offset += 32) {
7309 sta((char *)dva + offset, ASI_BLOCKCOPY, (char *)sva + offset);
7310 }
7311
7312 sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
7313 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
7314 sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
7315 setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
7316 kpreempt_enable();
7317 }
7318 #endif /* SUN4M || SUN4D */
7319
7320 /*
7321 * Turn off cache for a given (va, number of pages).
7322 *
7323 * We just assert PG_NC for each PTE; the addresses must reside
7324 * in locked kernel space. A cache flush is also done.
7325 */
7326 void
kvm_uncache(char * va,int npages)7327 kvm_uncache(char *va, int npages)
7328 {
7329 struct vm_page *pg;
7330 int pte;
7331
7332 if (CPU_HAS_SRMMU) {
7333 #if defined(SUN4M) || defined(SUN4D)
7334 for (; --npages >= 0; va = (char *)va + NBPG) {
7335 pte = getpte4m((vaddr_t) va);
7336 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
7337 panic("kvm_uncache: table entry not pte");
7338
7339 if ((pte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
7340 if ((pg = pvhead4m(pte)) != NULL) {
7341 pv_uncache(pg);
7342 return;
7343 }
7344 cache_flush_page((vaddr_t)va, 0);
7345 }
7346
7347 pte &= ~SRMMU_PG_C;
7348 setpte4m((vaddr_t)va, pte);
7349 }
7350 #endif
7351 } else {
7352 #if defined(SUN4) || defined(SUN4C)
7353 for (; --npages >= 0; va += NBPG) {
7354 pte = getpte4(va);
7355 if ((pte & PG_V) == 0)
7356 panic("kvm_uncache !pg_v");
7357
7358 if ((pte & PG_TYPE) == PG_OBMEM) {
7359 if ((pg = pvhead4_4c(pte)) != NULL) {
7360 pv_uncache(pg);
7361 return;
7362 }
7363 cache_flush_page((vaddr_t)va, 0);
7364 }
7365 pte |= PG_NC;
7366 setpte4(va, pte);
7367 }
7368 #endif
7369 }
7370 }
7371
7372 #if 0 /* not used */
7373 /*
7374 * Turn on IO cache for a given (va, number of pages).
7375 *
7376 * We just assert PG_NC for each PTE; the addresses must reside
7377 * in locked kernel space. A cache flush is also done.
7378 */
7379 void
7380 kvm_iocache(char *va, int npages)
7381 {
7382
7383 #if defined(SUN4M)
7384 if (CPU_ISSUN4M) /* %%%: Implement! */
7385 panic("kvm_iocache: 4m iocache not implemented");
7386 #endif
7387 #if defined(SUN4D)
7388 if (CPU_ISSUN4D) /* %%%: Implement! */
7389 panic("kvm_iocache: 4d iocache not implemented");
7390 #endif
7391 #if defined(SUN4) || defined(SUN4C)
7392 for (; --npages >= 0; va += NBPG) {
7393 int pte = getpte4(va);
7394 if ((pte & PG_V) == 0)
7395 panic("kvm_iocache !pg_v");
7396 pte |= PG_IOC;
7397 setpte4(va, pte);
7398 }
7399 #endif
7400 }
7401 #endif
7402
7403 /*
7404 * Find first virtual address >= *va that is
7405 * least likely to cause cache aliases.
7406 * (This will just seg-align mappings.)
7407 */
7408 void
pmap_prefer(vaddr_t foff,vaddr_t * vap,size_t size,int td)7409 pmap_prefer(vaddr_t foff, vaddr_t *vap, size_t size, int td)
7410 {
7411 vaddr_t va = *vap;
7412 long m;
7413
7414 m = CACHE_ALIAS_DIST;
7415 if (m == 0) /* m=0 => no cache aliasing */
7416 return;
7417
7418 if (VA_INHOLE(va)) {
7419 if (td)
7420 va = MMU_HOLE_START - size;
7421 else
7422 va = MMU_HOLE_END;
7423 }
7424
7425 va = (va & ~(m - 1)) | (foff & (m - 1));
7426
7427 if (td) {
7428 if (va > *vap)
7429 va -= m;
7430 } else {
7431 if (va < *vap)
7432 va += m;
7433 }
7434 *vap = va;
7435 }
7436
7437 void
pmap_redzone(void)7438 pmap_redzone(void)
7439 {
7440
7441 pmap_remove(pmap_kernel(), KERNBASE, KERNBASE+NBPG);
7442 }
7443
7444 /*
7445 * Activate the address space for the specified process. If the
7446 * process is the current process, load the new MMU context.
7447 */
7448 void
pmap_activate(struct lwp * l)7449 pmap_activate(struct lwp *l)
7450 {
7451 pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap;
7452
7453 if (pm == pmap_kernel() || l != curlwp) {
7454 return;
7455 }
7456
7457 mutex_spin_enter(&ctx_lock);
7458 if (pm->pm_ctx == NULL) {
7459 ctx_alloc(pm); /* performs setcontext() */
7460 } else {
7461 setcontext(pm->pm_ctxnum);
7462 }
7463 PMAP_SET_CPUSET(pm, &cpuinfo);
7464 mutex_spin_exit(&ctx_lock);
7465 }
7466
7467 /*
7468 * Deactivate the address space of the specified process.
7469 */
7470 void
pmap_deactivate(struct lwp * l)7471 pmap_deactivate(struct lwp *l)
7472 {
7473 struct proc *p = l->l_proc;
7474 pmap_t pm = p->p_vmspace->vm_map.pmap;
7475
7476 if (pm == pmap_kernel() || l != curlwp) {
7477 return;
7478 }
7479
7480 write_user_windows();
7481 mutex_spin_enter(&ctx_lock);
7482 if (pm->pm_ctx) {
7483 (*cpuinfo.pure_vcache_flush)();
7484
7485 #if defined(SUN4M) || defined(SUN4D)
7486 if (CPU_HAS_SRMMU)
7487 sp_tlb_flush(0, pm->pm_ctxnum, ASI_SRMMUFP_L0);
7488 #endif
7489 }
7490
7491 /* we no longer need broadcast tlb flushes for this pmap. */
7492 PMAP_CLR_CPUSET(pm, &cpuinfo);
7493 mutex_spin_exit(&ctx_lock);
7494 }
7495
7496 #ifdef DEBUG
7497 /*
7498 * Check consistency of a pmap (time consuming!).
7499 */
7500 void
pm_check(char * s,struct pmap * pm)7501 pm_check(char *s, struct pmap *pm)
7502 {
7503
7504 if (pm == pmap_kernel())
7505 pm_check_k(s, pm);
7506 else
7507 pm_check_u(s, pm);
7508 }
7509
7510 void
pm_check_u(char * s,struct pmap * pm)7511 pm_check_u(char *s, struct pmap *pm)
7512 {
7513 struct regmap *rp;
7514 struct segmap *sp;
7515 int cpu, n, vs, vr, j, m, *pte;
7516
7517 cpu = cpuinfo.ci_cpuid;
7518
7519 if (pm->pm_regmap == NULL)
7520 panic("%s: CPU %d: CHK(pmap %p): no region mapping",
7521 s, cpu, pm);
7522
7523 #if defined(SUN4M) || defined(SUN4D)
7524 if (CPU_HAS_SRMMU &&
7525 (pm->pm_reg_ptps[cpu] == NULL ||
7526 pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
7527 panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: "
7528 "tblva=%p, tblpa=0x%x",
7529 s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);
7530
7531 if (CPU_HAS_SRMMU && pm->pm_ctx != NULL &&
7532 (cpuinfo.ctx_tbl[pm->pm_ctxnum] != ((VA2PA((void *)pm->pm_reg_ptps[cpu])
7533 >> SRMMU_PPNPASHIFT) |
7534 SRMMU_TEPTD)))
7535 panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
7536 "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], pm->pm_ctxnum);
7537 #endif
7538
7539 for (vr = 0; vr < NUREG; vr++) {
7540 rp = &pm->pm_regmap[vr];
7541 if (rp->rg_nsegmap == 0)
7542 continue;
7543 if (rp->rg_segmap == NULL)
7544 panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
7545 s, cpu, vr, rp->rg_nsegmap);
7546 #if defined(SUN4M) || defined(SUN4D)
7547 if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
7548 panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
7549 s, cpu, vr, rp->rg_nsegmap);
7550 if (CPU_HAS_SRMMU &&
7551 pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
7552 SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
7553 panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
7554 s, cpu, vr);
7555 #endif
7556 if ((unsigned int)rp < KERNBASE)
7557 panic("%s: CPU %d: rp=%p", s, cpu, rp);
7558 n = 0;
7559 for (vs = 0; vs < NSEGRG; vs++) {
7560 sp = &rp->rg_segmap[vs];
7561 if ((unsigned int)sp < KERNBASE)
7562 panic("%s: CPU %d: sp=%p", s, cpu, sp);
7563 if (sp->sg_npte != 0) {
7564 n++;
7565 if (sp->sg_pte == NULL)
7566 panic("%s: CPU %d: CHK(vr %d, vs %d): npte=%d, "
7567 "pte=NULL", s, cpu, vr, vs, sp->sg_npte);
7568 #if defined(SUN4M) || defined(SUN4D)
7569 if (CPU_HAS_SRMMU &&
7570 rp->rg_seg_ptps[vs] !=
7571 ((VA2PA((void *)sp->sg_pte)
7572 >> SRMMU_PPNPASHIFT) |
7573 SRMMU_TEPTD))
7574 panic("%s: CPU %d: CHK(vr %d, vs %d): SRMMU page "
7575 "table not installed correctly",
7576 s, cpu, vr, vs);
7577 #endif
7578 pte=sp->sg_pte;
7579 m = 0;
7580 for (j=0; j<NPTESG; j++,pte++)
7581 if ((CPU_HAS_SRMMU
7582 ?((*pte & SRMMU_TETYPE) == SRMMU_TEPTE)
7583 :(*pte & PG_V)))
7584 m++;
7585 if (m != sp->sg_npte)
7586 printf("%s: CPU %d: user CHK(vr %d, vs %d): "
7587 "npte(%d) != # valid(%d)\n",
7588 s, cpu, vr, vs, sp->sg_npte, m);
7589 }
7590 }
7591 if (n != rp->rg_nsegmap)
7592 panic("%s: CPU %d: CHK(vr %d): inconsistent "
7593 "# of pte's: %d, should be %d",
7594 s, cpu, vr, rp->rg_nsegmap, n);
7595 }
7596 return;
7597 }
7598
7599 /* Note: not as extensive as pm_check_u. */
7600 void
pm_check_k(char * s,struct pmap * pm)7601 pm_check_k(char *s, struct pmap *pm)
7602 {
7603 struct regmap *rp;
7604 int cpu, vr, vs, n;
7605
7606 cpu = cpu_number();
7607
7608 if (pm->pm_regmap == NULL)
7609 panic("%s: CHK(pmap %p): no region mapping", s, pm);
7610
7611 #if defined(SUN4M) || defined(SUN4D)
7612 if (CPU_HAS_SRMMU &&
7613 (pm->pm_reg_ptps[cpu] == NULL ||
7614 pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
7615 panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: tblva=%p, tblpa=0x%x",
7616 s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);
7617
7618 if (CPU_HAS_SRMMU &&
7619 (cpuinfo.ctx_tbl[0] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) >>
7620 SRMMU_PPNPASHIFT) | SRMMU_TEPTD)))
7621 panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
7622 "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], 0);
7623 #endif
7624 for (vr = NUREG; vr < NUREG+NKREG; vr++) {
7625 rp = &pm->pm_regmap[vr];
7626 if (rp->rg_segmap == NULL)
7627 panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
7628 s, cpu, vr, rp->rg_nsegmap);
7629 if (rp->rg_nsegmap == 0)
7630 continue;
7631 #if defined(SUN4M) || defined(SUN4D)
7632 if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
7633 panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
7634 s, cpu, vr, rp->rg_nsegmap);
7635
7636 if (CPU_HAS_SRMMU && vr != NUREG /* 1st kseg is per CPU */ &&
7637 pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
7638 SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
7639 panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
7640 s, cpu, vr);
7641 #endif
7642 if (CPU_HAS_SRMMU) {
7643 n = NSEGRG;
7644 } else {
7645 for (n = 0, vs = 0; vs < NSEGRG; vs++) {
7646 if (rp->rg_segmap[vs].sg_npte)
7647 n++;
7648 }
7649 }
7650 if (n != rp->rg_nsegmap)
7651 printf("%s: CPU %d: kernel CHK(vr %d): inconsistent "
7652 "# of pte's: %d, should be %d\n",
7653 s, cpu, vr, rp->rg_nsegmap, n);
7654 }
7655 return;
7656 }
7657 #endif
7658
7659 /*
7660 * Return the number of disk blocks that pmap_dumpmmu() will dump.
7661 */
7662 int
pmap_dumpsize(void)7663 pmap_dumpsize(void)
7664 {
7665 int sz;
7666
7667 sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t));
7668 sz += npmemarr * sizeof(phys_ram_seg_t);
7669 sz += sizeof(kernel_segmap_store);
7670
7671 if (CPU_HAS_SUNMMU)
7672 /* For each pmeg in the MMU, we'll write NPTESG PTEs. */
7673 sz += (seginval + 1) * NPTESG * sizeof(int);
7674
7675 return btodb(sz + DEV_BSIZE - 1);
7676 }
7677
7678 /*
7679 * Write the core dump headers and MD data to the dump device.
7680 * We dump the following items:
7681 *
7682 * kcore_seg_t MI header defined in <sys/kcore.h>)
7683 * cpu_kcore_hdr_t MD header defined in <machine/kcore.h>)
7684 * phys_ram_seg_t[npmemarr] physical memory segments
7685 * segmap_t[NKREG*NSEGRG] the kernel's segment map
7686 * the MMU pmegs on sun4/sun4c
7687 */
7688 int
pmap_dumpmmu(int (* dump)(dev_t,daddr_t,void *,size_t),daddr_t blkno)7689 pmap_dumpmmu(int (*dump)(dev_t, daddr_t, void *, size_t),
7690 daddr_t blkno)
7691 {
7692 kcore_seg_t *ksegp;
7693 cpu_kcore_hdr_t *kcpup;
7694 phys_ram_seg_t memseg;
7695 int error = 0;
7696 int i, memsegoffset, segmapoffset, pmegoffset;
7697 int buffer[dbtob(1) / sizeof(int)];
7698 int *bp, *ep;
7699 #if defined(SUN4C) || defined(SUN4)
7700 int pmeg;
7701 #endif
7702
7703 #define EXPEDITE(p,n) do { \
7704 int *sp = (int *)(p); \
7705 int sz = (n); \
7706 while (sz > 0) { \
7707 *bp++ = *sp++; \
7708 if (bp >= ep) { \
7709 error = (*dump)(dumpdev, blkno, \
7710 (void *)buffer, dbtob(1)); \
7711 if (error != 0) \
7712 return (error); \
7713 ++blkno; \
7714 bp = buffer; \
7715 } \
7716 sz -= 4; \
7717 } \
7718 } while (0)
7719
7720 setcontext(0);
7721
7722 /* Setup bookkeeping pointers */
7723 bp = buffer;
7724 ep = &buffer[sizeof(buffer) / sizeof(buffer[0])];
7725
7726 /* Fill in MI segment header */
7727 ksegp = (kcore_seg_t *)bp;
7728 CORE_SETMAGIC(*ksegp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
7729 ksegp->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t));
7730
7731 /* Fill in MD segment header (interpreted by MD part of libkvm) */
7732 kcpup = (cpu_kcore_hdr_t *)((int)bp + ALIGN(sizeof(kcore_seg_t)));
7733 kcpup->cputype = cputyp;
7734 kcpup->kernbase = KERNBASE;
7735 kcpup->nmemseg = npmemarr;
7736 kcpup->memsegoffset = memsegoffset = ALIGN(sizeof(cpu_kcore_hdr_t));
7737 kcpup->nsegmap = NKREG*NSEGRG;
7738 kcpup->segmapoffset = segmapoffset =
7739 memsegoffset + npmemarr * sizeof(phys_ram_seg_t);
7740
7741 kcpup->npmeg = (CPU_HAS_SUNMMU) ? seginval + 1 : 0;
7742 kcpup->pmegoffset = pmegoffset =
7743 segmapoffset + kcpup->nsegmap * sizeof(struct segmap);
7744
7745 /* Note: we have assumed everything fits in buffer[] so far... */
7746 bp = (int *)((int)kcpup + ALIGN(sizeof(cpu_kcore_hdr_t)));
7747
7748 #if 0
7749 /* Align storage for upcoming quad-aligned segment array */
7750 while (bp != (int *)ALIGN(bp)) {
7751 int dummy = 0;
7752 EXPEDITE(&dummy, 4);
7753 }
7754 #endif
7755
7756 for (i = 0; i < npmemarr; i++) {
7757 memseg.start = pmemarr[i].addr;
7758 memseg.size = pmemarr[i].len;
7759 EXPEDITE((void *)&memseg, sizeof(phys_ram_seg_t));
7760 }
7761
7762 EXPEDITE(&kernel_segmap_store, sizeof(kernel_segmap_store));
7763
7764 if (CPU_HAS_SRMMU)
7765 goto out;
7766
7767 #if defined(SUN4C) || defined(SUN4)
7768 /*
7769 * dump page table entries
7770 *
7771 * We dump each pmeg in order (by segment number). Since the MMU
7772 * automatically maps the given virtual segment to a pmeg we must
7773 * iterate over the segments by incrementing an unused segment slot
7774 * in the MMU. This fixed segment number is used in the virtual
7775 * address argument to getpte().
7776 */
7777
7778 /*
7779 * Go through the pmegs and dump each one.
7780 */
7781 for (pmeg = 0; pmeg <= seginval; ++pmeg) {
7782 int va = 0;
7783
7784 setsegmap(va, pmeg);
7785 i = NPTESG;
7786 do {
7787 int pte = getpte4(va);
7788 EXPEDITE(&pte, sizeof(pte));
7789 va += NBPG;
7790 } while (--i > 0);
7791 }
7792 setsegmap(0, seginval);
7793 #endif
7794
7795 out:
7796 if (bp != buffer)
7797 error = (*dump)(dumpdev, blkno++, (void *)buffer, dbtob(1));
7798
7799 return (error);
7800 }
7801
7802 /*
7803 * Helper function for debuggers.
7804 */
7805 void
pmap_writetext(unsigned char * dst,int ch)7806 pmap_writetext(unsigned char *dst, int ch)
7807 {
7808 int s, pte0, pte, ctx;
7809 vaddr_t va;
7810
7811 s = splvm();
7812 va = (unsigned long)dst & (~PGOFSET);
7813 cache_flush(dst, 1);
7814
7815 ctx = getcontext();
7816 setcontext(0);
7817
7818 #if defined(SUN4M) || defined(SUN4D)
7819 if (CPU_HAS_SRMMU) {
7820 pte0 = getpte4m(va);
7821 if ((pte0 & SRMMU_TETYPE) != SRMMU_TEPTE) {
7822 goto out;
7823 }
7824 pte = pte0 | PPROT_WRITE;
7825 setpte4m(va, pte);
7826 *dst = (unsigned char)ch;
7827 setpte4m(va, pte0);
7828
7829 }
7830 #endif
7831 #if defined(SUN4) || defined(SUN4C)
7832 if (CPU_ISSUN4C || CPU_ISSUN4) {
7833 pte0 = getpte4(va);
7834 if ((pte0 & PG_V) == 0) {
7835 goto out;
7836 }
7837 pte = pte0 | PG_W;
7838 setpte4(va, pte);
7839 *dst = (unsigned char)ch;
7840 setpte4(va, pte0);
7841 }
7842 #endif
7843 cache_flush(dst, 1);
7844
7845 out:
7846 setcontext(ctx);
7847 splx(s);
7848 }
7849
7850 #ifdef EXTREME_DEBUG
7851
7852 void debug_pagetables(void);
7853 void print_fe_map(void);
7854
7855 static void test_region(int, int, int);
7856
7857
7858 void
debug_pagetables(void)7859 debug_pagetables(void)
7860 {
7861 struct promvec *promvec = romp;
7862 int *regtbl;
7863 int te;
7864 int i;
7865
7866 printf("\nncontext=%d. ", ncontext);
7867 printf("Context table is at va %p. Level 0 PTP: 0x%x\n",
7868 cpuinfo.ctx_tbl, cpuinfo.ctx_tbl[0]);
7869 printf("Context 0 region table is at va %p, pa 0x%x. Contents:\n",
7870 pmap_kernel()->pm_reg_ptps[0], pmap_kernel()->pm_reg_ptps_pa[0]);
7871
7872 regtbl = pmap_kernel()->pm_reg_ptps[0];
7873
7874 printf("PROM vector is at %p\n", promvec);
7875 printf("PROM reboot routine is at %p\n", promvec->pv_reboot);
7876 printf("PROM abort routine is at %p\n", promvec->pv_abort);
7877 printf("PROM halt routine is at %p\n", promvec->pv_halt);
7878
7879 printf("Testing region 0xfe: ");
7880 test_region(0xfe,0,16*1024*1024);
7881 printf("Testing region 0xff: ");
7882 test_region(0xff,0,16*1024*1024);
7883 printf("Testing kernel region 0x%x: ", VA_VREG(KERNBASE));
7884 test_region(VA_VREG(KERNBASE), 4096, avail_start);
7885 cngetc();
7886
7887 for (i = 0; i < SRMMU_L1SIZE; i++) {
7888 te = regtbl[i];
7889 if ((te & SRMMU_TETYPE) == SRMMU_TEINVALID)
7890 continue;
7891 printf("Region 0x%x: PTE=0x%x <%s> L2PA=0x%x kernL2VA=%p\n",
7892 i, te, ((te & SRMMU_TETYPE) == SRMMU_TEPTE ? "pte" :
7893 ((te & SRMMU_TETYPE) == SRMMU_TEPTD ? "ptd" :
7894 ((te & SRMMU_TETYPE) == SRMMU_TEINVALID ?
7895 "invalid" : "reserved"))),
7896 (te & ~0x3) << SRMMU_PPNPASHIFT,
7897 pmap_kernel()->pm_regmap[i].rg_seg_ptps);
7898 }
7899 printf("Press q to halt...\n");
7900 if (cngetc()=='q')
7901 callrom();
7902 }
7903
7904 static u_int
VA2PAsw(int ctx,void * addr,int * pte)7905 VA2PAsw(int ctx, void *addr, int *pte)
7906 {
7907 int *curtbl;
7908 int curpte;
7909
7910 #ifdef EXTREME_EXTREME_DEBUG
7911 printf("Looking up addr 0x%x in context 0x%x\n",addr,ctx);
7912 #endif
7913 /* L0 */
7914 *pte = curpte = cpuinfo.ctx_tbl[ctx];
7915 #ifdef EXTREME_EXTREME_DEBUG
7916 printf("Got L0 pte 0x%x\n",pte);
7917 #endif
7918 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
7919 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7920 ((u_int)addr & 0xffffffff));
7921 }
7922 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
7923 printf("Bad context table entry 0x%x for context 0x%x\n",
7924 curpte, ctx);
7925 return 0;
7926 }
7927 /* L1 */
7928 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
7929 *pte = curpte = curtbl[VA_VREG(addr)];
7930 #ifdef EXTREME_EXTREME_DEBUG
7931 printf("L1 table at 0x%x.\nGot L1 pte 0x%x\n",curtbl,curpte);
7932 #endif
7933 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
7934 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7935 ((u_int)addr & 0xffffff));
7936 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
7937 printf("Bad region table entry 0x%x for region 0x%x\n",
7938 curpte, VA_VREG(addr));
7939 return 0;
7940 }
7941 /* L2 */
7942 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
7943 *pte = curpte = curtbl[VA_VSEG(addr)];
7944 #ifdef EXTREME_EXTREME_DEBUG
7945 printf("L2 table at 0x%x.\nGot L2 pte 0x%x\n",curtbl,curpte);
7946 #endif
7947 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
7948 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7949 ((u_int)addr & 0x3ffff));
7950 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
7951 printf("Bad segment table entry 0x%x for reg 0x%x, seg 0x%x\n",
7952 curpte, VA_VREG(addr), VA_VSEG(addr));
7953 return 0;
7954 }
7955 /* L3 */
7956 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
7957 *pte = curpte = curtbl[VA_VPG(addr)];
7958 #ifdef EXTREME_EXTREME_DEBUG
7959 printf("L3 table at %p.\nGot L3 pte 0x%x\n", curtbl, curpte);
7960 #endif
7961 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
7962 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
7963 ((u_int)addr & 0xfff));
7964 else {
7965 printf("Bad L3 pte 0x%x for reg 0x%x, seg 0x%x, pg 0x%x\n",
7966 curpte, VA_VREG(addr), VA_VSEG(addr), VA_VPG(addr));
7967 return 0;
7968 }
7969 printf("Bizarreness with address %p!\n", addr);
7970 }
7971
7972 static void
test_region(int reg,int start,int stop)7973 test_region(int reg, int start, int stop)
7974 {
7975 int i;
7976 int addr;
7977 int pte;
7978 int ptesw;
7979 /* int cnt=0;
7980 */
7981
7982 for (i = start; i < stop; i += NBPG) {
7983 addr = (reg << RGSHIFT) | i;
7984 pte = lda(((u_int)(addr)) | ASI_SRMMUFP_LN, ASI_SRMMUFP);
7985 if (pte) {
7986 /* printf("Valid address 0x%x\n",addr);
7987 if (++cnt == 20) {
7988 cngetc();
7989 cnt = 0;
7990 }
7991 */
7992 if (VA2PA((void *)addr) != VA2PAsw(0, (void *)addr, &ptesw)) {
7993 printf("Mismatch at address 0x%x.\n", addr);
7994 if (cngetc() == 'q')
7995 break;
7996 }
7997 if (reg == VA_VREG(KERNBASE))
7998 /* kernel permissions are different */
7999 continue;
8000 if ((pte & SRMMU_PROT_MASK) != (ptesw & SRMMU_PROT_MASK)) {
8001 printf("Mismatched protections at address "
8002 "0x%x; pte=0x%x, ptesw=0x%x\n",
8003 addr, pte, ptesw);
8004 if (cngetc() == 'q')
8005 break;
8006 }
8007 }
8008 }
8009 printf("done.\n");
8010 }
8011
8012
8013 void
print_fe_map(void)8014 print_fe_map(void)
8015 {
8016 u_int i, pte;
8017
8018 printf("map of region 0xfe:\n");
8019 for (i = 0xfe000000; i < 0xff000000; i += 4096) {
8020 if (((pte = getpte4m(i)) & SRMMU_TETYPE) != SRMMU_TEPTE)
8021 continue;
8022 printf("0x%x -> 0x%x%x (pte 0x%x)\n", i, pte >> 28,
8023 (pte & ~0xff) << 4, pte);
8024 }
8025 printf("done\n");
8026 }
8027 #endif /* EXTREME_DEBUG */
8028
8029 #ifdef DDB
8030 int pmap_dump(struct pmap *pm);
8031
8032 int
pmap_dump(struct pmap * pm)8033 pmap_dump(struct pmap *pm)
8034 {
8035 int startvr, endvr, vr, vs, i, n;
8036 struct regmap *rp;
8037 struct segmap *sp;
8038
8039 if (pm == NULL)
8040 pm = pmap_kernel();
8041
8042 if (pm == pmap_kernel()) {
8043 startvr = NUREG;
8044 endvr = 256;
8045 } else {
8046 startvr = 0;
8047 endvr = NUREG;
8048 }
8049
8050 for (vr = startvr; vr < endvr; vr++) {
8051 rp = &pm->pm_regmap[vr];
8052 if (rp->rg_nsegmap == 0)
8053 continue;
8054 printf("vr %d: %d segments", vr, rp->rg_nsegmap);
8055 if (rp->rg_segmap == NULL) {
8056 printf("[no segments]\n");
8057 continue;
8058 }
8059 for (vs = 0; vs < NSEGRG; vs++) {
8060 sp = &rp->rg_segmap[vs];
8061 if (sp->sg_npte == 0)
8062 continue;
8063 if ((vs & 3) == 0)
8064 printf("\n ");
8065 printf(" %d: n %d w %d p %d,", vs,
8066 sp->sg_npte, sp->sg_nwired, sp->sg_pmeg);
8067 if (sp->sg_pte == NULL) {
8068 printf("[no ptes]");
8069 continue;
8070 }
8071 for (n = 0, i = 0; i < NPTESG; i++) {
8072 if (CPU_HAS_SUNMMU && sp->sg_pte[i] & PG_WIRED)
8073 n++;
8074 if (CPU_HAS_SRMMU && sp->sg_wiremap & (1 << i))
8075 n++;
8076 }
8077 if (n != sp->sg_nwired)
8078 printf("[wired count %d]", n);
8079 }
8080 printf("\n");
8081 }
8082
8083 return (0);
8084 }
8085 #endif /* DDB */
8086