1 /* $NetBSD: uvm_page.c,v 1.256 2024/03/05 14:33:50 thorpej Exp $ */
2
3 /*-
4 * Copyright (c) 2019, 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c) 1997 Charles D. Cranor and Washington University.
34 * Copyright (c) 1991, 1993, The Regents of the University of California.
35 *
36 * All rights reserved.
37 *
38 * This code is derived from software contributed to Berkeley by
39 * The Mach Operating System project at Carnegie-Mellon University.
40 *
41 * Redistribution and use in source and binary forms, with or without
42 * modification, are permitted provided that the following conditions
43 * are met:
44 * 1. Redistributions of source code must retain the above copyright
45 * notice, this list of conditions and the following disclaimer.
46 * 2. Redistributions in binary form must reproduce the above copyright
47 * notice, this list of conditions and the following disclaimer in the
48 * documentation and/or other materials provided with the distribution.
49 * 3. Neither the name of the University nor the names of its contributors
50 * may be used to endorse or promote products derived from this software
51 * without specific prior written permission.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63 * SUCH DAMAGE.
64 *
65 * @(#)vm_page.c 8.3 (Berkeley) 3/21/94
66 * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
67 *
68 *
69 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
70 * All rights reserved.
71 *
72 * Permission to use, copy, modify and distribute this software and
73 * its documentation is hereby granted, provided that both the copyright
74 * notice and this permission notice appear in all copies of the
75 * software, derivative works or modified versions, and any portions
76 * thereof, and that both notices appear in supporting documentation.
77 *
78 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
79 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
80 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
81 *
82 * Carnegie Mellon requests users of this software to return to
83 *
84 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
85 * School of Computer Science
86 * Carnegie Mellon University
87 * Pittsburgh PA 15213-3890
88 *
89 * any improvements or extensions that they make and grant Carnegie the
90 * rights to redistribute these changes.
91 */
92
93 /*
94 * uvm_page.c: page ops.
95 */
96
97 #include <sys/cdefs.h>
98 __KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.256 2024/03/05 14:33:50 thorpej Exp $");
99
100 #include "opt_ddb.h"
101 #include "opt_uvm.h"
102 #include "opt_uvmhist.h"
103 #include "opt_readahead.h"
104
105 #include <sys/param.h>
106 #include <sys/systm.h>
107 #include <sys/sched.h>
108 #include <sys/kernel.h>
109 #include <sys/vnode.h>
110 #include <sys/proc.h>
111 #include <sys/radixtree.h>
112 #include <sys/atomic.h>
113 #include <sys/cpu.h>
114
115 #include <ddb/db_active.h>
116
117 #include <uvm/uvm.h>
118 #include <uvm/uvm_ddb.h>
119 #include <uvm/uvm_pdpolicy.h>
120 #include <uvm/uvm_pgflcache.h>
121
122 /*
123 * number of pages per-CPU to reserve for the kernel.
124 */
125 #ifndef UVM_RESERVED_PAGES_PER_CPU
126 #define UVM_RESERVED_PAGES_PER_CPU 5
127 #endif
128 int vm_page_reserve_kernel = UVM_RESERVED_PAGES_PER_CPU;
129
130 /*
131 * physical memory size;
132 */
133 psize_t physmem;
134
135 /*
136 * local variables
137 */
138
139 /*
140 * these variables record the values returned by vm_page_bootstrap,
141 * for debugging purposes. The implementation of uvm_pageboot_alloc
142 * and pmap_startup here also uses them internally.
143 */
144
145 static vaddr_t virtual_space_start;
146 static vaddr_t virtual_space_end;
147
148 /*
149 * we allocate an initial number of page colors in uvm_page_init(),
150 * and remember them. We may re-color pages as cache sizes are
151 * discovered during the autoconfiguration phase. But we can never
152 * free the initial set of buckets, since they are allocated using
153 * uvm_pageboot_alloc().
154 */
155
156 static size_t recolored_pages_memsize /* = 0 */;
157 static char *recolored_pages_mem;
158
159 /*
160 * freelist locks - one per bucket.
161 */
162
163 union uvm_freelist_lock uvm_freelist_locks[PGFL_MAX_BUCKETS]
164 __cacheline_aligned;
165
166 /*
167 * basic NUMA information.
168 */
169
170 static struct uvm_page_numa_region {
171 struct uvm_page_numa_region *next;
172 paddr_t start;
173 paddr_t size;
174 u_int numa_id;
175 } *uvm_page_numa_region;
176
177 #ifdef DEBUG
178 kmutex_t uvm_zerochecklock __cacheline_aligned;
179 vaddr_t uvm_zerocheckkva;
180 #endif /* DEBUG */
181
182 /*
183 * These functions are reserved for uvm(9) internal use and are not
184 * exported in the header file uvm_physseg.h
185 *
186 * Thus they are redefined here.
187 */
188 void uvm_physseg_init_seg(uvm_physseg_t, struct vm_page *);
189 void uvm_physseg_seg_chomp_slab(uvm_physseg_t, struct vm_page *, size_t);
190
191 /* returns a pgs array */
192 struct vm_page *uvm_physseg_seg_alloc_from_slab(uvm_physseg_t, size_t);
193
194 /*
195 * inline functions
196 */
197
198 /*
199 * uvm_pageinsert: insert a page in the object.
200 *
201 * => caller must lock object
202 * => call should have already set pg's object and offset pointers
203 * and bumped the version counter
204 */
205
206 static inline void
uvm_pageinsert_object(struct uvm_object * uobj,struct vm_page * pg)207 uvm_pageinsert_object(struct uvm_object *uobj, struct vm_page *pg)
208 {
209
210 KASSERT(uobj == pg->uobject);
211 KASSERT(rw_write_held(uobj->vmobjlock));
212 KASSERT((pg->flags & PG_TABLED) == 0);
213
214 if ((pg->flags & PG_STAT) != 0) {
215 /* Cannot use uvm_pagegetdirty(): not yet in radix tree. */
216 const unsigned int status = pg->flags & (PG_CLEAN | PG_DIRTY);
217
218 if ((pg->flags & PG_FILE) != 0) {
219 if (uobj->uo_npages == 0) {
220 struct vnode *vp = (struct vnode *)uobj;
221 mutex_enter(vp->v_interlock);
222 KASSERT((vp->v_iflag & VI_PAGES) == 0);
223 vp->v_iflag |= VI_PAGES;
224 vholdl(vp);
225 mutex_exit(vp->v_interlock);
226 }
227 if (UVM_OBJ_IS_VTEXT(uobj)) {
228 cpu_count(CPU_COUNT_EXECPAGES, 1);
229 }
230 cpu_count(CPU_COUNT_FILEUNKNOWN + status, 1);
231 } else {
232 cpu_count(CPU_COUNT_ANONUNKNOWN + status, 1);
233 }
234 }
235 pg->flags |= PG_TABLED;
236 uobj->uo_npages++;
237 }
238
239 static inline int
uvm_pageinsert_tree(struct uvm_object * uobj,struct vm_page * pg)240 uvm_pageinsert_tree(struct uvm_object *uobj, struct vm_page *pg)
241 {
242 const uint64_t idx = pg->offset >> PAGE_SHIFT;
243 int error;
244
245 KASSERT(rw_write_held(uobj->vmobjlock));
246
247 error = radix_tree_insert_node(&uobj->uo_pages, idx, pg);
248 if (error != 0) {
249 return error;
250 }
251 if ((pg->flags & PG_CLEAN) == 0) {
252 uvm_obj_page_set_dirty(pg);
253 }
254 KASSERT(((pg->flags & PG_CLEAN) == 0) ==
255 uvm_obj_page_dirty_p(pg));
256 return 0;
257 }
258
259 /*
260 * uvm_page_remove: remove page from object.
261 *
262 * => caller must lock object
263 */
264
265 static inline void
uvm_pageremove_object(struct uvm_object * uobj,struct vm_page * pg)266 uvm_pageremove_object(struct uvm_object *uobj, struct vm_page *pg)
267 {
268
269 KASSERT(uobj == pg->uobject);
270 KASSERT(rw_write_held(uobj->vmobjlock));
271 KASSERT(pg->flags & PG_TABLED);
272
273 if ((pg->flags & PG_STAT) != 0) {
274 /* Cannot use uvm_pagegetdirty(): no longer in radix tree. */
275 const unsigned int status = pg->flags & (PG_CLEAN | PG_DIRTY);
276
277 if ((pg->flags & PG_FILE) != 0) {
278 if (uobj->uo_npages == 1) {
279 struct vnode *vp = (struct vnode *)uobj;
280 mutex_enter(vp->v_interlock);
281 KASSERT((vp->v_iflag & VI_PAGES) != 0);
282 vp->v_iflag &= ~VI_PAGES;
283 holdrelel(vp);
284 mutex_exit(vp->v_interlock);
285 }
286 if (UVM_OBJ_IS_VTEXT(uobj)) {
287 cpu_count(CPU_COUNT_EXECPAGES, -1);
288 }
289 cpu_count(CPU_COUNT_FILEUNKNOWN + status, -1);
290 } else {
291 cpu_count(CPU_COUNT_ANONUNKNOWN + status, -1);
292 }
293 }
294 uobj->uo_npages--;
295 pg->flags &= ~PG_TABLED;
296 pg->uobject = NULL;
297 }
298
299 static inline void
uvm_pageremove_tree(struct uvm_object * uobj,struct vm_page * pg)300 uvm_pageremove_tree(struct uvm_object *uobj, struct vm_page *pg)
301 {
302 struct vm_page *opg __unused;
303
304 KASSERT(rw_write_held(uobj->vmobjlock));
305
306 opg = radix_tree_remove_node(&uobj->uo_pages, pg->offset >> PAGE_SHIFT);
307 KASSERT(pg == opg);
308 }
309
310 static void
uvm_page_init_bucket(struct pgfreelist * pgfl,struct pgflbucket * pgb,int num)311 uvm_page_init_bucket(struct pgfreelist *pgfl, struct pgflbucket *pgb, int num)
312 {
313 int i;
314
315 pgb->pgb_nfree = 0;
316 for (i = 0; i < uvmexp.ncolors; i++) {
317 LIST_INIT(&pgb->pgb_colors[i]);
318 }
319 pgfl->pgfl_buckets[num] = pgb;
320 }
321
322 /*
323 * uvm_page_init: init the page system. called from uvm_init().
324 *
325 * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
326 */
327
328 void
uvm_page_init(vaddr_t * kvm_startp,vaddr_t * kvm_endp)329 uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
330 {
331 static struct uvm_cpu uvm_boot_cpu __cacheline_aligned;
332 psize_t freepages, pagecount, bucketsize, n;
333 struct pgflbucket *pgb;
334 struct vm_page *pagearray;
335 char *bucketarray;
336 uvm_physseg_t bank;
337 int fl, b;
338
339 KASSERT(ncpu <= 1);
340
341 /*
342 * init the page queues and free page queue locks, except the
343 * free list; we allocate that later (with the initial vm_page
344 * structures).
345 */
346
347 curcpu()->ci_data.cpu_uvm = &uvm_boot_cpu;
348 uvmpdpol_init();
349 for (b = 0; b < __arraycount(uvm_freelist_locks); b++) {
350 mutex_init(&uvm_freelist_locks[b].lock, MUTEX_DEFAULT, IPL_VM);
351 }
352
353 /*
354 * allocate vm_page structures.
355 */
356
357 /*
358 * sanity check:
359 * before calling this function the MD code is expected to register
360 * some free RAM with the uvm_page_physload() function. our job
361 * now is to allocate vm_page structures for this memory.
362 */
363
364 if (uvm_physseg_get_last() == UVM_PHYSSEG_TYPE_INVALID)
365 panic("uvm_page_bootstrap: no memory pre-allocated");
366
367 /*
368 * first calculate the number of free pages...
369 *
370 * note that we use start/end rather than avail_start/avail_end.
371 * this allows us to allocate extra vm_page structures in case we
372 * want to return some memory to the pool after booting.
373 */
374
375 freepages = 0;
376
377 for (bank = uvm_physseg_get_first();
378 uvm_physseg_valid_p(bank) ;
379 bank = uvm_physseg_get_next(bank)) {
380 freepages += (uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank));
381 }
382
383 /*
384 * Let MD code initialize the number of colors, or default
385 * to 1 color if MD code doesn't care.
386 */
387 if (uvmexp.ncolors == 0)
388 uvmexp.ncolors = 1;
389 uvmexp.colormask = uvmexp.ncolors - 1;
390 KASSERT((uvmexp.colormask & uvmexp.ncolors) == 0);
391
392 /* We always start with only 1 bucket. */
393 uvm.bucketcount = 1;
394
395 /*
396 * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
397 * use. for each page of memory we use we need a vm_page structure.
398 * thus, the total number of pages we can use is the total size of
399 * the memory divided by the PAGE_SIZE plus the size of the vm_page
400 * structure. we add one to freepages as a fudge factor to avoid
401 * truncation errors (since we can only allocate in terms of whole
402 * pages).
403 */
404 pagecount = ((freepages + 1) << PAGE_SHIFT) /
405 (PAGE_SIZE + sizeof(struct vm_page));
406 bucketsize = offsetof(struct pgflbucket, pgb_colors[uvmexp.ncolors]);
407 bucketsize = roundup2(bucketsize, coherency_unit);
408 bucketarray = (void *)uvm_pageboot_alloc(
409 bucketsize * VM_NFREELIST +
410 pagecount * sizeof(struct vm_page));
411 pagearray = (struct vm_page *)
412 (bucketarray + bucketsize * VM_NFREELIST);
413
414 for (fl = 0; fl < VM_NFREELIST; fl++) {
415 pgb = (struct pgflbucket *)(bucketarray + bucketsize * fl);
416 uvm_page_init_bucket(&uvm.page_free[fl], pgb, 0);
417 }
418 memset(pagearray, 0, pagecount * sizeof(struct vm_page));
419
420 /*
421 * init the freelist cache in the disabled state.
422 */
423 uvm_pgflcache_init();
424
425 /*
426 * init the vm_page structures and put them in the correct place.
427 */
428 /* First init the extent */
429
430 for (bank = uvm_physseg_get_first(),
431 uvm_physseg_seg_chomp_slab(bank, pagearray, pagecount);
432 uvm_physseg_valid_p(bank);
433 bank = uvm_physseg_get_next(bank)) {
434
435 n = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank);
436 uvm_physseg_seg_alloc_from_slab(bank, n);
437 uvm_physseg_init_seg(bank, pagearray);
438
439 /* set up page array pointers */
440 pagearray += n;
441 pagecount -= n;
442 }
443
444 /*
445 * pass up the values of virtual_space_start and
446 * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
447 * layers of the VM.
448 */
449
450 *kvm_startp = round_page(virtual_space_start);
451 *kvm_endp = trunc_page(virtual_space_end);
452
453 /*
454 * init various thresholds.
455 */
456
457 uvmexp.reserve_pagedaemon = 1;
458 uvmexp.reserve_kernel = vm_page_reserve_kernel;
459
460 /*
461 * done!
462 */
463
464 uvm.page_init_done = true;
465 }
466
467 /*
468 * uvm_pgfl_lock: lock all freelist buckets
469 */
470
471 void
uvm_pgfl_lock(void)472 uvm_pgfl_lock(void)
473 {
474 int i;
475
476 for (i = 0; i < __arraycount(uvm_freelist_locks); i++) {
477 mutex_spin_enter(&uvm_freelist_locks[i].lock);
478 }
479 }
480
481 /*
482 * uvm_pgfl_unlock: unlock all freelist buckets
483 */
484
485 void
uvm_pgfl_unlock(void)486 uvm_pgfl_unlock(void)
487 {
488 int i;
489
490 for (i = 0; i < __arraycount(uvm_freelist_locks); i++) {
491 mutex_spin_exit(&uvm_freelist_locks[i].lock);
492 }
493 }
494
495 /*
496 * uvm_setpagesize: set the page size
497 *
498 * => sets page_shift and page_mask from uvmexp.pagesize.
499 */
500
501 void
uvm_setpagesize(void)502 uvm_setpagesize(void)
503 {
504
505 /*
506 * If uvmexp.pagesize is 0 at this point, we expect PAGE_SIZE
507 * to be a constant (indicated by being a non-zero value).
508 */
509 if (uvmexp.pagesize == 0) {
510 if (PAGE_SIZE == 0)
511 panic("uvm_setpagesize: uvmexp.pagesize not set");
512 uvmexp.pagesize = PAGE_SIZE;
513 }
514 uvmexp.pagemask = uvmexp.pagesize - 1;
515 if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
516 panic("uvm_setpagesize: page size %u (%#x) not a power of two",
517 uvmexp.pagesize, uvmexp.pagesize);
518 for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
519 if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
520 break;
521 }
522
523 /*
524 * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
525 */
526
527 vaddr_t
uvm_pageboot_alloc(vsize_t size)528 uvm_pageboot_alloc(vsize_t size)
529 {
530 static bool initialized = false;
531 vaddr_t addr;
532 #if !defined(PMAP_STEAL_MEMORY)
533 vaddr_t vaddr;
534 paddr_t paddr;
535 #endif
536
537 /*
538 * on first call to this function, initialize ourselves.
539 */
540 if (initialized == false) {
541 pmap_virtual_space(&virtual_space_start, &virtual_space_end);
542
543 /* round it the way we like it */
544 virtual_space_start = round_page(virtual_space_start);
545 virtual_space_end = trunc_page(virtual_space_end);
546
547 initialized = true;
548 }
549
550 /* round to page size */
551 size = round_page(size);
552 uvmexp.bootpages += atop(size);
553
554 #if defined(PMAP_STEAL_MEMORY)
555
556 /*
557 * defer bootstrap allocation to MD code (it may want to allocate
558 * from a direct-mapped segment). pmap_steal_memory should adjust
559 * virtual_space_start/virtual_space_end if necessary.
560 */
561
562 addr = pmap_steal_memory(size, &virtual_space_start,
563 &virtual_space_end);
564
565 return addr;
566
567 #else /* !PMAP_STEAL_MEMORY */
568
569 /*
570 * allocate virtual memory for this request
571 */
572 if (virtual_space_start == virtual_space_end ||
573 (virtual_space_end - virtual_space_start) < size)
574 panic("uvm_pageboot_alloc: out of virtual space");
575
576 addr = virtual_space_start;
577
578 #ifdef PMAP_GROWKERNEL
579 /*
580 * If the kernel pmap can't map the requested space,
581 * then allocate more resources for it.
582 */
583 if (uvm_maxkaddr < (addr + size)) {
584 uvm_maxkaddr = pmap_growkernel(addr + size);
585 if (uvm_maxkaddr < (addr + size))
586 panic("uvm_pageboot_alloc: pmap_growkernel() failed");
587 }
588 #endif
589
590 virtual_space_start += size;
591
592 /*
593 * allocate and mapin physical pages to back new virtual pages
594 */
595
596 for (vaddr = round_page(addr) ; vaddr < addr + size ;
597 vaddr += PAGE_SIZE) {
598
599 if (!uvm_page_physget(&paddr))
600 panic("uvm_pageboot_alloc: out of memory");
601
602 /*
603 * Note this memory is no longer managed, so using
604 * pmap_kenter is safe.
605 */
606 pmap_kenter_pa(vaddr, paddr, VM_PROT_READ|VM_PROT_WRITE, 0);
607 }
608 pmap_update(pmap_kernel());
609 return addr;
610 #endif /* PMAP_STEAL_MEMORY */
611 }
612
613 #if !defined(PMAP_STEAL_MEMORY)
614 /*
615 * uvm_page_physget: "steal" one page from the vm_physmem structure.
616 *
617 * => attempt to allocate it off the end of a segment in which the "avail"
618 * values match the start/end values. if we can't do that, then we
619 * will advance both values (making them equal, and removing some
620 * vm_page structures from the non-avail area).
621 * => return false if out of memory.
622 */
623
624 /* subroutine: try to allocate from memory chunks on the specified freelist */
625 static bool uvm_page_physget_freelist(paddr_t *, int);
626
627 static bool
uvm_page_physget_freelist(paddr_t * paddrp,int freelist)628 uvm_page_physget_freelist(paddr_t *paddrp, int freelist)
629 {
630 uvm_physseg_t lcv;
631
632 /* pass 1: try allocating from a matching end */
633 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
634 for (lcv = uvm_physseg_get_last(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_prev(lcv))
635 #else
636 for (lcv = uvm_physseg_get_first(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_next(lcv))
637 #endif
638 {
639 if (uvm.page_init_done == true)
640 panic("uvm_page_physget: called _after_ bootstrap");
641
642 /* Try to match at front or back on unused segment */
643 if (uvm_page_physunload(lcv, freelist, paddrp))
644 return true;
645 }
646
647 /* pass2: forget about matching ends, just allocate something */
648 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
649 for (lcv = uvm_physseg_get_last(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_prev(lcv))
650 #else
651 for (lcv = uvm_physseg_get_first(); uvm_physseg_valid_p(lcv); lcv = uvm_physseg_get_next(lcv))
652 #endif
653 {
654 /* Try the front regardless. */
655 if (uvm_page_physunload_force(lcv, freelist, paddrp))
656 return true;
657 }
658 return false;
659 }
660
661 bool
uvm_page_physget(paddr_t * paddrp)662 uvm_page_physget(paddr_t *paddrp)
663 {
664 int i;
665
666 /* try in the order of freelist preference */
667 for (i = 0; i < VM_NFREELIST; i++)
668 if (uvm_page_physget_freelist(paddrp, i) == true)
669 return (true);
670 return (false);
671 }
672 #endif /* PMAP_STEAL_MEMORY */
673
674 paddr_t
uvm_vm_page_to_phys(const struct vm_page * pg)675 uvm_vm_page_to_phys(const struct vm_page *pg)
676 {
677
678 return pg->phys_addr & ~(PAGE_SIZE - 1);
679 }
680
681 /*
682 * uvm_page_numa_load: load NUMA range description.
683 */
684 void
uvm_page_numa_load(paddr_t start,paddr_t size,u_int numa_id)685 uvm_page_numa_load(paddr_t start, paddr_t size, u_int numa_id)
686 {
687 struct uvm_page_numa_region *d;
688
689 KASSERT(numa_id < PGFL_MAX_BUCKETS);
690
691 d = kmem_alloc(sizeof(*d), KM_SLEEP);
692 d->start = start;
693 d->size = size;
694 d->numa_id = numa_id;
695 d->next = uvm_page_numa_region;
696 uvm_page_numa_region = d;
697 }
698
699 /*
700 * uvm_page_numa_lookup: lookup NUMA node for the given page.
701 */
702 static u_int
uvm_page_numa_lookup(struct vm_page * pg)703 uvm_page_numa_lookup(struct vm_page *pg)
704 {
705 struct uvm_page_numa_region *d;
706 static bool warned;
707 paddr_t pa;
708
709 KASSERT(uvm_page_numa_region != NULL);
710
711 pa = VM_PAGE_TO_PHYS(pg);
712 for (d = uvm_page_numa_region; d != NULL; d = d->next) {
713 if (pa >= d->start && pa < d->start + d->size) {
714 return d->numa_id;
715 }
716 }
717
718 if (!warned) {
719 printf("uvm_page_numa_lookup: failed, first pg=%p pa=%#"
720 PRIxPADDR "\n", pg, VM_PAGE_TO_PHYS(pg));
721 warned = true;
722 }
723
724 return 0;
725 }
726
727 /*
728 * uvm_page_redim: adjust freelist dimensions if they have changed.
729 */
730
731 static void
uvm_page_redim(int newncolors,int newnbuckets)732 uvm_page_redim(int newncolors, int newnbuckets)
733 {
734 struct pgfreelist npgfl;
735 struct pgflbucket *opgb, *npgb;
736 struct pgflist *ohead, *nhead;
737 struct vm_page *pg;
738 size_t bucketsize, bucketmemsize, oldbucketmemsize;
739 int fl, ob, oc, nb, nc, obuckets, ocolors;
740 char *bucketarray, *oldbucketmem, *bucketmem;
741
742 KASSERT(((newncolors - 1) & newncolors) == 0);
743
744 /* Anything to do? */
745 if (newncolors <= uvmexp.ncolors &&
746 newnbuckets == uvm.bucketcount) {
747 return;
748 }
749 if (uvm.page_init_done == false) {
750 uvmexp.ncolors = newncolors;
751 return;
752 }
753
754 bucketsize = offsetof(struct pgflbucket, pgb_colors[newncolors]);
755 bucketsize = roundup2(bucketsize, coherency_unit);
756 bucketmemsize = bucketsize * newnbuckets * VM_NFREELIST +
757 coherency_unit - 1;
758 bucketmem = kmem_zalloc(bucketmemsize, KM_SLEEP);
759 bucketarray = (char *)roundup2((uintptr_t)bucketmem, coherency_unit);
760
761 ocolors = uvmexp.ncolors;
762 obuckets = uvm.bucketcount;
763
764 /* Freelist cache mustn't be enabled. */
765 uvm_pgflcache_pause();
766
767 /* Make sure we should still do this. */
768 uvm_pgfl_lock();
769 if (newncolors <= uvmexp.ncolors &&
770 newnbuckets == uvm.bucketcount) {
771 uvm_pgfl_unlock();
772 uvm_pgflcache_resume();
773 kmem_free(bucketmem, bucketmemsize);
774 return;
775 }
776
777 uvmexp.ncolors = newncolors;
778 uvmexp.colormask = uvmexp.ncolors - 1;
779 uvm.bucketcount = newnbuckets;
780
781 for (fl = 0; fl < VM_NFREELIST; fl++) {
782 /* Init new buckets in new freelist. */
783 memset(&npgfl, 0, sizeof(npgfl));
784 for (nb = 0; nb < newnbuckets; nb++) {
785 npgb = (struct pgflbucket *)bucketarray;
786 uvm_page_init_bucket(&npgfl, npgb, nb);
787 bucketarray += bucketsize;
788 }
789 /* Now transfer pages from the old freelist. */
790 for (nb = ob = 0; ob < obuckets; ob++) {
791 opgb = uvm.page_free[fl].pgfl_buckets[ob];
792 for (oc = 0; oc < ocolors; oc++) {
793 ohead = &opgb->pgb_colors[oc];
794 while ((pg = LIST_FIRST(ohead)) != NULL) {
795 LIST_REMOVE(pg, pageq.list);
796 /*
797 * Here we decide on the NEW color &
798 * bucket for the page. For NUMA
799 * we'll use the info that the
800 * hardware gave us. For non-NUMA
801 * assign take physical page frame
802 * number and cache color into
803 * account. We do this to try and
804 * avoid defeating any memory
805 * interleaving in the hardware.
806 */
807 KASSERT(
808 uvm_page_get_bucket(pg) == ob);
809 KASSERT(fl ==
810 uvm_page_get_freelist(pg));
811 if (uvm_page_numa_region != NULL) {
812 nb = uvm_page_numa_lookup(pg);
813 } else {
814 nb = atop(VM_PAGE_TO_PHYS(pg))
815 / uvmexp.ncolors / 8
816 % newnbuckets;
817 }
818 uvm_page_set_bucket(pg, nb);
819 npgb = npgfl.pgfl_buckets[nb];
820 npgb->pgb_nfree++;
821 nc = VM_PGCOLOR(pg);
822 nhead = &npgb->pgb_colors[nc];
823 LIST_INSERT_HEAD(nhead, pg, pageq.list);
824 }
825 }
826 }
827 /* Install the new freelist. */
828 memcpy(&uvm.page_free[fl], &npgfl, sizeof(npgfl));
829 }
830
831 /* Unlock and free the old memory. */
832 oldbucketmemsize = recolored_pages_memsize;
833 oldbucketmem = recolored_pages_mem;
834 recolored_pages_memsize = bucketmemsize;
835 recolored_pages_mem = bucketmem;
836
837 uvm_pgfl_unlock();
838 uvm_pgflcache_resume();
839
840 if (oldbucketmemsize) {
841 kmem_free(oldbucketmem, oldbucketmemsize);
842 }
843
844 /*
845 * this calls uvm_km_alloc() which may want to hold
846 * uvm_freelist_lock.
847 */
848 uvm_pager_realloc_emerg();
849 }
850
851 /*
852 * uvm_page_recolor: Recolor the pages if the new color count is
853 * larger than the old one.
854 */
855
856 void
uvm_page_recolor(int newncolors)857 uvm_page_recolor(int newncolors)
858 {
859
860 uvm_page_redim(newncolors, uvm.bucketcount);
861 }
862
863 /*
864 * uvm_page_rebucket: Determine a bucket structure and redim the free
865 * lists to match.
866 */
867
868 void
uvm_page_rebucket(void)869 uvm_page_rebucket(void)
870 {
871 u_int min_numa, max_numa, npackage, shift;
872 struct cpu_info *ci, *ci2, *ci3;
873 CPU_INFO_ITERATOR cii;
874
875 /*
876 * If we have more than one NUMA node, and the maximum NUMA node ID
877 * is less than PGFL_MAX_BUCKETS, then we'll use NUMA distribution
878 * for free pages.
879 */
880 min_numa = (u_int)-1;
881 max_numa = 0;
882 for (CPU_INFO_FOREACH(cii, ci)) {
883 if (ci->ci_numa_id < min_numa) {
884 min_numa = ci->ci_numa_id;
885 }
886 if (ci->ci_numa_id > max_numa) {
887 max_numa = ci->ci_numa_id;
888 }
889 }
890 if (min_numa != max_numa && max_numa < PGFL_MAX_BUCKETS) {
891 aprint_debug("UVM: using NUMA allocation scheme\n");
892 for (CPU_INFO_FOREACH(cii, ci)) {
893 ci->ci_data.cpu_uvm->pgflbucket = ci->ci_numa_id;
894 }
895 uvm_page_redim(uvmexp.ncolors, max_numa + 1);
896 return;
897 }
898
899 /*
900 * Otherwise we'll go with a scheme to maximise L2/L3 cache locality
901 * and minimise lock contention. Count the total number of CPU
902 * packages, and then try to distribute the buckets among CPU
903 * packages evenly.
904 */
905 npackage = curcpu()->ci_nsibling[CPUREL_PACKAGE1ST];
906
907 /*
908 * Figure out how to arrange the packages & buckets, and the total
909 * number of buckets we need. XXX 2 may not be the best factor.
910 */
911 for (shift = 0; npackage > PGFL_MAX_BUCKETS; shift++) {
912 npackage >>= 1;
913 }
914 uvm_page_redim(uvmexp.ncolors, npackage);
915
916 /*
917 * Now tell each CPU which bucket to use. In the outer loop, scroll
918 * through all CPU packages.
919 */
920 npackage = 0;
921 ci = curcpu();
922 ci2 = ci->ci_sibling[CPUREL_PACKAGE1ST];
923 do {
924 /*
925 * In the inner loop, scroll through all CPUs in the package
926 * and assign the same bucket ID.
927 */
928 ci3 = ci2;
929 do {
930 ci3->ci_data.cpu_uvm->pgflbucket = npackage >> shift;
931 ci3 = ci3->ci_sibling[CPUREL_PACKAGE];
932 } while (ci3 != ci2);
933 npackage++;
934 ci2 = ci2->ci_sibling[CPUREL_PACKAGE1ST];
935 } while (ci2 != ci->ci_sibling[CPUREL_PACKAGE1ST]);
936
937 aprint_debug("UVM: using package allocation scheme, "
938 "%d package(s) per bucket\n", 1 << shift);
939 }
940
941 /*
942 * uvm_cpu_attach: initialize per-CPU data structures.
943 */
944
945 void
uvm_cpu_attach(struct cpu_info * ci)946 uvm_cpu_attach(struct cpu_info *ci)
947 {
948 struct uvm_cpu *ucpu;
949
950 /* Already done in uvm_page_init(). */
951 if (!CPU_IS_PRIMARY(ci)) {
952 /* Add more reserve pages for this CPU. */
953 uvmexp.reserve_kernel += vm_page_reserve_kernel;
954
955 /* Allocate per-CPU data structures. */
956 ucpu = kmem_zalloc(sizeof(struct uvm_cpu) + coherency_unit - 1,
957 KM_SLEEP);
958 ucpu = (struct uvm_cpu *)roundup2((uintptr_t)ucpu,
959 coherency_unit);
960 ci->ci_data.cpu_uvm = ucpu;
961 } else {
962 ucpu = ci->ci_data.cpu_uvm;
963 }
964
965 uvmpdpol_init_cpu(ucpu);
966 }
967
968 /*
969 * uvm_availmem: fetch the total amount of free memory in pages. this can
970 * have a detrimental effect on performance due to false sharing; don't call
971 * unless needed.
972 *
973 * some users can request the amount of free memory so often that it begins
974 * to impact upon performance. if calling frequently and an inexact value
975 * is okay, call with cached = true.
976 */
977
978 int
uvm_availmem(bool cached)979 uvm_availmem(bool cached)
980 {
981 int64_t fp;
982
983 cpu_count_sync(cached);
984 if ((fp = cpu_count_get(CPU_COUNT_FREEPAGES)) < 0) {
985 /*
986 * XXXAD could briefly go negative because it's impossible
987 * to get a clean snapshot. address this for other counters
988 * used as running totals before NetBSD 10 although less
989 * important for those.
990 */
991 fp = 0;
992 }
993 return (int)fp;
994 }
995
996 /*
997 * uvm_pagealloc_pgb: helper routine that tries to allocate any color from a
998 * specific freelist and specific bucket only.
999 *
1000 * => must be at IPL_VM or higher to protect per-CPU data structures.
1001 */
1002
1003 static struct vm_page *
uvm_pagealloc_pgb(struct uvm_cpu * ucpu,int f,int b,int * trycolorp,int flags)1004 uvm_pagealloc_pgb(struct uvm_cpu *ucpu, int f, int b, int *trycolorp, int flags)
1005 {
1006 int c, trycolor, colormask;
1007 struct pgflbucket *pgb;
1008 struct vm_page *pg;
1009 kmutex_t *lock;
1010 bool fill;
1011
1012 /*
1013 * Skip the bucket if empty, no lock needed. There could be many
1014 * empty freelists/buckets.
1015 */
1016 pgb = uvm.page_free[f].pgfl_buckets[b];
1017 if (pgb->pgb_nfree == 0) {
1018 return NULL;
1019 }
1020
1021 /* Skip bucket if low on memory. */
1022 lock = &uvm_freelist_locks[b].lock;
1023 mutex_spin_enter(lock);
1024 if (__predict_false(pgb->pgb_nfree <= uvmexp.reserve_kernel)) {
1025 if ((flags & UVM_PGA_USERESERVE) == 0 ||
1026 (pgb->pgb_nfree <= uvmexp.reserve_pagedaemon &&
1027 curlwp != uvm.pagedaemon_lwp)) {
1028 mutex_spin_exit(lock);
1029 return NULL;
1030 }
1031 fill = false;
1032 } else {
1033 fill = true;
1034 }
1035
1036 /* Try all page colors as needed. */
1037 c = trycolor = *trycolorp;
1038 colormask = uvmexp.colormask;
1039 do {
1040 pg = LIST_FIRST(&pgb->pgb_colors[c]);
1041 if (__predict_true(pg != NULL)) {
1042 /*
1043 * Got a free page! PG_FREE must be cleared under
1044 * lock because of uvm_pglistalloc().
1045 */
1046 LIST_REMOVE(pg, pageq.list);
1047 KASSERT(pg->flags == PG_FREE);
1048 pg->flags = PG_BUSY | PG_CLEAN | PG_FAKE;
1049 pgb->pgb_nfree--;
1050 CPU_COUNT(CPU_COUNT_FREEPAGES, -1);
1051
1052 /*
1053 * While we have the bucket locked and our data
1054 * structures fresh in L1 cache, we have an ideal
1055 * opportunity to grab some pages for the freelist
1056 * cache without causing extra contention. Only do
1057 * so if we found pages in this CPU's preferred
1058 * bucket.
1059 */
1060 if (__predict_true(b == ucpu->pgflbucket && fill)) {
1061 uvm_pgflcache_fill(ucpu, f, b, c);
1062 }
1063 mutex_spin_exit(lock);
1064 KASSERT(uvm_page_get_bucket(pg) == b);
1065 CPU_COUNT(c == trycolor ?
1066 CPU_COUNT_COLORHIT : CPU_COUNT_COLORMISS, 1);
1067 CPU_COUNT(CPU_COUNT_CPUMISS, 1);
1068 *trycolorp = c;
1069 return pg;
1070 }
1071 c = (c + 1) & colormask;
1072 } while (c != trycolor);
1073 mutex_spin_exit(lock);
1074
1075 return NULL;
1076 }
1077
1078 /*
1079 * uvm_pagealloc_pgfl: helper routine for uvm_pagealloc_strat that allocates
1080 * any color from any bucket, in a specific freelist.
1081 *
1082 * => must be at IPL_VM or higher to protect per-CPU data structures.
1083 */
1084
1085 static struct vm_page *
uvm_pagealloc_pgfl(struct uvm_cpu * ucpu,int f,int * trycolorp,int flags)1086 uvm_pagealloc_pgfl(struct uvm_cpu *ucpu, int f, int *trycolorp, int flags)
1087 {
1088 int b, trybucket, bucketcount;
1089 struct vm_page *pg;
1090
1091 /* Try for the exact thing in the per-CPU cache. */
1092 if ((pg = uvm_pgflcache_alloc(ucpu, f, *trycolorp)) != NULL) {
1093 CPU_COUNT(CPU_COUNT_CPUHIT, 1);
1094 CPU_COUNT(CPU_COUNT_COLORHIT, 1);
1095 return pg;
1096 }
1097
1098 /* Walk through all buckets, trying our preferred bucket first. */
1099 trybucket = ucpu->pgflbucket;
1100 b = trybucket;
1101 bucketcount = uvm.bucketcount;
1102 do {
1103 pg = uvm_pagealloc_pgb(ucpu, f, b, trycolorp, flags);
1104 if (pg != NULL) {
1105 return pg;
1106 }
1107 b = (b + 1 == bucketcount ? 0 : b + 1);
1108 } while (b != trybucket);
1109
1110 return NULL;
1111 }
1112
1113 /*
1114 * uvm_pagealloc_strat: allocate vm_page from a particular free list.
1115 *
1116 * => return null if no pages free
1117 * => wake up pagedaemon if number of free pages drops below low water mark
1118 * => if obj != NULL, obj must be locked (to put in obj's tree)
1119 * => if anon != NULL, anon must be locked (to put in anon)
1120 * => only one of obj or anon can be non-null
1121 * => caller must activate/deactivate page if it is not wired.
1122 * => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
1123 * => policy decision: it is more important to pull a page off of the
1124 * appropriate priority free list than it is to get a page from the
1125 * correct bucket or color bin. This is because we live with the
1126 * consequences of a bad free list decision for the entire
1127 * lifetime of the page, e.g. if the page comes from memory that
1128 * is slower to access.
1129 */
1130
1131 struct vm_page *
uvm_pagealloc_strat(struct uvm_object * obj,voff_t off,struct vm_anon * anon,int flags,int strat,int free_list)1132 uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
1133 int flags, int strat, int free_list)
1134 {
1135 int color, lcv, error, s;
1136 struct uvm_cpu *ucpu;
1137 struct vm_page *pg;
1138 lwp_t *l;
1139
1140 KASSERT(obj == NULL || anon == NULL);
1141 KASSERT(anon == NULL || (flags & UVM_FLAG_COLORMATCH) || off == 0);
1142 KASSERT(off == trunc_page(off));
1143 KASSERT(obj == NULL || rw_write_held(obj->vmobjlock));
1144 KASSERT(anon == NULL || anon->an_lock == NULL ||
1145 rw_write_held(anon->an_lock));
1146
1147 /*
1148 * This implements a global round-robin page coloring
1149 * algorithm.
1150 */
1151
1152 s = splvm();
1153 ucpu = curcpu()->ci_data.cpu_uvm;
1154 if (flags & UVM_FLAG_COLORMATCH) {
1155 color = atop(off) & uvmexp.colormask;
1156 } else {
1157 color = ucpu->pgflcolor;
1158 }
1159
1160 /*
1161 * fail if any of these conditions is true:
1162 * [1] there really are no free pages, or
1163 * [2] only kernel "reserved" pages remain and
1164 * reserved pages have not been requested.
1165 * [3] only pagedaemon "reserved" pages remain and
1166 * the requestor isn't the pagedaemon.
1167 * we make kernel reserve pages available if called by a
1168 * kernel thread.
1169 */
1170 l = curlwp;
1171 if (__predict_true(l != NULL) && (l->l_flag & LW_SYSTEM) != 0) {
1172 flags |= UVM_PGA_USERESERVE;
1173 }
1174
1175 again:
1176 switch (strat) {
1177 case UVM_PGA_STRAT_NORMAL:
1178 /* Check freelists: descending priority (ascending id) order. */
1179 for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
1180 pg = uvm_pagealloc_pgfl(ucpu, lcv, &color, flags);
1181 if (pg != NULL) {
1182 goto gotit;
1183 }
1184 }
1185
1186 /* No pages free! Have pagedaemon free some memory. */
1187 splx(s);
1188 uvm_kick_pdaemon();
1189 return NULL;
1190
1191 case UVM_PGA_STRAT_ONLY:
1192 case UVM_PGA_STRAT_FALLBACK:
1193 /* Attempt to allocate from the specified free list. */
1194 KASSERT(free_list >= 0);
1195 KASSERT(free_list < VM_NFREELIST);
1196 pg = uvm_pagealloc_pgfl(ucpu, free_list, &color, flags);
1197 if (pg != NULL) {
1198 goto gotit;
1199 }
1200
1201 /* Fall back, if possible. */
1202 if (strat == UVM_PGA_STRAT_FALLBACK) {
1203 strat = UVM_PGA_STRAT_NORMAL;
1204 goto again;
1205 }
1206
1207 /* No pages free! Have pagedaemon free some memory. */
1208 splx(s);
1209 uvm_kick_pdaemon();
1210 return NULL;
1211
1212 case UVM_PGA_STRAT_NUMA:
1213 /*
1214 * NUMA strategy (experimental): allocating from the correct
1215 * bucket is more important than observing freelist
1216 * priority. Look only to the current NUMA node; if that
1217 * fails, we need to look to other NUMA nodes, so retry with
1218 * the normal strategy.
1219 */
1220 for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
1221 pg = uvm_pgflcache_alloc(ucpu, lcv, color);
1222 if (pg != NULL) {
1223 CPU_COUNT(CPU_COUNT_CPUHIT, 1);
1224 CPU_COUNT(CPU_COUNT_COLORHIT, 1);
1225 goto gotit;
1226 }
1227 pg = uvm_pagealloc_pgb(ucpu, lcv,
1228 ucpu->pgflbucket, &color, flags);
1229 if (pg != NULL) {
1230 goto gotit;
1231 }
1232 }
1233 strat = UVM_PGA_STRAT_NORMAL;
1234 goto again;
1235
1236 default:
1237 panic("uvm_pagealloc_strat: bad strat %d", strat);
1238 /* NOTREACHED */
1239 }
1240
1241 gotit:
1242 /*
1243 * We now know which color we actually allocated from; set
1244 * the next color accordingly.
1245 */
1246
1247 ucpu->pgflcolor = (color + 1) & uvmexp.colormask;
1248
1249 /*
1250 * while still at IPL_VM, update allocation statistics.
1251 */
1252
1253 if (anon) {
1254 CPU_COUNT(CPU_COUNT_ANONCLEAN, 1);
1255 }
1256 splx(s);
1257 KASSERT(pg->flags == (PG_BUSY|PG_CLEAN|PG_FAKE));
1258
1259 /*
1260 * assign the page to the object. as the page was free, we know
1261 * that pg->uobject and pg->uanon are NULL. we only need to take
1262 * the page's interlock if we are changing the values.
1263 */
1264 if (anon != NULL || obj != NULL) {
1265 mutex_enter(&pg->interlock);
1266 }
1267 pg->offset = off;
1268 pg->uobject = obj;
1269 pg->uanon = anon;
1270 KASSERT(uvm_page_owner_locked_p(pg, true));
1271 if (anon) {
1272 anon->an_page = pg;
1273 pg->flags |= PG_ANON;
1274 mutex_exit(&pg->interlock);
1275 } else if (obj) {
1276 /*
1277 * set PG_FILE|PG_AOBJ before the first uvm_pageinsert.
1278 */
1279 if (UVM_OBJ_IS_VNODE(obj)) {
1280 pg->flags |= PG_FILE;
1281 } else if (UVM_OBJ_IS_AOBJ(obj)) {
1282 pg->flags |= PG_AOBJ;
1283 }
1284 uvm_pageinsert_object(obj, pg);
1285 mutex_exit(&pg->interlock);
1286 error = uvm_pageinsert_tree(obj, pg);
1287 if (error != 0) {
1288 mutex_enter(&pg->interlock);
1289 uvm_pageremove_object(obj, pg);
1290 mutex_exit(&pg->interlock);
1291 uvm_pagefree(pg);
1292 return NULL;
1293 }
1294 }
1295
1296 #if defined(UVM_PAGE_TRKOWN)
1297 pg->owner_tag = NULL;
1298 #endif
1299 UVM_PAGE_OWN(pg, "new alloc");
1300
1301 if (flags & UVM_PGA_ZERO) {
1302 /* A zero'd page is not clean. */
1303 if (obj != NULL || anon != NULL) {
1304 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
1305 }
1306 pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1307 }
1308
1309 return(pg);
1310 }
1311
1312 /*
1313 * uvm_pagereplace: replace a page with another
1314 *
1315 * => object must be locked
1316 * => page interlocks must be held
1317 */
1318
1319 void
uvm_pagereplace(struct vm_page * oldpg,struct vm_page * newpg)1320 uvm_pagereplace(struct vm_page *oldpg, struct vm_page *newpg)
1321 {
1322 struct uvm_object *uobj = oldpg->uobject;
1323 struct vm_page *pg __diagused;
1324 uint64_t idx;
1325
1326 KASSERT((oldpg->flags & PG_TABLED) != 0);
1327 KASSERT(uobj != NULL);
1328 KASSERT((newpg->flags & PG_TABLED) == 0);
1329 KASSERT(newpg->uobject == NULL);
1330 KASSERT(rw_write_held(uobj->vmobjlock));
1331 KASSERT(mutex_owned(&oldpg->interlock));
1332 KASSERT(mutex_owned(&newpg->interlock));
1333
1334 newpg->uobject = uobj;
1335 newpg->offset = oldpg->offset;
1336 idx = newpg->offset >> PAGE_SHIFT;
1337 pg = radix_tree_replace_node(&uobj->uo_pages, idx, newpg);
1338 KASSERT(pg == oldpg);
1339 if (((oldpg->flags ^ newpg->flags) & PG_CLEAN) != 0) {
1340 if ((newpg->flags & PG_CLEAN) != 0) {
1341 uvm_obj_page_clear_dirty(newpg);
1342 } else {
1343 uvm_obj_page_set_dirty(newpg);
1344 }
1345 }
1346 /*
1347 * oldpg's PG_STAT is stable. newpg is not reachable by others yet.
1348 */
1349 newpg->flags |=
1350 (newpg->flags & ~PG_STAT) | (oldpg->flags & PG_STAT);
1351 uvm_pageinsert_object(uobj, newpg);
1352 uvm_pageremove_object(uobj, oldpg);
1353 }
1354
1355 /*
1356 * uvm_pagerealloc: reallocate a page from one object to another
1357 *
1358 * => both objects must be locked
1359 */
1360
1361 int
uvm_pagerealloc(struct vm_page * pg,struct uvm_object * newobj,voff_t newoff)1362 uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff)
1363 {
1364 int error = 0;
1365
1366 /*
1367 * remove it from the old object
1368 */
1369
1370 if (pg->uobject) {
1371 uvm_pageremove_tree(pg->uobject, pg);
1372 uvm_pageremove_object(pg->uobject, pg);
1373 }
1374
1375 /*
1376 * put it in the new object
1377 */
1378
1379 if (newobj) {
1380 mutex_enter(&pg->interlock);
1381 pg->uobject = newobj;
1382 pg->offset = newoff;
1383 if (UVM_OBJ_IS_VNODE(newobj)) {
1384 pg->flags |= PG_FILE;
1385 } else if (UVM_OBJ_IS_AOBJ(newobj)) {
1386 pg->flags |= PG_AOBJ;
1387 }
1388 uvm_pageinsert_object(newobj, pg);
1389 mutex_exit(&pg->interlock);
1390 error = uvm_pageinsert_tree(newobj, pg);
1391 if (error != 0) {
1392 mutex_enter(&pg->interlock);
1393 uvm_pageremove_object(newobj, pg);
1394 mutex_exit(&pg->interlock);
1395 }
1396 }
1397
1398 return error;
1399 }
1400
1401 /*
1402 * uvm_pagefree: free page
1403 *
1404 * => erase page's identity (i.e. remove from object)
1405 * => put page on free list
1406 * => caller must lock owning object (either anon or uvm_object)
1407 * => assumes all valid mappings of pg are gone
1408 */
1409
1410 void
uvm_pagefree(struct vm_page * pg)1411 uvm_pagefree(struct vm_page *pg)
1412 {
1413 struct pgfreelist *pgfl;
1414 struct pgflbucket *pgb;
1415 struct uvm_cpu *ucpu;
1416 kmutex_t *lock;
1417 int bucket, s;
1418 bool locked;
1419
1420 #ifdef DEBUG
1421 if (pg->uobject == (void *)0xdeadbeef &&
1422 pg->uanon == (void *)0xdeadbeef) {
1423 panic("uvm_pagefree: freeing free page %p", pg);
1424 }
1425 #endif /* DEBUG */
1426
1427 KASSERT((pg->flags & PG_PAGEOUT) == 0);
1428 KASSERT(!(pg->flags & PG_FREE));
1429 KASSERT(pg->uobject == NULL || rw_write_held(pg->uobject->vmobjlock));
1430 KASSERT(pg->uobject != NULL || pg->uanon == NULL ||
1431 rw_write_held(pg->uanon->an_lock));
1432
1433 /*
1434 * remove the page from the object's tree before acquiring any page
1435 * interlocks: this can acquire locks to free radixtree nodes.
1436 */
1437 if (pg->uobject != NULL) {
1438 uvm_pageremove_tree(pg->uobject, pg);
1439 }
1440
1441 /*
1442 * if the page is loaned, resolve the loan instead of freeing.
1443 */
1444
1445 if (pg->loan_count) {
1446 KASSERT(pg->wire_count == 0);
1447
1448 /*
1449 * if the page is owned by an anon then we just want to
1450 * drop anon ownership. the kernel will free the page when
1451 * it is done with it. if the page is owned by an object,
1452 * remove it from the object and mark it dirty for the benefit
1453 * of possible anon owners.
1454 *
1455 * regardless of previous ownership, wakeup any waiters,
1456 * unbusy the page, and we're done.
1457 */
1458
1459 uvm_pagelock(pg);
1460 locked = true;
1461 if (pg->uobject != NULL) {
1462 uvm_pageremove_object(pg->uobject, pg);
1463 pg->flags &= ~(PG_FILE|PG_AOBJ);
1464 } else if (pg->uanon != NULL) {
1465 if ((pg->flags & PG_ANON) == 0) {
1466 pg->loan_count--;
1467 } else {
1468 const unsigned status = uvm_pagegetdirty(pg);
1469 pg->flags &= ~PG_ANON;
1470 cpu_count(CPU_COUNT_ANONUNKNOWN + status, -1);
1471 }
1472 pg->uanon->an_page = NULL;
1473 pg->uanon = NULL;
1474 }
1475 if (pg->pqflags & PQ_WANTED) {
1476 wakeup(pg);
1477 }
1478 pg->pqflags &= ~PQ_WANTED;
1479 pg->flags &= ~(PG_BUSY|PG_RELEASED|PG_PAGER1);
1480 #ifdef UVM_PAGE_TRKOWN
1481 pg->owner_tag = NULL;
1482 #endif
1483 KASSERT((pg->flags & PG_STAT) == 0);
1484 if (pg->loan_count) {
1485 KASSERT(pg->uobject == NULL);
1486 if (pg->uanon == NULL) {
1487 uvm_pagedequeue(pg);
1488 }
1489 uvm_pageunlock(pg);
1490 return;
1491 }
1492 } else if (pg->uobject != NULL || pg->uanon != NULL ||
1493 pg->wire_count != 0) {
1494 uvm_pagelock(pg);
1495 locked = true;
1496 } else {
1497 locked = false;
1498 }
1499
1500 /*
1501 * remove page from its object or anon.
1502 */
1503 if (pg->uobject != NULL) {
1504 uvm_pageremove_object(pg->uobject, pg);
1505 } else if (pg->uanon != NULL) {
1506 const unsigned int status = uvm_pagegetdirty(pg);
1507 pg->uanon->an_page = NULL;
1508 pg->uanon = NULL;
1509 cpu_count(CPU_COUNT_ANONUNKNOWN + status, -1);
1510 }
1511
1512 /*
1513 * if the page was wired, unwire it now.
1514 */
1515
1516 if (pg->wire_count) {
1517 pg->wire_count = 0;
1518 atomic_dec_uint(&uvmexp.wired);
1519 }
1520 if (locked) {
1521 /*
1522 * wake anyone waiting on the page.
1523 */
1524 if ((pg->pqflags & PQ_WANTED) != 0) {
1525 pg->pqflags &= ~PQ_WANTED;
1526 wakeup(pg);
1527 }
1528
1529 /*
1530 * now remove the page from the queues.
1531 */
1532 uvm_pagedequeue(pg);
1533 uvm_pageunlock(pg);
1534 } else {
1535 KASSERT(!uvmpdpol_pageisqueued_p(pg));
1536 }
1537
1538 /*
1539 * and put on free queue
1540 */
1541
1542 #ifdef DEBUG
1543 pg->uobject = (void *)0xdeadbeef;
1544 pg->uanon = (void *)0xdeadbeef;
1545 #endif /* DEBUG */
1546
1547 /* Try to send the page to the per-CPU cache. */
1548 s = splvm();
1549 ucpu = curcpu()->ci_data.cpu_uvm;
1550 bucket = uvm_page_get_bucket(pg);
1551 if (bucket == ucpu->pgflbucket && uvm_pgflcache_free(ucpu, pg)) {
1552 splx(s);
1553 return;
1554 }
1555
1556 /* Didn't work. Never mind, send it to a global bucket. */
1557 pgfl = &uvm.page_free[uvm_page_get_freelist(pg)];
1558 pgb = pgfl->pgfl_buckets[bucket];
1559 lock = &uvm_freelist_locks[bucket].lock;
1560
1561 mutex_spin_enter(lock);
1562 /* PG_FREE must be set under lock because of uvm_pglistalloc(). */
1563 pg->flags = PG_FREE;
1564 LIST_INSERT_HEAD(&pgb->pgb_colors[VM_PGCOLOR(pg)], pg, pageq.list);
1565 pgb->pgb_nfree++;
1566 CPU_COUNT(CPU_COUNT_FREEPAGES, 1);
1567 mutex_spin_exit(lock);
1568 splx(s);
1569 }
1570
1571 /*
1572 * uvm_page_unbusy: unbusy an array of pages.
1573 *
1574 * => pages must either all belong to the same object, or all belong to anons.
1575 * => if pages are object-owned, object must be locked.
1576 * => if pages are anon-owned, anons must be locked.
1577 * => caller must make sure that anon-owned pages are not PG_RELEASED.
1578 */
1579
1580 void
uvm_page_unbusy(struct vm_page ** pgs,int npgs)1581 uvm_page_unbusy(struct vm_page **pgs, int npgs)
1582 {
1583 struct vm_page *pg;
1584 int i, pageout_done;
1585 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1586
1587 pageout_done = 0;
1588 for (i = 0; i < npgs; i++) {
1589 pg = pgs[i];
1590 if (pg == NULL || pg == PGO_DONTCARE) {
1591 continue;
1592 }
1593
1594 KASSERT(uvm_page_owner_locked_p(pg, true));
1595 KASSERT(pg->flags & PG_BUSY);
1596
1597 if (pg->flags & PG_PAGEOUT) {
1598 pg->flags &= ~PG_PAGEOUT;
1599 pg->flags |= PG_RELEASED;
1600 pageout_done++;
1601 atomic_inc_uint(&uvmexp.pdfreed);
1602 }
1603 if (pg->flags & PG_RELEASED) {
1604 UVMHIST_LOG(ubchist, "releasing pg %#jx",
1605 (uintptr_t)pg, 0, 0, 0);
1606 KASSERT(pg->uobject != NULL ||
1607 (pg->uanon != NULL && pg->uanon->an_ref > 0));
1608 pg->flags &= ~PG_RELEASED;
1609 uvm_pagefree(pg);
1610 } else {
1611 UVMHIST_LOG(ubchist, "unbusying pg %#jx",
1612 (uintptr_t)pg, 0, 0, 0);
1613 KASSERT((pg->flags & PG_FAKE) == 0);
1614 pg->flags &= ~PG_BUSY;
1615 uvm_pagelock(pg);
1616 uvm_pagewakeup(pg);
1617 uvm_pageunlock(pg);
1618 UVM_PAGE_OWN(pg, NULL);
1619 }
1620 }
1621 if (pageout_done != 0) {
1622 uvm_pageout_done(pageout_done);
1623 }
1624 }
1625
1626 /*
1627 * uvm_pagewait: wait for a busy page
1628 *
1629 * => page must be known PG_BUSY
1630 * => object must be read or write locked
1631 * => object will be unlocked on return
1632 */
1633
1634 void
uvm_pagewait(struct vm_page * pg,krwlock_t * lock,const char * wmesg)1635 uvm_pagewait(struct vm_page *pg, krwlock_t *lock, const char *wmesg)
1636 {
1637
1638 KASSERT(rw_lock_held(lock));
1639 KASSERT((pg->flags & PG_BUSY) != 0);
1640 KASSERT(uvm_page_owner_locked_p(pg, false));
1641
1642 mutex_enter(&pg->interlock);
1643 pg->pqflags |= PQ_WANTED;
1644 rw_exit(lock);
1645 UVM_UNLOCK_AND_WAIT(pg, &pg->interlock, false, wmesg, 0);
1646 }
1647
1648 /*
1649 * uvm_pagewakeup: wake anyone waiting on a page
1650 *
1651 * => page interlock must be held
1652 */
1653
1654 void
uvm_pagewakeup(struct vm_page * pg)1655 uvm_pagewakeup(struct vm_page *pg)
1656 {
1657 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1658
1659 KASSERT(mutex_owned(&pg->interlock));
1660
1661 UVMHIST_LOG(ubchist, "waking pg %#jx", (uintptr_t)pg, 0, 0, 0);
1662
1663 if ((pg->pqflags & PQ_WANTED) != 0) {
1664 wakeup(pg);
1665 pg->pqflags &= ~PQ_WANTED;
1666 }
1667 }
1668
1669 /*
1670 * uvm_pagewanted_p: return true if someone is waiting on the page
1671 *
1672 * => object must be write locked (lock out all concurrent access)
1673 */
1674
1675 bool
uvm_pagewanted_p(struct vm_page * pg)1676 uvm_pagewanted_p(struct vm_page *pg)
1677 {
1678
1679 KASSERT(uvm_page_owner_locked_p(pg, true));
1680
1681 return (atomic_load_relaxed(&pg->pqflags) & PQ_WANTED) != 0;
1682 }
1683
1684 #if defined(UVM_PAGE_TRKOWN)
1685 /*
1686 * uvm_page_own: set or release page ownership
1687 *
1688 * => this is a debugging function that keeps track of who sets PG_BUSY
1689 * and where they do it. it can be used to track down problems
1690 * such a process setting "PG_BUSY" and never releasing it.
1691 * => page's object [if any] must be locked
1692 * => if "tag" is NULL then we are releasing page ownership
1693 */
1694 void
uvm_page_own(struct vm_page * pg,const char * tag)1695 uvm_page_own(struct vm_page *pg, const char *tag)
1696 {
1697
1698 KASSERT((pg->flags & (PG_PAGEOUT|PG_RELEASED)) == 0);
1699 KASSERT(uvm_page_owner_locked_p(pg, true));
1700
1701 /* gain ownership? */
1702 if (tag) {
1703 KASSERT((pg->flags & PG_BUSY) != 0);
1704 if (pg->owner_tag) {
1705 printf("uvm_page_own: page %p already owned "
1706 "by proc %d.%d [%s]\n", pg,
1707 pg->owner, pg->lowner, pg->owner_tag);
1708 panic("uvm_page_own");
1709 }
1710 pg->owner = curproc->p_pid;
1711 pg->lowner = curlwp->l_lid;
1712 pg->owner_tag = tag;
1713 return;
1714 }
1715
1716 /* drop ownership */
1717 KASSERT((pg->flags & PG_BUSY) == 0);
1718 if (pg->owner_tag == NULL) {
1719 printf("uvm_page_own: dropping ownership of an non-owned "
1720 "page (%p)\n", pg);
1721 panic("uvm_page_own");
1722 }
1723 pg->owner_tag = NULL;
1724 }
1725 #endif
1726
1727 /*
1728 * uvm_pagelookup: look up a page
1729 *
1730 * => caller should lock object to keep someone from pulling the page
1731 * out from under it
1732 */
1733
1734 struct vm_page *
uvm_pagelookup(struct uvm_object * obj,voff_t off)1735 uvm_pagelookup(struct uvm_object *obj, voff_t off)
1736 {
1737 struct vm_page *pg;
1738
1739 KASSERT(db_active || rw_lock_held(obj->vmobjlock));
1740
1741 pg = radix_tree_lookup_node(&obj->uo_pages, off >> PAGE_SHIFT);
1742
1743 KASSERT(pg == NULL || obj->uo_npages != 0);
1744 KASSERT(pg == NULL || (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
1745 (pg->flags & PG_BUSY) != 0);
1746 return pg;
1747 }
1748
1749 /*
1750 * uvm_pagewire: wire the page, thus removing it from the daemon's grasp
1751 *
1752 * => caller must lock objects
1753 * => caller must hold pg->interlock
1754 */
1755
1756 void
uvm_pagewire(struct vm_page * pg)1757 uvm_pagewire(struct vm_page *pg)
1758 {
1759
1760 KASSERT(uvm_page_owner_locked_p(pg, true));
1761 KASSERT(mutex_owned(&pg->interlock));
1762 #if defined(READAHEAD_STATS)
1763 if ((pg->flags & PG_READAHEAD) != 0) {
1764 uvm_ra_hit.ev_count++;
1765 pg->flags &= ~PG_READAHEAD;
1766 }
1767 #endif /* defined(READAHEAD_STATS) */
1768 if (pg->wire_count == 0) {
1769 uvm_pagedequeue(pg);
1770 atomic_inc_uint(&uvmexp.wired);
1771 }
1772 pg->wire_count++;
1773 KASSERT(pg->wire_count > 0); /* detect wraparound */
1774 }
1775
1776 /*
1777 * uvm_pageunwire: unwire the page.
1778 *
1779 * => activate if wire count goes to zero.
1780 * => caller must lock objects
1781 * => caller must hold pg->interlock
1782 */
1783
1784 void
uvm_pageunwire(struct vm_page * pg)1785 uvm_pageunwire(struct vm_page *pg)
1786 {
1787
1788 KASSERT(uvm_page_owner_locked_p(pg, true));
1789 KASSERT(pg->wire_count != 0);
1790 KASSERT(!uvmpdpol_pageisqueued_p(pg));
1791 KASSERT(mutex_owned(&pg->interlock));
1792 pg->wire_count--;
1793 if (pg->wire_count == 0) {
1794 uvm_pageactivate(pg);
1795 KASSERT(uvmexp.wired != 0);
1796 atomic_dec_uint(&uvmexp.wired);
1797 }
1798 }
1799
1800 /*
1801 * uvm_pagedeactivate: deactivate page
1802 *
1803 * => caller must lock objects
1804 * => caller must check to make sure page is not wired
1805 * => object that page belongs to must be locked (so we can adjust pg->flags)
1806 * => caller must clear the reference on the page before calling
1807 * => caller must hold pg->interlock
1808 */
1809
1810 void
uvm_pagedeactivate(struct vm_page * pg)1811 uvm_pagedeactivate(struct vm_page *pg)
1812 {
1813
1814 KASSERT(uvm_page_owner_locked_p(pg, false));
1815 KASSERT(mutex_owned(&pg->interlock));
1816 if (pg->wire_count == 0) {
1817 KASSERT(uvmpdpol_pageisqueued_p(pg));
1818 uvmpdpol_pagedeactivate(pg);
1819 }
1820 }
1821
1822 /*
1823 * uvm_pageactivate: activate page
1824 *
1825 * => caller must lock objects
1826 * => caller must hold pg->interlock
1827 */
1828
1829 void
uvm_pageactivate(struct vm_page * pg)1830 uvm_pageactivate(struct vm_page *pg)
1831 {
1832
1833 KASSERT(uvm_page_owner_locked_p(pg, false));
1834 KASSERT(mutex_owned(&pg->interlock));
1835 #if defined(READAHEAD_STATS)
1836 if ((pg->flags & PG_READAHEAD) != 0) {
1837 uvm_ra_hit.ev_count++;
1838 pg->flags &= ~PG_READAHEAD;
1839 }
1840 #endif /* defined(READAHEAD_STATS) */
1841 if (pg->wire_count == 0) {
1842 uvmpdpol_pageactivate(pg);
1843 }
1844 }
1845
1846 /*
1847 * uvm_pagedequeue: remove a page from any paging queue
1848 *
1849 * => caller must lock objects
1850 * => caller must hold pg->interlock
1851 */
1852 void
uvm_pagedequeue(struct vm_page * pg)1853 uvm_pagedequeue(struct vm_page *pg)
1854 {
1855
1856 KASSERT(uvm_page_owner_locked_p(pg, true));
1857 KASSERT(mutex_owned(&pg->interlock));
1858 if (uvmpdpol_pageisqueued_p(pg)) {
1859 uvmpdpol_pagedequeue(pg);
1860 }
1861 }
1862
1863 /*
1864 * uvm_pageenqueue: add a page to a paging queue without activating.
1865 * used where a page is not really demanded (yet). eg. read-ahead
1866 *
1867 * => caller must lock objects
1868 * => caller must hold pg->interlock
1869 */
1870 void
uvm_pageenqueue(struct vm_page * pg)1871 uvm_pageenqueue(struct vm_page *pg)
1872 {
1873
1874 KASSERT(uvm_page_owner_locked_p(pg, false));
1875 KASSERT(mutex_owned(&pg->interlock));
1876 if (pg->wire_count == 0 && !uvmpdpol_pageisqueued_p(pg)) {
1877 uvmpdpol_pageenqueue(pg);
1878 }
1879 }
1880
1881 /*
1882 * uvm_pagelock: acquire page interlock
1883 */
1884 void
uvm_pagelock(struct vm_page * pg)1885 uvm_pagelock(struct vm_page *pg)
1886 {
1887
1888 mutex_enter(&pg->interlock);
1889 }
1890
1891 /*
1892 * uvm_pagelock2: acquire two page interlocks
1893 */
1894 void
uvm_pagelock2(struct vm_page * pg1,struct vm_page * pg2)1895 uvm_pagelock2(struct vm_page *pg1, struct vm_page *pg2)
1896 {
1897
1898 if (pg1 < pg2) {
1899 mutex_enter(&pg1->interlock);
1900 mutex_enter(&pg2->interlock);
1901 } else {
1902 mutex_enter(&pg2->interlock);
1903 mutex_enter(&pg1->interlock);
1904 }
1905 }
1906
1907 /*
1908 * uvm_pageunlock: release page interlock, and if a page replacement intent
1909 * is set on the page, pass it to uvmpdpol to make real.
1910 *
1911 * => caller must hold pg->interlock
1912 */
1913 void
uvm_pageunlock(struct vm_page * pg)1914 uvm_pageunlock(struct vm_page *pg)
1915 {
1916
1917 if ((pg->pqflags & PQ_INTENT_SET) == 0 ||
1918 (pg->pqflags & PQ_INTENT_QUEUED) != 0) {
1919 mutex_exit(&pg->interlock);
1920 return;
1921 }
1922 pg->pqflags |= PQ_INTENT_QUEUED;
1923 mutex_exit(&pg->interlock);
1924 uvmpdpol_pagerealize(pg);
1925 }
1926
1927 /*
1928 * uvm_pageunlock2: release two page interlocks, and for both pages if a
1929 * page replacement intent is set on the page, pass it to uvmpdpol to make
1930 * real.
1931 *
1932 * => caller must hold pg->interlock
1933 */
1934 void
uvm_pageunlock2(struct vm_page * pg1,struct vm_page * pg2)1935 uvm_pageunlock2(struct vm_page *pg1, struct vm_page *pg2)
1936 {
1937
1938 if ((pg1->pqflags & PQ_INTENT_SET) == 0 ||
1939 (pg1->pqflags & PQ_INTENT_QUEUED) != 0) {
1940 mutex_exit(&pg1->interlock);
1941 pg1 = NULL;
1942 } else {
1943 pg1->pqflags |= PQ_INTENT_QUEUED;
1944 mutex_exit(&pg1->interlock);
1945 }
1946
1947 if ((pg2->pqflags & PQ_INTENT_SET) == 0 ||
1948 (pg2->pqflags & PQ_INTENT_QUEUED) != 0) {
1949 mutex_exit(&pg2->interlock);
1950 pg2 = NULL;
1951 } else {
1952 pg2->pqflags |= PQ_INTENT_QUEUED;
1953 mutex_exit(&pg2->interlock);
1954 }
1955
1956 if (pg1 != NULL) {
1957 uvmpdpol_pagerealize(pg1);
1958 }
1959 if (pg2 != NULL) {
1960 uvmpdpol_pagerealize(pg2);
1961 }
1962 }
1963
1964 /*
1965 * uvm_pagezero: zero fill a page
1966 *
1967 * => if page is part of an object then the object should be locked
1968 * to protect pg->flags.
1969 */
1970
1971 void
uvm_pagezero(struct vm_page * pg)1972 uvm_pagezero(struct vm_page *pg)
1973 {
1974
1975 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
1976 pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1977 }
1978
1979 /*
1980 * uvm_pagecopy: copy a page
1981 *
1982 * => if page is part of an object then the object should be locked
1983 * to protect pg->flags.
1984 */
1985
1986 void
uvm_pagecopy(struct vm_page * src,struct vm_page * dst)1987 uvm_pagecopy(struct vm_page *src, struct vm_page *dst)
1988 {
1989
1990 uvm_pagemarkdirty(dst, UVM_PAGE_STATUS_DIRTY);
1991 pmap_copy_page(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst));
1992 }
1993
1994 /*
1995 * uvm_pageismanaged: test it see that a page (specified by PA) is managed.
1996 */
1997
1998 bool
uvm_pageismanaged(paddr_t pa)1999 uvm_pageismanaged(paddr_t pa)
2000 {
2001
2002 return (uvm_physseg_find(atop(pa), NULL) != UVM_PHYSSEG_TYPE_INVALID);
2003 }
2004
2005 /*
2006 * uvm_page_lookup_freelist: look up the free list for the specified page
2007 */
2008
2009 int
uvm_page_lookup_freelist(struct vm_page * pg)2010 uvm_page_lookup_freelist(struct vm_page *pg)
2011 {
2012 uvm_physseg_t upm;
2013
2014 upm = uvm_physseg_find(atop(VM_PAGE_TO_PHYS(pg)), NULL);
2015 KASSERT(upm != UVM_PHYSSEG_TYPE_INVALID);
2016 return uvm_physseg_get_free_list(upm);
2017 }
2018
2019 /*
2020 * uvm_page_owner_locked_p: return true if object associated with page is
2021 * locked. this is a weak check for runtime assertions only.
2022 */
2023
2024 bool
uvm_page_owner_locked_p(struct vm_page * pg,bool exclusive)2025 uvm_page_owner_locked_p(struct vm_page *pg, bool exclusive)
2026 {
2027
2028 if (pg->uobject != NULL) {
2029 return exclusive
2030 ? rw_write_held(pg->uobject->vmobjlock)
2031 : rw_lock_held(pg->uobject->vmobjlock);
2032 }
2033 if (pg->uanon != NULL) {
2034 return exclusive
2035 ? rw_write_held(pg->uanon->an_lock)
2036 : rw_lock_held(pg->uanon->an_lock);
2037 }
2038 return true;
2039 }
2040
2041 /*
2042 * uvm_pagereadonly_p: return if the page should be mapped read-only
2043 */
2044
2045 bool
uvm_pagereadonly_p(struct vm_page * pg)2046 uvm_pagereadonly_p(struct vm_page *pg)
2047 {
2048 struct uvm_object * const uobj = pg->uobject;
2049
2050 KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock));
2051 KASSERT(uobj != NULL || rw_lock_held(pg->uanon->an_lock));
2052 if ((pg->flags & PG_RDONLY) != 0) {
2053 return true;
2054 }
2055 if (uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) {
2056 return true;
2057 }
2058 if (uobj == NULL) {
2059 return false;
2060 }
2061 return UVM_OBJ_NEEDS_WRITEFAULT(uobj);
2062 }
2063
2064 #ifdef PMAP_DIRECT
2065 /*
2066 * Call pmap to translate physical address into a virtual and to run a callback
2067 * for it. Used to avoid actually mapping the pages, pmap most likely uses direct map
2068 * or equivalent.
2069 */
2070 int
uvm_direct_process(struct vm_page ** pgs,u_int npages,voff_t off,vsize_t len,int (* process)(void *,size_t,void *),void * arg)2071 uvm_direct_process(struct vm_page **pgs, u_int npages, voff_t off, vsize_t len,
2072 int (*process)(void *, size_t, void *), void *arg)
2073 {
2074 int error = 0;
2075 paddr_t pa;
2076 size_t todo;
2077 voff_t pgoff = (off & PAGE_MASK);
2078 struct vm_page *pg;
2079
2080 KASSERT(npages > 0);
2081 KASSERT(len > 0);
2082
2083 for (int i = 0; i < npages; i++) {
2084 pg = pgs[i];
2085
2086 KASSERT(len > 0);
2087
2088 /*
2089 * Caller is responsible for ensuring all the pages are
2090 * available.
2091 */
2092 KASSERT(pg != NULL);
2093 KASSERT(pg != PGO_DONTCARE);
2094
2095 pa = VM_PAGE_TO_PHYS(pg);
2096 todo = MIN(len, PAGE_SIZE - pgoff);
2097
2098 error = pmap_direct_process(pa, pgoff, todo, process, arg);
2099 if (error)
2100 break;
2101
2102 pgoff = 0;
2103 len -= todo;
2104 }
2105
2106 KASSERTMSG(error != 0 || len == 0, "len %lu != 0 for non-error", len);
2107 return error;
2108 }
2109 #endif /* PMAP_DIRECT */
2110
2111 #if defined(DDB) || defined(DEBUGPRINT)
2112
2113 /*
2114 * uvm_page_printit: actually print the page
2115 */
2116
2117 static const char page_flagbits[] = UVM_PGFLAGBITS;
2118 static const char page_pqflagbits[] = UVM_PQFLAGBITS;
2119
2120 void
uvm_page_printit(struct vm_page * pg,bool full,void (* pr)(const char *,...))2121 uvm_page_printit(struct vm_page *pg, bool full,
2122 void (*pr)(const char *, ...))
2123 {
2124 struct vm_page *tpg;
2125 struct uvm_object *uobj;
2126 struct pgflbucket *pgb;
2127 struct pgflist *pgl;
2128 char pgbuf[128];
2129
2130 (*pr)("PAGE %p:\n", pg);
2131 snprintb(pgbuf, sizeof(pgbuf), page_flagbits, pg->flags);
2132 (*pr)(" flags=%s\n", pgbuf);
2133 snprintb(pgbuf, sizeof(pgbuf), page_pqflagbits, pg->pqflags);
2134 (*pr)(" pqflags=%s\n", pgbuf);
2135 (*pr)(" uobject=%p, uanon=%p, offset=0x%llx\n",
2136 pg->uobject, pg->uanon, (long long)pg->offset);
2137 (*pr)(" loan_count=%d wire_count=%d bucket=%d freelist=%d\n",
2138 pg->loan_count, pg->wire_count, uvm_page_get_bucket(pg),
2139 uvm_page_get_freelist(pg));
2140 (*pr)(" pa=0x%lx\n", (long)VM_PAGE_TO_PHYS(pg));
2141 #if defined(UVM_PAGE_TRKOWN)
2142 if (pg->flags & PG_BUSY)
2143 (*pr)(" owning process = %d.%d, tag=%s\n",
2144 pg->owner, pg->lowner, pg->owner_tag);
2145 else
2146 (*pr)(" page not busy, no owner\n");
2147 #else
2148 (*pr)(" [page ownership tracking disabled]\n");
2149 #endif
2150
2151 if (!full)
2152 return;
2153
2154 /* cross-verify object/anon */
2155 if ((pg->flags & PG_FREE) == 0) {
2156 if (pg->flags & PG_ANON) {
2157 if (pg->uanon == NULL || pg->uanon->an_page != pg)
2158 (*pr)(" >>> ANON DOES NOT POINT HERE <<< (%p)\n",
2159 (pg->uanon) ? pg->uanon->an_page : NULL);
2160 else
2161 (*pr)(" anon backpointer is OK\n");
2162 } else {
2163 uobj = pg->uobject;
2164 if (uobj) {
2165 (*pr)(" checking object list\n");
2166 tpg = uvm_pagelookup(uobj, pg->offset);
2167 if (tpg)
2168 (*pr)(" page found on object list\n");
2169 else
2170 (*pr)(" >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n");
2171 }
2172 }
2173 }
2174
2175 /* cross-verify page queue */
2176 if (pg->flags & PG_FREE) {
2177 int fl = uvm_page_get_freelist(pg);
2178 int b = uvm_page_get_bucket(pg);
2179 pgb = uvm.page_free[fl].pgfl_buckets[b];
2180 pgl = &pgb->pgb_colors[VM_PGCOLOR(pg)];
2181 (*pr)(" checking pageq list\n");
2182 LIST_FOREACH(tpg, pgl, pageq.list) {
2183 if (tpg == pg) {
2184 break;
2185 }
2186 }
2187 if (tpg)
2188 (*pr)(" page found on pageq list\n");
2189 else
2190 (*pr)(" >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n");
2191 }
2192 }
2193
2194 /*
2195 * uvm_page_printall - print a summary of all managed pages
2196 */
2197
2198 void
uvm_page_printall(void (* pr)(const char *,...))2199 uvm_page_printall(void (*pr)(const char *, ...))
2200 {
2201 uvm_physseg_t i;
2202 paddr_t pfn;
2203 struct vm_page *pg;
2204
2205 (*pr)("%18s %4s %4s %18s %18s"
2206 #ifdef UVM_PAGE_TRKOWN
2207 " OWNER"
2208 #endif
2209 "\n", "PAGE", "FLAG", "PQ", "UOBJECT", "UANON");
2210 for (i = uvm_physseg_get_first();
2211 uvm_physseg_valid_p(i);
2212 i = uvm_physseg_get_next(i)) {
2213 for (pfn = uvm_physseg_get_start(i);
2214 pfn < uvm_physseg_get_end(i);
2215 pfn++) {
2216 pg = PHYS_TO_VM_PAGE(ptoa(pfn));
2217
2218 (*pr)("%18p %04x %08x %18p %18p",
2219 pg, pg->flags, pg->pqflags, pg->uobject,
2220 pg->uanon);
2221 #ifdef UVM_PAGE_TRKOWN
2222 if (pg->flags & PG_BUSY)
2223 (*pr)(" %d [%s]", pg->owner, pg->owner_tag);
2224 #endif
2225 (*pr)("\n");
2226 }
2227 }
2228 }
2229
2230 /*
2231 * uvm_page_print_freelists - print a summary freelists
2232 */
2233
2234 void
uvm_page_print_freelists(void (* pr)(const char *,...))2235 uvm_page_print_freelists(void (*pr)(const char *, ...))
2236 {
2237 struct pgfreelist *pgfl;
2238 struct pgflbucket *pgb;
2239 int fl, b, c;
2240
2241 (*pr)("There are %d freelists with %d buckets of %d colors.\n\n",
2242 VM_NFREELIST, uvm.bucketcount, uvmexp.ncolors);
2243
2244 for (fl = 0; fl < VM_NFREELIST; fl++) {
2245 pgfl = &uvm.page_free[fl];
2246 (*pr)("freelist(%d) @ %p\n", fl, pgfl);
2247 for (b = 0; b < uvm.bucketcount; b++) {
2248 pgb = uvm.page_free[fl].pgfl_buckets[b];
2249 (*pr)(" bucket(%d) @ %p, nfree = %d, lock @ %p:\n",
2250 b, pgb, pgb->pgb_nfree,
2251 &uvm_freelist_locks[b].lock);
2252 for (c = 0; c < uvmexp.ncolors; c++) {
2253 (*pr)(" color(%d) @ %p, ", c,
2254 &pgb->pgb_colors[c]);
2255 (*pr)("first page = %p\n",
2256 LIST_FIRST(&pgb->pgb_colors[c]));
2257 }
2258 }
2259 }
2260 }
2261
2262 #endif /* DDB || DEBUGPRINT */
2263