1 /* $NetBSD: uvm_fault.c,v 1.237 2024/03/15 07:09:37 andvar Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
28 */
29
30 /*
31 * uvm_fault.c: fault handler
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.237 2024/03/15 07:09:37 andvar Exp $");
36
37 #include "opt_uvmhist.h"
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/atomic.h>
42 #include <sys/kernel.h>
43 #include <sys/mman.h>
44
45 #include <uvm/uvm.h>
46 #include <uvm/uvm_pdpolicy.h>
47 #include <uvm/uvm_rndsource.h>
48
49 /*
50 *
51 * a word on page faults:
52 *
53 * types of page faults we handle:
54 *
55 * CASE 1: upper layer faults CASE 2: lower layer faults
56 *
57 * CASE 1A CASE 1B CASE 2A CASE 2B
58 * read/write1 write>1 read/write +-cow_write/zero
59 * | | | |
60 * +--|--+ +--|--+ +-----+ + | + | +-----+
61 * amap | V | | ---------> new | | | | ^ |
62 * +-----+ +-----+ +-----+ + | + | +--|--+
63 * | | |
64 * +-----+ +-----+ +--|--+ | +--|--+
65 * uobj | d/c | | d/c | | V | +----+ |
66 * +-----+ +-----+ +-----+ +-----+
67 *
68 * d/c = don't care
69 *
70 * case [0]: layerless fault
71 * no amap or uobj is present. this is an error.
72 *
73 * case [1]: upper layer fault [anon active]
74 * 1A: [read] or [write with anon->an_ref == 1]
75 * I/O takes place in upper level anon and uobj is not touched.
76 * 1B: [write with anon->an_ref > 1]
77 * new anon is alloc'd and data is copied off ["COW"]
78 *
79 * case [2]: lower layer fault [uobj]
80 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
81 * I/O takes place directly in object.
82 * 2B: [write to copy_on_write] or [read on NULL uobj]
83 * data is "promoted" from uobj to a new anon.
84 * if uobj is null, then we zero fill.
85 *
86 * we follow the standard UVM locking protocol ordering:
87 *
88 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
89 * we hold a PG_BUSY page if we unlock for I/O
90 *
91 *
92 * the code is structured as follows:
93 *
94 * - init the "IN" params in the ufi structure
95 * ReFault: (ERESTART returned to the loop in uvm_fault_internal)
96 * - do lookups [locks maps], check protection, handle needs_copy
97 * - check for case 0 fault (error)
98 * - establish "range" of fault
99 * - if we have an amap lock it and extract the anons
100 * - if sequential advice deactivate pages behind us
101 * - at the same time check pmap for unmapped areas and anon for pages
102 * that we could map in (and do map it if found)
103 * - check object for resident pages that we could map in
104 * - if (case 2) goto Case2
105 * - >>> handle case 1
106 * - ensure source anon is resident in RAM
107 * - if case 1B alloc new anon and copy from source
108 * - map the correct page in
109 * Case2:
110 * - >>> handle case 2
111 * - ensure source page is resident (if uobj)
112 * - if case 2B alloc new anon and copy from source (could be zero
113 * fill if uobj == NULL)
114 * - map the correct page in
115 * - done!
116 *
117 * note on paging:
118 * if we have to do I/O we place a PG_BUSY page in the correct object,
119 * unlock everything, and do the I/O. when I/O is done we must reverify
120 * the state of the world before assuming that our data structures are
121 * valid. [because mappings could change while the map is unlocked]
122 *
123 * alternative 1: unbusy the page in question and restart the page fault
124 * from the top (ReFault). this is easy but does not take advantage
125 * of the information that we already have from our previous lookup,
126 * although it is possible that the "hints" in the vm_map will help here.
127 *
128 * alternative 2: the system already keeps track of a "version" number of
129 * a map. [i.e. every time you write-lock a map (e.g. to change a
130 * mapping) you bump the version number up by one...] so, we can save
131 * the version number of the map before we release the lock and start I/O.
132 * then when I/O is done we can relock and check the version numbers
133 * to see if anything changed. this might save us some over 1 because
134 * we don't have to unbusy the page and may be less compares(?).
135 *
136 * alternative 3: put in backpointers or a way to "hold" part of a map
137 * in place while I/O is in progress. this could be complex to
138 * implement (especially with structures like amap that can be referenced
139 * by multiple map entries, and figuring out what should wait could be
140 * complex as well...).
141 *
142 * we use alternative 2. given that we are multi-threaded now we may want
143 * to reconsider the choice.
144 */
145
146 /*
147 * local data structures
148 */
149
150 struct uvm_advice {
151 int advice;
152 int nback;
153 int nforw;
154 };
155
156 /*
157 * page range array:
158 * note: index in array must match "advice" value
159 * XXX: borrowed numbers from freebsd. do they work well for us?
160 */
161
162 static const struct uvm_advice uvmadvice[] = {
163 { UVM_ADV_NORMAL, 3, 4 },
164 { UVM_ADV_RANDOM, 0, 0 },
165 { UVM_ADV_SEQUENTIAL, 8, 7},
166 };
167
168 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */
169
170 /*
171 * private prototypes
172 */
173
174 /*
175 * inline functions
176 */
177
178 /*
179 * uvmfault_anonflush: try and deactivate pages in specified anons
180 *
181 * => does not have to deactivate page if it is busy
182 */
183
184 static inline void
uvmfault_anonflush(struct vm_anon ** anons,int n)185 uvmfault_anonflush(struct vm_anon **anons, int n)
186 {
187 int lcv;
188 struct vm_page *pg;
189
190 for (lcv = 0; lcv < n; lcv++) {
191 if (anons[lcv] == NULL)
192 continue;
193 KASSERT(rw_lock_held(anons[lcv]->an_lock));
194 pg = anons[lcv]->an_page;
195 if (pg && (pg->flags & PG_BUSY) == 0) {
196 uvm_pagelock(pg);
197 uvm_pagedeactivate(pg);
198 uvm_pageunlock(pg);
199 }
200 }
201 }
202
203 /*
204 * normal functions
205 */
206
207 /*
208 * uvmfault_amapcopy: clear "needs_copy" in a map.
209 *
210 * => called with VM data structures unlocked (usually, see below)
211 * => we get a write lock on the maps and clear needs_copy for a VA
212 * => if we are out of RAM we sleep (waiting for more)
213 */
214
215 static void
uvmfault_amapcopy(struct uvm_faultinfo * ufi)216 uvmfault_amapcopy(struct uvm_faultinfo *ufi)
217 {
218 for (;;) {
219
220 /*
221 * no mapping? give up.
222 */
223
224 if (uvmfault_lookup(ufi, true) == false)
225 return;
226
227 /*
228 * copy if needed.
229 */
230
231 if (UVM_ET_ISNEEDSCOPY(ufi->entry))
232 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
233 ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
234
235 /*
236 * didn't work? must be out of RAM. unlock and sleep.
237 */
238
239 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
240 uvmfault_unlockmaps(ufi, true);
241 uvm_wait("fltamapcopy");
242 continue;
243 }
244
245 /*
246 * got it! unlock and return.
247 */
248
249 uvmfault_unlockmaps(ufi, true);
250 return;
251 }
252 /*NOTREACHED*/
253 }
254
255 /*
256 * uvmfault_anonget: get data in an anon into a non-busy, non-released
257 * page in that anon.
258 *
259 * => Map, amap and thus anon should be locked by caller.
260 * => If we fail, we unlock everything and error is returned.
261 * => If we are successful, return with everything still locked.
262 * => We do not move the page on the queues [gets moved later]. If we
263 * allocate a new page [we_own], it gets put on the queues. Either way,
264 * the result is that the page is on the queues at return time
265 * => For pages which are on loan from a uvm_object (and thus are not owned
266 * by the anon): if successful, return with the owning object locked.
267 * The caller must unlock this object when it unlocks everything else.
268 */
269
270 int
uvmfault_anonget(struct uvm_faultinfo * ufi,struct vm_amap * amap,struct vm_anon * anon)271 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
272 struct vm_anon *anon)
273 {
274 struct vm_page *pg;
275 krw_t lock_type;
276 int error __unused; /* used for VMSWAP */
277
278 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
279 KASSERT(rw_lock_held(anon->an_lock));
280 KASSERT(anon->an_lock == amap->am_lock);
281
282 /* Increment the counters.*/
283 cpu_count(CPU_COUNT_FLTANGET, 1);
284 if (anon->an_page) {
285 curlwp->l_ru.ru_minflt++;
286 } else {
287 curlwp->l_ru.ru_majflt++;
288 }
289 error = 0;
290
291 /*
292 * Loop until we get the anon data, or fail.
293 */
294
295 for (;;) {
296 bool we_own, locked;
297 /*
298 * Note: 'we_own' will become true if we set PG_BUSY on a page.
299 */
300 we_own = false;
301 pg = anon->an_page;
302
303 /*
304 * If there is a resident page and it is loaned, then anon
305 * may not own it. Call out to uvm_anon_lockloanpg() to
306 * identify and lock the real owner of the page.
307 */
308
309 if (pg && pg->loan_count)
310 pg = uvm_anon_lockloanpg(anon);
311
312 /*
313 * Is page resident? Make sure it is not busy/released.
314 */
315
316 lock_type = rw_lock_op(anon->an_lock);
317 if (pg) {
318
319 /*
320 * at this point, if the page has a uobject [meaning
321 * we have it on loan], then that uobject is locked
322 * by us! if the page is busy, we drop all the
323 * locks (including uobject) and try again.
324 */
325
326 if ((pg->flags & PG_BUSY) == 0) {
327 UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
328 return 0;
329 }
330 cpu_count(CPU_COUNT_FLTPGWAIT, 1);
331
332 /*
333 * The last unlock must be an atomic unlock and wait
334 * on the owner of page.
335 */
336
337 if (pg->uobject) {
338 /* Owner of page is UVM object. */
339 uvmfault_unlockall(ufi, amap, NULL);
340 UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
341 0,0,0);
342 uvm_pagewait(pg, pg->uobject->vmobjlock, "anonget1");
343 } else {
344 /* Owner of page is anon. */
345 uvmfault_unlockall(ufi, NULL, NULL);
346 UVMHIST_LOG(maphist, " unlock+wait on anon",0,
347 0,0,0);
348 uvm_pagewait(pg, anon->an_lock, "anonget2");
349 }
350 } else {
351 #if defined(VMSWAP)
352 /*
353 * No page, therefore allocate one. A write lock is
354 * required for this. If the caller didn't supply
355 * one, fail now and have them retry.
356 */
357
358 if (lock_type == RW_READER) {
359 return ENOLCK;
360 }
361 pg = uvm_pagealloc(NULL,
362 ufi != NULL ? ufi->orig_rvaddr : 0,
363 anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0);
364 if (pg == NULL) {
365 /* Out of memory. Wait a little. */
366 uvmfault_unlockall(ufi, amap, NULL);
367 cpu_count(CPU_COUNT_FLTNORAM, 1);
368 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
369 0,0,0);
370 if (!uvm_reclaimable()) {
371 return ENOMEM;
372 }
373 uvm_wait("flt_noram1");
374 } else {
375 /* PG_BUSY bit is set. */
376 we_own = true;
377 uvmfault_unlockall(ufi, amap, NULL);
378
379 /*
380 * Pass a PG_BUSY+PG_FAKE clean page into
381 * the uvm_swap_get() function with all data
382 * structures unlocked. Note that it is OK
383 * to read an_swslot here, because we hold
384 * PG_BUSY on the page.
385 */
386 cpu_count(CPU_COUNT_PAGEINS, 1);
387 error = uvm_swap_get(pg, anon->an_swslot,
388 PGO_SYNCIO);
389
390 /*
391 * We clean up after the I/O below in the
392 * 'we_own' case.
393 */
394 }
395 #else
396 panic("%s: no page", __func__);
397 #endif /* defined(VMSWAP) */
398 }
399
400 /*
401 * Re-lock the map and anon.
402 */
403
404 locked = uvmfault_relock(ufi);
405 if (locked || we_own) {
406 rw_enter(anon->an_lock, lock_type);
407 }
408
409 /*
410 * If we own the page (i.e. we set PG_BUSY), then we need
411 * to clean up after the I/O. There are three cases to
412 * consider:
413 *
414 * 1) Page was released during I/O: free anon and ReFault.
415 * 2) I/O not OK. Free the page and cause the fault to fail.
416 * 3) I/O OK! Activate the page and sync with the non-we_own
417 * case (i.e. drop anon lock if not locked).
418 */
419
420 if (we_own) {
421 KASSERT(lock_type == RW_WRITER);
422 #if defined(VMSWAP)
423 if (error) {
424
425 /*
426 * Remove the swap slot from the anon and
427 * mark the anon as having no real slot.
428 * Do not free the swap slot, thus preventing
429 * it from being used again.
430 */
431
432 if (anon->an_swslot > 0) {
433 uvm_swap_markbad(anon->an_swslot, 1);
434 }
435 anon->an_swslot = SWSLOT_BAD;
436
437 if ((pg->flags & PG_RELEASED) != 0) {
438 goto released;
439 }
440
441 /*
442 * Note: page was never !PG_BUSY, so it
443 * cannot be mapped and thus no need to
444 * pmap_page_protect() it.
445 */
446
447 uvm_pagefree(pg);
448
449 if (locked) {
450 uvmfault_unlockall(ufi, NULL, NULL);
451 }
452 rw_exit(anon->an_lock);
453 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
454 return error;
455 }
456
457 if ((pg->flags & PG_RELEASED) != 0) {
458 released:
459 KASSERT(anon->an_ref == 0);
460
461 /*
462 * Released while we had unlocked amap.
463 */
464
465 if (locked) {
466 uvmfault_unlockall(ufi, NULL, NULL);
467 }
468 uvm_anon_release(anon);
469
470 if (error) {
471 UVMHIST_LOG(maphist,
472 "<- ERROR/RELEASED", 0,0,0,0);
473 return error;
474 }
475
476 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
477 return ERESTART;
478 }
479
480 /*
481 * We have successfully read the page, activate it.
482 */
483
484 uvm_pagelock(pg);
485 uvm_pageactivate(pg);
486 uvm_pagewakeup(pg);
487 uvm_pageunlock(pg);
488 pg->flags &= ~(PG_BUSY|PG_FAKE);
489 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN);
490 UVM_PAGE_OWN(pg, NULL);
491 #else
492 panic("%s: we_own", __func__);
493 #endif /* defined(VMSWAP) */
494 }
495
496 /*
497 * We were not able to re-lock the map - restart the fault.
498 */
499
500 if (!locked) {
501 if (we_own) {
502 rw_exit(anon->an_lock);
503 }
504 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
505 return ERESTART;
506 }
507
508 /*
509 * Verify that no one has touched the amap and moved
510 * the anon on us.
511 */
512
513 if (ufi != NULL && amap_lookup(&ufi->entry->aref,
514 ufi->orig_rvaddr - ufi->entry->start) != anon) {
515
516 uvmfault_unlockall(ufi, amap, NULL);
517 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
518 return ERESTART;
519 }
520
521 /*
522 * Retry..
523 */
524
525 cpu_count(CPU_COUNT_FLTANRETRY, 1);
526 continue;
527 }
528 /*NOTREACHED*/
529 }
530
531 /*
532 * uvmfault_promote: promote data to a new anon. used for 1B and 2B.
533 *
534 * 1. allocate an anon and a page.
535 * 2. fill its contents.
536 * 3. put it into amap.
537 *
538 * => if we fail (result != 0) we unlock everything.
539 * => on success, return a new locked anon via 'nanon'.
540 * (*nanon)->an_page will be a resident, locked, dirty page.
541 * => it's caller's responsibility to put the promoted nanon->an_page to the
542 * page queue.
543 */
544
545 static int
uvmfault_promote(struct uvm_faultinfo * ufi,struct vm_anon * oanon,struct vm_page * uobjpage,struct vm_anon ** nanon,struct vm_anon ** spare)546 uvmfault_promote(struct uvm_faultinfo *ufi,
547 struct vm_anon *oanon,
548 struct vm_page *uobjpage,
549 struct vm_anon **nanon, /* OUT: allocated anon */
550 struct vm_anon **spare)
551 {
552 struct vm_amap *amap = ufi->entry->aref.ar_amap;
553 struct uvm_object *uobj;
554 struct vm_anon *anon;
555 struct vm_page *pg;
556 struct vm_page *opg;
557 int error;
558 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
559
560 if (oanon) {
561 /* anon COW */
562 opg = oanon->an_page;
563 KASSERT(opg != NULL);
564 KASSERT(opg->uobject == NULL || opg->loan_count > 0);
565 } else if (uobjpage != PGO_DONTCARE) {
566 /* object-backed COW */
567 opg = uobjpage;
568 KASSERT(rw_lock_held(opg->uobject->vmobjlock));
569 } else {
570 /* ZFOD */
571 opg = NULL;
572 }
573 if (opg != NULL) {
574 uobj = opg->uobject;
575 } else {
576 uobj = NULL;
577 }
578
579 KASSERT(amap != NULL);
580 KASSERT(uobjpage != NULL);
581 KASSERT(rw_write_held(amap->am_lock));
582 KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock);
583 KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock));
584
585 if (*spare != NULL) {
586 anon = *spare;
587 *spare = NULL;
588 } else {
589 anon = uvm_analloc();
590 }
591 if (anon) {
592
593 /*
594 * The new anon is locked.
595 *
596 * if opg == NULL, we want a zero'd, dirty page,
597 * so have uvm_pagealloc() do that for us.
598 */
599
600 KASSERT(anon->an_lock == NULL);
601 anon->an_lock = amap->am_lock;
602 pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon,
603 UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0));
604 if (pg == NULL) {
605 anon->an_lock = NULL;
606 }
607 } else {
608 pg = NULL;
609 }
610
611 /*
612 * out of memory resources?
613 */
614
615 if (pg == NULL) {
616 /* save anon for the next try. */
617 if (anon != NULL) {
618 *spare = anon;
619 }
620
621 /* unlock and fail ... */
622 uvmfault_unlockall(ufi, amap, uobj);
623 if (!uvm_reclaimable()) {
624 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
625 cpu_count(CPU_COUNT_FLTNOANON, 1);
626 error = ENOMEM;
627 goto done;
628 }
629
630 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
631 cpu_count(CPU_COUNT_FLTNORAM, 1);
632 uvm_wait("flt_noram5");
633 error = ERESTART;
634 goto done;
635 }
636
637 /*
638 * copy the page [pg now dirty]
639 *
640 * Remove the pmap entry now for the old page at this address
641 * so that no thread can modify the new page while any thread
642 * might still see the old page.
643 */
644 if (opg) {
645 pmap_remove(vm_map_pmap(ufi->orig_map), ufi->orig_rvaddr,
646 ufi->orig_rvaddr + PAGE_SIZE);
647 pmap_update(vm_map_pmap(ufi->orig_map));
648 uvm_pagecopy(opg, pg);
649 }
650 KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
651
652 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
653 oanon != NULL);
654
655 /*
656 * from this point on am_lock won't be dropped until the page is
657 * entered, so it's safe to unbusy the page up front.
658 *
659 * uvm_fault_{upper,lower}_done will activate or enqueue the page.
660 */
661
662 pg = anon->an_page;
663 pg->flags &= ~(PG_BUSY|PG_FAKE);
664 UVM_PAGE_OWN(pg, NULL);
665
666 *nanon = anon;
667 error = 0;
668 done:
669 return error;
670 }
671
672 /*
673 * Update statistics after fault resolution.
674 * - maxrss
675 */
676 void
uvmfault_update_stats(struct uvm_faultinfo * ufi)677 uvmfault_update_stats(struct uvm_faultinfo *ufi)
678 {
679 struct vm_map *map;
680 struct vmspace *vm;
681 struct proc *p;
682 vsize_t res;
683
684 map = ufi->orig_map;
685
686 p = curproc;
687 KASSERT(p != NULL);
688 vm = p->p_vmspace;
689
690 if (&vm->vm_map != map)
691 return;
692
693 res = pmap_resident_count(map->pmap);
694 if (vm->vm_rssmax < res)
695 vm->vm_rssmax = res;
696 }
697
698 /*
699 * F A U L T - m a i n e n t r y p o i n t
700 */
701
702 /*
703 * uvm_fault: page fault handler
704 *
705 * => called from MD code to resolve a page fault
706 * => VM data structures usually should be unlocked. however, it is
707 * possible to call here with the main map locked if the caller
708 * gets a write lock, sets it recursive, and then calls us (c.f.
709 * uvm_map_pageable). this should be avoided because it keeps
710 * the map locked off during I/O.
711 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
712 */
713
714 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
715 ~VM_PROT_WRITE : VM_PROT_ALL)
716
717 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
718 #define UVM_FAULT_WIRE (1 << 0)
719 #define UVM_FAULT_MAXPROT (1 << 1)
720
721 struct uvm_faultctx {
722
723 /*
724 * the following members are set up by uvm_fault_check() and
725 * read-only after that.
726 *
727 * note that narrow is used by uvm_fault_check() to change
728 * the behaviour after ERESTART.
729 *
730 * most of them might change after RESTART if the underlying
731 * map entry has been changed behind us. an exception is
732 * wire_paging, which does never change.
733 */
734 vm_prot_t access_type;
735 vaddr_t startva;
736 int npages;
737 int centeridx;
738 bool narrow; /* work on a single requested page only */
739 bool wire_mapping; /* request a PMAP_WIRED mapping
740 (UVM_FAULT_WIRE or VM_MAPENT_ISWIRED) */
741 bool wire_paging; /* request uvm_pagewire
742 (true for UVM_FAULT_WIRE) */
743 bool cow_now; /* VM_PROT_WRITE is actually requested
744 (ie. should break COW and page loaning) */
745
746 /*
747 * enter_prot is set up by uvm_fault_check() and clamped
748 * (ie. drop the VM_PROT_WRITE bit) in various places in case
749 * of !cow_now.
750 */
751 vm_prot_t enter_prot; /* prot at which we want to enter pages in */
752
753 /*
754 * the following member is for uvmfault_promote() and ERESTART.
755 */
756 struct vm_anon *anon_spare;
757
758 /*
759 * the following is actually a uvm_fault_lower() internal.
760 * it's here merely for debugging.
761 * (or due to the mechanical separation of the function?)
762 */
763 bool promote;
764
765 /*
766 * type of lock to acquire on objects in both layers.
767 */
768 krw_t lower_lock_type;
769 krw_t upper_lock_type;
770 };
771
772 static inline int uvm_fault_check(
773 struct uvm_faultinfo *, struct uvm_faultctx *,
774 struct vm_anon ***, bool);
775
776 static int uvm_fault_upper(
777 struct uvm_faultinfo *, struct uvm_faultctx *,
778 struct vm_anon **);
779 static inline int uvm_fault_upper_lookup(
780 struct uvm_faultinfo *, const struct uvm_faultctx *,
781 struct vm_anon **, struct vm_page **);
782 static inline void uvm_fault_upper_neighbor(
783 struct uvm_faultinfo *, const struct uvm_faultctx *,
784 vaddr_t, struct vm_page *, bool);
785 static inline int uvm_fault_upper_loan(
786 struct uvm_faultinfo *, struct uvm_faultctx *,
787 struct vm_anon *, struct uvm_object **);
788 static inline int uvm_fault_upper_promote(
789 struct uvm_faultinfo *, struct uvm_faultctx *,
790 struct uvm_object *, struct vm_anon *);
791 static inline int uvm_fault_upper_direct(
792 struct uvm_faultinfo *, struct uvm_faultctx *,
793 struct uvm_object *, struct vm_anon *);
794 static int uvm_fault_upper_enter(
795 struct uvm_faultinfo *, const struct uvm_faultctx *,
796 struct uvm_object *, struct vm_anon *,
797 struct vm_page *, struct vm_anon *);
798 static inline void uvm_fault_upper_done(
799 struct uvm_faultinfo *, const struct uvm_faultctx *,
800 struct vm_anon *, struct vm_page *);
801
802 static int uvm_fault_lower(
803 struct uvm_faultinfo *, struct uvm_faultctx *,
804 struct vm_page **);
805 static inline void uvm_fault_lower_lookup(
806 struct uvm_faultinfo *, const struct uvm_faultctx *,
807 struct vm_page **);
808 static inline void uvm_fault_lower_neighbor(
809 struct uvm_faultinfo *, const struct uvm_faultctx *,
810 vaddr_t, struct vm_page *);
811 static inline int uvm_fault_lower_io(
812 struct uvm_faultinfo *, struct uvm_faultctx *,
813 struct uvm_object **, struct vm_page **);
814 static inline int uvm_fault_lower_direct(
815 struct uvm_faultinfo *, struct uvm_faultctx *,
816 struct uvm_object *, struct vm_page *);
817 static inline int uvm_fault_lower_direct_loan(
818 struct uvm_faultinfo *, struct uvm_faultctx *,
819 struct uvm_object *, struct vm_page **,
820 struct vm_page **);
821 static inline int uvm_fault_lower_promote(
822 struct uvm_faultinfo *, struct uvm_faultctx *,
823 struct uvm_object *, struct vm_page *);
824 static int uvm_fault_lower_enter(
825 struct uvm_faultinfo *, const struct uvm_faultctx *,
826 struct uvm_object *,
827 struct vm_anon *, struct vm_page *);
828 static inline void uvm_fault_lower_done(
829 struct uvm_faultinfo *, const struct uvm_faultctx *,
830 struct uvm_object *, struct vm_page *);
831
832 int
uvm_fault_internal(struct vm_map * orig_map,vaddr_t vaddr,vm_prot_t access_type,int fault_flag)833 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
834 vm_prot_t access_type, int fault_flag)
835 {
836 struct uvm_faultinfo ufi;
837 struct uvm_faultctx flt = {
838 .access_type = access_type,
839
840 /* don't look for neighborhood * pages on "wire" fault */
841 .narrow = (fault_flag & UVM_FAULT_WIRE) != 0,
842
843 /* "wire" fault causes wiring of both mapping and paging */
844 .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0,
845 .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0,
846
847 /*
848 * default lock type to acquire on upper & lower layer
849 * objects: reader. this can be upgraded at any point
850 * during the fault from read -> write and uvm_faultctx
851 * changed to match, but is never downgraded write -> read.
852 */
853 #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */
854 .upper_lock_type = RW_WRITER,
855 .lower_lock_type = RW_WRITER,
856 #else
857 .upper_lock_type = RW_READER,
858 .lower_lock_type = RW_READER,
859 #endif
860 };
861 const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0;
862 struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
863 struct vm_page *pages_store[UVM_MAXRANGE], **pages;
864 int error;
865
866 UVMHIST_FUNC(__func__);
867 UVMHIST_CALLARGS(maphist, "(map=%#jx, vaddr=%#jx, at=%jd, ff=%jd)",
868 (uintptr_t)orig_map, vaddr, access_type, fault_flag);
869
870 /* Don't count anything until user interaction is possible */
871 kpreempt_disable();
872 if (__predict_true(start_init_exec)) {
873 struct cpu_info *ci = curcpu();
874 CPU_COUNT(CPU_COUNT_NFAULT, 1);
875 /* Don't flood RNG subsystem with samples. */
876 if (++(ci->ci_faultrng) == 503) {
877 ci->ci_faultrng = 0;
878 rnd_add_uint32(&uvm_fault_rndsource,
879 sizeof(vaddr_t) == sizeof(uint32_t) ?
880 (uint32_t)vaddr : sizeof(vaddr_t) ==
881 sizeof(uint64_t) ?
882 (uint32_t)vaddr :
883 (uint32_t)ci->ci_counts[CPU_COUNT_NFAULT]);
884 }
885 }
886 kpreempt_enable();
887
888 /*
889 * init the IN parameters in the ufi
890 */
891
892 ufi.orig_map = orig_map;
893 ufi.orig_rvaddr = trunc_page(vaddr);
894 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
895
896 error = ERESTART;
897 while (error == ERESTART) { /* ReFault: */
898 anons = anons_store;
899 pages = pages_store;
900
901 error = uvm_fault_check(&ufi, &flt, &anons, maxprot);
902 if (error != 0)
903 continue;
904
905 error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
906 if (error != 0)
907 continue;
908
909 if (pages[flt.centeridx] == PGO_DONTCARE)
910 error = uvm_fault_upper(&ufi, &flt, anons);
911 else {
912 struct uvm_object * const uobj =
913 ufi.entry->object.uvm_obj;
914
915 if (uobj && uobj->pgops->pgo_fault != NULL) {
916 /*
917 * invoke "special" fault routine.
918 */
919 rw_enter(uobj->vmobjlock, RW_WRITER);
920 /* locked: maps(read), amap(if there), uobj */
921 error = uobj->pgops->pgo_fault(&ufi,
922 flt.startva, pages, flt.npages,
923 flt.centeridx, flt.access_type,
924 PGO_LOCKED|PGO_SYNCIO);
925
926 /*
927 * locked: nothing, pgo_fault has unlocked
928 * everything
929 */
930
931 /*
932 * object fault routine responsible for
933 * pmap_update().
934 */
935
936 /*
937 * Wake up the pagedaemon if the fault method
938 * failed for lack of memory but some can be
939 * reclaimed.
940 */
941 if (error == ENOMEM && uvm_reclaimable()) {
942 uvm_wait("pgo_fault");
943 error = ERESTART;
944 }
945 } else {
946 error = uvm_fault_lower(&ufi, &flt, pages);
947 }
948 }
949 }
950
951 if (flt.anon_spare != NULL) {
952 flt.anon_spare->an_ref--;
953 KASSERT(flt.anon_spare->an_ref == 0);
954 KASSERT(flt.anon_spare->an_lock == NULL);
955 uvm_anfree(flt.anon_spare);
956 }
957 return error;
958 }
959
960 /*
961 * uvm_fault_check: check prot, handle needs-copy, etc.
962 *
963 * 1. lookup entry.
964 * 2. check protection.
965 * 3. adjust fault condition (mainly for simulated fault).
966 * 4. handle needs-copy (lazy amap copy).
967 * 5. establish range of interest for neighbor fault (aka pre-fault).
968 * 6. look up anons (if amap exists).
969 * 7. flush pages (if MADV_SEQUENTIAL)
970 *
971 * => called with nothing locked.
972 * => if we fail (result != 0) we unlock everything.
973 * => initialize/adjust many members of flt.
974 */
975
976 static int
uvm_fault_check(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct vm_anon *** ranons,bool maxprot)977 uvm_fault_check(
978 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
979 struct vm_anon ***ranons, bool maxprot)
980 {
981 struct vm_amap *amap;
982 struct uvm_object *uobj;
983 vm_prot_t check_prot;
984 int nback, nforw;
985 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
986
987 /*
988 * lookup and lock the maps
989 */
990
991 if (uvmfault_lookup(ufi, false) == false) {
992 UVMHIST_LOG(maphist, "<- no mapping @ %#jx", ufi->orig_rvaddr,
993 0,0,0);
994 return EFAULT;
995 }
996 /* locked: maps(read) */
997
998 #ifdef DIAGNOSTIC
999 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) {
1000 printf("Page fault on non-pageable map:\n");
1001 printf("ufi->map = %p\n", ufi->map);
1002 printf("ufi->orig_map = %p\n", ufi->orig_map);
1003 printf("ufi->orig_rvaddr = %#lx\n", (u_long) ufi->orig_rvaddr);
1004 panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0");
1005 }
1006 #endif
1007
1008 /*
1009 * check protection
1010 */
1011
1012 check_prot = maxprot ?
1013 ufi->entry->max_protection : ufi->entry->protection;
1014 if ((check_prot & flt->access_type) != flt->access_type) {
1015 UVMHIST_LOG(maphist,
1016 "<- protection failure (prot=%#jx, access=%#jx)",
1017 ufi->entry->protection, flt->access_type, 0, 0);
1018 uvmfault_unlockmaps(ufi, false);
1019 return EFAULT;
1020 }
1021
1022 /*
1023 * "enter_prot" is the protection we want to enter the page in at.
1024 * for certain pages (e.g. copy-on-write pages) this protection can
1025 * be more strict than ufi->entry->protection. "wired" means either
1026 * the entry is wired or we are fault-wiring the pg.
1027 */
1028
1029 flt->enter_prot = ufi->entry->protection;
1030 if (VM_MAPENT_ISWIRED(ufi->entry)) {
1031 flt->wire_mapping = true;
1032 flt->wire_paging = true;
1033 flt->narrow = true;
1034 }
1035
1036 if (flt->wire_mapping) {
1037 flt->access_type = flt->enter_prot; /* full access for wired */
1038 flt->cow_now = (check_prot & VM_PROT_WRITE) != 0;
1039 } else {
1040 flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0;
1041 }
1042
1043 if (flt->wire_paging) {
1044 /* wiring pages requires a write lock. */
1045 flt->upper_lock_type = RW_WRITER;
1046 flt->lower_lock_type = RW_WRITER;
1047 }
1048
1049 flt->promote = false;
1050
1051 /*
1052 * handle "needs_copy" case. if we need to copy the amap we will
1053 * have to drop our readlock and relock it with a write lock. (we
1054 * need a write lock to change anything in a map entry [e.g.
1055 * needs_copy]).
1056 */
1057
1058 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
1059 if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) {
1060 KASSERT(!maxprot);
1061 /* need to clear */
1062 UVMHIST_LOG(maphist,
1063 " need to clear needs_copy and refault",0,0,0,0);
1064 uvmfault_unlockmaps(ufi, false);
1065 uvmfault_amapcopy(ufi);
1066 cpu_count(CPU_COUNT_FLTAMCOPY, 1);
1067 return ERESTART;
1068
1069 } else {
1070
1071 /*
1072 * ensure that we pmap_enter page R/O since
1073 * needs_copy is still true
1074 */
1075
1076 flt->enter_prot &= ~VM_PROT_WRITE;
1077 }
1078 }
1079
1080 /*
1081 * identify the players
1082 */
1083
1084 amap = ufi->entry->aref.ar_amap; /* upper layer */
1085 uobj = ufi->entry->object.uvm_obj; /* lower layer */
1086
1087 /*
1088 * check for a case 0 fault. if nothing backing the entry then
1089 * error now.
1090 */
1091
1092 if (amap == NULL && uobj == NULL) {
1093 uvmfault_unlockmaps(ufi, false);
1094 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
1095 return EFAULT;
1096 }
1097
1098 /*
1099 * for a case 2B fault waste no time on adjacent pages because
1100 * they are likely already entered.
1101 */
1102
1103 if (uobj != NULL && amap != NULL &&
1104 (flt->access_type & VM_PROT_WRITE) != 0) {
1105 /* wide fault (!narrow) */
1106 flt->narrow = true;
1107 }
1108
1109 /*
1110 * establish range of interest based on advice from mapper
1111 * and then clip to fit map entry. note that we only want
1112 * to do this the first time through the fault. if we
1113 * ReFault we will disable this by setting "narrow" to true.
1114 */
1115
1116 if (flt->narrow == false) {
1117
1118 /* wide fault (!narrow) */
1119 KASSERT(uvmadvice[ufi->entry->advice].advice ==
1120 ufi->entry->advice);
1121 nback = MIN(uvmadvice[ufi->entry->advice].nback,
1122 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT);
1123 flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT);
1124 /*
1125 * note: "-1" because we don't want to count the
1126 * faulting page as forw
1127 */
1128 nforw = MIN(uvmadvice[ufi->entry->advice].nforw,
1129 ((ufi->entry->end - ufi->orig_rvaddr) >>
1130 PAGE_SHIFT) - 1);
1131 flt->npages = nback + nforw + 1;
1132 flt->centeridx = nback;
1133
1134 flt->narrow = true; /* ensure only once per-fault */
1135
1136 } else {
1137
1138 /* narrow fault! */
1139 nback = nforw = 0;
1140 flt->startva = ufi->orig_rvaddr;
1141 flt->npages = 1;
1142 flt->centeridx = 0;
1143
1144 }
1145 /* offset from entry's start to pgs' start */
1146 const voff_t eoff = flt->startva - ufi->entry->start;
1147
1148 /* locked: maps(read) */
1149 UVMHIST_LOG(maphist, " narrow=%jd, back=%jd, forw=%jd, startva=%#jx",
1150 flt->narrow, nback, nforw, flt->startva);
1151 UVMHIST_LOG(maphist, " entry=%#jx, amap=%#jx, obj=%#jx",
1152 (uintptr_t)ufi->entry, (uintptr_t)amap, (uintptr_t)uobj, 0);
1153
1154 /*
1155 * guess at the most suitable lock types to acquire.
1156 * if we've got an amap then lock it and extract current anons.
1157 */
1158
1159 if (amap) {
1160 if ((amap_flags(amap) & AMAP_SHARED) == 0) {
1161 /*
1162 * the amap isn't shared. get a writer lock to
1163 * avoid the cost of upgrading the lock later if
1164 * needed.
1165 *
1166 * XXX nice for PostgreSQL, but consider threads.
1167 */
1168 flt->upper_lock_type = RW_WRITER;
1169 } else if ((flt->access_type & VM_PROT_WRITE) != 0) {
1170 /*
1171 * assume we're about to COW.
1172 */
1173 flt->upper_lock_type = RW_WRITER;
1174 }
1175 amap_lock(amap, flt->upper_lock_type);
1176 amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages);
1177 } else {
1178 if ((flt->access_type & VM_PROT_WRITE) != 0) {
1179 /*
1180 * we are about to dirty the object and that
1181 * requires a write lock.
1182 */
1183 flt->lower_lock_type = RW_WRITER;
1184 }
1185 *ranons = NULL; /* to be safe */
1186 }
1187
1188 /* locked: maps(read), amap(if there) */
1189 KASSERT(amap == NULL ||
1190 rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1191
1192 /*
1193 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
1194 * now and then forget about them (for the rest of the fault).
1195 */
1196
1197 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) {
1198
1199 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
1200 0,0,0,0);
1201 /* flush back-page anons? */
1202 if (amap)
1203 uvmfault_anonflush(*ranons, nback);
1204
1205 /*
1206 * flush object? change lock type to RW_WRITER, to avoid
1207 * excessive competition between read/write locks if many
1208 * threads doing "sequential access".
1209 */
1210 if (uobj) {
1211 voff_t uoff;
1212
1213 flt->lower_lock_type = RW_WRITER;
1214 uoff = ufi->entry->offset + eoff;
1215 rw_enter(uobj->vmobjlock, RW_WRITER);
1216 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
1217 (nback << PAGE_SHIFT), PGO_DEACTIVATE);
1218 }
1219
1220 /* now forget about the backpages */
1221 if (amap)
1222 *ranons += nback;
1223 flt->startva += (nback << PAGE_SHIFT);
1224 flt->npages -= nback;
1225 flt->centeridx = 0;
1226 }
1227 /*
1228 * => startva is fixed
1229 * => npages is fixed
1230 */
1231 KASSERT(flt->startva <= ufi->orig_rvaddr);
1232 KASSERT(ufi->orig_rvaddr + ufi->orig_size <=
1233 flt->startva + (flt->npages << PAGE_SHIFT));
1234 return 0;
1235 }
1236
1237 /*
1238 * uvm_fault_upper_upgrade: upgrade upper lock, reader -> writer
1239 */
1240
1241 static inline int
uvm_fault_upper_upgrade(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct vm_amap * amap,struct uvm_object * uobj)1242 uvm_fault_upper_upgrade(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1243 struct vm_amap *amap, struct uvm_object *uobj)
1244 {
1245 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1246
1247 KASSERT(amap != NULL);
1248 KASSERT(flt->upper_lock_type == rw_lock_op(amap->am_lock));
1249
1250 /*
1251 * fast path.
1252 */
1253
1254 if (__predict_true(flt->upper_lock_type == RW_WRITER)) {
1255 return 0;
1256 }
1257
1258 /*
1259 * otherwise try for the upgrade. if we don't get it, unlock
1260 * everything, restart the fault and next time around get a writer
1261 * lock.
1262 */
1263
1264 flt->upper_lock_type = RW_WRITER;
1265 if (__predict_false(!rw_tryupgrade(amap->am_lock))) {
1266 uvmfault_unlockall(ufi, amap, uobj);
1267 cpu_count(CPU_COUNT_FLTNOUP, 1);
1268 UVMHIST_LOG(maphist, " !upgrade upper", 0, 0,0,0);
1269 return ERESTART;
1270 }
1271 cpu_count(CPU_COUNT_FLTUP, 1);
1272 KASSERT(flt->upper_lock_type == rw_lock_op(amap->am_lock));
1273 return 0;
1274 }
1275
1276 /*
1277 * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
1278 *
1279 * iterate range of interest:
1280 * 1. check if h/w mapping exists. if yes, we don't care
1281 * 2. check if anon exists. if not, page is lower.
1282 * 3. if anon exists, enter h/w mapping for neighbors.
1283 *
1284 * => called with amap locked (if exists).
1285 */
1286
1287 static int
uvm_fault_upper_lookup(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,struct vm_anon ** anons,struct vm_page ** pages)1288 uvm_fault_upper_lookup(
1289 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1290 struct vm_anon **anons, struct vm_page **pages)
1291 {
1292 struct vm_amap *amap = ufi->entry->aref.ar_amap;
1293 int lcv;
1294 vaddr_t currva;
1295 bool shadowed __unused;
1296 bool entered;
1297 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1298
1299 /* locked: maps(read), amap(if there) */
1300 KASSERT(amap == NULL ||
1301 rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1302
1303 /*
1304 * map in the backpages and frontpages we found in the amap in hopes
1305 * of preventing future faults. we also init the pages[] array as
1306 * we go.
1307 */
1308
1309 currva = flt->startva;
1310 shadowed = false;
1311 entered = false;
1312 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1313 /*
1314 * unmapped or center page. check if any anon at this level.
1315 */
1316 if (amap == NULL || anons[lcv] == NULL) {
1317 pages[lcv] = NULL;
1318 continue;
1319 }
1320
1321 /*
1322 * check for present page and map if possible.
1323 */
1324
1325 pages[lcv] = PGO_DONTCARE;
1326 if (lcv == flt->centeridx) { /* save center for later! */
1327 shadowed = true;
1328 continue;
1329 }
1330
1331 struct vm_anon *anon = anons[lcv];
1332 struct vm_page *pg = anon->an_page;
1333
1334 KASSERT(anon->an_lock == amap->am_lock);
1335
1336 /*
1337 * ignore loaned and busy pages.
1338 * don't play with VAs that are already mapped.
1339 */
1340
1341 if (pg && pg->loan_count == 0 && (pg->flags & PG_BUSY) == 0 &&
1342 !pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
1343 uvm_fault_upper_neighbor(ufi, flt, currva,
1344 pg, anon->an_ref > 1);
1345 entered = true;
1346 }
1347 }
1348 if (entered) {
1349 pmap_update(ufi->orig_map->pmap);
1350 }
1351
1352 /* locked: maps(read), amap(if there) */
1353 KASSERT(amap == NULL ||
1354 rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1355 /* (shadowed == true) if there is an anon at the faulting address */
1356 UVMHIST_LOG(maphist, " shadowed=%jd, will_get=%jd", shadowed,
1357 (ufi->entry->object.uvm_obj && shadowed != false),0,0);
1358
1359 return 0;
1360 }
1361
1362 /*
1363 * uvm_fault_upper_neighbor: enter single upper neighbor page.
1364 *
1365 * => called with amap and anon locked.
1366 */
1367
1368 static void
uvm_fault_upper_neighbor(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,vaddr_t currva,struct vm_page * pg,bool readonly)1369 uvm_fault_upper_neighbor(
1370 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1371 vaddr_t currva, struct vm_page *pg, bool readonly)
1372 {
1373 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1374
1375 /* locked: amap, anon */
1376
1377 KASSERT(pg->uobject == NULL);
1378 KASSERT(pg->uanon != NULL);
1379 KASSERT(rw_lock_op(pg->uanon->an_lock) == flt->upper_lock_type);
1380 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
1381
1382 /*
1383 * there wasn't a direct fault on the page, so avoid the cost of
1384 * activating it.
1385 */
1386
1387 if (!uvmpdpol_pageisqueued_p(pg) && pg->wire_count == 0) {
1388 uvm_pagelock(pg);
1389 uvm_pageenqueue(pg);
1390 uvm_pageunlock(pg);
1391 }
1392
1393 UVMHIST_LOG(maphist,
1394 " MAPPING: n anon: pm=%#jx, va=%#jx, pg=%#jx",
1395 (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0);
1396 cpu_count(CPU_COUNT_FLTNAMAP, 1);
1397
1398 /*
1399 * Since this page isn't the page that's actually faulting,
1400 * ignore pmap_enter() failures; it's not critical that we
1401 * enter these right now.
1402 */
1403
1404 (void) pmap_enter(ufi->orig_map->pmap, currva,
1405 VM_PAGE_TO_PHYS(pg),
1406 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1407 flt->enter_prot,
1408 PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1409 }
1410
1411 /*
1412 * uvm_fault_upper: handle upper fault.
1413 *
1414 * 1. acquire anon lock.
1415 * 2. get anon. let uvmfault_anonget do the dirty work.
1416 * 3. handle loan.
1417 * 4. dispatch direct or promote handlers.
1418 */
1419
1420 static int
uvm_fault_upper(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct vm_anon ** anons)1421 uvm_fault_upper(
1422 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1423 struct vm_anon **anons)
1424 {
1425 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1426 struct vm_anon * const anon = anons[flt->centeridx];
1427 struct uvm_object *uobj;
1428 int error;
1429 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1430
1431 /* locked: maps(read), amap, anon */
1432 KASSERT(rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1433 KASSERT(anon->an_lock == amap->am_lock);
1434
1435 /*
1436 * handle case 1: fault on an anon in our amap
1437 */
1438
1439 UVMHIST_LOG(maphist, " case 1 fault: anon=%#jx",
1440 (uintptr_t)anon, 0, 0, 0);
1441
1442 /*
1443 * no matter if we have case 1A or case 1B we are going to need to
1444 * have the anon's memory resident. ensure that now.
1445 */
1446
1447 /*
1448 * let uvmfault_anonget do the dirty work.
1449 * if it fails (!OK) it will unlock everything for us.
1450 * if it succeeds, locks are still valid and locked.
1451 * also, if it is OK, then the anon's page is on the queues.
1452 * if the page is on loan from a uvm_object, then anonget will
1453 * lock that object for us if it does not fail.
1454 */
1455 retry:
1456 error = uvmfault_anonget(ufi, amap, anon);
1457 switch (error) {
1458 case 0:
1459 break;
1460
1461 case ERESTART:
1462 return ERESTART;
1463
1464 case EAGAIN:
1465 kpause("fltagain1", false, hz/2, NULL);
1466 return ERESTART;
1467
1468 case ENOLCK:
1469 /* it needs a write lock: retry */
1470 error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL);
1471 if (error != 0) {
1472 return error;
1473 }
1474 KASSERT(rw_write_held(amap->am_lock));
1475 goto retry;
1476
1477 default:
1478 return error;
1479 }
1480
1481 /*
1482 * uobj is non null if the page is on loan from an object (i.e. uobj)
1483 */
1484
1485 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
1486
1487 /* locked: maps(read), amap, anon, uobj(if one) */
1488 KASSERT(rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1489 KASSERT(anon->an_lock == amap->am_lock);
1490 KASSERT(uobj == NULL ||
1491 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
1492
1493 /*
1494 * special handling for loaned pages
1495 */
1496
1497 if (anon->an_page->loan_count) {
1498 error = uvm_fault_upper_loan(ufi, flt, anon, &uobj);
1499 if (error != 0)
1500 return error;
1501 }
1502
1503 /*
1504 * if we are case 1B then we will need to allocate a new blank
1505 * anon to transfer the data into. note that we have a lock
1506 * on anon, so no one can busy or release the page until we are done.
1507 * also note that the ref count can't drop to zero here because
1508 * it is > 1 and we are only dropping one ref.
1509 *
1510 * in the (hopefully very rare) case that we are out of RAM we
1511 * will unlock, wait for more RAM, and refault.
1512 *
1513 * if we are out of anon VM we kill the process (XXX: could wait?).
1514 */
1515
1516 if (flt->cow_now && anon->an_ref > 1) {
1517 flt->promote = true;
1518 error = uvm_fault_upper_promote(ufi, flt, uobj, anon);
1519 } else {
1520 error = uvm_fault_upper_direct(ufi, flt, uobj, anon);
1521 }
1522 return error;
1523 }
1524
1525 /*
1526 * uvm_fault_upper_loan: handle loaned upper page.
1527 *
1528 * 1. if not cow'ing now, simply adjust flt->enter_prot.
1529 * 2. if cow'ing now, and if ref count is 1, break loan.
1530 */
1531
1532 static int
uvm_fault_upper_loan(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct vm_anon * anon,struct uvm_object ** ruobj)1533 uvm_fault_upper_loan(
1534 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1535 struct vm_anon *anon, struct uvm_object **ruobj)
1536 {
1537 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1538 int error = 0;
1539 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1540
1541 if (!flt->cow_now) {
1542
1543 /*
1544 * for read faults on loaned pages we just cap the
1545 * protection at read-only.
1546 */
1547
1548 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1549
1550 } else {
1551 /*
1552 * note that we can't allow writes into a loaned page!
1553 *
1554 * if we have a write fault on a loaned page in an
1555 * anon then we need to look at the anon's ref count.
1556 * if it is greater than one then we are going to do
1557 * a normal copy-on-write fault into a new anon (this
1558 * is not a problem). however, if the reference count
1559 * is one (a case where we would normally allow a
1560 * write directly to the page) then we need to kill
1561 * the loan before we continue.
1562 */
1563
1564 /* >1 case is already ok */
1565 if (anon->an_ref == 1) {
1566 /* breaking loan requires a write lock. */
1567 error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL);
1568 if (error != 0) {
1569 return error;
1570 }
1571 KASSERT(rw_write_held(amap->am_lock));
1572
1573 error = uvm_loanbreak_anon(anon, *ruobj);
1574 if (error != 0) {
1575 uvmfault_unlockall(ufi, amap, *ruobj);
1576 uvm_wait("flt_noram2");
1577 return ERESTART;
1578 }
1579 /* if we were a loan receiver uobj is gone */
1580 if (*ruobj)
1581 *ruobj = NULL;
1582 }
1583 }
1584 return error;
1585 }
1586
1587 /*
1588 * uvm_fault_upper_promote: promote upper page.
1589 *
1590 * 1. call uvmfault_promote.
1591 * 2. enqueue page.
1592 * 3. deref.
1593 * 4. pass page to uvm_fault_upper_enter.
1594 */
1595
1596 static int
uvm_fault_upper_promote(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_anon * anon)1597 uvm_fault_upper_promote(
1598 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1599 struct uvm_object *uobj, struct vm_anon *anon)
1600 {
1601 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1602 struct vm_anon * const oanon = anon;
1603 struct vm_page *pg;
1604 int error;
1605 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1606
1607 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1608
1609 /* promoting requires a write lock. */
1610 error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL);
1611 if (error != 0) {
1612 return error;
1613 }
1614 KASSERT(rw_write_held(amap->am_lock));
1615
1616 cpu_count(CPU_COUNT_FLT_ACOW, 1);
1617
1618 error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon,
1619 &flt->anon_spare);
1620 switch (error) {
1621 case 0:
1622 break;
1623 case ERESTART:
1624 return ERESTART;
1625 default:
1626 return error;
1627 }
1628 pg = anon->an_page;
1629
1630 KASSERT(anon->an_lock == oanon->an_lock);
1631 KASSERT((pg->flags & (PG_BUSY | PG_FAKE)) == 0);
1632
1633 /* deref: can not drop to zero here by defn! */
1634 KASSERT(oanon->an_ref > 1);
1635 oanon->an_ref--;
1636
1637 /*
1638 * note: oanon is still locked, as is the new anon. we
1639 * need to check for this later when we unlock oanon; if
1640 * oanon != anon, we'll have to unlock anon, too.
1641 */
1642
1643 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1644 }
1645
1646 /*
1647 * uvm_fault_upper_direct: handle direct fault.
1648 */
1649
1650 static int
uvm_fault_upper_direct(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_anon * anon)1651 uvm_fault_upper_direct(
1652 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1653 struct uvm_object *uobj, struct vm_anon *anon)
1654 {
1655 struct vm_anon * const oanon = anon;
1656 struct vm_page *pg;
1657 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1658
1659 cpu_count(CPU_COUNT_FLT_ANON, 1);
1660 pg = anon->an_page;
1661 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1662 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1663
1664 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1665 }
1666
1667 /*
1668 * uvm_fault_upper_enter: enter h/w mapping of upper page.
1669 */
1670
1671 static int
uvm_fault_upper_enter(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_anon * anon,struct vm_page * pg,struct vm_anon * oanon)1672 uvm_fault_upper_enter(
1673 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1674 struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg,
1675 struct vm_anon *oanon)
1676 {
1677 struct pmap *pmap = ufi->orig_map->pmap;
1678 vaddr_t va = ufi->orig_rvaddr;
1679 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1680 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1681
1682 /* locked: maps(read), amap, oanon, anon(if different from oanon) */
1683 KASSERT(rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1684 KASSERT(anon->an_lock == amap->am_lock);
1685 KASSERT(oanon->an_lock == amap->am_lock);
1686 KASSERT(uobj == NULL ||
1687 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
1688 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
1689
1690 /*
1691 * now map the page in.
1692 */
1693
1694 UVMHIST_LOG(maphist,
1695 " MAPPING: anon: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd",
1696 (uintptr_t)pmap, va, (uintptr_t)pg, flt->promote);
1697 if (pmap_enter(pmap, va, VM_PAGE_TO_PHYS(pg),
1698 flt->enter_prot, flt->access_type | PMAP_CANFAIL |
1699 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1700
1701 /*
1702 * If pmap_enter() fails, it must not leave behind an existing
1703 * pmap entry. In particular, a now-stale entry for a different
1704 * page would leave the pmap inconsistent with the vm_map.
1705 * This is not to imply that pmap_enter() should remove an
1706 * existing mapping in such a situation (since that could create
1707 * different problems, eg. if the existing mapping is wired),
1708 * but rather that the pmap should be designed such that it
1709 * never needs to fail when the new mapping is replacing an
1710 * existing mapping and the new page has no existing mappings.
1711 *
1712 * XXX This can't be asserted safely any more because many
1713 * LWPs and/or many processes could simultaneously fault on
1714 * the same VA and some might succeed.
1715 */
1716
1717 /* KASSERT(!pmap_extract(pmap, va, NULL)); */
1718
1719 /*
1720 * ensure that the page is queued in the case that
1721 * we just promoted.
1722 */
1723
1724 uvm_pagelock(pg);
1725 uvm_pageenqueue(pg);
1726 uvm_pageunlock(pg);
1727
1728 /*
1729 * No need to undo what we did; we can simply think of
1730 * this as the pmap throwing away the mapping information.
1731 *
1732 * We do, however, have to go through the ReFault path,
1733 * as the map may change while we're asleep.
1734 */
1735
1736 uvmfault_unlockall(ufi, amap, uobj);
1737 if (!uvm_reclaimable()) {
1738 UVMHIST_LOG(maphist,
1739 "<- failed. out of VM",0,0,0,0);
1740 /* XXX instrumentation */
1741 return ENOMEM;
1742 }
1743 /* XXX instrumentation */
1744 uvm_wait("flt_pmfail1");
1745 return ERESTART;
1746 }
1747
1748 uvm_fault_upper_done(ufi, flt, anon, pg);
1749
1750 /*
1751 * done case 1! finish up by unlocking everything and returning success
1752 */
1753
1754 pmap_update(pmap);
1755 uvmfault_unlockall(ufi, amap, uobj);
1756 return 0;
1757 }
1758
1759 /*
1760 * uvm_fault_upper_done: queue upper center page.
1761 */
1762
1763 static void
uvm_fault_upper_done(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,struct vm_anon * anon,struct vm_page * pg)1764 uvm_fault_upper_done(
1765 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1766 struct vm_anon *anon, struct vm_page *pg)
1767 {
1768 const bool wire_paging = flt->wire_paging;
1769
1770 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1771
1772 /*
1773 * ... update the page queues.
1774 */
1775
1776 if (wire_paging) {
1777 uvm_pagelock(pg);
1778 uvm_pagewire(pg);
1779 uvm_pageunlock(pg);
1780
1781 /*
1782 * since the now-wired page cannot be paged out,
1783 * release its swap resources for others to use.
1784 * and since an anon with no swap cannot be clean,
1785 * mark it dirty now.
1786 */
1787
1788 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
1789 uvm_anon_dropswap(anon);
1790 } else if (uvmpdpol_pageactivate_p(pg)) {
1791 /*
1792 * avoid re-activating the page unless needed,
1793 * to avoid false sharing on multiprocessor.
1794 */
1795
1796 uvm_pagelock(pg);
1797 uvm_pageactivate(pg);
1798 uvm_pageunlock(pg);
1799 }
1800 }
1801
1802 /*
1803 * uvm_fault_lower_upgrade: upgrade lower lock, reader -> writer
1804 */
1805
1806 static inline int
uvm_fault_lower_upgrade(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct vm_amap * amap,struct uvm_object * uobj,struct vm_page * uobjpage)1807 uvm_fault_lower_upgrade(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1808 struct vm_amap *amap, struct uvm_object *uobj, struct vm_page *uobjpage)
1809 {
1810
1811 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1812
1813 KASSERT(uobj != NULL);
1814 KASSERT(flt->lower_lock_type == rw_lock_op(uobj->vmobjlock));
1815
1816 /*
1817 * fast path.
1818 */
1819
1820 if (__predict_true(flt->lower_lock_type == RW_WRITER)) {
1821 return 0;
1822 }
1823
1824 /*
1825 * otherwise try for the upgrade. if we don't get it, unlock
1826 * everything, restart the fault and next time around get a writer
1827 * lock.
1828 */
1829
1830 flt->lower_lock_type = RW_WRITER;
1831 if (__predict_false(!rw_tryupgrade(uobj->vmobjlock))) {
1832 uvmfault_unlockall(ufi, amap, uobj);
1833 cpu_count(CPU_COUNT_FLTNOUP, 1);
1834 UVMHIST_LOG(maphist, " !upgrade lower", 0, 0,0,0);
1835 return ERESTART;
1836 }
1837 cpu_count(CPU_COUNT_FLTUP, 1);
1838 KASSERT(flt->lower_lock_type == rw_lock_op(uobj->vmobjlock));
1839 return 0;
1840 }
1841
1842 /*
1843 * uvm_fault_lower: handle lower fault.
1844 *
1845 * 1. check uobj
1846 * 1.1. if null, ZFOD.
1847 * 1.2. if not null, look up unmapped neighbor pages.
1848 * 2. for center page, check if promote.
1849 * 2.1. ZFOD always needs promotion.
1850 * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode).
1851 * 3. if uobj is not ZFOD and page is not found, do i/o.
1852 * 4. dispatch either direct / promote fault.
1853 */
1854
1855 static int
uvm_fault_lower(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct vm_page ** pages)1856 uvm_fault_lower(
1857 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1858 struct vm_page **pages)
1859 {
1860 struct vm_amap *amap __diagused = ufi->entry->aref.ar_amap;
1861 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1862 struct vm_page *uobjpage;
1863 int error;
1864 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1865
1866 /*
1867 * now, if the desired page is not shadowed by the amap and we have
1868 * a backing object that does not have a special fault routine, then
1869 * we ask (with pgo_get) the object for resident pages that we care
1870 * about and attempt to map them in. we do not let pgo_get block
1871 * (PGO_LOCKED).
1872 */
1873
1874 if (uobj == NULL) {
1875 /* zero fill; don't care neighbor pages */
1876 uobjpage = NULL;
1877 } else {
1878 uvm_fault_lower_lookup(ufi, flt, pages);
1879 uobjpage = pages[flt->centeridx];
1880 }
1881
1882 /*
1883 * note that at this point we are done with any front or back pages.
1884 * we are now going to focus on the center page (i.e. the one we've
1885 * faulted on). if we have faulted on the upper (anon) layer
1886 * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1887 * not touched it yet). if we have faulted on the bottom (uobj)
1888 * layer [i.e. case 2] and the page was both present and available,
1889 * then we've got a pointer to it as "uobjpage" and we've already
1890 * made it BUSY.
1891 */
1892
1893 /*
1894 * locked:
1895 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1896 */
1897 KASSERT(amap == NULL ||
1898 rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1899 KASSERT(uobj == NULL ||
1900 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
1901
1902 /*
1903 * note that uobjpage can not be PGO_DONTCARE at this point. we now
1904 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1905 * have a backing object, check and see if we are going to promote
1906 * the data up to an anon during the fault.
1907 */
1908
1909 if (uobj == NULL) {
1910 uobjpage = PGO_DONTCARE;
1911 flt->promote = true; /* always need anon here */
1912 } else {
1913 KASSERT(uobjpage != PGO_DONTCARE);
1914 flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry);
1915 }
1916 UVMHIST_LOG(maphist, " case 2 fault: promote=%jd, zfill=%jd",
1917 flt->promote, (uobj == NULL), 0,0);
1918
1919 /*
1920 * if uobjpage is not null then we do not need to do I/O to get the
1921 * uobjpage.
1922 *
1923 * if uobjpage is null, then we need to unlock and ask the pager to
1924 * get the data for us. once we have the data, we need to reverify
1925 * the state the world. we are currently not holding any resources.
1926 */
1927
1928 if (uobjpage) {
1929 /* update rusage counters */
1930 curlwp->l_ru.ru_minflt++;
1931 } else {
1932 error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage);
1933 if (error != 0)
1934 return error;
1935 }
1936
1937 /*
1938 * locked:
1939 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1940 */
1941 KASSERT(amap == NULL ||
1942 rw_lock_op(amap->am_lock) == flt->upper_lock_type);
1943 KASSERT(uobj == NULL ||
1944 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
1945
1946 /*
1947 * notes:
1948 * - at this point uobjpage can not be NULL
1949 * - at this point uobjpage can not be PG_RELEASED (since we checked
1950 * for it above)
1951 * - at this point uobjpage could be waited on (handle later)
1952 * - uobjpage can be from a different object if tmpfs (vnode vs UAO)
1953 */
1954
1955 KASSERT(uobjpage != NULL);
1956 KASSERT(uobj == NULL ||
1957 uobjpage->uobject->vmobjlock == uobj->vmobjlock);
1958 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
1959 uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_CLEAN);
1960
1961 if (!flt->promote) {
1962 error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage);
1963 } else {
1964 error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage);
1965 }
1966 return error;
1967 }
1968
1969 /*
1970 * uvm_fault_lower_lookup: look up on-memory uobj pages.
1971 *
1972 * 1. get on-memory pages.
1973 * 2. if failed, give up (get only center page later).
1974 * 3. if succeeded, enter h/w mapping of neighbor pages.
1975 */
1976
1977 static void
uvm_fault_lower_lookup(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,struct vm_page ** pages)1978 uvm_fault_lower_lookup(
1979 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1980 struct vm_page **pages)
1981 {
1982 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1983 int lcv, gotpages;
1984 vaddr_t currva;
1985 bool entered;
1986 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1987
1988 rw_enter(uobj->vmobjlock, flt->lower_lock_type);
1989
1990 /*
1991 * Locked: maps(read), amap(if there), uobj
1992 */
1993
1994 cpu_count(CPU_COUNT_FLTLGET, 1);
1995 gotpages = flt->npages;
1996 (void) uobj->pgops->pgo_get(uobj,
1997 ufi->entry->offset + flt->startva - ufi->entry->start,
1998 pages, &gotpages, flt->centeridx,
1999 flt->access_type & MASK(ufi->entry), ufi->entry->advice,
2000 PGO_LOCKED);
2001
2002 KASSERT(rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
2003
2004 /*
2005 * check for pages to map, if we got any
2006 */
2007
2008 if (gotpages == 0) {
2009 pages[flt->centeridx] = NULL;
2010 return;
2011 }
2012
2013 entered = false;
2014 currva = flt->startva;
2015 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
2016 struct vm_page *curpg;
2017
2018 curpg = pages[lcv];
2019 if (curpg == NULL || curpg == PGO_DONTCARE) {
2020 continue;
2021 }
2022
2023 /*
2024 * in the case of tmpfs, the pages might be from a different
2025 * uvm_object. just make sure that they have the same lock.
2026 */
2027
2028 KASSERT(curpg->uobject->vmobjlock == uobj->vmobjlock);
2029 KASSERT((curpg->flags & PG_BUSY) == 0);
2030
2031 /*
2032 * leave the centre page for later. don't screw with
2033 * existing mappings (needless & expensive).
2034 */
2035
2036 if (lcv == flt->centeridx) {
2037 UVMHIST_LOG(maphist, " got uobjpage (%#jx) "
2038 "with locked get", (uintptr_t)curpg, 0, 0, 0);
2039 } else if (!pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
2040 uvm_fault_lower_neighbor(ufi, flt, currva, curpg);
2041 entered = true;
2042 }
2043 }
2044 if (entered) {
2045 pmap_update(ufi->orig_map->pmap);
2046 }
2047 }
2048
2049 /*
2050 * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page.
2051 */
2052
2053 static void
uvm_fault_lower_neighbor(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,vaddr_t currva,struct vm_page * pg)2054 uvm_fault_lower_neighbor(
2055 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2056 vaddr_t currva, struct vm_page *pg)
2057 {
2058 const bool readonly = uvm_pagereadonly_p(pg) || pg->loan_count > 0;
2059 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2060
2061 /* locked: maps(read), amap(if there), uobj */
2062
2063 /*
2064 * calling pgo_get with PGO_LOCKED returns us pages which
2065 * are neither busy nor released, so we don't need to check
2066 * for this. we can just directly enter the pages.
2067 *
2068 * there wasn't a direct fault on the page, so avoid the cost of
2069 * activating it.
2070 */
2071
2072 if (!uvmpdpol_pageisqueued_p(pg) && pg->wire_count == 0) {
2073 uvm_pagelock(pg);
2074 uvm_pageenqueue(pg);
2075 uvm_pageunlock(pg);
2076 }
2077
2078 UVMHIST_LOG(maphist,
2079 " MAPPING: n obj: pm=%#jx, va=%#jx, pg=%#jx",
2080 (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0);
2081 cpu_count(CPU_COUNT_FLTNOMAP, 1);
2082
2083 /*
2084 * Since this page isn't the page that's actually faulting,
2085 * ignore pmap_enter() failures; it's not critical that we
2086 * enter these right now.
2087 * NOTE: page can't be waited on or PG_RELEASED because we've
2088 * held the lock the whole time we've had the handle.
2089 */
2090 KASSERT((pg->flags & PG_PAGEOUT) == 0);
2091 KASSERT((pg->flags & PG_RELEASED) == 0);
2092 KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) ||
2093 uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN);
2094 KASSERT((pg->flags & PG_BUSY) == 0);
2095 KASSERT(rw_lock_op(pg->uobject->vmobjlock) == flt->lower_lock_type);
2096
2097 const vm_prot_t mapprot =
2098 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
2099 flt->enter_prot & MASK(ufi->entry);
2100 const u_int mapflags =
2101 PMAP_CANFAIL | (flt->wire_mapping ? (mapprot | PMAP_WIRED) : 0);
2102 (void) pmap_enter(ufi->orig_map->pmap, currva,
2103 VM_PAGE_TO_PHYS(pg), mapprot, mapflags);
2104 }
2105
2106 /*
2107 * uvm_fault_lower_io: get lower page from backing store.
2108 *
2109 * 1. unlock everything, because i/o will block.
2110 * 2. call pgo_get.
2111 * 3. if failed, recover.
2112 * 4. if succeeded, relock everything and verify things.
2113 */
2114
2115 static int
uvm_fault_lower_io(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct uvm_object ** ruobj,struct vm_page ** ruobjpage)2116 uvm_fault_lower_io(
2117 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2118 struct uvm_object **ruobj, struct vm_page **ruobjpage)
2119 {
2120 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2121 struct uvm_object *uobj = *ruobj;
2122 struct vm_page *pg;
2123 bool locked;
2124 int gotpages;
2125 int error;
2126 voff_t uoff;
2127 vm_prot_t access_type;
2128 int advice;
2129 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2130
2131 /* grab everything we need from the entry before we unlock */
2132 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
2133 access_type = flt->access_type & MASK(ufi->entry);
2134 advice = ufi->entry->advice;
2135
2136 /* Locked: maps(read), amap(if there), uobj */
2137 KASSERT(rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
2138
2139 /* Upgrade to a write lock if needed. */
2140 error = uvm_fault_lower_upgrade(ufi, flt, amap, uobj, NULL);
2141 if (error != 0) {
2142 return error;
2143 }
2144 uvmfault_unlockall(ufi, amap, NULL);
2145
2146 /* update rusage counters */
2147 curlwp->l_ru.ru_majflt++;
2148
2149 /* Locked: uobj(write) */
2150 KASSERT(rw_write_held(uobj->vmobjlock));
2151
2152 cpu_count(CPU_COUNT_FLTGET, 1);
2153 gotpages = 1;
2154 pg = NULL;
2155 error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages,
2156 0, access_type, advice, PGO_SYNCIO);
2157 /* locked: pg(if no error) */
2158
2159 /*
2160 * recover from I/O
2161 */
2162
2163 if (error) {
2164 if (error == EAGAIN) {
2165 UVMHIST_LOG(maphist,
2166 " pgo_get says TRY AGAIN!",0,0,0,0);
2167 kpause("fltagain2", false, hz/2, NULL);
2168 return ERESTART;
2169 }
2170
2171 #if 0
2172 KASSERT(error != ERESTART);
2173 #else
2174 /* XXXUEBS don't re-fault? */
2175 if (error == ERESTART)
2176 error = EIO;
2177 #endif
2178
2179 UVMHIST_LOG(maphist, "<- pgo_get failed (code %jd)",
2180 error, 0,0,0);
2181 return error;
2182 }
2183
2184 /*
2185 * re-verify the state of the world by first trying to relock
2186 * the maps. always relock the object.
2187 */
2188
2189 locked = uvmfault_relock(ufi);
2190 if (locked && amap)
2191 amap_lock(amap, flt->upper_lock_type);
2192
2193 /* might be changed */
2194 uobj = pg->uobject;
2195
2196 rw_enter(uobj->vmobjlock, flt->lower_lock_type);
2197 KASSERT((pg->flags & PG_BUSY) != 0);
2198 KASSERT(flt->lower_lock_type == RW_WRITER);
2199
2200 uvm_pagelock(pg);
2201 uvm_pageactivate(pg);
2202 uvm_pageunlock(pg);
2203
2204 /* locked(locked): maps(read), amap(if !null), uobj, pg */
2205 /* locked(!locked): uobj, pg */
2206
2207 /*
2208 * verify that the page has not be released and re-verify
2209 * that amap slot is still free. if there is a problem,
2210 * we unlock and clean up.
2211 */
2212
2213 if ((pg->flags & PG_RELEASED) != 0 ||
2214 (locked && amap && amap_lookup(&ufi->entry->aref,
2215 ufi->orig_rvaddr - ufi->entry->start))) {
2216 if (locked)
2217 uvmfault_unlockall(ufi, amap, NULL);
2218 locked = false;
2219 }
2220
2221 /*
2222 * unbusy/release the page.
2223 */
2224
2225 if ((pg->flags & PG_RELEASED) == 0) {
2226 pg->flags &= ~PG_BUSY;
2227 uvm_pagelock(pg);
2228 uvm_pagewakeup(pg);
2229 uvm_pageunlock(pg);
2230 UVM_PAGE_OWN(pg, NULL);
2231 } else {
2232 cpu_count(CPU_COUNT_FLTPGRELE, 1);
2233 uvm_pagefree(pg);
2234 }
2235
2236 /*
2237 * didn't get the lock? retry.
2238 */
2239
2240 if (locked == false) {
2241 UVMHIST_LOG(maphist,
2242 " wasn't able to relock after fault: retry",
2243 0,0,0,0);
2244 rw_exit(uobj->vmobjlock);
2245 return ERESTART;
2246 }
2247
2248 /*
2249 * we have the data in pg. we are holding object lock (so the page
2250 * can't be released on us).
2251 */
2252
2253 /* locked: maps(read), amap(if !null), uobj */
2254
2255 *ruobj = uobj;
2256 *ruobjpage = pg;
2257 return 0;
2258 }
2259
2260 /*
2261 * uvm_fault_lower_direct: fault lower center page
2262 *
2263 * 1. adjust flt->enter_prot.
2264 * 2. if page is loaned, resolve.
2265 */
2266
2267 int
uvm_fault_lower_direct(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_page * uobjpage)2268 uvm_fault_lower_direct(
2269 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2270 struct uvm_object *uobj, struct vm_page *uobjpage)
2271 {
2272 struct vm_page *pg;
2273 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2274
2275 /*
2276 * we are not promoting. if the mapping is COW ensure that we
2277 * don't give more access than we should (e.g. when doing a read
2278 * fault on a COPYONWRITE mapping we want to map the COW page in
2279 * R/O even though the entry protection could be R/W).
2280 *
2281 * set "pg" to the page we want to map in (uobjpage, usually)
2282 */
2283
2284 cpu_count(CPU_COUNT_FLT_OBJ, 1);
2285 if (UVM_ET_ISCOPYONWRITE(ufi->entry) ||
2286 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
2287 flt->enter_prot &= ~VM_PROT_WRITE;
2288 pg = uobjpage; /* map in the actual object */
2289
2290 KASSERT(uobjpage != PGO_DONTCARE);
2291
2292 /*
2293 * we are faulting directly on the page. be careful
2294 * about writing to loaned pages...
2295 */
2296
2297 if (uobjpage->loan_count) {
2298 uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage);
2299 }
2300 KASSERT(pg == uobjpage);
2301 KASSERT((pg->flags & PG_BUSY) == 0);
2302 return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg);
2303 }
2304
2305 /*
2306 * uvm_fault_lower_direct_loan: resolve loaned page.
2307 *
2308 * 1. if not cow'ing, adjust flt->enter_prot.
2309 * 2. if cow'ing, break loan.
2310 */
2311
2312 static int
uvm_fault_lower_direct_loan(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_page ** rpg,struct vm_page ** ruobjpage)2313 uvm_fault_lower_direct_loan(
2314 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2315 struct uvm_object *uobj, struct vm_page **rpg,
2316 struct vm_page **ruobjpage)
2317 {
2318 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2319 struct vm_page *pg;
2320 struct vm_page *uobjpage = *ruobjpage;
2321 int error;
2322 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2323
2324 if (!flt->cow_now) {
2325 /* read fault: cap the protection at readonly */
2326 /* cap! */
2327 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
2328 } else {
2329 /*
2330 * write fault: must break the loan here. to do this
2331 * we need a write lock on the object.
2332 */
2333
2334 error = uvm_fault_lower_upgrade(ufi, flt, amap, uobj, uobjpage);
2335 if (error != 0) {
2336 return error;
2337 }
2338 KASSERT(rw_write_held(uobj->vmobjlock));
2339
2340 pg = uvm_loanbreak(uobjpage);
2341 if (pg == NULL) {
2342
2343 uvmfault_unlockall(ufi, amap, uobj);
2344 UVMHIST_LOG(maphist,
2345 " out of RAM breaking loan, waiting",
2346 0,0,0,0);
2347 cpu_count(CPU_COUNT_FLTNORAM, 1);
2348 uvm_wait("flt_noram4");
2349 return ERESTART;
2350 }
2351 *rpg = pg;
2352 *ruobjpage = pg;
2353
2354 /*
2355 * drop ownership of page while still holding object lock,
2356 * which won't be dropped until the page is entered.
2357 */
2358
2359 uvm_pagelock(pg);
2360 uvm_pagewakeup(pg);
2361 uvm_pageunlock(pg);
2362 pg->flags &= ~PG_BUSY;
2363 UVM_PAGE_OWN(pg, NULL);
2364 }
2365 return 0;
2366 }
2367
2368 /*
2369 * uvm_fault_lower_promote: promote lower page.
2370 *
2371 * 1. call uvmfault_promote.
2372 * 2. fill in data.
2373 * 3. if not ZFOD, dispose old page.
2374 */
2375
2376 int
uvm_fault_lower_promote(struct uvm_faultinfo * ufi,struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_page * uobjpage)2377 uvm_fault_lower_promote(
2378 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2379 struct uvm_object *uobj, struct vm_page *uobjpage)
2380 {
2381 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2382 struct vm_anon *anon;
2383 struct vm_page *pg;
2384 int error;
2385 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2386
2387 KASSERT(amap != NULL);
2388
2389 /* promoting requires a write lock. */
2390 error = uvm_fault_upper_upgrade(ufi, flt, amap, uobj);
2391 if (error != 0) {
2392 return error;
2393 }
2394 KASSERT(rw_write_held(amap->am_lock));
2395 KASSERT(uobj == NULL ||
2396 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
2397
2398 /*
2399 * If we are going to promote the data to an anon we
2400 * allocate a blank anon here and plug it into our amap.
2401 */
2402 error = uvmfault_promote(ufi, NULL, uobjpage, &anon, &flt->anon_spare);
2403 switch (error) {
2404 case 0:
2405 break;
2406 case ERESTART:
2407 return ERESTART;
2408 default:
2409 return error;
2410 }
2411
2412 pg = anon->an_page;
2413
2414 /*
2415 * Fill in the data.
2416 */
2417
2418 if (uobjpage != PGO_DONTCARE) {
2419 cpu_count(CPU_COUNT_FLT_PRCOPY, 1);
2420
2421 /*
2422 * promote to shared amap? make sure all sharing
2423 * procs see it
2424 */
2425
2426 if ((amap_flags(amap) & AMAP_SHARED) != 0) {
2427 pmap_page_protect(uobjpage, VM_PROT_NONE);
2428 /*
2429 * XXX: PAGE MIGHT BE WIRED!
2430 */
2431 }
2432
2433 UVMHIST_LOG(maphist,
2434 " promote uobjpage %#jx to anon/page %#jx/%#jx",
2435 (uintptr_t)uobjpage, (uintptr_t)anon, (uintptr_t)pg, 0);
2436
2437 } else {
2438 cpu_count(CPU_COUNT_FLT_PRZERO, 1);
2439
2440 /*
2441 * Page is zero'd and marked dirty by
2442 * uvmfault_promote().
2443 */
2444
2445 UVMHIST_LOG(maphist," zero fill anon/page %#jx/%#jx",
2446 (uintptr_t)anon, (uintptr_t)pg, 0, 0);
2447 }
2448
2449 return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg);
2450 }
2451
2452 /*
2453 * uvm_fault_lower_enter: enter h/w mapping of lower page or anon page promoted
2454 * from the lower page.
2455 */
2456
2457 int
uvm_fault_lower_enter(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_anon * anon,struct vm_page * pg)2458 uvm_fault_lower_enter(
2459 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2460 struct uvm_object *uobj,
2461 struct vm_anon *anon, struct vm_page *pg)
2462 {
2463 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2464 const bool readonly = uvm_pagereadonly_p(pg);
2465 int error;
2466 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2467
2468 /*
2469 * Locked:
2470 *
2471 * maps(read), amap(if !null), uobj(if !null),
2472 * anon(if !null), pg(if anon), unlock_uobj(if !null)
2473 *
2474 * anon must be write locked (promotion). uobj can be either.
2475 *
2476 * Note: pg is either the uobjpage or the new page in the new anon.
2477 */
2478
2479 KASSERT(amap == NULL ||
2480 rw_lock_op(amap->am_lock) == flt->upper_lock_type);
2481 KASSERT(uobj == NULL ||
2482 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type);
2483 KASSERT(anon == NULL || anon->an_lock == amap->am_lock);
2484
2485 /*
2486 * note that pg can't be PG_RELEASED or PG_BUSY since we did
2487 * not drop the object lock since the last time we checked.
2488 */
2489
2490 KASSERT((pg->flags & PG_RELEASED) == 0);
2491 KASSERT((pg->flags & PG_BUSY) == 0);
2492
2493 /*
2494 * all resources are present. we can now map it in and free our
2495 * resources.
2496 */
2497
2498 UVMHIST_LOG(maphist,
2499 " MAPPING: case2: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd",
2500 (uintptr_t)ufi->orig_map->pmap, ufi->orig_rvaddr,
2501 (uintptr_t)pg, flt->promote);
2502 KASSERTMSG((flt->access_type & VM_PROT_WRITE) == 0 || !readonly,
2503 "promote=%u cow_now=%u access_type=%x enter_prot=%x cow=%u "
2504 "entry=%p map=%p orig_rvaddr=%p pg=%p",
2505 flt->promote, flt->cow_now, flt->access_type, flt->enter_prot,
2506 UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map,
2507 (void *)ufi->orig_rvaddr, pg);
2508 KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly);
2509 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
2510 VM_PAGE_TO_PHYS(pg),
2511 readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
2512 flt->access_type | PMAP_CANFAIL |
2513 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
2514
2515 /*
2516 * No need to undo what we did; we can simply think of
2517 * this as the pmap throwing away the mapping information.
2518 *
2519 * We do, however, have to go through the ReFault path,
2520 * as the map may change while we're asleep.
2521 */
2522
2523 /*
2524 * ensure that the page is queued in the case that
2525 * we just promoted the page.
2526 */
2527
2528 if (anon != NULL) {
2529 uvm_pagelock(pg);
2530 uvm_pageenqueue(pg);
2531 uvm_pagewakeup(pg);
2532 uvm_pageunlock(pg);
2533 }
2534
2535 uvmfault_unlockall(ufi, amap, uobj);
2536 if (!uvm_reclaimable()) {
2537 UVMHIST_LOG(maphist,
2538 "<- failed. out of VM",0,0,0,0);
2539 /* XXX instrumentation */
2540 error = ENOMEM;
2541 return error;
2542 }
2543 /* XXX instrumentation */
2544 uvm_wait("flt_pmfail2");
2545 return ERESTART;
2546 }
2547
2548 uvm_fault_lower_done(ufi, flt, uobj, pg);
2549 pmap_update(ufi->orig_map->pmap);
2550 uvmfault_unlockall(ufi, amap, uobj);
2551
2552 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
2553 return 0;
2554 }
2555
2556 /*
2557 * uvm_fault_lower_done: queue lower center page.
2558 */
2559
2560 void
uvm_fault_lower_done(struct uvm_faultinfo * ufi,const struct uvm_faultctx * flt,struct uvm_object * uobj,struct vm_page * pg)2561 uvm_fault_lower_done(
2562 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2563 struct uvm_object *uobj, struct vm_page *pg)
2564 {
2565
2566 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
2567
2568 if (flt->wire_paging) {
2569 uvm_pagelock(pg);
2570 uvm_pagewire(pg);
2571 uvm_pageunlock(pg);
2572 if (pg->flags & PG_AOBJ) {
2573
2574 /*
2575 * since the now-wired page cannot be paged out,
2576 * release its swap resources for others to use.
2577 * since an aobj page with no swap cannot be clean,
2578 * mark it dirty now.
2579 *
2580 * use pg->uobject here. if the page is from a
2581 * tmpfs vnode, the pages are backed by its UAO and
2582 * not the vnode.
2583 */
2584
2585 KASSERT(uobj != NULL);
2586 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock);
2587 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
2588 uao_dropswap(pg->uobject, pg->offset >> PAGE_SHIFT);
2589 }
2590 } else if (uvmpdpol_pageactivate_p(pg)) {
2591 /*
2592 * avoid re-activating the page unless needed,
2593 * to avoid false sharing on multiprocessor.
2594 */
2595
2596 uvm_pagelock(pg);
2597 uvm_pageactivate(pg);
2598 uvm_pageunlock(pg);
2599 }
2600 }
2601
2602
2603 /*
2604 * uvm_fault_wire: wire down a range of virtual addresses in a map.
2605 *
2606 * => map may be read-locked by caller, but MUST NOT be write-locked.
2607 * => if map is read-locked, any operations which may cause map to
2608 * be write-locked in uvm_fault() must be taken care of by
2609 * the caller. See uvm_map_pageable().
2610 */
2611
2612 int
uvm_fault_wire(struct vm_map * map,vaddr_t start,vaddr_t end,vm_prot_t access_type,int maxprot)2613 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
2614 vm_prot_t access_type, int maxprot)
2615 {
2616 vaddr_t va;
2617 int error;
2618
2619 /*
2620 * now fault it in a page at a time. if the fault fails then we have
2621 * to undo what we have done. note that in uvm_fault VM_PROT_NONE
2622 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
2623 */
2624
2625 /*
2626 * XXX work around overflowing a vaddr_t. this prevents us from
2627 * wiring the last page in the address space, though.
2628 */
2629 if (start > end) {
2630 return EFAULT;
2631 }
2632
2633 for (va = start; va < end; va += PAGE_SIZE) {
2634 error = uvm_fault_internal(map, va, access_type,
2635 (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE);
2636 if (error) {
2637 if (va != start) {
2638 uvm_fault_unwire(map, start, va);
2639 }
2640 return error;
2641 }
2642 }
2643 return 0;
2644 }
2645
2646 /*
2647 * uvm_fault_unwire(): unwire range of virtual space.
2648 */
2649
2650 void
uvm_fault_unwire(struct vm_map * map,vaddr_t start,vaddr_t end)2651 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
2652 {
2653 vm_map_lock_read(map);
2654 uvm_fault_unwire_locked(map, start, end);
2655 vm_map_unlock_read(map);
2656 }
2657
2658 /*
2659 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
2660 *
2661 * => map must be at least read-locked.
2662 */
2663
2664 void
uvm_fault_unwire_locked(struct vm_map * map,vaddr_t start,vaddr_t end)2665 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
2666 {
2667 struct vm_map_entry *entry, *oentry;
2668 pmap_t pmap = vm_map_pmap(map);
2669 vaddr_t va;
2670 paddr_t pa;
2671 struct vm_page *pg;
2672
2673 /*
2674 * we assume that the area we are unwiring has actually been wired
2675 * in the first place. this means that we should be able to extract
2676 * the PAs from the pmap. we also lock out the page daemon so that
2677 * we can call uvm_pageunwire.
2678 */
2679
2680 /*
2681 * find the beginning map entry for the region.
2682 */
2683
2684 KASSERT(start >= vm_map_min(map));
2685 KASSERT(end <= vm_map_max(map));
2686 if (uvm_map_lookup_entry(map, start, &entry) == false)
2687 panic("uvm_fault_unwire_locked: address not in map");
2688
2689 oentry = NULL;
2690 for (va = start; va < end; va += PAGE_SIZE) {
2691
2692 /*
2693 * find the map entry for the current address.
2694 */
2695
2696 KASSERT(va >= entry->start);
2697 while (va >= entry->end) {
2698 KASSERT(entry->next != &map->header);
2699 KASSERT(entry->next->start <= entry->end);
2700 entry = entry->next;
2701 }
2702
2703 /*
2704 * lock it.
2705 */
2706
2707 if (entry != oentry) {
2708 if (oentry != NULL) {
2709 uvm_map_unlock_entry(oentry);
2710 }
2711 uvm_map_lock_entry(entry, RW_WRITER);
2712 oentry = entry;
2713 }
2714
2715 /*
2716 * if the entry is no longer wired, tell the pmap.
2717 */
2718
2719 if (!pmap_extract(pmap, va, &pa))
2720 continue;
2721
2722 if (VM_MAPENT_ISWIRED(entry) == 0)
2723 pmap_unwire(pmap, va);
2724
2725 pg = PHYS_TO_VM_PAGE(pa);
2726 if (pg) {
2727 uvm_pagelock(pg);
2728 uvm_pageunwire(pg);
2729 uvm_pageunlock(pg);
2730 }
2731 }
2732
2733 if (oentry != NULL) {
2734 uvm_map_unlock_entry(entry);
2735 }
2736 }
2737