1 /* $OpenBSD: uvm_pdaemon.c,v 1.25 2002/05/24 13:10:53 art Exp $ */
2 /* $NetBSD: uvm_pdaemon.c,v 1.23 2000/08/20 10:24:14 bjh21 Exp $ */
3
4 /*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * Copyright (c) 1991, 1993, The Regents of the University of California.
7 *
8 * All rights reserved.
9 *
10 * This code is derived from software contributed to Berkeley by
11 * The Mach Operating System project at Carnegie-Mellon University.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 * must display the following acknowledgement:
23 * This product includes software developed by Charles D. Cranor,
24 * Washington University, the University of California, Berkeley and
25 * its contributors.
26 * 4. Neither the name of the University nor the names of its contributors
27 * may be used to endorse or promote products derived from this software
28 * without specific prior written permission.
29 *
30 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
31 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
34 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
36 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
37 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
38 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
39 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 * SUCH DAMAGE.
41 *
42 * @(#)vm_pageout.c 8.5 (Berkeley) 2/14/94
43 * from: Id: uvm_pdaemon.c,v 1.1.2.32 1998/02/06 05:26:30 chs Exp
44 *
45 *
46 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
47 * All rights reserved.
48 *
49 * Permission to use, copy, modify and distribute this software and
50 * its documentation is hereby granted, provided that both the copyright
51 * notice and this permission notice appear in all copies of the
52 * software, derivative works or modified versions, and any portions
53 * thereof, and that both notices appear in supporting documentation.
54 *
55 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
58 *
59 * Carnegie Mellon requests users of this software to return to
60 *
61 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
62 * School of Computer Science
63 * Carnegie Mellon University
64 * Pittsburgh PA 15213-3890
65 *
66 * any improvements or extensions that they make and grant Carnegie the
67 * rights to redistribute these changes.
68 */
69
70 /*
71 * uvm_pdaemon.c: the page daemon
72 */
73
74 #include <sys/param.h>
75 #include <sys/proc.h>
76 #include <sys/systm.h>
77 #include <sys/kernel.h>
78 #include <sys/pool.h>
79 #include <sys/buf.h>
80 #include <sys/vnode.h>
81
82 #include <uvm/uvm.h>
83
84 /*
85 * UVMPD_NUMDIRTYREACTS is how many dirty pages the pagedeamon will reactivate
86 * in a pass thru the inactive list when swap is full. the value should be
87 * "small"... if it's too large we'll cycle the active pages thru the inactive
88 * queue too quickly to for them to be referenced and avoid being freed.
89 */
90
91 #define UVMPD_NUMDIRTYREACTS 16
92
93
94 /*
95 * local prototypes
96 */
97
98 static void uvmpd_scan(void);
99 static boolean_t uvmpd_scan_inactive(struct pglist *);
100 static void uvmpd_tune(void);
101
102 /*
103 * uvm_wait: wait (sleep) for the page daemon to free some pages
104 *
105 * => should be called with all locks released
106 * => should _not_ be called by the page daemon (to avoid deadlock)
107 */
108
109 void
uvm_wait(wmsg)110 uvm_wait(wmsg)
111 const char *wmsg;
112 {
113 int timo = 0;
114 int s = splbio();
115
116 /*
117 * check for page daemon going to sleep (waiting for itself)
118 */
119
120 if (curproc == uvm.pagedaemon_proc) {
121 /*
122 * now we have a problem: the pagedaemon wants to go to
123 * sleep until it frees more memory. but how can it
124 * free more memory if it is asleep? that is a deadlock.
125 * we have two options:
126 * [1] panic now
127 * [2] put a timeout on the sleep, thus causing the
128 * pagedaemon to only pause (rather than sleep forever)
129 *
130 * note that option [2] will only help us if we get lucky
131 * and some other process on the system breaks the deadlock
132 * by exiting or freeing memory (thus allowing the pagedaemon
133 * to continue). for now we panic if DEBUG is defined,
134 * otherwise we hope for the best with option [2] (better
135 * yet, this should never happen in the first place!).
136 */
137
138 printf("pagedaemon: deadlock detected!\n");
139 timo = hz >> 3; /* set timeout */
140 #if defined(DEBUG)
141 /* DEBUG: panic so we can debug it */
142 panic("pagedaemon deadlock");
143 #endif
144 }
145
146 simple_lock(&uvm.pagedaemon_lock);
147 wakeup(&uvm.pagedaemon); /* wake the daemon! */
148 UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvm.pagedaemon_lock, FALSE, wmsg,
149 timo);
150
151 splx(s);
152 }
153
154
155 /*
156 * uvmpd_tune: tune paging parameters
157 *
158 * => called when ever memory is added (or removed?) to the system
159 * => caller must call with page queues locked
160 */
161
162 static void
uvmpd_tune()163 uvmpd_tune()
164 {
165 UVMHIST_FUNC("uvmpd_tune"); UVMHIST_CALLED(pdhist);
166
167 uvmexp.freemin = uvmexp.npages / 30;
168
169 /* between 16k and 512k */
170 /* XXX: what are these values good for? */
171 uvmexp.freemin = max(uvmexp.freemin, (16*1024) >> PAGE_SHIFT);
172 uvmexp.freemin = min(uvmexp.freemin, (512*1024) >> PAGE_SHIFT);
173
174 /* Make sure there's always a user page free. */
175 if (uvmexp.freemin < uvmexp.reserve_kernel + 1)
176 uvmexp.freemin = uvmexp.reserve_kernel + 1;
177
178 uvmexp.freetarg = (uvmexp.freemin * 4) / 3;
179 if (uvmexp.freetarg <= uvmexp.freemin)
180 uvmexp.freetarg = uvmexp.freemin + 1;
181
182 /* uvmexp.inactarg: computed in main daemon loop */
183
184 uvmexp.wiredmax = uvmexp.npages / 3;
185 UVMHIST_LOG(pdhist, "<- done, freemin=%d, freetarg=%d, wiredmax=%d",
186 uvmexp.freemin, uvmexp.freetarg, uvmexp.wiredmax, 0);
187 }
188
189 /*
190 * uvm_pageout: the main loop for the pagedaemon
191 */
192
193 void
uvm_pageout(void * arg)194 uvm_pageout(void *arg)
195 {
196 int npages = 0;
197 UVMHIST_FUNC("uvm_pageout"); UVMHIST_CALLED(pdhist);
198
199 UVMHIST_LOG(pdhist,"<starting uvm pagedaemon>", 0, 0, 0, 0);
200
201 /*
202 * ensure correct priority and set paging parameters...
203 */
204
205 uvm.pagedaemon_proc = curproc;
206 (void) spl0();
207 uvm_lock_pageq();
208 npages = uvmexp.npages;
209 uvmpd_tune();
210 uvm_unlock_pageq();
211
212 /*
213 * main loop
214 */
215
216 for (;;) {
217 simple_lock(&uvm.pagedaemon_lock);
218
219 UVMHIST_LOG(pdhist," <<SLEEPING>>",0,0,0,0);
220 UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon,
221 &uvm.pagedaemon_lock, FALSE, "pgdaemon", 0);
222 uvmexp.pdwoke++;
223 UVMHIST_LOG(pdhist," <<WOKE UP>>",0,0,0,0);
224
225 /* drain pool resources */
226 pool_drain(0);
227
228 /*
229 * now lock page queues and recompute inactive count
230 */
231
232 uvm_lock_pageq();
233 if (npages != uvmexp.npages) { /* check for new pages? */
234 npages = uvmexp.npages;
235 uvmpd_tune();
236 }
237
238 uvmexp.inactarg = (uvmexp.active + uvmexp.inactive) / 3;
239 if (uvmexp.inactarg <= uvmexp.freetarg) {
240 uvmexp.inactarg = uvmexp.freetarg + 1;
241 }
242
243 UVMHIST_LOG(pdhist," free/ftarg=%d/%d, inact/itarg=%d/%d",
244 uvmexp.free, uvmexp.freetarg, uvmexp.inactive,
245 uvmexp.inactarg);
246
247 /*
248 * scan if needed
249 */
250
251 #ifdef UBC
252 if (uvmexp.free + uvmexp.paging < uvmexp.freetarg ||
253 uvmexp.inactive < uvmexp.inactarg ||
254 uvm_pgcnt_vnode >
255 (uvmexp.active + uvmexp.inactive + uvmexp.wired +
256 uvmexp.free) * 13 / 16) {
257 #else
258 if (uvmexp.free < uvmexp.freetarg ||
259 uvmexp.inactive < uvmexp.inactarg) {
260 #endif
261 uvmpd_scan();
262 }
263
264 /*
265 * if there's any free memory to be had,
266 * wake up any waiters.
267 */
268
269 if (uvmexp.free > uvmexp.reserve_kernel ||
270 uvmexp.paging == 0) {
271 wakeup(&uvmexp.free);
272 }
273
274 /*
275 * scan done. unlock page queues (the only lock we are holding)
276 */
277
278 uvm_unlock_pageq();
279 }
280 /*NOTREACHED*/
281 }
282
283
284 /*
285 * uvm_aiodone_daemon: main loop for the aiodone daemon.
286 */
287
288 void
289 uvm_aiodone_daemon(void *arg)
290 {
291 int s, free;
292 struct buf *bp, *nbp;
293 UVMHIST_FUNC("uvm_aiodoned"); UVMHIST_CALLED(pdhist);
294
295 for (;;) {
296
297 /*
298 * carefully attempt to go to sleep (without losing "wakeups"!).
299 * we need splbio because we want to make sure the aio_done list
300 * is totally empty before we go to sleep.
301 */
302
303 s = splbio();
304 simple_lock(&uvm.aiodoned_lock);
305 if (TAILQ_FIRST(&uvm.aio_done) == NULL) {
306 UVMHIST_LOG(pdhist," <<SLEEPING>>",0,0,0,0);
307 UVM_UNLOCK_AND_WAIT(&uvm.aiodoned,
308 &uvm.aiodoned_lock, FALSE, "aiodoned", 0);
309 UVMHIST_LOG(pdhist," <<WOKE UP>>",0,0,0,0);
310
311 /* relock aiodoned_lock, still at splbio */
312 simple_lock(&uvm.aiodoned_lock);
313 }
314
315 /*
316 * check for done aio structures
317 */
318
319 bp = TAILQ_FIRST(&uvm.aio_done);
320 if (bp) {
321 TAILQ_INIT(&uvm.aio_done);
322 }
323
324 simple_unlock(&uvm.aiodoned_lock);
325 splx(s);
326
327 /*
328 * process each i/o that's done.
329 */
330
331 free = uvmexp.free;
332 while (bp != NULL) {
333 if (bp->b_flags & B_PDAEMON) {
334 uvmexp.paging -= bp->b_bufsize >> PAGE_SHIFT;
335 }
336 nbp = TAILQ_NEXT(bp, b_freelist);
337 s = splbio(); /* b_iodone must by called at splbio */
338 (*bp->b_iodone)(bp);
339 splx(s);
340 bp = nbp;
341 }
342 if (free <= uvmexp.reserve_kernel) {
343 s = uvm_lock_fpageq();
344 wakeup(&uvm.pagedaemon);
345 uvm_unlock_fpageq(s);
346 } else {
347 simple_lock(&uvm.pagedaemon_lock);
348 wakeup(&uvmexp.free);
349 simple_unlock(&uvm.pagedaemon_lock);
350 }
351 }
352 }
353
354
355
356 /*
357 * uvmpd_scan_inactive: scan an inactive list for pages to clean or free.
358 *
359 * => called with page queues locked
360 * => we work on meeting our free target by converting inactive pages
361 * into free pages.
362 * => we handle the building of swap-backed clusters
363 * => we return TRUE if we are exiting because we met our target
364 */
365
366 static boolean_t
367 uvmpd_scan_inactive(pglst)
368 struct pglist *pglst;
369 {
370 boolean_t retval = FALSE; /* assume we haven't hit target */
371 int s, free, result;
372 struct vm_page *p, *nextpg;
373 struct uvm_object *uobj;
374 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp;
375 int npages;
376 struct vm_page *swpps[MAXBSIZE >> PAGE_SHIFT]; /* XXX: see below */
377 int swnpages, swcpages; /* XXX: see below */
378 int swslot;
379 struct vm_anon *anon;
380 boolean_t swap_backed;
381 vaddr_t start;
382 int dirtyreacts;
383 UVMHIST_FUNC("uvmpd_scan_inactive"); UVMHIST_CALLED(pdhist);
384
385 /*
386 * note: we currently keep swap-backed pages on a separate inactive
387 * list from object-backed pages. however, merging the two lists
388 * back together again hasn't been ruled out. thus, we keep our
389 * swap cluster in "swpps" rather than in pps (allows us to mix
390 * clustering types in the event of a mixed inactive queue).
391 */
392
393 /*
394 * swslot is non-zero if we are building a swap cluster. we want
395 * to stay in the loop while we have a page to scan or we have
396 * a swap-cluster to build.
397 */
398
399 swslot = 0;
400 swnpages = swcpages = 0;
401 free = 0;
402 dirtyreacts = 0;
403
404 for (p = TAILQ_FIRST(pglst); p != NULL || swslot != 0; p = nextpg) {
405
406 /*
407 * note that p can be NULL iff we have traversed the whole
408 * list and need to do one final swap-backed clustered pageout.
409 */
410
411 uobj = NULL;
412 anon = NULL;
413
414 if (p) {
415
416 /*
417 * update our copy of "free" and see if we've met
418 * our target
419 */
420
421 s = uvm_lock_fpageq();
422 free = uvmexp.free;
423 uvm_unlock_fpageq(s);
424
425 if (free + uvmexp.paging >= uvmexp.freetarg << 2 ||
426 dirtyreacts == UVMPD_NUMDIRTYREACTS) {
427 UVMHIST_LOG(pdhist," met free target: "
428 "exit loop", 0, 0, 0, 0);
429 retval = TRUE;
430
431 if (swslot == 0) {
432 /* exit now if no swap-i/o pending */
433 break;
434 }
435
436 /* set p to null to signal final swap i/o */
437 p = NULL;
438 }
439 }
440
441 if (p) { /* if (we have a new page to consider) */
442
443 /*
444 * we are below target and have a new page to consider.
445 */
446 uvmexp.pdscans++;
447 nextpg = TAILQ_NEXT(p, pageq);
448
449 /*
450 * move referenced pages back to active queue and
451 * skip to next page (unlikely to happen since
452 * inactive pages shouldn't have any valid mappings
453 * and we cleared reference before deactivating).
454 */
455
456 if (pmap_is_referenced(p)) {
457 uvm_pageactivate(p);
458 uvmexp.pdreact++;
459 continue;
460 }
461
462 /*
463 * first we attempt to lock the object that this page
464 * belongs to. if our attempt fails we skip on to
465 * the next page (no harm done). it is important to
466 * "try" locking the object as we are locking in the
467 * wrong order (pageq -> object) and we don't want to
468 * deadlock.
469 *
470 * the only time we expect to see an ownerless page
471 * (i.e. a page with no uobject and !PQ_ANON) is if an
472 * anon has loaned a page from a uvm_object and the
473 * uvm_object has dropped the ownership. in that
474 * case, the anon can "take over" the loaned page
475 * and make it its own.
476 */
477
478 /* is page part of an anon or ownerless ? */
479 if ((p->pqflags & PQ_ANON) || p->uobject == NULL) {
480 anon = p->uanon;
481 KASSERT(anon != NULL);
482 if (!simple_lock_try(&anon->an_lock)) {
483 /* lock failed, skip this page */
484 continue;
485 }
486
487 /*
488 * if the page is ownerless, claim it in the
489 * name of "anon"!
490 */
491
492 if ((p->pqflags & PQ_ANON) == 0) {
493 KASSERT(p->loan_count > 0);
494 p->loan_count--;
495 p->pqflags |= PQ_ANON;
496 /* anon now owns it */
497 }
498 if (p->flags & PG_BUSY) {
499 simple_unlock(&anon->an_lock);
500 uvmexp.pdbusy++;
501 /* someone else owns page, skip it */
502 continue;
503 }
504 uvmexp.pdanscan++;
505 } else {
506 uobj = p->uobject;
507 KASSERT(uobj != NULL);
508 if (!simple_lock_try(&uobj->vmobjlock)) {
509 /* lock failed, skip this page */
510 continue;
511 }
512 if (p->flags & PG_BUSY) {
513 simple_unlock(&uobj->vmobjlock);
514 uvmexp.pdbusy++;
515 /* someone else owns page, skip it */
516 continue;
517 }
518 uvmexp.pdobscan++;
519 }
520
521 /*
522 * we now have the object and the page queues locked.
523 * the page is not busy. if the page is clean we
524 * can free it now and continue.
525 */
526
527 if (p->flags & PG_CLEAN) {
528 if (p->pqflags & PQ_SWAPBACKED) {
529 /* this page now lives only in swap */
530 simple_lock(&uvm.swap_data_lock);
531 uvmexp.swpgonly++;
532 simple_unlock(&uvm.swap_data_lock);
533 }
534
535 /* zap all mappings with pmap_page_protect... */
536 pmap_page_protect(p, VM_PROT_NONE);
537 uvm_pagefree(p);
538 uvmexp.pdfreed++;
539
540 if (anon) {
541
542 /*
543 * an anonymous page can only be clean
544 * if it has backing store assigned.
545 */
546
547 KASSERT(anon->an_swslot != 0);
548
549 /* remove from object */
550 anon->u.an_page = NULL;
551 simple_unlock(&anon->an_lock);
552 } else {
553 /* pagefree has already removed the
554 * page from the object */
555 simple_unlock(&uobj->vmobjlock);
556 }
557 continue;
558 }
559
560 /*
561 * this page is dirty, skip it if we'll have met our
562 * free target when all the current pageouts complete.
563 */
564
565 if (free + uvmexp.paging > uvmexp.freetarg << 2) {
566 if (anon) {
567 simple_unlock(&anon->an_lock);
568 } else {
569 simple_unlock(&uobj->vmobjlock);
570 }
571 continue;
572 }
573
574 /*
575 * this page is dirty, but we can't page it out
576 * since all pages in swap are only in swap.
577 * reactivate it so that we eventually cycle
578 * all pages thru the inactive queue.
579 */
580
581 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
582 if ((p->pqflags & PQ_SWAPBACKED) &&
583 uvmexp.swpgonly == uvmexp.swpages) {
584 dirtyreacts++;
585 uvm_pageactivate(p);
586 if (anon) {
587 simple_unlock(&anon->an_lock);
588 } else {
589 simple_unlock(&uobj->vmobjlock);
590 }
591 continue;
592 }
593
594 /*
595 * if the page is swap-backed and dirty and swap space
596 * is full, free any swap allocated to the page
597 * so that other pages can be paged out.
598 */
599
600 KASSERT(uvmexp.swpginuse <= uvmexp.swpages);
601 if ((p->pqflags & PQ_SWAPBACKED) &&
602 uvmexp.swpginuse == uvmexp.swpages) {
603
604 if ((p->pqflags & PQ_ANON) &&
605 p->uanon->an_swslot) {
606 uvm_swap_free(p->uanon->an_swslot, 1);
607 p->uanon->an_swslot = 0;
608 }
609 if (p->pqflags & PQ_AOBJ) {
610 uao_dropswap(p->uobject,
611 p->offset >> PAGE_SHIFT);
612 }
613 }
614
615 /*
616 * the page we are looking at is dirty. we must
617 * clean it before it can be freed. to do this we
618 * first mark the page busy so that no one else will
619 * touch the page. we write protect all the mappings
620 * of the page so that no one touches it while it is
621 * in I/O.
622 */
623
624 swap_backed = ((p->pqflags & PQ_SWAPBACKED) != 0);
625 p->flags |= PG_BUSY; /* now we own it */
626 UVM_PAGE_OWN(p, "scan_inactive");
627 pmap_page_protect(p, VM_PROT_READ);
628 uvmexp.pgswapout++;
629
630 /*
631 * for swap-backed pages we need to (re)allocate
632 * swap space.
633 */
634
635 if (swap_backed) {
636
637 /*
638 * free old swap slot (if any)
639 */
640
641 if (anon) {
642 if (anon->an_swslot) {
643 uvm_swap_free(anon->an_swslot,
644 1);
645 anon->an_swslot = 0;
646 }
647 } else {
648 uao_dropswap(uobj,
649 p->offset >> PAGE_SHIFT);
650 }
651
652 /*
653 * start new cluster (if necessary)
654 */
655
656 if (swslot == 0) {
657 swnpages = MAXBSIZE >> PAGE_SHIFT;
658 swslot = uvm_swap_alloc(&swnpages,
659 TRUE);
660 if (swslot == 0) {
661 /* no swap? give up! */
662 p->flags &= ~PG_BUSY;
663 UVM_PAGE_OWN(p, NULL);
664 if (anon)
665 simple_unlock(
666 &anon->an_lock);
667 else
668 simple_unlock(
669 &uobj->vmobjlock);
670 continue;
671 }
672 swcpages = 0; /* cluster is empty */
673 }
674
675 /*
676 * add block to cluster
677 */
678
679 swpps[swcpages] = p;
680 if (anon)
681 anon->an_swslot = swslot + swcpages;
682 else
683 uao_set_swslot(uobj,
684 p->offset >> PAGE_SHIFT,
685 swslot + swcpages);
686 swcpages++;
687 }
688 } else {
689
690 /* if p == NULL we must be doing a last swap i/o */
691 swap_backed = TRUE;
692 }
693
694 /*
695 * now consider doing the pageout.
696 *
697 * for swap-backed pages, we do the pageout if we have either
698 * filled the cluster (in which case (swnpages == swcpages) or
699 * run out of pages (p == NULL).
700 *
701 * for object pages, we always do the pageout.
702 */
703
704 if (swap_backed) {
705 if (p) { /* if we just added a page to cluster */
706 if (anon)
707 simple_unlock(&anon->an_lock);
708 else
709 simple_unlock(&uobj->vmobjlock);
710
711 /* cluster not full yet? */
712 if (swcpages < swnpages)
713 continue;
714 }
715
716 /* starting I/O now... set up for it */
717 npages = swcpages;
718 ppsp = swpps;
719 /* for swap-backed pages only */
720 start = (vaddr_t) swslot;
721
722 /* if this is final pageout we could have a few
723 * extra swap blocks */
724 if (swcpages < swnpages) {
725 uvm_swap_free(swslot + swcpages,
726 (swnpages - swcpages));
727 }
728 } else {
729 /* normal object pageout */
730 ppsp = pps;
731 npages = sizeof(pps) / sizeof(struct vm_page *);
732 /* not looked at because PGO_ALLPAGES is set */
733 start = 0;
734 }
735
736 /*
737 * now do the pageout.
738 *
739 * for swap_backed pages we have already built the cluster.
740 * for !swap_backed pages, uvm_pager_put will call the object's
741 * "make put cluster" function to build a cluster on our behalf.
742 *
743 * we pass the PGO_PDFREECLUST flag to uvm_pager_put to instruct
744 * it to free the cluster pages for us on a successful I/O (it
745 * always does this for un-successful I/O requests). this
746 * allows us to do clustered pageout without having to deal
747 * with cluster pages at this level.
748 *
749 * note locking semantics of uvm_pager_put with PGO_PDFREECLUST:
750 * IN: locked: uobj (if !swap_backed), page queues
751 * OUT: locked: uobj (if !swap_backed && result !=VM_PAGER_PEND)
752 * !locked: pageqs, uobj (if swap_backed || VM_PAGER_PEND)
753 *
754 * [the bit about VM_PAGER_PEND saves us one lock-unlock pair]
755 */
756
757 /* locked: uobj (if !swap_backed), page queues */
758 uvmexp.pdpageouts++;
759 result = uvm_pager_put(swap_backed ? NULL : uobj, p,
760 &ppsp, &npages, PGO_ALLPAGES|PGO_PDFREECLUST, start, 0);
761 /* locked: uobj (if !swap_backed && result != PEND) */
762 /* unlocked: pageqs, object (if swap_backed ||result == PEND) */
763
764 /*
765 * if we did i/o to swap, zero swslot to indicate that we are
766 * no longer building a swap-backed cluster.
767 */
768
769 if (swap_backed)
770 swslot = 0; /* done with this cluster */
771
772 /*
773 * first, we check for VM_PAGER_PEND which means that the
774 * async I/O is in progress and the async I/O done routine
775 * will clean up after us. in this case we move on to the
776 * next page.
777 *
778 * there is a very remote chance that the pending async i/o can
779 * finish _before_ we get here. if that happens, our page "p"
780 * may no longer be on the inactive queue. so we verify this
781 * when determining the next page (starting over at the head if
782 * we've lost our inactive page).
783 */
784
785 if (result == VM_PAGER_PEND) {
786 uvmexp.paging += npages;
787 uvm_lock_pageq();
788 uvmexp.pdpending++;
789 if (p) {
790 if (p->pqflags & PQ_INACTIVE)
791 nextpg = TAILQ_NEXT(p, pageq);
792 else
793 nextpg = TAILQ_FIRST(pglst);
794 } else {
795 nextpg = NULL;
796 }
797 continue;
798 }
799
800 #ifdef UBC
801 if (result == VM_PAGER_ERROR &&
802 curproc == uvm.pagedaemon_proc) {
803 uvm_lock_pageq();
804 nextpg = TAILQ_NEXT(p, pageq);
805 uvm_pageactivate(p);
806 continue;
807 }
808 #endif
809
810 /*
811 * clean up "p" if we have one
812 */
813
814 if (p) {
815 /*
816 * the I/O request to "p" is done and uvm_pager_put
817 * has freed any cluster pages it may have allocated
818 * during I/O. all that is left for us to do is
819 * clean up page "p" (which is still PG_BUSY).
820 *
821 * our result could be one of the following:
822 * VM_PAGER_OK: successful pageout
823 *
824 * VM_PAGER_AGAIN: tmp resource shortage, we skip
825 * to next page
826 * VM_PAGER_{FAIL,ERROR,BAD}: an error. we
827 * "reactivate" page to get it out of the way (it
828 * will eventually drift back into the inactive
829 * queue for a retry).
830 * VM_PAGER_UNLOCK: should never see this as it is
831 * only valid for "get" operations
832 */
833
834 /* relock p's object: page queues not lock yet, so
835 * no need for "try" */
836
837 /* !swap_backed case: already locked... */
838 if (swap_backed) {
839 if (anon)
840 simple_lock(&anon->an_lock);
841 else
842 simple_lock(&uobj->vmobjlock);
843 }
844
845 #ifdef DIAGNOSTIC
846 if (result == VM_PAGER_UNLOCK)
847 panic("pagedaemon: pageout returned "
848 "invalid 'unlock' code");
849 #endif
850
851 /* handle PG_WANTED now */
852 if (p->flags & PG_WANTED)
853 /* still holding object lock */
854 wakeup(p);
855
856 p->flags &= ~(PG_BUSY|PG_WANTED);
857 UVM_PAGE_OWN(p, NULL);
858
859 /* released during I/O? */
860 if (p->flags & PG_RELEASED) {
861 if (anon) {
862 /* remove page so we can get nextpg */
863 anon->u.an_page = NULL;
864
865 simple_unlock(&anon->an_lock);
866 uvm_anfree(anon); /* kills anon */
867 pmap_page_protect(p, VM_PROT_NONE);
868 anon = NULL;
869 uvm_lock_pageq();
870 nextpg = TAILQ_NEXT(p, pageq);
871 /* free released page */
872 uvm_pagefree(p);
873
874 } else {
875
876 /*
877 * pgo_releasepg nukes the page and
878 * gets "nextpg" for us. it returns
879 * with the page queues locked (when
880 * given nextpg ptr).
881 */
882
883 if (!uobj->pgops->pgo_releasepg(p,
884 &nextpg))
885 /* uobj died after release */
886 uobj = NULL;
887
888 /*
889 * lock page queues here so that they're
890 * always locked at the end of the loop.
891 */
892
893 uvm_lock_pageq();
894 }
895 } else { /* page was not released during I/O */
896 uvm_lock_pageq();
897 nextpg = TAILQ_NEXT(p, pageq);
898 if (result != VM_PAGER_OK) {
899 /* pageout was a failure... */
900 if (result != VM_PAGER_AGAIN)
901 uvm_pageactivate(p);
902 pmap_clear_reference(p);
903 /* XXXCDC: if (swap_backed) FREE p's
904 * swap block? */
905 } else {
906 /* pageout was a success... */
907 pmap_clear_reference(p);
908 pmap_clear_modify(p);
909 p->flags |= PG_CLEAN;
910 }
911 }
912
913 /*
914 * drop object lock (if there is an object left). do
915 * a safety check of nextpg to make sure it is on the
916 * inactive queue (it should be since PG_BUSY pages on
917 * the inactive queue can't be re-queued [note: not
918 * true for active queue]).
919 */
920
921 if (anon)
922 simple_unlock(&anon->an_lock);
923 else if (uobj)
924 simple_unlock(&uobj->vmobjlock);
925
926 } else {
927
928 /*
929 * if p is null in this loop, make sure it stays null
930 * in the next loop.
931 */
932
933 nextpg = NULL;
934
935 /*
936 * lock page queues here just so they're always locked
937 * at the end of the loop.
938 */
939
940 uvm_lock_pageq();
941 }
942
943 if (nextpg && (nextpg->pqflags & PQ_INACTIVE) == 0) {
944 nextpg = TAILQ_FIRST(pglst); /* reload! */
945 }
946 }
947 return (retval);
948 }
949
950 /*
951 * uvmpd_scan: scan the page queues and attempt to meet our targets.
952 *
953 * => called with pageq's locked
954 */
955
956 void
957 uvmpd_scan()
958 {
959 int s, free, inactive_shortage, swap_shortage, pages_freed;
960 struct vm_page *p, *nextpg;
961 struct uvm_object *uobj;
962 boolean_t got_it;
963 UVMHIST_FUNC("uvmpd_scan"); UVMHIST_CALLED(pdhist);
964
965 uvmexp.pdrevs++; /* counter */
966 uobj = NULL;
967
968 /*
969 * get current "free" page count
970 */
971 s = uvm_lock_fpageq();
972 free = uvmexp.free;
973 uvm_unlock_fpageq(s);
974
975 #ifndef __SWAP_BROKEN
976 /*
977 * swap out some processes if we are below our free target.
978 * we need to unlock the page queues for this.
979 */
980 if (free < uvmexp.freetarg) {
981 uvmexp.pdswout++;
982 UVMHIST_LOG(pdhist," free %d < target %d: swapout", free,
983 uvmexp.freetarg, 0, 0);
984 uvm_unlock_pageq();
985 uvm_swapout_threads();
986 uvm_lock_pageq();
987
988 }
989 #endif
990
991 /*
992 * now we want to work on meeting our targets. first we work on our
993 * free target by converting inactive pages into free pages. then
994 * we work on meeting our inactive target by converting active pages
995 * to inactive ones.
996 */
997
998 UVMHIST_LOG(pdhist, " starting 'free' loop",0,0,0,0);
999
1000 /*
1001 * alternate starting queue between swap and object based on the
1002 * low bit of uvmexp.pdrevs (which we bump by one each call).
1003 */
1004
1005 got_it = FALSE;
1006 pages_freed = uvmexp.pdfreed;
1007 if ((uvmexp.pdrevs & 1) != 0 && uvmexp.nswapdev != 0)
1008 got_it = uvmpd_scan_inactive(&uvm.page_inactive_swp);
1009 if (!got_it)
1010 got_it = uvmpd_scan_inactive(&uvm.page_inactive_obj);
1011 if (!got_it && (uvmexp.pdrevs & 1) == 0 && uvmexp.nswapdev != 0)
1012 (void) uvmpd_scan_inactive(&uvm.page_inactive_swp);
1013 pages_freed = uvmexp.pdfreed - pages_freed;
1014
1015 /*
1016 * we have done the scan to get free pages. now we work on meeting
1017 * our inactive target.
1018 */
1019
1020 inactive_shortage = uvmexp.inactarg - uvmexp.inactive;
1021
1022 /*
1023 * detect if we're not going to be able to page anything out
1024 * until we free some swap resources from active pages.
1025 */
1026
1027 swap_shortage = 0;
1028 if (uvmexp.free < uvmexp.freetarg &&
1029 uvmexp.swpginuse == uvmexp.swpages &&
1030 uvmexp.swpgonly < uvmexp.swpages &&
1031 pages_freed == 0) {
1032 swap_shortage = uvmexp.freetarg - uvmexp.free;
1033 }
1034
1035 UVMHIST_LOG(pdhist, " loop 2: inactive_shortage=%d swap_shortage=%d",
1036 inactive_shortage, swap_shortage,0,0);
1037 for (p = TAILQ_FIRST(&uvm.page_active);
1038 p != NULL && (inactive_shortage > 0 || swap_shortage > 0);
1039 p = nextpg) {
1040 nextpg = TAILQ_NEXT(p, pageq);
1041 if (p->flags & PG_BUSY)
1042 continue; /* quick check before trying to lock */
1043
1044 /*
1045 * lock the page's owner.
1046 */
1047 /* is page anon owned or ownerless? */
1048 if ((p->pqflags & PQ_ANON) || p->uobject == NULL) {
1049 KASSERT(p->uanon != NULL);
1050 if (!simple_lock_try(&p->uanon->an_lock))
1051 continue;
1052
1053 /* take over the page? */
1054 if ((p->pqflags & PQ_ANON) == 0) {
1055 KASSERT(p->loan_count > 0);
1056 p->loan_count--;
1057 p->pqflags |= PQ_ANON;
1058 }
1059 } else {
1060 if (!simple_lock_try(&p->uobject->vmobjlock))
1061 continue;
1062 }
1063
1064 /*
1065 * skip this page if it's busy.
1066 */
1067
1068 if ((p->flags & PG_BUSY) != 0) {
1069 if (p->pqflags & PQ_ANON)
1070 simple_unlock(&p->uanon->an_lock);
1071 else
1072 simple_unlock(&p->uobject->vmobjlock);
1073 continue;
1074 }
1075
1076 /*
1077 * if there's a shortage of swap, free any swap allocated
1078 * to this page so that other pages can be paged out.
1079 */
1080
1081 if (swap_shortage > 0) {
1082 if ((p->pqflags & PQ_ANON) && p->uanon->an_swslot) {
1083 uvm_swap_free(p->uanon->an_swslot, 1);
1084 p->uanon->an_swslot = 0;
1085 p->flags &= ~PG_CLEAN;
1086 swap_shortage--;
1087 }
1088 if (p->pqflags & PQ_AOBJ) {
1089 int slot = uao_set_swslot(p->uobject,
1090 p->offset >> PAGE_SHIFT, 0);
1091 if (slot) {
1092 uvm_swap_free(slot, 1);
1093 p->flags &= ~PG_CLEAN;
1094 swap_shortage--;
1095 }
1096 }
1097 }
1098
1099 /*
1100 * deactivate this page if there's a shortage of
1101 * inactive pages.
1102 */
1103
1104 if (inactive_shortage > 0) {
1105 pmap_page_protect(p, VM_PROT_NONE);
1106 /* no need to check wire_count as pg is "active" */
1107 uvm_pagedeactivate(p);
1108 uvmexp.pddeact++;
1109 inactive_shortage--;
1110 }
1111 if (p->pqflags & PQ_ANON)
1112 simple_unlock(&p->uanon->an_lock);
1113 else
1114 simple_unlock(&p->uobject->vmobjlock);
1115 }
1116 }
1117