1 /*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 * Copyright (c) 2005 Yahoo! Technologies Norway AS
9 * All rights reserved.
10 *
11 * This code is derived from software contributed to Berkeley by
12 * The Mach Operating System project at Carnegie-Mellon University.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
16 * are met:
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 * 3. All advertising materials mentioning features or use of this software
23 * must display the following acknowledgement:
24 * This product includes software developed by the University of
25 * California, Berkeley and its contributors.
26 * 4. Neither the name of the University nor the names of its contributors
27 * may be used to endorse or promote products derived from this software
28 * without specific prior written permission.
29 *
30 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
31 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
34 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
36 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
37 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
38 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
39 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 * SUCH DAMAGE.
41 *
42 * from: @(#)vm_pageout.c 7.4 (Berkeley) 5/7/91
43 *
44 *
45 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46 * All rights reserved.
47 *
48 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
49 *
50 * Permission to use, copy, modify and distribute this software and
51 * its documentation is hereby granted, provided that both the copyright
52 * notice and this permission notice appear in all copies of the
53 * software, derivative works or modified versions, and any portions
54 * thereof, and that both notices appear in supporting documentation.
55 *
56 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
57 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
58 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
59 *
60 * Carnegie Mellon requests users of this software to return to
61 *
62 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
63 * School of Computer Science
64 * Carnegie Mellon University
65 * Pittsburgh PA 15213-3890
66 *
67 * any improvements or extensions that they make and grant Carnegie the
68 * rights to redistribute these changes.
69 */
70
71 /*
72 * The proverbial page-out daemon.
73 */
74
75 #include <sys/cdefs.h>
76 __FBSDID("$FreeBSD$");
77
78 #include "opt_vm.h"
79
80 #include <sys/param.h>
81 #include <sys/systm.h>
82 #include <sys/kernel.h>
83 #include <sys/eventhandler.h>
84 #include <sys/lock.h>
85 #include <sys/mutex.h>
86 #include <sys/proc.h>
87 #include <sys/kthread.h>
88 #include <sys/ktr.h>
89 #include <sys/mount.h>
90 #include <sys/racct.h>
91 #include <sys/resourcevar.h>
92 #include <sys/sched.h>
93 #include <sys/sdt.h>
94 #include <sys/signalvar.h>
95 #include <sys/smp.h>
96 #include <sys/time.h>
97 #include <sys/vnode.h>
98 #include <sys/vmmeter.h>
99 #include <sys/rwlock.h>
100 #include <sys/sx.h>
101 #include <sys/sysctl.h>
102
103 #include <vm/vm.h>
104 #include <vm/vm_param.h>
105 #include <vm/vm_object.h>
106 #include <vm/vm_page.h>
107 #include <vm/vm_map.h>
108 #include <vm/vm_pageout.h>
109 #include <vm/vm_pager.h>
110 #include <vm/vm_phys.h>
111 #include <vm/vm_reserv.h>
112 #include <vm/swap_pager.h>
113 #include <vm/vm_extern.h>
114 #include <vm/uma.h>
115
116 /*
117 * System initialization
118 */
119
120 /* the kernel process "vm_pageout"*/
121 static void vm_pageout(void);
122 static void vm_pageout_init(void);
123 static int vm_pageout_clean(vm_page_t m);
124 static int vm_pageout_cluster(vm_page_t m);
125 static void vm_pageout_scan(struct vm_domain *vmd, int pass);
126 static boolean_t vm_pageout_candidate(vm_page_t);
127 static boolean_t vm_pageout_contig(vm_page_t p_start, u_long npages, int level);
128 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
129 int starting_page_shortage);
130
131 SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
132 NULL);
133
134 struct proc *pageproc;
135
136 static struct kproc_desc page_kp = {
137 "pagedaemon",
138 vm_pageout,
139 &pageproc
140 };
141 SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
142 &page_kp);
143
144 SDT_PROVIDER_DEFINE(vm);
145 SDT_PROBE_DEFINE(vm, , , vm__lowmem_cache);
146 SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan);
147
148 #if !defined(NO_SWAPPING)
149 /* the kernel process "vm_daemon"*/
150 static void vm_daemon(void);
151 static struct proc *vmproc;
152
153 static struct kproc_desc vm_kp = {
154 "vmdaemon",
155 vm_daemon,
156 &vmproc
157 };
158 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
159 #endif
160
161
162 int vm_pages_needed; /* Event on which pageout daemon sleeps */
163 int vm_pageout_deficit; /* Estimated number of pages deficit */
164 int vm_pageout_wakeup_thresh;
165 static int vm_pageout_oom_seq = 12;
166
167 #if !defined(NO_SWAPPING)
168 static int vm_pageout_req_swapout; /* XXX */
169 static int vm_daemon_needed;
170 static struct mtx vm_daemon_mtx;
171 /* Allow for use by vm_pageout before vm_daemon is initialized. */
172 MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
173 #endif
174 static int vm_max_launder = 32;
175 static int vm_pageout_update_period;
176 static int defer_swap_pageouts;
177 static int disable_swap_pageouts;
178 static int lowmem_period = 10;
179 static time_t lowmem_uptime;
180
181 #if defined(NO_SWAPPING)
182 static int vm_swap_enabled = 0;
183 static int vm_swap_idle_enabled = 0;
184 #else
185 static int vm_swap_enabled = 1;
186 static int vm_swap_idle_enabled = 0;
187 #endif
188
189 static int vm_panic_on_oom = 0;
190
191 SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
192 CTLFLAG_RWTUN, &vm_panic_on_oom, 0,
193 "panic on out of memory instead of killing the largest process");
194
195 SYSCTL_INT(_vm, OID_AUTO, pageout_wakeup_thresh,
196 CTLFLAG_RW, &vm_pageout_wakeup_thresh, 0,
197 "free page threshold for waking up the pageout daemon");
198
199 SYSCTL_INT(_vm, OID_AUTO, max_launder,
200 CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
201
202 SYSCTL_INT(_vm, OID_AUTO, pageout_update_period,
203 CTLFLAG_RW, &vm_pageout_update_period, 0,
204 "Maximum active LRU update period");
205
206 SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RW, &lowmem_period, 0,
207 "Low memory callback period");
208
209 #if defined(NO_SWAPPING)
210 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
211 CTLFLAG_RD, &vm_swap_enabled, 0, "Enable entire process swapout");
212 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
213 CTLFLAG_RD, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
214 #else
215 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
216 CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
217 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
218 CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
219 #endif
220
221 SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
222 CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
223
224 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
225 CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
226
227 static int pageout_lock_miss;
228 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
229 CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
230
231 SYSCTL_INT(_vm, OID_AUTO, pageout_oom_seq,
232 CTLFLAG_RW, &vm_pageout_oom_seq, 0,
233 "back-to-back calls to oom detector to start OOM");
234
235 #define VM_PAGEOUT_PAGE_COUNT 16
236 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
237
238 int vm_page_max_wired; /* XXX max # of wired pages system-wide */
239 SYSCTL_INT(_vm, OID_AUTO, max_wired,
240 CTLFLAG_RW, &vm_page_max_wired, 0, "System-wide limit to wired page count");
241
242 static boolean_t vm_pageout_fallback_object_lock(vm_page_t, vm_page_t *);
243 #if !defined(NO_SWAPPING)
244 static void vm_pageout_map_deactivate_pages(vm_map_t, long);
245 static void vm_pageout_object_deactivate_pages(pmap_t, vm_object_t, long);
246 static void vm_req_vmdaemon(int req);
247 #endif
248 static boolean_t vm_pageout_page_lock(vm_page_t, vm_page_t *);
249
250 /*
251 * Initialize a dummy page for marking the caller's place in the specified
252 * paging queue. In principle, this function only needs to set the flag
253 * PG_MARKER. Nonetheless, it wirte busies and initializes the hold count
254 * to one as safety precautions.
255 */
256 static void
vm_pageout_init_marker(vm_page_t marker,u_short queue)257 vm_pageout_init_marker(vm_page_t marker, u_short queue)
258 {
259
260 bzero(marker, sizeof(*marker));
261 marker->flags = PG_MARKER;
262 marker->busy_lock = VPB_SINGLE_EXCLUSIVER;
263 marker->queue = queue;
264 marker->hold_count = 1;
265 }
266
267 /*
268 * vm_pageout_fallback_object_lock:
269 *
270 * Lock vm object currently associated with `m'. VM_OBJECT_TRYWLOCK is
271 * known to have failed and page queue must be either PQ_ACTIVE or
272 * PQ_INACTIVE. To avoid lock order violation, unlock the page queues
273 * while locking the vm object. Use marker page to detect page queue
274 * changes and maintain notion of next page on page queue. Return
275 * TRUE if no changes were detected, FALSE otherwise. vm object is
276 * locked on return.
277 *
278 * This function depends on both the lock portion of struct vm_object
279 * and normal struct vm_page being type stable.
280 */
281 static boolean_t
vm_pageout_fallback_object_lock(vm_page_t m,vm_page_t * next)282 vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
283 {
284 struct vm_page marker;
285 struct vm_pagequeue *pq;
286 boolean_t unchanged;
287 u_short queue;
288 vm_object_t object;
289
290 VM_ASSERT((m->flags & PG_PAQUEUE) == 0);
291 queue = m->queue;
292 vm_pageout_init_marker(&marker, queue);
293 pq = vm_page_pagequeue(m);
294 object = m->object;
295
296 TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
297 vm_pagequeue_unlock(pq);
298 vm_page_unlock(m);
299 VM_OBJECT_WLOCK(object);
300 vm_page_lock(m);
301 vm_pagequeue_lock(pq);
302
303 /*
304 * The page's object might have changed, and/or the page might
305 * have moved from its original position in the queue. If the
306 * page's object has changed, then the caller should abandon
307 * processing the page because the wrong object lock was
308 * acquired. Use the marker's plinks.q, not the page's, to
309 * determine if the page has been moved. The state of the
310 * page's plinks.q can be indeterminate; whereas, the marker's
311 * plinks.q must be valid.
312 */
313 *next = TAILQ_NEXT(&marker, plinks.q);
314 unchanged = m->object == object &&
315 m == TAILQ_PREV(&marker, pglist, plinks.q);
316 KASSERT(!unchanged || m->queue == queue,
317 ("page %p queue %d %d", m, queue, m->queue));
318 TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
319 return (unchanged);
320 }
321
322 /*
323 * Lock the page while holding the page queue lock. Use marker page
324 * to detect page queue changes and maintain notion of next page on
325 * page queue. Return TRUE if no changes were detected, FALSE
326 * otherwise. The page is locked on return. The page queue lock might
327 * be dropped and reacquired.
328 *
329 * This function depends on normal struct vm_page being type stable.
330 */
331 static boolean_t
vm_pageout_page_lock(vm_page_t m,vm_page_t * next)332 vm_pageout_page_lock(vm_page_t m, vm_page_t *next)
333 {
334 struct vm_page marker;
335 struct vm_pagequeue *pq;
336 boolean_t unchanged;
337 u_short queue;
338
339 vm_page_lock_assert(m, MA_NOTOWNED);
340 if (vm_page_trylock(m))
341 return (TRUE);
342
343 VM_ASSERT((m->flags & PG_PAQUEUE) == 0);
344 queue = m->queue;
345 vm_pageout_init_marker(&marker, queue);
346 pq = vm_page_pagequeue(m);
347
348 TAILQ_INSERT_AFTER(&pq->pq_pl, m, &marker, plinks.q);
349 vm_pagequeue_unlock(pq);
350 vm_page_lock(m);
351 vm_pagequeue_lock(pq);
352
353 /* Page queue might have changed. */
354 *next = TAILQ_NEXT(&marker, plinks.q);
355 unchanged = m == TAILQ_PREV(&marker, pglist, plinks.q);
356 KASSERT(!unchanged || m->queue == queue,
357 ("page %p queue %d %d", m, queue, m->queue));
358 TAILQ_REMOVE(&pq->pq_pl, &marker, plinks.q);
359 return (unchanged);
360 }
361
362 /*
363 * vm_pageout_clean:
364 *
365 * Clean the page and remove it from the laundry.
366 *
367 * We set the busy bit to cause potential page faults on this page to
368 * block. Note the careful timing, however, the busy bit isn't set till
369 * late and we cannot do anything that will mess with the page.
370 */
371 static int
vm_pageout_cluster(vm_page_t m)372 vm_pageout_cluster(vm_page_t m)
373 {
374 vm_object_t object;
375 vm_page_t mc[2*vm_pageout_page_count], pb, ps;
376 int pageout_count;
377 int ib, is, page_base;
378 vm_pindex_t pindex = m->pindex;
379
380 vm_page_lock_assert(m, MA_OWNED);
381 object = m->object;
382 VM_OBJECT_ASSERT_WLOCKED(object);
383
384 /*
385 * It doesn't cost us anything to pageout OBJT_DEFAULT or OBJT_SWAP
386 * with the new swapper, but we could have serious problems paging
387 * out other object types if there is insufficient memory.
388 *
389 * Unfortunately, checking free memory here is far too late, so the
390 * check has been moved up a procedural level.
391 */
392
393 /*
394 * Can't clean the page if it's busy or held.
395 */
396 vm_page_assert_unbusied(m);
397 KASSERT(m->hold_count == 0, ("vm_pageout_clean: page %p is held", m));
398 vm_page_unlock(m);
399
400 mc[vm_pageout_page_count] = pb = ps = m;
401 pageout_count = 1;
402 page_base = vm_pageout_page_count;
403 ib = 1;
404 is = 1;
405
406 /*
407 * Scan object for clusterable pages.
408 *
409 * We can cluster ONLY if: ->> the page is NOT
410 * clean, wired, busy, held, or mapped into a
411 * buffer, and one of the following:
412 * 1) The page is inactive, or a seldom used
413 * active page.
414 * -or-
415 * 2) we force the issue.
416 *
417 * During heavy mmap/modification loads the pageout
418 * daemon can really fragment the underlying file
419 * due to flushing pages out of order and not trying
420 * align the clusters (which leave sporatic out-of-order
421 * holes). To solve this problem we do the reverse scan
422 * first and attempt to align our cluster, then do a
423 * forward scan if room remains.
424 */
425 more:
426 while (ib && pageout_count < vm_pageout_page_count) {
427 vm_page_t p;
428
429 if (ib > pindex) {
430 ib = 0;
431 break;
432 }
433
434 if ((p = vm_page_prev(pb)) == NULL || vm_page_busied(p)) {
435 ib = 0;
436 break;
437 }
438 vm_page_test_dirty(p);
439 if (p->dirty == 0) {
440 ib = 0;
441 break;
442 }
443 vm_page_lock(p);
444 if (p->queue != PQ_INACTIVE ||
445 p->hold_count != 0) { /* may be undergoing I/O */
446 vm_page_unlock(p);
447 ib = 0;
448 break;
449 }
450 vm_page_unlock(p);
451 mc[--page_base] = pb = p;
452 ++pageout_count;
453 ++ib;
454 /*
455 * alignment boundry, stop here and switch directions. Do
456 * not clear ib.
457 */
458 if ((pindex - (ib - 1)) % vm_pageout_page_count == 0)
459 break;
460 }
461
462 while (pageout_count < vm_pageout_page_count &&
463 pindex + is < object->size) {
464 vm_page_t p;
465
466 if ((p = vm_page_next(ps)) == NULL || vm_page_busied(p))
467 break;
468 vm_page_test_dirty(p);
469 if (p->dirty == 0)
470 break;
471 vm_page_lock(p);
472 if (p->queue != PQ_INACTIVE ||
473 p->hold_count != 0) { /* may be undergoing I/O */
474 vm_page_unlock(p);
475 break;
476 }
477 vm_page_unlock(p);
478 mc[page_base + pageout_count] = ps = p;
479 ++pageout_count;
480 ++is;
481 }
482
483 /*
484 * If we exhausted our forward scan, continue with the reverse scan
485 * when possible, even past a page boundry. This catches boundry
486 * conditions.
487 */
488 if (ib && pageout_count < vm_pageout_page_count)
489 goto more;
490
491 /*
492 * we allow reads during pageouts...
493 */
494 return (vm_pageout_flush(&mc[page_base], pageout_count, 0, 0, NULL,
495 NULL));
496 }
497
498 /*
499 * vm_pageout_flush() - launder the given pages
500 *
501 * The given pages are laundered. Note that we setup for the start of
502 * I/O ( i.e. busy the page ), mark it read-only, and bump the object
503 * reference count all in here rather then in the parent. If we want
504 * the parent to do more sophisticated things we may have to change
505 * the ordering.
506 *
507 * Returned runlen is the count of pages between mreq and first
508 * page after mreq with status VM_PAGER_AGAIN.
509 * *eio is set to TRUE if pager returned VM_PAGER_ERROR or VM_PAGER_FAIL
510 * for any page in runlen set.
511 */
512 int
vm_pageout_flush(vm_page_t * mc,int count,int flags,int mreq,int * prunlen,boolean_t * eio)513 vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen,
514 boolean_t *eio)
515 {
516 vm_object_t object = mc[0]->object;
517 int pageout_status[count];
518 int numpagedout = 0;
519 int i, runlen;
520
521 VM_OBJECT_ASSERT_WLOCKED(object);
522
523 /*
524 * Initiate I/O. Bump the vm_page_t->busy counter and
525 * mark the pages read-only.
526 *
527 * We do not have to fixup the clean/dirty bits here... we can
528 * allow the pager to do it after the I/O completes.
529 *
530 * NOTE! mc[i]->dirty may be partial or fragmented due to an
531 * edge case with file fragments.
532 */
533 for (i = 0; i < count; i++) {
534 KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL,
535 ("vm_pageout_flush: partially invalid page %p index %d/%d",
536 mc[i], i, count));
537 vm_page_sbusy(mc[i]);
538 pmap_remove_write(mc[i]);
539 }
540 vm_object_pip_add(object, count);
541
542 vm_pager_put_pages(object, mc, count, flags, pageout_status);
543
544 runlen = count - mreq;
545 if (eio != NULL)
546 *eio = FALSE;
547 for (i = 0; i < count; i++) {
548 vm_page_t mt = mc[i];
549
550 KASSERT(pageout_status[i] == VM_PAGER_PEND ||
551 !pmap_page_is_write_mapped(mt),
552 ("vm_pageout_flush: page %p is not write protected", mt));
553 switch (pageout_status[i]) {
554 case VM_PAGER_OK:
555 case VM_PAGER_PEND:
556 numpagedout++;
557 break;
558 case VM_PAGER_BAD:
559 /*
560 * Page outside of range of object. Right now we
561 * essentially lose the changes by pretending it
562 * worked.
563 */
564 vm_page_undirty(mt);
565 break;
566 case VM_PAGER_ERROR:
567 case VM_PAGER_FAIL:
568 /*
569 * If page couldn't be paged out, then reactivate the
570 * page so it doesn't clog the inactive list. (We
571 * will try paging out it again later).
572 */
573 vm_page_lock(mt);
574 vm_page_activate(mt);
575 vm_page_unlock(mt);
576 if (eio != NULL && i >= mreq && i - mreq < runlen)
577 *eio = TRUE;
578 break;
579 case VM_PAGER_AGAIN:
580 if (i >= mreq && i - mreq < runlen)
581 runlen = i - mreq;
582 break;
583 }
584
585 /*
586 * If the operation is still going, leave the page busy to
587 * block all other accesses. Also, leave the paging in
588 * progress indicator set so that we don't attempt an object
589 * collapse.
590 */
591 if (pageout_status[i] != VM_PAGER_PEND) {
592 vm_object_pip_wakeup(object);
593 vm_page_sunbusy(mt);
594 }
595 }
596 if (prunlen != NULL)
597 *prunlen = runlen;
598 return (numpagedout);
599 }
600
601 static __noinline boolean_t
vm_pageout_candidate(vm_page_t p)602 vm_pageout_candidate(vm_page_t p)
603 {
604 /* Can't pageout wired, busy, or held pages */
605 if (p->wire_count || p->hold_count ||
606 (p->oflags & VPO_UNMANAGED) != 0 ||
607 vm_page_busied(p))
608 return (FALSE);
609 return (TRUE);
610 }
611
612 /*
613 * Count the number of pages requiring pageout processing to create
614 * a contiguous free block.
615 *
616 * The interpretation of level is as follows:
617 * 0: only inactive or free pages
618 * 1: include dirty pages
619 * 2: include active pages
620 *
621 * Held, busy, or wired pages cause a return of -1.
622 */
623 int
vm_pageout_count_pages(vm_page_t p_start,u_long npages,int level)624 vm_pageout_count_pages(vm_page_t p_start, u_long npages, int level)
625 {
626 vm_page_t p;
627 int i, workpages;
628
629 workpages = 0;
630 for (i = 0; i < npages; i++) {
631 /*
632 * All of these checks are done locklessly
633 * because the results can change as soon
634 * as we return. This is a best-effort
635 * interface.
636 */
637 p = &p_start[i];
638 if (!vm_pageout_candidate(p))
639 return (-1);
640 /* Dirty is not coherent until we check pmap. */
641 if (p->dirty) {
642 if (level < 1)
643 return (-1);
644 workpages++;
645 continue;
646 }
647 if (p->queue == PQ_ACTIVE) {
648 if (level < 2)
649 return (-1);
650 workpages++;
651 continue;
652 }
653 /*
654 * Only count INACTIVE and reservations against level 0.
655 */
656 if (level > 0)
657 continue;
658 if (p->queue == PQ_INACTIVE)
659 workpages++;
660 else if (p->order == VM_NFREEORDER && p->queue == PQ_NONE)
661 workpages++;
662 }
663
664 return (workpages);
665 }
666
667 static boolean_t
vm_pageout_contig(vm_page_t p_start,u_long npages,int level)668 vm_pageout_contig(vm_page_t p_start, u_long npages, int level)
669 {
670 vm_object_t object;
671 vm_page_t p;
672 int i;
673
674 if (level > 1) {
675 /* Scan and deactivate. */
676 for (i = 0; i < npages; i++) {
677 p = &p_start[i];
678 vm_page_lock(p);
679 if (!vm_pageout_candidate(p)) {
680 vm_page_unlock(p);
681 return (FALSE);
682 }
683 if (p->queue != PQ_ACTIVE) {
684 vm_page_unlock(p);
685 continue;
686 }
687 vm_page_deactivate(p);
688 vm_page_unlock(p);
689 }
690 }
691 /* Scan and free, cleaning if allowed. */
692 for (i = 0; i < npages; i++) {
693 p = &p_start[i];
694 object = p->object;
695 if (object == NULL) {
696 vm_page_lock(p);
697 if (object != NULL) {
698 vm_page_unlock(p);
699 i--; /* Retry */
700 continue;
701 }
702 if (!vm_pageout_candidate(p)) {
703 vm_page_unlock(p);
704 return (FALSE);
705 }
706 #if VM_NRESERVLEVEL > 0
707 if (p->order == VM_NFREEORDER && p->queue == PQ_NONE) {
708 mtx_lock(&vm_page_queue_free_mtx);
709 vm_page_unlock(p);
710 vm_reserv_reclaim_page(p);
711 mtx_unlock(&vm_page_queue_free_mtx);
712 } else
713 #endif
714 vm_page_unlock(p);
715 continue;
716 }
717 VM_OBJECT_WLOCK(object);
718 if (object != p->object) {
719 VM_OBJECT_WUNLOCK(object);
720 i--; /* Retry this page. */
721 continue;
722 }
723 vm_page_lock(p);
724 if (!vm_pageout_candidate(p))
725 goto fail;
726 if (p->queue != PQ_INACTIVE) {
727 vm_page_unlock(p);
728 VM_OBJECT_WUNLOCK(object);
729 continue;
730 }
731 pmap_remove_all(p);
732 if (p->dirty) {
733 /* Have we been asked to clean dirty pages? */
734 if (level < 1)
735 goto fail;
736 /*
737 * We don't bother paging objects that are "dead".
738 */
739 if (object->flags & OBJ_DEAD)
740 goto fail;
741 if (disable_swap_pageouts &&
742 (object->type == OBJT_SWAP ||
743 object->type == OBJT_DEFAULT))
744 goto fail;
745 if (vm_pageout_clean(p))
746 return (FALSE);
747 } else {
748 vm_page_free(p);
749 vm_page_unlock(p);
750 VM_OBJECT_WUNLOCK(object);
751 }
752 }
753 return (TRUE);
754
755 fail:
756 vm_page_unlock(p);
757 VM_OBJECT_WUNLOCK(object);
758 return (FALSE);
759 }
760 /*
761 * Attempt to free a contiguous region of physical memory within the
762 * specified boundaries.
763 *
764 * 0: All clean, inactive pages within the specified physical address range
765 * are freed. Will not sleep.
766 * 1: The vm_lowmem handlers are called. All inactive pages within
767 * the specified physical address range are reclaimed. May sleep.
768 * 2: The vm_lowmem handlers are called. All inactive and active pages
769 * within the specified physical address range are freed. May sleep.
770 */
771 int
vm_pageout_reclaim_contig(u_long npages,vm_paddr_t low,vm_paddr_t high,u_long alignment,vm_paddr_t boundary,int level)772 vm_pageout_reclaim_contig(u_long npages, vm_paddr_t low, vm_paddr_t high,
773 u_long alignment, vm_paddr_t boundary, int level)
774 {
775 vm_page_t p, lastp;
776
777 if (level > 0) {
778 /*
779 * Decrease registered cache sizes. The vm_lowmem handlers
780 * may acquire locks and/or sleep, so they can only be invoked
781 * when "level" is greater than zero.
782 */
783 SDT_PROBE0(vm, , , vm__lowmem_cache);
784 EVENTHANDLER_INVOKE(vm_lowmem, 0);
785
786 /*
787 * We do this explicitly after the caches have been drained
788 * above.
789 */
790 uma_reclaim();
791 }
792 lastp = NULL;
793 for (;;) {
794 mtx_lock(&vm_page_queue_free_mtx);
795 p = vm_phys_reclaim_contig(npages, low, high, alignment, boundary,
796 level);
797 mtx_unlock(&vm_page_queue_free_mtx);
798 if (p == NULL)
799 break;
800 if (vm_pageout_contig(p, npages, level) == 0)
801 return (0);
802 /*
803 * Prevent looping if pageout_pages failed but the same
804 * group was selected again.
805 */
806 if (p == lastp)
807 break;
808 lastp = p;
809 }
810 return (ENOENT);
811 }
812
813 #if !defined(NO_SWAPPING)
814 /*
815 * vm_pageout_object_deactivate_pages
816 *
817 * Deactivate enough pages to satisfy the inactive target
818 * requirements.
819 *
820 * The object and map must be locked.
821 */
822 static void
vm_pageout_object_deactivate_pages(pmap_t pmap,vm_object_t first_object,long desired)823 vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
824 long desired)
825 {
826 vm_object_t backing_object, object;
827 vm_page_t p;
828 int act_delta, remove_mode;
829
830 VM_OBJECT_ASSERT_LOCKED(first_object);
831 if ((first_object->flags & OBJ_FICTITIOUS) != 0)
832 return;
833 for (object = first_object;; object = backing_object) {
834 if (pmap_resident_count(pmap) <= desired)
835 goto unlock_return;
836 VM_OBJECT_ASSERT_LOCKED(object);
837 if ((object->flags & OBJ_UNMANAGED) != 0 ||
838 object->paging_in_progress != 0)
839 goto unlock_return;
840
841 remove_mode = 0;
842 if (object->shadow_count > 1)
843 remove_mode = 1;
844 /*
845 * Scan the object's entire memory queue.
846 */
847 TAILQ_FOREACH(p, &object->memq, listq) {
848 if (pmap_resident_count(pmap) <= desired)
849 goto unlock_return;
850 if (vm_page_busied(p))
851 continue;
852 PCPU_INC(cnt.v_pdpages);
853 vm_page_lock(p);
854 if (p->wire_count != 0 || p->hold_count != 0 ||
855 !pmap_page_exists_quick(pmap, p)) {
856 vm_page_unlock(p);
857 continue;
858 }
859 act_delta = pmap_ts_referenced(p);
860 if ((p->aflags & PGA_REFERENCED) != 0) {
861 if (act_delta == 0)
862 act_delta = 1;
863 vm_page_aflag_clear(p, PGA_REFERENCED);
864 }
865 if (p->queue != PQ_ACTIVE && act_delta != 0) {
866 vm_page_activate(p);
867 p->act_count += act_delta;
868 } else if (p->queue == PQ_ACTIVE) {
869 if (act_delta == 0) {
870 p->act_count -= min(p->act_count,
871 ACT_DECLINE);
872 if (!remove_mode && p->act_count == 0) {
873 pmap_remove_all(p);
874 vm_page_deactivate(p);
875 } else
876 vm_page_requeue(p);
877 } else {
878 vm_page_activate(p);
879 if (p->act_count < ACT_MAX -
880 ACT_ADVANCE)
881 p->act_count += ACT_ADVANCE;
882 vm_page_requeue(p);
883 }
884 } else if (p->queue == PQ_INACTIVE)
885 pmap_remove_all(p);
886 vm_page_unlock(p);
887 }
888 if ((backing_object = object->backing_object) == NULL)
889 goto unlock_return;
890 VM_OBJECT_RLOCK(backing_object);
891 if (object != first_object)
892 VM_OBJECT_RUNLOCK(object);
893 }
894 unlock_return:
895 if (object != first_object)
896 VM_OBJECT_RUNLOCK(object);
897 }
898
899 /*
900 * deactivate some number of pages in a map, try to do it fairly, but
901 * that is really hard to do.
902 */
903 static void
vm_pageout_map_deactivate_pages(map,desired)904 vm_pageout_map_deactivate_pages(map, desired)
905 vm_map_t map;
906 long desired;
907 {
908 vm_map_entry_t tmpe;
909 vm_object_t obj, bigobj;
910 int nothingwired;
911
912 if (!vm_map_trylock(map))
913 return;
914
915 bigobj = NULL;
916 nothingwired = TRUE;
917
918 /*
919 * first, search out the biggest object, and try to free pages from
920 * that.
921 */
922 tmpe = map->header.next;
923 while (tmpe != &map->header) {
924 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
925 obj = tmpe->object.vm_object;
926 if (obj != NULL && VM_OBJECT_TRYRLOCK(obj)) {
927 if (obj->shadow_count <= 1 &&
928 (bigobj == NULL ||
929 bigobj->resident_page_count < obj->resident_page_count)) {
930 if (bigobj != NULL)
931 VM_OBJECT_RUNLOCK(bigobj);
932 bigobj = obj;
933 } else
934 VM_OBJECT_RUNLOCK(obj);
935 }
936 }
937 if (tmpe->wired_count > 0)
938 nothingwired = FALSE;
939 tmpe = tmpe->next;
940 }
941
942 if (bigobj != NULL) {
943 vm_pageout_object_deactivate_pages(map->pmap, bigobj, desired);
944 VM_OBJECT_RUNLOCK(bigobj);
945 }
946 /*
947 * Next, hunt around for other pages to deactivate. We actually
948 * do this search sort of wrong -- .text first is not the best idea.
949 */
950 tmpe = map->header.next;
951 while (tmpe != &map->header) {
952 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
953 break;
954 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
955 obj = tmpe->object.vm_object;
956 if (obj != NULL) {
957 VM_OBJECT_RLOCK(obj);
958 vm_pageout_object_deactivate_pages(map->pmap, obj, desired);
959 VM_OBJECT_RUNLOCK(obj);
960 }
961 }
962 tmpe = tmpe->next;
963 }
964
965 /*
966 * Remove all mappings if a process is swapped out, this will free page
967 * table pages.
968 */
969 if (desired == 0 && nothingwired) {
970 pmap_remove(vm_map_pmap(map), vm_map_min(map),
971 vm_map_max(map));
972 }
973
974 vm_map_unlock(map);
975 }
976 #endif /* !defined(NO_SWAPPING) */
977
978 /*
979 * Attempt to acquire all of the necessary locks to launder a page and
980 * then call through the clustering layer to PUTPAGES. Wait a short
981 * time for a vnode lock.
982 *
983 * Requires the page and object lock on entry, releases both before return.
984 * Returns 0 on success and an errno otherwise.
985 */
986 static int
vm_pageout_clean(vm_page_t m)987 vm_pageout_clean(vm_page_t m)
988 {
989 struct vnode *vp;
990 struct mount *mp;
991 vm_object_t object;
992 vm_pindex_t pindex;
993 int error, lockmode;
994
995 vm_page_assert_locked(m);
996 object = m->object;
997 VM_OBJECT_ASSERT_WLOCKED(object);
998 error = 0;
999 vp = NULL;
1000 mp = NULL;
1001
1002 /*
1003 * The object is already known NOT to be dead. It
1004 * is possible for the vget() to block the whole
1005 * pageout daemon, but the new low-memory handling
1006 * code should prevent it.
1007 *
1008 * We can't wait forever for the vnode lock, we might
1009 * deadlock due to a vn_read() getting stuck in
1010 * vm_wait while holding this vnode. We skip the
1011 * vnode if we can't get it in a reasonable amount
1012 * of time.
1013 */
1014 if (object->type == OBJT_VNODE) {
1015 vm_page_unlock(m);
1016 vp = object->handle;
1017 if (vp->v_type == VREG &&
1018 vn_start_write(vp, &mp, V_NOWAIT) != 0) {
1019 mp = NULL;
1020 error = EDEADLK;
1021 goto unlock_all;
1022 }
1023 KASSERT(mp != NULL,
1024 ("vp %p with NULL v_mount", vp));
1025 vm_object_reference_locked(object);
1026 pindex = m->pindex;
1027 VM_OBJECT_WUNLOCK(object);
1028 lockmode = MNT_SHARED_WRITES(vp->v_mount) ?
1029 LK_SHARED : LK_EXCLUSIVE;
1030 if (vget(vp, lockmode | LK_TIMELOCK, curthread)) {
1031 vp = NULL;
1032 error = EDEADLK;
1033 goto unlock_mp;
1034 }
1035 VM_OBJECT_WLOCK(object);
1036 vm_page_lock(m);
1037 /*
1038 * While the object and page were unlocked, the page
1039 * may have been:
1040 * (1) moved to a different queue,
1041 * (2) reallocated to a different object,
1042 * (3) reallocated to a different offset, or
1043 * (4) cleaned.
1044 */
1045 if (m->queue != PQ_INACTIVE || m->object != object ||
1046 m->pindex != pindex || m->dirty == 0) {
1047 vm_page_unlock(m);
1048 error = ENXIO;
1049 goto unlock_all;
1050 }
1051
1052 /*
1053 * The page may have been busied or held while the object
1054 * and page locks were released.
1055 */
1056 if (vm_page_busied(m) || m->hold_count != 0) {
1057 vm_page_unlock(m);
1058 error = EBUSY;
1059 goto unlock_all;
1060 }
1061 }
1062
1063 /*
1064 * If a page is dirty, then it is either being washed
1065 * (but not yet cleaned) or it is still in the
1066 * laundry. If it is still in the laundry, then we
1067 * start the cleaning operation.
1068 */
1069 if (vm_pageout_cluster(m) == 0)
1070 error = EIO;
1071
1072 unlock_all:
1073 VM_OBJECT_WUNLOCK(object);
1074
1075 unlock_mp:
1076 vm_page_lock_assert(m, MA_NOTOWNED);
1077 if (mp != NULL) {
1078 if (vp != NULL)
1079 vput(vp);
1080 vm_object_deallocate(object);
1081 vn_finished_write(mp);
1082 }
1083
1084 return (error);
1085 }
1086
1087 /*
1088 * vm_pageout_scan does the dirty work for the pageout daemon.
1089 *
1090 * pass 0 - Update active LRU/deactivate pages
1091 * pass 1 - Move inactive to cache or free
1092 * pass 2 - Launder dirty pages
1093 */
1094 static void
vm_pageout_scan(struct vm_domain * vmd,int pass)1095 vm_pageout_scan(struct vm_domain *vmd, int pass)
1096 {
1097 vm_page_t m, next;
1098 struct vm_pagequeue *pq;
1099 vm_object_t object;
1100 long min_scan;
1101 int act_delta, addl_page_shortage, deficit, error, maxlaunder, maxscan;
1102 int page_shortage, scan_tick, scanned, starting_page_shortage;
1103 int vnodes_skipped;
1104 boolean_t pageout_ok, queues_locked;
1105
1106 /*
1107 * If we need to reclaim memory ask kernel caches to return
1108 * some. We rate limit to avoid thrashing.
1109 */
1110 if (vmd == &vm_dom[0] && pass > 0 &&
1111 (time_uptime - lowmem_uptime) >= lowmem_period) {
1112 /*
1113 * Decrease registered cache sizes.
1114 */
1115 SDT_PROBE0(vm, , , vm__lowmem_scan);
1116 EVENTHANDLER_INVOKE(vm_lowmem, 0);
1117 /*
1118 * We do this explicitly after the caches have been
1119 * drained above.
1120 */
1121 uma_reclaim();
1122 lowmem_uptime = time_uptime;
1123 }
1124
1125 /*
1126 * The addl_page_shortage is the number of temporarily
1127 * stuck pages in the inactive queue. In other words, the
1128 * number of pages from the inactive count that should be
1129 * discounted in setting the target for the active queue scan.
1130 */
1131 addl_page_shortage = 0;
1132
1133 /*
1134 * Calculate the number of pages we want to either free or move
1135 * to the cache.
1136 */
1137 if (pass > 0) {
1138 deficit = atomic_readandclear_int(&vm_pageout_deficit);
1139 page_shortage = vm_paging_target() + deficit;
1140 } else
1141 page_shortage = deficit = 0;
1142 starting_page_shortage = page_shortage;
1143
1144 /*
1145 * maxlaunder limits the number of dirty pages we flush per scan.
1146 * For most systems a smaller value (16 or 32) is more robust under
1147 * extreme memory and disk pressure because any unnecessary writes
1148 * to disk can result in extreme performance degredation. However,
1149 * systems with excessive dirty pages (especially when MAP_NOSYNC is
1150 * used) will die horribly with limited laundering. If the pageout
1151 * daemon cannot clean enough pages in the first pass, we let it go
1152 * all out in succeeding passes.
1153 */
1154 if ((maxlaunder = vm_max_launder) <= 1)
1155 maxlaunder = 1;
1156 if (pass > 1)
1157 maxlaunder = 10000;
1158
1159 vnodes_skipped = 0;
1160
1161 /*
1162 * Start scanning the inactive queue for pages we can move to the
1163 * cache or free. The scan will stop when the target is reached or
1164 * we have scanned the entire inactive queue. Note that m->act_count
1165 * is not used to form decisions for the inactive queue, only for the
1166 * active queue.
1167 */
1168 pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
1169 maxscan = pq->pq_cnt;
1170 vm_pagequeue_lock(pq);
1171 queues_locked = TRUE;
1172 for (m = TAILQ_FIRST(&pq->pq_pl);
1173 m != NULL && maxscan-- > 0 && page_shortage > 0;
1174 m = next) {
1175 vm_pagequeue_assert_locked(pq);
1176 KASSERT(queues_locked, ("unlocked queues"));
1177 KASSERT(m->queue == PQ_INACTIVE, ("Inactive queue %p", m));
1178
1179 PCPU_INC(cnt.v_pdpages);
1180 next = TAILQ_NEXT(m, plinks.q);
1181
1182 /*
1183 * skip marker pages
1184 */
1185 if (m->flags & PG_MARKER)
1186 continue;
1187
1188 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1189 ("Fictitious page %p cannot be in inactive queue", m));
1190 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1191 ("Unmanaged page %p cannot be in inactive queue", m));
1192
1193 /*
1194 * The page or object lock acquisitions fail if the
1195 * page was removed from the queue or moved to a
1196 * different position within the queue. In either
1197 * case, addl_page_shortage should not be incremented.
1198 */
1199 if (!vm_pageout_page_lock(m, &next))
1200 goto unlock_page;
1201 else if (m->hold_count != 0) {
1202 /*
1203 * Held pages are essentially stuck in the
1204 * queue. So, they ought to be discounted
1205 * from the inactive count. See the
1206 * calculation of the page_shortage for the
1207 * loop over the active queue below.
1208 */
1209 addl_page_shortage++;
1210 goto unlock_page;
1211 }
1212 vm_page_queue_fixup_locked(m);
1213 object = m->object;
1214 if (!VM_OBJECT_TRYWLOCK(object)) {
1215 if (!vm_pageout_fallback_object_lock(m, &next))
1216 goto unlock_object;
1217 else if (m->hold_count != 0) {
1218 addl_page_shortage++;
1219 goto unlock_object;
1220 }
1221 }
1222 if (vm_page_busied(m)) {
1223 /*
1224 * Don't mess with busy pages. Leave them at
1225 * the front of the queue. Most likely, they
1226 * are being paged out and will leave the
1227 * queue shortly after the scan finishes. So,
1228 * they ought to be discounted from the
1229 * inactive count.
1230 */
1231 addl_page_shortage++;
1232 unlock_object:
1233 VM_OBJECT_WUNLOCK(object);
1234 unlock_page:
1235 vm_page_unlock(m);
1236 continue;
1237 }
1238 KASSERT(m->hold_count == 0, ("Held page %p", m));
1239
1240 /*
1241 * We unlock the inactive page queue, invalidating the
1242 * 'next' pointer. Use our marker to remember our
1243 * place.
1244 */
1245 TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_marker, plinks.q);
1246 vm_pagequeue_unlock(pq);
1247 queues_locked = FALSE;
1248
1249 /*
1250 * Invalid pages can be easily freed. They cannot be
1251 * mapped, vm_page_free() asserts this.
1252 */
1253 if (m->valid == 0)
1254 goto free_page;
1255
1256 /*
1257 * If the page has been referenced and the object is not dead,
1258 * reactivate or requeue the page depending on whether the
1259 * object is mapped.
1260 */
1261 if ((m->aflags & PGA_REFERENCED) != 0) {
1262 vm_page_aflag_clear(m, PGA_REFERENCED);
1263 act_delta = 1;
1264 } else
1265 act_delta = 0;
1266 if (object->ref_count != 0) {
1267 act_delta += pmap_ts_referenced(m);
1268 } else {
1269 KASSERT(!pmap_page_is_mapped(m),
1270 ("vm_pageout_scan: page %p is mapped", m));
1271 }
1272 if (act_delta != 0) {
1273 if (object->ref_count != 0) {
1274 vm_page_activate(m);
1275
1276 /*
1277 * Increase the activation count if the page
1278 * was referenced while in the inactive queue.
1279 * This makes it less likely that the page will
1280 * be returned prematurely to the inactive
1281 * queue.
1282 */
1283 m->act_count += act_delta + ACT_ADVANCE;
1284 goto drop_page;
1285 } else if ((object->flags & OBJ_DEAD) == 0)
1286 goto requeue_page;
1287 }
1288
1289 /*
1290 * If the page appears to be clean at the machine-independent
1291 * layer, then remove all of its mappings from the pmap in
1292 * anticipation of placing it onto the cache queue. If,
1293 * however, any of the page's mappings allow write access,
1294 * then the page may still be modified until the last of those
1295 * mappings are removed.
1296 */
1297 if (object->ref_count != 0) {
1298 vm_page_test_dirty(m);
1299 if (m->dirty == 0)
1300 pmap_remove_all(m);
1301 }
1302
1303 if (m->dirty == 0) {
1304 /*
1305 * Clean pages can be freed.
1306 */
1307 free_page:
1308 vm_page_free(m);
1309 PCPU_INC(cnt.v_dfree);
1310 --page_shortage;
1311 } else if ((object->flags & OBJ_DEAD) != 0) {
1312 /*
1313 * Leave dirty pages from dead objects at the front of
1314 * the queue. They are being paged out and freed by
1315 * the thread that destroyed the object. They will
1316 * leave the queue shortly after the scan finishes, so
1317 * they should be discounted from the inactive count.
1318 */
1319 addl_page_shortage++;
1320 } else if ((m->flags & PG_WINATCFLS) == 0 && pass < 2) {
1321 /*
1322 * Dirty pages need to be paged out, but flushing
1323 * a page is extremely expensive versus freeing
1324 * a clean page. Rather then artificially limiting
1325 * the number of pages we can flush, we instead give
1326 * dirty pages extra priority on the inactive queue
1327 * by forcing them to be cycled through the queue
1328 * twice before being flushed, after which the
1329 * (now clean) page will cycle through once more
1330 * before being freed. This significantly extends
1331 * the thrash point for a heavily loaded machine.
1332 */
1333 m->flags |= PG_WINATCFLS;
1334 requeue_page:
1335 vm_pagequeue_lock(pq);
1336 queues_locked = TRUE;
1337 vm_page_requeue_locked(m);
1338 } else if (maxlaunder > 0) {
1339 /*
1340 * We always want to try to flush some dirty pages if
1341 * we encounter them, to keep the system stable.
1342 * Normally this number is small, but under extreme
1343 * pressure where there are insufficient clean pages
1344 * on the inactive queue, we may have to go all out.
1345 */
1346
1347 if (object->type != OBJT_SWAP &&
1348 object->type != OBJT_DEFAULT)
1349 pageout_ok = TRUE;
1350 else if (disable_swap_pageouts)
1351 pageout_ok = FALSE;
1352 else if (defer_swap_pageouts)
1353 pageout_ok = vm_page_count_min();
1354 else
1355 pageout_ok = TRUE;
1356 if (!pageout_ok)
1357 goto requeue_page;
1358 error = vm_pageout_clean(m);
1359 /*
1360 * Decrement page_shortage on success to account for
1361 * the (future) cleaned page. Otherwise we could wind
1362 * up laundering or cleaning too many pages.
1363 */
1364 if (error == 0) {
1365 page_shortage--;
1366 maxlaunder--;
1367 } else if (error == EDEADLK) {
1368 pageout_lock_miss++;
1369 vnodes_skipped++;
1370 } else if (error == EBUSY) {
1371 addl_page_shortage++;
1372 }
1373 vm_page_lock_assert(m, MA_NOTOWNED);
1374 goto relock_queues;
1375 }
1376 drop_page:
1377 vm_page_unlock(m);
1378 VM_OBJECT_WUNLOCK(object);
1379 relock_queues:
1380 if (!queues_locked) {
1381 vm_pagequeue_lock(pq);
1382 queues_locked = TRUE;
1383 }
1384 next = TAILQ_NEXT(&vmd->vmd_marker, plinks.q);
1385 TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_marker, plinks.q);
1386 }
1387 vm_pagequeue_unlock(pq);
1388
1389 #if !defined(NO_SWAPPING)
1390 /*
1391 * Wakeup the swapout daemon if we didn't cache or free the targeted
1392 * number of pages.
1393 */
1394 if (vm_swap_enabled && page_shortage > 0)
1395 vm_req_vmdaemon(VM_SWAP_NORMAL);
1396 #endif
1397
1398 /*
1399 * Wakeup the sync daemon if we skipped a vnode in a writeable object
1400 * and we didn't cache or free enough pages.
1401 */
1402 if (vnodes_skipped > 0 && page_shortage > vm_cnt.v_free_target -
1403 vm_cnt.v_free_min)
1404 (void)speedup_syncer();
1405
1406 /*
1407 * If the inactive queue scan fails repeatedly to meet its
1408 * target, kill the largest process.
1409 */
1410 vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);
1411
1412 /*
1413 * Compute the number of pages we want to try to move from the
1414 * active queue to the inactive queue.
1415 */
1416 page_shortage = vm_cnt.v_inactive_target - vm_cnt.v_inactive_count +
1417 vm_paging_target() + deficit + addl_page_shortage;
1418
1419 pq = &vmd->vmd_pagequeues[PQ_ACTIVE];
1420 vm_pagequeue_lock(pq);
1421 maxscan = pq->pq_cnt;
1422
1423 /*
1424 * If we're just idle polling attempt to visit every
1425 * active page within 'update_period' seconds.
1426 */
1427 scan_tick = ticks;
1428 if (vm_pageout_update_period != 0) {
1429 min_scan = pq->pq_cnt;
1430 min_scan *= scan_tick - vmd->vmd_last_active_scan;
1431 min_scan /= hz * vm_pageout_update_period;
1432 } else
1433 min_scan = 0;
1434 if (min_scan > 0 || (page_shortage > 0 && maxscan > 0))
1435 vmd->vmd_last_active_scan = scan_tick;
1436
1437 /*
1438 * Scan the active queue for pages that can be deactivated. Update
1439 * the per-page activity counter and use it to identify deactivation
1440 * candidates.
1441 */
1442 for (m = TAILQ_FIRST(&pq->pq_pl), scanned = 0; m != NULL && (scanned <
1443 min_scan || (page_shortage > 0 && scanned < maxscan)); m = next,
1444 scanned++) {
1445
1446 KASSERT(m->queue == PQ_ACTIVE,
1447 ("vm_pageout_scan: page %p isn't active", m));
1448
1449 next = TAILQ_NEXT(m, plinks.q);
1450 if ((m->flags & PG_MARKER) != 0)
1451 continue;
1452 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1453 ("Fictitious page %p cannot be in active queue", m));
1454 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1455 ("Unmanaged page %p cannot be in active queue", m));
1456 if (!vm_pageout_page_lock(m, &next)) {
1457 vm_page_unlock(m);
1458 continue;
1459 }
1460
1461 /*
1462 * The count for pagedaemon pages is done after checking the
1463 * page for eligibility...
1464 */
1465 PCPU_INC(cnt.v_pdpages);
1466
1467 /*
1468 * Check to see "how much" the page has been used.
1469 */
1470 if ((m->aflags & PGA_REFERENCED) != 0) {
1471 vm_page_aflag_clear(m, PGA_REFERENCED);
1472 act_delta = 1;
1473 } else
1474 act_delta = 0;
1475
1476 /*
1477 * Unlocked object ref count check. Two races are possible.
1478 * 1) The ref was transitioning to zero and we saw non-zero,
1479 * the pmap bits will be checked unnecessarily.
1480 * 2) The ref was transitioning to one and we saw zero.
1481 * The page lock prevents a new reference to this page so
1482 * we need not check the reference bits.
1483 */
1484 if (m->object->ref_count != 0)
1485 act_delta += pmap_ts_referenced(m);
1486
1487 /*
1488 * Advance or decay the act_count based on recent usage.
1489 */
1490 if (act_delta != 0) {
1491 m->act_count += ACT_ADVANCE + act_delta;
1492 if (m->act_count > ACT_MAX)
1493 m->act_count = ACT_MAX;
1494 } else
1495 m->act_count -= min(m->act_count, ACT_DECLINE);
1496
1497 /*
1498 * Move this page to the tail of the active or inactive
1499 * queue depending on usage.
1500 */
1501 if (m->act_count == 0) {
1502 /* Dequeue to avoid later lock recursion. */
1503 vm_page_dequeue_locked(m);
1504 vm_page_deactivate(m);
1505 page_shortage--;
1506 } else
1507 vm_page_requeue_locked(m);
1508 vm_page_unlock(m);
1509 }
1510 vm_pagequeue_unlock(pq);
1511 #if !defined(NO_SWAPPING)
1512 /*
1513 * Idle process swapout -- run once per second.
1514 */
1515 if (vm_swap_idle_enabled) {
1516 static long lsec;
1517 if (time_second != lsec) {
1518 vm_req_vmdaemon(VM_SWAP_IDLE);
1519 lsec = time_second;
1520 }
1521 }
1522 #endif
1523 }
1524
1525 static int vm_pageout_oom_vote;
1526
1527 /*
1528 * The pagedaemon threads randlomly select one to perform the
1529 * OOM. Trying to kill processes before all pagedaemons
1530 * failed to reach free target is premature.
1531 */
1532 static void
vm_pageout_mightbe_oom(struct vm_domain * vmd,int page_shortage,int starting_page_shortage)1533 vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
1534 int starting_page_shortage)
1535 {
1536 int old_vote;
1537
1538 if (starting_page_shortage <= 0 || starting_page_shortage !=
1539 page_shortage)
1540 vmd->vmd_oom_seq = 0;
1541 else
1542 vmd->vmd_oom_seq++;
1543 if (vmd->vmd_oom_seq < vm_pageout_oom_seq) {
1544 if (vmd->vmd_oom) {
1545 vmd->vmd_oom = FALSE;
1546 atomic_subtract_int(&vm_pageout_oom_vote, 1);
1547 }
1548 return;
1549 }
1550
1551 /*
1552 * Do not follow the call sequence until OOM condition is
1553 * cleared.
1554 */
1555 vmd->vmd_oom_seq = 0;
1556
1557 if (vmd->vmd_oom)
1558 return;
1559
1560 vmd->vmd_oom = TRUE;
1561 old_vote = atomic_fetchadd_int(&vm_pageout_oom_vote, 1);
1562 if (old_vote != vm_ndomains - 1)
1563 return;
1564
1565 /*
1566 * The current pagedaemon thread is the last in the quorum to
1567 * start OOM. Initiate the selection and signaling of the
1568 * victim.
1569 */
1570 vm_pageout_oom(VM_OOM_MEM);
1571
1572 /*
1573 * After one round of OOM terror, recall our vote. On the
1574 * next pass, current pagedaemon would vote again if the low
1575 * memory condition is still there, due to vmd_oom being
1576 * false.
1577 */
1578 vmd->vmd_oom = FALSE;
1579 atomic_subtract_int(&vm_pageout_oom_vote, 1);
1580 }
1581
1582 /*
1583 * The OOM killer is the page daemon's action of last resort when
1584 * memory allocation requests have been stalled for a prolonged period
1585 * of time because it cannot reclaim memory. This function computes
1586 * the approximate number of physical pages that could be reclaimed if
1587 * the specified address space is destroyed.
1588 *
1589 * Private, anonymous memory owned by the address space is the
1590 * principal resource that we expect to recover after an OOM kill.
1591 * Since the physical pages mapped by the address space's COW entries
1592 * are typically shared pages, they are unlikely to be released and so
1593 * they are not counted.
1594 *
1595 * To get to the point where the page daemon runs the OOM killer, its
1596 * efforts to write-back vnode-backed pages may have stalled. This
1597 * could be caused by a memory allocation deadlock in the write path
1598 * that might be resolved by an OOM kill. Therefore, physical pages
1599 * belonging to vnode-backed objects are counted, because they might
1600 * be freed without being written out first if the address space holds
1601 * the last reference to an unlinked vnode.
1602 *
1603 * Similarly, physical pages belonging to OBJT_PHYS objects are
1604 * counted because the address space might hold the last reference to
1605 * the object.
1606 */
1607 static long
vm_pageout_oom_pagecount(struct vmspace * vmspace)1608 vm_pageout_oom_pagecount(struct vmspace *vmspace)
1609 {
1610 vm_map_t map;
1611 vm_map_entry_t entry;
1612 vm_object_t obj;
1613 long res;
1614
1615 map = &vmspace->vm_map;
1616 KASSERT(!map->system_map, ("system map"));
1617 sx_assert(&map->lock, SA_LOCKED);
1618 res = 0;
1619 for (entry = map->header.next; entry != &map->header;
1620 entry = entry->next) {
1621 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
1622 continue;
1623 obj = entry->object.vm_object;
1624 if (obj == NULL)
1625 continue;
1626 if ((entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0 &&
1627 obj->ref_count != 1)
1628 continue;
1629 switch (obj->type) {
1630 case OBJT_DEFAULT:
1631 case OBJT_SWAP:
1632 case OBJT_PHYS:
1633 case OBJT_VNODE:
1634 res += obj->resident_page_count;
1635 break;
1636 }
1637 }
1638 return (res);
1639 }
1640
1641 void
vm_pageout_oom(int shortage)1642 vm_pageout_oom(int shortage)
1643 {
1644 struct proc *p, *bigproc;
1645 vm_offset_t size, bigsize;
1646 struct thread *td;
1647 struct vmspace *vm;
1648
1649 /*
1650 * We keep the process bigproc locked once we find it to keep anyone
1651 * from messing with it; however, there is a possibility of
1652 * deadlock if process B is bigproc and one of it's child processes
1653 * attempts to propagate a signal to B while we are waiting for A's
1654 * lock while walking this list. To avoid this, we don't block on
1655 * the process lock but just skip a process if it is already locked.
1656 */
1657 bigproc = NULL;
1658 bigsize = 0;
1659 sx_slock(&allproc_lock);
1660 FOREACH_PROC_IN_SYSTEM(p) {
1661 int breakout;
1662
1663 PROC_LOCK(p);
1664
1665 /*
1666 * If this is a system, protected or killed process, skip it.
1667 */
1668 if (p->p_state != PRS_NORMAL || (p->p_flag & (P_INEXEC |
1669 P_PROTECTED | P_SYSTEM | P_WEXIT)) != 0 ||
1670 p->p_pid == 1 || P_KILLED(p) ||
1671 (p->p_pid < 48 && swap_pager_avail != 0)) {
1672 PROC_UNLOCK(p);
1673 continue;
1674 }
1675 /*
1676 * If the process is in a non-running type state,
1677 * don't touch it. Check all the threads individually.
1678 */
1679 breakout = 0;
1680 FOREACH_THREAD_IN_PROC(p, td) {
1681 thread_lock(td);
1682 if (!TD_ON_RUNQ(td) &&
1683 !TD_IS_RUNNING(td) &&
1684 !TD_IS_SLEEPING(td) &&
1685 !TD_IS_SUSPENDED(td) &&
1686 !TD_IS_SWAPPED(td)) {
1687 thread_unlock(td);
1688 breakout = 1;
1689 break;
1690 }
1691 thread_unlock(td);
1692 }
1693 if (breakout) {
1694 PROC_UNLOCK(p);
1695 continue;
1696 }
1697 /*
1698 * get the process size
1699 */
1700 vm = vmspace_acquire_ref(p);
1701 if (vm == NULL) {
1702 PROC_UNLOCK(p);
1703 continue;
1704 }
1705 _PHOLD(p);
1706 if (!vm_map_trylock_read(&vm->vm_map)) {
1707 _PRELE(p);
1708 PROC_UNLOCK(p);
1709 vmspace_free(vm);
1710 continue;
1711 }
1712 PROC_UNLOCK(p);
1713 size = vmspace_swap_count(vm);
1714 if (shortage == VM_OOM_MEM)
1715 size += vm_pageout_oom_pagecount(vm);
1716 vm_map_unlock_read(&vm->vm_map);
1717 vmspace_free(vm);
1718
1719 /*
1720 * If this process is bigger than the biggest one,
1721 * remember it.
1722 */
1723 if (size > bigsize) {
1724 if (bigproc != NULL)
1725 PRELE(bigproc);
1726 bigproc = p;
1727 bigsize = size;
1728 } else {
1729 PRELE(p);
1730 }
1731 }
1732 sx_sunlock(&allproc_lock);
1733 if (bigproc != NULL) {
1734 if (vm_panic_on_oom != 0)
1735 panic("out of swap space");
1736 PROC_LOCK(bigproc);
1737 killproc(bigproc, "out of swap space");
1738 sched_nice(bigproc, PRIO_MIN);
1739 _PRELE(bigproc);
1740 PROC_UNLOCK(bigproc);
1741 wakeup(&vm_cnt.v_free_count);
1742 }
1743 }
1744
1745 static void
vm_pageout_worker(void * arg)1746 vm_pageout_worker(void *arg)
1747 {
1748 struct vm_domain *domain;
1749 int domidx;
1750
1751 domidx = (uintptr_t)arg;
1752 domain = &vm_dom[domidx];
1753
1754 /*
1755 * XXXKIB It could be useful to bind pageout daemon threads to
1756 * the cores belonging to the domain, from which vm_page_array
1757 * is allocated.
1758 */
1759
1760 KASSERT(domain->vmd_segs != 0, ("domain without segments"));
1761 domain->vmd_last_active_scan = ticks;
1762 vm_pageout_init_marker(&domain->vmd_marker, PQ_INACTIVE);
1763 vm_pageout_init_marker(&domain->vmd_inacthead, PQ_INACTIVE);
1764 TAILQ_INSERT_HEAD(&domain->vmd_pagequeues[PQ_INACTIVE].pq_pl,
1765 &domain->vmd_inacthead, plinks.q);
1766
1767 /*
1768 * The pageout daemon worker is never done, so loop forever.
1769 */
1770 while (TRUE) {
1771 /*
1772 * If we have enough free memory, wakeup waiters. Do
1773 * not clear vm_pages_needed until we reach our target,
1774 * otherwise we may be woken up over and over again and
1775 * waste a lot of cpu.
1776 */
1777 mtx_lock(&vm_page_queue_free_mtx);
1778 if (vm_pages_needed && !vm_page_count_min()) {
1779 if (!vm_paging_needed())
1780 vm_pages_needed = 0;
1781 wakeup(&vm_cnt.v_free_count);
1782 }
1783 if (vm_pages_needed) {
1784 /*
1785 * We're still not done. Either vm_pages_needed was
1786 * set by another thread during the previous scan
1787 * (typically, this happens during a level 0 scan) or
1788 * vm_pages_needed was already set and the scan failed
1789 * to free enough pages. If we haven't yet performed
1790 * a level >= 2 scan (unlimited dirty cleaning), then
1791 * upgrade the level and scan again now. Otherwise,
1792 * sleep a bit and try again later. While sleeping,
1793 * vm_pages_needed can be cleared.
1794 */
1795 if (domain->vmd_pass > 1)
1796 msleep(&vm_pages_needed,
1797 &vm_page_queue_free_mtx, PVM, "psleep",
1798 hz / 2);
1799 } else {
1800 /*
1801 * Good enough, sleep until required to refresh
1802 * stats.
1803 */
1804 msleep(&vm_pages_needed, &vm_page_queue_free_mtx,
1805 PVM, "psleep", hz);
1806 }
1807 if (vm_pages_needed) {
1808 vm_cnt.v_pdwakeups++;
1809 domain->vmd_pass++;
1810 } else
1811 domain->vmd_pass = 0;
1812 mtx_unlock(&vm_page_queue_free_mtx);
1813 vm_pageout_scan(domain, domain->vmd_pass);
1814 }
1815 }
1816
1817 /*
1818 * vm_pageout_init initialises basic pageout daemon settings.
1819 */
1820 static void
vm_pageout_init(void)1821 vm_pageout_init(void)
1822 {
1823 /*
1824 * Initialize some paging parameters.
1825 */
1826 vm_cnt.v_interrupt_free_min = 2;
1827 if (vm_cnt.v_page_count < 2000)
1828 vm_pageout_page_count = 8;
1829
1830 /*
1831 * v_free_reserved needs to include enough for the largest
1832 * swap pager structures plus enough for any pv_entry structs
1833 * when paging.
1834 */
1835 if (vm_cnt.v_page_count > 1024)
1836 vm_cnt.v_free_min = 4 + (vm_cnt.v_page_count - 1024) / 200;
1837 else
1838 vm_cnt.v_free_min = 4;
1839 vm_cnt.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
1840 vm_cnt.v_interrupt_free_min;
1841 vm_cnt.v_free_reserved = vm_pageout_page_count +
1842 vm_cnt.v_pageout_free_min + (vm_cnt.v_page_count / 768);
1843 vm_cnt.v_free_severe = vm_cnt.v_free_min / 2;
1844 vm_cnt.v_free_target = 4 * vm_cnt.v_free_min + vm_cnt.v_free_reserved;
1845 vm_cnt.v_free_min += vm_cnt.v_free_reserved;
1846 vm_cnt.v_free_severe += vm_cnt.v_free_reserved;
1847 vm_cnt.v_inactive_target = (3 * vm_cnt.v_free_target) / 2;
1848 if (vm_cnt.v_inactive_target > vm_cnt.v_free_count / 3)
1849 vm_cnt.v_inactive_target = vm_cnt.v_free_count / 3;
1850
1851 /*
1852 * Set the default wakeup threshold to be 10% above the minimum
1853 * page limit. This keeps the steady state out of shortfall.
1854 */
1855 vm_pageout_wakeup_thresh = (vm_cnt.v_free_min / 10) * 11;
1856
1857 /*
1858 * Set interval in seconds for active scan. We want to visit each
1859 * page at least once every ten minutes. This is to prevent worst
1860 * case paging behaviors with stale active LRU.
1861 */
1862 if (vm_pageout_update_period == 0)
1863 vm_pageout_update_period = 600;
1864
1865 /* XXX does not really belong here */
1866 if (vm_page_max_wired == 0)
1867 vm_page_max_wired = vm_cnt.v_free_count / 3;
1868 }
1869
1870 /*
1871 * vm_pageout is the high level pageout daemon.
1872 */
1873 static void
vm_pageout(void)1874 vm_pageout(void)
1875 {
1876 int error;
1877 #if MAXMEMDOM > 1
1878 int i;
1879 #endif
1880
1881 swap_pager_swap_init();
1882 #if MAXMEMDOM > 1
1883 for (i = 1; i < vm_ndomains; i++) {
1884 error = kthread_add(vm_pageout_worker, (void *)(uintptr_t)i,
1885 curproc, NULL, 0, 0, "dom%d", i);
1886 if (error != 0) {
1887 panic("starting pageout for domain %d, error %d\n",
1888 i, error);
1889 }
1890 }
1891 #endif
1892 error = kthread_add(uma_reclaim_worker, NULL, curproc, NULL,
1893 0, 0, "uma");
1894 if (error != 0)
1895 panic("starting uma_reclaim helper, error %d\n", error);
1896 vm_pageout_worker((void *)(uintptr_t)0);
1897 }
1898
1899 /*
1900 * Unless the free page queue lock is held by the caller, this function
1901 * should be regarded as advisory. Specifically, the caller should
1902 * not msleep() on &vm_cnt.v_free_count following this function unless
1903 * the free page queue lock is held until the msleep() is performed.
1904 */
1905 void
pagedaemon_wakeup(void)1906 pagedaemon_wakeup(void)
1907 {
1908
1909 if (!vm_pages_needed && curthread->td_proc != pageproc) {
1910 vm_pages_needed = 1;
1911 wakeup(&vm_pages_needed);
1912 }
1913 }
1914
1915 #if !defined(NO_SWAPPING)
1916 static void
vm_req_vmdaemon(int req)1917 vm_req_vmdaemon(int req)
1918 {
1919 static int lastrun = 0;
1920
1921 mtx_lock(&vm_daemon_mtx);
1922 vm_pageout_req_swapout |= req;
1923 if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
1924 wakeup(&vm_daemon_needed);
1925 lastrun = ticks;
1926 }
1927 mtx_unlock(&vm_daemon_mtx);
1928 }
1929
1930 static void
vm_daemon(void)1931 vm_daemon(void)
1932 {
1933 struct rlimit rsslim;
1934 struct proc *p;
1935 struct thread *td;
1936 struct vmspace *vm;
1937 int breakout, swapout_flags, tryagain, attempts;
1938 #ifdef RACCT
1939 uint64_t rsize, ravailable;
1940 #endif
1941
1942 while (TRUE) {
1943 mtx_lock(&vm_daemon_mtx);
1944 msleep(&vm_daemon_needed, &vm_daemon_mtx, PPAUSE, "psleep",
1945 #ifdef RACCT
1946 racct_enable ? hz : 0
1947 #else
1948 0
1949 #endif
1950 );
1951 swapout_flags = vm_pageout_req_swapout;
1952 vm_pageout_req_swapout = 0;
1953 mtx_unlock(&vm_daemon_mtx);
1954 if (swapout_flags)
1955 swapout_procs(swapout_flags);
1956
1957 /*
1958 * scan the processes for exceeding their rlimits or if
1959 * process is swapped out -- deactivate pages
1960 */
1961 tryagain = 0;
1962 attempts = 0;
1963 again:
1964 attempts++;
1965 sx_slock(&allproc_lock);
1966 FOREACH_PROC_IN_SYSTEM(p) {
1967 vm_pindex_t limit, size;
1968
1969 /*
1970 * if this is a system process or if we have already
1971 * looked at this process, skip it.
1972 */
1973 PROC_LOCK(p);
1974 if (p->p_state != PRS_NORMAL ||
1975 p->p_flag & (P_INEXEC | P_SYSTEM | P_WEXIT)) {
1976 PROC_UNLOCK(p);
1977 continue;
1978 }
1979 /*
1980 * if the process is in a non-running type state,
1981 * don't touch it.
1982 */
1983 breakout = 0;
1984 FOREACH_THREAD_IN_PROC(p, td) {
1985 thread_lock(td);
1986 if (!TD_ON_RUNQ(td) &&
1987 !TD_IS_RUNNING(td) &&
1988 !TD_IS_SLEEPING(td) &&
1989 !TD_IS_SUSPENDED(td)) {
1990 thread_unlock(td);
1991 breakout = 1;
1992 break;
1993 }
1994 thread_unlock(td);
1995 }
1996 if (breakout) {
1997 PROC_UNLOCK(p);
1998 continue;
1999 }
2000 /*
2001 * get a limit
2002 */
2003 lim_rlimit_proc(p, RLIMIT_RSS, &rsslim);
2004 limit = OFF_TO_IDX(
2005 qmin(rsslim.rlim_cur, rsslim.rlim_max));
2006
2007 /*
2008 * let processes that are swapped out really be
2009 * swapped out set the limit to nothing (will force a
2010 * swap-out.)
2011 */
2012 if ((p->p_flag & P_INMEM) == 0)
2013 limit = 0; /* XXX */
2014 vm = vmspace_acquire_ref(p);
2015 PROC_UNLOCK(p);
2016 if (vm == NULL)
2017 continue;
2018
2019 size = vmspace_resident_count(vm);
2020 if (size >= limit) {
2021 vm_pageout_map_deactivate_pages(
2022 &vm->vm_map, limit);
2023 }
2024 #ifdef RACCT
2025 if (racct_enable) {
2026 rsize = IDX_TO_OFF(size);
2027 PROC_LOCK(p);
2028 racct_set(p, RACCT_RSS, rsize);
2029 ravailable = racct_get_available(p, RACCT_RSS);
2030 PROC_UNLOCK(p);
2031 if (rsize > ravailable) {
2032 /*
2033 * Don't be overly aggressive; this
2034 * might be an innocent process,
2035 * and the limit could've been exceeded
2036 * by some memory hog. Don't try
2037 * to deactivate more than 1/4th
2038 * of process' resident set size.
2039 */
2040 if (attempts <= 8) {
2041 if (ravailable < rsize -
2042 (rsize / 4)) {
2043 ravailable = rsize -
2044 (rsize / 4);
2045 }
2046 }
2047 vm_pageout_map_deactivate_pages(
2048 &vm->vm_map,
2049 OFF_TO_IDX(ravailable));
2050 /* Update RSS usage after paging out. */
2051 size = vmspace_resident_count(vm);
2052 rsize = IDX_TO_OFF(size);
2053 PROC_LOCK(p);
2054 racct_set(p, RACCT_RSS, rsize);
2055 PROC_UNLOCK(p);
2056 if (rsize > ravailable)
2057 tryagain = 1;
2058 }
2059 }
2060 #endif
2061 vmspace_free(vm);
2062 }
2063 sx_sunlock(&allproc_lock);
2064 if (tryagain != 0 && attempts <= 10)
2065 goto again;
2066 }
2067 }
2068 #endif /* !defined(NO_SWAPPING) */
2069