1 /* $OpenBSD: uvm_pager.c,v 1.37 2005/07/26 07:11:55 art Exp $ */
2 /* $NetBSD: uvm_pager.c,v 1.36 2000/11/27 18:26:41 chs Exp $ */
3
4 /*
5 *
6 * Copyright (c) 1997 Charles D. Cranor and Washington University.
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by Charles D. Cranor and
20 * Washington University.
21 * 4. The name of the author may not be used to endorse or promote products
22 * derived from this software without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 *
35 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp
36 */
37
38 /*
39 * uvm_pager.c: generic functions used to assist the pagers.
40 */
41
42 #define UVM_PAGER
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/proc.h>
46 #include <sys/malloc.h>
47 #include <sys/pool.h>
48 #include <sys/vnode.h>
49 #include <sys/buf.h>
50
51 #include <uvm/uvm.h>
52
53 struct pool *uvm_aiobuf_pool;
54
55 struct uvm_pagerops *uvmpagerops[] = {
56 &aobj_pager,
57 &uvm_deviceops,
58 &uvm_vnodeops,
59 };
60
61 /*
62 * the pager map: provides KVA for I/O
63 */
64
65 vm_map_t pager_map; /* XXX */
66 simple_lock_data_t pager_map_wanted_lock;
67 boolean_t pager_map_wanted; /* locked by pager map */
68 static vaddr_t emergva;
69 static boolean_t emerginuse;
70
71 /*
72 * uvm_pager_init: init pagers (at boot time)
73 */
74
75 void
uvm_pager_init()76 uvm_pager_init()
77 {
78 int lcv;
79
80 /*
81 * init pager map
82 */
83
84 pager_map = uvm_km_suballoc(kernel_map, &uvm.pager_sva, &uvm.pager_eva,
85 PAGER_MAP_SIZE, 0, FALSE, NULL);
86 simple_lock_init(&pager_map_wanted_lock);
87 pager_map_wanted = FALSE;
88 emergva = uvm_km_valloc(kernel_map, MAXBSIZE);
89 emerginuse = FALSE;
90
91 /*
92 * init ASYNC I/O queue
93 */
94
95 TAILQ_INIT(&uvm.aio_done);
96
97 /*
98 * call pager init functions
99 */
100 for (lcv = 0 ; lcv < sizeof(uvmpagerops)/sizeof(struct uvm_pagerops *);
101 lcv++) {
102 if (uvmpagerops[lcv]->pgo_init)
103 uvmpagerops[lcv]->pgo_init();
104 }
105 }
106
107 /*
108 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings
109 *
110 * we basically just map in a blank map entry to reserve the space in the
111 * map and then use pmap_enter() to put the mappings in by hand.
112 */
113
114 vaddr_t
uvm_pagermapin(pps,npages,flags)115 uvm_pagermapin(pps, npages, flags)
116 struct vm_page **pps;
117 int npages;
118 int flags;
119 {
120 vsize_t size;
121 vaddr_t kva;
122 vaddr_t cva;
123 struct vm_page *pp;
124 vm_prot_t prot;
125 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist);
126
127 UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d)", pps, npages,0,0);
128
129 /*
130 * compute protection. outgoing I/O only needs read
131 * access to the page, whereas incoming needs read/write.
132 */
133
134 prot = VM_PROT_READ;
135 if (flags & UVMPAGER_MAPIN_READ)
136 prot |= VM_PROT_WRITE;
137
138 ReStart:
139 size = npages << PAGE_SHIFT;
140 kva = 0; /* let system choose VA */
141
142 if (uvm_map(pager_map, &kva, size, NULL,
143 UVM_UNKNOWN_OFFSET, 0, UVM_FLAG_NOMERGE) != KERN_SUCCESS) {
144 if (curproc == uvm.pagedaemon_proc) {
145 simple_lock(&pager_map_wanted_lock);
146 if (emerginuse) {
147 UVM_UNLOCK_AND_WAIT(&emergva,
148 &pager_map_wanted_lock, FALSE,
149 "emergva", 0);
150 goto ReStart;
151 }
152 emerginuse = TRUE;
153 simple_unlock(&pager_map_wanted_lock);
154 kva = emergva;
155 KASSERT(npages <= MAXBSIZE >> PAGE_SHIFT);
156 goto enter;
157 }
158 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) {
159 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0);
160 return(0);
161 }
162 simple_lock(&pager_map_wanted_lock);
163 pager_map_wanted = TRUE;
164 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0);
165 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, FALSE,
166 "pager_map", 0);
167 goto ReStart;
168 }
169
170 enter:
171 /* got it */
172 for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) {
173 pp = *pps++;
174 KASSERT(pp);
175 KASSERT(pp->flags & PG_BUSY);
176 pmap_enter(vm_map_pmap(pager_map), cva, VM_PAGE_TO_PHYS(pp),
177 prot, PMAP_WIRED | prot);
178 }
179 pmap_update(vm_map_pmap(pager_map));
180
181 UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0);
182 return(kva);
183 }
184
185 /*
186 * uvm_pagermapout: remove pager_map mapping
187 *
188 * we remove our mappings by hand and then remove the mapping (waking
189 * up anyone wanting space).
190 */
191
192 void
uvm_pagermapout(kva,npages)193 uvm_pagermapout(kva, npages)
194 vaddr_t kva;
195 int npages;
196 {
197 vsize_t size = npages << PAGE_SHIFT;
198 vm_map_entry_t entries;
199 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist);
200
201 UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0);
202
203 /*
204 * duplicate uvm_unmap, but add in pager_map_wanted handling.
205 */
206
207 if (kva == emergva) {
208 simple_lock(&pager_map_wanted_lock);
209 emerginuse = FALSE;
210 wakeup(&emergva);
211 simple_unlock(&pager_map_wanted_lock);
212 entries = NULL;
213 goto remove;
214 }
215
216 vm_map_lock(pager_map);
217 uvm_unmap_remove(pager_map, kva, kva + size, &entries);
218 simple_lock(&pager_map_wanted_lock);
219 if (pager_map_wanted) {
220 pager_map_wanted = FALSE;
221 wakeup(pager_map);
222 }
223 simple_unlock(&pager_map_wanted_lock);
224 vm_map_unlock(pager_map);
225 remove:
226 pmap_remove(pmap_kernel(), kva, kva + (npages << PAGE_SHIFT));
227 if (entries)
228 uvm_unmap_detach(entries, 0);
229
230 pmap_update(pmap_kernel());
231 UVMHIST_LOG(maphist,"<- done",0,0,0,0);
232 }
233
234 /*
235 * uvm_mk_pcluster
236 *
237 * generic "make 'pager put' cluster" function. a pager can either
238 * [1] set pgo_mk_pcluster to NULL (never cluster), [2] set it to this
239 * generic function, or [3] set it to a pager specific function.
240 *
241 * => caller must lock object _and_ pagequeues (since we need to look
242 * at active vs. inactive bits, etc.)
243 * => caller must make center page busy and write-protect it
244 * => we mark all cluster pages busy for the caller
245 * => the caller must unbusy all pages (and check wanted/released
246 * status if it drops the object lock)
247 * => flags:
248 * PGO_ALLPAGES: all pages in object are valid targets
249 * !PGO_ALLPAGES: use "lo" and "hi" to limit range of cluster
250 * PGO_DOACTCLUST: include active pages in cluster.
251 * NOTE: the caller should clear PG_CLEANCHK bits if PGO_DOACTCLUST.
252 * PG_CLEANCHK is only a hint, but clearing will help reduce
253 * the number of calls we make to the pmap layer.
254 */
255
256 struct vm_page **
uvm_mk_pcluster(uobj,pps,npages,center,flags,mlo,mhi)257 uvm_mk_pcluster(uobj, pps, npages, center, flags, mlo, mhi)
258 struct uvm_object *uobj; /* IN */
259 struct vm_page **pps, *center; /* IN/OUT, IN */
260 int *npages, flags; /* IN/OUT, IN */
261 voff_t mlo, mhi; /* IN (if !PGO_ALLPAGES) */
262 {
263 struct vm_page **ppsp, *pclust;
264 voff_t lo, hi, curoff;
265 int center_idx, forward, incr;
266 UVMHIST_FUNC("uvm_mk_pcluster"); UVMHIST_CALLED(maphist);
267
268 /*
269 * center page should already be busy and write protected. XXX:
270 * suppose page is wired? if we lock, then a process could
271 * fault/block on it. if we don't lock, a process could write the
272 * pages in the middle of an I/O. (consider an msync()). let's
273 * lock it for now (better to delay than corrupt data?).
274 */
275
276 /*
277 * get cluster boundaries, check sanity, and apply our limits as well.
278 */
279
280 uobj->pgops->pgo_cluster(uobj, center->offset, &lo, &hi);
281 if ((flags & PGO_ALLPAGES) == 0) {
282 if (lo < mlo)
283 lo = mlo;
284 if (hi > mhi)
285 hi = mhi;
286 }
287 if ((hi - lo) >> PAGE_SHIFT > *npages) { /* pps too small, bail out! */
288 pps[0] = center;
289 *npages = 1;
290 return(pps);
291 }
292
293 /*
294 * now determine the center and attempt to cluster around the
295 * edges
296 */
297
298 center_idx = (center->offset - lo) >> PAGE_SHIFT;
299 pps[center_idx] = center; /* plug in the center page */
300 ppsp = &pps[center_idx];
301 *npages = 1;
302
303 /*
304 * attempt to cluster around the left [backward], and then
305 * the right side [forward].
306 *
307 * note that for inactive pages (pages that have been deactivated)
308 * there are no valid mappings and PG_CLEAN should be up to date.
309 * [i.e. there is no need to query the pmap with pmap_is_modified
310 * since there are no mappings].
311 */
312
313 for (forward = 0 ; forward <= 1 ; forward++) {
314 incr = forward ? PAGE_SIZE : -PAGE_SIZE;
315 curoff = center->offset + incr;
316 for ( ;(forward == 0 && curoff >= lo) ||
317 (forward && curoff < hi);
318 curoff += incr) {
319
320 pclust = uvm_pagelookup(uobj, curoff); /* lookup page */
321 if (pclust == NULL) {
322 break; /* no page */
323 }
324 /* handle active pages */
325 /* NOTE: inactive pages don't have pmap mappings */
326 if ((pclust->pqflags & PQ_INACTIVE) == 0) {
327 if ((flags & PGO_DOACTCLUST) == 0) {
328 /* dont want mapped pages at all */
329 break;
330 }
331
332 /* make sure "clean" bit is sync'd */
333 if ((pclust->flags & PG_CLEANCHK) == 0) {
334 if ((pclust->flags & (PG_CLEAN|PG_BUSY))
335 == PG_CLEAN &&
336 pmap_is_modified(pclust))
337 pclust->flags &= ~PG_CLEAN;
338 /* now checked */
339 pclust->flags |= PG_CLEANCHK;
340 }
341 }
342
343 /* is page available for cleaning and does it need it */
344 if ((pclust->flags & (PG_CLEAN|PG_BUSY)) != 0) {
345 break; /* page is already clean or is busy */
346 }
347
348 /* yes! enroll the page in our array */
349 pclust->flags |= PG_BUSY; /* busy! */
350 UVM_PAGE_OWN(pclust, "uvm_mk_pcluster");
351
352 /* XXX: protect wired page? see above comment. */
353 pmap_page_protect(pclust, VM_PROT_READ);
354 if (!forward) {
355 ppsp--; /* back up one page */
356 *ppsp = pclust;
357 } else {
358 /* move forward one page */
359 ppsp[*npages] = pclust;
360 }
361 (*npages)++;
362 }
363 }
364
365 /*
366 * done! return the cluster array to the caller!!!
367 */
368
369 UVMHIST_LOG(maphist, "<- done",0,0,0,0);
370 return(ppsp);
371 }
372
373 /*
374 * uvm_pager_put: high level pageout routine
375 *
376 * we want to pageout page "pg" to backing store, clustering if
377 * possible.
378 *
379 * => page queues must be locked by caller
380 * => if page is not swap-backed, then "uobj" points to the object
381 * backing it. this object should be locked by the caller.
382 * => if page is swap-backed, then "uobj" should be NULL.
383 * => "pg" should be PG_BUSY (by caller), and !PG_CLEAN
384 * for swap-backed memory, "pg" can be NULL if there is no page
385 * of interest [sometimes the case for the pagedaemon]
386 * => "ppsp_ptr" should point to an array of npages vm_page pointers
387 * for possible cluster building
388 * => flags (first two for non-swap-backed pages)
389 * PGO_ALLPAGES: all pages in uobj are valid targets
390 * PGO_DOACTCLUST: include "PQ_ACTIVE" pages as valid targets
391 * PGO_SYNCIO: do SYNC I/O (no async)
392 * PGO_PDFREECLUST: pagedaemon: drop cluster on successful I/O
393 * => start/stop: if (uobj && !PGO_ALLPAGES) limit targets to this range
394 * if (!uobj) start is the (daddr_t) of the starting swapblk
395 * => return state:
396 * 1. we return the VM_PAGER status code of the pageout
397 * 2. we return with the page queues unlocked
398 * 3. if (uobj != NULL) [!swap_backed] we return with
399 * uobj locked _only_ if PGO_PDFREECLUST is set
400 * AND result != VM_PAGER_PEND. in all other cases
401 * we return with uobj unlocked. [this is a hack
402 * that allows the pagedaemon to save one lock/unlock
403 * pair in the !swap_backed case since we have to
404 * lock the uobj to drop the cluster anyway]
405 * 4. on errors we always drop the cluster. thus, if we return
406 * !PEND, !OK, then the caller only has to worry about
407 * un-busying the main page (not the cluster pages).
408 * 5. on success, if !PGO_PDFREECLUST, we return the cluster
409 * with all pages busy (caller must un-busy and check
410 * wanted/released flags).
411 */
412
413 int
uvm_pager_put(uobj,pg,ppsp_ptr,npages,flags,start,stop)414 uvm_pager_put(uobj, pg, ppsp_ptr, npages, flags, start, stop)
415 struct uvm_object *uobj; /* IN */
416 struct vm_page *pg, ***ppsp_ptr;/* IN, IN/OUT */
417 int *npages; /* IN/OUT */
418 int flags; /* IN */
419 voff_t start, stop; /* IN, IN */
420 {
421 int result;
422 daddr_t swblk;
423 struct vm_page **ppsp = *ppsp_ptr;
424 UVMHIST_FUNC("uvm_pager_put"); UVMHIST_CALLED(pdhist);
425
426 /*
427 * note that uobj is null if we are doing a swap-backed pageout.
428 * note that uobj is !null if we are doing normal object pageout.
429 * note that the page queues must be locked to cluster.
430 */
431
432 if (uobj) { /* if !swap-backed */
433
434 /*
435 * attempt to build a cluster for pageout using its
436 * make-put-cluster function (if it has one).
437 */
438
439 if (uobj->pgops->pgo_mk_pcluster) {
440 ppsp = uobj->pgops->pgo_mk_pcluster(uobj, ppsp,
441 npages, pg, flags, start, stop);
442 *ppsp_ptr = ppsp; /* update caller's pointer */
443 } else {
444 ppsp[0] = pg;
445 *npages = 1;
446 }
447
448 swblk = 0; /* XXX: keep gcc happy */
449
450 } else {
451
452 /*
453 * for swap-backed pageout, the caller (the pagedaemon) has
454 * already built the cluster for us. the starting swap
455 * block we are writing to has been passed in as "start."
456 * "pg" could be NULL if there is no page we are especially
457 * interested in (in which case the whole cluster gets dropped
458 * in the event of an error or a sync "done").
459 */
460 swblk = (daddr_t) start;
461 /* ppsp and npages should be ok */
462 }
463
464 /* now that we've clustered we can unlock the page queues */
465 uvm_unlock_pageq();
466
467 /*
468 * now attempt the I/O. if we have a failure and we are
469 * clustered, we will drop the cluster and try again.
470 */
471
472 ReTry:
473 if (uobj) {
474 /* object is locked */
475 result = uobj->pgops->pgo_put(uobj, ppsp, *npages, flags);
476 UVMHIST_LOG(pdhist, "put -> %d", result, 0,0,0);
477 /* object is now unlocked */
478 } else {
479 /* nothing locked */
480 result = uvm_swap_put(swblk, ppsp, *npages, flags);
481 /* nothing locked */
482 }
483
484 /*
485 * we have attempted the I/O.
486 *
487 * if the I/O was a success then:
488 * if !PGO_PDFREECLUST, we return the cluster to the
489 * caller (who must un-busy all pages)
490 * else we un-busy cluster pages for the pagedaemon
491 *
492 * if I/O is pending (async i/o) then we return the pending code.
493 * [in this case the async i/o done function must clean up when
494 * i/o is done...]
495 */
496
497 if (result == VM_PAGER_PEND || result == VM_PAGER_OK) {
498 if (result == VM_PAGER_OK && (flags & PGO_PDFREECLUST)) {
499 /*
500 * drop cluster and relock object (only if I/O is
501 * not pending)
502 */
503 if (uobj)
504 /* required for dropcluster */
505 simple_lock(&uobj->vmobjlock);
506 if (*npages > 1 || pg == NULL)
507 uvm_pager_dropcluster(uobj, pg, ppsp, npages,
508 PGO_PDFREECLUST);
509 /* if (uobj): object still locked, as per
510 * return-state item #3 */
511 }
512 return (result);
513 }
514
515 /*
516 * a pager error occured (even after dropping the cluster, if there
517 * was one). give up! the caller only has one page ("pg")
518 * to worry about.
519 */
520
521 if (*npages > 1 || pg == NULL) {
522 if (uobj) {
523 simple_lock(&uobj->vmobjlock);
524 }
525 uvm_pager_dropcluster(uobj, pg, ppsp, npages, PGO_REALLOCSWAP);
526
527 /*
528 * for failed swap-backed pageouts with a "pg",
529 * we need to reset pg's swslot to either:
530 * "swblk" (for transient errors, so we can retry),
531 * or 0 (for hard errors).
532 */
533
534 if (uobj == NULL && pg != NULL) {
535 int nswblk = (result == VM_PAGER_AGAIN) ? swblk : 0;
536 if (pg->pqflags & PQ_ANON) {
537 simple_lock(&pg->uanon->an_lock);
538 pg->uanon->an_swslot = nswblk;
539 simple_unlock(&pg->uanon->an_lock);
540 } else {
541 simple_lock(&pg->uobject->vmobjlock);
542 uao_set_swslot(pg->uobject,
543 pg->offset >> PAGE_SHIFT,
544 nswblk);
545 simple_unlock(&pg->uobject->vmobjlock);
546 }
547 }
548 if (result == VM_PAGER_AGAIN) {
549
550 /*
551 * for transient failures, free all the swslots that
552 * we're not going to retry with.
553 */
554
555 if (uobj == NULL) {
556 if (pg) {
557 uvm_swap_free(swblk + 1, *npages - 1);
558 } else {
559 uvm_swap_free(swblk, *npages);
560 }
561 }
562 if (pg) {
563 ppsp[0] = pg;
564 *npages = 1;
565 goto ReTry;
566 }
567 } else if (uobj == NULL) {
568
569 /*
570 * for hard errors on swap-backed pageouts,
571 * mark the swslots as bad. note that we do not
572 * free swslots that we mark bad.
573 */
574
575 uvm_swap_markbad(swblk, *npages);
576 }
577 }
578
579 /*
580 * a pager error occurred (even after dropping the cluster, if there
581 * was one). give up! the caller only has one page ("pg")
582 * to worry about.
583 */
584
585 if (uobj && (flags & PGO_PDFREECLUST) != 0)
586 simple_lock(&uobj->vmobjlock);
587 return(result);
588 }
589
590 /*
591 * uvm_pager_dropcluster: drop a cluster we have built (because we
592 * got an error, or, if PGO_PDFREECLUST we are un-busying the
593 * cluster pages on behalf of the pagedaemon).
594 *
595 * => uobj, if non-null, is a non-swap-backed object that is
596 * locked by the caller. we return with this object still
597 * locked.
598 * => page queues are not locked
599 * => pg is our page of interest (the one we clustered around, can be null)
600 * => ppsp/npages is our current cluster
601 * => flags: PGO_PDFREECLUST: pageout was a success: un-busy cluster
602 * pages on behalf of the pagedaemon.
603 * PGO_REALLOCSWAP: drop previously allocated swap slots for
604 * clustered swap-backed pages (except for "pg" if !NULL)
605 * "swblk" is the start of swap alloc (e.g. for ppsp[0])
606 * [only meaningful if swap-backed (uobj == NULL)]
607 */
608
609 void
uvm_pager_dropcluster(uobj,pg,ppsp,npages,flags)610 uvm_pager_dropcluster(uobj, pg, ppsp, npages, flags)
611 struct uvm_object *uobj; /* IN */
612 struct vm_page *pg, **ppsp; /* IN, IN/OUT */
613 int *npages; /* IN/OUT */
614 int flags;
615 {
616 int lcv;
617 boolean_t obj_is_alive;
618 struct uvm_object *saved_uobj;
619
620 /*
621 * drop all pages but "pg"
622 */
623
624 for (lcv = 0 ; lcv < *npages ; lcv++) {
625
626 /* skip "pg" or empty slot */
627 if (ppsp[lcv] == pg || ppsp[lcv] == NULL)
628 continue;
629
630 /*
631 * if swap-backed, gain lock on object that owns page. note
632 * that PQ_ANON bit can't change as long as we are holding
633 * the PG_BUSY bit (so there is no need to lock the page
634 * queues to test it).
635 *
636 * once we have the lock, dispose of the pointer to swap, if
637 * requested
638 */
639 if (!uobj) {
640 if (ppsp[lcv]->pqflags & PQ_ANON) {
641 simple_lock(&ppsp[lcv]->uanon->an_lock);
642 if (flags & PGO_REALLOCSWAP)
643 /* zap swap block */
644 ppsp[lcv]->uanon->an_swslot = 0;
645 } else {
646 simple_lock(&ppsp[lcv]->uobject->vmobjlock);
647 if (flags & PGO_REALLOCSWAP)
648 uao_set_swslot(ppsp[lcv]->uobject,
649 ppsp[lcv]->offset >> PAGE_SHIFT, 0);
650 }
651 }
652
653 /* did someone want the page while we had it busy-locked? */
654 if (ppsp[lcv]->flags & PG_WANTED) {
655 /* still holding obj lock */
656 wakeup(ppsp[lcv]);
657 }
658
659 /* if page was released, release it. otherwise un-busy it */
660 if (ppsp[lcv]->flags & PG_RELEASED) {
661
662 if (ppsp[lcv]->pqflags & PQ_ANON) {
663 /* so that anfree will free */
664 ppsp[lcv]->flags &= ~(PG_BUSY);
665 UVM_PAGE_OWN(ppsp[lcv], NULL);
666
667 pmap_page_protect(ppsp[lcv], VM_PROT_NONE);
668 simple_unlock(&ppsp[lcv]->uanon->an_lock);
669 /* kills anon and frees pg */
670 uvm_anfree(ppsp[lcv]->uanon);
671
672 continue;
673 }
674
675 /*
676 * pgo_releasepg will dump the page for us
677 */
678
679 saved_uobj = ppsp[lcv]->uobject;
680 obj_is_alive =
681 saved_uobj->pgops->pgo_releasepg(ppsp[lcv], NULL);
682
683 /* for normal objects, "pg" is still PG_BUSY by us,
684 * so obj can't die */
685 KASSERT(!uobj || obj_is_alive);
686
687 /* only unlock the object if it is still alive... */
688 if (obj_is_alive && saved_uobj != uobj)
689 simple_unlock(&saved_uobj->vmobjlock);
690
691 /*
692 * XXXCDC: suppose uobj died in the pgo_releasepg?
693 * how pass that
694 * info up to caller. we are currently ignoring it...
695 */
696
697 continue; /* next page */
698
699 } else {
700 ppsp[lcv]->flags &= ~(PG_BUSY|PG_WANTED|PG_FAKE);
701 UVM_PAGE_OWN(ppsp[lcv], NULL);
702 }
703
704 /*
705 * if we are operating on behalf of the pagedaemon and we
706 * had a successful pageout update the page!
707 */
708 if (flags & PGO_PDFREECLUST) {
709 pmap_clear_reference(ppsp[lcv]);
710 pmap_clear_modify(ppsp[lcv]);
711 ppsp[lcv]->flags |= PG_CLEAN;
712 }
713
714 /* if anonymous cluster, unlock object and move on */
715 if (!uobj) {
716 if (ppsp[lcv]->pqflags & PQ_ANON)
717 simple_unlock(&ppsp[lcv]->uanon->an_lock);
718 else
719 simple_unlock(&ppsp[lcv]->uobject->vmobjlock);
720 }
721 }
722 }
723
724 #ifdef UBC
725 /*
726 * interrupt-context iodone handler for nested i/o bufs.
727 *
728 * => must be at splbio().
729 */
730
731 void
uvm_aio_biodone1(bp)732 uvm_aio_biodone1(bp)
733 struct buf *bp;
734 {
735 struct buf *mbp = bp->b_private;
736
737 splassert(IPL_BIO);
738
739 KASSERT(mbp != bp);
740 if (bp->b_flags & B_ERROR) {
741 mbp->b_flags |= B_ERROR;
742 mbp->b_error = bp->b_error;
743 }
744 mbp->b_resid -= bp->b_bcount;
745 pool_put(&bufpool, bp);
746 if (mbp->b_resid == 0) {
747 biodone(mbp);
748 }
749 }
750 #endif
751
752 /*
753 * interrupt-context iodone handler for single-buf i/os
754 * or the top-level buf of a nested-buf i/o.
755 *
756 * => must be at splbio().
757 */
758
759 void
uvm_aio_biodone(bp)760 uvm_aio_biodone(bp)
761 struct buf *bp;
762 {
763 splassert(IPL_BIO);
764
765 /* reset b_iodone for when this is a single-buf i/o. */
766 bp->b_iodone = uvm_aio_aiodone;
767
768 simple_lock(&uvm.aiodoned_lock); /* locks uvm.aio_done */
769 TAILQ_INSERT_TAIL(&uvm.aio_done, bp, b_freelist);
770 wakeup(&uvm.aiodoned);
771 simple_unlock(&uvm.aiodoned_lock);
772 }
773
774 /*
775 * uvm_aio_aiodone: do iodone processing for async i/os.
776 * this should be called in thread context, not interrupt context.
777 */
778
779 void
uvm_aio_aiodone(bp)780 uvm_aio_aiodone(bp)
781 struct buf *bp;
782 {
783 int npages = bp->b_bufsize >> PAGE_SHIFT;
784 struct vm_page *pg, *pgs[npages];
785 struct uvm_object *uobj;
786 int i, error;
787 boolean_t write, swap = FALSE;
788 UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(pdhist);
789 UVMHIST_LOG(pdhist, "bp %p", bp, 0,0,0);
790
791 splassert(IPL_BIO);
792
793 error = (bp->b_flags & B_ERROR) ? (bp->b_error ? bp->b_error : EIO) : 0;
794 write = (bp->b_flags & B_READ) == 0;
795 #ifdef UBC
796 /* XXXUBC B_NOCACHE is for swap pager, should be done differently */
797 if (write && !(bp->b_flags & B_NOCACHE) && bioops.io_pageiodone) {
798 (*bioops.io_pageiodone)(bp);
799 }
800 #endif
801
802 uobj = NULL;
803 for (i = 0; i < npages; i++) {
804 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT));
805 UVMHIST_LOG(pdhist, "pgs[%d] = %p", i, pgs[i],0,0);
806 }
807 uvm_pagermapout((vaddr_t)bp->b_data, npages);
808 #ifdef UVM_SWAP_ENCRYPT
809 /*
810 * XXX - assumes that we only get ASYNC writes. used to be above.
811 */
812 if (pgs[0]->pqflags & PQ_ENCRYPT) {
813 uvm_swap_freepages(pgs, npages);
814 goto freed;
815 }
816 #endif /* UVM_SWAP_ENCRYPT */
817 for (i = 0; i < npages; i++) {
818 pg = pgs[i];
819
820 if (i == 0) {
821 swap = (pg->pqflags & PQ_SWAPBACKED) != 0;
822 if (!swap) {
823 uobj = pg->uobject;
824 simple_lock(&uobj->vmobjlock);
825 }
826 }
827 KASSERT(swap || pg->uobject == uobj);
828 if (swap) {
829 if (pg->pqflags & PQ_ANON) {
830 simple_lock(&pg->uanon->an_lock);
831 } else {
832 simple_lock(&pg->uobject->vmobjlock);
833 }
834 }
835
836 /*
837 * if this is a read and we got an error, mark the pages
838 * PG_RELEASED so that uvm_page_unbusy() will free them.
839 */
840
841 if (!write && error) {
842 pg->flags |= PG_RELEASED;
843 continue;
844 }
845 KASSERT(!write || (pgs[i]->flags & PG_FAKE) == 0);
846
847 /*
848 * if this is a read and the page is PG_FAKE,
849 * or this was a successful write,
850 * mark the page PG_CLEAN and not PG_FAKE.
851 */
852
853 if ((pgs[i]->flags & PG_FAKE) || (write && error != ENOMEM)) {
854 pmap_clear_reference(pgs[i]);
855 pmap_clear_modify(pgs[i]);
856 pgs[i]->flags |= PG_CLEAN;
857 pgs[i]->flags &= ~PG_FAKE;
858 }
859 if (swap) {
860 if (pg->pqflags & PQ_ANON) {
861 simple_unlock(&pg->uanon->an_lock);
862 } else {
863 simple_unlock(&pg->uobject->vmobjlock);
864 }
865 }
866 }
867 uvm_page_unbusy(pgs, npages);
868 if (!swap) {
869 simple_unlock(&uobj->vmobjlock);
870 }
871
872 #ifdef UVM_SWAP_ENCRYPT
873 freed:
874 #endif
875 if (write && (bp->b_flags & B_AGE) != 0 && bp->b_vp != NULL) {
876 vwakeup(bp->b_vp);
877 }
878 pool_put(&bufpool, bp);
879 }
880
881 /*
882 * translate unix errno values to VM_PAGER_*.
883 */
884
885 int
uvm_errno2vmerror(errno)886 uvm_errno2vmerror(errno)
887 int errno;
888 {
889 switch (errno) {
890 case 0:
891 return VM_PAGER_OK;
892 case EINVAL:
893 return VM_PAGER_BAD;
894 case EINPROGRESS:
895 return VM_PAGER_PEND;
896 case EIO:
897 return VM_PAGER_ERROR;
898 case EAGAIN:
899 return VM_PAGER_AGAIN;
900 case EBUSY:
901 return VM_PAGER_UNLOCK;
902 default:
903 return VM_PAGER_ERROR;
904 }
905 }
906