1 /* $NetBSD: vfs_bio.c,v 1.306 2024/12/07 02:27:38 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2007, 2008, 2009, 2019, 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran, and by Wasabi Systems, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*-
33 * Copyright (c) 1982, 1986, 1989, 1993
34 * The Regents of the University of California. All rights reserved.
35 * (c) UNIX System Laboratories, Inc.
36 * All or some portions of this file are derived from material licensed
37 * to the University of California by American Telephone and Telegraph
38 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
39 * the permission of UNIX System Laboratories, Inc.
40 *
41 * Redistribution and use in source and binary forms, with or without
42 * modification, are permitted provided that the following conditions
43 * are met:
44 * 1. Redistributions of source code must retain the above copyright
45 * notice, this list of conditions and the following disclaimer.
46 * 2. Redistributions in binary form must reproduce the above copyright
47 * notice, this list of conditions and the following disclaimer in the
48 * documentation and/or other materials provided with the distribution.
49 * 3. Neither the name of the University nor the names of its contributors
50 * may be used to endorse or promote products derived from this software
51 * without specific prior written permission.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63 * SUCH DAMAGE.
64 *
65 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94
66 */
67
68 /*-
69 * Copyright (c) 1994 Christopher G. Demetriou
70 *
71 * Redistribution and use in source and binary forms, with or without
72 * modification, are permitted provided that the following conditions
73 * are met:
74 * 1. Redistributions of source code must retain the above copyright
75 * notice, this list of conditions and the following disclaimer.
76 * 2. Redistributions in binary form must reproduce the above copyright
77 * notice, this list of conditions and the following disclaimer in the
78 * documentation and/or other materials provided with the distribution.
79 * 3. All advertising materials mentioning features or use of this software
80 * must display the following acknowledgement:
81 * This product includes software developed by the University of
82 * California, Berkeley and its contributors.
83 * 4. Neither the name of the University nor the names of its contributors
84 * may be used to endorse or promote products derived from this software
85 * without specific prior written permission.
86 *
87 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
88 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
89 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
90 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
91 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
92 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
93 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
94 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
95 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
96 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
97 * SUCH DAMAGE.
98 *
99 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94
100 */
101
102 /*
103 * The buffer cache subsystem.
104 *
105 * Some references:
106 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986)
107 * Leffler, et al.: The Design and Implementation of the 4.3BSD
108 * UNIX Operating System (Addison Welley, 1989)
109 *
110 * Locking
111 *
112 * There are three locks:
113 * - bufcache_lock: protects global buffer cache state.
114 * - BC_BUSY: a long term per-buffer lock.
115 * - buf_t::b_objlock: lock on completion (biowait vs biodone).
116 *
117 * For buffers associated with vnodes (a most common case) b_objlock points
118 * to the vnode_t::v_interlock. Otherwise, it points to generic buffer_lock.
119 *
120 * Lock order:
121 * bufcache_lock ->
122 * buf_t::b_objlock
123 */
124
125 #include <sys/cdefs.h>
126 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.306 2024/12/07 02:27:38 riastradh Exp $");
127
128 #ifdef _KERNEL_OPT
129 #include "opt_biohist.h"
130 #include "opt_bufcache.h"
131 #include "opt_dtrace.h"
132 #endif
133
134 #include <sys/param.h>
135 #include <sys/types.h>
136
137 #include <sys/bitops.h>
138 #include <sys/buf.h>
139 #include <sys/conf.h>
140 #include <sys/cprng.h>
141 #include <sys/cpu.h>
142 #include <sys/fstrans.h>
143 #include <sys/intr.h>
144 #include <sys/kauth.h>
145 #include <sys/kernel.h>
146 #include <sys/mount.h>
147 #include <sys/proc.h>
148 #include <sys/resourcevar.h>
149 #include <sys/sdt.h>
150 #include <sys/sysctl.h>
151 #include <sys/systm.h>
152 #include <sys/vnode.h>
153 #include <sys/wapbl.h>
154
155 #include <uvm/uvm.h> /* extern struct uvm uvm */
156
157 #include <miscfs/specfs/specdev.h>
158
159 SDT_PROVIDER_DEFINE(io);
160
161 SDT_PROBE_DEFINE4(io, kernel, , bbusy__start,
162 "struct buf *"/*bp*/,
163 "bool"/*intr*/, "int"/*timo*/, "kmutex_t *"/*interlock*/);
164 SDT_PROBE_DEFINE5(io, kernel, , bbusy__done,
165 "struct buf *"/*bp*/,
166 "bool"/*intr*/,
167 "int"/*timo*/,
168 "kmutex_t *"/*interlock*/,
169 "int"/*error*/);
170 SDT_PROBE_DEFINE0(io, kernel, , getnewbuf__start);
171 SDT_PROBE_DEFINE1(io, kernel, , getnewbuf__done, "struct buf *"/*bp*/);
172 SDT_PROBE_DEFINE3(io, kernel, , getblk__start,
173 "struct vnode *"/*vp*/, "daddr_t"/*blkno*/, "int"/*size*/);
174 SDT_PROBE_DEFINE4(io, kernel, , getblk__done,
175 "struct vnode *"/*vp*/, "daddr_t"/*blkno*/, "int"/*size*/,
176 "struct buf *"/*bp*/);
177 SDT_PROBE_DEFINE2(io, kernel, , brelse, "struct buf *"/*bp*/, "int"/*set*/);
178 SDT_PROBE_DEFINE1(io, kernel, , wait__start, "struct buf *"/*bp*/);
179 SDT_PROBE_DEFINE1(io, kernel, , wait__done, "struct buf *"/*bp*/);
180
181 #ifndef BUFPAGES
182 # define BUFPAGES 0
183 #endif
184
185 #ifdef BUFCACHE
186 # if (BUFCACHE < 5) || (BUFCACHE > 95)
187 # error BUFCACHE is not between 5 and 95
188 # endif
189 #else
190 # define BUFCACHE 15
191 #endif
192
193 u_int nbuf; /* desired number of buffer headers */
194 u_int bufpages = BUFPAGES; /* optional hardwired count */
195 u_int bufcache = BUFCACHE; /* max % of RAM to use for buffer cache */
196
197 /*
198 * Definitions for the buffer free lists.
199 */
200 #define BQUEUES 3 /* number of free buffer queues */
201
202 #define BQ_LOCKED 0 /* super-blocks &c */
203 #define BQ_LRU 1 /* lru, useful buffers */
204 #define BQ_AGE 2 /* rubbish */
205
206 struct bqueue {
207 TAILQ_HEAD(, buf) bq_queue;
208 uint64_t bq_bytes;
209 buf_t *bq_marker;
210 };
211 static struct bqueue bufqueues[BQUEUES] __cacheline_aligned;
212
213 /* Function prototypes */
214 static void buf_setwm(void);
215 static int buf_trim(void);
216 static void *bufpool_page_alloc(struct pool *, int);
217 static void bufpool_page_free(struct pool *, void *);
218 static buf_t *bio_doread(struct vnode *, daddr_t, int, int);
219 static buf_t *getnewbuf(int, int, int);
220 static int buf_lotsfree(void);
221 static int buf_canrelease(void);
222 static u_long buf_mempoolidx(u_long);
223 static u_long buf_roundsize(u_long);
224 static void *buf_alloc(size_t);
225 static void buf_mrelease(void *, size_t);
226 static void binsheadfree(buf_t *, struct bqueue *);
227 static void binstailfree(buf_t *, struct bqueue *);
228 #ifdef DEBUG
229 static int checkfreelist(buf_t *, struct bqueue *, int);
230 #endif
231 static void biointr(void *);
232 static void biodone2(buf_t *);
233 static void sysctl_kern_buf_setup(void);
234 static void sysctl_vm_buf_setup(void);
235
236 /* Initialization for biohist */
237
238 #include <sys/biohist.h>
239
240 BIOHIST_DEFINE(biohist);
241
242 void
biohist_init(void)243 biohist_init(void)
244 {
245
246 BIOHIST_INIT(biohist, BIOHIST_SIZE);
247 }
248
249 /*
250 * Definitions for the buffer hash lists.
251 */
252 #define BUFHASH(dvp, lbn) \
253 (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash])
254 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
255 u_long bufhash;
256
257 static int bufhash_stats(struct hashstat_sysctl *, bool);
258
259 static kcondvar_t needbuffer_cv;
260
261 /*
262 * Buffer queue lock.
263 */
264 kmutex_t bufcache_lock __cacheline_aligned;
265 kmutex_t buffer_lock __cacheline_aligned;
266
267 /* Software ISR for completed transfers. */
268 static void *biodone_sih;
269
270 /* Buffer pool for I/O buffers. */
271 static pool_cache_t buf_cache;
272 static pool_cache_t bufio_cache;
273
274 #define MEMPOOL_INDEX_OFFSET (ilog2(DEV_BSIZE)) /* smallest pool is 512 bytes */
275 #define NMEMPOOLS (ilog2(MAXBSIZE) - MEMPOOL_INDEX_OFFSET + 1)
276 __CTASSERT((1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) == MAXBSIZE);
277
278 /* Buffer memory pools */
279 static struct pool bmempools[NMEMPOOLS];
280
281 static struct vm_map *buf_map;
282
283 /*
284 * Buffer memory pool allocator.
285 */
286 static void *
bufpool_page_alloc(struct pool * pp,int flags)287 bufpool_page_alloc(struct pool *pp, int flags)
288 {
289
290 return (void *)uvm_km_alloc(buf_map,
291 MAXBSIZE, MAXBSIZE,
292 ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT|UVM_KMF_TRYLOCK)
293 | UVM_KMF_WIRED);
294 }
295
296 static void
bufpool_page_free(struct pool * pp,void * v)297 bufpool_page_free(struct pool *pp, void *v)
298 {
299
300 uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED);
301 }
302
303 static struct pool_allocator bufmempool_allocator = {
304 .pa_alloc = bufpool_page_alloc,
305 .pa_free = bufpool_page_free,
306 .pa_pagesz = MAXBSIZE,
307 };
308
309 /* Buffer memory management variables */
310 u_long bufmem_valimit;
311 u_long bufmem_hiwater;
312 u_long bufmem_lowater;
313 u_long bufmem;
314
315 /*
316 * MD code can call this to set a hard limit on the amount
317 * of virtual memory used by the buffer cache.
318 */
319 int
buf_setvalimit(vsize_t sz)320 buf_setvalimit(vsize_t sz)
321 {
322
323 /* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */
324 if (sz < NMEMPOOLS * MAXBSIZE)
325 return SET_ERROR(EINVAL);
326
327 bufmem_valimit = sz;
328 return 0;
329 }
330
331 static void
buf_setwm(void)332 buf_setwm(void)
333 {
334
335 bufmem_hiwater = buf_memcalc();
336 /* lowater is approx. 2% of memory (with bufcache = 15) */
337 #define BUFMEM_WMSHIFT 3
338 #define BUFMEM_HIWMMIN (64 * 1024 << BUFMEM_WMSHIFT)
339 if (bufmem_hiwater < BUFMEM_HIWMMIN)
340 /* Ensure a reasonable minimum value */
341 bufmem_hiwater = BUFMEM_HIWMMIN;
342 bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT;
343 }
344
345 #ifdef DEBUG
346 int debug_verify_freelist = 0;
347 static int
checkfreelist(buf_t * bp,struct bqueue * dp,int ison)348 checkfreelist(buf_t *bp, struct bqueue *dp, int ison)
349 {
350 buf_t *b;
351
352 if (!debug_verify_freelist)
353 return 1;
354
355 TAILQ_FOREACH(b, &dp->bq_queue, b_freelist) {
356 if (b == bp)
357 return ison ? 1 : 0;
358 }
359
360 return ison ? 0 : 1;
361 }
362 #endif
363
364 /*
365 * Insq/Remq for the buffer hash lists.
366 * Call with buffer queue locked.
367 */
368 static void
binsheadfree(buf_t * bp,struct bqueue * dp)369 binsheadfree(buf_t *bp, struct bqueue *dp)
370 {
371
372 KASSERT(mutex_owned(&bufcache_lock));
373 KASSERT(bp->b_freelistindex == -1);
374 TAILQ_INSERT_HEAD(&dp->bq_queue, bp, b_freelist);
375 dp->bq_bytes += bp->b_bufsize;
376 bp->b_freelistindex = dp - bufqueues;
377 }
378
379 static void
binstailfree(buf_t * bp,struct bqueue * dp)380 binstailfree(buf_t *bp, struct bqueue *dp)
381 {
382
383 KASSERT(mutex_owned(&bufcache_lock));
384 KASSERTMSG(bp->b_freelistindex == -1, "double free of buffer? "
385 "bp=%p, b_freelistindex=%d\n", bp, bp->b_freelistindex);
386 TAILQ_INSERT_TAIL(&dp->bq_queue, bp, b_freelist);
387 dp->bq_bytes += bp->b_bufsize;
388 bp->b_freelistindex = dp - bufqueues;
389 }
390
391 void
bremfree(buf_t * bp)392 bremfree(buf_t *bp)
393 {
394 struct bqueue *dp;
395 int bqidx = bp->b_freelistindex;
396
397 KASSERT(mutex_owned(&bufcache_lock));
398
399 KASSERT(bqidx != -1);
400 dp = &bufqueues[bqidx];
401 KDASSERT(checkfreelist(bp, dp, 1));
402 KASSERT(dp->bq_bytes >= bp->b_bufsize);
403 TAILQ_REMOVE(&dp->bq_queue, bp, b_freelist);
404 dp->bq_bytes -= bp->b_bufsize;
405
406 /* For the sysctl helper. */
407 if (bp == dp->bq_marker)
408 dp->bq_marker = NULL;
409
410 #if defined(DIAGNOSTIC)
411 bp->b_freelistindex = -1;
412 #endif /* defined(DIAGNOSTIC) */
413 }
414
415 /*
416 * note that for some ports this is used by pmap bootstrap code to
417 * determine kva size.
418 */
419 u_long
buf_memcalc(void)420 buf_memcalc(void)
421 {
422 u_long n;
423 vsize_t mapsz = 0;
424
425 /*
426 * Determine the upper bound of memory to use for buffers.
427 *
428 * - If bufpages is specified, use that as the number
429 * pages.
430 *
431 * - Otherwise, use bufcache as the percentage of
432 * physical memory.
433 */
434 if (bufpages != 0) {
435 n = bufpages;
436 } else {
437 if (bufcache < 5) {
438 printf("forcing bufcache %d -> 5", bufcache);
439 bufcache = 5;
440 }
441 if (bufcache > 95) {
442 printf("forcing bufcache %d -> 95", bufcache);
443 bufcache = 95;
444 }
445 if (buf_map != NULL)
446 mapsz = vm_map_max(buf_map) - vm_map_min(buf_map);
447 n = calc_cache_size(mapsz, bufcache,
448 (buf_map != kernel_map) ? 100 : BUFCACHE_VA_MAXPCT)
449 / PAGE_SIZE;
450 }
451
452 n <<= PAGE_SHIFT;
453 if (bufmem_valimit != 0 && n > bufmem_valimit)
454 n = bufmem_valimit;
455
456 return n;
457 }
458
459 /*
460 * Initialize buffers and hash links for buffers.
461 */
462 void
bufinit(void)463 bufinit(void)
464 {
465 struct bqueue *dp;
466 int use_std;
467 u_int i;
468
469 biodone_vfs = biodone;
470
471 mutex_init(&bufcache_lock, MUTEX_DEFAULT, IPL_NONE);
472 mutex_init(&buffer_lock, MUTEX_DEFAULT, IPL_NONE);
473 cv_init(&needbuffer_cv, "needbuf");
474
475 if (bufmem_valimit != 0) {
476 vaddr_t minaddr = 0, maxaddr;
477 buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
478 bufmem_valimit, 0, false, 0);
479 if (buf_map == NULL)
480 panic("bufinit: cannot allocate submap");
481 } else
482 buf_map = kernel_map;
483
484 /*
485 * Initialize buffer cache memory parameters.
486 */
487 bufmem = 0;
488 buf_setwm();
489
490 /* On "small" machines use small pool page sizes where possible */
491 use_std = (physmem < atop(16*1024*1024));
492
493 /*
494 * Also use them on systems that can map the pool pages using
495 * a direct-mapped segment.
496 */
497 #ifdef PMAP_MAP_POOLPAGE
498 use_std = 1;
499 #endif
500
501 buf_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
502 "bufpl", NULL, IPL_SOFTBIO, NULL, NULL, NULL);
503 bufio_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
504 "biopl", NULL, IPL_BIO, NULL, NULL, NULL);
505
506 for (i = 0; i < NMEMPOOLS; i++) {
507 struct pool_allocator *pa;
508 struct pool *pp = &bmempools[i];
509 u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET);
510 char *name = kmem_alloc(8, KM_SLEEP); /* XXX: never freed */
511
512 if (__predict_false(size >= 1048576))
513 (void)snprintf(name, 8, "buf%um", size / 1048576);
514 else if (__predict_true(size >= 1024))
515 (void)snprintf(name, 8, "buf%uk", size / 1024);
516 else
517 (void)snprintf(name, 8, "buf%ub", size);
518 pa = (size <= PAGE_SIZE && use_std)
519 ? &pool_allocator_nointr
520 : &bufmempool_allocator;
521 pool_init(pp, size, DEV_BSIZE, 0, 0, name, pa, IPL_NONE);
522 pool_setlowat(pp, 1);
523 pool_sethiwat(pp, 1);
524 }
525
526 /* Initialize the buffer queues */
527 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) {
528 TAILQ_INIT(&dp->bq_queue);
529 dp->bq_bytes = 0;
530 }
531
532 /*
533 * Estimate hash table size based on the amount of memory we
534 * intend to use for the buffer cache. The average buffer
535 * size is dependent on our clients (i.e. filesystems).
536 *
537 * For now, use an empirical 3K per buffer.
538 */
539 nbuf = (bufmem_hiwater / 1024) / 3;
540 bufhashtbl = hashinit(nbuf, HASH_LIST, true, &bufhash);
541
542 sysctl_kern_buf_setup();
543 sysctl_vm_buf_setup();
544 hashstat_register("bufhash", bufhash_stats);
545 }
546
547 void
bufinit2(void)548 bufinit2(void)
549 {
550
551 biodone_sih = softint_establish(SOFTINT_BIO | SOFTINT_MPSAFE, biointr,
552 NULL);
553 if (biodone_sih == NULL)
554 panic("bufinit2: can't establish soft interrupt");
555 }
556
557 static int
buf_lotsfree(void)558 buf_lotsfree(void)
559 {
560 u_long guess;
561
562 /* Always allocate if less than the low water mark. */
563 if (bufmem < bufmem_lowater)
564 return 1;
565
566 /* Never allocate if greater than the high water mark. */
567 if (bufmem > bufmem_hiwater)
568 return 0;
569
570 /* If there's anything on the AGE list, it should be eaten. */
571 if (TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue) != NULL)
572 return 0;
573
574 /*
575 * The probabily of getting a new allocation is inversely
576 * proportional to the current size of the cache above
577 * the low water mark. Divide the total first to avoid overflows
578 * in the product.
579 */
580 guess = cprng_fast32() % 16;
581
582 if ((bufmem_hiwater - bufmem_lowater) / 16 * guess >=
583 (bufmem - bufmem_lowater))
584 return 1;
585
586 /* Otherwise don't allocate. */
587 return 0;
588 }
589
590 /*
591 * Return estimate of bytes we think need to be
592 * released to help resolve low memory conditions.
593 *
594 * => called with bufcache_lock held.
595 */
596 static int
buf_canrelease(void)597 buf_canrelease(void)
598 {
599 int pagedemand, ninvalid = 0;
600
601 KASSERT(mutex_owned(&bufcache_lock));
602
603 if (bufmem < bufmem_lowater)
604 return 0;
605
606 if (bufmem > bufmem_hiwater)
607 return bufmem - bufmem_hiwater;
608
609 ninvalid += bufqueues[BQ_AGE].bq_bytes;
610
611 pagedemand = uvmexp.freetarg - uvm_availmem(false);
612 if (pagedemand < 0)
613 return ninvalid;
614 return MAX(ninvalid, MIN(2 * MAXBSIZE,
615 MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE)));
616 }
617
618 /*
619 * Buffer memory allocation helper functions
620 */
621 static u_long
buf_mempoolidx(u_long size)622 buf_mempoolidx(u_long size)
623 {
624 u_int n = 0;
625
626 size -= 1;
627 size >>= MEMPOOL_INDEX_OFFSET;
628 while (size) {
629 size >>= 1;
630 n += 1;
631 }
632 if (n >= NMEMPOOLS)
633 panic("buf mem pool index %d", n);
634 return n;
635 }
636
637 static u_long
buf_roundsize(u_long size)638 buf_roundsize(u_long size)
639 {
640
641 /* Round up to nearest power of 2 */
642 return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET));
643 }
644
645 static void *
buf_alloc(size_t size)646 buf_alloc(size_t size)
647 {
648 u_int n = buf_mempoolidx(size);
649 void *addr;
650
651 while (1) {
652 addr = pool_get(&bmempools[n], PR_NOWAIT);
653 if (addr != NULL)
654 break;
655
656 /* No memory, see if we can free some. If so, try again */
657 mutex_enter(&bufcache_lock);
658 if (buf_drain(1) > 0) {
659 mutex_exit(&bufcache_lock);
660 continue;
661 }
662
663 if (curlwp == uvm.pagedaemon_lwp) {
664 mutex_exit(&bufcache_lock);
665 return NULL;
666 }
667
668 /* Wait for buffers to arrive on the LRU queue */
669 cv_timedwait(&needbuffer_cv, &bufcache_lock, hz / 4);
670 mutex_exit(&bufcache_lock);
671 }
672
673 return addr;
674 }
675
676 static void
buf_mrelease(void * addr,size_t size)677 buf_mrelease(void *addr, size_t size)
678 {
679
680 pool_put(&bmempools[buf_mempoolidx(size)], addr);
681 }
682
683 /*
684 * bread()/breadn() helper.
685 */
686 static buf_t *
bio_doread(struct vnode * vp,daddr_t blkno,int size,int async)687 bio_doread(struct vnode *vp, daddr_t blkno, int size, int async)
688 {
689 buf_t *bp;
690 struct mount *mp;
691
692 bp = getblk(vp, blkno, size, 0, 0);
693
694 /*
695 * getblk() may return NULL if we are the pagedaemon.
696 */
697 if (bp == NULL) {
698 KASSERT(curlwp == uvm.pagedaemon_lwp);
699 return NULL;
700 }
701
702 /*
703 * If buffer does not have data valid, start a read.
704 * Note that if buffer is BC_INVAL, getblk() won't return it.
705 * Therefore, it's valid if its I/O has completed or been delayed.
706 */
707 if (!ISSET(bp->b_oflags, (BO_DONE | BO_DELWRI))) {
708 /* Start I/O for the buffer. */
709 SET(bp->b_flags, B_READ | async);
710 if (async)
711 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
712 else
713 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
714 VOP_STRATEGY(vp, bp);
715
716 /* Pay for the read. */
717 curlwp->l_ru.ru_inblock++;
718 } else if (async)
719 brelse(bp, 0);
720
721 if (vp->v_type == VBLK)
722 mp = spec_node_getmountedfs(vp);
723 else
724 mp = vp->v_mount;
725
726 /*
727 * Collect statistics on synchronous and asynchronous reads.
728 * Reads from block devices are charged to their associated
729 * filesystem (if any).
730 */
731 if (mp != NULL) {
732 if (async == 0)
733 mp->mnt_stat.f_syncreads++;
734 else
735 mp->mnt_stat.f_asyncreads++;
736 }
737
738 return bp;
739 }
740
741 /*
742 * Read a disk block.
743 * This algorithm described in Bach (p.54).
744 */
745 int
bread(struct vnode * vp,daddr_t blkno,int size,int flags,buf_t ** bpp)746 bread(struct vnode *vp, daddr_t blkno, int size, int flags, buf_t **bpp)
747 {
748 buf_t *bp;
749 int error;
750
751 BIOHIST_FUNC(__func__); BIOHIST_CALLED(biohist);
752
753 /* Get buffer for block. */
754 bp = *bpp = bio_doread(vp, blkno, size, 0);
755 if (bp == NULL)
756 return SET_ERROR(ENOMEM);
757
758 /* Wait for the read to complete, and return result. */
759 error = biowait(bp);
760 if (error == 0 && (flags & B_MODIFY) != 0)
761 error = fscow_run(bp, true);
762 if (error) {
763 brelse(bp, 0);
764 *bpp = NULL;
765 }
766
767 return error;
768 }
769
770 /*
771 * Read-ahead multiple disk blocks. The first is sync, the rest async.
772 * Trivial modification to the breada algorithm presented in Bach (p.55).
773 */
774 int
breadn(struct vnode * vp,daddr_t blkno,int size,daddr_t * rablks,int * rasizes,int nrablks,int flags,buf_t ** bpp)775 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks,
776 int *rasizes, int nrablks, int flags, buf_t **bpp)
777 {
778 buf_t *bp;
779 int error, i;
780
781 BIOHIST_FUNC(__func__); BIOHIST_CALLED(biohist);
782
783 bp = *bpp = bio_doread(vp, blkno, size, 0);
784 if (bp == NULL)
785 return SET_ERROR(ENOMEM);
786
787 /*
788 * For each of the read-ahead blocks, start a read, if necessary.
789 */
790 mutex_enter(&bufcache_lock);
791 for (i = 0; i < nrablks; i++) {
792 /* If it's in the cache, just go on to next one. */
793 if (incore(vp, rablks[i]))
794 continue;
795
796 /* Get a buffer for the read-ahead block */
797 mutex_exit(&bufcache_lock);
798 (void) bio_doread(vp, rablks[i], rasizes[i], B_ASYNC);
799 mutex_enter(&bufcache_lock);
800 }
801 mutex_exit(&bufcache_lock);
802
803 /* Otherwise, we had to start a read for it; wait until it's valid. */
804 error = biowait(bp);
805 if (error == 0 && (flags & B_MODIFY) != 0)
806 error = fscow_run(bp, true);
807 if (error) {
808 brelse(bp, 0);
809 *bpp = NULL;
810 }
811
812 return error;
813 }
814
815 /*
816 * Block write. Described in Bach (p.56)
817 */
818 int
bwrite(buf_t * bp)819 bwrite(buf_t *bp)
820 {
821 int rv, sync, wasdelayed;
822 struct vnode *vp;
823 struct mount *mp;
824
825 BIOHIST_FUNC(__func__); BIOHIST_CALLARGS(biohist, "bp=%#jx",
826 (uintptr_t)bp, 0, 0, 0);
827
828 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
829 KASSERT(!cv_has_waiters(&bp->b_done));
830
831 vp = bp->b_vp;
832
833 /*
834 * dholland 20160728 AFAICT vp==NULL must be impossible as it
835 * will crash upon reaching VOP_STRATEGY below... see further
836 * analysis on tech-kern.
837 */
838 KASSERTMSG(vp != NULL, "bwrite given buffer with null vnode");
839
840 if (vp != NULL) {
841 KASSERT(bp->b_objlock == vp->v_interlock);
842 if (vp->v_type == VBLK)
843 mp = spec_node_getmountedfs(vp);
844 else
845 mp = vp->v_mount;
846 } else {
847 mp = NULL;
848 }
849
850 if (mp && mp->mnt_wapbl) {
851 if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
852 bdwrite(bp);
853 return 0;
854 }
855 }
856
857 /*
858 * Remember buffer type, to switch on it later. If the write was
859 * synchronous, but the file system was mounted with MNT_ASYNC,
860 * convert it to a delayed write.
861 * XXX note that this relies on delayed tape writes being converted
862 * to async, not sync writes (which is safe, but ugly).
863 */
864 sync = !ISSET(bp->b_flags, B_ASYNC);
865 if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) {
866 bdwrite(bp);
867 return 0;
868 }
869
870 /*
871 * Collect statistics on synchronous and asynchronous writes.
872 * Writes to block devices are charged to their associated
873 * filesystem (if any).
874 */
875 if (mp != NULL) {
876 if (sync)
877 mp->mnt_stat.f_syncwrites++;
878 else
879 mp->mnt_stat.f_asyncwrites++;
880 }
881
882 /*
883 * Pay for the I/O operation and make sure the buf is on the correct
884 * vnode queue.
885 */
886 bp->b_error = 0;
887 wasdelayed = ISSET(bp->b_oflags, BO_DELWRI);
888 CLR(bp->b_flags, B_READ);
889 if (wasdelayed) {
890 mutex_enter(&bufcache_lock);
891 mutex_enter(bp->b_objlock);
892 CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
893 reassignbuf(bp, bp->b_vp);
894 /* Wake anyone trying to busy the buffer via vnode's lists. */
895 cv_broadcast(&bp->b_busy);
896 mutex_exit(&bufcache_lock);
897 } else {
898 curlwp->l_ru.ru_oublock++;
899 mutex_enter(bp->b_objlock);
900 CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
901 }
902 if (vp != NULL)
903 vp->v_numoutput++;
904 mutex_exit(bp->b_objlock);
905
906 /* Initiate disk write. */
907 if (sync)
908 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
909 else
910 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
911
912 VOP_STRATEGY(vp, bp);
913
914 if (sync) {
915 /* If I/O was synchronous, wait for it to complete. */
916 rv = biowait(bp);
917
918 /* Release the buffer. */
919 brelse(bp, 0);
920
921 return rv;
922 } else {
923 return 0;
924 }
925 }
926
927 int
vn_bwrite(void * v)928 vn_bwrite(void *v)
929 {
930 struct vop_bwrite_args *ap = v;
931
932 return bwrite(ap->a_bp);
933 }
934
935 /*
936 * Delayed write.
937 *
938 * The buffer is marked dirty, but is not queued for I/O.
939 * This routine should be used when the buffer is expected
940 * to be modified again soon, typically a small write that
941 * partially fills a buffer.
942 *
943 * NB: magnetic tapes cannot be delayed; they must be
944 * written in the order that the writes are requested.
945 *
946 * Described in Leffler, et al. (pp. 208-213).
947 */
948 void
bdwrite(buf_t * bp)949 bdwrite(buf_t *bp)
950 {
951
952 BIOHIST_FUNC(__func__); BIOHIST_CALLARGS(biohist, "bp=%#jx",
953 (uintptr_t)bp, 0, 0, 0);
954
955 KASSERT(bp->b_vp == NULL || bp->b_vp->v_tag != VT_UFS ||
956 bp->b_vp->v_type == VBLK || ISSET(bp->b_flags, B_COWDONE));
957 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
958 KASSERT(!cv_has_waiters(&bp->b_done));
959
960 /* If this is a tape block, write the block now. */
961 if (bdev_type(bp->b_dev) == D_TAPE) {
962 bawrite(bp);
963 return;
964 }
965
966 if (wapbl_vphaswapbl(bp->b_vp)) {
967 struct mount *mp = wapbl_vptomp(bp->b_vp);
968
969 if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
970 WAPBL_ADD_BUF(mp, bp);
971 }
972 }
973
974 /*
975 * If the block hasn't been seen before:
976 * (1) Mark it as having been seen,
977 * (2) Charge for the write,
978 * (3) Make sure it's on its vnode's correct block list.
979 */
980 KASSERT(bp->b_vp == NULL || bp->b_objlock == bp->b_vp->v_interlock);
981
982 if (!ISSET(bp->b_oflags, BO_DELWRI)) {
983 mutex_enter(&bufcache_lock);
984 mutex_enter(bp->b_objlock);
985 SET(bp->b_oflags, BO_DELWRI);
986 curlwp->l_ru.ru_oublock++;
987 reassignbuf(bp, bp->b_vp);
988 /* Wake anyone trying to busy the buffer via vnode's lists. */
989 cv_broadcast(&bp->b_busy);
990 mutex_exit(&bufcache_lock);
991 } else {
992 mutex_enter(bp->b_objlock);
993 }
994 /* Otherwise, the "write" is done, so mark and release the buffer. */
995 CLR(bp->b_oflags, BO_DONE);
996 mutex_exit(bp->b_objlock);
997
998 brelse(bp, 0);
999 }
1000
1001 /*
1002 * Asynchronous block write; just an asynchronous bwrite().
1003 */
1004 void
bawrite(buf_t * bp)1005 bawrite(buf_t *bp)
1006 {
1007
1008 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1009 KASSERT(bp->b_vp != NULL);
1010
1011 SET(bp->b_flags, B_ASYNC);
1012 VOP_BWRITE(bp->b_vp, bp);
1013 }
1014
1015 /*
1016 * Release a buffer on to the free lists.
1017 * Described in Bach (p. 46).
1018 */
1019 void
brelsel(buf_t * bp,int set)1020 brelsel(buf_t *bp, int set)
1021 {
1022 struct bqueue *bufq;
1023 struct vnode *vp;
1024
1025 SDT_PROBE2(io, kernel, , brelse, bp, set);
1026
1027 KASSERT(bp != NULL);
1028 KASSERT(mutex_owned(&bufcache_lock));
1029 KASSERT(!cv_has_waiters(&bp->b_done));
1030
1031 SET(bp->b_cflags, set);
1032
1033 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1034 KASSERT(bp->b_iodone == NULL);
1035
1036 /* Wake up any processes waiting for any buffer to become free. */
1037 cv_signal(&needbuffer_cv);
1038
1039 /* Wake up any proceeses waiting for _this_ buffer to become free */
1040 if (ISSET(bp->b_cflags, BC_WANTED))
1041 CLR(bp->b_cflags, BC_WANTED|BC_AGE);
1042
1043 /* If it's clean clear the copy-on-write flag. */
1044 if (ISSET(bp->b_flags, B_COWDONE)) {
1045 mutex_enter(bp->b_objlock);
1046 if (!ISSET(bp->b_oflags, BO_DELWRI))
1047 CLR(bp->b_flags, B_COWDONE);
1048 mutex_exit(bp->b_objlock);
1049 }
1050
1051 /*
1052 * Determine which queue the buffer should be on, then put it there.
1053 */
1054
1055 /* If it's locked, don't report an error; try again later. */
1056 if (ISSET(bp->b_flags, B_LOCKED))
1057 bp->b_error = 0;
1058
1059 /* If it's not cacheable, or an error, mark it invalid. */
1060 if (ISSET(bp->b_cflags, BC_NOCACHE) || bp->b_error != 0)
1061 SET(bp->b_cflags, BC_INVAL);
1062
1063 if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1064 /*
1065 * This is a delayed write buffer that was just flushed to
1066 * disk. It is still on the LRU queue. If it's become
1067 * invalid, then we need to move it to a different queue;
1068 * otherwise leave it in its current position.
1069 */
1070 CLR(bp->b_cflags, BC_VFLUSH);
1071 if (!ISSET(bp->b_cflags, BC_INVAL|BC_AGE) &&
1072 !ISSET(bp->b_flags, B_LOCKED) && bp->b_error == 0) {
1073 KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 1));
1074 goto already_queued;
1075 } else {
1076 bremfree(bp);
1077 }
1078 }
1079
1080 KDASSERT(checkfreelist(bp, &bufqueues[BQ_AGE], 0));
1081 KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 0));
1082 KDASSERT(checkfreelist(bp, &bufqueues[BQ_LOCKED], 0));
1083
1084 if ((bp->b_bufsize <= 0) || ISSET(bp->b_cflags, BC_INVAL)) {
1085 /*
1086 * If it's invalid or empty, dissociate it from its vnode
1087 * and put on the head of the appropriate queue.
1088 */
1089 if (ISSET(bp->b_flags, B_LOCKED)) {
1090 if (wapbl_vphaswapbl(vp = bp->b_vp)) {
1091 struct mount *mp = wapbl_vptomp(vp);
1092
1093 KASSERT(bp->b_iodone !=
1094 mp->mnt_wapbl_op->wo_wapbl_biodone);
1095 WAPBL_REMOVE_BUF(mp, bp);
1096 }
1097 }
1098
1099 mutex_enter(bp->b_objlock);
1100 CLR(bp->b_oflags, BO_DONE|BO_DELWRI);
1101 if ((vp = bp->b_vp) != NULL) {
1102 KASSERT(bp->b_objlock == vp->v_interlock);
1103 reassignbuf(bp, bp->b_vp);
1104 brelvp(bp);
1105 mutex_exit(vp->v_interlock);
1106 } else {
1107 KASSERT(bp->b_objlock == &buffer_lock);
1108 mutex_exit(bp->b_objlock);
1109 }
1110 /* We want to dispose of the buffer, so wake everybody. */
1111 cv_broadcast(&bp->b_busy);
1112 if (bp->b_bufsize <= 0)
1113 /* no data */
1114 goto already_queued;
1115 else
1116 /* invalid data */
1117 bufq = &bufqueues[BQ_AGE];
1118 binsheadfree(bp, bufq);
1119 } else {
1120 /*
1121 * It has valid data. Put it on the end of the appropriate
1122 * queue, so that it'll stick around for as long as possible.
1123 * If buf is AGE, but has dependencies, must put it on last
1124 * bufqueue to be scanned, ie LRU. This protects against the
1125 * livelock where BQ_AGE only has buffers with dependencies,
1126 * and we thus never get to the dependent buffers in BQ_LRU.
1127 */
1128 if (ISSET(bp->b_flags, B_LOCKED)) {
1129 /* locked in core */
1130 bufq = &bufqueues[BQ_LOCKED];
1131 } else if (!ISSET(bp->b_cflags, BC_AGE)) {
1132 /* valid data */
1133 bufq = &bufqueues[BQ_LRU];
1134 } else {
1135 /* stale but valid data */
1136 bufq = &bufqueues[BQ_AGE];
1137 }
1138 binstailfree(bp, bufq);
1139 }
1140 already_queued:
1141 /* Unlock the buffer. */
1142 CLR(bp->b_cflags, BC_AGE|BC_BUSY|BC_NOCACHE);
1143 CLR(bp->b_flags, B_ASYNC);
1144
1145 /*
1146 * Wake only the highest priority waiter on the lock, in order to
1147 * prevent a thundering herd: many LWPs simultaneously awakening and
1148 * competing for the buffer's lock. Testing in 2019 revealed this
1149 * to reduce contention on bufcache_lock tenfold during a kernel
1150 * compile. Here and elsewhere, when the buffer is changing
1151 * identity, being disposed of, or moving from one list to another,
1152 * we wake all lock requestors.
1153 */
1154 if (bp->b_bufsize <= 0) {
1155 cv_broadcast(&bp->b_busy);
1156 buf_destroy(bp);
1157 #ifdef DEBUG
1158 memset((char *)bp, 0, sizeof(*bp));
1159 #endif
1160 pool_cache_put(buf_cache, bp);
1161 } else
1162 cv_signal(&bp->b_busy);
1163 }
1164
1165 void
brelse(buf_t * bp,int set)1166 brelse(buf_t *bp, int set)
1167 {
1168
1169 mutex_enter(&bufcache_lock);
1170 brelsel(bp, set);
1171 mutex_exit(&bufcache_lock);
1172 }
1173
1174 /*
1175 * Determine if a block is in the cache.
1176 * Just look on what would be its hash chain. If it's there, return
1177 * a pointer to it, unless it's marked invalid. If it's marked invalid,
1178 * we normally don't return the buffer, unless the caller explicitly
1179 * wants us to.
1180 */
1181 buf_t *
incore(struct vnode * vp,daddr_t blkno)1182 incore(struct vnode *vp, daddr_t blkno)
1183 {
1184 buf_t *bp;
1185
1186 KASSERT(mutex_owned(&bufcache_lock));
1187
1188 /* Search hash chain */
1189 LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) {
1190 if (bp->b_lblkno == blkno && bp->b_vp == vp &&
1191 !ISSET(bp->b_cflags, BC_INVAL)) {
1192 KASSERT(bp->b_objlock == vp->v_interlock);
1193 return (bp);
1194 }
1195 }
1196
1197 return NULL;
1198 }
1199
1200 /*
1201 * Get a block of requested size that is associated with
1202 * a given vnode and block offset. If it is found in the
1203 * block cache, mark it as having been found, make it busy
1204 * and return it. Otherwise, return an empty block of the
1205 * correct size. It is up to the caller to insure that the
1206 * cached blocks be of the correct size.
1207 */
1208 buf_t *
getblk(struct vnode * vp,daddr_t blkno,int size,int slpflag,int slptimeo)1209 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo)
1210 {
1211 int err, preserve;
1212 buf_t *bp;
1213
1214 mutex_enter(&bufcache_lock);
1215 SDT_PROBE3(io, kernel, , getblk__start, vp, blkno, size);
1216 loop:
1217 bp = incore(vp, blkno);
1218 if (bp != NULL) {
1219 err = bbusy(bp, ((slpflag & PCATCH) != 0), slptimeo, NULL);
1220 if (err != 0) {
1221 if (err == EPASSTHROUGH)
1222 goto loop;
1223 mutex_exit(&bufcache_lock);
1224 SDT_PROBE4(io, kernel, , getblk__done,
1225 vp, blkno, size, NULL);
1226 return NULL;
1227 }
1228 KASSERT(!cv_has_waiters(&bp->b_done));
1229 #ifdef DIAGNOSTIC
1230 if (ISSET(bp->b_oflags, BO_DONE|BO_DELWRI) &&
1231 bp->b_bcount < size && vp->v_type != VBLK)
1232 panic("getblk: block size invariant failed");
1233 #endif
1234 bremfree(bp);
1235 preserve = 1;
1236 } else {
1237 if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL)
1238 goto loop;
1239
1240 if (incore(vp, blkno) != NULL) {
1241 /* The block has come into memory in the meantime. */
1242 brelsel(bp, 0);
1243 goto loop;
1244 }
1245
1246 LIST_INSERT_HEAD(BUFHASH(vp, blkno), bp, b_hash);
1247 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno;
1248 mutex_enter(vp->v_interlock);
1249 bgetvp(vp, bp);
1250 mutex_exit(vp->v_interlock);
1251 preserve = 0;
1252 }
1253 mutex_exit(&bufcache_lock);
1254
1255 /*
1256 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes)
1257 * if we re-size buffers here.
1258 */
1259 if (ISSET(bp->b_flags, B_LOCKED)) {
1260 KASSERT(bp->b_bufsize >= size);
1261 } else {
1262 if (allocbuf(bp, size, preserve)) {
1263 mutex_enter(&bufcache_lock);
1264 LIST_REMOVE(bp, b_hash);
1265 brelsel(bp, BC_INVAL);
1266 mutex_exit(&bufcache_lock);
1267 SDT_PROBE4(io, kernel, , getblk__done,
1268 vp, blkno, size, NULL);
1269 return NULL;
1270 }
1271 }
1272 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1273 SDT_PROBE4(io, kernel, , getblk__done, vp, blkno, size, bp);
1274 return bp;
1275 }
1276
1277 /*
1278 * Get an empty, disassociated buffer of given size.
1279 */
1280 buf_t *
geteblk(int size)1281 geteblk(int size)
1282 {
1283 buf_t *bp;
1284 int error __diagused;
1285
1286 mutex_enter(&bufcache_lock);
1287 while ((bp = getnewbuf(0, 0, 0)) == NULL)
1288 continue;
1289
1290 SET(bp->b_cflags, BC_INVAL);
1291 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1292 mutex_exit(&bufcache_lock);
1293 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1294 error = allocbuf(bp, size, 0);
1295 KASSERT(error == 0);
1296 return bp;
1297 }
1298
1299 /*
1300 * Expand or contract the actual memory allocated to a buffer.
1301 *
1302 * If the buffer shrinks, data is lost, so it's up to the
1303 * caller to have written it out *first*; this routine will not
1304 * start a write. If the buffer grows, it's the callers
1305 * responsibility to fill out the buffer's additional contents.
1306 */
1307 int
allocbuf(buf_t * bp,int size,int preserve)1308 allocbuf(buf_t *bp, int size, int preserve)
1309 {
1310 void *addr;
1311 vsize_t oldsize, desired_size;
1312 int oldcount;
1313 int delta;
1314
1315 desired_size = buf_roundsize(size);
1316 if (desired_size > MAXBSIZE)
1317 printf("allocbuf: buffer larger than MAXBSIZE requested");
1318
1319 oldcount = bp->b_bcount;
1320
1321 bp->b_bcount = size;
1322
1323 oldsize = bp->b_bufsize;
1324 if (oldsize == desired_size) {
1325 /*
1326 * Do not short cut the WAPBL resize, as the buffer length
1327 * could still have changed and this would corrupt the
1328 * tracking of the transaction length.
1329 */
1330 goto out;
1331 }
1332
1333 /*
1334 * If we want a buffer of a different size, re-allocate the
1335 * buffer's memory; copy old content only if needed.
1336 */
1337 addr = buf_alloc(desired_size);
1338 if (addr == NULL)
1339 return SET_ERROR(ENOMEM);
1340 if (preserve)
1341 memcpy(addr, bp->b_data, MIN(oldsize,desired_size));
1342 if (bp->b_data != NULL)
1343 buf_mrelease(bp->b_data, oldsize);
1344 bp->b_data = addr;
1345 bp->b_bufsize = desired_size;
1346
1347 /*
1348 * Update overall buffer memory counter (protected by bufcache_lock)
1349 */
1350 delta = (long)desired_size - (long)oldsize;
1351
1352 mutex_enter(&bufcache_lock);
1353 if ((bufmem += delta) > bufmem_hiwater) {
1354 /*
1355 * Need to trim overall memory usage.
1356 */
1357 while (buf_canrelease()) {
1358 if (preempt_needed()) {
1359 mutex_exit(&bufcache_lock);
1360 preempt();
1361 mutex_enter(&bufcache_lock);
1362 }
1363 if (buf_trim() == 0)
1364 break;
1365 }
1366 }
1367 mutex_exit(&bufcache_lock);
1368
1369 out:
1370 if (wapbl_vphaswapbl(bp->b_vp)) {
1371 WAPBL_RESIZE_BUF(wapbl_vptomp(bp->b_vp), bp,
1372 oldsize, oldcount);
1373 }
1374
1375 return 0;
1376 }
1377
1378 /*
1379 * Find a buffer which is available for use.
1380 * Select something from a free list.
1381 * Preference is to AGE list, then LRU list.
1382 *
1383 * Called with the buffer queues locked.
1384 * Return buffer locked.
1385 */
1386 static buf_t *
getnewbuf(int slpflag,int slptimeo,int from_bufq)1387 getnewbuf(int slpflag, int slptimeo, int from_bufq)
1388 {
1389 buf_t *bp;
1390 struct vnode *vp;
1391 struct mount *transmp = NULL;
1392
1393 SDT_PROBE0(io, kernel, , getnewbuf__start);
1394
1395 start:
1396 KASSERT(mutex_owned(&bufcache_lock));
1397
1398 /*
1399 * Get a new buffer from the pool.
1400 */
1401 if (!from_bufq && buf_lotsfree()) {
1402 mutex_exit(&bufcache_lock);
1403 bp = pool_cache_get(buf_cache, PR_NOWAIT);
1404 if (bp != NULL) {
1405 memset((char *)bp, 0, sizeof(*bp));
1406 buf_init(bp);
1407 SET(bp->b_cflags, BC_BUSY); /* mark buffer busy */
1408 mutex_enter(&bufcache_lock);
1409 #if defined(DIAGNOSTIC)
1410 bp->b_freelistindex = -1;
1411 #endif /* defined(DIAGNOSTIC) */
1412 SDT_PROBE1(io, kernel, , getnewbuf__done, bp);
1413 return bp;
1414 }
1415 mutex_enter(&bufcache_lock);
1416 }
1417
1418 KASSERT(mutex_owned(&bufcache_lock));
1419 if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue)) != NULL) {
1420 KASSERT(!ISSET(bp->b_oflags, BO_DELWRI));
1421 } else {
1422 TAILQ_FOREACH(bp, &bufqueues[BQ_LRU].bq_queue, b_freelist) {
1423 if (ISSET(bp->b_cflags, BC_VFLUSH) ||
1424 !ISSET(bp->b_oflags, BO_DELWRI))
1425 break;
1426 if (fstrans_start_nowait(bp->b_vp->v_mount) == 0) {
1427 KASSERT(transmp == NULL);
1428 transmp = bp->b_vp->v_mount;
1429 break;
1430 }
1431 }
1432 }
1433 if (bp != NULL) {
1434 KASSERT(!ISSET(bp->b_cflags, BC_BUSY) ||
1435 ISSET(bp->b_cflags, BC_VFLUSH));
1436 bremfree(bp);
1437
1438 /* Buffer is no longer on free lists. */
1439 SET(bp->b_cflags, BC_BUSY);
1440
1441 /* Wake anyone trying to lock the old identity. */
1442 cv_broadcast(&bp->b_busy);
1443 } else {
1444 /*
1445 * XXX: !from_bufq should be removed.
1446 */
1447 if (!from_bufq || curlwp != uvm.pagedaemon_lwp) {
1448 /* wait for a free buffer of any kind */
1449 if ((slpflag & PCATCH) != 0)
1450 (void)cv_timedwait_sig(&needbuffer_cv,
1451 &bufcache_lock, slptimeo);
1452 else
1453 (void)cv_timedwait(&needbuffer_cv,
1454 &bufcache_lock, slptimeo);
1455 }
1456 SDT_PROBE1(io, kernel, , getnewbuf__done, NULL);
1457 return NULL;
1458 }
1459
1460 #ifdef DIAGNOSTIC
1461 if (bp->b_bufsize <= 0)
1462 panic("buffer %p: on queue but empty", bp);
1463 #endif
1464
1465 if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1466 /*
1467 * This is a delayed write buffer being flushed to disk. Make
1468 * sure it gets aged out of the queue when it's finished, and
1469 * leave it off the LRU queue.
1470 */
1471 CLR(bp->b_cflags, BC_VFLUSH);
1472 SET(bp->b_cflags, BC_AGE);
1473 goto start;
1474 }
1475
1476 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1477 KASSERT(!cv_has_waiters(&bp->b_done));
1478
1479 /*
1480 * If buffer was a delayed write, start it and return NULL
1481 * (since we might sleep while starting the write).
1482 */
1483 if (ISSET(bp->b_oflags, BO_DELWRI)) {
1484 /*
1485 * This buffer has gone through the LRU, so make sure it gets
1486 * reused ASAP.
1487 */
1488 SET(bp->b_cflags, BC_AGE);
1489 mutex_exit(&bufcache_lock);
1490 bawrite(bp);
1491 KASSERT(transmp != NULL);
1492 fstrans_done(transmp);
1493 mutex_enter(&bufcache_lock);
1494 SDT_PROBE1(io, kernel, , getnewbuf__done, NULL);
1495 return NULL;
1496 }
1497
1498 KASSERT(transmp == NULL);
1499
1500 vp = bp->b_vp;
1501
1502 /* clear out various other fields */
1503 bp->b_cflags = BC_BUSY;
1504 bp->b_oflags = 0;
1505 bp->b_flags = 0;
1506 bp->b_dev = NODEV;
1507 bp->b_blkno = 0;
1508 bp->b_lblkno = 0;
1509 bp->b_rawblkno = 0;
1510 bp->b_iodone = 0;
1511 bp->b_error = 0;
1512 bp->b_resid = 0;
1513 bp->b_bcount = 0;
1514
1515 LIST_REMOVE(bp, b_hash);
1516
1517 /* Disassociate us from our vnode, if we had one... */
1518 if (vp != NULL) {
1519 mutex_enter(vp->v_interlock);
1520 brelvp(bp);
1521 mutex_exit(vp->v_interlock);
1522 }
1523
1524 SDT_PROBE1(io, kernel, , getnewbuf__done, bp);
1525 return bp;
1526 }
1527
1528 /*
1529 * Invalidate the specified buffer if it exists.
1530 */
1531 void
binvalbuf(struct vnode * vp,daddr_t blkno)1532 binvalbuf(struct vnode *vp, daddr_t blkno)
1533 {
1534 buf_t *bp;
1535 int err;
1536
1537 mutex_enter(&bufcache_lock);
1538
1539 loop:
1540 bp = incore(vp, blkno);
1541 if (bp != NULL) {
1542 err = bbusy(bp, 0, 0, NULL);
1543 if (err == EPASSTHROUGH)
1544 goto loop;
1545 bremfree(bp);
1546 if (ISSET(bp->b_oflags, BO_DELWRI)) {
1547 SET(bp->b_cflags, BC_NOCACHE);
1548 mutex_exit(&bufcache_lock);
1549 bwrite(bp);
1550 } else {
1551 brelsel(bp, BC_INVAL);
1552 mutex_exit(&bufcache_lock);
1553 }
1554 } else
1555 mutex_exit(&bufcache_lock);
1556 }
1557
1558 /*
1559 * Attempt to free an aged buffer off the queues.
1560 * Called with queue lock held.
1561 * Returns the amount of buffer memory freed.
1562 */
1563 static int
buf_trim(void)1564 buf_trim(void)
1565 {
1566 buf_t *bp;
1567 long size;
1568
1569 KASSERT(mutex_owned(&bufcache_lock));
1570
1571 /* Instruct getnewbuf() to get buffers off the queues */
1572 if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL)
1573 return 0;
1574
1575 KASSERT((bp->b_cflags & BC_WANTED) == 0);
1576 size = bp->b_bufsize;
1577 bufmem -= size;
1578 if (size > 0) {
1579 buf_mrelease(bp->b_data, size);
1580 bp->b_bcount = bp->b_bufsize = 0;
1581 }
1582 /* brelse() will return the buffer to the global buffer pool */
1583 brelsel(bp, 0);
1584 return size;
1585 }
1586
1587 int
buf_drain(int n)1588 buf_drain(int n)
1589 {
1590 int size = 0, sz;
1591
1592 KASSERT(mutex_owned(&bufcache_lock));
1593
1594 while (size < n && bufmem > bufmem_lowater) {
1595 sz = buf_trim();
1596 if (sz <= 0)
1597 break;
1598 size += sz;
1599 }
1600
1601 return size;
1602 }
1603
1604 /*
1605 * Wait for operations on the buffer to complete.
1606 * When they do, extract and return the I/O's error value.
1607 */
1608 int
biowait(buf_t * bp)1609 biowait(buf_t *bp)
1610 {
1611
1612 BIOHIST_FUNC(__func__);
1613
1614 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1615
1616 SDT_PROBE1(io, kernel, , wait__start, bp);
1617
1618 mutex_enter(bp->b_objlock);
1619
1620 BIOHIST_CALLARGS(biohist, "bp=%#jx, oflags=0x%jx, ret_addr=%#jx",
1621 (uintptr_t)bp, bp->b_oflags,
1622 (uintptr_t)__builtin_return_address(0), 0);
1623
1624 while (!ISSET(bp->b_oflags, BO_DONE | BO_DELWRI)) {
1625 BIOHIST_LOG(biohist, "waiting bp=%#jx",
1626 (uintptr_t)bp, 0, 0, 0);
1627 cv_wait(&bp->b_done, bp->b_objlock);
1628 }
1629 mutex_exit(bp->b_objlock);
1630
1631 SDT_PROBE1(io, kernel, , wait__done, bp);
1632
1633 BIOHIST_LOG(biohist, "return %jd", bp->b_error, 0, 0, 0);
1634
1635 return bp->b_error;
1636 }
1637
1638 /*
1639 * Mark I/O complete on a buffer.
1640 *
1641 * If a callback has been requested, e.g. the pageout
1642 * daemon, do so. Otherwise, awaken waiting processes.
1643 *
1644 * [ Leffler, et al., says on p.247:
1645 * "This routine wakes up the blocked process, frees the buffer
1646 * for an asynchronous write, or, for a request by the pagedaemon
1647 * process, invokes a procedure specified in the buffer structure" ]
1648 *
1649 * In real life, the pagedaemon (or other system processes) wants
1650 * to do async stuff too, and doesn't want the buffer brelse()'d.
1651 * (for swap pager, that puts swap buffers on the free lists (!!!),
1652 * for the vn device, that puts allocated buffers on the free lists!)
1653 */
1654 void
biodone(buf_t * bp)1655 biodone(buf_t *bp)
1656 {
1657 int s;
1658
1659 BIOHIST_FUNC(__func__);
1660
1661 KASSERT(!ISSET(bp->b_oflags, BO_DONE));
1662
1663 if (cpu_intr_p()) {
1664 /* From interrupt mode: defer to a soft interrupt. */
1665 s = splvm();
1666 TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_biodone, bp, b_actq);
1667
1668 BIOHIST_CALLARGS(biohist, "bp=%#jx, softint scheduled",
1669 (uintptr_t)bp, 0, 0, 0);
1670 softint_schedule(biodone_sih);
1671 splx(s);
1672 } else {
1673 /* Process now - the buffer may be freed soon. */
1674 biodone2(bp);
1675 }
1676 }
1677
1678 SDT_PROBE_DEFINE1(io, kernel, , done, "struct buf *"/*bp*/);
1679
1680 static void
biodone2(buf_t * bp)1681 biodone2(buf_t *bp)
1682 {
1683 void (*callout)(buf_t *);
1684
1685 SDT_PROBE1(io, kernel, ,done, bp);
1686
1687 BIOHIST_FUNC(__func__);
1688 BIOHIST_CALLARGS(biohist, "bp=%#jx", (uintptr_t)bp, 0, 0, 0);
1689
1690 mutex_enter(bp->b_objlock);
1691 /* Note that the transfer is done. */
1692 if (ISSET(bp->b_oflags, BO_DONE))
1693 panic("biodone2 already");
1694 CLR(bp->b_flags, B_COWDONE);
1695 SET(bp->b_oflags, BO_DONE);
1696 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1697
1698 /* Wake up waiting writers. */
1699 if (!ISSET(bp->b_flags, B_READ))
1700 vwakeup(bp);
1701
1702 if ((callout = bp->b_iodone) != NULL) {
1703 BIOHIST_LOG(biohist, "callout %#jx", (uintptr_t)callout,
1704 0, 0, 0);
1705
1706 /* Note callout done, then call out. */
1707 KASSERT(!cv_has_waiters(&bp->b_done));
1708 bp->b_iodone = NULL;
1709 mutex_exit(bp->b_objlock);
1710 (*callout)(bp);
1711 } else if (ISSET(bp->b_flags, B_ASYNC)) {
1712 /* If async, release. */
1713 BIOHIST_LOG(biohist, "async", 0, 0, 0, 0);
1714 KASSERT(!cv_has_waiters(&bp->b_done));
1715 mutex_exit(bp->b_objlock);
1716 brelse(bp, 0);
1717 } else {
1718 /* Otherwise just wake up waiters in biowait(). */
1719 BIOHIST_LOG(biohist, "wake-up", 0, 0, 0, 0);
1720 cv_broadcast(&bp->b_done);
1721 mutex_exit(bp->b_objlock);
1722 }
1723 }
1724
1725 static void
biointr(void * cookie)1726 biointr(void *cookie)
1727 {
1728 struct cpu_info *ci;
1729 buf_t *bp;
1730 int s;
1731
1732 BIOHIST_FUNC(__func__); BIOHIST_CALLED(biohist);
1733
1734 ci = curcpu();
1735
1736 s = splvm();
1737 while (!TAILQ_EMPTY(&ci->ci_data.cpu_biodone)) {
1738 KASSERT(curcpu() == ci);
1739
1740 bp = TAILQ_FIRST(&ci->ci_data.cpu_biodone);
1741 TAILQ_REMOVE(&ci->ci_data.cpu_biodone, bp, b_actq);
1742 splx(s);
1743
1744 BIOHIST_LOG(biohist, "bp=%#jx", (uintptr_t)bp, 0, 0, 0);
1745 biodone2(bp);
1746
1747 s = splvm();
1748 }
1749 splx(s);
1750 }
1751
1752 static void
sysctl_fillbuf(const buf_t * i,struct buf_sysctl * o)1753 sysctl_fillbuf(const buf_t *i, struct buf_sysctl *o)
1754 {
1755 const bool allowaddr = get_expose_address(curproc);
1756
1757 memset(o, 0, sizeof(*o));
1758
1759 o->b_flags = i->b_flags | i->b_cflags | i->b_oflags;
1760 o->b_error = i->b_error;
1761 o->b_prio = i->b_prio;
1762 o->b_dev = i->b_dev;
1763 o->b_bufsize = i->b_bufsize;
1764 o->b_bcount = i->b_bcount;
1765 o->b_resid = i->b_resid;
1766 COND_SET_VALUE(o->b_addr, PTRTOUINT64(i->b_data), allowaddr);
1767 o->b_blkno = i->b_blkno;
1768 o->b_rawblkno = i->b_rawblkno;
1769 COND_SET_VALUE(o->b_iodone, PTRTOUINT64(i->b_iodone), allowaddr);
1770 COND_SET_VALUE(o->b_proc, PTRTOUINT64(i->b_proc), allowaddr);
1771 COND_SET_VALUE(o->b_vp, PTRTOUINT64(i->b_vp), allowaddr);
1772 COND_SET_VALUE(o->b_saveaddr, PTRTOUINT64(i->b_saveaddr), allowaddr);
1773 o->b_lblkno = i->b_lblkno;
1774 }
1775
1776 static int
sysctl_dobuf(SYSCTLFN_ARGS)1777 sysctl_dobuf(SYSCTLFN_ARGS)
1778 {
1779 buf_t *bp;
1780 struct buf_sysctl bs;
1781 struct bqueue *bq;
1782 char *dp;
1783 u_int i, op, arg;
1784 size_t len, needed, elem_size, out_size;
1785 int error, elem_count, retries;
1786
1787 if (namelen == 1 && name[0] == CTL_QUERY)
1788 return sysctl_query(SYSCTLFN_CALL(rnode));
1789
1790 if (namelen != 4)
1791 return SET_ERROR(EINVAL);
1792
1793 retries = 100;
1794 retry:
1795 dp = oldp;
1796 len = (oldp != NULL) ? *oldlenp : 0;
1797 op = name[0];
1798 arg = name[1];
1799 elem_size = name[2];
1800 elem_count = name[3];
1801 out_size = MIN(sizeof(bs), elem_size);
1802
1803 /*
1804 * at the moment, these are just "placeholders" to make the
1805 * API for retrieving kern.buf data more extensible in the
1806 * future.
1807 *
1808 * XXX kern.buf currently has "netbsd32" issues. hopefully
1809 * these will be resolved at a later point.
1810 */
1811 if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL ||
1812 elem_size < 1 || elem_count < 0)
1813 return SET_ERROR(EINVAL);
1814
1815 if (oldp == NULL) {
1816 /* count only, don't run through the buffer queues */
1817 needed = pool_cache_nget(buf_cache) -
1818 pool_cache_nput(buf_cache);
1819 *oldlenp = (needed + KERN_BUFSLOP) * elem_size;
1820
1821 return 0;
1822 }
1823
1824 error = 0;
1825 needed = 0;
1826 sysctl_unlock();
1827 mutex_enter(&bufcache_lock);
1828 for (i = 0; i < BQUEUES; i++) {
1829 bq = &bufqueues[i];
1830 TAILQ_FOREACH(bp, &bq->bq_queue, b_freelist) {
1831 bq->bq_marker = bp;
1832 if (len >= elem_size && elem_count > 0) {
1833 sysctl_fillbuf(bp, &bs);
1834 mutex_exit(&bufcache_lock);
1835 error = copyout(&bs, dp, out_size);
1836 mutex_enter(&bufcache_lock);
1837 if (error)
1838 break;
1839 if (bq->bq_marker != bp) {
1840 /*
1841 * This sysctl node is only for
1842 * statistics. Retry; if the
1843 * queue keeps changing, then
1844 * bail out.
1845 */
1846 if (retries-- == 0) {
1847 error = SET_ERROR(EAGAIN);
1848 break;
1849 }
1850 mutex_exit(&bufcache_lock);
1851 sysctl_relock();
1852 goto retry;
1853 }
1854 dp += elem_size;
1855 len -= elem_size;
1856 }
1857 needed += elem_size;
1858 if (elem_count > 0 && elem_count != INT_MAX)
1859 elem_count--;
1860 }
1861 if (error != 0)
1862 break;
1863 }
1864 mutex_exit(&bufcache_lock);
1865 sysctl_relock();
1866
1867 *oldlenp = needed;
1868
1869 return error;
1870 }
1871
1872 static int
sysctl_bufvm_update(SYSCTLFN_ARGS)1873 sysctl_bufvm_update(SYSCTLFN_ARGS)
1874 {
1875 int error, rv;
1876 struct sysctlnode node;
1877 unsigned int temp_bufcache;
1878 unsigned long temp_water;
1879
1880 /* Take a copy of the supplied node and its data */
1881 node = *rnode;
1882 if (node.sysctl_data == &bufcache) {
1883 node.sysctl_data = &temp_bufcache;
1884 temp_bufcache = *(unsigned int *)rnode->sysctl_data;
1885 } else {
1886 node.sysctl_data = &temp_water;
1887 temp_water = *(unsigned long *)rnode->sysctl_data;
1888 }
1889
1890 /* Update the copy */
1891 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1892 if (error || newp == NULL)
1893 return error;
1894
1895 if (rnode->sysctl_data == &bufcache) {
1896 if (temp_bufcache > 100)
1897 return SET_ERROR(EINVAL);
1898 bufcache = temp_bufcache;
1899 buf_setwm();
1900 } else if (rnode->sysctl_data == &bufmem_lowater) {
1901 if (bufmem_hiwater - temp_water < 16)
1902 return SET_ERROR(EINVAL);
1903 bufmem_lowater = temp_water;
1904 } else if (rnode->sysctl_data == &bufmem_hiwater) {
1905 if (temp_water - bufmem_lowater < 16)
1906 return SET_ERROR(EINVAL);
1907 bufmem_hiwater = temp_water;
1908 } else
1909 return SET_ERROR(EINVAL);
1910
1911 /* Drain until below new high water mark */
1912 sysctl_unlock();
1913 mutex_enter(&bufcache_lock);
1914 while (bufmem > bufmem_hiwater) {
1915 rv = buf_drain((bufmem - bufmem_hiwater) / (2 * 1024));
1916 if (rv <= 0)
1917 break;
1918 }
1919 mutex_exit(&bufcache_lock);
1920 sysctl_relock();
1921
1922 return 0;
1923 }
1924
1925 static struct sysctllog *vfsbio_sysctllog;
1926
1927 static void
sysctl_kern_buf_setup(void)1928 sysctl_kern_buf_setup(void)
1929 {
1930
1931 sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1932 CTLFLAG_PERMANENT,
1933 CTLTYPE_NODE, "buf",
1934 SYSCTL_DESCR("Kernel buffer cache information"),
1935 sysctl_dobuf, 0, NULL, 0,
1936 CTL_KERN, KERN_BUF, CTL_EOL);
1937 }
1938
1939 static void
sysctl_vm_buf_setup(void)1940 sysctl_vm_buf_setup(void)
1941 {
1942
1943 sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1944 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1945 CTLTYPE_INT, "bufcache",
1946 SYSCTL_DESCR("Percentage of physical memory to use for "
1947 "buffer cache"),
1948 sysctl_bufvm_update, 0, &bufcache, 0,
1949 CTL_VM, CTL_CREATE, CTL_EOL);
1950 sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1951 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
1952 CTLTYPE_LONG, "bufmem",
1953 SYSCTL_DESCR("Amount of kernel memory used by buffer cache"),
1954 NULL, 0, &bufmem, 0,
1955 CTL_VM, CTL_CREATE, CTL_EOL);
1956 sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1957 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1958 CTLTYPE_LONG, "bufmem_lowater",
1959 SYSCTL_DESCR("Minimum amount of kernel memory to reserve for "
1960 "buffer cache"),
1961 sysctl_bufvm_update, 0, &bufmem_lowater, 0,
1962 CTL_VM, CTL_CREATE, CTL_EOL);
1963 sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1964 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1965 CTLTYPE_LONG, "bufmem_hiwater",
1966 SYSCTL_DESCR("Maximum amount of kernel memory to use for "
1967 "buffer cache"),
1968 sysctl_bufvm_update, 0, &bufmem_hiwater, 0,
1969 CTL_VM, CTL_CREATE, CTL_EOL);
1970 }
1971
1972 static int
bufhash_stats(struct hashstat_sysctl * hs,bool fill)1973 bufhash_stats(struct hashstat_sysctl *hs, bool fill)
1974 {
1975 buf_t *bp;
1976 uint64_t chain;
1977
1978 strlcpy(hs->hash_name, "bufhash", sizeof(hs->hash_name));
1979 strlcpy(hs->hash_desc, "buffer hash", sizeof(hs->hash_desc));
1980 if (!fill)
1981 return 0;
1982
1983 hs->hash_size = bufhash + 1;
1984
1985 for (size_t i = 0; i < hs->hash_size; i++) {
1986 chain = 0;
1987
1988 mutex_enter(&bufcache_lock);
1989 LIST_FOREACH(bp, &bufhashtbl[i], b_hash) {
1990 chain++;
1991 }
1992 mutex_exit(&bufcache_lock);
1993
1994 if (chain > 0) {
1995 hs->hash_used++;
1996 hs->hash_items += chain;
1997 if (chain > hs->hash_maxchain)
1998 hs->hash_maxchain = chain;
1999 }
2000 preempt_point();
2001 }
2002
2003 return 0;
2004 }
2005
2006 #ifdef DEBUG
2007 /*
2008 * Print out statistics on the current allocation of the buffer pool.
2009 * Can be enabled to print out on every ``sync'' by setting "syncprt"
2010 * in vfs_syscalls.c using sysctl.
2011 */
2012 void
vfs_bufstats(void)2013 vfs_bufstats(void)
2014 {
2015 int i, j, count;
2016 buf_t *bp;
2017 struct bqueue *dp;
2018 int counts[MAXBSIZE / MIN_PAGE_SIZE + 1];
2019 static const char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" };
2020
2021 for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
2022 count = 0;
2023 memset(counts, 0, sizeof(counts));
2024 TAILQ_FOREACH(bp, &dp->bq_queue, b_freelist) {
2025 counts[bp->b_bufsize / PAGE_SIZE]++;
2026 count++;
2027 }
2028 printf("%s: total-%d", bname[i], count);
2029 for (j = 0; j <= MAXBSIZE / PAGE_SIZE; j++)
2030 if (counts[j] != 0)
2031 printf(", %d-%d", j * PAGE_SIZE, counts[j]);
2032 printf("\n");
2033 }
2034 }
2035 #endif /* DEBUG */
2036
2037 /* ------------------------------ */
2038
2039 buf_t *
getiobuf(struct vnode * vp,bool waitok)2040 getiobuf(struct vnode *vp, bool waitok)
2041 {
2042 buf_t *bp;
2043
2044 bp = pool_cache_get(bufio_cache, (waitok ? PR_WAITOK : PR_NOWAIT));
2045 if (bp == NULL)
2046 return bp;
2047
2048 buf_init(bp);
2049
2050 if ((bp->b_vp = vp) != NULL) {
2051 bp->b_objlock = vp->v_interlock;
2052 } else {
2053 KASSERT(bp->b_objlock == &buffer_lock);
2054 }
2055
2056 return bp;
2057 }
2058
2059 void
putiobuf(buf_t * bp)2060 putiobuf(buf_t *bp)
2061 {
2062
2063 buf_destroy(bp);
2064 pool_cache_put(bufio_cache, bp);
2065 }
2066
2067 /*
2068 * nestiobuf_iodone: b_iodone callback for nested buffers.
2069 */
2070
2071 void
nestiobuf_iodone(buf_t * bp)2072 nestiobuf_iodone(buf_t *bp)
2073 {
2074 buf_t *mbp = bp->b_private;
2075 int error;
2076 int donebytes;
2077
2078 KASSERT(bp->b_bcount <= bp->b_bufsize);
2079 KASSERT(mbp != bp);
2080
2081 error = bp->b_error;
2082 if (bp->b_error == 0 &&
2083 (bp->b_bcount < bp->b_bufsize || bp->b_resid > 0)) {
2084 /*
2085 * Not all got transferred, raise an error. We have no way to
2086 * propagate these conditions to mbp.
2087 */
2088 error = SET_ERROR(EIO);
2089 }
2090
2091 donebytes = bp->b_bufsize;
2092
2093 putiobuf(bp);
2094 nestiobuf_done(mbp, donebytes, error);
2095 }
2096
2097 /*
2098 * nestiobuf_setup: setup a "nested" buffer.
2099 *
2100 * => 'mbp' is a "master" buffer which is being divided into sub pieces.
2101 * => 'bp' should be a buffer allocated by getiobuf.
2102 * => 'offset' is a byte offset in the master buffer.
2103 * => 'size' is a size in bytes of this nested buffer.
2104 */
2105
2106 void
nestiobuf_setup(buf_t * mbp,buf_t * bp,int offset,size_t size)2107 nestiobuf_setup(buf_t *mbp, buf_t *bp, int offset, size_t size)
2108 {
2109 const int b_pass = mbp->b_flags & (B_READ|B_PHYS|B_RAW|B_MEDIA_FLAGS);
2110 struct vnode *vp = mbp->b_vp;
2111
2112 KASSERT(mbp->b_bcount >= offset + size);
2113 bp->b_vp = vp;
2114 bp->b_dev = mbp->b_dev;
2115 bp->b_objlock = mbp->b_objlock;
2116 bp->b_cflags = BC_BUSY;
2117 bp->b_flags = B_ASYNC | b_pass;
2118 bp->b_iodone = nestiobuf_iodone;
2119 bp->b_data = (char *)mbp->b_data + offset;
2120 bp->b_resid = bp->b_bcount = size;
2121 bp->b_bufsize = bp->b_bcount;
2122 bp->b_private = mbp;
2123 BIO_COPYPRIO(bp, mbp);
2124 if (BUF_ISWRITE(bp) && vp != NULL) {
2125 mutex_enter(vp->v_interlock);
2126 vp->v_numoutput++;
2127 mutex_exit(vp->v_interlock);
2128 }
2129 }
2130
2131 /*
2132 * nestiobuf_done: propagate completion to the master buffer.
2133 *
2134 * => 'donebytes' specifies how many bytes in the 'mbp' is completed.
2135 * => 'error' is an errno(2) that 'donebytes' has been completed with.
2136 */
2137
2138 void
nestiobuf_done(buf_t * mbp,int donebytes,int error)2139 nestiobuf_done(buf_t *mbp, int donebytes, int error)
2140 {
2141
2142 if (donebytes == 0) {
2143 return;
2144 }
2145 mutex_enter(mbp->b_objlock);
2146 KASSERT(mbp->b_resid >= donebytes);
2147 mbp->b_resid -= donebytes;
2148 if (error)
2149 mbp->b_error = error;
2150 if (mbp->b_resid == 0) {
2151 if (mbp->b_error)
2152 mbp->b_resid = mbp->b_bcount;
2153 mutex_exit(mbp->b_objlock);
2154 biodone(mbp);
2155 } else
2156 mutex_exit(mbp->b_objlock);
2157 }
2158
2159 void
buf_init(buf_t * bp)2160 buf_init(buf_t *bp)
2161 {
2162
2163 cv_init(&bp->b_busy, "biolock");
2164 cv_init(&bp->b_done, "biowait");
2165 bp->b_dev = NODEV;
2166 bp->b_error = 0;
2167 bp->b_flags = 0;
2168 bp->b_cflags = 0;
2169 bp->b_oflags = 0;
2170 bp->b_objlock = &buffer_lock;
2171 bp->b_iodone = NULL;
2172 bp->b_dev = NODEV;
2173 bp->b_vnbufs.le_next = NOLIST;
2174 BIO_SETPRIO(bp, BPRIO_DEFAULT);
2175 }
2176
2177 void
buf_destroy(buf_t * bp)2178 buf_destroy(buf_t *bp)
2179 {
2180
2181 cv_destroy(&bp->b_done);
2182 cv_destroy(&bp->b_busy);
2183 }
2184
2185 int
bbusy(buf_t * bp,bool intr,int timo,kmutex_t * interlock)2186 bbusy(buf_t *bp, bool intr, int timo, kmutex_t *interlock)
2187 {
2188 int error;
2189
2190 KASSERT(mutex_owned(&bufcache_lock));
2191
2192 SDT_PROBE4(io, kernel, , bbusy__start, bp, intr, timo, interlock);
2193
2194 if ((bp->b_cflags & BC_BUSY) != 0) {
2195 if (curlwp == uvm.pagedaemon_lwp) {
2196 error = SET_ERROR(EDEADLK);
2197 goto out;
2198 }
2199 bp->b_cflags |= BC_WANTED;
2200 if (interlock != NULL)
2201 mutex_exit(interlock);
2202 if (intr) {
2203 error = cv_timedwait_sig(&bp->b_busy, &bufcache_lock,
2204 timo);
2205 } else {
2206 error = cv_timedwait(&bp->b_busy, &bufcache_lock,
2207 timo);
2208 }
2209 /*
2210 * At this point the buffer may be gone: don't touch it
2211 * again. The caller needs to find it again and retry.
2212 */
2213 if (interlock != NULL)
2214 mutex_enter(interlock);
2215 if (error == 0)
2216 error = SET_ERROR(EPASSTHROUGH);
2217 } else {
2218 bp->b_cflags |= BC_BUSY;
2219 error = 0;
2220 }
2221
2222 out: SDT_PROBE5(io, kernel, , bbusy__done,
2223 bp, intr, timo, interlock, error);
2224 return error;
2225 }
2226
2227 /*
2228 * Nothing outside this file should really need to know about nbuf,
2229 * but a few things still want to read it, so give them a way to do that.
2230 */
2231 u_int
buf_nbuf(void)2232 buf_nbuf(void)
2233 {
2234
2235 return nbuf;
2236 }
2237