111 #define	BQ_LOCKPTR(bq)		(&(bq)->bq_lock)  argument
112 #define	BQ_LOCK(bq)		mtx_lock(BQ_LOCKPTR((bq)))  argument
113 #define	BQ_UNLOCK(bq)		mtx_unlock(BQ_LOCKPTR((bq)))  argument
114 #define	BQ_ASSERT_LOCKED(bq)	mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED)  argument
183 static void bq_remove(struct bufqueue *bq, struct buf *bp);
184 static void bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock);
186 static void bq_init(struct bufqueue *bq, int qindex, int cpu,
1497 	struct bufqueue *bq, *nbq;  in bufqueue_acquire()  local
1504 	bq = bufqueue(bp);  in bufqueue_acquire()
1505 	BQ_LOCK(bq);  in bufqueue_acquire()
1508 		if (bq == nbq)  in bufqueue_acquire()
1510 		BQ_UNLOCK(bq);  in bufqueue_acquire()
1512 		bq = nbq;  in bufqueue_acquire()
1514 	return (bq);  in bufqueue_acquire()
1527 	struct bufqueue *bq;  in binsfree()  local
1543 		bq = bufqueue_acquire(bp);  in binsfree()
1544 		bq_remove(bq, bp);  in binsfree()
1545 		BQ_UNLOCK(bq);  in binsfree()
1550 			bq = &bd->bd_subq[PCPU_GET(cpuid)];  in binsfree()
1552 			bq = bd->bd_cleanq;  in binsfree()
1554 		bq = &bd->bd_dirtyq;  in binsfree()
1555 	bq_insert(bq, bp, true);  in binsfree()
1622 	struct bufqueue *bq;  in buf_release()  local
1626 	bq = &bqempty;  in buf_release()
1627 	BQ_LOCK(bq);  in buf_release()
1631 		TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);  in buf_release()
1633 		bq->bq_len++;  in buf_release()
1634 		bp->b_qindex = bq->bq_index;  in buf_release()
1636 	BQ_UNLOCK(bq);  in buf_release()
1719 	struct bufqueue *bq;  in buf_recycle()  local
1725 	bq = bd->bd_cleanq;  in buf_recycle()
1726 	BQ_LOCK(bq);  in buf_recycle()
1727 	KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd),  in buf_recycle()
1729 	nbp = TAILQ_FIRST(&bq->bq_queue);  in buf_recycle()
1757 			TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);  in buf_recycle()
1758 			TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);  in buf_recycle()
1782 		bq_remove(bq, bp);  in buf_recycle()
1783 		BQ_UNLOCK(bq);  in buf_recycle()
1791 			BQ_LOCK(bq);  in buf_recycle()
1792 			nbp = TAILQ_FIRST(&bq->bq_queue);  in buf_recycle()
1800 	BQ_UNLOCK(bq);  in buf_recycle()
1834 	struct bufqueue *bq;  in bremfreef()  local
1836 	bq = bufqueue_acquire(bp);  in bremfreef()
1837 	bq_remove(bq, bp);  in bremfreef()
1838 	BQ_UNLOCK(bq);  in bremfreef()
1842 bq_init(struct bufqueue *bq, int qindex, int subqueue, const char *lockname)  in bq_init()  argument
1845 	mtx_init(&bq->bq_lock, lockname, NULL, MTX_DEF);  in bq_init()
1846 	TAILQ_INIT(&bq->bq_queue);  in bq_init()
1847 	bq->bq_len = 0;  in bq_init()
1848 	bq->bq_index = qindex;  in bq_init()
1849 	bq->bq_subqueue = subqueue;  in bq_init()
1873 bq_remove(struct bufqueue *bq, struct buf *bp)  in bq_remove()  argument
1880 	KASSERT(bufqueue(bp) == bq,  in bq_remove()
1883 	BQ_ASSERT_LOCKED(bq);  in bq_remove()
1887 	KASSERT(bq->bq_len >= 1,  in bq_remove()
1889 	TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);  in bq_remove()
1890 	bq->bq_len--;  in bq_remove()
1896 bd_flush(struct bufdomain *bd, struct bufqueue *bq)  in bd_flush()  argument
1900 	BQ_ASSERT_LOCKED(bq);  in bd_flush()
1901 	if (bq != bd->bd_cleanq) {  in bd_flush()
1903 		while ((bp = TAILQ_FIRST(&bq->bq_queue)) != NULL) {  in bd_flush()
1904 			TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);  in bd_flush()
1909 		bd->bd_cleanq->bq_len += bq->bq_len;  in bd_flush()
1910 		bq->bq_len = 0;  in bd_flush()
1916 	if (bq != bd->bd_cleanq)  in bd_flush()
1923 	struct bufqueue *bq;  in bd_flushall()  local
1931 		bq = &bd->bd_subq[i];  in bd_flushall()
1932 		if (bq->bq_len == 0)  in bd_flushall()
1934 		BQ_LOCK(bq);  in bd_flushall()
1935 		bd_flush(bd, bq);  in bd_flushall()
1936 		BQ_UNLOCK(bq);  in bd_flushall()
1944 bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock)  in bq_insert()  argument
1954 		if (bq->bq_index == QUEUE_CLEAN)  in bq_insert()
1955 			bq = bd->bd_cleanq;  in bq_insert()
1956 		BQ_LOCK(bq);  in bq_insert()
1957 		TAILQ_INSERT_HEAD(&bq->bq_queue, bp, b_freelist);  in bq_insert()
1959 		BQ_LOCK(bq);  in bq_insert()
1960 		TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);  in bq_insert()
1963 	bq->bq_len++;  in bq_insert()
1964 	bp->b_qindex = bq->bq_index;  in bq_insert()
1965 	bp->b_subqueue = bq->bq_subqueue;  in bq_insert()
1978 		if (bd->bd_wanted || (bq != bd->bd_cleanq &&  in bq_insert()
1979 		    bq->bq_len >= bd->bd_lim))  in bq_insert()
1980 			bd_flush(bd, bq);  in bq_insert()
1982 	BQ_UNLOCK(bq);  in bq_insert()
3440 	struct bufqueue *bq;  in flushbufqueues()  local
3451 	bq = &bd->bd_dirtyq;  in flushbufqueues()
3455 	BQ_LOCK(bq);  in flushbufqueues()
3456 	TAILQ_INSERT_HEAD(&bq->bq_queue, sentinel, b_freelist);  in flushbufqueues()
3457 	BQ_UNLOCK(bq);  in flushbufqueues()
3460 		BQ_LOCK(bq);  in flushbufqueues()
3463 			TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);  in flushbufqueues()
3464 			TAILQ_INSERT_AFTER(&bq->bq_queue, bp, sentinel,  in flushbufqueues()
3467 			BQ_UNLOCK(bq);  in flushbufqueues()
3479 			BQ_UNLOCK(bq);  in flushbufqueues()
3483 		BQ_UNLOCK(bq);  in flushbufqueues()
3563 	BQ_LOCK(bq);  in flushbufqueues()
3564 	TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);  in flushbufqueues()
3565 	BQ_UNLOCK(bq);  in flushbufqueues()