105 #define	VM_RESERV_INDEX(object, pindex)	\  argument
106     (((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1))
194 	vm_object_t	object;			/* (o, r) containing object */  member
300 #define	vm_reserv_object_lock_idx(object)			\  argument
301 	    (((uintptr_t)object / sizeof(*object)) % VM_RESERV_OBJ_LOCK_COUNT)
302 #define	vm_reserv_object_lock_ptr(object)			\  argument
303 	    &vm_reserv_object_mtx[vm_reserv_object_lock_idx((object))]
304 #define	vm_reserv_object_lock(object)				\  argument
305 	    mtx_lock(vm_reserv_object_lock_ptr((object)))
306 #define	vm_reserv_object_unlock(object)				\  argument
307 	    mtx_unlock(vm_reserv_object_lock_ptr((object)))
388 	vm_object_t object;  in vm_reserv_remove()  local
392 	    __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);  in vm_reserv_remove()
393 	KASSERT(rv->object != NULL,  in vm_reserv_remove()
397 	object = rv->object;  in vm_reserv_remove()
398 	vm_reserv_object_lock(object);  in vm_reserv_remove()
400 	rv->object = NULL;  in vm_reserv_remove()
401 	vm_reserv_object_unlock(object);  in vm_reserv_remove()
408 vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex)  in vm_reserv_insert()  argument
415 	    __FUNCTION__, rv, rv->pages, rv->object, object,  in vm_reserv_insert()
417 	KASSERT(rv->object == NULL,  in vm_reserv_insert()
426 	vm_reserv_object_lock(object);  in vm_reserv_insert()
428 	rv->object = object;  in vm_reserv_insert()
430 	LIST_INSERT_HEAD(&object->rvq, rv, objq);  in vm_reserv_insert()
431 	vm_reserv_object_unlock(object);  in vm_reserv_insert()
447 	    __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);  in vm_reserv_depopulate()
448 	KASSERT(rv->object != NULL,  in vm_reserv_depopulate()
506 vm_reserv_from_object(vm_object_t object, vm_pindex_t pindex,  in vm_reserv_from_object()  argument
514 		KASSERT(mpred->object == object,  in vm_reserv_from_object()
519 		if (rv->object == object && vm_reserv_has_pindex(rv, pindex))  in vm_reserv_from_object()
523 		msucc = TAILQ_FIRST(&object->memq);  in vm_reserv_from_object()
528 		if (rv->object == object && vm_reserv_has_pindex(rv, pindex))  in vm_reserv_from_object()
560 	    __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);  in vm_reserv_populate()
561 	KASSERT(rv->object != NULL,  in vm_reserv_populate()
607 vm_reserv_extend_contig(int req, vm_object_t object, vm_pindex_t pindex,  in vm_reserv_extend_contig()  argument
617 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_reserv_extend_contig()
623 	if (pindex < VM_RESERV_INDEX(object, pindex) ||  in vm_reserv_extend_contig()
624 	    pindex + npages > object->size || object->resident_page_count == 0)  in vm_reserv_extend_contig()
637 	pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT;  in vm_reserv_extend_contig()
647 	rv = vm_reserv_from_object(object, pindex, mpred, &msucc);  in vm_reserv_extend_contig()
650 	KASSERT(object != kernel_object || rv->domain == domain,  in vm_reserv_extend_contig()
652 	index = VM_RESERV_INDEX(object, pindex);  in vm_reserv_extend_contig()
659 	if (rv->object != object)  in vm_reserv_extend_contig()
702 vm_reserv_alloc_contig(int req, vm_object_t object, vm_pindex_t pindex, int domain,  in vm_reserv_alloc_contig()  argument
714 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_reserv_alloc_contig()
720 	if (pindex < VM_RESERV_INDEX(object, pindex) ||  in vm_reserv_alloc_contig()
721 	    pindex + npages > object->size)  in vm_reserv_alloc_contig()
734 	pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT;  in vm_reserv_alloc_contig()
746 	rv = vm_reserv_from_object(object, pindex, mpred, &msucc);  in vm_reserv_alloc_contig()
759 	first = pindex - VM_RESERV_INDEX(object, pindex);  in vm_reserv_alloc_contig()
760 	minpages = VM_RESERV_INDEX(object, pindex) + npages;  in vm_reserv_alloc_contig()
763 	vm_reserv_object_lock(object);  in vm_reserv_alloc_contig()
765 		if ((rv = vm_reserv_from_page(mpred))->object != object)  in vm_reserv_alloc_contig()
770 			vm_reserv_object_unlock(object);  in vm_reserv_alloc_contig()
775 		if ((rv = vm_reserv_from_page(msucc))->object != object)  in vm_reserv_alloc_contig()
781 				vm_reserv_object_unlock(object);  in vm_reserv_alloc_contig()
794 	vm_reserv_object_unlock(object);  in vm_reserv_alloc_contig()
799 	if (first + maxpages > object->size) {  in vm_reserv_alloc_contig()
804 		if (object->type == OBJT_VNODE ||  in vm_reserv_alloc_contig()
805 		    (object->backing_object != NULL &&  in vm_reserv_alloc_contig()
806 		    object->backing_object->type == OBJT_VNODE)) {  in vm_reserv_alloc_contig()
845 	index = VM_RESERV_INDEX(object, pindex);  in vm_reserv_alloc_contig()
852 		vm_reserv_insert(rv, object, first);  in vm_reserv_alloc_contig()
879 vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain,  in vm_reserv_extend()  argument
887 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_reserv_extend()
892 	if (pindex < VM_RESERV_INDEX(object, pindex) ||  in vm_reserv_extend()
893 	    pindex >= object->size || object->resident_page_count == 0)  in vm_reserv_extend()
899 	rv = vm_reserv_from_object(object, pindex, mpred, &msucc);  in vm_reserv_extend()
903 	KASSERT(object != kernel_object || rv->domain == domain,  in vm_reserv_extend()
907 	index = VM_RESERV_INDEX(object, pindex);  in vm_reserv_extend()
911 	if (rv->object != object ||  in vm_reserv_extend()
938 vm_reserv_alloc_page(int req, vm_object_t object, vm_pindex_t pindex, int domain,  in vm_reserv_alloc_page()  argument
947 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_reserv_alloc_page()
952 	if (pindex < VM_RESERV_INDEX(object, pindex) ||  in vm_reserv_alloc_page()
953 	    pindex >= object->size)  in vm_reserv_alloc_page()
961 	rv = vm_reserv_from_object(object, pindex, mpred, &msucc);  in vm_reserv_alloc_page()
973 	first = pindex - VM_RESERV_INDEX(object, pindex);  in vm_reserv_alloc_page()
974 	vm_reserv_object_lock(object);  in vm_reserv_alloc_page()
976 		if ((rv = vm_reserv_from_page(mpred))->object != object)  in vm_reserv_alloc_page()
981 			vm_reserv_object_unlock(object);  in vm_reserv_alloc_page()
986 		if ((rv = vm_reserv_from_page(msucc))->object != object)  in vm_reserv_alloc_page()
991 			vm_reserv_object_unlock(object);  in vm_reserv_alloc_page()
995 	vm_reserv_object_unlock(object);  in vm_reserv_alloc_page()
1000 	if (first + VM_LEVEL_0_NPAGES > object->size) {  in vm_reserv_alloc_page()
1005 		if (object->type == OBJT_VNODE ||  in vm_reserv_alloc_page()
1006 		    (object->backing_object != NULL &&  in vm_reserv_alloc_page()
1007 		    object->backing_object->type == OBJT_VNODE))  in vm_reserv_alloc_page()
1032 	vm_reserv_insert(rv, object, first);  in vm_reserv_alloc_page()
1033 	index = VM_RESERV_INDEX(object, pindex);  in vm_reserv_alloc_page()
1055 	    __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);  in vm_reserv_break()
1108 vm_reserv_break_all(vm_object_t object)  in vm_reserv_break_all()  argument
1119 	while ((rv = LIST_FIRST(&object->rvq)) != NULL) {  in vm_reserv_break_all()
1122 		if (rv->object != object) {  in vm_reserv_break_all()
1148 	if (rv->object == NULL)  in vm_reserv_free_page()
1152 	if (rv->object != NULL) {  in vm_reserv_free_page()
1223 	if (rv->object == NULL)  in vm_reserv_is_page_free()
1238 	return (rv->object != NULL ? 0 : -1);  in vm_reserv_level()
1264 	    __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);  in vm_reserv_dequeue()
1282 	    __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);  in vm_reserv_reclaim()
1513 	if (rv->object == old_object) {  in vm_reserv_rename()
1517 		    __FUNCTION__, rv, rv->object, new_object, rv->popcnt,  in vm_reserv_rename()
1519 		if (rv->object == old_object) {  in vm_reserv_rename()
1521 			rv->object = NULL;  in vm_reserv_rename()
1525 			rv->object = new_object;  in vm_reserv_rename()
1608 	VM_OBJECT_ASSERT_LOCKED(m->object);  in vm_reserv_to_superpage()
1610 	if (rv->object == m->object && rv->popcnt == VM_LEVEL_0_NPAGES)  in vm_reserv_to_superpage()