49 static void os_pages_unmap(void *addr, size_t size);
54 os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) {  in os_pages_map()  argument
56 	assert(ALIGNMENT_CEILING(size, os_page) == size);  in os_pages_map()
57 	assert(size != 0);  in os_pages_map()
69 	ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),  in os_pages_map()
79 		ret = mmap(addr, size, prot, mmap_flags, -1, 0);  in os_pages_map()
89 		os_pages_unmap(ret, size);  in os_pages_map()
99 os_pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,  in os_pages_trim()  argument
103 	assert(alloc_size >= leadsize + size);  in os_pages_trim()
106 	void *new_addr = os_pages_map(ret, size, PAGE, commit);  in os_pages_trim()
111 		os_pages_unmap(new_addr, size);  in os_pages_trim()
115 	size_t trailsize = alloc_size - leadsize - size;  in os_pages_trim()
121 		os_pages_unmap((void *)((uintptr_t)ret + size), trailsize);  in os_pages_trim()
128 os_pages_unmap(void *addr, size_t size) {  in os_pages_unmap()  argument
130 	assert(ALIGNMENT_CEILING(size, os_page) == size);  in os_pages_unmap()
135 	if (munmap(addr, size) == -1)  in os_pages_unmap()
155 pages_map_slow(size_t size, size_t alignment, bool *commit) {  in pages_map_slow()  argument
156 	size_t alloc_size = size + alignment - os_page;  in pages_map_slow()
158 	if (alloc_size < size) {  in pages_map_slow()
170 		ret = os_pages_trim(pages, alloc_size, leadsize, size, commit);  in pages_map_slow()
179 pages_map(void *addr, size_t size, size_t alignment, bool *commit) {  in pages_map()  argument
197 	void *ret = os_pages_map(addr, size, os_page, commit);  in pages_map()
203 		os_pages_unmap(ret, size);  in pages_map()
204 		return pages_map_slow(size, alignment, commit);  in pages_map()
212 pages_unmap(void *addr, size_t size) {  in pages_unmap()  argument
214 	assert(PAGE_CEILING(size) == size);  in pages_unmap()
216 	os_pages_unmap(addr, size);  in pages_unmap()
220 pages_commit_impl(void *addr, size_t size, bool commit) {  in pages_commit_impl()  argument
222 	assert(PAGE_CEILING(size) == size);  in pages_commit_impl()
229 	return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,  in pages_commit_impl()
230 	    PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));  in pages_commit_impl()
234 		void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,  in pages_commit_impl()
244 			os_pages_unmap(result, size);  in pages_commit_impl()
253 pages_commit(void *addr, size_t size) {  in pages_commit()  argument
254 	return pages_commit_impl(addr, size, true);  in pages_commit()
258 pages_decommit(void *addr, size_t size) {  in pages_decommit()  argument
259 	return pages_commit_impl(addr, size, false);  in pages_decommit()
263 pages_purge_lazy(void *addr, size_t size) {  in pages_purge_lazy()  argument
265 	assert(PAGE_CEILING(size) == size);  in pages_purge_lazy()
279 	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);  in pages_purge_lazy()
282 	return (madvise(addr, size,  in pages_purge_lazy()
291 	return (madvise(addr, size, MADV_DONTNEED) != 0);  in pages_purge_lazy()
298 pages_purge_forced(void *addr, size_t size) {  in pages_purge_forced()  argument
300 	assert(PAGE_CEILING(size) == size);  in pages_purge_forced()
308 	return (madvise(addr, size, MADV_DONTNEED) != 0);  in pages_purge_forced()
311 	return pages_commit(addr, size);  in pages_purge_forced()
318 pages_huge_impl(void *addr, size_t size, bool aligned) {  in pages_huge_impl()  argument
321 		assert(HUGEPAGE_CEILING(size) == size);  in pages_huge_impl()
324 	return (madvise(addr, size, MADV_HUGEPAGE) != 0);  in pages_huge_impl()
331 pages_huge(void *addr, size_t size) {  in pages_huge()  argument
332 	return pages_huge_impl(addr, size, true);  in pages_huge()
336 pages_huge_unaligned(void *addr, size_t size) {  in pages_huge_unaligned()  argument
337 	return pages_huge_impl(addr, size, false);  in pages_huge_unaligned()
341 pages_nohuge_impl(void *addr, size_t size, bool aligned) {  in pages_nohuge_impl()  argument
344 		assert(HUGEPAGE_CEILING(size) == size);  in pages_nohuge_impl()
348 	return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);  in pages_nohuge_impl()
355 pages_nohuge(void *addr, size_t size) {  in pages_nohuge()  argument
356 	return pages_nohuge_impl(addr, size, true);  in pages_nohuge()
360 pages_nohuge_unaligned(void *addr, size_t size) {  in pages_nohuge_unaligned()  argument
361 	return pages_nohuge_impl(addr, size, false);  in pages_nohuge_unaligned()
365 pages_dontdump(void *addr, size_t size) {  in pages_dontdump()  argument
367 	assert(PAGE_CEILING(size) == size);  in pages_dontdump()
369 	return madvise(addr, size, MADV_DONTDUMP) != 0;  in pages_dontdump()
376 pages_dodump(void *addr, size_t size) {  in pages_dodump()  argument
378 	assert(PAGE_CEILING(size) == size);  in pages_dodump()
380 	return madvise(addr, size, MADV_DODUMP) != 0;  in pages_dodump()
497 pages_set_thp_state (void *ptr, size_t size) {  in pages_set_thp_state()  argument
507 		pages_huge_unaligned(ptr, size);  in pages_set_thp_state()
511 		pages_nohuge_unaligned(ptr, size);  in pages_set_thp_state()