1 /* $NetBSD: uvm_physseg.c,v 1.20 2024/01/13 09:44:42 tnn Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993, The Regents of the University of California.
6 *
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)vm_page.h 7.3 (Berkeley) 4/21/91
37 * from: Id: uvm_page.h,v 1.1.2.6 1998/02/04 02:31:42 chuck Exp
38 *
39 *
40 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
41 * All rights reserved.
42 *
43 * Permission to use, copy, modify and distribute this software and
44 * its documentation is hereby granted, provided that both the copyright
45 * notice and this permission notice appear in all copies of the
46 * software, derivative works or modified versions, and any portions
47 * thereof, and that both notices appear in supporting documentation.
48 *
49 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
50 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
51 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
52 *
53 * Carnegie Mellon requests users of this software to return to
54 *
55 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
56 * School of Computer Science
57 * Carnegie Mellon University
58 * Pittsburgh PA 15213-3890
59 *
60 * any improvements or extensions that they make and grant Carnegie the
61 * rights to redistribute these changes.
62 */
63
64 /*
65 * Consolidated API from uvm_page.c and others.
66 * Consolidated and designed by Cherry G. Mathew <cherry@zyx.in>
67 * rbtree(3) backing implementation by:
68 * Santhosh N. Raju <santhosh.raju@gmail.com>
69 */
70
71 #ifdef _KERNEL_OPT
72 #include "opt_uvm.h"
73 #endif
74
75 #include <sys/param.h>
76 #include <sys/types.h>
77 #include <sys/extent.h>
78 #include <sys/kmem.h>
79
80 #include <uvm/uvm.h>
81 #include <uvm/uvm_page.h>
82 #include <uvm/uvm_param.h>
83 #include <uvm/uvm_pdpolicy.h>
84 #include <uvm/uvm_physseg.h>
85
86 /*
87 * uvm_physseg: describes one segment of physical memory
88 */
89 struct uvm_physseg {
90 /* used during RB tree lookup for PHYS_TO_VM_PAGE(). */
91 #if defined(UVM_HOTPLUG)
92 struct rb_node rb_node; /* tree information */
93 #endif
94 paddr_t start; /* PF# of first page in segment */
95 paddr_t end; /* (PF# of last page in segment) + 1 */
96 struct vm_page *pgs; /* vm_page structures (from start) */
97
98 /* less performance sensitive fields. */
99 paddr_t avail_start; /* PF# of first free page in segment */
100 paddr_t avail_end; /* (PF# of last free page in segment) +1 */
101 struct extent *ext; /* extent(9) structure to manage pgs[] */
102 int free_list; /* which free list they belong on */
103 u_long start_hint; /* start looking for free pages here */
104 #ifdef __HAVE_PMAP_PHYSSEG
105 struct pmap_physseg pmseg; /* pmap specific (MD) data */
106 #endif
107 };
108
109 /*
110 * These functions are reserved for uvm(9) internal use and are not
111 * exported in the header file uvm_physseg.h
112 *
113 * Thus they are redefined here.
114 */
115 void uvm_physseg_init_seg(uvm_physseg_t, struct vm_page *);
116 void uvm_physseg_seg_chomp_slab(uvm_physseg_t, struct vm_page *, size_t);
117
118 /* returns a pgs array */
119 struct vm_page *uvm_physseg_seg_alloc_from_slab(uvm_physseg_t, size_t);
120
121 #if defined(UVM_HOTPLUG) /* rbtree impementation */
122
123 #define HANDLE_TO_PHYSSEG_NODE(h) ((struct uvm_physseg *)(h))
124 #define PHYSSEG_NODE_TO_HANDLE(u) ((uvm_physseg_t)(u))
125
126 struct uvm_physseg_graph {
127 struct rb_tree rb_tree; /* Tree for entries */
128 int nentries; /* Number of entries */
129 } __aligned(COHERENCY_UNIT);
130
131 static struct uvm_physseg_graph uvm_physseg_graph __read_mostly;
132
133 /*
134 * Note on kmem(9) allocator usage:
135 * We take the conservative approach that plug/unplug are allowed to
136 * fail in high memory stress situations.
137 *
138 * We want to avoid re-entrant situations in which one plug/unplug
139 * operation is waiting on a previous one to complete, since this
140 * makes the design more complicated than necessary.
141 *
142 * We may review this and change its behaviour, once the use cases
143 * become more obvious.
144 */
145
146 /*
147 * Special alloc()/free() functions for boot time support:
148 * We assume that alloc() at boot time is only for new 'vm_physseg's
149 * This allows us to use a static array for memory allocation at boot
150 * time. Thus we avoid using kmem(9) which is not ready at this point
151 * in boot.
152 *
153 * After kmem(9) is ready, we use it. We currently discard any free()s
154 * to this static array, since the size is small enough to be a
155 * trivial waste on all architectures we run on.
156 */
157
158 static size_t nseg = 0;
159 static struct uvm_physseg uvm_physseg[VM_PHYSSEG_MAX];
160
161 static void *
uvm_physseg_alloc(size_t sz)162 uvm_physseg_alloc(size_t sz)
163 {
164 /*
165 * During boot time, we only support allocating vm_physseg
166 * entries from the static array.
167 * We need to assert for this.
168 */
169
170 if (__predict_false(uvm.page_init_done == false)) {
171 if (sz % sizeof(struct uvm_physseg))
172 panic("%s: tried to alloc size other than multiple"
173 " of struct uvm_physseg at boot\n", __func__);
174
175 size_t n = sz / sizeof(struct uvm_physseg);
176 nseg += n;
177
178 KASSERT(nseg > 0);
179 KASSERT(nseg <= VM_PHYSSEG_MAX);
180
181 return &uvm_physseg[nseg - n];
182 }
183
184 return kmem_zalloc(sz, KM_NOSLEEP);
185 }
186
187 static void
uvm_physseg_free(void * p,size_t sz)188 uvm_physseg_free(void *p, size_t sz)
189 {
190 /*
191 * This is a bit tricky. We do allow simulation of free()
192 * during boot (for eg: when MD code is "steal"ing memory,
193 * and the segment has been exhausted (and thus needs to be
194 * free() - ed.
195 * free() also complicates things because we leak the
196 * free(). Therefore calling code can't assume that free()-ed
197 * memory is available for alloc() again, at boot time.
198 *
199 * Thus we can't explicitly disallow free()s during
200 * boot time. However, the same restriction for alloc()
201 * applies to free(). We only allow uvm_physseg related free()s
202 * via this function during boot time.
203 */
204
205 if (__predict_false(uvm.page_init_done == false)) {
206 if (sz % sizeof(struct uvm_physseg))
207 panic("%s: tried to free size other than struct uvm_physseg"
208 " at boot\n", __func__);
209
210 }
211
212 /*
213 * Could have been in a single if(){} block - split for
214 * clarity
215 */
216
217 if ((struct uvm_physseg *)p >= uvm_physseg &&
218 (struct uvm_physseg *)p < (uvm_physseg + VM_PHYSSEG_MAX)) {
219 if (sz % sizeof(struct uvm_physseg))
220 panic("%s: tried to free() other than struct uvm_physseg"
221 " from static array\n", __func__);
222
223 if ((sz / sizeof(struct uvm_physseg)) >= VM_PHYSSEG_MAX)
224 panic("%s: tried to free() the entire static array!", __func__);
225 return; /* Nothing to free */
226 }
227
228 kmem_free(p, sz);
229 }
230
231 /* XXX: Multi page size */
232 bool
uvm_physseg_plug(paddr_t pfn,size_t pages,uvm_physseg_t * psp)233 uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp)
234 {
235 int preload;
236 size_t slabpages;
237 struct uvm_physseg *ps, *current_ps = NULL;
238 struct vm_page *slab = NULL, *pgs = NULL;
239
240 #ifdef DEBUG
241 paddr_t off;
242 uvm_physseg_t upm;
243 upm = uvm_physseg_find(pfn, &off);
244
245 ps = HANDLE_TO_PHYSSEG_NODE(upm);
246
247 if (ps != NULL) /* XXX; do we allow "update" plugs ? */
248 return false;
249 #endif
250
251 /*
252 * do we have room?
253 */
254
255 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg));
256 if (ps == NULL) {
257 printf("uvm_page_physload: unable to load physical memory "
258 "segment\n");
259 printf("\t%d segments allocated, ignoring 0x%"PRIxPADDR" -> 0x%"PRIxPADDR"\n",
260 VM_PHYSSEG_MAX, pfn, pfn + pages + 1);
261 printf("\tincrease VM_PHYSSEG_MAX\n");
262 return false;
263 }
264
265 /* span init */
266 ps->start = pfn;
267 ps->end = pfn + pages;
268
269 /*
270 * XXX: Ugly hack because uvmexp.npages accounts for only
271 * those pages in the segment included below as well - this
272 * should be legacy and removed.
273 */
274
275 ps->avail_start = ps->start;
276 ps->avail_end = ps->end;
277
278 /*
279 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been
280 * called yet, so kmem is not available).
281 */
282
283 preload = 1; /* We are going to assume it is a preload */
284
285 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) {
286 /* If there are non NULL pages then we are not in a preload */
287 if (current_ps->pgs != NULL) {
288 preload = 0;
289 /* Try to scavenge from earlier unplug()s. */
290 pgs = uvm_physseg_seg_alloc_from_slab(current_ps, pages);
291
292 if (pgs != NULL) {
293 break;
294 }
295 }
296 }
297
298
299 /*
300 * if VM is already running, attempt to kmem_alloc vm_page structures
301 */
302
303 if (!preload) {
304 if (pgs == NULL) { /* Brand new */
305 /* Iteratively try alloc down from uvmexp.npages */
306 for (slabpages = (size_t) uvmexp.npages; slabpages >= pages; slabpages--) {
307 slab = kmem_zalloc(sizeof *pgs * (long unsigned int)slabpages, KM_NOSLEEP);
308 if (slab != NULL)
309 break;
310 }
311
312 if (slab == NULL) {
313 uvm_physseg_free(ps, sizeof(struct uvm_physseg));
314 return false;
315 }
316
317 uvm_physseg_seg_chomp_slab(ps, slab, (size_t) slabpages);
318 /* We allocate enough for this plug */
319 pgs = uvm_physseg_seg_alloc_from_slab(ps, pages);
320
321 if (pgs == NULL) {
322 printf("unable to uvm_physseg_seg_alloc_from_slab() from backend\n");
323 return false;
324 }
325 } else {
326 /* Reuse scavenged extent */
327 ps->ext = current_ps->ext;
328 }
329
330 physmem += pages;
331 uvmpdpol_reinit();
332 } else { /* Boot time - see uvm_page.c:uvm_page_init() */
333 pgs = NULL;
334 ps->pgs = pgs;
335 }
336
337 /*
338 * now insert us in the proper place in uvm_physseg_graph.rb_tree
339 */
340
341 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps);
342 if (current_ps != ps) {
343 panic("uvm_page_physload: Duplicate address range detected!");
344 }
345 uvm_physseg_graph.nentries++;
346
347 /*
348 * uvm_pagefree() requires the PHYS_TO_VM_PAGE(pgs[i]) on the
349 * newly allocated pgs[] to return the correct value. This is
350 * a bit of a chicken and egg problem, since it needs
351 * uvm_physseg_find() to succeed. For this, the node needs to
352 * be inserted *before* uvm_physseg_init_seg() happens.
353 *
354 * During boot, this happens anyway, since
355 * uvm_physseg_init_seg() is called later on and separately
356 * from uvm_page.c:uvm_page_init().
357 * In the case of hotplug we need to ensure this.
358 */
359
360 if (__predict_true(!preload))
361 uvm_physseg_init_seg(ps, pgs);
362
363 if (psp != NULL)
364 *psp = ps;
365
366 return true;
367 }
368
369 static int
uvm_physseg_compare_nodes(void * ctx,const void * nnode1,const void * nnode2)370 uvm_physseg_compare_nodes(void *ctx, const void *nnode1, const void *nnode2)
371 {
372 const struct uvm_physseg *enode1 = nnode1;
373 const struct uvm_physseg *enode2 = nnode2;
374
375 KASSERT(enode1->start < enode2->start || enode1->start >= enode2->end);
376 KASSERT(enode2->start < enode1->start || enode2->start >= enode1->end);
377
378 if (enode1->start < enode2->start)
379 return -1;
380 if (enode1->start >= enode2->end)
381 return 1;
382 return 0;
383 }
384
385 static int
uvm_physseg_compare_key(void * ctx,const void * nnode,const void * pkey)386 uvm_physseg_compare_key(void *ctx, const void *nnode, const void *pkey)
387 {
388 const struct uvm_physseg *enode = nnode;
389 const paddr_t pa = *(const paddr_t *) pkey;
390
391 if(enode->start <= pa && pa < enode->end)
392 return 0;
393 if (enode->start < pa)
394 return -1;
395 if (enode->end > pa)
396 return 1;
397
398 return 0;
399 }
400
401 static const rb_tree_ops_t uvm_physseg_tree_ops = {
402 .rbto_compare_nodes = uvm_physseg_compare_nodes,
403 .rbto_compare_key = uvm_physseg_compare_key,
404 .rbto_node_offset = offsetof(struct uvm_physseg, rb_node),
405 .rbto_context = NULL
406 };
407
408 /*
409 * uvm_physseg_init: init the physmem
410 *
411 * => physmem unit should not be in use at this point
412 */
413
414 void
uvm_physseg_init(void)415 uvm_physseg_init(void)
416 {
417 rb_tree_init(&(uvm_physseg_graph.rb_tree), &uvm_physseg_tree_ops);
418 uvm_physseg_graph.nentries = 0;
419 }
420
421 uvm_physseg_t
uvm_physseg_get_next(uvm_physseg_t upm)422 uvm_physseg_get_next(uvm_physseg_t upm)
423 {
424 /* next of invalid is invalid, not fatal */
425 if (uvm_physseg_valid_p(upm) == false)
426 return UVM_PHYSSEG_TYPE_INVALID;
427
428 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm,
429 RB_DIR_RIGHT);
430 }
431
432 uvm_physseg_t
uvm_physseg_get_prev(uvm_physseg_t upm)433 uvm_physseg_get_prev(uvm_physseg_t upm)
434 {
435 /* prev of invalid is invalid, not fatal */
436 if (uvm_physseg_valid_p(upm) == false)
437 return UVM_PHYSSEG_TYPE_INVALID;
438
439 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm,
440 RB_DIR_LEFT);
441 }
442
443 uvm_physseg_t
uvm_physseg_get_last(void)444 uvm_physseg_get_last(void)
445 {
446 return (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree));
447 }
448
449 uvm_physseg_t
uvm_physseg_get_first(void)450 uvm_physseg_get_first(void)
451 {
452 return (uvm_physseg_t) RB_TREE_MIN(&(uvm_physseg_graph.rb_tree));
453 }
454
455 paddr_t
uvm_physseg_get_highest_frame(void)456 uvm_physseg_get_highest_frame(void)
457 {
458 struct uvm_physseg *ps =
459 (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree));
460
461 return ps->end - 1;
462 }
463
464 /*
465 * uvm_page_physunload: unload physical memory and return it to
466 * caller.
467 */
468 bool
uvm_page_physunload(uvm_physseg_t upm,int freelist,paddr_t * paddrp)469 uvm_page_physunload(uvm_physseg_t upm, int freelist, paddr_t *paddrp)
470 {
471 struct uvm_physseg *seg;
472
473 if (__predict_true(uvm.page_init_done == true))
474 panic("%s: unload attempted after uvm_page_init()\n", __func__);
475
476 seg = HANDLE_TO_PHYSSEG_NODE(upm);
477
478 if (seg->free_list != freelist) {
479 return false;
480 }
481
482 /*
483 * During cold boot, what we're about to unplug hasn't been
484 * put on the uvm freelist, nor has uvmexp.npages been
485 * updated. (This happens in uvm_page.c:uvm_page_init())
486 *
487 * For hotplug, we assume here that the pages being unloaded
488 * here are completely out of sight of uvm (ie; not on any uvm
489 * lists), and that uvmexp.npages has been suitably
490 * decremented before we're called.
491 *
492 * XXX: will avail_end == start if avail_start < avail_end?
493 */
494
495 /* try from front */
496 if (seg->avail_start == seg->start &&
497 seg->avail_start < seg->avail_end) {
498 *paddrp = ctob(seg->avail_start);
499 return uvm_physseg_unplug(seg->avail_start, 1);
500 }
501
502 /* try from rear */
503 if (seg->avail_end == seg->end &&
504 seg->avail_start < seg->avail_end) {
505 *paddrp = ctob(seg->avail_end - 1);
506 return uvm_physseg_unplug(seg->avail_end - 1, 1);
507 }
508
509 return false;
510 }
511
512 bool
uvm_page_physunload_force(uvm_physseg_t upm,int freelist,paddr_t * paddrp)513 uvm_page_physunload_force(uvm_physseg_t upm, int freelist, paddr_t *paddrp)
514 {
515 struct uvm_physseg *seg;
516
517 seg = HANDLE_TO_PHYSSEG_NODE(upm);
518
519 if (__predict_true(uvm.page_init_done == true))
520 panic("%s: unload attempted after uvm_page_init()\n", __func__);
521 /* any room in this bank? */
522 if (seg->avail_start >= seg->avail_end) {
523 return false; /* nope */
524 }
525
526 *paddrp = ctob(seg->avail_start);
527
528 /* Always unplug from front */
529 return uvm_physseg_unplug(seg->avail_start, 1);
530 }
531
532
533 /*
534 * vm_physseg_find: find vm_physseg structure that belongs to a PA
535 */
536 uvm_physseg_t
uvm_physseg_find(paddr_t pframe,psize_t * offp)537 uvm_physseg_find(paddr_t pframe, psize_t *offp)
538 {
539 struct uvm_physseg * ps = NULL;
540
541 ps = rb_tree_find_node(&(uvm_physseg_graph.rb_tree), &pframe);
542
543 if(ps != NULL && offp != NULL)
544 *offp = pframe - ps->start;
545
546 return ps;
547 }
548
549 #else /* UVM_HOTPLUG */
550
551 /*
552 * physical memory config is stored in vm_physmem.
553 */
554
555 #define VM_PHYSMEM_PTR(i) (&vm_physmem[i])
556 #if VM_PHYSSEG_MAX == 1
557 #define VM_PHYSMEM_PTR_SWAP(i, j) /* impossible */
558 #else
559 #define VM_PHYSMEM_PTR_SWAP(i, j) \
560 do { vm_physmem[(i)] = vm_physmem[(j)]; } while (0)
561 #endif
562
563 #define HANDLE_TO_PHYSSEG_NODE(h) (VM_PHYSMEM_PTR((int)h))
564 #define PHYSSEG_NODE_TO_HANDLE(u) ((int)((vsize_t) (u - vm_physmem) / sizeof(struct uvm_physseg)))
565
566 /* XXXCDC: uvm.physmem */
567 static struct uvm_physseg vm_physmem[VM_PHYSSEG_MAX] __read_mostly;
568 /* XXXCDC: uvm.nphysseg */
569 static int vm_nphysseg __read_mostly = 0;
570 #define vm_nphysmem vm_nphysseg
571
572 void
uvm_physseg_init(void)573 uvm_physseg_init(void)
574 {
575 /* XXX: Provisioning for rb_tree related init(s) */
576 return;
577 }
578
579 int
uvm_physseg_get_next(uvm_physseg_t lcv)580 uvm_physseg_get_next(uvm_physseg_t lcv)
581 {
582 /* next of invalid is invalid, not fatal */
583 if (uvm_physseg_valid_p(lcv) == false)
584 return UVM_PHYSSEG_TYPE_INVALID;
585
586 return (lcv + 1);
587 }
588
589 int
uvm_physseg_get_prev(uvm_physseg_t lcv)590 uvm_physseg_get_prev(uvm_physseg_t lcv)
591 {
592 /* prev of invalid is invalid, not fatal */
593 if (uvm_physseg_valid_p(lcv) == false)
594 return UVM_PHYSSEG_TYPE_INVALID;
595
596 return (lcv - 1);
597 }
598
599 int
uvm_physseg_get_last(void)600 uvm_physseg_get_last(void)
601 {
602 return (vm_nphysseg - 1);
603 }
604
605 int
uvm_physseg_get_first(void)606 uvm_physseg_get_first(void)
607 {
608 return 0;
609 }
610
611 paddr_t
uvm_physseg_get_highest_frame(void)612 uvm_physseg_get_highest_frame(void)
613 {
614 int lcv;
615 paddr_t last = 0;
616 struct uvm_physseg *ps;
617
618 for (lcv = 0; lcv < vm_nphysseg; lcv++) {
619 ps = VM_PHYSMEM_PTR(lcv);
620 if (last < ps->end)
621 last = ps->end;
622 }
623
624 return last;
625 }
626
627
628 static struct vm_page *
uvm_post_preload_check(void)629 uvm_post_preload_check(void)
630 {
631 int preload, lcv;
632
633 /*
634 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been
635 * called yet, so kmem is not available).
636 */
637
638 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) {
639 if (VM_PHYSMEM_PTR(lcv)->pgs)
640 break;
641 }
642 preload = (lcv == vm_nphysmem);
643
644 /*
645 * if VM is already running, attempt to kmem_alloc vm_page structures
646 */
647
648 if (!preload) {
649 panic("Tried to add RAM after uvm_page_init");
650 }
651
652 return NULL;
653 }
654
655 /*
656 * uvm_page_physunload: unload physical memory and return it to
657 * caller.
658 */
659 bool
uvm_page_physunload(uvm_physseg_t psi,int freelist,paddr_t * paddrp)660 uvm_page_physunload(uvm_physseg_t psi, int freelist, paddr_t *paddrp)
661 {
662 int x;
663 struct uvm_physseg *seg;
664
665 uvm_post_preload_check();
666
667 seg = VM_PHYSMEM_PTR(psi);
668
669 if (seg->free_list != freelist) {
670 return false;
671 }
672
673 /* try from front */
674 if (seg->avail_start == seg->start &&
675 seg->avail_start < seg->avail_end) {
676 *paddrp = ctob(seg->avail_start);
677 seg->avail_start++;
678 seg->start++;
679 /* nothing left? nuke it */
680 if (seg->avail_start == seg->end) {
681 if (vm_nphysmem == 1)
682 panic("uvm_page_physget: out of memory!");
683 vm_nphysmem--;
684 for (x = psi ; x < vm_nphysmem ; x++)
685 /* structure copy */
686 VM_PHYSMEM_PTR_SWAP(x, x + 1);
687 }
688 return (true);
689 }
690
691 /* try from rear */
692 if (seg->avail_end == seg->end &&
693 seg->avail_start < seg->avail_end) {
694 *paddrp = ctob(seg->avail_end - 1);
695 seg->avail_end--;
696 seg->end--;
697 /* nothing left? nuke it */
698 if (seg->avail_end == seg->start) {
699 if (vm_nphysmem == 1)
700 panic("uvm_page_physget: out of memory!");
701 vm_nphysmem--;
702 for (x = psi ; x < vm_nphysmem ; x++)
703 /* structure copy */
704 VM_PHYSMEM_PTR_SWAP(x, x + 1);
705 }
706 return (true);
707 }
708
709 return false;
710 }
711
712 bool
uvm_page_physunload_force(uvm_physseg_t psi,int freelist,paddr_t * paddrp)713 uvm_page_physunload_force(uvm_physseg_t psi, int freelist, paddr_t *paddrp)
714 {
715 int x;
716 struct uvm_physseg *seg;
717
718 uvm_post_preload_check();
719
720 seg = VM_PHYSMEM_PTR(psi);
721
722 /* any room in this bank? */
723 if (seg->avail_start >= seg->avail_end) {
724 return false; /* nope */
725 }
726
727 *paddrp = ctob(seg->avail_start);
728 seg->avail_start++;
729 /* truncate! */
730 seg->start = seg->avail_start;
731
732 /* nothing left? nuke it */
733 if (seg->avail_start == seg->end) {
734 if (vm_nphysmem == 1)
735 panic("uvm_page_physget: out of memory!");
736 vm_nphysmem--;
737 for (x = psi ; x < vm_nphysmem ; x++)
738 /* structure copy */
739 VM_PHYSMEM_PTR_SWAP(x, x + 1);
740 }
741 return (true);
742 }
743
744 bool
uvm_physseg_plug(paddr_t pfn,size_t pages,uvm_physseg_t * psp)745 uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp)
746 {
747 int lcv;
748 struct vm_page *pgs;
749 struct uvm_physseg *ps;
750
751 #ifdef DEBUG
752 paddr_t off;
753 uvm_physseg_t upm;
754 upm = uvm_physseg_find(pfn, &off);
755
756 if (uvm_physseg_valid_p(upm)) /* XXX; do we allow "update" plugs ? */
757 return false;
758 #endif
759
760 paddr_t start = pfn;
761 paddr_t end = pfn + pages;
762 paddr_t avail_start = start;
763 paddr_t avail_end = end;
764
765 if (uvmexp.pagesize == 0)
766 panic("uvm_page_physload: page size not set!");
767
768 /*
769 * do we have room?
770 */
771
772 if (vm_nphysmem == VM_PHYSSEG_MAX) {
773 printf("uvm_page_physload: unable to load physical memory "
774 "segment\n");
775 printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
776 VM_PHYSSEG_MAX, (long long)start, (long long)end);
777 printf("\tincrease VM_PHYSSEG_MAX\n");
778 if (psp != NULL)
779 *psp = UVM_PHYSSEG_TYPE_INVALID_OVERFLOW;
780 return false;
781 }
782
783 /*
784 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been
785 * called yet, so kmem is not available).
786 */
787 pgs = uvm_post_preload_check();
788
789 /*
790 * now insert us in the proper place in vm_physmem[]
791 */
792
793 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM)
794 /* random: put it at the end (easy!) */
795 ps = VM_PHYSMEM_PTR(vm_nphysmem);
796 lcv = vm_nphysmem;
797 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
798 {
799 int x;
800 /* sort by address for binary search */
801 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
802 if (start < VM_PHYSMEM_PTR(lcv)->start)
803 break;
804 ps = VM_PHYSMEM_PTR(lcv);
805 /* move back other entries, if necessary ... */
806 for (x = vm_nphysmem ; x > lcv ; x--)
807 /* structure copy */
808 VM_PHYSMEM_PTR_SWAP(x, x - 1);
809 }
810 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
811 {
812 int x;
813 /* sort by largest segment first */
814 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
815 if ((end - start) >
816 (VM_PHYSMEM_PTR(lcv)->end - VM_PHYSMEM_PTR(lcv)->start))
817 break;
818 ps = VM_PHYSMEM_PTR(lcv);
819 /* move back other entries, if necessary ... */
820 for (x = vm_nphysmem ; x > lcv ; x--)
821 /* structure copy */
822 VM_PHYSMEM_PTR_SWAP(x, x - 1);
823 }
824 #else
825 panic("uvm_page_physload: unknown physseg strategy selected!");
826 #endif
827
828 ps->start = start;
829 ps->end = end;
830 ps->avail_start = avail_start;
831 ps->avail_end = avail_end;
832
833 ps->pgs = pgs;
834
835 vm_nphysmem++;
836
837 if (psp != NULL)
838 *psp = lcv;
839
840 return true;
841 }
842
843 /*
844 * when VM_PHYSSEG_MAX is 1, we can simplify these functions
845 */
846
847 #if VM_PHYSSEG_MAX == 1
848 static inline int vm_physseg_find_contig(struct uvm_physseg *, int, paddr_t, psize_t *);
849 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
850 static inline int vm_physseg_find_bsearch(struct uvm_physseg *, int, paddr_t, psize_t *);
851 #else
852 static inline int vm_physseg_find_linear(struct uvm_physseg *, int, paddr_t, psize_t *);
853 #endif
854
855 /*
856 * vm_physseg_find: find vm_physseg structure that belongs to a PA
857 */
858 inline int
uvm_physseg_find(paddr_t pframe,psize_t * offp)859 uvm_physseg_find(paddr_t pframe, psize_t *offp)
860 {
861
862 #if VM_PHYSSEG_MAX == 1
863 return vm_physseg_find_contig(vm_physmem, vm_nphysseg, pframe, offp);
864 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
865 return vm_physseg_find_bsearch(vm_physmem, vm_nphysseg, pframe, offp);
866 #else
867 return vm_physseg_find_linear(vm_physmem, vm_nphysseg, pframe, offp);
868 #endif
869 }
870
871 #if VM_PHYSSEG_MAX == 1
872 static inline int
vm_physseg_find_contig(struct uvm_physseg * segs,int nsegs,paddr_t pframe,psize_t * offp)873 vm_physseg_find_contig(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp)
874 {
875
876 /* 'contig' case */
877 if (pframe >= segs[0].start && pframe < segs[0].end) {
878 if (offp)
879 *offp = pframe - segs[0].start;
880 return(0);
881 }
882 return(-1);
883 }
884
885 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
886
887 static inline int
vm_physseg_find_bsearch(struct uvm_physseg * segs,int nsegs,paddr_t pframe,psize_t * offp)888 vm_physseg_find_bsearch(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp)
889 {
890 /* binary search for it */
891 int start, len, guess;
892
893 /*
894 * if try is too large (thus target is less than try) we reduce
895 * the length to trunc(len/2) [i.e. everything smaller than "try"]
896 *
897 * if the try is too small (thus target is greater than try) then
898 * we set the new start to be (try + 1). this means we need to
899 * reduce the length to (round(len/2) - 1).
900 *
901 * note "adjust" below which takes advantage of the fact that
902 * (round(len/2) - 1) == trunc((len - 1) / 2)
903 * for any value of len we may have
904 */
905
906 for (start = 0, len = nsegs ; len != 0 ; len = len / 2) {
907 guess = start + (len / 2); /* try in the middle */
908
909 /* start past our try? */
910 if (pframe >= segs[guess].start) {
911 /* was try correct? */
912 if (pframe < segs[guess].end) {
913 if (offp)
914 *offp = pframe - segs[guess].start;
915 return guess; /* got it */
916 }
917 start = guess + 1; /* next time, start here */
918 len--; /* "adjust" */
919 } else {
920 /*
921 * pframe before try, just reduce length of
922 * region, done in "for" loop
923 */
924 }
925 }
926 return(-1);
927 }
928
929 #else
930
931 static inline int
vm_physseg_find_linear(struct uvm_physseg * segs,int nsegs,paddr_t pframe,psize_t * offp)932 vm_physseg_find_linear(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp)
933 {
934 /* linear search for it */
935 int lcv;
936
937 for (lcv = 0; lcv < nsegs; lcv++) {
938 if (pframe >= segs[lcv].start &&
939 pframe < segs[lcv].end) {
940 if (offp)
941 *offp = pframe - segs[lcv].start;
942 return(lcv); /* got it */
943 }
944 }
945 return(-1);
946 }
947 #endif
948 #endif /* UVM_HOTPLUG */
949
950 /*
951 * PHYS_TO_VM_PAGE: find vm_page for a PA. used by MI code to get vm_pages
952 * back from an I/O mapping (ugh!). used in some MD code as well. it can
953 * be prominent in flamegraphs, so optimise it and try to make it easy for
954 * the compiler by including next to the inline lookup routines.
955 */
956 struct vm_page *
uvm_phys_to_vm_page(paddr_t pa)957 uvm_phys_to_vm_page(paddr_t pa)
958 {
959 #if VM_PHYSSEG_STRAT != VM_PSTRAT_BSEARCH
960 /* 'contig' and linear cases */
961 KASSERT(vm_nphysseg > 0);
962 struct uvm_physseg *ps = &vm_physmem[0];
963 struct uvm_physseg *end = &vm_physmem[vm_nphysseg];
964 paddr_t pframe = atop(pa);
965 do {
966 if (pframe >= ps->start && pframe < ps->end) {
967 return &ps->pgs[pframe - ps->start];
968 }
969 } while (VM_PHYSSEG_MAX > 1 && __predict_false(++ps < end));
970 return NULL;
971 #else
972 /* binary search for it */
973 paddr_t pf = atop(pa);
974 paddr_t off;
975 uvm_physseg_t upm;
976
977 upm = uvm_physseg_find(pf, &off);
978 if (upm != UVM_PHYSSEG_TYPE_INVALID)
979 return uvm_physseg_get_pg(upm, off);
980 return(NULL);
981 #endif
982 }
983
984 bool
uvm_physseg_valid_p(uvm_physseg_t upm)985 uvm_physseg_valid_p(uvm_physseg_t upm)
986 {
987 struct uvm_physseg *ps;
988
989 if (upm == UVM_PHYSSEG_TYPE_INVALID ||
990 upm == UVM_PHYSSEG_TYPE_INVALID_EMPTY ||
991 upm == UVM_PHYSSEG_TYPE_INVALID_OVERFLOW)
992 return false;
993
994 /*
995 * This is the delicate init dance -
996 * needs to go with the dance.
997 */
998 if (uvm.page_init_done != true)
999 return true;
1000
1001 ps = HANDLE_TO_PHYSSEG_NODE(upm);
1002
1003 /* Extra checks needed only post uvm_page_init() */
1004 if (ps->pgs == NULL)
1005 return false;
1006
1007 /* XXX: etc. */
1008
1009 return true;
1010
1011 }
1012
1013 /*
1014 * Boot protocol dictates that these must be able to return partially
1015 * initialised segments.
1016 */
1017 paddr_t
uvm_physseg_get_start(uvm_physseg_t upm)1018 uvm_physseg_get_start(uvm_physseg_t upm)
1019 {
1020 if (uvm_physseg_valid_p(upm) == false)
1021 return (paddr_t) -1;
1022
1023 return HANDLE_TO_PHYSSEG_NODE(upm)->start;
1024 }
1025
1026 paddr_t
uvm_physseg_get_end(uvm_physseg_t upm)1027 uvm_physseg_get_end(uvm_physseg_t upm)
1028 {
1029 if (uvm_physseg_valid_p(upm) == false)
1030 return (paddr_t) -1;
1031
1032 return HANDLE_TO_PHYSSEG_NODE(upm)->end;
1033 }
1034
1035 paddr_t
uvm_physseg_get_avail_start(uvm_physseg_t upm)1036 uvm_physseg_get_avail_start(uvm_physseg_t upm)
1037 {
1038 if (uvm_physseg_valid_p(upm) == false)
1039 return (paddr_t) -1;
1040
1041 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_start;
1042 }
1043
1044 #if defined(UVM_PHYSSEG_LEGACY)
1045 void
uvm_physseg_set_avail_start(uvm_physseg_t upm,paddr_t avail_start)1046 uvm_physseg_set_avail_start(uvm_physseg_t upm, paddr_t avail_start)
1047 {
1048 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm);
1049
1050 #if defined(DIAGNOSTIC)
1051 paddr_t avail_end;
1052 avail_end = uvm_physseg_get_avail_end(upm);
1053 KASSERT(uvm_physseg_valid_p(upm));
1054 KASSERT(avail_start < avail_end);
1055 KASSERT(avail_start >= ps->start);
1056 #endif
1057
1058 ps->avail_start = avail_start;
1059 }
1060
1061 void
uvm_physseg_set_avail_end(uvm_physseg_t upm,paddr_t avail_end)1062 uvm_physseg_set_avail_end(uvm_physseg_t upm, paddr_t avail_end)
1063 {
1064 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm);
1065
1066 #if defined(DIAGNOSTIC)
1067 paddr_t avail_start;
1068 avail_start = uvm_physseg_get_avail_start(upm);
1069 KASSERT(uvm_physseg_valid_p(upm));
1070 KASSERT(avail_end > avail_start);
1071 KASSERT(avail_end <= ps->end);
1072 #endif
1073
1074 ps->avail_end = avail_end;
1075 }
1076
1077 #endif /* UVM_PHYSSEG_LEGACY */
1078
1079 paddr_t
uvm_physseg_get_avail_end(uvm_physseg_t upm)1080 uvm_physseg_get_avail_end(uvm_physseg_t upm)
1081 {
1082 if (uvm_physseg_valid_p(upm) == false)
1083 return (paddr_t) -1;
1084
1085 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_end;
1086 }
1087
1088 inline struct vm_page *
uvm_physseg_get_pg(uvm_physseg_t upm,paddr_t idx)1089 uvm_physseg_get_pg(uvm_physseg_t upm, paddr_t idx)
1090 {
1091 KASSERT(uvm_physseg_valid_p(upm));
1092 return &HANDLE_TO_PHYSSEG_NODE(upm)->pgs[idx];
1093 }
1094
1095 #ifdef __HAVE_PMAP_PHYSSEG
1096 struct pmap_physseg *
uvm_physseg_get_pmseg(uvm_physseg_t upm)1097 uvm_physseg_get_pmseg(uvm_physseg_t upm)
1098 {
1099 KASSERT(uvm_physseg_valid_p(upm));
1100 return &(HANDLE_TO_PHYSSEG_NODE(upm)->pmseg);
1101 }
1102 #endif
1103
1104 int
uvm_physseg_get_free_list(uvm_physseg_t upm)1105 uvm_physseg_get_free_list(uvm_physseg_t upm)
1106 {
1107 KASSERT(uvm_physseg_valid_p(upm));
1108 return HANDLE_TO_PHYSSEG_NODE(upm)->free_list;
1109 }
1110
1111 u_long
uvm_physseg_get_start_hint(uvm_physseg_t upm)1112 uvm_physseg_get_start_hint(uvm_physseg_t upm)
1113 {
1114 KASSERT(uvm_physseg_valid_p(upm));
1115 return HANDLE_TO_PHYSSEG_NODE(upm)->start_hint;
1116 }
1117
1118 bool
uvm_physseg_set_start_hint(uvm_physseg_t upm,u_long start_hint)1119 uvm_physseg_set_start_hint(uvm_physseg_t upm, u_long start_hint)
1120 {
1121 if (uvm_physseg_valid_p(upm) == false)
1122 return false;
1123
1124 HANDLE_TO_PHYSSEG_NODE(upm)->start_hint = start_hint;
1125 return true;
1126 }
1127
1128 void
uvm_physseg_init_seg(uvm_physseg_t upm,struct vm_page * pgs)1129 uvm_physseg_init_seg(uvm_physseg_t upm, struct vm_page *pgs)
1130 {
1131 psize_t i;
1132 psize_t n;
1133 paddr_t paddr;
1134 struct uvm_physseg *seg;
1135 struct vm_page *pg;
1136
1137 KASSERT(upm != UVM_PHYSSEG_TYPE_INVALID);
1138 KASSERT(pgs != NULL);
1139
1140 seg = HANDLE_TO_PHYSSEG_NODE(upm);
1141 KASSERT(seg != NULL);
1142 KASSERT(seg->pgs == NULL);
1143
1144 n = seg->end - seg->start;
1145 seg->pgs = pgs;
1146
1147 /* init and free vm_pages (we've already zeroed them) */
1148 paddr = ctob(seg->start);
1149 for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
1150 pg = &seg->pgs[i];
1151 pg->phys_addr = paddr;
1152 #ifdef __HAVE_VM_PAGE_MD
1153 VM_MDPAGE_INIT(pg);
1154 #endif
1155 if (atop(paddr) >= seg->avail_start &&
1156 atop(paddr) < seg->avail_end) {
1157 uvmexp.npages++;
1158 /* add page to free pool */
1159 uvm_page_set_freelist(pg,
1160 uvm_page_lookup_freelist(pg));
1161 /* Disable LOCKDEBUG: too many and too early. */
1162 mutex_init(&pg->interlock, MUTEX_NODEBUG, IPL_NONE);
1163 uvm_pagefree(pg);
1164 }
1165 }
1166 }
1167
1168 void
uvm_physseg_seg_chomp_slab(uvm_physseg_t upm,struct vm_page * pgs,size_t n)1169 uvm_physseg_seg_chomp_slab(uvm_physseg_t upm, struct vm_page *pgs, size_t n)
1170 {
1171 struct uvm_physseg *seg = HANDLE_TO_PHYSSEG_NODE(upm);
1172
1173 /* max number of pre-boot unplug()s allowed */
1174 #define UVM_PHYSSEG_BOOT_UNPLUG_MAX VM_PHYSSEG_MAX
1175
1176 static char btslab_ex_storage[EXTENT_FIXED_STORAGE_SIZE(UVM_PHYSSEG_BOOT_UNPLUG_MAX)];
1177
1178 if (__predict_false(uvm.page_init_done == false)) {
1179 seg->ext = extent_create("Boot time slab", (u_long) pgs, (u_long) (pgs + n),
1180 (void *)btslab_ex_storage, sizeof(btslab_ex_storage), 0);
1181 } else {
1182 seg->ext = extent_create("Hotplug slab", (u_long) pgs, (u_long) (pgs + n), NULL, 0, 0);
1183 }
1184
1185 KASSERT(seg->ext != NULL);
1186
1187 }
1188
1189 struct vm_page *
uvm_physseg_seg_alloc_from_slab(uvm_physseg_t upm,size_t pages)1190 uvm_physseg_seg_alloc_from_slab(uvm_physseg_t upm, size_t pages)
1191 {
1192 int err;
1193 struct uvm_physseg *seg;
1194 struct vm_page *pgs = NULL;
1195
1196 KASSERT(pages > 0);
1197
1198 seg = HANDLE_TO_PHYSSEG_NODE(upm);
1199
1200 if (__predict_false(seg->ext == NULL)) {
1201 /*
1202 * This is a situation unique to boot time.
1203 * It shouldn't happen at any point other than from
1204 * the first uvm_page.c:uvm_page_init() call
1205 * Since we're in a loop, we can get away with the
1206 * below.
1207 */
1208 KASSERT(uvm.page_init_done != true);
1209
1210 uvm_physseg_t upmp = uvm_physseg_get_prev(upm);
1211 KASSERT(upmp != UVM_PHYSSEG_TYPE_INVALID);
1212
1213 seg->ext = HANDLE_TO_PHYSSEG_NODE(upmp)->ext;
1214
1215 KASSERT(seg->ext != NULL);
1216 }
1217
1218 /* We allocate enough for this segment */
1219 err = extent_alloc(seg->ext, sizeof(*pgs) * pages, 1, 0, EX_BOUNDZERO, (u_long *)&pgs);
1220
1221 if (err != 0) {
1222 #ifdef DEBUG
1223 printf("%s: extent_alloc failed with error: %d \n",
1224 __func__, err);
1225 #endif
1226 }
1227
1228 return pgs;
1229 }
1230
1231 /*
1232 * uvm_page_physload: load physical memory into VM system
1233 *
1234 * => all args are PFs
1235 * => all pages in start/end get vm_page structures
1236 * => areas marked by avail_start/avail_end get added to the free page pool
1237 * => we are limited to VM_PHYSSEG_MAX physical memory segments
1238 */
1239
1240 uvm_physseg_t
uvm_page_physload(paddr_t start,paddr_t end,paddr_t avail_start,paddr_t avail_end,int free_list)1241 uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start,
1242 paddr_t avail_end, int free_list)
1243 {
1244 struct uvm_physseg *ps;
1245 uvm_physseg_t upm;
1246
1247 if (__predict_true(uvm.page_init_done == true))
1248 panic("%s: unload attempted after uvm_page_init()\n", __func__);
1249 if (uvmexp.pagesize == 0)
1250 panic("uvm_page_physload: page size not set!");
1251 if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT)
1252 panic("uvm_page_physload: bad free list %d", free_list);
1253 if (start >= end)
1254 panic("uvm_page_physload: start[%" PRIxPADDR "] >= end[%"
1255 PRIxPADDR "]", start, end);
1256
1257 if (uvm_physseg_plug(start, end - start, &upm) == false) {
1258 panic("uvm_physseg_plug() failed at boot.");
1259 /* NOTREACHED */
1260 return UVM_PHYSSEG_TYPE_INVALID; /* XXX: correct type */
1261 }
1262
1263 ps = HANDLE_TO_PHYSSEG_NODE(upm);
1264
1265 /* Legacy */
1266 ps->avail_start = avail_start;
1267 ps->avail_end = avail_end;
1268
1269 ps->free_list = free_list; /* XXX: */
1270
1271
1272 return upm;
1273 }
1274
1275 bool
uvm_physseg_unplug(paddr_t pfn,size_t pages)1276 uvm_physseg_unplug(paddr_t pfn, size_t pages)
1277 {
1278 uvm_physseg_t upm;
1279 paddr_t off = 0, start __diagused, end;
1280 struct uvm_physseg *seg;
1281
1282 upm = uvm_physseg_find(pfn, &off);
1283
1284 if (!uvm_physseg_valid_p(upm)) {
1285 printf("%s: Tried to unplug from unknown offset\n", __func__);
1286 return false;
1287 }
1288
1289 seg = HANDLE_TO_PHYSSEG_NODE(upm);
1290
1291 start = uvm_physseg_get_start(upm);
1292 end = uvm_physseg_get_end(upm);
1293
1294 if (end < (pfn + pages)) {
1295 printf("%s: Tried to unplug oversized span \n", __func__);
1296 return false;
1297 }
1298
1299 KASSERT(pfn == start + off); /* sanity */
1300
1301 if (__predict_true(uvm.page_init_done == true)) {
1302 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1303 if (extent_free(seg->ext, (u_long)(seg->pgs + off), sizeof(struct vm_page) * pages, EX_MALLOCOK | EX_NOWAIT) != 0)
1304 return false;
1305 }
1306
1307 if (off == 0 && (pfn + pages) == end) {
1308 #if defined(UVM_HOTPLUG) /* rbtree implementation */
1309 int segcount = 0;
1310 struct uvm_physseg *current_ps;
1311 /* Complete segment */
1312 if (uvm_physseg_graph.nentries == 1)
1313 panic("%s: out of memory!", __func__);
1314
1315 if (__predict_true(uvm.page_init_done == true)) {
1316 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) {
1317 if (seg->ext == current_ps->ext)
1318 segcount++;
1319 }
1320 KASSERT(segcount > 0);
1321
1322 if (segcount == 1) {
1323 extent_destroy(seg->ext);
1324 }
1325
1326 /*
1327 * We assume that the unplug will succeed from
1328 * this point onwards
1329 */
1330 uvmexp.npages -= (int) pages;
1331 }
1332
1333 rb_tree_remove_node(&(uvm_physseg_graph.rb_tree), upm);
1334 memset(seg, 0, sizeof(struct uvm_physseg));
1335 uvm_physseg_free(seg, sizeof(struct uvm_physseg));
1336 uvm_physseg_graph.nentries--;
1337 #else /* UVM_HOTPLUG */
1338 int x;
1339 if (vm_nphysmem == 1)
1340 panic("uvm_page_physget: out of memory!");
1341 vm_nphysmem--;
1342 for (x = upm ; x < vm_nphysmem ; x++)
1343 /* structure copy */
1344 VM_PHYSMEM_PTR_SWAP(x, x + 1);
1345 #endif /* UVM_HOTPLUG */
1346 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1347 return true;
1348 }
1349
1350 if (off > 0 &&
1351 (pfn + pages) < end) {
1352 #if defined(UVM_HOTPLUG) /* rbtree implementation */
1353 /* middle chunk - need a new segment */
1354 struct uvm_physseg *ps, *current_ps;
1355 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg));
1356 if (ps == NULL) {
1357 printf("%s: Unable to allocated new fragment vm_physseg \n",
1358 __func__);
1359 return false;
1360 }
1361
1362 /* Remove middle chunk */
1363 if (__predict_true(uvm.page_init_done == true)) {
1364 KASSERT(seg->ext != NULL);
1365 ps->ext = seg->ext;
1366
1367 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1368 /*
1369 * We assume that the unplug will succeed from
1370 * this point onwards
1371 */
1372 uvmexp.npages -= (int) pages;
1373 }
1374
1375 ps->start = pfn + pages;
1376 ps->avail_start = ps->start; /* XXX: Legacy */
1377
1378 ps->end = seg->end;
1379 ps->avail_end = ps->end; /* XXX: Legacy */
1380
1381 seg->end = pfn;
1382 seg->avail_end = seg->end; /* XXX: Legacy */
1383
1384
1385 /*
1386 * The new pgs array points to the beginning of the
1387 * tail fragment.
1388 */
1389 if (__predict_true(uvm.page_init_done == true))
1390 ps->pgs = seg->pgs + off + pages;
1391
1392 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps);
1393 if (current_ps != ps) {
1394 panic("uvm_page_physload: Duplicate address range detected!");
1395 }
1396 uvm_physseg_graph.nentries++;
1397 #else /* UVM_HOTPLUG */
1398 panic("%s: can't unplug() from the middle of a segment without"
1399 " UVM_HOTPLUG\n", __func__);
1400 /* NOTREACHED */
1401 #endif /* UVM_HOTPLUG */
1402 return true;
1403 }
1404
1405 if (off == 0 && (pfn + pages) < end) {
1406 /* Remove front chunk */
1407 if (__predict_true(uvm.page_init_done == true)) {
1408 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */
1409 /*
1410 * We assume that the unplug will succeed from
1411 * this point onwards
1412 */
1413 uvmexp.npages -= (int) pages;
1414 }
1415
1416 /* Truncate */
1417 seg->start = pfn + pages;
1418 seg->avail_start = seg->start; /* XXX: Legacy */
1419
1420 /*
1421 * Move the pgs array start to the beginning of the
1422 * tail end.
1423 */
1424 if (__predict_true(uvm.page_init_done == true))
1425 seg->pgs += pages;
1426
1427 return true;
1428 }
1429
1430 if (off > 0 && (pfn + pages) == end) {
1431 /* back chunk */
1432
1433
1434 /* Truncate! */
1435 seg->end = pfn;
1436 seg->avail_end = seg->end; /* XXX: Legacy */
1437
1438 uvmexp.npages -= (int) pages;
1439
1440 return true;
1441 }
1442
1443 printf("%s: Tried to unplug unknown range \n", __func__);
1444
1445 return false;
1446 }
1447